Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ef76c9034d | ||
|
|
cee281ff55 | ||
|
|
5c509831b8 | ||
|
|
ee57aeea4a | ||
|
|
803c1d3385 | ||
|
|
b8ec62beef | ||
|
|
59bfd32b9e | ||
|
|
3432a33d9a | ||
|
|
a1b081d328 | ||
|
|
bbecff9c04 | ||
|
|
c1553bd5d6 | ||
|
|
1a550bda24 | ||
|
|
902f86192d | ||
|
|
6a40ac86e5 | ||
|
|
410ba6e992 | ||
|
|
6c6f02e5dd | ||
|
|
d9006d1fda | ||
|
|
36375010d3 | ||
|
|
5efa0dba7c | ||
|
|
e32733fbf6 | ||
|
|
b02799327d | ||
|
|
946aa934cc | ||
|
|
125921c11b | ||
|
|
9dd5288381 | ||
|
|
3f2c74dd9e | ||
|
|
51e41dee2a | ||
|
|
dbf1b39d47 | ||
|
|
845f20a28d | ||
|
|
a5083776da | ||
|
|
76a81f1684 | ||
|
|
24b81c7e5a | ||
|
|
0e33f84f38 | ||
|
|
c7ac039523 | ||
|
|
e771cc9da6 | ||
|
|
7f9f82a068 | ||
|
|
695b5c8c25 | ||
|
|
5b2188d909 | ||
|
|
e5da916d54 | ||
|
|
de640bba1b | ||
|
|
f0f5719be0 | ||
|
|
6200b298ac | ||
|
|
394a9aae22 | ||
|
|
12d4af1b89 | ||
|
|
9d7c55d3c0 | ||
|
|
5a104bfd88 | ||
|
|
2ada8f9ad5 | ||
|
|
1493f1806d | ||
|
|
ea28e25535 | ||
|
|
d6295d62ce | ||
|
|
a9f416ce44 | ||
|
|
448edc0432 | ||
|
|
4a7703b06a | ||
|
|
6fc297544e | ||
|
|
583d3c8edb | ||
|
|
9883757190 | ||
|
|
a690109975 | ||
|
|
36d4d0e43e | ||
|
|
a0b9d9794b | ||
|
|
3e9f70d9ba | ||
|
|
2b5aaef409 | ||
|
|
478f80be9f | ||
|
|
e674b25213 | ||
|
|
1116da9f64 | ||
|
|
684153350a | ||
|
|
ec206b02d0 | ||
|
|
88bdfeeb58 | ||
|
|
3f5b0250f4 | ||
|
|
c41bfb2cdb | ||
|
|
1b2632f968 | ||
|
|
7dbca70a4b | ||
|
|
1a053f06e5 | ||
|
|
58b3a33e82 | ||
|
|
17b0632098 | ||
|
|
376e6500ab | ||
|
|
82a1d5e240 | ||
|
|
3eec5a4cc3 | ||
|
|
6144c7fc7f | ||
|
|
ea4ab661b4 | ||
|
|
84827ca617 | ||
|
|
23552e823d | ||
|
|
d1a567f7e3 | ||
|
|
93c1fc3c1f | ||
|
|
4815b1b281 | ||
|
|
ad7bc3348b | ||
|
|
34970b961c | ||
|
|
b3f7cad4db | ||
|
|
cd8a84c0a2 | ||
|
|
09c384d7d0 | ||
|
|
81ad313ee5 |
@@ -9,7 +9,7 @@ on:
|
|||||||
jobs:
|
jobs:
|
||||||
lint:
|
lint:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
container: gitea.tap-tap.win/taptap/fastsync-ci:v10
|
container: gitea.tap-tap.win/taptap/fastsync-ci:v11
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||||
@@ -26,7 +26,7 @@ jobs:
|
|||||||
# suite) run on merge to dev/main, so PR CI stays well under ~3 minutes.
|
# suite) run on merge to dev/main, so PR CI stays well under ~3 minutes.
|
||||||
build-and-test:
|
build-and-test:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
container: gitea.tap-tap.win/taptap/fastsync-ci:v10
|
container: gitea.tap-tap.win/taptap/fastsync-ci:v11
|
||||||
needs: lint
|
needs: lint
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
@@ -51,7 +51,7 @@ jobs:
|
|||||||
|
|
||||||
sanitizers:
|
sanitizers:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
container: gitea.tap-tap.win/taptap/fastsync-ci:v10
|
container: gitea.tap-tap.win/taptap/fastsync-ci:v11
|
||||||
needs: lint
|
needs: lint
|
||||||
if: github.event_name == 'push'
|
if: github.event_name == 'push'
|
||||||
strategy:
|
strategy:
|
||||||
@@ -72,7 +72,7 @@ jobs:
|
|||||||
|
|
||||||
fuzz-build:
|
fuzz-build:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
container: gitea.tap-tap.win/taptap/fastsync-ci:v10
|
container: gitea.tap-tap.win/taptap/fastsync-ci:v11
|
||||||
needs: lint
|
needs: lint
|
||||||
if: github.event_name == 'push'
|
if: github.event_name == 'push'
|
||||||
steps:
|
steps:
|
||||||
@@ -94,7 +94,7 @@ jobs:
|
|||||||
|
|
||||||
coverage:
|
coverage:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
container: gitea.tap-tap.win/taptap/fastsync-ci:v10
|
container: gitea.tap-tap.win/taptap/fastsync-ci:v11
|
||||||
needs: lint
|
needs: lint
|
||||||
if: github.event_name == 'push'
|
if: github.event_name == 'push'
|
||||||
steps:
|
steps:
|
||||||
@@ -118,7 +118,7 @@ jobs:
|
|||||||
|
|
||||||
valgrind:
|
valgrind:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
container: gitea.tap-tap.win/taptap/fastsync-ci:v10
|
container: gitea.tap-tap.win/taptap/fastsync-ci:v11
|
||||||
needs: lint
|
needs: lint
|
||||||
if: github.event_name == 'push'
|
if: github.event_name == 'push'
|
||||||
steps:
|
steps:
|
||||||
|
|||||||
@@ -55,6 +55,16 @@ if(NOT ZSTD_LIBRARY)
|
|||||||
message(FATAL_ERROR "zstd library not found. Ensure it is in your nix-shell!")
|
message(FATAL_ERROR "zstd library not found. Ensure it is in your nix-shell!")
|
||||||
endif()
|
endif()
|
||||||
|
|
||||||
|
find_library(ZLIB_LIBRARY z)
|
||||||
|
if(NOT ZLIB_LIBRARY)
|
||||||
|
message(FATAL_ERROR "zlib library not found. Ensure zlib1g-dev / nix zlib is available!")
|
||||||
|
endif()
|
||||||
|
|
||||||
|
find_library(LZ4_LIBRARY lz4)
|
||||||
|
if(NOT LZ4_LIBRARY)
|
||||||
|
message(FATAL_ERROR "lz4 library not found. Ensure liblz4-dev / nix lz4 is available!")
|
||||||
|
endif()
|
||||||
|
|
||||||
find_package(OpenSSL REQUIRED)
|
find_package(OpenSSL REQUIRED)
|
||||||
|
|
||||||
file(GLOB SHARED_SRCS "src/shared/*.c")
|
file(GLOB SHARED_SRCS "src/shared/*.c")
|
||||||
@@ -64,15 +74,15 @@ file(GLOB TEST_SRCS "tests/*.c")
|
|||||||
|
|
||||||
add_executable(server ${SERVER_SRCS} ${SHARED_SRCS})
|
add_executable(server ${SERVER_SRCS} ${SHARED_SRCS})
|
||||||
target_include_directories(server PRIVATE src/shared src/server src/client)
|
target_include_directories(server PRIVATE src/shared src/server src/client)
|
||||||
target_link_libraries(server PRIVATE Threads::Threads ${ZSTD_LIBRARY} OpenSSL::SSL OpenSSL::Crypto xxhash)
|
target_link_libraries(server PRIVATE Threads::Threads ${ZSTD_LIBRARY} ${ZLIB_LIBRARY} ${LZ4_LIBRARY} OpenSSL::SSL OpenSSL::Crypto xxhash)
|
||||||
|
|
||||||
add_executable(client ${CLIENT_SRCS} ${SHARED_SRCS})
|
add_executable(client ${CLIENT_SRCS} ${SHARED_SRCS})
|
||||||
target_include_directories(client PRIVATE src/shared src/server src/client)
|
target_include_directories(client PRIVATE src/shared src/server src/client)
|
||||||
target_link_libraries(client PRIVATE Threads::Threads ${ZSTD_LIBRARY} OpenSSL::SSL OpenSSL::Crypto xxhash)
|
target_link_libraries(client PRIVATE Threads::Threads ${ZSTD_LIBRARY} ${ZLIB_LIBRARY} ${LZ4_LIBRARY} OpenSSL::SSL OpenSSL::Crypto xxhash)
|
||||||
|
|
||||||
add_executable(tests ${TEST_SRCS} ${SHARED_SRCS} src/client/scanner.c)
|
add_executable(tests ${TEST_SRCS} ${SHARED_SRCS} src/client/scanner.c)
|
||||||
target_include_directories(tests PRIVATE tests src/shared src/server src/client)
|
target_include_directories(tests PRIVATE tests src/shared src/server src/client)
|
||||||
target_link_libraries(tests PRIVATE Threads::Threads ${ZSTD_LIBRARY} OpenSSL::SSL OpenSSL::Crypto xxhash)
|
target_link_libraries(tests PRIVATE Threads::Threads ${ZSTD_LIBRARY} ${ZLIB_LIBRARY} ${LZ4_LIBRARY} OpenSSL::SSL OpenSSL::Crypto xxhash)
|
||||||
```
|
```
|
||||||
|
|
||||||
### Source Layout
|
### Source Layout
|
||||||
@@ -85,17 +95,23 @@ tests/integration/ — Python pytest integration tests
|
|||||||
```
|
```
|
||||||
|
|
||||||
### Dependencies
|
### Dependencies
|
||||||
- **zstd** — found via `find_library(ZSTD_LIBRARY zstd)`
|
- **zstd** — found via `find_library(ZSTD_LIBRARY zstd)` (default compression codec)
|
||||||
|
- **zlib** — found via `find_library(ZLIB_LIBRARY z)` (the `zlib`/`zlibx` codecs)
|
||||||
|
- **lz4** — found via `find_library(LZ4_LIBRARY lz4)` (the `lz4` codec)
|
||||||
- **OpenSSL** — found via `find_package(OpenSSL REQUIRED)` (TLS 1.2+ transport)
|
- **OpenSSL** — found via `find_package(OpenSSL REQUIRED)` (TLS 1.2+ transport)
|
||||||
- **xxHash** — fetched via `FetchContent` from the upstream repository (delta transfer hashing, v0.8.3)
|
- **xxHash** — fetched via `FetchContent` from the upstream repository (delta transfer hashing, v0.8.3)
|
||||||
- **pthreads** — found via `find_package(Threads REQUIRED)`
|
- **pthreads** — found via `find_package(Threads REQUIRED)`
|
||||||
- **C11 standard** — required
|
- **C11 standard** — required
|
||||||
- **CMake 3.22+** — minimum version
|
- **CMake 3.22+** — minimum version
|
||||||
|
|
||||||
|
The codec matrix (protocol 2.26.0) uses zstd/zlib/lz4 for compression and
|
||||||
|
xxHash/OpenSSL for the `xxh128`/`xxh3`/`xxh64`/`md5`/`md4`/`sha1` checksums
|
||||||
|
(`none` needs no library); both codec families are negotiated per transfer.
|
||||||
|
|
||||||
## Conventions
|
## Conventions
|
||||||
|
|
||||||
- Use `file(GLOB ...)` for source collection (existing pattern).
|
- Use `file(GLOB ...)` for source collection (existing pattern).
|
||||||
- All targets link `Threads::Threads`, `${ZSTD_LIBRARY}`, `OpenSSL::SSL`, `OpenSSL::Crypto`, and `xxhash`.
|
- All targets link `Threads::Threads`, `${ZSTD_LIBRARY}`, `${ZLIB_LIBRARY}`, `${LZ4_LIBRARY}`, `OpenSSL::SSL`, `OpenSSL::Crypto`, and `xxhash`.
|
||||||
- Include directories: `src/shared`, `src/server`, `src/client`, `tests` (for test target).
|
- Include directories: `src/shared`, `src/server`, `src/client`, `tests` (for test target).
|
||||||
- Sanitizer support: pass `-DSANITIZER=address`, `-DSANITIZER=thread`, or `-DSANITIZER=undefined` to cmake (live option in CMakeLists.txt).
|
- Sanitizer support: pass `-DSANITIZER=address`, `-DSANITIZER=thread`, or `-DSANITIZER=undefined` to cmake (live option in CMakeLists.txt).
|
||||||
- Build with `cmake -B build -S . && cmake --build build -j$(nproc)`.
|
- Build with `cmake -B build -S . && cmake --build build -j$(nproc)`.
|
||||||
|
|||||||
@@ -116,7 +116,7 @@ The project uses Gitea Actions. Key jobs:
|
|||||||
jobs:
|
jobs:
|
||||||
new-job:
|
new-job:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
container: gitea.tap-tap.win/taptap/fastsync-ci:v10
|
container: gitea.tap-tap.win/taptap/fastsync-ci:v11
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
- name: Configure
|
- name: Configure
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ Ask the user or determine from context:
|
|||||||
- **Minor** (x.Y.0) — new features, backward compatible
|
- **Minor** (x.Y.0) — new features, backward compatible
|
||||||
- **Patch** (x.y.Z) — bug fixes, no protocol changes
|
- **Patch** (x.y.Z) — bug fixes, no protocol changes
|
||||||
|
|
||||||
Current version: `PROTOCOL_VERSION "2.21.0"` in `src/shared/config.h`
|
Current version: `PROTOCOL_VERSION "2.26.0"` in `src/shared/config.h`
|
||||||
|
|
||||||
### Step 2: Check Protocol Version
|
### Step 2: Check Protocol Version
|
||||||
|
|
||||||
|
|||||||
@@ -4,18 +4,19 @@ FastSync is a high-performance file synchronization system written in C11. It su
|
|||||||
|
|
||||||
## Dependency installation
|
## Dependency installation
|
||||||
|
|
||||||
**CI rule:** never add `apt-get install` / `pip install` steps to CI workflows — use the custom Docker image instead. The image is built from the repo-root `Dockerfile` and is the same image CI uses: `gitea.tap-tap.win/taptap/fastsync-ci:v10`. It contains the full toolchain: gcc/g++, CMake, libzstd-dev, libssl-dev, make, git, cppcheck, clang-format, python3 + pytest + pytest-xdist, openssh-client, and Node.js.
|
**CI rule:** never add `apt-get install` / `pip install` steps to CI workflows — use the custom Docker image instead. The image is built from the repo-root `Dockerfile` and is the same image CI uses: `gitea.tap-tap.win/taptap/fastsync-ci:v11`. It contains the full toolchain: gcc/g++, CMake, libzstd-dev, libssl-dev, make, git, cppcheck, clang-format, python3 + pytest + pytest-xdist, openssh-client, Node.js, plus `rsync` 3.4.1 (with zstd/xxhash/lz4), `acl` and `attr` (setfacl/getfacl, setfattr/getfattr) for drop-in parity tests.
|
||||||
|
|
||||||
**Host rule:** for local development, use `nix-shell` (see `README.md`) which provides zstd, OpenSSL, CMake, and gcc. The Docker image can also be used locally for CI parity.
|
**Host rule:** for local development, use `nix-shell` (see `README.md`) which provides zstd, OpenSSL, CMake, and gcc. The Docker image can also be used locally for CI parity.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Use the prebuilt CI image directly (faster, guaranteed CI parity)
|
# Use the prebuilt CI image directly (faster, guaranteed CI parity)
|
||||||
docker pull gitea.tap-tap.win/taptap/fastsync-ci:v10
|
docker pull gitea.tap-tap.win/taptap/fastsync-ci:v11
|
||||||
docker tag gitea.tap-tap.win/taptap/fastsync-ci:v10 fastsync-ci:local
|
docker tag gitea.tap-tap.win/taptap/fastsync-ci:v11 fastsync-ci:local
|
||||||
|
|
||||||
# Or build the image from the repo-root Dockerfile
|
# Or build the image from the repo-root Dockerfile
|
||||||
# (Note: the prebuilt :v10 image reflects the previous Dockerfile state;
|
# (Note: the prebuilt :v11 image is built from the current Dockerfile and
|
||||||
# rebuild from source to pick up any newly added packages like lcov/valgrind.)
|
# includes rsync 3.4.1 plus acl/attr; rebuild from source after changing
|
||||||
|
# the Dockerfile.)
|
||||||
docker build -t fastsync-ci:local .
|
docker build -t fastsync-ci:local .
|
||||||
|
|
||||||
# Build, run unit tests, and run integration tests inside the container
|
# Build, run unit tests, and run integration tests inside the container
|
||||||
@@ -66,14 +67,14 @@ When running the CI workflow via `tea` (the task execution agent), always set a
|
|||||||
### If lint (clang-format) fails
|
### If lint (clang-format) fails
|
||||||
Run clang-format in the CI Docker image to match the exact CI version:
|
Run clang-format in the CI Docker image to match the exact CI version:
|
||||||
```bash
|
```bash
|
||||||
docker run --rm -v "$PWD:/workspace" -w /workspace gitea.tap-tap.win/taptap/fastsync-ci:v10 \
|
docker run --rm -v "$PWD:/workspace" -w /workspace gitea.tap-tap.win/taptap/fastsync-ci:v11 \
|
||||||
sh -c 'find src/ tests/ -name "*.c" -o -name "*.h" | xargs clang-format -i'
|
sh -c 'find src/ tests/ -name "*.c" -o -name "*.h" | xargs clang-format -i'
|
||||||
```
|
```
|
||||||
|
|
||||||
### If cppcheck fails
|
### If cppcheck fails
|
||||||
Fix reported issues locally, then verify with:
|
Fix reported issues locally, then verify with:
|
||||||
```bash
|
```bash
|
||||||
docker run --rm -v "$PWD:/workspace" -w /workspace gitea.tap-tap.win/taptap/fastsync-ci:v10 \
|
docker run --rm -v "$PWD:/workspace" -w /workspace gitea.tap-tap.win/taptap/fastsync-ci:v11 \
|
||||||
sh -c 'cppcheck --enable=warning,style,performance,portability --suppress=missingIncludeSystem --error-exitcode=1 --inline-suppr src/ tests/'
|
sh -c 'cppcheck --enable=warning,style,performance,portability --suppress=missingIncludeSystem --error-exitcode=1 --inline-suppr src/ tests/'
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|||||||
+181
@@ -4,6 +4,187 @@ All notable changes to FastSync are documented here. Versions match
|
|||||||
`PROTOCOL_VERSION` (printed by `fastsync --version`); the client and server must
|
`PROTOCOL_VERSION` (printed by `fastsync --version`); the client and server must
|
||||||
run the same version because the handshake is strict.
|
run the same version because the handshake is strict.
|
||||||
|
|
||||||
|
## [2.26.0] - 2026-09-17
|
||||||
|
|
||||||
|
### Added
|
||||||
|
|
||||||
|
- **Parity-completion wave.** Closed the remaining rsync-parity gaps against
|
||||||
|
rsync 3.4.1 and reclassified the inherently non-rsync rows. It moved the wire
|
||||||
|
protocol three times (`2.23.0 → 2.24.0 → 2.25.0 → 2.26.0`).
|
||||||
|
- **Delete timing (2.24.0):** per-directory delete plans
|
||||||
|
(`STATUS_DELETE_PLAN`) for `--delete-during`/`--delete-delay`. An interrupted
|
||||||
|
during-transfer has already removed the reached directories' extras, while a
|
||||||
|
delayed transfer commits per directory only after the whole transfer
|
||||||
|
succeeds (a late-created extra survives `--delete-delay` but not
|
||||||
|
`--delete-after`). `-R --delete` is scoped to the transferred prefix; empty
|
||||||
|
in-scope source directories survive; dry-run never deletes.
|
||||||
|
- **Wire stats (2.25.0):** `STATUS_STATS` carries the receiver counters
|
||||||
|
(matched data, deleted files) and the dry-run would-delete list. `--stats`
|
||||||
|
prints rsync's protocol-independent lines; `--progress`/`-P` print per-file
|
||||||
|
blocks; `--out-format` gains `%b` (wire bytes), `%c` (block-sum bytes) and
|
||||||
|
`%C` (whole-file digest); `-n --delete` prints escaped `*deleting` lines in
|
||||||
|
the sequential and `--threads` paths.
|
||||||
|
- **Codecs (2.26.0):** `lz4`/`zlib`/`zlibx` compression and `md4`/`sha1`/
|
||||||
|
`none` checksums, with rsync-style `auto` negotiation (default `xxh128` +
|
||||||
|
`zstd`) and exit-4 rejection of unknown names; the resolved `compression_algo`
|
||||||
|
crosses the wire.
|
||||||
|
- General `-R`/`--relative` (including the `/./` cut) and `--no-implied-dirs`;
|
||||||
|
one-level `-d`/`--dirs` listing for `dir`, `dir/` and `.`; the full filter
|
||||||
|
grammar (`merge`/`dir-merge`/`hide`/`show`/`protect`/`risk`/`clear` and
|
||||||
|
modifiers) with `-f` bound to `--filter`; a single `-F` transfers
|
||||||
|
`.rsync-filter` and `-FF` excludes it.
|
||||||
|
- Receiver-side `--chown`/`--usermap`/`--groupmap` TO-name resolution; absolute
|
||||||
|
basis directories and a `--link-dest` relink of an up-to-date destination;
|
||||||
|
a receiver-side `--ignore-existing` short-circuit before any payload;
|
||||||
|
`--preallocate` now wins over `--sparse` via `fallocate(2)`.
|
||||||
|
- Client quick wins: `--iconv=.`/`-`/`--no-iconv`, a lone `-h` prints help, an
|
||||||
|
empty `--files-from` succeeds (exit 0), a broken referent under
|
||||||
|
`-L`/`--copy-unsafe-links` exits 23, the full `--info`/`--debug`
|
||||||
|
vocabularies, and the aliases `--ignore-non-existing`, `--protect-args`,
|
||||||
|
`--msgs2stderr`.
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- `PROTOCOL_VERSION` bumped `2.23.0 → 2.24.0` (delete plans),
|
||||||
|
`2.24.0 → 2.25.0` (`STATUS_STATS` + `report_stats`), and
|
||||||
|
`2.25.0 → 2.26.0` (codec negotiation + `md4`/`sha1`/`none`).
|
||||||
|
- `--checksum-choice`/`--cc` now accepts `md4`, `sha1`, `none` and the two-name
|
||||||
|
form; the negotiated whole-file default is `xxh128`.
|
||||||
|
- `--compress-choice`/`--zc` now accepts `lz4`, `zlib`, `zlibx`.
|
||||||
|
- `RSYNC_COMPAT.md` reclassifies the matrix: 9 already-parity rows to ✅, 17
|
||||||
|
inherently non-rsync rows to ❌ (native daemon config/auth, batch, privileged
|
||||||
|
xattr namespaces, and the safe-subset device/privilege flags), and the genuine
|
||||||
|
fixes to ✅; new rows cover `--bwlimit`, `--partial`, `--partial-dir`,
|
||||||
|
`--no-whole-file`, `--inc-recursive`/`--no-inc-recursive`, `--protect-args`
|
||||||
|
and `--msgs2stderr`.
|
||||||
|
- The client `--help` `--max-delete` text now describes the implemented partial
|
||||||
|
semantics (delete up to N, skip the rest, exit 25).
|
||||||
|
|
||||||
|
### Notes
|
||||||
|
|
||||||
|
- Remaining documented divergences include the `--stats` per-type file-count
|
||||||
|
breakdown, `%b`/`%c` being FastSync wire counts, `-n --delete` line ordering,
|
||||||
|
the default `--delete` timing (delete-after, not rsync's delete-during),
|
||||||
|
destination-only exclude protection (still sender-derived), `--temp-dir`
|
||||||
|
absolute paths, basis-dir attribute re-application and the 256 MiB whole-file
|
||||||
|
cap, `--fuzzy` tie-breaking, `--bwlimit=0`/decimal rates, `zlibx`==`zlib`, and
|
||||||
|
recursive empty-directory creation.
|
||||||
|
- Build: adds zlib and lz4 as link dependencies.
|
||||||
|
|
||||||
|
## [2.23.0] - 2026-09-16
|
||||||
|
|
||||||
|
### Added
|
||||||
|
|
||||||
|
- **Rsync-parity wave.** Closed the remaining CLI, filesystem, ownership,
|
||||||
|
deletion, and output gaps against rsync 3.4.1.
|
||||||
|
- Short options `-r` (`--recursive`), `-b` (`--backup`), `-L`
|
||||||
|
(`--copy-links`), and `-B` (`--block-size`/`--delta-block`); rsync
|
||||||
|
short-option clustering (`-av`, `-aAX`, `-rlpt`) and attached/inline values
|
||||||
|
(`--opt=value`, `-B1000`, `-essh`, `-MOPT`). A value that starts with `-`
|
||||||
|
is not mistaken for a cluster.
|
||||||
|
- `-c`/`--checksum` now implies the incremental checksum quick-check (and,
|
||||||
|
like rsync, does not imply `-t`).
|
||||||
|
- `--checksum-choice`/`--cc` accepts `xxh64`/`xxhash`/`xxh3`/`xxh128`/`md5`/
|
||||||
|
`auto` and rejects `md4`/`sha1`/`none` and the two-name form by name;
|
||||||
|
`--checksum-seed=0` (the default) is randomized per transfer and the chosen
|
||||||
|
seed is sent to the receiver.
|
||||||
|
- `--compress-choice`/`--zc` accepts `zstd`/`none`/`auto` and rejects
|
||||||
|
`lz4`/`zlib`/`zlibx` by name; `--skip-compress` defaults to rsync 3.4.1's
|
||||||
|
built-in suffix list; `--no-whole-file` is accepted.
|
||||||
|
- `--timeout` defaults to 0 (disabled) and `--contimeout` to 60 s (both `0`
|
||||||
|
disables), matching rsync; `--max-alloc=0` means no local limit.
|
||||||
|
- `--temp-dir` is confined to the receive root (absolute/`..` rejected by the
|
||||||
|
receiver) and an `EXDEV` install falls back to a non-atomic copy.
|
||||||
|
- `--numeric-ids` is documented as a mapping modifier only;
|
||||||
|
`--usermap`/`--groupmap` support inclusive `LOW-HIGH` ranges, `*`,
|
||||||
|
empty-`FROM` (unnamed ids), and receiver-resolved `TO` names; `--chown`
|
||||||
|
conflicts with a map on the same side are rejected.
|
||||||
|
- `--fake-super` records the *resolved* owner (never a real chown) and replays
|
||||||
|
mode/time; directory ownership and directory xattrs/ACLs are preserved.
|
||||||
|
- `-l`/`--links` stores symlink targets verbatim (absolute and `..`-bearing
|
||||||
|
included), matching rsync; `--safe-links`/`--copy-unsafe-links` are applied
|
||||||
|
sender-side and `--munge-links` uses rsync's `/rsyncd-munged/` marker;
|
||||||
|
`--trust-sender` no longer affects symlink targets.
|
||||||
|
- `--specials` recreates unix sockets with `mknod(S_IFSOCK)` (so `-D` covers
|
||||||
|
the full rsync node set).
|
||||||
|
- Deletion: the manifest carries a synchronized-directory section so
|
||||||
|
`--files-from` subsets no longer delete untransmitted paths;
|
||||||
|
`--delete-excluded` leaves size-pruned mirrors protected; extraneous
|
||||||
|
destination symlinks are unlinked (never followed); `--max-delete=N` is
|
||||||
|
partial (delete up to N, skip the rest, exit 25) and `--delete-missing-args`
|
||||||
|
removals draw from the same budget; `--force` is honored during
|
||||||
|
`--delay-updates` publication.
|
||||||
|
- `-x`/`--one-file-system` emits the mount-point directory entry; the
|
||||||
|
`--include`/`--exclude` layers are an ordered first-match rule list.
|
||||||
|
- `--chmod` is a faithful port of rsync 3.4.1 (numeric/symbolic, `D`/`F`/`X`,
|
||||||
|
`s`/`t`, append semantics, no `-p` implication, no sanitization).
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- `PROTOCOL_VERSION` bumped `2.22.0 → 2.23.0`: the delete manifest gains a
|
||||||
|
synchronized-directory section and the terminal status gains
|
||||||
|
`STATUS_DELETE_LIMIT` (client exit 25 on a `--max-delete`-capped commit).
|
||||||
|
- **The 2.22.0 mode-masking divergence is removed.** Under `-p` the source mode
|
||||||
|
is copied exactly, including `S_IWGRP`/`S_IWOTH` and setuid/setgid/sticky;
|
||||||
|
`--chmod` no longer implies `-p`. New files without `-p` still use
|
||||||
|
`source_mode & ~umask` when metadata is present (else `0644`), and new
|
||||||
|
directories without `-p` still use the `0755` creation default.
|
||||||
|
- `--protocol=NUM` accepts only the current `2.23.0` version string.
|
||||||
|
|
||||||
|
### Notes
|
||||||
|
|
||||||
|
- The rsync-compatibility matrix (`RSYNC_COMPAT.md`) now classifies every row
|
||||||
|
as **parity**, **caveat** (works with a documented divergence), or
|
||||||
|
**divergent** (not supported/no-op/impossible), replacing the previous
|
||||||
|
misleading "N implemented / 0 divergence" summary. Durable documented
|
||||||
|
divergences remain: receiver-side symlink target containment is not enforced
|
||||||
|
by default (verbatim storage is rsync parity; use `--safe-links`),
|
||||||
|
`--temp-dir` rejects absolute/foreign-filesystem paths, `--copy-devices`
|
||||||
|
reads a bounded `st_size`, a broken referent under `--copy-links` exits 0,
|
||||||
|
new directories without `-p` use `0755`, `--stats` receiver-only counters are
|
||||||
|
0, and `--password-file`/`--early-input`/`--hash-credentials`/`--iterations`
|
||||||
|
and the batch format are FastSync-native.
|
||||||
|
|
||||||
|
## [2.22.0] - 2026-09-15
|
||||||
|
|
||||||
|
### Added
|
||||||
|
|
||||||
|
- **Per-attribute metadata preservation (protocol 2.22.0).** The former single
|
||||||
|
metadata bundle is split into four independent, rsync-compatible flags:
|
||||||
|
`-p/--perms`, `-t/--times`, `-o/--owner`, and `-g/--group`, each applied
|
||||||
|
independently on the receiver, with negations `--no-perms`/`--no-times`/
|
||||||
|
`--no-owner`/`--no-group` (short `--no-p`/`--no-t`/`--no-o`/`--no-g`) and
|
||||||
|
`--no-preserve` clearing all four. `-a/--archive` is now full rsync
|
||||||
|
`-rlptgoD` (owner and group included; their application stays
|
||||||
|
privilege-gated). `-A/--acls` and `--chmod` imply `-p`, `-X/--xattrs` does
|
||||||
|
not, `-E/--executability` sets only executability, and `-U`/`-N` do not imply
|
||||||
|
`-t`. `--incremental`/`--delta` still auto-preserve perms+times unless the
|
||||||
|
user explicitly negated them.
|
||||||
|
- Receiver applies directory modes under `-p` (at the end of the transfer,
|
||||||
|
alongside the deferred directory times) and symlink mode under `-p`; `-O`
|
||||||
|
suppresses directory times only.
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- `PROTOCOL_VERSION` bumped `2.21.0 → 2.22.0`: the binary config frame gains
|
||||||
|
four appended booleans (`preserve_perms`/`preserve_times`/`preserve_owner`/
|
||||||
|
`preserve_group`) after `omit_link_times`. The fixed-width `FileMetadata`
|
||||||
|
layout is unchanged; the receiver derives the metadata-frame gate
|
||||||
|
(`use_metadata`) from the four attributes.
|
||||||
|
|
||||||
|
### Notes
|
||||||
|
|
||||||
|
- Documented divergences from rsync: a client-supplied mode never grants
|
||||||
|
group/other write (`S_IWGRP|S_IWOTH` are stripped for files, directories,
|
||||||
|
symlinks, and specials; rsync's `-p` preserves them exactly); a brand-new file
|
||||||
|
without `-p` gets `source_mode & ~umask` (sanitized) when metadata is present,
|
||||||
|
else the historical fixed `0644`; `--chmod` implies `-p` (rsync does not);
|
||||||
|
`-o`/`-g` map by name on the receiver with a raw-numeric fallback (only
|
||||||
|
numeric ids cross the wire); and a daemon module without `client owner = yes`
|
||||||
|
does not refuse a plain `-a`/`-o`/`-g` but forces super-user activities off,
|
||||||
|
applies no ownership, and logs a warning (explicit `--chown`/`--usermap`/
|
||||||
|
`--groupmap`/`--numeric-ids`/`--copy-as`/`--super` are still refused).
|
||||||
|
|
||||||
## [2.21.0] - 2026-09-14
|
## [2.21.0] - 2026-09-14
|
||||||
|
|
||||||
### Added
|
### Added
|
||||||
|
|||||||
+18
-5
@@ -1,6 +1,6 @@
|
|||||||
cmake_minimum_required(VERSION 3.22)
|
cmake_minimum_required(VERSION 3.22)
|
||||||
|
|
||||||
project(FastFileTransfer VERSION 2.21.0)
|
project(FastFileTransfer VERSION 2.26.0)
|
||||||
|
|
||||||
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
|
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
|
||||||
set(CMAKE_C_STANDARD 11)
|
set(CMAKE_C_STANDARD 11)
|
||||||
@@ -68,6 +68,16 @@ if(NOT ZSTD_LIBRARY)
|
|||||||
message(FATAL_ERROR "zstd library not found. Ensure it is in your nix-shell!")
|
message(FATAL_ERROR "zstd library not found. Ensure it is in your nix-shell!")
|
||||||
endif()
|
endif()
|
||||||
|
|
||||||
|
find_library(ZLIB_LIBRARY z)
|
||||||
|
if(NOT ZLIB_LIBRARY)
|
||||||
|
message(FATAL_ERROR "zlib library not found. Ensure zlib1g-dev / nix zlib is available!")
|
||||||
|
endif()
|
||||||
|
|
||||||
|
find_library(LZ4_LIBRARY lz4)
|
||||||
|
if(NOT LZ4_LIBRARY)
|
||||||
|
message(FATAL_ERROR "lz4 library not found. Ensure liblz4-dev / nix lz4 is available!")
|
||||||
|
endif()
|
||||||
|
|
||||||
find_package(OpenSSL REQUIRED)
|
find_package(OpenSSL REQUIRED)
|
||||||
|
|
||||||
# --- Explicit source lists ---
|
# --- Explicit source lists ---
|
||||||
@@ -89,6 +99,7 @@ set(SHARED_SRCS
|
|||||||
src/shared/daemon_limits.c
|
src/shared/daemon_limits.c
|
||||||
src/shared/data.c
|
src/shared/data.c
|
||||||
src/shared/delay_updates.c
|
src/shared/delay_updates.c
|
||||||
|
src/shared/delete_plan.c
|
||||||
src/shared/delta.c
|
src/shared/delta.c
|
||||||
src/shared/file.c
|
src/shared/file.c
|
||||||
src/shared/file_list.c
|
src/shared/file_list.c
|
||||||
@@ -96,6 +107,7 @@ set(SHARED_SRCS
|
|||||||
src/shared/file_send.c
|
src/shared/file_send.c
|
||||||
src/shared/file_store.c
|
src/shared/file_store.c
|
||||||
src/shared/filter.c
|
src/shared/filter.c
|
||||||
|
src/shared/format.c
|
||||||
src/shared/hardlink.c
|
src/shared/hardlink.c
|
||||||
src/shared/identity.c
|
src/shared/identity.c
|
||||||
src/shared/log.c
|
src/shared/log.c
|
||||||
@@ -134,8 +146,8 @@ set(CLIENT_MAIN_SRCS src/client/client_cli.c)
|
|||||||
# --- Library targets ---
|
# --- Library targets ---
|
||||||
add_library(fastsync_shared STATIC ${SHARED_SRCS})
|
add_library(fastsync_shared STATIC ${SHARED_SRCS})
|
||||||
target_include_directories(fastsync_shared PUBLIC src/shared)
|
target_include_directories(fastsync_shared PUBLIC src/shared)
|
||||||
target_link_libraries(fastsync_shared PUBLIC Threads::Threads ${ZSTD_LIBRARY} OpenSSL::SSL
|
target_link_libraries(fastsync_shared PUBLIC Threads::Threads ${ZSTD_LIBRARY} ${ZLIB_LIBRARY}
|
||||||
OpenSSL::Crypto xxhash)
|
${LZ4_LIBRARY} OpenSSL::SSL OpenSSL::Crypto xxhash)
|
||||||
|
|
||||||
add_library(fastsync_client_core STATIC ${CLIENT_CORE_SRCS})
|
add_library(fastsync_client_core STATIC ${CLIENT_CORE_SRCS})
|
||||||
target_include_directories(fastsync_client_core PUBLIC src/client)
|
target_include_directories(fastsync_client_core PUBLIC src/client)
|
||||||
@@ -213,6 +225,7 @@ set(TEST_SRCS
|
|||||||
tests/test_file.c
|
tests/test_file.c
|
||||||
tests/test_file_list.c
|
tests/test_file_list.c
|
||||||
tests/test_file_sendfile.c
|
tests/test_file_sendfile.c
|
||||||
|
tests/test_format.c
|
||||||
tests/test_fuzz_smoke.c
|
tests/test_fuzz_smoke.c
|
||||||
tests/test_glob.c
|
tests/test_glob.c
|
||||||
tests/test_hardlink.c
|
tests/test_hardlink.c
|
||||||
@@ -273,7 +286,7 @@ if(ENABLE_FUZZ)
|
|||||||
target_include_directories(${FUZZ_NAME} PRIVATE tests src/shared src/server)
|
target_include_directories(${FUZZ_NAME} PRIVATE tests src/shared src/server)
|
||||||
target_compile_options(${FUZZ_NAME} PRIVATE -fsanitize=fuzzer,address,undefined -fno-omit-frame-pointer)
|
target_compile_options(${FUZZ_NAME} PRIVATE -fsanitize=fuzzer,address,undefined -fno-omit-frame-pointer)
|
||||||
target_link_options(${FUZZ_NAME} PRIVATE -fsanitize=fuzzer,address,undefined)
|
target_link_options(${FUZZ_NAME} PRIVATE -fsanitize=fuzzer,address,undefined)
|
||||||
target_link_libraries(${FUZZ_NAME} PRIVATE Threads::Threads ${ZSTD_LIBRARY} OpenSSL::SSL
|
target_link_libraries(${FUZZ_NAME} PRIVATE Threads::Threads ${ZSTD_LIBRARY} ${ZLIB_LIBRARY}
|
||||||
OpenSSL::Crypto xxhash)
|
${LZ4_LIBRARY} OpenSSL::SSL OpenSSL::Crypto xxhash)
|
||||||
endforeach()
|
endforeach()
|
||||||
endif()
|
endif()
|
||||||
|
|||||||
+15
-1
@@ -2,8 +2,22 @@ FROM ubuntu:24.04
|
|||||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
gcc g++ make libc6-dev cmake libzstd-dev libssl-dev git ca-certificates curl cppcheck clang-format \
|
gcc g++ make libc6-dev cmake libzstd-dev libssl-dev git ca-certificates curl cppcheck clang-format \
|
||||||
python3 python3-pip python3-venv openssl openssh-client \
|
python3 python3-pip python3-venv openssl openssh-client \
|
||||||
lcov valgrind clang libclang-rt-18-dev && \
|
lcov valgrind clang libclang-rt-18-dev \
|
||||||
|
acl attr zlib1g-dev liblz4-dev libxxhash-dev && \
|
||||||
pip3 install --break-system-packages pytest pytest-xdist && \
|
pip3 install --break-system-packages pytest pytest-xdist && \
|
||||||
curl -fsSL https://deb.nodesource.com/setup_20.x | bash - && \
|
curl -fsSL https://deb.nodesource.com/setup_20.x | bash - && \
|
||||||
apt-get install -y --no-install-recommends nodejs && \
|
apt-get install -y --no-install-recommends nodejs && \
|
||||||
rm -rf /var/lib/apt/lists/*
|
rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
# rsync is used as the reference implementation for drop-in parity tests.
|
||||||
|
# Ubuntu 24.04 ships 3.2.7, so build the pinned 3.4.1 reference from source.
|
||||||
|
ARG RSYNC_VERSION=3.4.1
|
||||||
|
ARG RSYNC_SHA256=2924bcb3a1ed8b551fc101f740b9f0fe0a202b115027647cf69850d65fd88c52
|
||||||
|
RUN curl -fsSL "https://download.samba.org/pub/rsync/src/rsync-${RSYNC_VERSION}.tar.gz" -o /tmp/rsync.tar.gz && \
|
||||||
|
echo "${RSYNC_SHA256} /tmp/rsync.tar.gz" | sha256sum -c - && \
|
||||||
|
tar -xzf /tmp/rsync.tar.gz -C /tmp && \
|
||||||
|
cd "/tmp/rsync-${RSYNC_VERSION}" && \
|
||||||
|
./configure --enable-zstd --enable-xxhash --enable-lz4 && \
|
||||||
|
make -j"$(nproc)" && \
|
||||||
|
make install && \
|
||||||
|
rm -rf "/tmp/rsync-${RSYNC_VERSION}" /tmp/rsync.tar.gz
|
||||||
|
|||||||
+67
@@ -0,0 +1,67 @@
|
|||||||
|
# FastSync — Session Handoff (2026-09-17)
|
||||||
|
|
||||||
|
## Current status
|
||||||
|
- **Release `v2.21.0`** tagged (`919a729`, "Release v2.21.0"); full CI green
|
||||||
|
(run 552: lint, build-and-test, ASan, UBSan, fuzz-build, coverage, valgrind).
|
||||||
|
`dev` has the release commit plus later doc-only merges (a README refresh and
|
||||||
|
this handoff).
|
||||||
|
- **Release PR #284 (`dev` -> `main`)** open, CI green (run 553).
|
||||||
|
`main` is protected: it needs review/approval to merge.
|
||||||
|
https://gitea.tap-tap.win/TapTap/FastSync/pulls/284
|
||||||
|
- **`PROTOCOL_VERSION` = `"2.26.0"`** (`src/shared/config.h`); CMake
|
||||||
|
`project(FastFileTransfer VERSION 2.26.0)`.
|
||||||
|
- Working tree clean; no wave worktrees remain.
|
||||||
|
|
||||||
|
## What landed this session
|
||||||
|
1. **Wave 8 (refactors):** Config X-macro wire table; single-owner `authorized_root`;
|
||||||
|
daemon per-module/per-host caps + cross-process auth lockout (`daemon_limits.[ch]`);
|
||||||
|
`Data` charge returns to its owning `ProtocolSession`.
|
||||||
|
2. **Wave 9 (protocol 2.21.0):** optional `STATUS_ERROR_DETAIL` rejection reasons;
|
||||||
|
server-contacting `--dry-run` (`STATUS_DRY_RUN_TRANSFER`, receiver mutates nothing).
|
||||||
|
3. **Security wave:** ran 5 parallel audits (wire parsing; daemon/transport/TLS/auth;
|
||||||
|
receiver confinement; client/CLI/SSH; crypto/memory/limits). Fixed all HIGH and the
|
||||||
|
confirmed MEDIUMs:
|
||||||
|
- SSH `-o ProxyCommand=…` argument injection (RCE) — reject leading `-`, insert `--`.
|
||||||
|
- Truncated zstd frame infinite CPU loop (remote DoS).
|
||||||
|
- FIFO receiver opens lacked `O_NONBLOCK` (indefinite hang).
|
||||||
|
- `--inplace` could write a FIFO/device (bypass of `--write-devices` gate).
|
||||||
|
- `--force` not gated by server `--allow-delete`.
|
||||||
|
- Privileged standalone server defaulted super activities on; added `--allow-super`
|
||||||
|
(never honored with `--stdio`).
|
||||||
|
- `--dry-run` content/hash oracle on `read only`/basis files removed.
|
||||||
|
- Empty `hosts allow`/`deny`/`auth users` now rejected.
|
||||||
|
- TLS: AEAD-only 1.2 + server preference, TOCTOU-safe key load, IP-SAN verify,
|
||||||
|
CN-truncation guard. Glob backtracking bounded; line reads bounded; ACL xattrs
|
||||||
|
gated on `--acls`; decompression/chunk memory charged; pre-auth `basis_count`
|
||||||
|
NULL-deref fixed.
|
||||||
|
4. **Tooling:** benchmark accuracy (data mix, verification, percentiles, `tc`,
|
||||||
|
`build-bench/`, `--warm` mode); `shell.nix` full toolchain and no build-on-entry;
|
||||||
|
docs state push-only / remote-source unsupported.
|
||||||
|
5. **Preserve-attribute split (protocol 2.22.0)** landed on `feat/preserve-attr-split`: per-attribute `-p/-t/-o/-g` + `--no-*` negations, `-a` = `-rlptgoD`, and the 2.21.0 → 2.22.0 wire bump.
|
||||||
|
6. **Rsync-parity wave (protocol 2.23.0)** on `feat/rsync-parity`: rsync short options/clustering/attached values (`-r`/`-b`/`-L`/`-B`, `-av`, `-aAX`, `-B1000`, `-essh`, `-MOPT`), `-c` checksum quick-check, `--checksum-choice`/`--compress-choice` validation and seed randomization, rsync timeout/max-alloc defaults, temp-dir confinement + `EXDEV` fallback, ownership/mapping parity (numeric-ids modifier, map ranges/`*`/empty-FROM, `--chown`+map conflicts, fake-super resolved-owner record), verbatim symlink storage with rsync `--safe-links`/`--munge-links`, socket recreation under `--specials`, `--chmod` 3.4.1 semantics, and delete scoping + `--max-delete` partial/exit-25. Wire: appended delete-manifest synchronized-directory section and `STATUS_DELETE_LIMIT`.
|
||||||
|
7. **Parity-completion wave (protocol 2.24.0 → 2.26.0)** on `feat/parity-completion`: per-directory delete plans (`STATUS_DELETE_PLAN`) for `--delete-during`/`--delete-delay`; receiver `STATUS_STATS` counters feeding `--stats`/`--progress` and `--out-format %b/%c/%C`, plus `-n --delete` lines; `lz4`/`zlib`/`zlibx` compression and `md4`/`sha1`/`none` checksums with `auto` negotiation (default `xxh128`/`zstd`); general `-R`/`--no-implied-dirs`/`-d`; the full filter grammar (`merge`/`dir-merge`/`hide`/`show`/`protect`/`risk`/`clear` + modifiers) and corrected `-F`/`-FF`; receiver-side `--chown`/map TO-name resolution; absolute basis dirs + `--link-dest` relink; receiver-side `--ignore-existing` short-circuit; `--preallocate` over `--sparse` via `fallocate(2)`; `--iconv=.`/`-`/`--no-iconv`; lone `-h` help; aliases `--ignore-non-existing`/`--protect-args`/`--msgs2stderr`; and the full `--info`/`--debug` vocabulary. `RSYNC_COMPAT.md` reclassifies the matrix to 106 ✅ / 27 ⚠️ / 23 ❌.
|
||||||
|
|
||||||
|
## Next steps
|
||||||
|
1. **Merge PR #284** (`dev` -> `main`) once reviewed (protected branch).
|
||||||
|
2. **Deferred security items** (documented, not implemented):
|
||||||
|
- Pre-auth config/daemon-auth handshake has no aggregate wall-clock deadline
|
||||||
|
(per-message timeout only) — slowloris holds connection slots.
|
||||||
|
- Per-source registry fails open when the shared table is full (per-module/global
|
||||||
|
caps and host ACLs still apply); consider fail-closed or larger/evicting table.
|
||||||
|
- SCRAM-like daemon auth has no TLS channel binding (and is not RFC 5802).
|
||||||
|
- `cleanup()` signal handler calls non-async-signal-safe teardown; daemon `umask(0)`.
|
||||||
|
- Wire protocol assumes homogeneous word size/endianness (lengths are native
|
||||||
|
`size_t`) — document or move to fixed-width framing.
|
||||||
|
3. **Out of scope / intentional:** pull (remote source) mode is **not** planned —
|
||||||
|
FastSync is push-only; see `RSYNC_COMPAT.md#direction`.
|
||||||
|
|
||||||
|
## Key facts / commands
|
||||||
|
- CI image: `gitea.tap-tap.win/taptap/fastsync-ci:v11` (alias `fastsync-ci:local`).
|
||||||
|
- Build/test: `cmake -B build -S . -DSTRICT_WARNINGS=ON && cmake --build build -j$(nproc) && ./build/tests`
|
||||||
|
then `python3 -m pytest tests/integration/ -n 4 --dist=load -m "not setpriv"`.
|
||||||
|
- Dev shell: `nix-shell` (provides clang-format, cppcheck, pytest-xdist, openssh,
|
||||||
|
rsync, iproute2, valgrind, lcov; does not build on entry).
|
||||||
|
- Gitea API token: supplied out-of-band via the `TOKEN` environment variable; it is
|
||||||
|
intentionally **not** recorded in this file.
|
||||||
|
- CI polling: `GET /api/v1/repos/TapTap/FastSync/actions/runs?limit=N`, match `head_sha`,
|
||||||
|
then `/actions/runs/<id>/jobs`.
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
#FastSync
|
# FastSync
|
||||||
|
|
||||||
FastSync is a high-performance file synchronization tool designed to become a
|
FastSync is a high-performance file synchronization tool designed to become a
|
||||||
drop-in replacement for common `rsync` workflows. It keeps the familiar
|
drop-in replacement for common `rsync` workflows. It keeps the familiar
|
||||||
@@ -7,7 +7,7 @@ multithreading, streaming zstd compression, chunking, zero-copy TCP transfers,
|
|||||||
and native TCP/TLS transports.
|
and native TCP/TLS transports.
|
||||||
|
|
||||||
The release version is FastSync's client/server protocol version (printed by
|
The release version is FastSync's client/server protocol version (printed by
|
||||||
`fastsync --version`); client and server must match. See
|
`./build/client --version`); client and server must match. See
|
||||||
[CHANGELOG.md](CHANGELOG.md) for the history.
|
[CHANGELOG.md](CHANGELOG.md) for the history.
|
||||||
|
|
||||||
The compatibility target is straightforward:
|
The compatibility target is straightforward:
|
||||||
@@ -28,7 +28,7 @@ FastSync uses a producer-consumer transfer pipeline and can combine several
|
|||||||
optimizations for large or high-latency transfers:
|
optimizations for large or high-latency transfers:
|
||||||
|
|
||||||
- Multithreaded scanning, loading, and sending.
|
- Multithreaded scanning, loading, and sending.
|
||||||
- Streaming zstd compression with levels 1 through 22.
|
- Streaming compression (zstd by default, plus lz4/zlib/zlibx) with levels 1 through 22.
|
||||||
- Configurable file chunking and compact chunk serialization.
|
- Configurable file chunking and compact chunk serialization.
|
||||||
- `sendfile()` zero-copy transfers over TCP.
|
- `sendfile()` zero-copy transfers over TCP.
|
||||||
- Batched incremental checks to reduce round trips.
|
- Batched incremental checks to reduce round trips.
|
||||||
@@ -51,28 +51,51 @@ replacement for every rsync feature or protocol mode.
|
|||||||
- Rsync-style source and destination arguments.
|
- Rsync-style source and destination arguments.
|
||||||
- SSH transport using `user@host:destination` paths below the remote authorized root.
|
- SSH transport using `user@host:destination` paths below the remote authorized root.
|
||||||
- TCP client/server transfers.
|
- TCP client/server transfers.
|
||||||
- Dry runs, excludes, includes, size filters, backups, statistics, and
|
- Dry runs (server-contacting since protocol 2.21.0 for server-routed targets),
|
||||||
bandwidth limiting.
|
excludes, includes, size filters, backups, statistics, and bandwidth
|
||||||
- Incremental size/mtime checks and optional xxHash64 content checks.
|
limiting.
|
||||||
|
- Incremental size/mtime checks and optional content checks (`xxh128` by
|
||||||
|
default, selectable with `--checksum-choice`).
|
||||||
- FastSync-native delta transfer for changed files.
|
- FastSync-native delta transfer for changed files.
|
||||||
- Optional mode and timestamp preservation.
|
- Optional mode and timestamp preservation.
|
||||||
- Delete manifests with server-side delete authorization.
|
- Delete manifests with server-side delete authorization.
|
||||||
- Temporary-file writes with atomic rename by default.
|
- Temporary-file writes with atomic rename by default.
|
||||||
- Path traversal checks and destination-root confinement.
|
- Path traversal checks and destination-root confinement.
|
||||||
|
|
||||||
### Not yet equivalent to rsync
|
### Boundaries and documented divergences
|
||||||
|
|
||||||
|
The items below summarize FastSync's rsync compatibility status — recently
|
||||||
|
closed gaps and the remaining known divergences. Each row of the detailed
|
||||||
|
matrix is classified as parity, caveat, or divergent in
|
||||||
|
[`RSYNC_COMPAT.md`](RSYNC_COMPAT.md).
|
||||||
|
|
||||||
- The FastSync wire protocol is not the rsync wire protocol.
|
- The FastSync wire protocol is not the rsync wire protocol.
|
||||||
- SSH mode requires `fastsync-server` on the remote host.
|
- SSH mode requires `fastsync-server` on the remote host.
|
||||||
- Archive mode does not yet provide all of rsync's `-rlptgoD` behavior.
|
- Archive mode covers rsync's `-rlptgoD` behavior — links, permissions, times,
|
||||||
- Symlink transfer is incomplete; link targets are not yet recreated in all
|
owner, group, devices, and special files — and does not imply compression or
|
||||||
modes.
|
multithreading (see [Client](#client)). Ownership application is still
|
||||||
- Owner/group, ACL, xattr, and hard-link handling is incomplete or
|
privilege-gated: a receiver that cannot `chown` logs a warning and skips it.
|
||||||
unavailable.
|
Under `-p` the source mode is copied exactly, including setuid/setgid/sticky
|
||||||
|
and group/other-write bits (strict rsync parity; see
|
||||||
|
[`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)).
|
||||||
|
- Symlink transfer stores targets **verbatim** (`-l`/`--links`), including
|
||||||
|
absolute and `..`-bearing targets, matching rsync. The receiver does not
|
||||||
|
enforce a containment predicate by default; `--safe-links` drops unsafe
|
||||||
|
targets on the sender, and `--munge-links` rewrites them with rsync's
|
||||||
|
`/rsyncd-munged/` marker. `--trust-sender` does not affect symlink targets.
|
||||||
|
A destination later consumed by a link-following tool can therefore follow a
|
||||||
|
link outside the receive root — use `--safe-links` for untrusted sources.
|
||||||
|
- Hard links (`-H`/`--hard-links`), extended attributes (`-X`/`--xattrs`), and
|
||||||
|
POSIX ACLs (`-A`/`--acls`) are preserved; owner/group is applied through
|
||||||
|
`-o`/`-g` (or an `-a`/`--archive` transfer), through the opt-in identity flags
|
||||||
|
(`--chown`/`--usermap`/`--groupmap`/`--numeric-ids`/`--copy-as`), and only when
|
||||||
|
the receiver has permission. See
|
||||||
|
[`RSYNC_COMPAT.md`](RSYNC_COMPAT.md) for the exact semantics and documented
|
||||||
|
divergences.
|
||||||
- Device and special-file preservation is implemented with documented
|
- Device and special-file preservation is implemented with documented
|
||||||
divergences: recreated device nodes require `CAP_MKNOD` on the receiver (a
|
divergences: recreated device nodes require `CAP_MKNOD` on the receiver (a
|
||||||
non-root receiver skips the entry), and sockets cannot be recreated (FIFOs
|
non-root receiver skips the entry), while FIFOs **and unix sockets** are
|
||||||
are).
|
recreated (`--specials`).
|
||||||
- Sparse-file hole preservation (`-S`, `--sparse`) is implemented receiver-side:
|
- Sparse-file hole preservation (`-S`, `--sparse`) is implemented receiver-side:
|
||||||
long all-zero runs are written as holes (no wire change; the full file image
|
long all-zero runs are written as holes (no wire change; the full file image
|
||||||
is already in memory).
|
is already in memory).
|
||||||
@@ -80,78 +103,156 @@ replacement for every rsync feature or protocol mode.
|
|||||||
the write atomic (temp + rename). With `--partial`, a failed/interrupted write
|
the write atomic (temp + rename). With `--partial`, a failed/interrupted write
|
||||||
now retains the already-written temp at the destination path (best-effort) so
|
now retains the already-written temp at the destination path (best-effort) so
|
||||||
a later `--append`/`--append-verify` run can resume it.
|
a later `--append`/`--append-verify` run can resume it.
|
||||||
- `--dirs` is not implemented. Its compatibility aliases `--old-dirs` and
|
- `-d`/`--dirs` and its aliases `--old-dirs`/`--old-d` transfer the named
|
||||||
`--old-d` are recognized but rejected explicitly rather than silently using
|
directory entries without recursing into their contents.
|
||||||
FastSync's recursive directory behavior.
|
|
||||||
- Short-option names are now rsync-parity (Phase 7 Wave A): FastSync's former
|
- Short-option names are now rsync-parity (Phase 7 Wave A): FastSync's former
|
||||||
collisions were renamed (`-j`/`--threads`, `--preserve`, `--sendfile`,
|
collisions were renamed (`-j`/`--threads`, `--preserve`, `--sendfile`,
|
||||||
`--chunk-serialization`, `--timeout`, `--ssh-port`), so `-m`, `-M`, `-f`,
|
`--chunk-serialization`, `--timeout`, `--ssh-port`), so `-m`, `-M`, `-f`,
|
||||||
`-s`, `-T`, `-p`, `-c`, `-a`, and `-z` follow rsync. See `RSYNC_COMPAT.md`.
|
`-s`, `-T`, `-p`, `-c`, `-a`, and `-z` follow rsync.
|
||||||
|
- Short-option clustering (`-av`, `-aAX`, `-rlpt`) and attached values
|
||||||
|
(`-B1000`, `-essh`, `-MOPT`, `--opt=value`) are accepted, matching rsync.
|
||||||
|
- `-r`, `-b`, `-L`, and `-B` are parsed with the rsync short names.
|
||||||
|
- `--stats` prints the counters FastSync can observe plus the receiver-only
|
||||||
|
counters (`Matched data`, deleted files) reported over the wire; rsync's
|
||||||
|
per-type `Number of files` breakdown is not reproduced. `--progress` prints
|
||||||
|
rsync-style per-file blocks (without rsync's leading `./` line).
|
||||||
|
- Codecs match rsync 3.4.1: `zstd`/`lz4`/`zlib`/`zlibx` compression and
|
||||||
|
`xxh128`/`xxh3`/`xxh64`/`md5`/`md4`/`sha1`/`none` checksums, negotiated with
|
||||||
|
`auto`; `zlibx` behaves as `zlib`, and the transfer checksum is not separately
|
||||||
|
selectable.
|
||||||
|
|
||||||
The detailed flag matrix is maintained in
|
The detailed flag matrix is maintained in
|
||||||
[`RSYNC_COMPAT.md`](RSYNC_COMPAT.md). It distinguishes implemented,
|
[`RSYNC_COMPAT.md`](RSYNC_COMPAT.md). It reports each row as **parity**,
|
||||||
partial, alternate, and planned behavior.
|
**caveat** (works with a documented divergence), or **divergent** (not
|
||||||
|
supported), rather than treating "parsed" as parity.
|
||||||
|
|
||||||
## Quick Start
|
## Quick Start
|
||||||
|
|
||||||
### Build
|
### Build
|
||||||
|
|
||||||
`compile_commands.json` is a symlink to `build/compile_commands.json` and is used by clangd/editor tooling; its target is generated by the build, so it dangles until the first build.
|
```bash
|
||||||
|
cmake -B build -S .
|
||||||
|
cmake --build build -j$(nproc)
|
||||||
|
```
|
||||||
|
|
||||||
|
This produces `./build/client` and `./build/server`. `compile_commands.json` is a symlink to `build/compile_commands.json` and is used by clangd/editor tooling; its target is generated by the build, so it dangles until the first build.
|
||||||
|
|
||||||
### Client
|
### Client
|
||||||
|
|
||||||
| Argument | Description |
|
| Argument | Description |
|
||||||
|----------|-------------|
|
|----------|-------------|
|
||||||
| Positional | `<source> <dest>` — automatic SSH detection if dest contains `:` |
|
| Positional | `<source> <dest>` — automatic SSH detection if dest contains `:` |
|
||||||
| `-c, --checksum` | Verify content by checksum instead of size+mtime |
|
| `-c, --checksum` | Verify content by checksum instead of size+mtime (implies the incremental checksum quick-check) |
|
||||||
| `-z, --compress [level]` | Enable streaming zstd compression (level 1–22, default 5) |
|
| `--checksum-choice <alg>` | Whole-file checksum algorithm: `xxh128` (default), `xxh3`, `xxh64`/`xxhash`, `md5`, `md4`, `sha1`, `none`, or `auto` (plus rsync's two-name `transfer,pre-transfer` form) |
|
||||||
| `-a, --archive` | rsync archive mode (`-rlptgoD`): links, metadata, devices and specials (not compression/multithreading) |
|
| `-z, --compress [level]` | Enable streaming compression (default `zstd`; level 1–22, default 5) |
|
||||||
|
| `--compress-choice <alg>` | Compression algorithm: `zstd` (default), `lz4`, `zlib`, `zlibx`, `none`, or `auto` |
|
||||||
|
| `--skip-compress <list>` | Skip compression for suffixes (`/`- or `,`-separated); defaults to rsync 3.4.1's built-in suffix list |
|
||||||
|
| `-a, --archive` | rsync archive mode (`-rlptgoD`): links, perms, times, owner, group, devices and specials; ownership application stays privilege-gated (not compression/multithreading) |
|
||||||
| `-j, --threads[=N]` | Multithreading mode; `N` (1–256) sets the parallel scanner worker count, bare `-j`/`--threads` uses the default |
|
| `-j, --threads[=N]` | Multithreading mode; `N` (1–256) sets the parallel scanner worker count, bare `-j`/`--threads` uses the default |
|
||||||
| `-m` | rsync `--prune-empty-dirs` (short form now rsync-parity) |
|
| `-m` | rsync `--prune-empty-dirs` (short form now rsync-parity) |
|
||||||
|
| `-r, --recursive` | Recurse into directories (FastSync is always recursive; accepted for rsync compatibility) |
|
||||||
|
| `-d, --dirs` | Transfer the named directory entries without recursing into their contents; aliases `--old-dirs`/`--old-d` |
|
||||||
|
| `-R, --relative` | Use rsync's relative path semantics (including the `/./` cut); with `--files-from`, preserve each listed entry's relative path below the destination root |
|
||||||
| `--chunk-serialization` | Chunk serialization (batch all files per chunk; long form only) |
|
| `--chunk-serialization` | Chunk serialization (batch all files per chunk; long form only) |
|
||||||
| `-s` | rsync `--secluded-args` compatibility no-op (remote SSH argv is already injection-safe) |
|
| `-s` | rsync `--secluded-args` compatibility no-op (remote SSH argv is already injection-safe) |
|
||||||
| `--sendfile` | Sendfile zero-copy. Incompatible with compression / chunk serialization. TCP only. Long form only. |
|
| `--sendfile` | Sendfile zero-copy. Incompatible with compression / chunk serialization. TCP only. Long form only. |
|
||||||
| `--preserve` | Preserve supported file metadata (mode and mtime; ownership and atime are unsupported) |
|
| `--preallocate` | Allocate destination file space up front (fail-fast on a full disk) |
|
||||||
| `-n, --dry-run` | Scan and print what would be transferred |
|
| `--append` | Resume a shorter destination by appending only its tail (prefix not verified; requires `--incremental`) |
|
||||||
| `-p, --perms` | Preserve permission bits (part of the metadata bundle) |
|
| `--append-verify` | Like `--append`, but verifies the retained prefix checksum first (falls back to a full transfer on mismatch) |
|
||||||
| `--ssh-port <port>` | SSH port (default: 22) |
|
| `-W, --whole-file` | Transfer changed files without delta processing; `--no-whole-file` clears it |
|
||||||
| `-v, --verbose` | Enable debug logging |
|
| `-B <n>, --block-size <n>` | Delta block size in bytes (alias `--delta-block`) |
|
||||||
| `-q, --quiet` | Suppress non-error output |
|
| `--checksum-seed <n>` | Seed for the whole-file xxHash digest; an unset/`0` seed is randomized per transfer, matching rsync |
|
||||||
| `--progress` | Show real-time transfer speed |
|
| `-I, --ignore-times` | Transfer files even when size and mtime match |
|
||||||
| `-P` | Enables partial-transfer mode + progress output; interrupted writes retain the already-written temp for resumption |
|
| `--size-only` | Skip incremental files matching in size, ignoring mtime |
|
||||||
| `--delete` | Delete files on receiver not present in source (default timing: delete-after, i.e. only after the whole transfer succeeded) |
|
| `--preserve` | Preserve mode and mtime (`-p` + `-t`; add `-o`/`-g` for owner/group or `-U`/`--atimes` for atime; `-N`/`--crtimes` captures birth time but cannot apply it) |
|
||||||
|
| `-U, --atimes` | Preserve access times. Captured with the metadata payload; does not enable ownership. |
|
||||||
|
| `-N, --crtimes` | Capture birth time; cannot be applied (documented divergence) |
|
||||||
|
| `-p, --perms` | Preserve permission bits. Strict rsync parity: the source mode is copied exactly, including setuid/setgid/sticky and group/other-write bits |
|
||||||
|
| `-t, --times` | Preserve modification times |
|
||||||
|
| `-o, --owner` | Preserve the source owner (privilege-gated; mapped by name on the receiver with a numeric fallback) |
|
||||||
|
| `-g, --group` | Preserve the source group (privilege-gated; mapped by name on the receiver with a numeric fallback) |
|
||||||
|
| `--no-perms`, `--no-times`, `--no-owner`, `--no-group`, `--no-preserve` | Negate the per-attribute flags (short `--no-p`/`--no-t`/`--no-o`/`--no-g`; `--no-preserve` clears all four) |
|
||||||
|
| `-E, --executability` | Preserve executable permission bits |
|
||||||
|
| `-X, --xattrs` | Preserve user `user.*` extended attributes |
|
||||||
|
| `-A, --acls` | Preserve POSIX ACLs |
|
||||||
|
| `--chmod <changes>` | Modify transferred permissions (rsync syntax) |
|
||||||
|
| `--chown=USER:GROUP` | Override the ownership of transferred files |
|
||||||
|
| `--usermap=MAP` | Map usernames when applying ownership |
|
||||||
|
| `--groupmap=MAP` | Map group names when applying ownership |
|
||||||
|
| `--numeric-ids` | Apply source numeric uid/gid directly instead of mapping by name |
|
||||||
|
| `--copy-as=USER[:GROUP]` | Force every written entry to USER[:GROUP] (requires a privileged receiver) |
|
||||||
|
| `--fake-super` | Record the resolved owner plus mode/time in a reserved `user.fastsync.stat` xattr and replay mode/time; never performs a real chown |
|
||||||
|
| `--super` | Permit the receiver to attempt confined super-user activities (device nodes) |
|
||||||
|
| `-D` | Preserve device and special files (implies `--devices --specials`) |
|
||||||
|
| `--devices` | Recreate device nodes on the destination (privileged; skipped without `CAP_MKNOD`) |
|
||||||
|
| `--specials` | Recreate special files: FIFOs and unix sockets |
|
||||||
|
| `--remove-source-files` | Remove regular source files after a successful transfer |
|
||||||
|
| `--exclude <pattern>` | Exclude files matching glob pattern (repeatable) |
|
||||||
|
| `--exclude-from <file>` | Read exclude patterns from a file (one per line) |
|
||||||
|
| `--include <pattern>` | Only transfer files matching glob pattern (repeatable, whitelist) |
|
||||||
|
| `--include-from <file>` | Read include patterns from a file |
|
||||||
|
| `--files-from <file>` | Read the source file list from FILE (paths relative to the source root) |
|
||||||
|
| `--max-size <n>` | Skip files larger than n bytes |
|
||||||
|
| `--min-size <n>` | Skip files smaller than n bytes |
|
||||||
|
| `-x, --one-file-system` | Do not cross filesystem boundaries; the mount-point directory entry is emitted (empty at the destination) without descending |
|
||||||
|
| `--max-alloc <SIZE>` | Maximum single allocation (binary units: B, K, M, G, T, P, E; default 1G; `0` = no local limit, matching rsync) |
|
||||||
|
| `-u, --update` | Skip files newer than the source on the receiver |
|
||||||
|
| `--incremental` | Skip files unchanged since last transfer (size + mtime). Auto-enables `--preserve`. Incompatible with `--chunk-serialization`. |
|
||||||
|
| `--existing` | Skip files not already present at the destination; update existing files normally. |
|
||||||
|
| `--compare-dest <dir>` | Extra comparison basis: unchanged files are not transferred (requires/implies `--incremental`) |
|
||||||
|
| `--copy-dest <dir>` | Like `--compare-dest`, but copies the unchanged file from DIR into the destination |
|
||||||
|
| `--link-dest <dir>` | Like `--copy-dest`, but hard-links the unchanged file from DIR (repeatable; earlier DIRs win) |
|
||||||
|
| `--delete` | Delete files on receiver not present in source (default timing: delete-after, i.e. only after the whole transfer succeeded). Scoped to the synchronized directories, so `--files-from` subsets are safe |
|
||||||
| `--delete-before` | Delete extras before the transfer starts (implies `--delete`) |
|
| `--delete-before` | Delete extras before the transfer starts (implies `--delete`) |
|
||||||
| `--delete-during`, `--del` | Delete extras once the keep-set is known, before data is applied (implies `--delete`) |
|
| `--delete-during`, `--del` | Delete extras once the keep-set is known, before data is applied (implies `--delete`) |
|
||||||
| `--delete-delay` | Delete extras only after a successful transfer (implies `--delete`) |
|
| `--delete-delay` | Delete extras only after a successful transfer (implies `--delete`) |
|
||||||
| `--delete-after` | Explicit delete-after timing (implies `--delete`) |
|
| `--delete-after` | Explicit delete-after timing (implies `--delete`) |
|
||||||
| `--exclude <pattern>` | Exclude files matching glob pattern (repeatable) |
|
| `--delete-excluded` | Also delete filter-excluded destination mirrors (size-pruned mirrors stay protected) |
|
||||||
| `--exclude-from <file>` | Read exclude patterns from a file (one per line) |
|
| `--max-delete <n>` | Delete at most n destination entries; the rest are skipped and the run exits 25 (partial), matching rsync |
|
||||||
| `--include <pattern>` | Only transfer files matching glob pattern (repeatable, whitelist) |
|
| `--delay-updates` | Put updated files into place only at the end of the transfer (`--force` is honored at publication) |
|
||||||
| `--max-size <n>` | Skip files larger than n bytes |
|
| `-T, --temp-dir <dir>` | Scratch directory for temp files before the atomic install; confined to the receive root (relative only), with an `EXDEV` non-atomic copy fallback |
|
||||||
| `--min-size <n>` | Skip files smaller than n bytes |
|
| `-n, --dry-run` | Report what would be transferred without mutating the destination. Since protocol 2.21.0 a server-routed target contacts the receiver and reports would-transfer based on receiver state; a plain local destination keeps the client-side scan. Never mutates or deletes. |
|
||||||
| `--max-alloc <SIZE>` | Maximum single allocation (binary units: B, K, M, G, T, P, E; default 1G) |
|
| `-v, --verbose` | Enable debug logging |
|
||||||
| `--incremental` | Skip files unchanged since last transfer (size + mtime). Auto-enables `--preserve`. Incompatible with `--chunk-serialization`. |
|
| `-q, --quiet` | Suppress non-error output |
|
||||||
| `--existing` | Skip files not already present at the destination; update existing files normally. |
|
| `--progress` | Show rsync-style per-file progress blocks from the receiver's wire counters (FastSync does not print rsync's leading `./` line) |
|
||||||
| `--bwlimit <KB/s>` | Bandwidth limit in kilobytes per second |
|
| `-P` | Enables partial-transfer mode + progress output; interrupted writes retain the already-written temp for resumption |
|
||||||
| `--chunk-size <n>` | Chunk size in bytes (default: 10485760) |
|
| `--stats` | Print transfer statistics at end (bytes, files, timing), including the receiver-only counters reported over the wire; rsync's per-type `Number of files` breakdown is not reproduced |
|
||||||
| `--timeout <sec>` | Positive I/O timeout in seconds, applied to both the socket (`SO_RCVTIMEO`/`SO_SNDTIMEO`, built-in default 30 s) and the per-message protocol poll deadline (built-in default 60 s). Omit the option to keep both built-ins; `0` is rejected. The server side keeps the built-in 60 s protocol window (the value is not sent on the wire). |
|
| `-i, --itemize-changes` | Print an rsync-style per-file change line |
|
||||||
| `--contimeout <sec>` | Connection timeout in seconds (default: 10) |
|
| `--out-format=FORMAT` | Output format for changed files (`%f %n %l %b %M %%`) |
|
||||||
| `--backup` | Backup existing destination files before overwriting |
|
| `--list-only` | List source files instead of transferring |
|
||||||
| `--backup-dir <dir>` | Target directory for backups (requires `--backup`) |
|
| `--fsync` | Fsync every written file before publication |
|
||||||
| `--stats` | Print transfer statistics at end (bytes, files, timing) |
|
| `-h, --human-readable` | Format transfer byte/rate counts with rsync's decimal (base-1000) units |
|
||||||
| `-h, --human-readable` | Format transfer byte sizes with binary units |
|
|
||||||
| `--max-depth <n>` | Maximum directory depth to recurse (0 = unlimited, default: 0) |
|
| `--max-depth <n>` | Maximum directory depth to recurse (0 = unlimited, default: 0) |
|
||||||
| `--log-file <path>` | Write log messages to file instead of stderr |
|
| `--log-file <path>` | Write log messages to file instead of stderr |
|
||||||
|
| `--write-batch=FILE` | Run the normal live transfer and also emit a self-contained batch file of the source tree |
|
||||||
|
| `--only-write-batch=FILE` | Emit the batch file only (no destination, no server) |
|
||||||
|
| `--read-batch=FILE` | Apply a batch file to the destination (no source, no server) |
|
||||||
| `--source-dir <path>` | Source directory (overrides `FASTSYNC_SOURCE_DIR`) |
|
| `--source-dir <path>` | Source directory (overrides `FASTSYNC_SOURCE_DIR`) |
|
||||||
| `--dest-dir <path>` | Server destination directory (overrides `FASTSYNC_DEST_DIR`) |
|
| `--dest-dir <path>` | Server destination directory (overrides `FASTSYNC_DEST_DIR`) |
|
||||||
| `--save-to-disk` | Write received files to disk |
|
| `--save-to-disk` | Write received files to disk |
|
||||||
| `--server-host <ip>` | Server IP address (default: `127.0.0.1`) |
|
| `--server-host <ip>` | Server IP address (default: `127.0.0.1`) |
|
||||||
| `--server-port <n>` | Server port (default: `8080`) |
|
| `--server-port <n>` | Server port (default: `8080`) |
|
||||||
|
| `--ssh-port <port>` | SSH port (default: 22) |
|
||||||
|
| `-e, --rsh <command>` | Remote shell to launch for the SSH transport (default: `ssh`; may include arguments, e.g. `-e "ssh -p 2222"`) |
|
||||||
|
| `-M, --remote-option=OPT` | Append OPT to the remote server invocation over SSH (repeatable) |
|
||||||
|
| `--address <ip>` | Bind the outgoing client socket to this source address |
|
||||||
|
| `-4, --ipv4` | Force IPv4 for destination resolution |
|
||||||
|
| `-6, --ipv6` | Force IPv6 for destination resolution |
|
||||||
|
| `--sockopts=OPTS` | Comma-separated OPT=VAL socket options applied before connect (`TCP_NODELAY`, `SO_KEEPALIVE`, `SO_RCVBUF`, `SO_SNDBUF`, `SO_REUSEADDR`) |
|
||||||
|
| `--bwlimit <KB/s>` | Bandwidth limit in kilobytes per second |
|
||||||
|
| `--chunk-size <n>` | Chunk size in bytes (default: 10485760) |
|
||||||
|
| `--timeout <sec>` | I/O timeout in seconds, applied to both the socket (`SO_RCVTIMEO`/`SO_SNDTIMEO`) and the per-message protocol poll deadline. Default `0` = disabled (matching rsync); `0` disables it. `--no-timeout` is the negation. The value is not sent on the wire; the server side keeps its own safe floor. |
|
||||||
|
| `--contimeout <sec>` | Connection timeout in seconds (default: 60, matching rsync); `0` disables it (`--no-contimeout` is the negation) |
|
||||||
|
| `--stop-after=MINS` | Stop the transfer after MINS minutes (a positive integer); whatever was already transferred is kept |
|
||||||
|
| `--stop-at=TIME` | Stop at an absolute time (`HH:MM`, `HH:MM:SS`, or `now+N[smhd]`); an early stop skips the late `--delete` keep-set |
|
||||||
|
| `-b, --backup` | Backup existing destination files before overwriting |
|
||||||
|
| `--backup-dir <dir>` | Target directory for backups (requires `--backup`) |
|
||||||
| `--tls` | Enable TLS encryption |
|
| `--tls` | Enable TLS encryption |
|
||||||
| `--cert <path>` | TLS certificate file (PEM) |
|
| `--cert <path>` | TLS certificate file (PEM) |
|
||||||
| `--key <path>` | TLS private key file (PEM) |
|
| `--key <path>` | TLS private key file (PEM) |
|
||||||
| `--ca <path>` | TLS CA certificate file for verification (PEM) |
|
| `--ca <path>` | TLS CA certificate file for verification (PEM) |
|
||||||
| `--client-cn <name>` | TLS client certificate common name; mandatory with `--tls` (a TLS connection always verifies the client CN) |
|
|
||||||
|
The exhaustive rsync flag matrix is in [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md).
|
||||||
|
|
||||||
**Per-message vs. connection timeouts.** `--timeout` bounds each individual protocol
|
**Per-message vs. connection timeouts.** `--timeout` bounds each individual protocol
|
||||||
send/receive (the `poll()` deadline), so a peer that stops mid-frame is dropped. It
|
send/receive (the `poll()` deadline), so a peer that stops mid-frame is dropped. It
|
||||||
@@ -190,60 +291,64 @@ transfer is never aborted.
|
|||||||
| `FASTSYNC_SOURCE_DIR` | — | Source directory fallback |
|
| `FASTSYNC_SOURCE_DIR` | — | Source directory fallback |
|
||||||
| `FASTSYNC_DEST_DIR` | — | Destination directory fallback |
|
| `FASTSYNC_DEST_DIR` | — | Destination directory fallback |
|
||||||
| `FASTSYNC_SAVE_TO_DISK` | `false` | Disk persistence fallback |
|
| `FASTSYNC_SAVE_TO_DISK` | `false` | Disk persistence fallback |
|
||||||
| `FASTSYNC_SSH_PORT` | `22` | Default SSH port |
|
|
||||||
| `FASTSYNC_SERVER_HOST` | `127.0.0.1` | Default server host |
|
|
||||||
| `FASTSYNC_SERVER_PORT` | `8080` | Default server port |
|
|
||||||
| `FASTSYNC_TLS_CERT` | — | Default TLS certificate path |
|
|
||||||
| `FASTSYNC_TLS_KEY` | — | Default TLS private key path |
|
|
||||||
| `FASTSYNC_TLS_CA` | — | Default TLS CA certificate path |
|
|
||||||
|
|
||||||
## Implementation Details
|
## Implementation Details
|
||||||
|
|
||||||
### Data Structures
|
### Data Structures
|
||||||
1. **Chunk** — collection of files (~10 MB total by default)
|
|
||||||
2. **File** — path, content (`Data`), optional `FileMetadata` pointer
|
1. **Chunk** — collection of files (~10 MB total by default).
|
||||||
3. **FileMetadata** — `mode`, `uid`, `gid`, `mtime_sec`, `mtime_nsec`;
|
2. **File** — path, content (`Data`), optional `FileMetadata` pointer.
|
||||||
uid / gid are advisory wire fields and are never applied by the receiver;
|
3. **FileMetadata** — `mode`, `uid`, `gid`, `mtime_sec`, `mtime_nsec` (plus
|
||||||
atime is unsupported
|
atime/crtime fields). `uid`/`gid` are applied only through the opt-in
|
||||||
4. **Config** — runtime parameters (transported over wire, TLS settings excluded). Includes `timeout`, `contimeout`, `quiet`, `backup`, `backup_dir`, `stats`, `max_depth`, `log_file`.
|
identity path; atime is preserved with `-U`/`--atimes`; crtime is captured
|
||||||
5. **Queue** — thread-safe bounded queue with condition variables
|
but cannot be set on the destination.
|
||||||
6. **DirectoryScanner** — recursive BFS traversal with exclude and include pattern support, max-depth enforcement
|
4. **Config** — runtime parameters. Most cross the wire (TLS settings
|
||||||
|
excluded); `backup` and `backup_dir` are in the serialized wire table, while
|
||||||
|
`timeout`, `contimeout`, `quiet`, `stats`, `max_depth`, and `log_file` are
|
||||||
|
client-only.
|
||||||
|
5. **Queue** — thread-safe bounded queue with condition variables.
|
||||||
|
6. **DirectoryScanner** — recursive BFS traversal with exclude and include
|
||||||
|
pattern support, max-depth enforcement.
|
||||||
|
|
||||||
### Key Algorithms
|
### Key Algorithms
|
||||||
1. **File scanning** — BFS directory traversal;
|
|
||||||
entries matched against exclude and include patterns,
|
1. **File scanning** — BFS directory traversal; entries matched against exclude
|
||||||
max - depth enforced 2. * *Chunking ** — files accumulated until `chunk_size` threshold,
|
and include patterns, with max-depth enforced.
|
||||||
then flushed 3. *
|
2. **Chunking** — files accumulated until the `chunk_size` threshold (default
|
||||||
*Compression ** — streaming zstd
|
10 MiB) is reached, then flushed.
|
||||||
via `ZSTD_compressStream2` / `ZSTD_decompressStream` 4. *
|
3. **Compression** — streaming zstd via `ZSTD_compressStream2()` /
|
||||||
*Network protocol ** — status -
|
`ZSTD_decompressStream()`.
|
||||||
code - driven exchange with metadata packing,
|
4. **Network protocol** — status-code-driven exchange with metadata packing,
|
||||||
keep - alive,
|
keep-alive, and abort support.
|
||||||
and abort support 5. * *Incremental check ** — client sends `STATUS_CHECK` + path + size +
|
5. **Incremental check** — the client sends `STATUS_CHECK` + path + size +
|
||||||
mtime and,
|
mtime and, with `--checksum`, a whole-file content checksum (`xxh128` by
|
||||||
with `--checksum`, XXH64 content checksum; server compares against destination. Can be batched via `STATUS_CHECK_BATCH` for reduced round-trips.
|
default; selectable via `--checksum-choice`/`--cc`, seeded by
|
||||||
6. **Bandwidth limiting** — token-bucket algorithm with `nanosleep` throttling on 64 KB write chunks
|
`--checksum-seed`); the server compares against the destination. Can be
|
||||||
7. **Metadata restoration** — `chmod()`, `chown()`, `utimensat()` on the receiving side
|
batched via `STATUS_CHECK_BATCH` for reduced round-trips.
|
||||||
8. **`--delete`** — sender tracks all sent paths;
|
6. **Bandwidth limiting** — token-bucket algorithm with sleep throttling on
|
||||||
receiver walks destination tree and removes unlisted files / directories 9. *
|
64 KiB write chunks.
|
||||||
*SSH transport *
|
7. **Metadata restoration** — mode via `chmod()`/`fchmod()`, times via
|
||||||
* — `socketpair()` + `fork()` + `execvp("ssh",
|
`utimensat()`/`futimens()`, and ownership only with an identity flag via
|
||||||
...)` with `ControlMaster` and port support
|
fd-relative `fchown()`/`fchownat()`.
|
||||||
10. *
|
8. **`--delete`** — the sender tracks all sent paths; the receiver walks the
|
||||||
*TLS transport ** — OpenSSL `SSL_CTX` with TLS
|
destination tree and removes unlisted files and directories.
|
||||||
1.2 minimum,
|
9. **SSH transport** — `socketpair()` + `fork()` + `execvp("ssh", ...)` with
|
||||||
mutual CA verification,
|
`ControlMaster` and port support.
|
||||||
transparent `SSL_read`/`SSL_write` via `io_set_ssl()` 11. *
|
10. **TLS transport** — OpenSSL `SSL_CTX` with TLS 1.2 minimum, mutual CA
|
||||||
*Path traversal protection ** — `has_path_traversal()` rejects any file path
|
verification, and transparent `SSL_read()`/`SSL_write()` via
|
||||||
containing `..` components,
|
`io_set_ssl()`.
|
||||||
preventing directory escape attacks 12. *
|
11. **Path traversal protection** — `has_path_traversal()` rejects any file
|
||||||
*Connection limiting ** — server tracks active connections and rejects
|
path containing `..` components, preventing directory escape attacks.
|
||||||
new ones beyond `max_connections` (default 100)13. *
|
12. **Connection limiting** — the server tracks active connections and rejects
|
||||||
*Keep
|
new ones beyond `max_connections` (default 100).
|
||||||
- alive ** — idle connections receive periodic `STATUS_KEEPALIVE` to detect half
|
13. **Keep-alive** — idle connections receive periodic `STATUS_KEEPALIVE` to
|
||||||
- open TCP connections 14. * *Abort handling ** — `SIGINT` sets an abort flag; the next protocol operation sends `STATUS_ABORT` for clean server cleanup
|
detect half-open TCP connections.
|
||||||
15. **Atomic writes** — files are written to a `.tmp` suffix then atomically renamed via `rename()`, preventing partial files
|
14. **Abort handling** — `SIGINT` sets an abort flag; the next protocol
|
||||||
16. **Backup** — before overwriting, existing files are moved to `--backup-dir` (or same directory with `~` suffix) preserving the original
|
operation sends `STATUS_ABORT` for clean server cleanup.
|
||||||
|
15. **Atomic writes** — files are written to a `.tmp` suffix then atomically
|
||||||
|
renamed via `rename()`, preventing partial files.
|
||||||
|
16. **Backup** — before overwriting, existing files are moved to `--backup-dir`
|
||||||
|
(or the same directory with a `~` suffix), preserving the original.
|
||||||
|
|
||||||
## Security Features
|
## Security Features
|
||||||
|
|
||||||
@@ -300,7 +405,8 @@ cmake --build build -j$(nproc)
|
|||||||
|
|
||||||
### SSH transfer
|
### SSH transfer
|
||||||
|
|
||||||
The remote host must have `fastsync-server` available in `PATH`, or use
|
The remote host must have `fastsync-server` available in `PATH` (install or
|
||||||
|
copy the built `./build/server` there as `fastsync-server`), or use
|
||||||
`--fastsync-server-path`. SSH starts `fastsync-server --stdio` in its remote
|
`--fastsync-server-path`. SSH starts `fastsync-server --stdio` in its remote
|
||||||
working directory, so use a destination below that directory unless the
|
working directory, so use a destination below that directory unless the
|
||||||
remote server is otherwise configured with a matching authorized root.
|
remote server is otherwise configured with a matching authorized root.
|
||||||
@@ -341,8 +447,13 @@ Plain TCP requires the explicit `--allow-unauthenticated` server option. Use TLS
|
|||||||
authenticated network connections.
|
authenticated network connections.
|
||||||
|
|
||||||
### TLS transfer
|
### TLS transfer
|
||||||
|
|
||||||
|
Server TLS requires `--cert`, `--key`, `--ca`, and `--client-cn`; the client
|
||||||
|
requires `--cert`, `--key`, and `--ca`.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
./build/server --destination-root /path/to --tls --cert server.pem --key server-key.pem -p 8443
|
./build/server --destination-root /path/to --tls --cert server.pem --key server-key.pem \
|
||||||
|
--ca ca.pem --client-cn client -p 8443
|
||||||
./build/client --tls --cert client.pem --key client-key.pem --ca ca.pem \
|
./build/client --tls --cert client.pem --key client-key.pem --ca ca.pem \
|
||||||
--server-host example.com --server-port 8443 \
|
--server-host example.com --server-port 8443 \
|
||||||
--source-dir /path/to/source --dest-dir /path/to/destination \
|
--source-dir /path/to/source --dest-dir /path/to/destination \
|
||||||
@@ -355,32 +466,32 @@ These examples show the intended rsync-style workflow. Options marked as
|
|||||||
FastSync-native are optional performance or transport extensions.
|
FastSync-native are optional performance or transport extensions.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
#Basic synchronization
|
# Basic synchronization
|
||||||
./build/client /source/ /destination/
|
./build/client /source/ /destination/
|
||||||
|
|
||||||
#Archive - style synchronization(current FastSync archive behavior)
|
# Archive-style synchronization (current FastSync archive behavior)
|
||||||
./build/client -a /source/ user@host:destination/
|
./build/client -a /source/ user@host:destination/
|
||||||
|
|
||||||
#Preview a transfer without changing the destination
|
# Preview a transfer without changing the destination
|
||||||
./build/client -n /source/ /destination/
|
./build/client -n /source/ /destination/
|
||||||
|
|
||||||
#Exclude temporary and object files
|
# Exclude temporary and object files
|
||||||
./build/client --exclude '*.tmp' --exclude '*.o' \
|
./build/client --exclude '*.tmp' --exclude '*.o' \
|
||||||
/source/ user@host:destination/
|
/source/ user@host:destination/
|
||||||
|
|
||||||
#Remove destination entries not present in the source
|
# Remove destination entries not present in the source
|
||||||
./build/client --delete /source/ user@host:destination/
|
./build/client --delete /source/ user@host:destination/
|
||||||
|
|
||||||
#Skip unchanged files using size and modification time
|
# Skip unchanged files using size and modification time
|
||||||
./build/client --incremental /source/ user@host:destination/
|
./build/client --incremental /source/ user@host:destination/
|
||||||
|
|
||||||
#Verify content when size and time are not sufficient
|
# Verify content when size and time are not sufficient
|
||||||
./build/client --incremental --checksum /source/ user@host:destination/
|
./build/client --incremental --checksum /source/ user@host:destination/
|
||||||
|
|
||||||
#Preserve supported mode and timestamp metadata
|
# Preserve supported mode and timestamp metadata
|
||||||
./build/client --preserve /source/ user@host:destination/
|
./build/client --preserve /source/ user@host:destination/
|
||||||
|
|
||||||
#Keep backups of overwritten destination files
|
# Keep backups of overwritten destination files
|
||||||
./build/client --backup --backup-dir backups \
|
./build/client --backup --backup-dir backups \
|
||||||
/source/ user@host:destination/
|
/source/ user@host:destination/
|
||||||
```
|
```
|
||||||
@@ -393,11 +504,11 @@ features without changing the meaning of ordinary compatibility options.
|
|||||||
| Option | Purpose |
|
| Option | Purpose |
|
||||||
|---|---|
|
|---|---|
|
||||||
| `-j`, `--threads[=N]` | Enable the multithreaded scanner/loader/sender pipeline. `N` (1–256) sets the parallel scanner worker count; bare `-j`/`--threads` uses the default. |
|
| `-j`, `--threads[=N]` | Enable the multithreaded scanner/loader/sender pipeline. `N` (1–256) sets the parallel scanner worker count; bare `-j`/`--threads` uses the default. |
|
||||||
| `-z [level]`, `--compress [level]` | Enable streaming zstd compression, levels 1-22. |
|
| `-z [level]`, `--compress [level]` | Enable streaming compression (default `zstd`), levels 1-22. |
|
||||||
| `--compress-level <n>` | Set the zstd compression level. |
|
| `--compress-level <n>` | Set the compression level. |
|
||||||
| `--zc <alg>` | Alias for `--compress-choice`. FastSync supports `zstd` and `none`. |
|
| `--zc <alg>` | Alias for `--compress-choice`. FastSync supports `zstd` (default), `lz4`, `zlib`, `zlibx`, `none`, and `auto`; `zlibx` behaves as `zlib`. |
|
||||||
| `--zl <n>` | Alias for `--compress-level`. |
|
| `--zl <n>` | Alias for `--compress-level`. |
|
||||||
| `--skip-compress <list>` | Skip compression for comma-separated suffixes; incompatible with `--chunk-serialization`. |
|
| `--skip-compress <list>` | Skip compression for `/`- or `,`-separated suffixes; defaults to rsync 3.4.1's built-in list. Incompatible with `--chunk-serialization`. |
|
||||||
| `--compress-threads <n>` | Use `n` zstd compression workers. Requires compression and a zstd build with threaded support; the setting affects sender CPU work only. |
|
| `--compress-threads <n>` | Use `n` zstd compression workers. Requires compression and a zstd build with threaded support; the setting affects sender CPU work only. |
|
||||||
| `--chunk-size <bytes>` | Set the transfer chunk size. |
|
| `--chunk-size <bytes>` | Set the transfer chunk size. |
|
||||||
| `--chunk-serialization` | Enable FastSync chunk serialization (long form only; `-s` is rsync's `--secluded-args`). |
|
| `--chunk-serialization` | Enable FastSync chunk serialization (long form only; `-s` is rsync's `--secluded-args`). |
|
||||||
@@ -409,10 +520,10 @@ features without changing the meaning of ordinary compatibility options.
|
|||||||
| `--server-port <port>` | Select the TCP server port (`--port <port>` and `--port=<port>` are rsync-friendly aliases). |
|
| `--server-port <port>` | Select the TCP server port (`--port <port>` and `--port=<port>` are rsync-friendly aliases). |
|
||||||
| `--tls` | Enable TLS for TCP transport. |
|
| `--tls` | Enable TLS for TCP transport. |
|
||||||
| `--bwlimit <KB/s>` | Apply token-bucket bandwidth limiting. |
|
| `--bwlimit <KB/s>` | Apply token-bucket bandwidth limiting. |
|
||||||
| `--progress` | Show transfer progress and throughput. |
|
| `--progress` | Show rsync-style per-file progress blocks from the receiver's wire counters (FastSync omits rsync's leading `./` line). |
|
||||||
| `--stats` | Print transfer statistics. |
|
| `--stats` | Print transfer statistics, including the receiver-only counters reported over the wire; rsync's per-type `Number of files` breakdown is not reproduced. |
|
||||||
| `--timeout <seconds>` | Set the socket **and** per-message protocol I/O timeout (positive seconds). Omit to keep the built-in 30 s socket / 60 s protocol defaults. |
|
| `--timeout <seconds>` | Set the socket **and** per-message protocol I/O timeout. Default `0` = disabled (matching rsync); `0` disables it. |
|
||||||
| `--contimeout <seconds>` | Set connection timeout. |
|
| `--contimeout <seconds>` | Connection timeout (default 60, matching rsync); `0` disables it. |
|
||||||
|
|
||||||
Short-option conflicts with rsync have been resolved for the CLI namespace
|
Short-option conflicts with rsync have been resolved for the CLI namespace
|
||||||
(Phase 7): `-c` is now rsync's `--checksum`, `-m` is `--prune-empty-dirs`, `-M`
|
(Phase 7): `-c` is now rsync's `--checksum`, `-m` is `--prune-empty-dirs`, `-M`
|
||||||
@@ -421,7 +532,8 @@ is `--remote-option`, `-f` is `--filter`, `-s` is `--secluded-args`, `-p` is
|
|||||||
long-form-only or new shorts: multithreading is `-j`/`--threads`, metadata
|
long-form-only or new shorts: multithreading is `-j`/`--threads`, metadata
|
||||||
is `--preserve`, sendfile is `--sendfile`, chunk serialization is
|
is `--preserve`, sendfile is `--sendfile`, chunk serialization is
|
||||||
`--chunk-serialization`, timeout is `--timeout`, and SSH port is `--ssh-port`.
|
`--chunk-serialization`, timeout is `--timeout`, and SSH port is `--ssh-port`.
|
||||||
`-a`/`--archive` is now real rsync archive (`-rlptgoD`).
|
`-a`/`--archive` is now rsync archive `-rlptgoD` (owner/group implied, but the
|
||||||
|
receiver still needs privilege to apply them).
|
||||||
|
|
||||||
`--secluded-args` (and its short form `-s`) is accepted as a compatibility
|
`--secluded-args` (and its short form `-s`) is accepted as a compatibility
|
||||||
no-op. It does not change FastSync's transport or protocol behavior, because
|
no-op. It does not change FastSync's transport or protocol behavior, because
|
||||||
@@ -433,42 +545,94 @@ remote SSH argv is already built injection-safe.
|
|||||||
|
|
||||||
| Option | Description |
|
| Option | Description |
|
||||||
|---|---|
|
|---|---|
|
||||||
| `-a`, `--archive` | rsync archive mode (`-rlptgoD`): links, metadata, devices and specials. |
|
| `-a`, `--archive` | rsync archive mode (`-rlptgoD`): links, perms, times, owner, group, devices and specials; ownership application stays privilege-gated. |
|
||||||
| `-n`, `--dry-run` | Scan and report without writing files. |
|
| `-n`, `--dry-run` | Report what would be transferred without mutating the destination. Since protocol 2.21.0 a server-routed target contacts the receiver and reports would-transfer based on receiver state; a plain local destination keeps the client-side scan. Never mutates or deletes. |
|
||||||
| `--delete` | Request removal of destination entries absent from the source. The server must allow deletion. Default timing is delete-after: extras are removed only after the whole transfer succeeded. |
|
| `--remove-source-files` | Remove regular source files after a successful transfer. |
|
||||||
|
| `--incremental` | Skip files matching destination size and mtime. Auto-enables `--preserve`. Incompatible with `--chunk-serialization`. |
|
||||||
|
| `-c, --checksum` | Verify content by checksum (implies the incremental quick-check). Algorithm selectable with `--checksum-choice`. |
|
||||||
|
| `--checksum-choice <alg>` | Whole-file checksum algorithm: `xxh64`/`xxhash` (default), `xxh3`, `xxh128`, `md5`, or `auto`. |
|
||||||
|
| `--checksum-seed <n>` | Seed for the whole-file xxHash digest; an unset/`0` seed is randomized per transfer, matching rsync. |
|
||||||
|
| `--size-only` | Skip incremental files matching in size, ignoring mtime. |
|
||||||
|
| `-I, --ignore-times` | Transfer files even when size and mtime match. |
|
||||||
|
| `-u, --update` | Skip files newer than the source on the receiver. |
|
||||||
|
| `-W, --whole-file` | Transfer changed files without delta processing (`--no-whole-file` clears it). |
|
||||||
|
| `-B <n>, --block-size <n>` | Delta block size in bytes (alias `--delta-block`). |
|
||||||
|
| `-d, --dirs` | Transfer the named directory entries without recursing into their contents (aliases `--old-dirs`/`--old-d`). |
|
||||||
|
| `-R, --relative` | Use rsync's relative path semantics (including the `/./` cut); with `--files-from`, preserve each listed entry's relative path below the destination root. |
|
||||||
|
| `--files-from <file>` | Read the source file list from FILE (paths relative to the source root). |
|
||||||
|
| `--delay-updates` | Put updated files into place only at the end of the transfer. |
|
||||||
|
| `--compare-dest <dir>` | Extra comparison basis: unchanged files are not transferred (requires/implies `--incremental`). |
|
||||||
|
| `--copy-dest <dir>` | Like `--compare-dest`, but copies the unchanged file from DIR into the destination. |
|
||||||
|
| `--link-dest <dir>` | Like `--copy-dest`, but hard-links the unchanged file from DIR (repeatable; earlier DIRs win). |
|
||||||
|
| `--preallocate` | Allocate destination file space up front (fail-fast on a full disk). |
|
||||||
|
| `--append` | Resume a shorter destination by appending only its tail (prefix not verified; requires `--incremental`). |
|
||||||
|
| `--append-verify` | Like `--append`, but verifies the retained prefix checksum first (falls back to a full transfer on mismatch). |
|
||||||
|
| `--delete` | Request removal of destination entries absent from the source. The server must allow deletion. Default timing is delete-after: extras are removed only after the whole transfer succeeded. Scoped to the synchronized directories, so `--files-from` subsets are safe. |
|
||||||
| `--delete-before` | Delete extras before the transfer starts (implies `--delete`). |
|
| `--delete-before` | Delete extras before the transfer starts (implies `--delete`). |
|
||||||
| `--delete-during`, `--del` | Delete extras once the keep-set manifest is known, before data is applied (implies `--delete`; early mode, same engine behaviour as `--delete-before`). |
|
| `--delete-during`, `--del` | Delete extras once the keep-set manifest is known, before data is applied (implies `--delete`; early mode, same engine behaviour as `--delete-before`). |
|
||||||
| `--delete-delay` | Delete extras only after a successful transfer (implies `--delete`; commit mode, same behaviour as `--delete-after`). |
|
| `--delete-delay` | Delete extras only after a successful transfer (implies `--delete`; commit mode, same behaviour as `--delete-after`). |
|
||||||
| `--delete-after` | Explicit delete-after timing: delete only after the transfer succeeded (implies `--delete`). |
|
| `--delete-after` | Explicit delete-after timing: delete only after the transfer succeeded (implies `--delete`). |
|
||||||
|
| `--delete-excluded` | Also delete filter-excluded destination mirrors (size-pruned mirrors stay protected). |
|
||||||
|
| `--max-delete <n>` | Delete at most n destination entries; the rest are skipped and the run exits 25 (partial), matching rsync. |
|
||||||
|
| `--force` | Allow an incoming file/symlink to replace a destination directory (also during `--delay-updates` publication). |
|
||||||
| `--exclude <pattern>` | Exclude matching paths. Repeatable. |
|
| `--exclude <pattern>` | Exclude matching paths. Repeatable. |
|
||||||
| `--include <pattern>` | Include matching paths. Repeatable. |
|
| `--include <pattern>` | Include matching paths. Repeatable. |
|
||||||
| `--exclude-from <file>` | Read exclude patterns from a file. |
|
| `--exclude-from <file>` | Read exclude patterns from a file. |
|
||||||
| `--include-from <file>` | Read include patterns from a file. |
|
| `--include-from <file>` | Read include patterns from a file. |
|
||||||
|
| `-f, --filter=RULE` | Add an rsync-style filter rule (`+`/`-`, `include`/`exclude`, `merge`/`.`, `dir-merge`/`:`, `hide`/`H`, `show`/`S`, `protect`/`P`, `risk`/`R`, `clear`/`!`, and modifiers; repeatable). |
|
||||||
| `--max-size <bytes>` | Skip files larger than the limit. |
|
| `--max-size <bytes>` | Skip files larger than the limit. |
|
||||||
| `--min-size <bytes>` | Skip files smaller than the limit. |
|
| `--min-size <bytes>` | Skip files smaller than the limit. |
|
||||||
| `--max-depth <n>` | Limit recursive scanning depth;
|
| `--max-alloc <SIZE>` | Maximum single allocation (binary units; default 1G; `0` = no local limit). |
|
||||||
zero means unlimited.| | `--incremental` | Skip files matching destination size and mtime.|
|
| `--max-depth <n>` | Limit recursive scanning depth; zero means unlimited. |
|
||||||
| `--checksum` | Include xxHash64 content checks in incremental comparisons.| | `--backup` |
|
| `-b, --backup` | Back up overwritten files. |
|
||||||
Back up overwritten files.| | `--backup - dir<dir>` | Store backups under a separate directory.|
|
| `-T, --temp-dir <dir>` | Scratch directory for temp files before the atomic install (confined to the receive root; `EXDEV` falls back to a non-atomic copy). |
|
||||||
| `--suffix<suffix>` | Set the backup filename suffix.| | `--partial` |
|
| `--backup-dir <dir>` | Store backups under a separate directory (requires `--backup`). |
|
||||||
Select partial - transfer handling. On failed/interrupted writes the
|
| `--suffix <suffix>` | Set the backup filename suffix (default: `~`). |
|
||||||
already-written temp file is retained (best-effort) for resumption.|
|
| `--partial` | Select partial-transfer handling. On failed/interrupted writes the already-written temp file is retained (best-effort) for resumption. With `--partial --partial-dir <dir>`, completed files are written under the partial directory and installed atomically. |
|
||||||
With `--partial --partial-dir <dir>`, completed files are written under the
|
| `--partial-dir <dir>` | Set a relative partial-transfer directory below the server destination root. Use with `--partial`. |
|
||||||
partial directory and installed atomically. | | `--partial - dir<dir>` |
|
|
||||||
Set a relative partial - transfer directory below the server destination root.
|
|
||||||
Use with `--partial`. |
|
|
||||||
| `--inplace` | Write directly to the destination instead of using a temporary file. |
|
| `--inplace` | Write directly to the destination instead of using a temporary file. |
|
||||||
|
| `--fsync` | Fsync every written file before publication. |
|
||||||
|
| `--write-batch=FILE` | Run the normal live transfer and also emit a self-contained batch file of the source tree. |
|
||||||
|
| `--only-write-batch=FILE` | Emit the batch file only (no destination, no server). |
|
||||||
|
| `--read-batch=FILE` | Apply a batch file to the destination (no source, no server). |
|
||||||
|
| `--stop-after=MINS` | Stop the transfer after MINS minutes; whatever was already transferred is kept. |
|
||||||
|
| `--stop-at=TIME` | Stop at an absolute time (`HH:MM`, `HH:MM:SS`, or `now+N[smhd]`). An early stop skips the late `--delete` keep-set. |
|
||||||
|
|
||||||
### Metadata and links
|
### Metadata and links
|
||||||
|
|
||||||
| Option | Description |
|
| Option | Description |
|
||||||
|---|---|
|
|---|---|
|
||||||
| `--preserve` | Preserve supported file metadata, currently mode and modification time (long form only). |
|
| `--preserve` | Preserve mode and mtime (long form only; equivalent to `-p` + `-t`). Add `-o`/`-g` for owner/group, `-U`/`--atimes` for atime, or an identity flag (`--chown`/`--usermap`/`--groupmap`/`--numeric-ids`/`--copy-as`) for mapped ownership. |
|
||||||
| `-l`, `--links` | Request symlink preservation;
|
| `-U`, `--atimes` | Preserve access times. Captured with the metadata payload; does not enable ownership. |
|
||||||
link-target transfer remains incomplete. |
|
| `-N`, `--crtimes` | Capture birth time and transmit it; it cannot be applied because no portable filesystem call can set a birth time (documented divergence). |
|
||||||
| `--copy-links` | Copy symlink referents. |
|
| `-p`, `--perms` | Preserve permission bits. One of the four per-attribute preserve flags (with `-t`/`-o`/`-g`); under `-p` the source mode is copied exactly (setuid/setgid/sticky and group/other-write included), matching rsync. |
|
||||||
| `--safe-links` | Skip symlinks that point outside the transfer tree. |
|
| `-t`, `--times` | Preserve modification times. Independent of the other attributes; `-O`/`--omit-dir-times` suppresses directories only. |
|
||||||
|
| `-o`, `--owner` | Preserve the source owner (uid). Mapped by name on the receiver with a raw-numeric fallback (only numeric ids cross the wire); application is privilege-gated. |
|
||||||
|
| `-g`, `--group` | Preserve the source group (gid). Same name-mapping/numeric-fallback and privilege gating as `-o`. |
|
||||||
|
| `--no-perms`, `--no-times`, `--no-owner`, `--no-group` | Negate each per-attribute flag (also `--no-p`/`--no-t`/`--no-o`/`--no-g`); `--no-preserve` clears all four. |
|
||||||
|
| `-E`, `--executability` | Preserve executable permission bits. |
|
||||||
|
| `-X`, `--xattrs` | Preserve user `user.*` extended attributes. |
|
||||||
|
| `-A`, `--acls` | Preserve POSIX ACLs. |
|
||||||
|
| `--chmod <changes>` | Modify transferred permissions (rsync syntax, including `D`/`F`/`X` selectors and `s`/`t`); does not imply `-p`. |
|
||||||
|
| `--chown=USER:GROUP` | Override the ownership of transferred files (`USER:GROUP`, `USER`, or `:GROUP`); conflicts with `--usermap`/`--groupmap` on the same side. |
|
||||||
|
| `--usermap=MAP` | Map usernames when applying ownership (`FROM:TO` rules; names, ids, `LOW-HIGH` ranges, `*`, empty-`FROM`). |
|
||||||
|
| `--groupmap=MAP` | Map group names when applying ownership (same syntax as `--usermap`). |
|
||||||
|
| `--numeric-ids` | Mapping modifier: apply the source numeric uid/gid directly instead of mapping by name (combine with `-o`/`-g`, `-a`, or a map). |
|
||||||
|
| `--copy-as=USER[:GROUP]` | Force every written entry to USER[:GROUP]; requires a privileged receiver. |
|
||||||
|
| `--fake-super` | Record the resolved owner plus mode/time in a reserved `user.fastsync.stat` xattr and replay mode/time; never performs a real chown. |
|
||||||
|
| `--super` | Permit the receiver to attempt confined super-user activities (device nodes). |
|
||||||
|
| `--no-super` | Forbid those super-user activities even when the receiver is root. |
|
||||||
|
| `-l`, `--links` | Copy symlinks as symlinks; the target is stored verbatim (absolute and `..`-bearing targets included), matching rsync. |
|
||||||
|
| `-L`, `--copy-links` | Copy symlink referents (a broken referent makes the run exit 23, matching rsync). |
|
||||||
|
| `--safe-links` | Skip symlinks whose target points outside the transfer tree (applied on the sender). |
|
||||||
| `--copy-unsafe-links` | Copy unsafe symlink referents. |
|
| `--copy-unsafe-links` | Copy unsafe symlink referents. |
|
||||||
|
| `--munge-links` | Rewrite stored symlink targets with rsync's `/rsyncd-munged/` marker. |
|
||||||
|
| `-k`, `--copy-dirlinks` | Treat a symlink to a directory as a real directory on the sender. |
|
||||||
|
| `-K`, `--keep-dirlinks` | Follow an existing destination symlink-to-directory (confined to the receive root). |
|
||||||
|
| `-H`, `--hard-links` | Preserve hard-link relationships across the transfer. |
|
||||||
|
| `-D` | Preserve device and special files (implies `--devices --specials`). |
|
||||||
|
| `--devices` | Recreate device nodes on the destination (privileged; skipped without `CAP_MKNOD`). |
|
||||||
|
| `--specials` | Recreate special files: FIFOs and unix sockets. |
|
||||||
| `-S`, `--sparse` | Sparse-file handling: receiver preserves holes (zero runs are written as holes; no wire change). |
|
| `-S`, `--sparse` | Sparse-file handling: receiver preserves holes (zero runs are written as holes; no wire change). |
|
||||||
|
|
||||||
### Output and logging
|
### Output and logging
|
||||||
@@ -476,8 +640,12 @@ link-target transfer remains incomplete. |
|
|||||||
| Option | Description |
|
| Option | Description |
|
||||||
|---|---|
|
|---|---|
|
||||||
| `-v`, `--verbose` | Enable debug logging. |
|
| `-v`, `--verbose` | Enable debug logging. |
|
||||||
| `--progress` | Show live transfer progress. |
|
| `-q`, `--quiet` | Suppress non-error output. |
|
||||||
| `--stats` | Print transfer statistics. |
|
| `--progress` | Show rsync-style per-file progress blocks (not rsync's leading `./` line). |
|
||||||
|
| `--stats` | Print transfer statistics, including the receiver-only counters reported over the wire. |
|
||||||
|
| `-i`, `--itemize-changes` | Print an rsync-style per-file change line. |
|
||||||
|
| `--out-format=FORMAT` | Output format for changed files (`%f %n %l %b %M %%`). |
|
||||||
|
| `--list-only` | List source files instead of transferring. |
|
||||||
| `--log-file <path>` | Write log output to a file. |
|
| `--log-file <path>` | Write log output to a file. |
|
||||||
| `-V`, `--version` | Print the FastSync protocol version. |
|
| `-V`, `--version` | Print the FastSync protocol version. |
|
||||||
| `--help` | Print command usage. |
|
| `--help` | Print command usage. |
|
||||||
@@ -487,31 +655,56 @@ link-target transfer remains incomplete. |
|
|||||||
| Option | Description |
|
| Option | Description |
|
||||||
|---|---|
|
|---|---|
|
||||||
| `--ssh-port <port>` | SSH port for the SSH transport (default: 22). Note the short `-p` is now rsync's `--perms`. |
|
| `--ssh-port <port>` | SSH port for the SSH transport (default: 22). Note the short `-p` is now rsync's `--perms`. |
|
||||||
| `--fastsync-server-path <path>` | Remote FastSync server path for SSH mode. |
|
| `-e`, `--rsh <command>` | Remote shell to launch for the SSH transport (default: `ssh`; may include arguments). |
|
||||||
|
| `--fastsync-server-path <path>` | Remote FastSync server path for SSH mode (client-only; never crosses the wire). |
|
||||||
|
| `--rsync-path <path>` | Alias for `--fastsync-server-path`. |
|
||||||
|
| `-M`, `--remote-option=OPT` | Append OPT to the remote server invocation over SSH (repeatable; rejected for daemon/TCP destinations). |
|
||||||
|
| `--trust-sender` | Receiver-local: trust the remote sender's file list and skip path re-validation (does not affect symlink targets). |
|
||||||
|
| `--timeout <sec>` | Socket + per-message I/O timeout; default `0` = disabled. |
|
||||||
|
| `--contimeout <sec>` | Connection timeout; default 60; `0` disables. |
|
||||||
| `--source-dir <path>` | Set the source directory explicitly. |
|
| `--source-dir <path>` | Set the source directory explicitly. |
|
||||||
| `--dest-dir <path>` | Set the destination directory explicitly. |
|
| `--dest-dir <path>` | Set the destination directory explicitly. |
|
||||||
| `--save-to-disk` | Enable server-side disk persistence. |
|
| `--save-to-disk` | Enable server-side disk persistence. |
|
||||||
| `--server-host <host>` | TCP server address. |
|
| `--server-host <host>` | TCP server address. |
|
||||||
| `--server-port <port>` | TCP server port. `--port <port>` / `--port=<port>` is an alias. |
|
| `--server-port <port>` | TCP server port. `--port <port>` / `--port=<port>` is an alias. |
|
||||||
| `--tls` | Enable TLS. Requires `--cert` and `--key`. |
|
| `--address <ip>` | Bind the outgoing client socket to this source address. |
|
||||||
|
| `-4`, `--ipv4` | Force IPv4 for destination resolution. |
|
||||||
|
| `-6`, `--ipv6` | Force IPv6 for destination resolution. |
|
||||||
|
| `--sockopts=OPTS` | Comma-separated OPT=VAL socket options applied before connect. |
|
||||||
|
| `--tls` | Enable TLS. Requires `--cert`, `--key`, and `--ca`. |
|
||||||
| `--cert <path>` | TLS certificate file. |
|
| `--cert <path>` | TLS certificate file. |
|
||||||
| `--key <path>` | TLS private key file. |
|
| `--key <path>` | TLS private key file. |
|
||||||
| `--ca <path>` | CA file for peer verification. |
|
| `--ca <path>` | CA file for peer verification (always required with `--tls`). |
|
||||||
|
|
||||||
## Server Options
|
## Server Options
|
||||||
|
|
||||||
| Option | Description |
|
| Option | Description |
|
||||||
|---|---|
|
|---|---|
|
||||||
| `--stdio` | Serve one SSH connection over standard input/output. |
|
| `--stdio` | Serve one SSH connection over standard input/output. |
|
||||||
| `-p <port>` | TCP listen port. |
|
| `--daemon` | Run as a persistent daemon listener using a module config file; the daemon default port is 873 (unlike `-p`, which defaults to 8080). |
|
||||||
|
| `--config=FILE` | Daemon config file (default: `~/.config/fastsync/fastsyncd.conf`, else `/etc/fastsyncd.conf`). Requires `--daemon`. |
|
||||||
|
| `--dparam=KEY=VALUE` | Override one global config key on the command line. Requires `--daemon`. |
|
||||||
|
| `--no-detach` | Stay in the foreground (default detaches to the background when running `--daemon`). |
|
||||||
|
| `-p, --port <port>` | TCP listen port (default: 8080, range: 1–65535). |
|
||||||
| `--tls` | Enable TLS. |
|
| `--tls` | Enable TLS. |
|
||||||
| `--cert <path>` | TLS certificate file. |
|
| `--cert <path>` | TLS certificate file (PEM). |
|
||||||
| `--key <path>` | TLS private key file. |
|
| `--key <path>` | TLS private key file (PEM). |
|
||||||
| `--ca <path>` | CA file for peer verification. |
|
| `--ca <path>` | CA file for peer verification (PEM). |
|
||||||
| `--destination-root <path>` | Confine received files to this server-side root;
|
| `--client-cn <name>` | TLS client certificate CN; mandatory with `--tls` (the server verifies the client CN). |
|
||||||
defaults to the current directory. |
|
| `--destination-root <path>` | Confine received files to this server-side root; defaults to the current directory. |
|
||||||
|
| `--address <addr>` | Bind the listening socket to this address. |
|
||||||
|
| `-4`, `--ipv4` | Bind an IPv4 socket (default). |
|
||||||
|
| `-6`, `--ipv6` | Bind an IPv6 socket. |
|
||||||
| `--allow-delete` | Permit client delete manifests. Deletion is refused by default. This also gates `--force` (which can recursively replace/remove a destination directory tree). |
|
| `--allow-delete` | Permit client delete manifests. Deletion is refused by default. This also gates `--force` (which can recursively replace/remove a destination directory tree). |
|
||||||
| `--allow-super` | Standalone TCP listener only: keep super-user activities enabled for a **root** receiver. Without it a root standalone server forces `SUPER_MODE_OFF`, so client `--devices`/`--write-devices`/`--super` and client-chosen ownership requests are skipped/refused. Rejected with `--stdio` (the SSH remote argv is client-composed; use a forced command if the default must hold). No effect when not root. Daemon modules opt in per module with `client owner = yes`. |
|
| `--allow-super` | Standalone TCP listener only: keep super-user activities enabled for a **root** receiver. Without it a root standalone server forces `SUPER_MODE_OFF`, so client `--devices`/`--write-devices`/`--super` and client-chosen ownership requests are skipped/refused. Rejected with `--stdio` (the SSH remote argv is client-composed; use a forced command if the default must hold). No effect when not root. Daemon modules opt in per module with `client owner = yes`. |
|
||||||
|
| `--trust-sender` | Trust the remote sender's file list: skip the receiver's up-front path-traversal re-validation (fewer checks, faster, potentially unsafe; off by default). It does not affect symlink targets, which are stored verbatim either way. |
|
||||||
|
| `--no-super` | Operator veto: never attempt super-user activities (ownership, device nodes) even as root, and refuse any client `--copy-as`/`--super` request. |
|
||||||
|
| `--allow-unauthenticated` | Permit plaintext/anonymous network clients; an auth-required module still accepts only opted-in loopback plaintext. |
|
||||||
|
| `--iconv=LOCAL[,REMOTE]` | Declare this server's LOCAL charset for file-name conversion. |
|
||||||
|
| `--password-file=FILE` | Credential store for modules that declare `auth users`. Requires `--daemon`. |
|
||||||
|
| `--early-input=FILE` | Second credential store layered over `--password-file`. Requires `--daemon`. |
|
||||||
|
| `--hash-credentials <file>` | Read `<file>`'s `user:password` lines and print PBKDF2 credential-store lines to stdout, then exit. Cannot be combined with `--daemon` or `--stdio`. |
|
||||||
|
| `--iterations N` | PBKDF2 iteration count for `--hash-credentials` (default 600000, range 100000–10000000). Requires `--hash-credentials`. |
|
||||||
| `-v`, `--verbose` | Enable debug logging. |
|
| `-v`, `--verbose` | Enable debug logging. |
|
||||||
| `--help` | Print server usage. |
|
| `--help` | Print server usage. |
|
||||||
|
|
||||||
@@ -540,8 +733,9 @@ and `address`, the global section accepts:
|
|||||||
- `hosts allow` / `hosts deny` — comma- and/or whitespace-separated host access
|
- `hosts allow` / `hosts deny` — comma- and/or whitespace-separated host access
|
||||||
patterns.
|
patterns.
|
||||||
|
|
||||||
A `[module]` may also set `max connections` (0 = unlimited; enforced per module
|
A `[module]` requires `path`, and may also set `read only`, `client owner`,
|
||||||
across all connection children) and its own `hosts allow`/`hosts deny`.
|
`auth users`, `max connections` (0 = unlimited; enforced per module across all
|
||||||
|
connection children), and its own `hosts allow`/`hosts deny`.
|
||||||
|
|
||||||
The per-host cap and the shared auth lockout identify a source by its numeric
|
The per-host cap and the shared auth lockout identify a source by its numeric
|
||||||
peer IP. **Loopback peers (127.0.0.0/8, IPv6 `::1`) are exempt**: every local
|
peer IP. **Loopback peers (127.0.0.0/8, IPv6 `::1`) are exempt**: every local
|
||||||
@@ -595,7 +789,7 @@ before the module list, before authentication, and the connecting peer address
|
|||||||
|
|
||||||
## Protocol and Security
|
## Protocol and Security
|
||||||
|
|
||||||
FastSync protocol version `2.21.0` is shared by the client and server. The
|
FastSync protocol version `2.26.0` is shared by the client and server. The
|
||||||
current protocol is sender-driven and includes configuration negotiation,
|
current protocol is sender-driven and includes configuration negotiation,
|
||||||
including the maximum allocation limit, incremental checks, checksums,
|
including the maximum allocation limit, incremental checks, checksums,
|
||||||
manifests, keep-alives, abort handling, per-file remove-source results, and
|
manifests, keep-alives, abort handling, per-file remove-source results, and
|
||||||
@@ -647,10 +841,9 @@ mandates `--client-cn`, so a TLS connection to an auth-required module always
|
|||||||
has its client CN verified (`--client-cn` matches the certificate's CN only, not
|
has its client CN verified (`--client-cn` matches the certificate's CN only, not
|
||||||
a subjectAltName, which is acceptable for a private CA).
|
a subjectAltName, which is acceptable for a private CA).
|
||||||
|
|
||||||
TLS provides encrypted TCP transport. Supplying `--ca` enables certificate
|
TLS provides encrypted TCP transport. Both the client and the server require
|
||||||
verification; without it, traffic is encrypted but peer identity is not
|
`--ca` together with `--tls`, so peer certificates are always verified
|
||||||
verified. Use certificate verification for deployments where authentication
|
(`SSL_VERIFY_PEER`, depth 4). The default TCP transport is not encrypted.
|
||||||
matters. The default TCP transport is not encrypted.
|
|
||||||
|
|
||||||
The receiver protects its destination root with path validation, `openat()`
|
The receiver protects its destination root with path validation, `openat()`
|
||||||
directory traversal, `O_NOFOLLOW`, temporary files, and atomic renames. Delete
|
directory traversal, `O_NOFOLLOW`, temporary files, and atomic renames. Delete
|
||||||
@@ -661,18 +854,27 @@ operations require the server's explicit `--allow-delete` policy.
|
|||||||
The project will reach the drop-in replacement goal in stages:
|
The project will reach the drop-in replacement goal in stages:
|
||||||
|
|
||||||
1. Correct rsync option meanings, including short options, combined options,
|
1. Correct rsync option meanings, including short options, combined options,
|
||||||
and `--option=value` syntax.
|
and `--option=value` syntax — **done** in the rsync-parity wave: `-r`/`-b`/
|
||||||
|
`-L`/`-B`, short-option clustering (`-av`, `-aAX`, `-rlpt`), and attached
|
||||||
|
values (`-B1000`, `-essh`, `-MOPT`) all parse.
|
||||||
2. Add differential tests that compare FastSync and rsync contents, metadata,
|
2. Add differential tests that compare FastSync and rsync contents, metadata,
|
||||||
links, deletes, filters, dry runs, and exit codes.
|
links, deletes, filters, dry runs, and exit codes — **done** for the
|
||||||
3. Make `-a` implement the expected recursive, links, permissions, times,
|
completion wave's scope; the tests live in `tests/integration/` and skip
|
||||||
owner/group, and supported special-file behavior.
|
cleanly when rsync is unavailable.
|
||||||
4. Complete symlink, sparse-file, metadata, delete-policy, and resumable-write
|
3. `-a` implements full rsync `-rlptgoD`; under `-p` the source mode is copied
|
||||||
semantics.
|
exactly (no masking). Ownership application stays privilege-gated, as in
|
||||||
|
rsync.
|
||||||
|
4. Symlink (verbatim storage), sparse-file, metadata, delete-policy (including
|
||||||
|
`--max-delete` partial + exit 25, per-directory `--delete-during`/
|
||||||
|
`--delete-delay`), codecs, and resumable-write semantics are implemented;
|
||||||
|
remaining work is the documented edge cases, which the **Parity Completion
|
||||||
|
Wave** section of `RSYNC_COMPAT.md` enumerates honestly.
|
||||||
5. Add rsync remote-shell and daemon protocol interoperability.
|
5. Add rsync remote-shell and daemon protocol interoperability.
|
||||||
6. Keep FastSync performance options as negotiated, optional extensions.
|
6. Keep FastSync performance options as negotiated, optional extensions.
|
||||||
|
|
||||||
The exhaustive implementation matrix and compatibility notes are in
|
The exhaustive implementation matrix and compatibility notes are in
|
||||||
[`RSYNC_COMPAT.md`](RSYNC_COMPAT.md).
|
[`RSYNC_COMPAT.md`](RSYNC_COMPAT.md); each row is classified as parity, caveat,
|
||||||
|
or divergent.
|
||||||
|
|
||||||
## Testing
|
## Testing
|
||||||
|
|
||||||
@@ -709,10 +911,13 @@ rsync protocol or filesystem-semantic compatibility.
|
|||||||
|
|
||||||
## Performance Guidance
|
## Performance Guidance
|
||||||
|
|
||||||
- Use `-m` for workloads with many files or enough CPU parallelism.
|
- Use `-j`/`--threads` for workloads with many files or enough CPU parallelism
|
||||||
- Use `-c` or `-z` when network bandwidth is more constrained than CPU.
|
(`-m` is `--prune-empty-dirs`).
|
||||||
|
- Use `-z` when network bandwidth is more constrained than CPU (`-c` is
|
||||||
|
`--checksum`, not a bandwidth option).
|
||||||
- Tune `--chunk-size` for file sizes, memory limits, and network latency.
|
- Tune `--chunk-size` for file sizes, memory limits, and network latency.
|
||||||
- Use `-f` for large uncompressed TCP transfers where zero-copy I/O helps.
|
- Use `--sendfile` for large uncompressed TCP transfers where zero-copy I/O
|
||||||
|
helps (`-f` is `--filter`).
|
||||||
- Use `--incremental` to avoid retransmitting unchanged files.
|
- Use `--incremental` to avoid retransmitting unchanged files.
|
||||||
- Use `--delta` for changed files when both endpoints are FastSync peers.
|
- Use `--delta` for changed files when both endpoints are FastSync peers.
|
||||||
- Use `--bwlimit` when sharing a link with other traffic.
|
- Use `--bwlimit` when sharing a link with other traffic.
|
||||||
|
|||||||
+579
-265
File diff suppressed because it is too large
Load Diff
@@ -38,6 +38,8 @@ pkgs.mkShell {
|
|||||||
|
|
||||||
buildInputs = with pkgs; [
|
buildInputs = with pkgs; [
|
||||||
zstd
|
zstd
|
||||||
|
zlib
|
||||||
|
lz4
|
||||||
openssl
|
openssl
|
||||||
];
|
];
|
||||||
|
|
||||||
@@ -54,6 +56,6 @@ pkgs.mkShell {
|
|||||||
echo "FastSync dev shell ready."
|
echo "FastSync dev shell ready."
|
||||||
echo " Build: cmake -B build -S . && cmake --build build -j\$(nproc)"
|
echo " Build: cmake -B build -S . && cmake --build build -j\$(nproc)"
|
||||||
echo " Unit: ./build/tests"
|
echo " Unit: ./build/tests"
|
||||||
echo " CI parity: docker run --rm --user \"\$(id -u):\$(id -g)\" -v \"\$PWD:/workspace\" -w /workspace gitea.tap-tap.win/taptap/fastsync-ci:v10 ..."
|
echo " CI parity: docker run --rm --user \"\$(id -u):\$(id -g)\" -v \"\$PWD:/workspace\" -w /workspace gitea.tap-tap.win/taptap/fastsync-ci:v11 ..."
|
||||||
'';
|
'';
|
||||||
}
|
}
|
||||||
|
|||||||
+455
-154
@@ -1,5 +1,7 @@
|
|||||||
#include "change_list.h"
|
#include "change_list.h"
|
||||||
|
#include "checksum.h"
|
||||||
#include "utils.h"
|
#include "utils.h"
|
||||||
|
#include <fcntl.h>
|
||||||
#include <limits.h>
|
#include <limits.h>
|
||||||
#include <stdint.h>
|
#include <stdint.h>
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
@@ -7,21 +9,7 @@
|
|||||||
#include <string.h>
|
#include <string.h>
|
||||||
#include <sys/stat.h>
|
#include <sys/stat.h>
|
||||||
#include <time.h>
|
#include <time.h>
|
||||||
|
#include <unistd.h>
|
||||||
/* Itemize code emitted for a transferred regular file.
|
|
||||||
*
|
|
||||||
* Layout (rsync-compatible 11-char item): `>f` marks a regular file that was
|
|
||||||
* transferred to the remote host; the trailing nine markers are, in order,
|
|
||||||
* c(hecksum) s(ize) t(ime) p(erms) o(wner) g(roup) u(ser/acl) a(ttrs) x(attrs).
|
|
||||||
* Every marker is `+` (FastSync does not compare each attribute on the
|
|
||||||
* receiving side, so a sent file is reported as fully updated). Files that
|
|
||||||
* are already up to date print no line at all, matching rsync's single -i
|
|
||||||
* which only itemizes changes.
|
|
||||||
*
|
|
||||||
* Because the scanner only yields regular-file transfer candidates, `>d`
|
|
||||||
* (directory) lines are never produced; directories are not transferred as
|
|
||||||
* items by FastSync. */
|
|
||||||
#define ITEMIZE_SENT_FILE ">f+++++++++"
|
|
||||||
|
|
||||||
typedef struct {
|
typedef struct {
|
||||||
char* data;
|
char* data;
|
||||||
@@ -80,103 +68,14 @@ static bool strbuf_append(StrBuf* buf, const char* text) {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
static bool strbuf_append_ull(StrBuf* buf, unsigned long long value) {
|
|
||||||
char digits[32];
|
|
||||||
int written = snprintf(digits, sizeof(digits), "%llu", value);
|
|
||||||
if (written < 0 || (size_t)written >= sizeof(digits))
|
|
||||||
return false;
|
|
||||||
return strbuf_append(buf, digits);
|
|
||||||
}
|
|
||||||
|
|
||||||
static bool strbuf_append_longlong(StrBuf* buf, long long value) {
|
|
||||||
char digits[32];
|
|
||||||
int written = snprintf(digits, sizeof(digits), "%lld", value);
|
|
||||||
if (written < 0 || (size_t)written >= sizeof(digits))
|
|
||||||
return false;
|
|
||||||
return strbuf_append(buf, digits);
|
|
||||||
}
|
|
||||||
|
|
||||||
bool change_list_enabled(const Config* config) {
|
bool change_list_enabled(const Config* config) {
|
||||||
return config != NULL && (config->itemize_changes || config->out_format != NULL ||
|
return config != NULL && (config->itemize_changes || config->out_format != NULL ||
|
||||||
(config->log_file != NULL && config->log_file_format != NULL));
|
(config->log_file != NULL && config->log_file_format != NULL));
|
||||||
}
|
}
|
||||||
|
|
||||||
char* change_render_itemize(const ChangeEvent* event) {
|
/* ---- Itemize code ---- */
|
||||||
if (event == NULL || event->decision != CHANGE_SENT)
|
|
||||||
return str_dup("");
|
|
||||||
const char* code = event->is_directory ? ">d+++++++++" : ITEMIZE_SENT_FILE;
|
|
||||||
StrBuf line = {0};
|
|
||||||
bool ok = strbuf_append(&line, code) && strbuf_append(&line, " ") &&
|
|
||||||
strbuf_append(&line, event->path != NULL ? event->path : "");
|
|
||||||
if (!ok) {
|
|
||||||
strbuf_free(&line);
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
return line.data;
|
|
||||||
}
|
|
||||||
|
|
||||||
static const char* leaf_name(const char* path) {
|
/* Format the permission bits as an `ls -l` string, e.g. `-rw-r--r--`. */
|
||||||
if (path == NULL)
|
|
||||||
return "";
|
|
||||||
const char* slash = strrchr(path, '/');
|
|
||||||
return slash != NULL && slash[1] != '\0' ? slash + 1 : path;
|
|
||||||
}
|
|
||||||
|
|
||||||
char* change_render_format(const char* format, const ChangeEvent* event) {
|
|
||||||
if (format == NULL)
|
|
||||||
return NULL;
|
|
||||||
StrBuf line = {0};
|
|
||||||
bool ok = true;
|
|
||||||
for (const char* p = format; *p != '\0' && ok;) {
|
|
||||||
if (*p != '%') {
|
|
||||||
ok = strbuf_append_char(&line, *p);
|
|
||||||
p++;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
char token = p[1];
|
|
||||||
if (token == '\0') {
|
|
||||||
ok = strbuf_append_char(&line, '%');
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
switch (token) {
|
|
||||||
case '%':
|
|
||||||
ok = strbuf_append_char(&line, '%');
|
|
||||||
break;
|
|
||||||
case 'f':
|
|
||||||
ok = strbuf_append(&line, event->path != NULL ? event->path : "");
|
|
||||||
break;
|
|
||||||
case 'n':
|
|
||||||
ok = strbuf_append(&line, leaf_name(event->path));
|
|
||||||
break;
|
|
||||||
case 'l':
|
|
||||||
ok = strbuf_append_ull(&line, event->size);
|
|
||||||
break;
|
|
||||||
case 'b':
|
|
||||||
ok = strbuf_append_ull(&line, event->bytes_sent);
|
|
||||||
break;
|
|
||||||
case 'M':
|
|
||||||
ok = strbuf_append_longlong(&line, (long long)event->mtime_sec);
|
|
||||||
break;
|
|
||||||
default:
|
|
||||||
/* Unknown escape sequences are preserved verbatim. */
|
|
||||||
ok = strbuf_append_char(&line, '%') && strbuf_append_char(&line, token);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
p += 2;
|
|
||||||
}
|
|
||||||
if (!ok) {
|
|
||||||
strbuf_free(&line);
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
if (line.data == NULL) {
|
|
||||||
line.data = str_dup("");
|
|
||||||
if (!line.data)
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
return line.data;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Format a mode as an `ls -l` permission string, e.g. `-rw-r--r--`. */
|
|
||||||
static void mode_to_ls_string(mode_t mode, char out[11]) {
|
static void mode_to_ls_string(mode_t mode, char out[11]) {
|
||||||
out[0] = S_ISDIR(mode) ? 'd'
|
out[0] = S_ISDIR(mode) ? 'd'
|
||||||
: S_ISLNK(mode) ? 'l'
|
: S_ISLNK(mode) ? 'l'
|
||||||
@@ -198,29 +97,94 @@ static void mode_to_ls_string(mode_t mode, char out[11]) {
|
|||||||
out[10] = '\0';
|
out[10] = '\0';
|
||||||
}
|
}
|
||||||
|
|
||||||
char* change_render_list_line(mode_t mode, unsigned long long size, time_t mtime,
|
static char itemize_type_char(const ChangeEvent* event) {
|
||||||
const char* path) {
|
if (event->is_directory)
|
||||||
char permission[11];
|
return 'd';
|
||||||
mode_to_ls_string(mode, permission);
|
if (event->is_symlink)
|
||||||
char date[32];
|
return 'L';
|
||||||
struct tm broken_down;
|
if (event->is_special) {
|
||||||
if (localtime_r(&mtime, &broken_down) != NULL) {
|
if (S_ISCHR(event->mode) || S_ISBLK(event->mode))
|
||||||
if (strftime(date, sizeof(date), "%Y/%m/%d %H:%M:%S", &broken_down) == 0)
|
return 'D';
|
||||||
snprintf(date, sizeof(date), "?");
|
return 'S';
|
||||||
} else {
|
|
||||||
snprintf(date, sizeof(date), "?");
|
|
||||||
}
|
}
|
||||||
|
return 'f';
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool times_match(const Config* config, const ChangeEvent* event) {
|
||||||
|
if (!event->dest.known || !event->dest.existed)
|
||||||
|
return false;
|
||||||
|
if (event->mtime_sec == event->dest.mtime_sec)
|
||||||
|
return event->mtime_nsec == event->dest.mtime_nsec;
|
||||||
|
long long delta = (long long)event->mtime_sec - (long long)event->dest.mtime_sec;
|
||||||
|
if (delta < 0)
|
||||||
|
delta = -delta;
|
||||||
|
return delta <= (long long)config->modify_window;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Fill the 11-character itemize code (10 chars + NUL). `created` means the
|
||||||
|
* destination entry did not exist, so every attribute marker is `+`. */
|
||||||
|
static void itemize_code(const Config* config, const ChangeEvent* event, char code[12]) {
|
||||||
|
bool known = event->dest.known;
|
||||||
|
bool created = !known || !event->dest.existed;
|
||||||
|
char update;
|
||||||
|
if (event->is_hardlink)
|
||||||
|
update = 'h';
|
||||||
|
else if (created)
|
||||||
|
update = (event->is_directory || event->is_symlink || event->is_special) ? 'c' : '>';
|
||||||
|
else
|
||||||
|
update = '>';
|
||||||
|
code[0] = update;
|
||||||
|
code[1] = itemize_type_char(event);
|
||||||
|
if (created) {
|
||||||
|
for (int i = 0; i < 9; i++)
|
||||||
|
code[2 + i] = '+';
|
||||||
|
code[11] = '\0';
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
bool size_diff = event->size != event->dest.size;
|
||||||
|
bool time_diff = !times_match(config, event);
|
||||||
|
bool perms_diff = (event->mode & 07777) != (event->dest.mode & 07777);
|
||||||
|
bool owner_diff = event->uid != (uid_t)event->dest.uid;
|
||||||
|
bool group_diff = event->gid != (gid_t)event->dest.gid;
|
||||||
|
code[2] = '.'; /* checksum: no destination digest available */
|
||||||
|
code[3] = size_diff ? 's' : '.';
|
||||||
|
code[4] = time_diff ? 't' : '.';
|
||||||
|
code[5] = (config->preserve_perms && perms_diff) ? 'p' : '.';
|
||||||
|
code[6] = (config->preserve_owner && owner_diff) ? 'o' : '.';
|
||||||
|
code[7] = (config->preserve_group && group_diff) ? 'g' : '.';
|
||||||
|
code[8] = '.'; /* reserved */
|
||||||
|
code[9] = '.'; /* acl: not compared */
|
||||||
|
code[10] = '.';
|
||||||
|
code[11] = '\0';
|
||||||
|
}
|
||||||
|
|
||||||
|
/* rsync %n: the transfer-relative name, with a trailing slash for directories. */
|
||||||
|
static bool append_name(StrBuf* buf, const ChangeEvent* event) {
|
||||||
|
if (!strbuf_append(buf, event->name != NULL ? event->name : ""))
|
||||||
|
return false;
|
||||||
|
if (event->is_directory && (event->name == NULL || event->name[0] == '\0' ||
|
||||||
|
event->name[strlen(event->name) - 1] != '/'))
|
||||||
|
return strbuf_append_char(buf, '/');
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* rsync %L: " -> target" for a symlink, " => target" for a hard link, else "". */
|
||||||
|
static bool append_link_suffix(StrBuf* buf, const ChangeEvent* event) {
|
||||||
|
if (event->is_symlink && event->symlink_target != NULL)
|
||||||
|
return strbuf_append(buf, " -> ") && strbuf_append(buf, event->symlink_target);
|
||||||
|
if (event->is_hardlink && event->hardlink_target != NULL)
|
||||||
|
return strbuf_append(buf, " => ") && strbuf_append(buf, event->hardlink_target);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
char* change_render_itemize(const Config* config, const ChangeEvent* event) {
|
||||||
|
if (event == NULL || event->decision != CHANGE_SENT)
|
||||||
|
return str_dup("");
|
||||||
|
char code[12];
|
||||||
|
itemize_code(config, event, code);
|
||||||
StrBuf line = {0};
|
StrBuf line = {0};
|
||||||
char size_field[32];
|
bool ok = strbuf_append(&line, code) && strbuf_append_char(&line, ' ') &&
|
||||||
int written = snprintf(size_field, sizeof(size_field), "%llu", size);
|
append_name(&line, event) && append_link_suffix(&line, event);
|
||||||
if (written < 0 || (size_t)written >= sizeof(size_field)) {
|
|
||||||
strbuf_free(&line);
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
bool ok = strbuf_append(&line, permission) && strbuf_append_char(&line, ' ') &&
|
|
||||||
strbuf_append(&line, size_field) && strbuf_append_char(&line, ' ') &&
|
|
||||||
strbuf_append(&line, date) && strbuf_append_char(&line, ' ') &&
|
|
||||||
strbuf_append(&line, path != NULL ? path : "");
|
|
||||||
if (!ok) {
|
if (!ok) {
|
||||||
strbuf_free(&line);
|
strbuf_free(&line);
|
||||||
return NULL;
|
return NULL;
|
||||||
@@ -228,6 +192,238 @@ char* change_render_list_line(mode_t mode, unsigned long long size, time_t mtime
|
|||||||
return line.data;
|
return line.data;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* ---- --out-format / --log-file-format ---- */
|
||||||
|
|
||||||
|
/* rsync 3.4.1's `%C` uses the negotiated transfer checksum; with the default
|
||||||
|
* "auto" choice on both ends that is xxh128. FastSync's internal XXH64 default
|
||||||
|
* is not an rsync algorithm, so map it to xxh128 for parity. */
|
||||||
|
static ChecksumAlgo out_format_checksum_algo(const Config* config) {
|
||||||
|
switch ((ChecksumAlgo)config->checksum_algo) {
|
||||||
|
case CHECKSUM_ALGO_MD5:
|
||||||
|
return CHECKSUM_ALGO_MD5;
|
||||||
|
case CHECKSUM_ALGO_XXH3:
|
||||||
|
return CHECKSUM_ALGO_XXH3;
|
||||||
|
case CHECKSUM_ALGO_XXH128:
|
||||||
|
return CHECKSUM_ALGO_XXH128;
|
||||||
|
case CHECKSUM_ALGO_XXH64:
|
||||||
|
default:
|
||||||
|
return CHECKSUM_ALGO_XXH128;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Render a digest as rsync's sum_as_hex: for xxh128 the HIGH 64-bit half is
|
||||||
|
* printed before the low half; every other algorithm prints its bytes in order. */
|
||||||
|
static void digest_to_hex(ChecksumAlgo algo, const uint8_t* digest, size_t len, char* out) {
|
||||||
|
if (algo == CHECKSUM_ALGO_XXH128 && len == 16) {
|
||||||
|
uint64_t low = 0;
|
||||||
|
uint64_t high = 0;
|
||||||
|
memcpy(&low, digest, sizeof(low));
|
||||||
|
memcpy(&high, digest + 8, sizeof(high));
|
||||||
|
snprintf(out, len * 2 + 1, "%016llx%016llx", (unsigned long long)high, (unsigned long long)low);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
static const char hex[] = "0123456789abcdef";
|
||||||
|
for (size_t i = 0; i < len; i++) {
|
||||||
|
out[i * 2] = hex[(digest[i] >> 4) & 0xf];
|
||||||
|
out[i * 2 + 1] = hex[digest[i] & 0xf];
|
||||||
|
}
|
||||||
|
out[len * 2] = '\0';
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool format_uses_checksum(const char* format) {
|
||||||
|
if (format == NULL)
|
||||||
|
return false;
|
||||||
|
for (const char* p = format; *p != '\0';) {
|
||||||
|
if (*p != '%') {
|
||||||
|
p++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
char token = p[1];
|
||||||
|
if (token == '\0')
|
||||||
|
break;
|
||||||
|
if (token == 'C')
|
||||||
|
return true;
|
||||||
|
p += 2;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Fill event->checksum/checksum_known for a transferred regular file. A
|
||||||
|
* non-regular entry (or a hard-link sibling) leaves checksum_known false, which
|
||||||
|
* renders as spaces like rsync. */
|
||||||
|
static void fill_event_checksum(const Config* config, const File* file, ChangeEvent* event) {
|
||||||
|
if (file == NULL || file->is_dir || file->is_symlink || file->is_special ||
|
||||||
|
(file->link_group != 0 && !file->link_first))
|
||||||
|
return;
|
||||||
|
if (!format_uses_checksum(config->out_format) && !format_uses_checksum(config->log_file_format))
|
||||||
|
return;
|
||||||
|
if (file->path == NULL)
|
||||||
|
return;
|
||||||
|
ChecksumAlgo algo = out_format_checksum_algo(config);
|
||||||
|
uint8_t digest[CHECKSUM_MAX_DIGEST_LEN];
|
||||||
|
size_t len = 0;
|
||||||
|
/* rsync's %C is the transfer checksum, which is always seeded with 0 (it is
|
||||||
|
* independent of --checksum-seed, as rsync 3.4.1 demonstrates). */
|
||||||
|
if (!checksum_digest_file(algo, 0, file->path, digest, sizeof(digest), &len))
|
||||||
|
return;
|
||||||
|
digest_to_hex(algo, digest, len, event->checksum);
|
||||||
|
event->checksum_known = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
char* change_render_format(const char* format, const Config* config, const ChangeEvent* event) {
|
||||||
|
if (format == NULL || event == NULL)
|
||||||
|
return NULL;
|
||||||
|
StrBuf line = {0};
|
||||||
|
bool ok = true;
|
||||||
|
for (const char* p = format; *p != '\0' && ok;) {
|
||||||
|
if (*p != '%') {
|
||||||
|
ok = strbuf_append_char(&line, *p);
|
||||||
|
p++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
char token = p[1];
|
||||||
|
if (token == '\0') {
|
||||||
|
ok = strbuf_append_char(&line, '%');
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
switch (token) {
|
||||||
|
case '%':
|
||||||
|
ok = strbuf_append_char(&line, '%');
|
||||||
|
break;
|
||||||
|
case 'i': {
|
||||||
|
if (event->deleted) {
|
||||||
|
/* rsync's ITEM_DELETED itemize code: `*deleting ` (11 chars). */
|
||||||
|
ok = strbuf_append(&line, "*deleting ");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
char code[12];
|
||||||
|
itemize_code(config, event, code);
|
||||||
|
ok = strbuf_append(&line, code);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case 'f':
|
||||||
|
ok = strbuf_append(&line, event->path != NULL ? event->path : "");
|
||||||
|
break;
|
||||||
|
case 'n':
|
||||||
|
ok = append_name(&line, event);
|
||||||
|
break;
|
||||||
|
case 'L':
|
||||||
|
ok = append_link_suffix(&line, event);
|
||||||
|
break;
|
||||||
|
case 'l': {
|
||||||
|
char digits[32];
|
||||||
|
int written = snprintf(digits, sizeof(digits), "%llu", event->size);
|
||||||
|
ok = written >= 0 && (size_t)written < sizeof(digits) && strbuf_append(&line, digits);
|
||||||
|
} break;
|
||||||
|
case 'b': {
|
||||||
|
char digits[32];
|
||||||
|
int written = snprintf(digits, sizeof(digits), "%llu", event->bytes_sent);
|
||||||
|
ok = written >= 0 && (size_t)written < sizeof(digits) && strbuf_append(&line, digits);
|
||||||
|
} break;
|
||||||
|
case 'c': {
|
||||||
|
char digits[32];
|
||||||
|
int written = snprintf(digits, sizeof(digits), "%llu", event->bytes_read);
|
||||||
|
ok = written >= 0 && (size_t)written < sizeof(digits) && strbuf_append(&line, digits);
|
||||||
|
} break;
|
||||||
|
case 'C': {
|
||||||
|
if (event->checksum_known) {
|
||||||
|
ok = strbuf_append(&line, event->checksum);
|
||||||
|
} else {
|
||||||
|
/* rsync pads a non-regular / untransferred entry with spaces. */
|
||||||
|
ChecksumAlgo algo = out_format_checksum_algo(config);
|
||||||
|
int width = checksum_digest_len(algo) * 2;
|
||||||
|
for (int i = 0; i < width && ok; i++)
|
||||||
|
ok = strbuf_append_char(&line, ' ');
|
||||||
|
}
|
||||||
|
} break;
|
||||||
|
case 'M': {
|
||||||
|
char when[32];
|
||||||
|
if (format_rsync_datetime(event->mtime_sec, true, when, sizeof(when)))
|
||||||
|
ok = strbuf_append(&line, when);
|
||||||
|
} break;
|
||||||
|
case 't': {
|
||||||
|
char when[32];
|
||||||
|
if (format_rsync_datetime(time(NULL), false, when, sizeof(when)))
|
||||||
|
ok = strbuf_append(&line, when);
|
||||||
|
} break;
|
||||||
|
case 'o':
|
||||||
|
ok = strbuf_append(&line, "send");
|
||||||
|
break;
|
||||||
|
case 'p': {
|
||||||
|
char digits[32];
|
||||||
|
int written = snprintf(digits, sizeof(digits), "%ld", (long)getpid());
|
||||||
|
ok = written >= 0 && (size_t)written < sizeof(digits) && strbuf_append(&line, digits);
|
||||||
|
} break;
|
||||||
|
case 'B': {
|
||||||
|
char permission[11];
|
||||||
|
mode_to_ls_string(event->mode, permission);
|
||||||
|
ok = strbuf_append(&line, permission + 1);
|
||||||
|
} break;
|
||||||
|
case 'U': {
|
||||||
|
char digits[32];
|
||||||
|
int written = snprintf(digits, sizeof(digits), "%u", (unsigned)event->uid);
|
||||||
|
ok = written >= 0 && (size_t)written < sizeof(digits) && strbuf_append(&line, digits);
|
||||||
|
} break;
|
||||||
|
case 'G': {
|
||||||
|
char digits[32];
|
||||||
|
int written = snprintf(digits, sizeof(digits), "%u", (unsigned)event->gid);
|
||||||
|
ok = written >= 0 && (size_t)written < sizeof(digits) && strbuf_append(&line, digits);
|
||||||
|
} break;
|
||||||
|
default:
|
||||||
|
/* Unknown escape sequences are preserved verbatim. */
|
||||||
|
ok = strbuf_append_char(&line, '%') && strbuf_append_char(&line, token);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
p += 2;
|
||||||
|
}
|
||||||
|
if (!ok) {
|
||||||
|
strbuf_free(&line);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
if (line.data == NULL) {
|
||||||
|
line.data = str_dup("");
|
||||||
|
if (!line.data)
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
return line.data;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ---- --list-only ---- */
|
||||||
|
|
||||||
|
char* change_render_list_line(const Config* config, const ChangeEvent* event) {
|
||||||
|
(void)config;
|
||||||
|
if (event == NULL)
|
||||||
|
return NULL;
|
||||||
|
char permission[11];
|
||||||
|
mode_to_ls_string(event->mode, permission);
|
||||||
|
char date[32];
|
||||||
|
if (!format_rsync_datetime(event->mtime_sec, false, date, sizeof(date)))
|
||||||
|
snprintf(date, sizeof(date), "?");
|
||||||
|
StrBuf line = {0};
|
||||||
|
char size_field[40];
|
||||||
|
char grouped[32];
|
||||||
|
if (!format_big_num(event->size, false, grouped, sizeof(grouped))) {
|
||||||
|
strbuf_free(&line);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
int written = snprintf(size_field, sizeof(size_field), "%15s", grouped);
|
||||||
|
if (written < 0 || (size_t)written >= sizeof(size_field)) {
|
||||||
|
strbuf_free(&line);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
const char* name = event->name != NULL && event->name[0] != '\0' ? event->name : ".";
|
||||||
|
bool ok = strbuf_append(&line, permission) && strbuf_append(&line, size_field) &&
|
||||||
|
strbuf_append_char(&line, ' ') && strbuf_append(&line, date) &&
|
||||||
|
strbuf_append_char(&line, ' ') && strbuf_append(&line, name);
|
||||||
|
if (!ok) {
|
||||||
|
strbuf_free(&line);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
return line.data;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ---- Event emission ---- */
|
||||||
|
|
||||||
static void print_escaped_line(FILE* stream, const char* line, bool eight_bit_output) {
|
static void print_escaped_line(FILE* stream, const char* line, bool eight_bit_output) {
|
||||||
char* escaped = output_escape(line, eight_bit_output);
|
char* escaped = output_escape(line, eight_bit_output);
|
||||||
if (escaped != NULL) {
|
if (escaped != NULL) {
|
||||||
@@ -247,15 +443,16 @@ void change_emit(const Config* config, const ChangeEvent* event) {
|
|||||||
bool to_stdout = config->itemize_changes || config->out_format != NULL;
|
bool to_stdout = config->itemize_changes || config->out_format != NULL;
|
||||||
bool to_log = config->log_file != NULL && config->log_file_format != NULL;
|
bool to_log = config->log_file != NULL && config->log_file_format != NULL;
|
||||||
if (to_stdout) {
|
if (to_stdout) {
|
||||||
char* line = config->out_format != NULL ? change_render_format(config->out_format, event)
|
char* line = config->out_format != NULL
|
||||||
: change_render_itemize(event);
|
? change_render_format(config->out_format, config, event)
|
||||||
|
: change_render_itemize(config, event);
|
||||||
if (line != NULL) {
|
if (line != NULL) {
|
||||||
print_escaped_line(stdout, line, config->eight_bit_output);
|
print_escaped_line(stdout, line, config->eight_bit_output);
|
||||||
free(line);
|
free(line);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (to_log) {
|
if (to_log) {
|
||||||
char* line = change_render_format(config->log_file_format, event);
|
char* line = change_render_format(config->log_file_format, config, event);
|
||||||
if (line != NULL) {
|
if (line != NULL) {
|
||||||
print_escaped_line(config->log_file, line, config->eight_bit_output);
|
print_escaped_line(config->log_file, line, config->eight_bit_output);
|
||||||
free(line);
|
free(line);
|
||||||
@@ -266,9 +463,6 @@ void change_emit(const Config* config, const ChangeEvent* event) {
|
|||||||
static bool format_uses_mtime(const char* format) {
|
static bool format_uses_mtime(const char* format) {
|
||||||
if (format == NULL)
|
if (format == NULL)
|
||||||
return false;
|
return false;
|
||||||
/* Mirror change_render_format's tokenizer: "%%" is a literal percent (so
|
|
||||||
* "%%M" does NOT expand %M) and unknown "%X" escapes consume both chars.
|
|
||||||
* This keeps the optional stat() fallback below in step with the renderer. */
|
|
||||||
for (const char* p = format; *p != '\0';) {
|
for (const char* p = format; *p != '\0';) {
|
||||||
if (*p != '%') {
|
if (*p != '%') {
|
||||||
p++;
|
p++;
|
||||||
@@ -284,46 +478,153 @@ static bool format_uses_mtime(const char* format) {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
void change_emit_file_sent(const Config* config, const File* file) {
|
/* Relative path of an entry below the transfer root (no leading slash). Uses
|
||||||
|
* the sender-side send_path override when present (bare-relative -R layout). */
|
||||||
|
static char* relative_name(const Config* config, const File* file) {
|
||||||
|
const char* full = file_wire_path(file);
|
||||||
|
if (file->send_path != NULL)
|
||||||
|
return str_dup(full != NULL ? full : "");
|
||||||
|
const char* root = config->send_directory;
|
||||||
|
if (root == NULL || full == NULL)
|
||||||
|
return str_dup(full != NULL ? full : "");
|
||||||
|
size_t root_len = strlen(root);
|
||||||
|
while (root_len > 1 && root[root_len - 1] == '/')
|
||||||
|
root_len--;
|
||||||
|
if (strncmp(root, full, root_len) == 0) {
|
||||||
|
if (full[root_len] == '\0')
|
||||||
|
return str_dup("");
|
||||||
|
if (full[root_len] == '/')
|
||||||
|
return str_dup(full + root_len + 1);
|
||||||
|
}
|
||||||
|
return str_dup(full);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* rsync %f long form: the source argument as typed (leading '/' removed,
|
||||||
|
* trailing '/' removed, leading "./" removed) joined to the relative name. */
|
||||||
|
static char* display_name(const Config* config, const char* name) {
|
||||||
|
const char* root = config->send_directory;
|
||||||
|
if (root == NULL)
|
||||||
|
return str_dup(name != NULL ? name : "");
|
||||||
|
const char* p = root;
|
||||||
|
while (*p == '/')
|
||||||
|
p++;
|
||||||
|
if (p[0] == '.' && p[1] == '/')
|
||||||
|
p += 2;
|
||||||
|
size_t root_len = strlen(p);
|
||||||
|
while (root_len > 0 && p[root_len - 1] == '/')
|
||||||
|
root_len--;
|
||||||
|
size_t name_len = name != NULL ? strlen(name) : 0;
|
||||||
|
if (root_len == 0 && name_len == 0)
|
||||||
|
return str_dup("");
|
||||||
|
char* out = malloc(root_len + (root_len > 0 && name_len > 0 ? 1 : 0) + name_len + 1);
|
||||||
|
if (!out)
|
||||||
|
return NULL;
|
||||||
|
size_t offset = 0;
|
||||||
|
if (root_len > 0) {
|
||||||
|
memcpy(out, p, root_len);
|
||||||
|
offset = root_len;
|
||||||
|
}
|
||||||
|
if (root_len > 0 && name_len > 0)
|
||||||
|
out[offset++] = '/';
|
||||||
|
if (name_len > 0)
|
||||||
|
memcpy(out + offset, name, name_len);
|
||||||
|
out[offset + name_len] = '\0';
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void fill_event_from_file(const Config* config, const File* file, ChangeEvent* event,
|
||||||
|
char** name_out, char** path_out) {
|
||||||
|
char* name = relative_name(config, file);
|
||||||
|
char* path = display_name(config, name);
|
||||||
|
event->name = name;
|
||||||
|
event->path = path;
|
||||||
|
*name_out = name;
|
||||||
|
*path_out = path;
|
||||||
|
if (file->metadata != NULL) {
|
||||||
|
event->mtime_sec = file->metadata->mtime_sec;
|
||||||
|
event->mtime_nsec = file->metadata->mtime_nsec;
|
||||||
|
event->mode = file->metadata->mode;
|
||||||
|
event->uid = file->metadata->uid;
|
||||||
|
event->gid = file->metadata->gid;
|
||||||
|
} else if (format_uses_mtime(config->out_format) || format_uses_mtime(config->log_file_format)) {
|
||||||
|
struct stat st;
|
||||||
|
if (file->path != NULL && stat(file->path, &st) == 0) {
|
||||||
|
event->mtime_sec = st.st_mtime;
|
||||||
|
event->mtime_nsec = st.st_mtim.tv_nsec;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void change_emit_file_sent_bytes(const Config* config, const File* file,
|
||||||
|
unsigned long long bytes_sent, unsigned long long bytes_read) {
|
||||||
if (file == NULL || !change_list_enabled(config))
|
if (file == NULL || !change_list_enabled(config))
|
||||||
return;
|
return;
|
||||||
ChangeEvent event;
|
ChangeEvent event;
|
||||||
memset(&event, 0, sizeof(event));
|
memset(&event, 0, sizeof(event));
|
||||||
/* The displayed path is the one transmitted (with -R + --files-from this is
|
|
||||||
the bare relative destination path); the metadata fallback below still
|
|
||||||
stats the local absolute path. */
|
|
||||||
event.path = file_wire_path(file);
|
|
||||||
event.decision = CHANGE_SENT;
|
event.decision = CHANGE_SENT;
|
||||||
event.is_directory = false;
|
event.is_directory = false;
|
||||||
|
event.is_symlink = false;
|
||||||
|
event.is_special = false;
|
||||||
|
event.is_hardlink = false;
|
||||||
event.size = file->data != NULL ? file->data->size : 0;
|
event.size = file->data != NULL ? file->data->size : 0;
|
||||||
/* FastSync has no wire-byte counter yet, so %b reports the source length
|
event.dest = file->dest_state;
|
||||||
* that had to be delivered (always equal to %l); the actual bytes written
|
if (file->is_symlink) {
|
||||||
* to the socket (compressed/delta) are not measured. */
|
event.is_symlink = true;
|
||||||
event.bytes_sent = event.size;
|
event.symlink_target = file->symlink_target;
|
||||||
if (file->metadata != NULL) {
|
event.size = file->symlink_target != NULL ? strlen(file->symlink_target) : 0;
|
||||||
event.mtime_sec = file->metadata->mtime_sec;
|
event.bytes_sent = 0;
|
||||||
} else if (format_uses_mtime(config->out_format) || format_uses_mtime(config->log_file_format)) {
|
} else if (file->is_special) {
|
||||||
/* Best-effort fallback for %M when no metadata was captured (no -M): the
|
event.is_special = true;
|
||||||
* path is stat()ed just to fill the field, and any failure leaves 0. */
|
event.bytes_sent = 0;
|
||||||
struct stat st;
|
} else if (file->link_group != 0 && !file->link_first) {
|
||||||
if (file->path != NULL && stat(file->path, &st) == 0)
|
event.is_hardlink = true;
|
||||||
event.mtime_sec = st.st_mtime;
|
event.hardlink_target = file->hardlink_target;
|
||||||
|
event.bytes_sent = 0;
|
||||||
|
} else {
|
||||||
|
event.bytes_sent = bytes_sent;
|
||||||
|
/* rsync's %c is the block-checksum bytes received for the file. Even a
|
||||||
|
* whole-file transfer (no basis; --append/--inplace included) receives
|
||||||
|
* rsync's 16-byte sum header, so rsync reports 16; a dry run transfers
|
||||||
|
* nothing and reports 0. FastSync's whole-file path has no sum header, so
|
||||||
|
* report rsync's value for parity. With delta enabled the real received
|
||||||
|
* bytes are kept, but FastSync's signature framing differs from rsync's so
|
||||||
|
* those stay numerically divergent. */
|
||||||
|
bool delta_active = config->use_delta && !config->whole_file;
|
||||||
|
event.bytes_read = (!config->dry_run && !delta_active) ? 16 : bytes_read;
|
||||||
}
|
}
|
||||||
|
char* name = NULL;
|
||||||
|
char* path = NULL;
|
||||||
|
fill_event_from_file(config, file, &event, &name, &path);
|
||||||
|
if (name != NULL && path != NULL) {
|
||||||
|
fill_event_checksum(config, file, &event);
|
||||||
change_emit(config, &event);
|
change_emit(config, &event);
|
||||||
|
}
|
||||||
|
free(name);
|
||||||
|
free(path);
|
||||||
|
}
|
||||||
|
|
||||||
|
void change_emit_file_sent(const Config* config, const File* file) {
|
||||||
|
if (file == NULL)
|
||||||
|
return;
|
||||||
|
unsigned long long payload = file->data != NULL ? file->data->size : 0;
|
||||||
|
change_emit_file_sent_bytes(config, file, payload, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Build and emit a CHANGE_SENT event for an explicit directory entry (-d). */
|
|
||||||
void change_emit_dir_sent(const Config* config, const File* file) {
|
void change_emit_dir_sent(const Config* config, const File* file) {
|
||||||
if (file == NULL || !change_list_enabled(config))
|
if (file == NULL || !change_list_enabled(config))
|
||||||
return;
|
return;
|
||||||
ChangeEvent event;
|
ChangeEvent event;
|
||||||
memset(&event, 0, sizeof(event));
|
memset(&event, 0, sizeof(event));
|
||||||
event.path = file_wire_path(file);
|
|
||||||
event.decision = CHANGE_SENT;
|
event.decision = CHANGE_SENT;
|
||||||
event.is_directory = true;
|
event.is_directory = true;
|
||||||
event.size = 0;
|
event.size = 0;
|
||||||
event.bytes_sent = 0;
|
event.bytes_sent = 0;
|
||||||
if (file->metadata != NULL)
|
event.dest = file->dest_state;
|
||||||
event.mtime_sec = file->metadata->mtime_sec;
|
char* name = NULL;
|
||||||
|
char* path = NULL;
|
||||||
|
fill_event_from_file(config, file, &event, &name, &path);
|
||||||
|
if (name != NULL && path != NULL)
|
||||||
change_emit(config, &event);
|
change_emit(config, &event);
|
||||||
|
free(name);
|
||||||
|
free(path);
|
||||||
}
|
}
|
||||||
|
|||||||
+51
-24
@@ -2,7 +2,9 @@
|
|||||||
#define CHANGE_LIST_H
|
#define CHANGE_LIST_H
|
||||||
|
|
||||||
#include "config.h"
|
#include "config.h"
|
||||||
|
#include "checksum.h"
|
||||||
#include "file_types.h"
|
#include "file_types.h"
|
||||||
|
#include "format.h"
|
||||||
#include <stdbool.h>
|
#include <stdbool.h>
|
||||||
#include <sys/stat.h>
|
#include <sys/stat.h>
|
||||||
#include <time.h>
|
#include <time.h>
|
||||||
@@ -26,42 +28,59 @@ typedef enum {
|
|||||||
} ChangeDecision;
|
} ChangeDecision;
|
||||||
|
|
||||||
typedef struct {
|
typedef struct {
|
||||||
const char* path; /* full source path */
|
const char* path; /* long-form display path (rsync %f) */
|
||||||
|
const char* name; /* transfer-relative path (rsync %n), no trailing slash */
|
||||||
ChangeDecision decision;
|
ChangeDecision decision;
|
||||||
bool is_directory;
|
bool is_directory;
|
||||||
|
bool is_symlink;
|
||||||
|
bool is_special;
|
||||||
|
bool is_hardlink; /* a hard-link sibling (linked, no data sent) */
|
||||||
|
bool deleted; /* a would-delete report (-n --delete); no source file */
|
||||||
|
const char* symlink_target;
|
||||||
|
const char* hardlink_target;
|
||||||
unsigned long long size; /* source file length in bytes */
|
unsigned long long size; /* source file length in bytes */
|
||||||
/* The number of bytes reported for a sent file. FastSync has no wire-byte
|
unsigned long long bytes_sent; /* wire bytes actually transferred (rsync %b) */
|
||||||
* counter, so this is always the source length (== size / %l); actual
|
unsigned long long bytes_read; /* wire bytes read back for this file (rsync %c) */
|
||||||
* post-compression/delta bytes on the wire are not counted. */
|
/* rsync %C: whole-file checksum hex for a transferred regular file. Only
|
||||||
unsigned long long bytes_sent;
|
* filled when the active format uses %C (checksum_known == false otherwise,
|
||||||
time_t mtime_sec; /* 0 when unknown */
|
* which renders as spaces like rsync for non-regular entries). */
|
||||||
|
bool checksum_known;
|
||||||
|
char checksum[CHECKSUM_MAX_DIGEST_LEN * 2 + 1];
|
||||||
|
time_t mtime_sec;
|
||||||
|
long mtime_nsec;
|
||||||
|
mode_t mode;
|
||||||
|
uid_t uid;
|
||||||
|
gid_t gid;
|
||||||
|
/* Receiver-reported pre-transfer destination state (OutputDestState.known is
|
||||||
|
* false when no report was requested/received). */
|
||||||
|
OutputDestState dest;
|
||||||
} ChangeEvent;
|
} ChangeEvent;
|
||||||
|
|
||||||
/* True when any output mode is active and per-file events matter. */
|
/* True when any output mode is active and per-file events matter. */
|
||||||
bool change_list_enabled(const Config* config);
|
bool change_list_enabled(const Config* config);
|
||||||
|
|
||||||
/* Render the rsync-style itemize line for a transferred file:
|
/* Render the rsync-style itemize line for a transferred item
|
||||||
* `>f+++++++++ <path>`
|
* (`%i %n%L`): `>f+++++++++ sub/b.txt`. Caller frees the result. */
|
||||||
* The 11-char code is `>f` (regular file transferred to the remote host)
|
char* change_render_itemize(const Config* config, const ChangeEvent* event);
|
||||||
* followed by c/s/t/p/o/g/u/a/x markers that are all `+` (value will be set
|
|
||||||
* / differs) because FastSync does not separately compare checksums, size,
|
|
||||||
* mtime, perms, owner, group, uid, acl, or xattr on the receiving side, so a
|
|
||||||
* sent file is reported as fully updated. Up-to-date files print no line
|
|
||||||
* (rsync single `-i` only shows changes). Caller frees the result. */
|
|
||||||
char* change_render_itemize(const ChangeEvent* event);
|
|
||||||
|
|
||||||
/* Expand an --out-format/--log-file-format template. Tokens:
|
/* Expand an --out-format/--log-file-format template. Supported tokens:
|
||||||
* %f full source path %b "bytes sent" == the source length (%l);
|
* %i itemize code %n transfer-relative name (dir: trailing /)
|
||||||
* %n leaf (base) name actual post-compression/delta wire bytes
|
* %f long display path %l file length in bytes
|
||||||
* %l file length in bytes are not counted
|
* %b wire bytes transferred %c block-checksum bytes received (rsync: 16
|
||||||
* %M mtime in whole seconds %% a literal percent sign
|
* for a whole-file transfer, 0 for a dry run)
|
||||||
|
* %C whole-file checksum hex (xxh128 by default; spaces for non-regular)
|
||||||
|
* %M mtime (YYYY/MM/DD-HH:MM:SS)
|
||||||
|
* %t current time %o operation ("send"/"del.")
|
||||||
|
* %p pid %B permission bits without the type char
|
||||||
|
* %U uid %G gid
|
||||||
|
* %L " -> target" / " => target" %% a literal percent sign
|
||||||
* Unknown %X sequences are preserved verbatim. Caller frees the result. */
|
* Unknown %X sequences are preserved verbatim. Caller frees the result. */
|
||||||
char* change_render_format(const char* format, const ChangeEvent* event);
|
char* change_render_format(const char* format, const Config* config, const ChangeEvent* event);
|
||||||
|
|
||||||
/* Render one --list-only long-listing entry:
|
/* Render one --list-only long-listing entry:
|
||||||
* `-rw-r--r-- 12 2026/09/06 10:00:00 <path>`
|
* `-rw-r--r-- 12 2026/09/06 10:00:00 sub/b.txt`
|
||||||
* (ls -l style columns; mtime in the local time zone). Caller frees it. */
|
* (ls -l style columns; mtime in the local time zone). Caller frees it. */
|
||||||
char* change_render_list_line(mode_t mode, unsigned long long size, time_t mtime, const char* path);
|
char* change_render_list_line(const Config* config, const ChangeEvent* event);
|
||||||
|
|
||||||
/* Emit an event to every active destination:
|
/* Emit an event to every active destination:
|
||||||
* stdout: --itemize-changes line, or the --out-format expansion when set;
|
* stdout: --itemize-changes line, or the --out-format expansion when set;
|
||||||
@@ -69,7 +88,15 @@ char* change_render_list_line(mode_t mode, unsigned long long size, time_t mtime
|
|||||||
* CHANGE_UP_TO_DATE events produce no output. */
|
* CHANGE_UP_TO_DATE events produce no output. */
|
||||||
void change_emit(const Config* config, const ChangeEvent* event);
|
void change_emit(const Config* config, const ChangeEvent* event);
|
||||||
|
|
||||||
/* Build and emit a CHANGE_SENT event for a file the client just sent. */
|
/* Build and emit a CHANGE_SENT event for a file the client just sent. `bytes_sent`
|
||||||
|
* is the process-wide wire-byte delta for this file (rsync's %b) and `bytes_read`
|
||||||
|
* the received bytes used for the delta handshake; pass 0 when unknown. For a
|
||||||
|
* whole-file transfer %c is pinned to rsync's 16-byte sum header regardless. */
|
||||||
|
void change_emit_file_sent_bytes(const Config* config, const File* file,
|
||||||
|
unsigned long long bytes_sent, unsigned long long bytes_read);
|
||||||
|
|
||||||
|
/* Build and emit a CHANGE_SENT event for a file the client just sent, deriving
|
||||||
|
* the wire byte counts from the source payload length. */
|
||||||
void change_emit_file_sent(const Config* config, const File* file);
|
void change_emit_file_sent(const Config* config, const File* file);
|
||||||
|
|
||||||
/* Build and emit a CHANGE_SENT event for an explicit directory entry (-d). */
|
/* Build and emit a CHANGE_SENT event for an explicit directory entry (-d). */
|
||||||
|
|||||||
+947
-210
File diff suppressed because it is too large
Load Diff
+966
-227
File diff suppressed because it is too large
Load Diff
@@ -60,6 +60,16 @@ bool validate_config(const Config* config) {
|
|||||||
log_message(LOG_LEVEL_ERROR, "-f/--sendfile is not supported with SSH transport");
|
log_message(LOG_LEVEL_ERROR, "-f/--sendfile is not supported with SSH transport");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
/* -M/--remote-option appends an option to the REMOTE server's argv, which
|
||||||
|
* only exists on the SSH (user@host:path) transport. A daemon
|
||||||
|
* (host::module/path) or local TCP destination has no remote command line,
|
||||||
|
* so the option would be silently ignored; reject it by name instead. */
|
||||||
|
if (config->remote_option_count > 0 && config->transport != TRANSPORT_SSH) {
|
||||||
|
log_message(LOG_LEVEL_ERROR,
|
||||||
|
"-M/--remote-option is only valid with the SSH transport (user@host:path); it "
|
||||||
|
"cannot be used with a daemon (host::module/path) or local TCP destination");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
/* -4 and -6 are mutually exclusive: a socket address family cannot be both. */
|
/* -4 and -6 are mutually exclusive: a socket address family cannot be both. */
|
||||||
if (config->ipv4 && config->ipv6) {
|
if (config->ipv4 && config->ipv6) {
|
||||||
log_message(LOG_LEVEL_ERROR, "-4/--ipv4 and -6/--ipv6 are mutually exclusive");
|
log_message(LOG_LEVEL_ERROR, "-4/--ipv4 and -6/--ipv6 are mutually exclusive");
|
||||||
|
|||||||
+753
-176
File diff suppressed because it is too large
Load Diff
+53
-9
@@ -63,8 +63,23 @@ typedef struct {
|
|||||||
const FileListSet* file_list; /* --files-from allow-set, or NULL */
|
const FileListSet* file_list; /* --files-from allow-set, or NULL */
|
||||||
const FilterRuleList* base_filters; /* command-line + -C rules, or NULL */
|
const FilterRuleList* base_filters; /* command-line + -C rules, or NULL */
|
||||||
bool per_dir_filters; /* -F: read .rsync-filter per directory */
|
bool per_dir_filters; /* -F: read .rsync-filter per directory */
|
||||||
|
/* --delete-excluded: per-directory plain rules become sender-only, so they no
|
||||||
|
longer protect the receiver from deletion. */
|
||||||
|
bool delete_excluded;
|
||||||
|
/* -FF: also exclude the per-directory filter files themselves from the
|
||||||
|
transfer (single -F transfers them). */
|
||||||
|
bool exclude_per_dir_filter_files;
|
||||||
bool dirs; /* -d/--dirs: transfer dir entries, no recursion */
|
bool dirs; /* -d/--dirs: transfer dir entries, no recursion */
|
||||||
bool relative; /* -R/--relative (dest rel paths, with --files-from) */
|
bool relative; /* -R/--relative (dest rel paths, with --files-from) */
|
||||||
|
/* -R/--relative outside --files-from: the destination-relative path prefix
|
||||||
|
* reconstructed from the source spec (rsync's '/./' cut point), or NULL when
|
||||||
|
* -R is off or --files-from is in use (the bare-relative path then comes from
|
||||||
|
* the listed entry). Borrowed read-only; owned by client_send. */
|
||||||
|
const char* relative_prefix;
|
||||||
|
/* --list-only: emit an is_dir File for every traversed directory (the listing
|
||||||
|
* includes directory entries, matching rsync). Client-only; never set on a
|
||||||
|
* real transfer, which relies on implicit parent creation. */
|
||||||
|
bool list_dirs;
|
||||||
/* --prune-empty-dirs (long only): in --dirs mode an empty source directory's
|
/* --prune-empty-dirs (long only): in --dirs mode an empty source directory's
|
||||||
explicit entry is omitted from the transfer file list (so nothing is
|
explicit entry is omitted from the transfer file list (so nothing is
|
||||||
created at the destination and it can be pruned by --delete); explicitly
|
created at the destination and it can be pruned by --delete); explicitly
|
||||||
@@ -73,17 +88,39 @@ typedef struct {
|
|||||||
bool prune_empty_dirs;
|
bool prune_empty_dirs;
|
||||||
/* Delete-excluded protection sink (optional): when non-NULL the scanner
|
/* Delete-excluded protection sink (optional): when non-NULL the scanner
|
||||||
* appends the destination-relative path of every entry it prunes because a
|
* appends the destination-relative path of every entry it prunes because a
|
||||||
* USER SELECTION rule excluded it (--filter/-C/per-dir rules, the legacy
|
* USER SELECTION rule excluded it (--filter/-C/per-dir rules and the legacy
|
||||||
* --exclude/--include layer, and --max-size/--min-size). The sender turns
|
* --exclude/--include layer). The sender turns this list into the manifest's
|
||||||
* this list into the manifest's protected prefixes so `--delete` leaves the
|
* protected prefixes so `--delete` leaves the destination mirror of excluded
|
||||||
* destination mirror of excluded source paths alone (rsync's default), and
|
* source paths alone (rsync's default), and drops it when --delete-excluded
|
||||||
* empties it when --delete-excluded opts back into deleting them. NOT
|
* opts back into deleting them. NOT recorded for --files-from subset pruning
|
||||||
* recorded for --files-from subset pruning (whose delete semantics stay
|
* (whose delete semantics derive from the synchronized-directory set) or for
|
||||||
* keep-set-only) or for -R/--files-from relative wire paths. When
|
* -R/--files-from relative wire paths. When `excluded_mutex` is non-NULL it
|
||||||
* `excluded_mutex` is non-NULL it is taken around every append (the parallel
|
* is taken around every append (the parallel scanner shares one list across
|
||||||
* scanner shares one list across its worker threads). */
|
* its worker threads). */
|
||||||
ArrayList* excluded_paths;
|
ArrayList* excluded_paths;
|
||||||
mtx_t* excluded_mutex;
|
mtx_t* excluded_mutex;
|
||||||
|
/* Size-prune protection sink (optional): when non-NULL the scanner appends
|
||||||
|
* the destination-relative path of every entry it skipped because of
|
||||||
|
* --max-size/--min-size. rsync never deletes a size-skipped source mirror,
|
||||||
|
* even under --delete-excluded, so the sender always transmits this list as
|
||||||
|
* protected prefixes (unlike excluded_paths, which --delete-excluded drops).
|
||||||
|
* Guarded by `excluded_mutex` like excluded_paths. */
|
||||||
|
ArrayList* size_skipped_paths;
|
||||||
|
/* Synchronized-directory sink (optional): when non-NULL the scanner appends
|
||||||
|
* the destination-relative path of every directory it is about to traverse
|
||||||
|
* that lies inside a --files-from listed directory (or of every traversed
|
||||||
|
* directory when there is no list). The sender sends this set with the delete
|
||||||
|
* manifest so the receiver confines its extras walk to synchronized
|
||||||
|
* directories, exactly like rsync; the receive root is the "." sentinel.
|
||||||
|
* Guarded by `excluded_mutex`. */
|
||||||
|
ArrayList* synced_dirs;
|
||||||
|
/* Delete-plan directory sink (optional): when non-NULL the scanner appends
|
||||||
|
* the destination-relative path of every directory it traverses (except the
|
||||||
|
* receive root). The per-directory --delete-during/--delete-delay plan
|
||||||
|
* builder uses this to keep an empty in-scope source directory (rsync keeps
|
||||||
|
* it) and to emit its plan after the data stream, when no file frame would
|
||||||
|
* otherwise trigger it. Guarded by `excluded_mutex`. */
|
||||||
|
ArrayList* plan_dirs;
|
||||||
/* --ignore-errors: an unreadable directory during the scan is recorded as an
|
/* --ignore-errors: an unreadable directory during the scan is recorded as an
|
||||||
* I/O error and skipped instead of aborting the scan. Client-only. */
|
* I/O error and skipped instead of aborting the scan. Client-only. */
|
||||||
bool ignore_io_errors;
|
bool ignore_io_errors;
|
||||||
@@ -194,6 +231,13 @@ bool scanner_same_filesystem(bool one_file_system, dev_t root_device, dev_t entr
|
|||||||
* "/". Exposed so tests can exercise the mapping directly. */
|
* "/". Exposed so tests can exercise the mapping directly. */
|
||||||
char* scanner_path_relative(const char* root, const char* fs_path);
|
char* scanner_path_relative(const char* root, const char* fs_path);
|
||||||
|
|
||||||
|
/* -R/--relative destination-relative prefix reconstructed from a source spec:
|
||||||
|
* the path after rsync's first '.' path component (the '/./' cut point), with
|
||||||
|
* leading/trailing slashes removed, or the whole spec (normalized) when there
|
||||||
|
* is no cut. Returns "" for the receive root, or NULL when `spec` is NULL or
|
||||||
|
* allocation fails. Exposed so tests can exercise the mapping directly. */
|
||||||
|
char* scanner_relative_prefix(const char* spec);
|
||||||
|
|
||||||
ParallelScanner* parallel_scanner_create_with_options(const char* root_directory,
|
ParallelScanner* parallel_scanner_create_with_options(const char* root_directory,
|
||||||
const ScannerOptions* options,
|
const ScannerOptions* options,
|
||||||
ProtocolSession* allocation_session);
|
ProtocolSession* allocation_session);
|
||||||
|
|||||||
+129
-72
@@ -20,11 +20,16 @@ void print_usage(void) {
|
|||||||
printf("Options:\n");
|
printf("Options:\n");
|
||||||
printf(" -c, --checksum Verify content by checksum instead of size+mtime\n");
|
printf(" -c, --checksum Verify content by checksum instead of size+mtime\n");
|
||||||
printf(" -z, --compress [level] Enable compression (level 1-22, default 5)\n");
|
printf(" -z, --compress [level] Enable compression (level 1-22, default 5)\n");
|
||||||
printf(" -a, --archive rsync archive mode (-rlptgoD): links, metadata,\n");
|
printf(" -a, --archive rsync archive mode (-rlptgoD): links, perms, times,\n");
|
||||||
printf(" devices and specials (not compression/multithreading)\n");
|
printf(" owner, group, devices and specials; not\n");
|
||||||
|
printf(" compression/multithreading\n");
|
||||||
|
printf(" -r, --recursive Recurse into directories (FastSync is always recursive)\n");
|
||||||
printf(" -n, --dry-run Show what would be transferred\n");
|
printf(" -n, --dry-run Show what would be transferred\n");
|
||||||
printf(" --remove-source-files Remove regular source files after successful transfer\n");
|
printf(" --remove-source-files Remove regular source files after successful transfer\n");
|
||||||
printf(" -p, --perms Preserve permission bits (part of the metadata bundle)\n");
|
printf(" -p, --perms Preserve permission bits\n");
|
||||||
|
printf(" -t, --times Preserve modification times\n");
|
||||||
|
printf(" -o, --owner Preserve owner (uid)\n");
|
||||||
|
printf(" -g, --group Preserve group (gid)\n");
|
||||||
printf(" --ssh-port <port> SSH port (default: 22)\n");
|
printf(" --ssh-port <port> SSH port (default: 22)\n");
|
||||||
printf(" -e, --rsh <command> Remote shell to launch on the client for the SSH\n");
|
printf(" -e, --rsh <command> Remote shell to launch on the client for the SSH\n");
|
||||||
printf(" transport (default: ssh). The command may include\n");
|
printf(" transport (default: ssh). The command may include\n");
|
||||||
@@ -54,24 +59,26 @@ void print_usage(void) {
|
|||||||
printf(" Emit the batch file only (no destination, no server)\n");
|
printf(" Emit the batch file only (no destination, no server)\n");
|
||||||
printf(" --read-batch=FILE Apply the batch file to the destination (no source, no\n");
|
printf(" --read-batch=FILE Apply the batch file to the destination (no source, no\n");
|
||||||
printf(" server); takes only the destination as an argument\n");
|
printf(" server); takes only the destination as an argument\n");
|
||||||
|
printf(" NOTE: the FastSync batch format is NOT interoperable with rsync's batch\n");
|
||||||
|
printf(" files (different container format); do not mix the two tools.\n");
|
||||||
printf(" --delete Delete files on receiver not in source\n");
|
printf(" --delete Delete files on receiver not in source\n");
|
||||||
printf(" (default timing: delete only after the whole\n");
|
printf(" (default timing: delete only after the whole\n");
|
||||||
printf(" transfer has succeeded)\n");
|
printf(" transfer has succeeded)\n");
|
||||||
printf(" --delete-before Delete extras before the transfer starts\n");
|
printf(" --delete-before Delete extras before the transfer starts\n");
|
||||||
printf(" (implies --delete)\n");
|
printf(" (implies --delete)\n");
|
||||||
printf(" --delete-during Delete extras once the keep-set manifest is known,\n");
|
printf(" --delete-during Delete a directory's extras as that directory is\n");
|
||||||
printf(" before the data is applied (implies --delete)\n");
|
printf(" processed (implies --delete)\n");
|
||||||
printf(" --del Alias for --delete-during\n");
|
printf(" --del Alias for --delete-during\n");
|
||||||
printf(" --delete-delay Delete extras only after a successful transfer\n");
|
printf(" --delete-delay Record the extras during the scan but remove them\n");
|
||||||
printf(" (implies --delete)\n");
|
printf(" only after a successful transfer (implies --delete)\n");
|
||||||
printf(" --delete-after Delete only after the whole transfer succeeded\n");
|
printf(" --delete-after Delete only after the whole transfer succeeded\n");
|
||||||
printf(" (the default --delete timing; implies --delete)\n");
|
printf(" (the default --delete timing; implies --delete)\n");
|
||||||
printf(" --delete-excluded Also delete destination files that were excluded on\n");
|
printf(" --delete-excluded Also delete destination files that were excluded on\n");
|
||||||
printf(" the source (default protects them, matching rsync)\n");
|
printf(" the source (default protects them, matching rsync)\n");
|
||||||
printf(" --max-delete=NUM Never delete more than NUM destination entries per run;\n");
|
printf(" --max-delete=NUM Delete at most NUM destination entries per run; if the\n");
|
||||||
printf(" if the extras would exceed NUM, nothing is deleted and\n");
|
printf(" extras exceed NUM, the rest are skipped and the run is\n");
|
||||||
printf(" the run fails with a clear error (implies --delete only\n");
|
printf(" reported as partial (exit 25, matching rsync). Only\n");
|
||||||
printf(" when used with it)\n");
|
printf(" applies together with --delete\n");
|
||||||
printf(" --ignore-errors Continue (and still delete) when a source directory is\n");
|
printf(" --ignore-errors Continue (and still delete) when a source directory is\n");
|
||||||
printf(" unreadable during the scan, instead of aborting with no\n");
|
printf(" unreadable during the scan, instead of aborting with no\n");
|
||||||
printf(" deletion\n");
|
printf(" deletion\n");
|
||||||
@@ -100,20 +107,23 @@ void print_usage(void) {
|
|||||||
printf(" parent directory is not itself listed\n");
|
printf(" parent directory is not itself listed\n");
|
||||||
printf(" --mkpath Create the destination root directory on the server when it\n");
|
printf(" --mkpath Create the destination root directory on the server when it\n");
|
||||||
printf(" does not exist yet\n");
|
printf(" does not exist yet\n");
|
||||||
printf(" --exclude <pattern> Exclude files matching pattern\n");
|
printf(" --exclude <pattern>, --exclude=<pattern> Exclude files matching pattern\n");
|
||||||
printf(" --include <pattern> Only include files matching pattern\n");
|
printf(" --include <pattern>, --include=<pattern> Only include files matching pattern\n");
|
||||||
printf(" --exclude-from <file> Read exclude patterns from file\n");
|
printf(" --exclude-from <file>, --exclude-from=<file> Read exclude patterns from file\n");
|
||||||
printf(" --include-from <file> Read include patterns from file\n");
|
printf(" --include-from <file>, --include-from=<file> Read include patterns from file\n");
|
||||||
printf(" --files-from <file> Read the source file list from FILE (paths relative to the "
|
printf(" --files-from <file> Read the source file list from FILE (paths relative to the "
|
||||||
"source root)\n");
|
"source root)\n");
|
||||||
printf(" -0, --from0 Entries in --files-from are NUL-delimited\n");
|
printf(" -0, --from0 Entries in --files-from are NUL-delimited\n");
|
||||||
printf(" -f, --filter=RULE rsync-style filter rule (+/- include/exclude; repeatable;\n");
|
printf(" -f, --filter=RULE rsync-style filter rule: exclude/- include/+ hide/H show/S\n");
|
||||||
printf(" both --filter=RULE and the -f RULE / -f=RULE short forms work)\n");
|
printf(" protect/P risk/R merge/. dir-merge/: clear/! with modifiers\n");
|
||||||
|
printf(" (repeatable; --filter=RULE and -f RULE / -f=RULE both work)\n");
|
||||||
printf(" -C, --cvs-exclude Auto-ignore common CVS/SCM files (.git/, .svn/, *.o, *~, ...)\n");
|
printf(" -C, --cvs-exclude Auto-ignore common CVS/SCM files (.git/, .svn/, *.o, *~, ...)\n");
|
||||||
printf(" -F Apply per-directory .rsync-filter files during the scan\n");
|
printf(" -F Apply per-directory .rsync-filter files; repeated -FF also\n");
|
||||||
|
printf(" excludes the .rsync-filter files themselves\n");
|
||||||
printf(" --max-size <n> Skip files larger than n bytes\n");
|
printf(" --max-size <n> Skip files larger than n bytes\n");
|
||||||
printf(" --min-size <n> Skip files smaller than n bytes\n");
|
printf(" --min-size <n> Skip files smaller than n bytes\n");
|
||||||
printf(" --max-alloc <SIZE> Maximum single allocation (default: 1G)\n");
|
printf(" --max-alloc <SIZE> Maximum single allocation (default: 1G; 0 = no limit,\n");
|
||||||
|
printf(" matching rsync)\n");
|
||||||
printf(" --incremental Skip files unchanged since last transfer\n");
|
printf(" --incremental Skip files unchanged since last transfer\n");
|
||||||
printf(" --size-only Skip incremental files matching in size, ignoring mtime\n");
|
printf(" --size-only Skip incremental files matching in size, ignoring mtime\n");
|
||||||
printf(" -I, --ignore-times Transfer files even when size and mtime match\n");
|
printf(" -I, --ignore-times Transfer files even when size and mtime match\n");
|
||||||
@@ -128,20 +138,24 @@ void print_usage(void) {
|
|||||||
printf(" --link-dest <dir> Like --copy-dest, but hard-links the unchanged file from DIR\n");
|
printf(" --link-dest <dir> Like --copy-dest, but hard-links the unchanged file from DIR\n");
|
||||||
printf(" into the destination (repeatable; earlier DIRs win)\n");
|
printf(" into the destination (repeatable; earlier DIRs win)\n");
|
||||||
printf(" --checksum-choice, --cc <alg> Whole-file checksum algorithm for --incremental/\n");
|
printf(" --checksum-choice, --cc <alg> Whole-file checksum algorithm for --incremental/\n");
|
||||||
printf(" --checksum compares (xxh64/xxhash or md5; default xxh64 with\n");
|
printf(" --checksum compares. Accepted: xxh128 (default), xxh3, xxh64\n");
|
||||||
printf(" seed 0). The seed comes from --checksum-seed\n");
|
printf(" (aka xxhash), md5, md4, sha1, or none. A two-name\n");
|
||||||
printf(" --checksum-seed <num> Seed for the whole-file xxHash64 digest (and the delta\n");
|
printf(" 'transfer,pre-transfer' form is accepted like rsync; 'none' as\n");
|
||||||
printf(" block strong hash, low 32 bits); md5 ignores the seed. The\n");
|
printf(" the pre-transfer algorithm is rejected with --checksum\n");
|
||||||
printf(" digest algorithm and seed must match on sender and receiver\n");
|
printf(" --checksum-seed <num> Seed for the whole-file xxHash digest (and the delta\n");
|
||||||
|
printf(" block strong hash, low 32 bits); md5 ignores the seed. A seed\n");
|
||||||
|
printf(" of 0 (the default) is randomized per transfer, exactly like\n");
|
||||||
|
printf(" rsync, and the chosen seed is sent to the receiver\n");
|
||||||
printf(" --delta Delta transfer for changed files (requires --incremental)\n");
|
printf(" --delta Delta transfer for changed files (requires --incremental)\n");
|
||||||
printf(" -W, --whole-file Transfer changed files without delta processing\n");
|
printf(" -W, --whole-file Transfer changed files without delta processing\n");
|
||||||
|
printf(" --no-whole-file rsync spelling that clears -W/--whole-file\n");
|
||||||
printf(" -y, --fuzzy Use a similar-named file already in the destination\n");
|
printf(" -y, --fuzzy Use a similar-named file already in the destination\n");
|
||||||
printf(" directory as the delta basis when the destination has no\n");
|
printf(" directory as the delta basis when the destination has no\n");
|
||||||
printf(" usable file at the exact path (saves bandwidth; implies\n");
|
printf(" usable file at the exact path (saves bandwidth; implies\n");
|
||||||
printf(" --incremental and --delta; inert with --whole-file,\n");
|
printf(" --incremental and --delta; inert with --whole-file,\n");
|
||||||
printf(" --no-delta, or --no-incremental)\n");
|
printf(" --no-delta, or --no-incremental)\n");
|
||||||
printf(" --no-fuzzy Disable --fuzzy\n");
|
printf(" --no-fuzzy Disable --fuzzy\n");
|
||||||
printf(" --delta-block <n>, --block-size <n>\n");
|
printf(" -B <n>, --block-size <n>, --delta-block <n>\n");
|
||||||
printf(" Delta block size in bytes (default: %d)\n", DELTA_BLOCK_SIZE_DEFAULT);
|
printf(" Delta block size in bytes (default: %d)\n", DELTA_BLOCK_SIZE_DEFAULT);
|
||||||
printf(" --delta-max <n> Max file size for delta transfer (default: %llu)\n",
|
printf(" --delta-max <n> Max file size for delta transfer (default: %llu)\n",
|
||||||
DELTA_MAX_FILE_SIZE);
|
DELTA_MAX_FILE_SIZE);
|
||||||
@@ -152,16 +166,26 @@ void print_usage(void) {
|
|||||||
printf(" --chunk-serialization Enable chunk serialization (long form only)\n");
|
printf(" --chunk-serialization Enable chunk serialization (long form only)\n");
|
||||||
printf(" -s, --secluded-args Protect-args compatibility option (no effect; remote\n");
|
printf(" -s, --secluded-args Protect-args compatibility option (no effect; remote\n");
|
||||||
printf(" SSH argv is already built injection-safe)\n");
|
printf(" SSH argv is already built injection-safe)\n");
|
||||||
printf(" --sendfile Enable sendfile zero-copy (TCP only; long form only)\n");
|
printf(" --sendfile Enable sendfile zero-copy (TCP only; long form only;\n");
|
||||||
printf(" --compress-choice <alg> Compression algorithm (default: zstd)\n");
|
printf(" -f is bound to --filter, not --sendfile)\n");
|
||||||
|
printf(" --compress-choice <alg> Compression algorithm: zstd (default), lz4, zlib,\n");
|
||||||
|
printf(" zlibx, none, or auto\n");
|
||||||
printf(" --zc <alg> Alias for --compress-choice\n");
|
printf(" --zc <alg> Alias for --compress-choice\n");
|
||||||
printf(" -v, --verbose Enable debug logging\n");
|
printf(" -v, --verbose Enable debug logging\n");
|
||||||
printf(" -q, --quiet Suppress non-error output\n");
|
printf(" -q, --quiet Suppress non-error output\n");
|
||||||
printf(" --debug=FLAGS Fine-grained debug logging (use --debug=help for flags)\n");
|
printf(" --debug=FLAGS Fine-grained debug logging (use --debug=help for flags)\n");
|
||||||
printf(" --info=FLAGS Fine-grained info: copy,misc,skip,stats,all,none\n");
|
printf(" --info=FLAGS Fine-grained info: copy,name,misc,skip,stats,all,none\n");
|
||||||
printf(" none suppresses info even with --verbose\n");
|
printf(" (use --info=help for flags; none suppresses --verbose)\n");
|
||||||
printf(" --preserve Preserve file metadata (long form only)\n");
|
printf(" --preserve Preserve permissions and times (= -pt; long form only)\n");
|
||||||
|
printf(" --no-perms Negate -p/--perms\n");
|
||||||
|
printf(" --no-times Negate -t/--times\n");
|
||||||
|
printf(" --no-owner Negate -o/--owner\n");
|
||||||
|
printf(" --no-group Negate -g/--group\n");
|
||||||
|
printf(" --no-preserve Disable metadata preservation (negates --preserve)\n");
|
||||||
printf(" -E, --executability Preserve executable permission bits\n");
|
printf(" -E, --executability Preserve executable permission bits\n");
|
||||||
|
printf(" -U, --atimes Preserve access times\n");
|
||||||
|
printf(" -N, --crtimes Capture birth time; cannot be applied (documented\n");
|
||||||
|
printf(" divergence)\n");
|
||||||
printf(" -X, --xattrs Preserve user extended attributes (user.* only;\n");
|
printf(" -X, --xattrs Preserve user extended attributes (user.* only;\n");
|
||||||
printf(" privileged security.*/trusted.* namespaces are\n");
|
printf(" privileged security.*/trusted.* namespaces are\n");
|
||||||
printf(" never captured or applied)\n");
|
printf(" never captured or applied)\n");
|
||||||
@@ -176,28 +200,32 @@ void print_usage(void) {
|
|||||||
printf(" (char/block device-node creation, --write-devices)\n");
|
printf(" (char/block device-node creation, --write-devices)\n");
|
||||||
printf(" within the confined receive root. Never elevates\n");
|
printf(" within the confined receive root. Never elevates\n");
|
||||||
printf(" privileges and never bypasses confinement; ownership\n");
|
printf(" privileges and never bypasses confinement; ownership\n");
|
||||||
printf(" is still applied only with an explicit identity flag\n");
|
printf(" is still applied only with -o/--owner, -g/--group, or an\n");
|
||||||
printf(" (--numeric-ids/--chown/--usermap/--groupmap/--copy-as)\n");
|
printf(" explicit identity flag (--chown/--usermap/--groupmap/\n");
|
||||||
|
printf(" --copy-as); --numeric-ids only changes how ids map\n");
|
||||||
printf(" --no-super Forbid those super-user activities even when the\n");
|
printf(" --no-super Forbid those super-user activities even when the\n");
|
||||||
printf(" receiver is running as root\n");
|
printf(" receiver is running as root\n");
|
||||||
printf(" --chmod <changes> Modify transferred permissions (rsync syntax)\n");
|
printf(
|
||||||
printf(" --numeric-ids Do not map uid/gid by name: use the source numeric\n");
|
" --chmod <changes> Modify new/transferred permissions (rsync syntax; implies no -p)\n");
|
||||||
printf(" ids directly when applying ownership\n");
|
printf(" --numeric-ids Map uid/gid by id instead of by name (a modifier, not\n");
|
||||||
|
printf(" an ownership request: combine with -o/-g or a map)\n");
|
||||||
printf(" --usermap=MAP Map usernames when applying ownership: comma-separated\n");
|
printf(" --usermap=MAP Map usernames when applying ownership: comma-separated\n");
|
||||||
printf(" FROM:TO rules, first match wins. FROM/TO are names\n");
|
printf(" FROM:TO rules, first match wins. FROM is a name (from\n");
|
||||||
printf(" (resolved on the source machine), * (match any /\n");
|
printf(" the source), an id, an inclusive LOW-HIGH range, *\n");
|
||||||
printf(" current user), or @N numeric ids. e.g. *:nobody\n");
|
printf(" (any id), or empty (ids with no name). TO is an id, *\n");
|
||||||
|
printf(" (current user), or a name resolved on the receiver.\n");
|
||||||
|
printf(" e.g. 0-99:nobody,*:normal (cannot mix with --chown)\n");
|
||||||
printf(" --groupmap=MAP Map group names when applying ownership (same syntax)\n");
|
printf(" --groupmap=MAP Map group names when applying ownership (same syntax)\n");
|
||||||
printf(" --chown=USER:GROUP Override the ownership of transferred files. Forms:\n");
|
printf(" --chown=USER:GROUP Override the ownership of transferred files. Forms:\n");
|
||||||
printf(" USER:GROUP, USER (owner only), :GROUP (group only); a\n");
|
printf(" USER:GROUP, USER (owner only), :GROUP (group only); a\n");
|
||||||
printf(" value of * means the current/root user as appropriate.\n");
|
printf(" value of * means the current/root user as appropriate.\n");
|
||||||
printf(" Names resolve on the source machine; @N for numerics.\n");
|
printf(" Names resolve on the source machine; @N for numerics.\n");
|
||||||
printf(" (Metadata is enabled with --preserve; -M now means\n");
|
printf(" (Implies owner/group metadata; -M now means rsync's\n");
|
||||||
printf(" rsync's --remote-option.)\n");
|
printf(" --remote-option.)\n");
|
||||||
printf(" --copy-as=USER[:GROUP] Force every written entry (files, dirs, symlinks\n");
|
printf(" --copy-as=USER[:GROUP] Force every written entry (files, dirs, symlinks\n");
|
||||||
printf(" and special nodes) to USER[:GROUP], resolved on the\n");
|
printf(" and special nodes) to USER[:GROUP], resolved on the\n");
|
||||||
printf(" source machine like --chown. Requires a privileged\n");
|
printf(" source machine like --chown. Requires a privileged\n");
|
||||||
printf(" (root) receiver and implies --preserve; an\n");
|
printf(" (root) receiver and implies owner/group metadata; an\n");
|
||||||
printf(" unprivileged receiver refuses the transfer. Never\n");
|
printf(" unprivileged receiver refuses the transfer. Never\n");
|
||||||
printf(" switches process credentials (safe-subset; see\n");
|
printf(" switches process credentials (safe-subset; see\n");
|
||||||
printf(" RSYNC_COMPAT.md). A daemon refuses it.\n");
|
printf(" RSYNC_COMPAT.md). A daemon refuses it.\n");
|
||||||
@@ -209,9 +237,10 @@ void print_usage(void) {
|
|||||||
printf(" --server-port <n> Server port (default: 8080)\n");
|
printf(" --server-port <n> Server port (default: 8080)\n");
|
||||||
printf(" --port <n> Alias for --server-port\n");
|
printf(" --port <n> Alias for --server-port\n");
|
||||||
printf(" --password-file <f> Authenticate a host::module/path daemon destination.\n");
|
printf(" --password-file <f> Authenticate a host::module/path daemon destination.\n");
|
||||||
printf(" The file's first user:password line supplies the\n");
|
printf(" FastSync-native SCRAM/PBKDF2 credential scheme (NOT\n");
|
||||||
printf(" username and password (only a SHA-256 digest of the\n");
|
printf(" rsync's --password-file): the file's first user:password\n");
|
||||||
printf(" password is sent; keep the file mode 0600)\n");
|
printf(" line supplies the username and password; no password or\n");
|
||||||
|
printf(" reusable digest is sent (keep the file mode 0600)\n");
|
||||||
printf(" --no-motd Suppress display of the daemon's MOTD (the server\n");
|
printf(" --no-motd Suppress display of the daemon's MOTD (the server\n");
|
||||||
printf(" still sends it; the client just does not show it)\n");
|
printf(" still sends it; the client just does not show it)\n");
|
||||||
printf(" --bwlimit <KB/s> Bandwidth limit in kilobytes per second\n");
|
printf(" --bwlimit <KB/s> Bandwidth limit in kilobytes per second\n");
|
||||||
@@ -219,55 +248,69 @@ void print_usage(void) {
|
|||||||
printf(" --cert <path> TLS certificate file (PEM)\n");
|
printf(" --cert <path> TLS certificate file (PEM)\n");
|
||||||
printf(" --key <path> TLS private key file (PEM)\n");
|
printf(" --key <path> TLS private key file (PEM)\n");
|
||||||
printf(" --ca <path> TLS CA certificate file (PEM)\n");
|
printf(" --ca <path> TLS CA certificate file (PEM)\n");
|
||||||
printf(" --timeout <sec> I/O timeout in seconds (default: 30; long form only)\n");
|
printf(" --timeout <sec> I/O timeout in seconds (default: 0 = disabled, matching\n");
|
||||||
printf(" --contimeout <sec> Connection timeout in seconds (default: 10)\n");
|
printf(" rsync). 0 disables it; --no-timeout is the same\n");
|
||||||
|
printf(" --contimeout <sec> Connection timeout in seconds (default: 60, matching\n");
|
||||||
|
printf(" rsync); 0 disables it (--no-contimeout)\n");
|
||||||
printf(" --stop-after=MINS Stop the transfer after MINS minutes (a positive\n");
|
printf(" --stop-after=MINS Stop the transfer after MINS minutes (a positive\n");
|
||||||
printf(" integer); whatever was already transferred is kept\n");
|
printf(" integer); whatever was already transferred is kept\n");
|
||||||
printf(" --stop-at=TIME Stop at an absolute time: HH:MM, HH:MM:SS, or\n");
|
printf(" --stop-at=TIME Stop at an absolute time. Accepts rsync's date form\n");
|
||||||
printf(" now+N[smhd] (a time already in the past stops the\n");
|
printf(" (Y-M-DTh:m, Y/M/DTh:m, abbreviable fields such as 12-31,\n");
|
||||||
printf(" transfer immediately; client-only). An early stop\n");
|
printf(" 14:00, :59, 1) plus FastSync's HH:MM[:SS] and now+N[smhd]\n");
|
||||||
printf(" skips the late --delete keep-set so it cannot delete\n");
|
printf(" (a time already in the past stops the transfer\n");
|
||||||
printf(" source mirrors that were not yet scanned\n");
|
printf(" immediately; client-only). An early stop skips the late\n");
|
||||||
|
printf(" --delete keep-set so it cannot delete source mirrors that\n");
|
||||||
|
printf(" were not yet scanned\n");
|
||||||
printf(" --address <ip> Bind the outgoing client socket to this source address\n");
|
printf(" --address <ip> Bind the outgoing client socket to this source address\n");
|
||||||
printf(" -4, --ipv4 Force IPv4 for destination resolution\n");
|
printf(" -4, --ipv4 Force IPv4 for destination resolution\n");
|
||||||
printf(" -6, --ipv6 Force IPv6 for destination resolution\n");
|
printf(" -6, --ipv6 Force IPv6 for destination resolution\n");
|
||||||
printf(" --sockopts=OPTS Comma-separated OPT=VAL socket options applied before connect:\n");
|
printf(" --sockopts=OPTS Comma-separated OPT=VAL socket options applied before connect:\n");
|
||||||
printf(" TCP_NODELAY, SO_KEEPALIVE, SO_RCVBUF, SO_SNDBUF, SO_REUSEADDR\n");
|
printf(" TCP_NODELAY, SO_KEEPALIVE, SO_RCVBUF, SO_SNDBUF, SO_REUSEADDR\n");
|
||||||
printf(" --backup Backup existing files before overwriting\n");
|
printf(" -b, --backup Backup existing files before overwriting\n");
|
||||||
printf(" --backup-dir <dir> Directory for backups (requires --backup)\n");
|
printf(" --backup-dir <dir> Directory for backups (requires --backup)\n");
|
||||||
printf(" --suffix <str> Backup suffix (default: ~)\n");
|
printf(" --suffix <str> Backup suffix (default: ~)\n");
|
||||||
printf(" --stats Print transfer statistics at end\n");
|
printf(" --stats Print transfer statistics at end\n");
|
||||||
printf(" -i, --itemize-changes Print an rsync-style per-file change line\n");
|
printf(" -i, --itemize-changes Print an rsync-style per-file change line\n");
|
||||||
printf(" --out-format=FORMAT Output format for changed files (%%f %%n %%l %%b %%M %%%%)\n");
|
printf(" --out-format=FORMAT Output format (%%f %%n %%l %%b %%c %%C %%i %%M %%%%)\n");
|
||||||
printf(" --list-only List source files instead of transferring\n");
|
printf(" --list-only List source files instead of transferring\n");
|
||||||
printf(" --log-file-format=FORMAT Per-file log line format (needs --log-file)\n");
|
printf(" --log-file-format=FORMAT Per-file log line format (needs --log-file)\n");
|
||||||
printf(" -h, --human-readable Print byte sizes in human-readable form\n");
|
printf(" -h, --human-readable Print byte sizes in human-readable form\n");
|
||||||
printf(" --max-depth <n> Maximum directory depth (0=unlimited)\n");
|
printf(" --max-depth <n> Maximum directory depth (0=unlimited)\n");
|
||||||
printf(" -x, --one-file-system Do not cross filesystem boundaries\n");
|
printf(" -x, --one-file-system Do not cross filesystem boundaries\n");
|
||||||
printf(" --log-file <path> Write log messages to file\n");
|
printf(" --log-file <path>, --log-file=<path> Write log messages to file\n");
|
||||||
printf(" --stderr=MODE Route logging to stderr: errors or all\n");
|
printf(" --stderr=MODE Route logging to stderr: errors or all\n");
|
||||||
printf(" --partial Keep partial files on interrupted transfer\n");
|
printf(" --partial Keep partial files on interrupted transfer\n");
|
||||||
printf(" --partial-dir <dir> Directory for partial files\n");
|
printf(" --partial-dir <dir> Directory for partial files\n");
|
||||||
printf(" -T, --temp-dir <dir> Scratch dir for temp files before atomic install\n");
|
printf(" -T, --temp-dir <dir> Scratch dir for temp files before atomic install.\n");
|
||||||
|
printf(" Confined to the receive root: a relative dir resolves below\n");
|
||||||
|
printf(" it and an absolute/traversal dir is rejected. The dir must\n");
|
||||||
|
printf(" already exist; a different filesystem falls back to a\n");
|
||||||
|
printf(" non-atomic copy instead of aborting\n");
|
||||||
printf(" --fastsync-server-path <path>\n");
|
printf(" --fastsync-server-path <path>\n");
|
||||||
printf(" Path to fastsync-server on remote (default: fastsync-server)\n");
|
printf(" Path to fastsync-server on remote (default: fastsync-server)\n");
|
||||||
printf(" --old-args Accepted for rsync CLI compatibility; no effect (the\n");
|
printf(" --old-args Accepted for rsync CLI compatibility; no effect (the\n");
|
||||||
printf(" remote server path is always safely quoted now)\n");
|
printf(" remote server path is always safely quoted now)\n");
|
||||||
printf(" -M, --remote-option=OPT Append OPT to the REMOTE server invocation over SSH\n");
|
printf(" -M, --remote-option=OPT Append OPT to the REMOTE server invocation. SSH\n");
|
||||||
printf(" (repeatable; each value is single-quote-escaped on the remote\n");
|
printf(" transport ONLY (user@host:path): a daemon (host::module) or\n");
|
||||||
printf(" command line; empty values and values with control characters\n");
|
printf(" local TCP destination rejects it (no remote command line to\n");
|
||||||
printf(" are rejected; -M OPT, -M=OPT and --remote-option=OPT work)\n");
|
printf(" append to). Repeatable; each value is single-quote-escaped on\n");
|
||||||
printf(" --trust-sender Trust the remote sender's file list: the receiver skips its\n");
|
printf(" the remote command line; empty values and values with control\n");
|
||||||
printf(" own up-front path-traversal/containment re-validation of the\n");
|
printf(" characters are rejected; -M OPT, -M=OPT and\n");
|
||||||
printf(" incoming file list (fewer checks, faster, potentially unsafe).\n");
|
printf(" --remote-option=OPT work\n");
|
||||||
printf(" Local receiver policy: never sent to the peer, off by default\n");
|
printf(" --trust-sender RECEIVER-LOCAL policy: trust the remote sender's file list\n");
|
||||||
|
printf(" and skip the receiver's own up-front path-traversal/\n");
|
||||||
|
printf(" containment re-validation of the incoming list (fewer checks,\n");
|
||||||
|
printf(" faster, potentially unsafe). It is never sent to the peer, so\n");
|
||||||
|
printf(" for a push it must be enabled on the receiving SERVER\n");
|
||||||
|
printf(" (fastsync-server --trust-sender) or forwarded with\n");
|
||||||
|
printf(" -M--trust-sender; the client flag alone has no effect\n");
|
||||||
printf(" -l, --links Copy symlinks as symlinks\n");
|
printf(" -l, --links Copy symlinks as symlinks\n");
|
||||||
printf(" --copy-links Transform symlinks into referent files\n");
|
printf(" -L, --copy-links Transform symlinks into referent files\n");
|
||||||
printf(" --safe-links Skip symlinks that point outside transfer tree\n");
|
printf(" --safe-links Skip symlinks whose target points outside the tree\n");
|
||||||
printf(" --copy-unsafe-links Only transform unsafe symlinks into referent files\n");
|
printf(" --copy-unsafe-links Copy unsafe symlinks (outside tree) as referent files\n");
|
||||||
printf(" -k, --copy-dirlinks Transform symlinks to directories into real dirs\n");
|
printf(" -k, --copy-dirlinks Transform symlinks to directories into real dirs\n");
|
||||||
printf(" -K, --keep-dirlinks Keep an existing symlink-to-dir as that dir\n");
|
printf(" -K, --keep-dirlinks Keep an existing symlink-to-dir as that dir\n");
|
||||||
printf(" --munge-links Munge symlink targets on the wire (sender)\n");
|
printf(" --munge-links Munge stored symlink targets (/rsyncd-munged/) on the receiver\n");
|
||||||
printf(" -H, --hard-links Preserve hard-link relationships across the transfer\n");
|
printf(" -H, --hard-links Preserve hard-link relationships across the transfer\n");
|
||||||
printf(" -S, --sparse Handle sparse files efficiently\n");
|
printf(" -S, --sparse Handle sparse files efficiently\n");
|
||||||
printf(
|
printf(
|
||||||
@@ -275,8 +318,7 @@ void print_usage(void) {
|
|||||||
printf(
|
printf(
|
||||||
" --devices Recreate device nodes on the destination (privileged; skipped when\n");
|
" --devices Recreate device nodes on the destination (privileged; skipped when\n");
|
||||||
printf(" the receiver lacks CAP_MKNOD)\n");
|
printf(" the receiver lacks CAP_MKNOD)\n");
|
||||||
printf(" --specials Recreate special files (FIFOs) on the destination (sockets "
|
printf(" --specials Recreate special files (FIFOs, sockets) on the destination\n");
|
||||||
"skipped)\n");
|
|
||||||
printf(" --copy-devices Copy a source device's content as a regular file instead\n");
|
printf(" --copy-devices Copy a source device's content as a regular file instead\n");
|
||||||
printf(" --write-devices Write received data into an existing destination device node\n");
|
printf(" --write-devices Write received data into an existing destination device node\n");
|
||||||
printf(" --inplace Update files in-place (no temp+rename)\n");
|
printf(" --inplace Update files in-place (no temp+rename)\n");
|
||||||
@@ -289,7 +331,9 @@ void print_usage(void) {
|
|||||||
printf(" --fsync Fsync every written file before publication\n");
|
printf(" --fsync Fsync every written file before publication\n");
|
||||||
printf(" --compress-level <n> Compression level (default: 5)\n");
|
printf(" --compress-level <n> Compression level (default: 5)\n");
|
||||||
printf(" --zl <n> Alias for --compress-level\n");
|
printf(" --zl <n> Alias for --compress-level\n");
|
||||||
printf(" --skip-compress=LIST Skip compression for comma-separated suffixes\n");
|
printf(" --skip-compress=LIST Skip compression for suffixes in LIST (separated by\n");
|
||||||
|
printf(" '/' as in rsync, or ','); a leading dot is optional. The\n");
|
||||||
|
printf(" default is rsync 3.4.1's built-in skip-compress list\n");
|
||||||
printf(" --compress-threads <n> Compression worker threads (requires zstd threaded support)\n");
|
printf(" --compress-threads <n> Compression worker threads (requires zstd threaded support)\n");
|
||||||
printf(" --no-OPTION Disable a supported boolean option\n");
|
printf(" --no-OPTION Disable a supported boolean option\n");
|
||||||
printf(" --help Show this help\n");
|
printf(" --help Show this help\n");
|
||||||
@@ -297,7 +341,20 @@ void print_usage(void) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
void print_debug_usage(void) {
|
void print_debug_usage(void) {
|
||||||
printf("Supported debug flags: IO,PROTO,PACK,UTIL,ALL,NONE\n");
|
printf("Emitting debug flags: IO,PROTO,PACK,UTIL,ALL,NONE\n");
|
||||||
|
printf("Also accepted for rsync CLI parity (silent): ACL,BACKUP,BIND,CHDIR,\n");
|
||||||
|
printf("CONNECT,CMD,DEL,DELTASUM,DUP,EXIT,FILTER,FLIST,FUZZY,GENR,HASH,HLINK,\n");
|
||||||
|
printf("ICONV,NSTR,OWN,RECV,SEND,TIME.\n");
|
||||||
printf("Flags may be comma-separated, for example: --debug=io,proto\n");
|
printf("Flags may be comma-separated, for example: --debug=io,proto\n");
|
||||||
printf("Other rsync debug flags are unsupported and rejected.\n");
|
printf("An optional level suffix is accepted (e.g. --debug=io2); level 0\n");
|
||||||
|
printf("silences that item. Unknown names are rejected.\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
void print_info_usage(void) {
|
||||||
|
printf("Emitting info flags: COPY,NAME,MISC,SKIP,STATS,ALL,NONE\n");
|
||||||
|
printf("Also accepted for rsync CLI parity (silent): BACKUP,DEL,FLIST,MOUNT,\n");
|
||||||
|
printf("NONREG,PROGRESS,REMOVE,SYMSAFE.\n");
|
||||||
|
printf("Flags may be comma-separated, for example: --info=name,stats\n");
|
||||||
|
printf("An optional level suffix is accepted (e.g. --info=stats2); level 0\n");
|
||||||
|
printf("silences that item. Unknown names are rejected.\n");
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,5 +3,6 @@
|
|||||||
|
|
||||||
void print_usage(void);
|
void print_usage(void);
|
||||||
void print_debug_usage(void);
|
void print_debug_usage(void);
|
||||||
|
void print_info_usage(void);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
+172
-32
@@ -3,6 +3,7 @@
|
|||||||
#include "charset.h"
|
#include "charset.h"
|
||||||
#include "chunk.h"
|
#include "chunk.h"
|
||||||
#include "config.h"
|
#include "config.h"
|
||||||
|
#include "delete_plan.h"
|
||||||
#include "delay_updates.h"
|
#include "delay_updates.h"
|
||||||
#include "file.h"
|
#include "file.h"
|
||||||
#include "file_receive.h"
|
#include "file_receive.h"
|
||||||
@@ -11,6 +12,7 @@
|
|||||||
#include "protocol.h"
|
#include "protocol.h"
|
||||||
#include "utils.h"
|
#include "utils.h"
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
#include <sys/stat.h>
|
#include <sys/stat.h>
|
||||||
#include <time.h>
|
#include <time.h>
|
||||||
|
|
||||||
@@ -43,17 +45,49 @@ void receiver_outcomes_destroy(ReceiverOutcomes* outcomes) {
|
|||||||
/* End-of-transfer success frame. When --remove-source-files was negotiated
|
/* End-of-transfer success frame. When --remove-source-files was negotiated
|
||||||
each processed data file is acknowledged first (STATUS_NEXT = written,
|
each processed data file is acknowledged first (STATUS_NEXT = written,
|
||||||
STATUS_OK = skipped) so the sender never removes a source the receiver did
|
STATUS_OK = skipped) so the sender never removes a source the receiver did
|
||||||
not actually store. The frame always ends with a plain STATUS_OK. */
|
not actually store. The frame ends with `final_status` (STATUS_OK, or
|
||||||
bool receiver_send_final_success(int fd, const Config* config, const ReceiverOutcomes* outcomes) {
|
STATUS_DELETE_LIMIT when a --max-delete commit was capped). */
|
||||||
|
bool receiver_send_final_success(int fd, const Config* config, const ReceiverOutcomes* outcomes,
|
||||||
|
Status final_status) {
|
||||||
if (!config->remove_source_files)
|
if (!config->remove_source_files)
|
||||||
return send_status(fd, STATUS_OK);
|
return send_status(fd, final_status);
|
||||||
size_t count = outcomes ? outcomes->count : 0;
|
size_t count = outcomes ? outcomes->count : 0;
|
||||||
for (size_t i = 0; i < count; i++) {
|
for (size_t i = 0; i < count; i++) {
|
||||||
Status per_file = outcomes->entries[i] == FILE_SAVE_WRITTEN ? STATUS_NEXT : STATUS_OK;
|
Status per_file = outcomes->entries[i] == FILE_SAVE_WRITTEN ? STATUS_NEXT : STATUS_OK;
|
||||||
if (!send_status(fd, per_file))
|
if (!send_status(fd, per_file))
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
return send_status(fd, STATUS_OK);
|
return send_status(fd, final_status);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool receiver_send_stats_frame(int fd, const Config* config, const ReceiverStats* stats,
|
||||||
|
const struct ArrayList* would_delete) {
|
||||||
|
if (!config->report_stats)
|
||||||
|
return true;
|
||||||
|
ReceiverStats local;
|
||||||
|
memset(&local, 0, sizeof(local));
|
||||||
|
const ReceiverStats* out = stats ? stats : &local;
|
||||||
|
size_t count = would_delete ? (size_t)would_delete->size : 0;
|
||||||
|
if (count > (size_t)MAX_MANIFEST_ENTRIES)
|
||||||
|
count = MAX_MANIFEST_ENTRIES;
|
||||||
|
ReceiverStats record = *out;
|
||||||
|
record.would_delete_count = count;
|
||||||
|
if (!send_status(fd, STATUS_STATS) || !format_stats_send(fd, &record) ||
|
||||||
|
!send_int(fd, (int)count))
|
||||||
|
return false;
|
||||||
|
for (size_t i = 0; i < count; i++) {
|
||||||
|
const char* path = (const char*)would_delete->items[i];
|
||||||
|
if (!send_wire_str(fd, path ? path : ""))
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Add a delete commit's tally to the sink's end-of-transfer wire counters (when
|
||||||
|
the sink reports them). Runs on the receiving thread, so no locking. */
|
||||||
|
static void receiver_tally_deleted(const ReceiverSink* sink, size_t deleted) {
|
||||||
|
if (sink && sink->stats && deleted > 0)
|
||||||
|
sink->stats->deleted_files += deleted;
|
||||||
}
|
}
|
||||||
|
|
||||||
static bool receiver_process_chunk(Chunk* chunk, const ReceiverSink* sink) {
|
static bool receiver_process_chunk(Chunk* chunk, const ReceiverSink* sink) {
|
||||||
@@ -242,7 +276,7 @@ static bool receiver_note_status(const struct timespec* session_start,
|
|||||||
}
|
}
|
||||||
|
|
||||||
int receiver_process(Config* config, int file_descriptor, const ReceiverSink* sink) {
|
int receiver_process(Config* config, int file_descriptor, const ReceiverSink* sink) {
|
||||||
return receiver_process_pending(config, file_descriptor, sink, NULL);
|
return receiver_process_pending(config, file_descriptor, sink, NULL, NULL);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Runs the whole receive loop. The delete manifest may legitimately arrive
|
/* Runs the whole receive loop. The delete manifest may legitimately arrive
|
||||||
@@ -256,7 +290,7 @@ int receiver_process(Config* config, int file_descriptor, const ReceiverSink* si
|
|||||||
the whole transfer succeeded. See receiver_process_pending() for how the -m
|
the whole transfer succeeded. See receiver_process_pending() for how the -m
|
||||||
receiver defers that commit until its disk writer has drained. */
|
receiver defers that commit until its disk writer has drained. */
|
||||||
int receiver_process_pending(Config* config, int file_descriptor, const ReceiverSink* sink,
|
int receiver_process_pending(Config* config, int file_descriptor, const ReceiverSink* sink,
|
||||||
DeleteManifest** pending_manifest) {
|
DeleteManifest** pending_manifest, DeletePlanSession** pending_plans) {
|
||||||
Status status;
|
Status status;
|
||||||
if (!receive_status(file_descriptor, &status))
|
if (!receive_status(file_descriptor, &status))
|
||||||
return -1;
|
return -1;
|
||||||
@@ -270,14 +304,22 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver
|
|||||||
if (!receiver_note_status(&session_start, &last_progress, status, file_descriptor, sink))
|
if (!receiver_note_status(&session_start, &last_progress, status, file_descriptor, sink))
|
||||||
return -1;
|
return -1;
|
||||||
bool early_delete = config_delete_timing_early(config);
|
bool early_delete = config_delete_timing_early(config);
|
||||||
|
bool per_dir_delete = config_delete_timing_per_dir(config);
|
||||||
/* Parked keep-set for the late/commit timing. Every exit path below frees it
|
/* Parked keep-set for the late/commit timing. Every exit path below frees it
|
||||||
exactly once; the only exception is the successful FINISHED handoff, which
|
exactly once; the only exception is the successful FINISHED handoff, which
|
||||||
transfers ownership to *pending_manifest (used by the -m receiver). */
|
transfers ownership to *pending_manifest (used by the -m receiver). */
|
||||||
DeleteManifest* deferred_manifest = NULL;
|
DeleteManifest* deferred_manifest = NULL;
|
||||||
|
/* Per-directory delete session for --delete-during/--delete-delay. During the
|
||||||
|
loop it applies plans inline (during) or snapshots their extras (delay); on
|
||||||
|
a successful FINISHED it is either committed here or handed to
|
||||||
|
*pending_plans so the -m caller commits after its disk writer drained. */
|
||||||
|
DeletePlanSession* plan_session = NULL;
|
||||||
|
bool delete_limit_noted = false;
|
||||||
while (status == STATUS_NEXT || status == STATUS_CHUNK || status == STATUS_CHECK ||
|
while (status == STATUS_NEXT || status == STATUS_CHUNK || status == STATUS_CHECK ||
|
||||||
status == STATUS_KEEPALIVE || status == STATUS_ABORT || status == STATUS_CHECK_BATCH ||
|
status == STATUS_KEEPALIVE || status == STATUS_ABORT || status == STATUS_CHECK_BATCH ||
|
||||||
status == STATUS_MKDIR || status == STATUS_MANIFEST || status == STATUS_HARDLINK ||
|
status == STATUS_MKDIR || status == STATUS_MANIFEST || status == STATUS_HARDLINK ||
|
||||||
status == STATUS_SYMLINK || status == STATUS_SPECIAL || status == STATUS_DIR_TIMES) {
|
status == STATUS_SYMLINK || status == STATUS_SPECIAL || status == STATUS_DIR_TIMES ||
|
||||||
|
status == STATUS_DELETE_PLAN) {
|
||||||
if (status == STATUS_KEEPALIVE) {
|
if (status == STATUS_KEEPALIVE) {
|
||||||
if (!send_status(file_descriptor, STATUS_KEEPALIVE))
|
if (!send_status(file_descriptor, STATUS_KEEPALIVE))
|
||||||
goto fail;
|
goto fail;
|
||||||
@@ -335,32 +377,44 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver
|
|||||||
if (config->dry_run) {
|
if (config->dry_run) {
|
||||||
/* Server-contacting --dry-run mutates nothing, so a keep-set manifest
|
/* Server-contacting --dry-run mutates nothing, so a keep-set manifest
|
||||||
is consumed and discarded. The early-delete mode still needs its ACK
|
is consumed and discarded. The early-delete mode still needs its ACK
|
||||||
so a sender blocked on the delete handshake is not left hanging. */
|
so a sender blocked on the delete handshake is not left hanging.
|
||||||
|
When would-delete reporting is armed, enumerate (read-only) the
|
||||||
|
destination extras so the terminal STATUS_STATS frame can list them. */
|
||||||
|
if (config->use_delete && sink->would_delete) {
|
||||||
|
size_t count = 0;
|
||||||
|
if (!manifest_would_delete_list(config, manifest, sink->would_delete, &count))
|
||||||
|
log_message(LOG_LEVEL_WARNING, "dry-run: could not enumerate would-delete paths");
|
||||||
|
}
|
||||||
delete_manifest_free(manifest);
|
delete_manifest_free(manifest);
|
||||||
if (early_delete && !send_status(file_descriptor, STATUS_OK))
|
if (early_delete && !send_status(file_descriptor, STATUS_OK))
|
||||||
goto fail;
|
goto fail;
|
||||||
goto next_status;
|
goto next_status;
|
||||||
}
|
}
|
||||||
if (early_delete) {
|
if (early_delete) {
|
||||||
/* --delete-before / --delete-during: the manifest is authoritative the
|
/* --delete-before: the whole-tree manifest is authoritative the moment
|
||||||
moment it arrives, before any file data. Delete now and acknowledge
|
it arrives, before any file data. Delete now and acknowledge so the
|
||||||
so the sender only starts streaming once the deletion committed (or
|
sender only starts streaming once the deletion committed (or failed).
|
||||||
failed). This is the rsync delete-before/delete-during window: a
|
A later transfer failure does not restore these deletions. A
|
||||||
later transfer failure does not restore these deletions. */
|
--max-delete-capped commit still succeeds and the transfer proceeds;
|
||||||
bool deletion_ok = (config->use_delete || config->delete_missing_args)
|
the terminal success frame reports the cap. */
|
||||||
? manifest_delete_all(config, manifest)
|
size_t deleted = 0;
|
||||||
: true;
|
DeleteCommitResult deletion = (config->use_delete || config->delete_missing_args)
|
||||||
|
? manifest_delete_all_counted(config, manifest, &deleted)
|
||||||
|
: DELETE_COMMIT_OK;
|
||||||
|
receiver_tally_deleted(sink, deleted);
|
||||||
delete_manifest_free(manifest);
|
delete_manifest_free(manifest);
|
||||||
if (!deletion_ok) {
|
if (deletion == DELETE_COMMIT_ERROR) {
|
||||||
send_status(file_descriptor, STATUS_ERROR);
|
send_status(file_descriptor, STATUS_ERROR);
|
||||||
goto fail;
|
goto fail;
|
||||||
}
|
}
|
||||||
|
if (deletion == DELETE_COMMIT_LIMIT_REACHED && sink->note_delete_limit)
|
||||||
|
sink->note_delete_limit(sink->context);
|
||||||
if (!send_status(file_descriptor, STATUS_OK))
|
if (!send_status(file_descriptor, STATUS_OK))
|
||||||
goto fail;
|
goto fail;
|
||||||
} else if (config->use_delete || config->delete_missing_args) {
|
} else if (config->use_delete || config->delete_missing_args) {
|
||||||
/* Plain --delete / --delete-after / --delete-delay and the
|
/* Plain --delete / --delete-after and the --delete-missing-args
|
||||||
--delete-missing-args exact-path deletions: hold the manifest and
|
exact-path deletions: hold the manifest and commit it only after
|
||||||
commit it only after STATUS_FINISHED. */
|
STATUS_FINISHED. The per-directory modes never send this frame. */
|
||||||
if (deferred_manifest) {
|
if (deferred_manifest) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Received a second delete manifest");
|
log_message(LOG_LEVEL_ERROR, "Received a second delete manifest");
|
||||||
delete_manifest_free(deferred_manifest);
|
delete_manifest_free(deferred_manifest);
|
||||||
@@ -374,6 +428,23 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver
|
|||||||
delete_manifest_free(manifest);
|
delete_manifest_free(manifest);
|
||||||
}
|
}
|
||||||
goto next_status;
|
goto next_status;
|
||||||
|
} else if (status == STATUS_DELETE_PLAN) {
|
||||||
|
if (!per_dir_delete) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Received a per-directory delete plan without a per-dir "
|
||||||
|
"delete timing");
|
||||||
|
send_status(file_descriptor, STATUS_ERROR);
|
||||||
|
goto fail;
|
||||||
|
}
|
||||||
|
if (!plan_session)
|
||||||
|
plan_session = delete_plan_session_create(config);
|
||||||
|
if (!plan_session || delete_plan_session_receive(plan_session, config, file_descriptor) != 0)
|
||||||
|
goto fail;
|
||||||
|
if (delete_plan_session_limit_reached(plan_session) && !delete_limit_noted &&
|
||||||
|
sink->note_delete_limit) {
|
||||||
|
sink->note_delete_limit(sink->context);
|
||||||
|
delete_limit_noted = true;
|
||||||
|
}
|
||||||
|
goto next_status;
|
||||||
} else {
|
} else {
|
||||||
File* file = file_receive(config, file_descriptor);
|
File* file = file_receive(config, file_descriptor);
|
||||||
if (!file) {
|
if (!file) {
|
||||||
@@ -407,13 +478,45 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver
|
|||||||
*pending_manifest = deferred_manifest;
|
*pending_manifest = deferred_manifest;
|
||||||
deferred_manifest = NULL;
|
deferred_manifest = NULL;
|
||||||
} else {
|
} else {
|
||||||
bool deletion_ok = manifest_delete_all(config, deferred_manifest);
|
size_t deleted = 0;
|
||||||
|
DeleteCommitResult deletion =
|
||||||
|
manifest_delete_all_counted(config, deferred_manifest, &deleted);
|
||||||
|
receiver_tally_deleted(sink, deleted);
|
||||||
delete_manifest_free(deferred_manifest);
|
delete_manifest_free(deferred_manifest);
|
||||||
deferred_manifest = NULL;
|
deferred_manifest = NULL;
|
||||||
if (!deletion_ok) {
|
if (deletion == DELETE_COMMIT_ERROR) {
|
||||||
send_status(file_descriptor, STATUS_ERROR);
|
send_status(file_descriptor, STATUS_ERROR);
|
||||||
goto fail;
|
goto fail;
|
||||||
}
|
}
|
||||||
|
if (deletion == DELETE_COMMIT_LIMIT_REACHED && sink->note_delete_limit)
|
||||||
|
sink->note_delete_limit(sink->context);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
/* Per-directory deletion: --delete-during already applied each plan inline, so
|
||||||
|
this only finishes the missing-args deletions; --delete-delay committed
|
||||||
|
nothing yet and applies its decompressed snapshot here. The -m receiver
|
||||||
|
hands the session to its caller instead, which commits after the disk
|
||||||
|
writer drained. */
|
||||||
|
if (plan_session) {
|
||||||
|
if (pending_plans) {
|
||||||
|
*pending_plans = plan_session;
|
||||||
|
plan_session = NULL;
|
||||||
|
} else if (config->dry_run) {
|
||||||
|
/* Central dry-run no-op: never commit a deletion for a -n run. */
|
||||||
|
delete_plan_session_destroy(plan_session);
|
||||||
|
plan_session = NULL;
|
||||||
|
} else {
|
||||||
|
DeleteCommitResult deletion = delete_plan_session_commit(plan_session, config);
|
||||||
|
bool limit = delete_plan_session_limit_reached(plan_session);
|
||||||
|
receiver_tally_deleted(sink, delete_plan_session_deleted(plan_session));
|
||||||
|
delete_plan_session_destroy(plan_session);
|
||||||
|
plan_session = NULL;
|
||||||
|
if (deletion == DELETE_COMMIT_ERROR) {
|
||||||
|
send_status(file_descriptor, STATUS_ERROR);
|
||||||
|
goto fail;
|
||||||
|
}
|
||||||
|
if (limit && !delete_limit_noted && sink->note_delete_limit)
|
||||||
|
sink->note_delete_limit(sink->context);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (sink->send_success) {
|
if (sink->send_success) {
|
||||||
@@ -428,11 +531,14 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver
|
|||||||
|
|
||||||
fail:
|
fail:
|
||||||
/* Failure exits that must not (or already did) report a STATUS_ERROR. The
|
/* Failure exits that must not (or already did) report a STATUS_ERROR. The
|
||||||
parked keep-set is dropped: never commit a deletion for a failed stream. */
|
parked keep-set/session is dropped: never commit a deletion for a failed
|
||||||
|
stream. */
|
||||||
if (deferred_manifest) {
|
if (deferred_manifest) {
|
||||||
delete_manifest_free(deferred_manifest);
|
delete_manifest_free(deferred_manifest);
|
||||||
deferred_manifest = NULL;
|
deferred_manifest = NULL;
|
||||||
}
|
}
|
||||||
|
if (plan_session)
|
||||||
|
delete_plan_session_destroy(plan_session);
|
||||||
return -1;
|
return -1;
|
||||||
|
|
||||||
receive_error:
|
receive_error:
|
||||||
@@ -440,6 +546,8 @@ receive_error:
|
|||||||
delete_manifest_free(deferred_manifest);
|
delete_manifest_free(deferred_manifest);
|
||||||
deferred_manifest = NULL;
|
deferred_manifest = NULL;
|
||||||
}
|
}
|
||||||
|
if (plan_session)
|
||||||
|
delete_plan_session_destroy(plan_session);
|
||||||
if (sink->send_error)
|
if (sink->send_error)
|
||||||
send_status(file_descriptor, STATUS_ERROR);
|
send_status(file_descriptor, STATUS_ERROR);
|
||||||
return -1;
|
return -1;
|
||||||
@@ -454,6 +562,13 @@ typedef struct {
|
|||||||
after the whole transfer (and its delete/publication phases) has run so a
|
after the whole transfer (and its delete/publication phases) has run so a
|
||||||
child write never clobbers a directory mtime. */
|
child write never clobbers a directory mtime. */
|
||||||
DirTimeList dir_times;
|
DirTimeList dir_times;
|
||||||
|
/* Set when a --max-delete commit was capped; the terminal frame then carries
|
||||||
|
STATUS_DELETE_LIMIT so the sender exits 25 like rsync. */
|
||||||
|
bool delete_limit_reached;
|
||||||
|
/* End-of-transfer wire counters (protocol 2.25.0) and the -n/--dry-run
|
||||||
|
--delete would-delete path list collected while processing the manifest. */
|
||||||
|
ReceiverStats stats;
|
||||||
|
ArrayList* would_delete;
|
||||||
} ReceiverSaveContext;
|
} ReceiverSaveContext;
|
||||||
|
|
||||||
static bool receiver_save_file(File* file, void* context_pointer) {
|
static bool receiver_save_file(File* file, void* context_pointer) {
|
||||||
@@ -470,12 +585,17 @@ static bool receiver_save_file(File* file, void* context_pointer) {
|
|||||||
} else {
|
} else {
|
||||||
result = file_save_to_disk_full(context->config->receive_root_directory, file, context->config);
|
result = file_save_to_disk_full(context->config->receive_root_directory, file, context->config);
|
||||||
}
|
}
|
||||||
/* A directory's times are deferred, never applied inline: collect the
|
/* Wire-stats tally: bytes reconstructed from the basis file (delta matches)
|
||||||
metadata now and apply it at the end. -O/--omit-dir-times is honored by
|
count as matched data in the end-of-transfer report. */
|
||||||
dir_time_list_apply's caller (see receiver_send_success_frame). */
|
if (result != FILE_SAVE_ERROR && file->matched_bytes > 0)
|
||||||
|
context->stats.matched_data += file->matched_bytes;
|
||||||
|
/* A directory's metadata is deferred, never applied inline: collect it now
|
||||||
|
and apply it at the end. -O/--omit-dir-times and --preserve_perms/-times
|
||||||
|
are honored by dir_metadata_list_apply's caller (see
|
||||||
|
receiver_send_success_frame). */
|
||||||
if (result != FILE_SAVE_ERROR && file->is_dir && file->metadata &&
|
if (result != FILE_SAVE_ERROR && file->is_dir && file->metadata &&
|
||||||
dir_times_should_capture(context->config) &&
|
dir_metadata_should_capture(context->config) &&
|
||||||
!dir_time_list_add(&context->dir_times, file->path, file->metadata)) {
|
!dir_time_list_add(&context->dir_times, file->path, file->metadata, file->xattrs)) {
|
||||||
file_destroy(file);
|
file_destroy(file);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
@@ -493,12 +613,20 @@ static bool receiver_save_file(File* file, void* context_pointer) {
|
|||||||
return result != FILE_SAVE_ERROR;
|
return result != FILE_SAVE_ERROR;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void receiver_note_delete_limit(void* context_pointer) {
|
||||||
|
ReceiverSaveContext* context = context_pointer;
|
||||||
|
context->delete_limit_reached = true;
|
||||||
|
}
|
||||||
|
|
||||||
static bool receiver_send_success_frame(int fd, void* context_pointer) {
|
static bool receiver_send_success_frame(int fd, void* context_pointer) {
|
||||||
ReceiverSaveContext* context = context_pointer;
|
ReceiverSaveContext* context = context_pointer;
|
||||||
|
Status final_status = context->delete_limit_reached ? STATUS_DELETE_LIMIT : STATUS_OK;
|
||||||
|
if (!receiver_send_stats_frame(fd, context->config, &context->stats, context->would_delete))
|
||||||
|
return false;
|
||||||
/* Server-contacting --dry-run: nothing was staged or written, so there is
|
/* Server-contacting --dry-run: nothing was staged or written, so there is
|
||||||
nothing to publish and no directory times to stamp. */
|
nothing to publish and no directory times to stamp. */
|
||||||
if (context->config->dry_run)
|
if (context->config->dry_run)
|
||||||
return receiver_send_final_success(fd, context->config, &context->outcomes);
|
return receiver_send_final_success(fd, context->config, &context->outcomes, final_status);
|
||||||
/* --delay-updates: the whole protocol stream (including manifest/delete
|
/* --delay-updates: the whole protocol stream (including manifest/delete
|
||||||
handling, which ran inside receiver_process) has succeeded and every
|
handling, which ran inside receiver_process) has succeeded and every
|
||||||
staged file was fully written. Publish them atomically now, before the
|
staged file was fully written. Publish them atomically now, before the
|
||||||
@@ -514,18 +642,30 @@ static bool receiver_send_success_frame(int fd, void* context_pointer) {
|
|||||||
phases have committed, so it is finally safe to stamp directory times.
|
phases have committed, so it is finally safe to stamp directory times.
|
||||||
This runs after the deferred deletion because receiver_process commits it
|
This runs after the deferred deletion because receiver_process commits it
|
||||||
before calling this success frame. */
|
before calling this success frame. */
|
||||||
dir_time_list_apply(&context->dir_times, context->config->receive_root_directory);
|
dir_metadata_list_apply(&context->dir_times, context->config->receive_root_directory,
|
||||||
return receiver_send_final_success(fd, context->config, &context->outcomes);
|
context->config);
|
||||||
|
return receiver_send_final_success(fd, context->config, &context->outcomes, final_status);
|
||||||
}
|
}
|
||||||
|
|
||||||
int receiver_receive_files(Config* config, int file_descriptor) {
|
int receiver_receive_files(Config* config, int file_descriptor) {
|
||||||
ReceiverSaveContext context = {.config = config, .outcomes = {0}};
|
ReceiverSaveContext context = {.config = config, .outcomes = {0}};
|
||||||
dir_time_list_init(&context.dir_times);
|
dir_time_list_init(&context.dir_times);
|
||||||
ReceiverSink sink = {receiver_save_file, &context, true, true, receiver_send_success_frame};
|
context.would_delete = array_list_create(free);
|
||||||
|
if (!context.would_delete)
|
||||||
|
return -1;
|
||||||
|
ReceiverSink sink = {receiver_save_file,
|
||||||
|
&context,
|
||||||
|
true,
|
||||||
|
true,
|
||||||
|
receiver_send_success_frame,
|
||||||
|
receiver_note_delete_limit,
|
||||||
|
&context.stats,
|
||||||
|
context.would_delete};
|
||||||
int ret = receiver_process(config, file_descriptor, &sink);
|
int ret = receiver_process(config, file_descriptor, &sink);
|
||||||
if (ret != 0 && config->delay_updates && config->delay_context)
|
if (ret != 0 && config->delay_updates && config->delay_context)
|
||||||
delay_updates_cleanup(config->delay_context);
|
delay_updates_cleanup(config->delay_context);
|
||||||
receiver_outcomes_destroy(&context.outcomes);
|
receiver_outcomes_destroy(&context.outcomes);
|
||||||
dir_time_list_free(&context.dir_times);
|
dir_time_list_free(&context.dir_times);
|
||||||
|
array_list_delete(context.would_delete);
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
|
|||||||
+33
-5
@@ -2,8 +2,10 @@
|
|||||||
#define RECEIVER_H
|
#define RECEIVER_H
|
||||||
|
|
||||||
#include "config.h"
|
#include "config.h"
|
||||||
|
#include "delete_plan.h"
|
||||||
#include "file.h"
|
#include "file.h"
|
||||||
#include "file_receive.h"
|
#include "file_receive.h"
|
||||||
|
#include "protocol.h"
|
||||||
#include <stdbool.h>
|
#include <stdbool.h>
|
||||||
#include <time.h>
|
#include <time.h>
|
||||||
|
|
||||||
@@ -21,6 +23,12 @@ typedef struct {
|
|||||||
|
|
||||||
typedef bool (*ReceiverSuccessFrame)(int fd, void* context);
|
typedef bool (*ReceiverSuccessFrame)(int fd, void* context);
|
||||||
|
|
||||||
|
/* Records that a --max-delete commit stopped with extras left over, so the
|
||||||
|
caller's terminal success frame can carry STATUS_DELETE_LIMIT instead of
|
||||||
|
STATUS_OK. The commit runs on the receiver thread, so the flag is stored in
|
||||||
|
the sink's own context rather than in a shared global. */
|
||||||
|
typedef void (*ReceiverNoteDeleteLimit)(void* context);
|
||||||
|
|
||||||
typedef struct {
|
typedef struct {
|
||||||
ReceiverFileSink store_file;
|
ReceiverFileSink store_file;
|
||||||
void* context;
|
void* context;
|
||||||
@@ -28,23 +36,43 @@ typedef struct {
|
|||||||
bool send_success;
|
bool send_success;
|
||||||
/* Emits the end-of-transfer success frame. When the sender requested
|
/* Emits the end-of-transfer success frame. When the sender requested
|
||||||
--remove-source-files this includes one per-file status per processed
|
--remove-source-files this includes one per-file status per processed
|
||||||
data file followed by the final STATUS_OK; otherwise just STATUS_OK. */
|
data file followed by the final status; otherwise just the final status. */
|
||||||
ReceiverSuccessFrame send_success_frame;
|
ReceiverSuccessFrame send_success_frame;
|
||||||
|
/* Optional; may be NULL when the sink has no --max-delete handling. */
|
||||||
|
ReceiverNoteDeleteLimit note_delete_limit;
|
||||||
|
/* Optional end-of-transfer wire counters (protocol 2.25.0). When non-NULL
|
||||||
|
and the wire config carries report_stats, the success frame is preceded by
|
||||||
|
a STATUS_STATS record; `would_delete` (optional, receiver-owned strings)
|
||||||
|
carries the -n/--dry-run --delete path list. */
|
||||||
|
ReceiverStats* stats;
|
||||||
|
struct ArrayList* would_delete;
|
||||||
} ReceiverSink;
|
} ReceiverSink;
|
||||||
|
|
||||||
bool receiver_outcomes_append(ReceiverOutcomes* outcomes, unsigned char code);
|
bool receiver_outcomes_append(ReceiverOutcomes* outcomes, unsigned char code);
|
||||||
void receiver_outcomes_destroy(ReceiverOutcomes* outcomes);
|
void receiver_outcomes_destroy(ReceiverOutcomes* outcomes);
|
||||||
bool receiver_send_final_success(int fd, const Config* config, const ReceiverOutcomes* outcomes);
|
|
||||||
|
/* Send the terminal success frame. `final_status` is usually STATUS_OK, or
|
||||||
|
STATUS_DELETE_LIMIT when a --max-delete commit was capped. */
|
||||||
|
bool receiver_send_final_success(int fd, const Config* config, const ReceiverOutcomes* outcomes,
|
||||||
|
Status final_status);
|
||||||
|
|
||||||
|
/* Emit STATUS_STATS (a fixed ReceiverStats record plus, when `would_delete` is
|
||||||
|
non-NULL, a count and that many wire strings) when the wire config requested
|
||||||
|
report_stats. A no-op otherwise. */
|
||||||
|
bool receiver_send_stats_frame(int fd, const Config* config, const ReceiverStats* stats,
|
||||||
|
const struct ArrayList* would_delete);
|
||||||
|
|
||||||
int receiver_process(Config* config, int file_descriptor, const ReceiverSink* sink);
|
int receiver_process(Config* config, int file_descriptor, const ReceiverSink* sink);
|
||||||
/* receiver_process with an escape hatch for the commit-style (late) deletion:
|
/* receiver_process with an escape hatch for the commit-style (late) deletion:
|
||||||
when `pending_manifest` is non-NULL the receiver does NOT delete at
|
when `pending_manifest` is non-NULL the receiver does NOT delete at
|
||||||
STATUS_FINISHED itself; instead it stores the owned keep-set manifest there
|
STATUS_FINISHED itself; instead it stores the owned keep-set manifest there
|
||||||
(leaving *pending_manifest untouched on early modes/errors) so the caller can
|
(leaving *pending_manifest untouched on early modes/errors) so the caller can
|
||||||
commit the deletion only after its disk writer has fully drained. Pass NULL
|
commit the deletion only after its disk writer has fully drained. Likewise,
|
||||||
to keep the default behaviour (delete before the success frame). */
|
when `pending_plans` is non-NULL the --delete-delay per-directory session is
|
||||||
|
handed to the caller instead of being committed at STATUS_FINISHED. Pass NULL
|
||||||
|
for either to keep the default behaviour (delete before the success frame). */
|
||||||
int receiver_process_pending(Config* config, int file_descriptor, const ReceiverSink* sink,
|
int receiver_process_pending(Config* config, int file_descriptor, const ReceiverSink* sink,
|
||||||
DeleteManifest** pending_manifest);
|
DeleteManifest** pending_manifest, DeletePlanSession** pending_plans);
|
||||||
int receiver_receive_files(Config* config, int file_descriptor);
|
int receiver_receive_files(Config* config, int file_descriptor);
|
||||||
|
|
||||||
/* ---- Connection time bounds (anti-slowloris) ----
|
/* ---- Connection time bounds (anti-slowloris) ----
|
||||||
|
|||||||
@@ -27,6 +27,10 @@ PipelineContextReceiver* pipeline_context_receiver_create(Config* config, Queue*
|
|||||||
context->queued_bytes = 0;
|
context->queued_bytes = 0;
|
||||||
context->max_queue_bytes = 0;
|
context->max_queue_bytes = 0;
|
||||||
context->deferred_manifest = NULL;
|
context->deferred_manifest = NULL;
|
||||||
|
context->deferred_plans = NULL;
|
||||||
|
context->delete_limit_reached = false;
|
||||||
|
memset(&context->stats, 0, sizeof(context->stats));
|
||||||
|
context->would_delete = NULL;
|
||||||
atomic_init(&context->cancelled, false);
|
atomic_init(&context->cancelled, false);
|
||||||
int init = 0;
|
int init = 0;
|
||||||
if (mtx_init(&context->mutex, mtx_plain) != thrd_success)
|
if (mtx_init(&context->mutex, mtx_plain) != thrd_success)
|
||||||
@@ -39,6 +43,9 @@ PipelineContextReceiver* pipeline_context_receiver_create(Config* config, Queue*
|
|||||||
goto fail;
|
goto fail;
|
||||||
// cppcheck-suppress unreadVariable
|
// cppcheck-suppress unreadVariable
|
||||||
init++;
|
init++;
|
||||||
|
context->would_delete = array_list_create(free);
|
||||||
|
if (!context->would_delete)
|
||||||
|
goto fail;
|
||||||
return context;
|
return context;
|
||||||
|
|
||||||
fail:
|
fail:
|
||||||
@@ -57,9 +64,13 @@ void pipeline_context_receiver_destroy(PipelineContextReceiver* context) {
|
|||||||
config_delete(context->config);
|
config_delete(context->config);
|
||||||
if (context->deferred_manifest)
|
if (context->deferred_manifest)
|
||||||
delete_manifest_free(context->deferred_manifest);
|
delete_manifest_free(context->deferred_manifest);
|
||||||
|
if (context->deferred_plans)
|
||||||
|
delete_plan_session_destroy(context->deferred_plans);
|
||||||
queue_destroy(context->queue);
|
queue_destroy(context->queue);
|
||||||
receiver_outcomes_destroy(&context->outcomes);
|
receiver_outcomes_destroy(&context->outcomes);
|
||||||
dir_time_list_free(&context->dir_times);
|
dir_time_list_free(&context->dir_times);
|
||||||
|
if (context->would_delete)
|
||||||
|
array_list_delete(context->would_delete);
|
||||||
mtx_destroy(&context->mutex);
|
mtx_destroy(&context->mutex);
|
||||||
cnd_destroy(&context->condition_not_full);
|
cnd_destroy(&context->condition_not_full);
|
||||||
cnd_destroy(&context->condition_not_empty);
|
cnd_destroy(&context->condition_not_empty);
|
||||||
@@ -132,9 +143,23 @@ bool pipeline_context_receiver_enqueue_file(PipelineContextReceiver* context, Fi
|
|||||||
|
|
||||||
static bool receiver_enqueue_file(File* file, void* context_pointer) {
|
static bool receiver_enqueue_file(File* file, void* context_pointer) {
|
||||||
PipelineContextReceiver* context = (PipelineContextReceiver*)context_pointer;
|
PipelineContextReceiver* context = (PipelineContextReceiver*)context_pointer;
|
||||||
|
if (file && file->matched_bytes > 0) {
|
||||||
|
mtx_lock(&context->mutex);
|
||||||
|
context->stats.matched_data += file->matched_bytes;
|
||||||
|
mtx_unlock(&context->mutex);
|
||||||
|
}
|
||||||
return pipeline_context_receiver_enqueue_file(context, file);
|
return pipeline_context_receiver_enqueue_file(context, file);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Early delete modes (--delete-before/--delete-during) commit the manifest
|
||||||
|
inside receiver_process_pending on this thread; record a capped commit so
|
||||||
|
server.c's terminal frame can report STATUS_DELETE_LIMIT. The plain bool is
|
||||||
|
safe: receive_thread writes it before the main thread joins the thread. */
|
||||||
|
static void receiver_pipeline_note_delete_limit(void* context_pointer) {
|
||||||
|
PipelineContextReceiver* context = (PipelineContextReceiver*)context_pointer;
|
||||||
|
context->delete_limit_reached = true;
|
||||||
|
}
|
||||||
|
|
||||||
static void receiver_thread_fail(PipelineContextReceiver* context) {
|
static void receiver_thread_fail(PipelineContextReceiver* context) {
|
||||||
mtx_lock(&context->mutex);
|
mtx_lock(&context->mutex);
|
||||||
atomic_store(&context->cancelled, true);
|
atomic_store(&context->cancelled, true);
|
||||||
@@ -152,9 +177,16 @@ int receive_thread(void* pipeline_context) {
|
|||||||
const Config* config = context->config;
|
const Config* config = context->config;
|
||||||
mtx_unlock(&context->mutex);
|
mtx_unlock(&context->mutex);
|
||||||
|
|
||||||
ReceiverSink sink = {receiver_enqueue_file, context, false, false, NULL};
|
ReceiverSink sink = {receiver_enqueue_file,
|
||||||
if (receiver_process_pending((Config*)config, file_descriptor, &sink,
|
context,
|
||||||
&context->deferred_manifest) != 0) {
|
false,
|
||||||
|
false,
|
||||||
|
NULL,
|
||||||
|
receiver_pipeline_note_delete_limit,
|
||||||
|
&context->stats,
|
||||||
|
context->would_delete};
|
||||||
|
if (receiver_process_pending((Config*)config, file_descriptor, &sink, &context->deferred_manifest,
|
||||||
|
&context->deferred_plans) != 0) {
|
||||||
receiver_thread_fail(context);
|
receiver_thread_fail(context);
|
||||||
protocol_session_unbind();
|
protocol_session_unbind();
|
||||||
return thrd_error;
|
return thrd_error;
|
||||||
@@ -220,8 +252,8 @@ int write_thread(void* pipeline_context) {
|
|||||||
write would clobber them); accumulate the metadata here and let the
|
write would clobber them); accumulate the metadata here and let the
|
||||||
caller apply it once every writer has drained. */
|
caller apply it once every writer has drained. */
|
||||||
if (!dry_run && result != FILE_SAVE_ERROR && file->is_dir && file->metadata &&
|
if (!dry_run && result != FILE_SAVE_ERROR && file->is_dir && file->metadata &&
|
||||||
dir_times_should_capture(context->config) &&
|
dir_metadata_should_capture(context->config) &&
|
||||||
!dir_time_list_add(&context->dir_times, file->path, file->metadata)) {
|
!dir_time_list_add(&context->dir_times, file->path, file->metadata, file->xattrs)) {
|
||||||
file_destroy(file);
|
file_destroy(file);
|
||||||
pipeline_context_receiver_note_bytes_released(context, file_bytes);
|
pipeline_context_receiver_note_bytes_released(context, file_bytes);
|
||||||
mtx_lock(&context->mutex);
|
mtx_lock(&context->mutex);
|
||||||
|
|||||||
@@ -41,10 +41,26 @@ typedef struct PipelineContextReceiver {
|
|||||||
transfer truly succeeded. NULL in the early delete modes (which delete at
|
transfer truly succeeded. NULL in the early delete modes (which delete at
|
||||||
the manifest). */
|
the manifest). */
|
||||||
DeleteManifest* deferred_manifest;
|
DeleteManifest* deferred_manifest;
|
||||||
|
/* Per-directory delete session for --delete-delay: receive_thread snapshots
|
||||||
|
each plan's extras as it arrives and hands the session here instead of
|
||||||
|
committing while the disk writer may still be draining; server.c commits it
|
||||||
|
after both threads joined. NULL for every other timing. */
|
||||||
|
DeletePlanSession* deferred_plans;
|
||||||
|
/* Set by server.c when the deferred delete commit hit the --max-delete
|
||||||
|
budget; the terminal success frame then carries STATUS_DELETE_LIMIT
|
||||||
|
(rsync exit 25) while the transfer itself still succeeds. */
|
||||||
|
bool delete_limit_reached;
|
||||||
/* P7 Wave D: directory metadata collected by write_thread from received
|
/* P7 Wave D: directory metadata collected by write_thread from received
|
||||||
directory entries. Only write_thread mutates it (before it joins); the
|
directory entries. Only write_thread mutates it (before it joins); the
|
||||||
caller (server.c) applies it after the delete/delay-updates phase. */
|
caller (server.c) applies it after the delete/delay-updates phase. */
|
||||||
DirTimeList dir_times;
|
DirTimeList dir_times;
|
||||||
|
/* End-of-transfer wire counters (protocol 2.25.0). receive_thread accumulates
|
||||||
|
matched_data under `mutex`; server.c adds the delete-commit tallies after
|
||||||
|
both threads join and emits the STATUS_STATS frame. */
|
||||||
|
ReceiverStats stats;
|
||||||
|
/* -n/--dry-run --delete would-delete path list, collected by receive_thread
|
||||||
|
and reported in the STATUS_STATS frame. */
|
||||||
|
struct ArrayList* would_delete;
|
||||||
} PipelineContextReceiver;
|
} PipelineContextReceiver;
|
||||||
|
|
||||||
PipelineContextReceiver* pipeline_context_receiver_create(Config* config, Queue* queue_receiver,
|
PipelineContextReceiver* pipeline_context_receiver_create(Config* config, Queue* queue_receiver,
|
||||||
|
|||||||
+78
-14
@@ -389,8 +389,12 @@ static const char* module_gate_check_ownership(const Config* config, const Daemo
|
|||||||
ModuleGateContext* gate_ctx) {
|
ModuleGateContext* gate_ctx) {
|
||||||
if (module->client_owner)
|
if (module->client_owner)
|
||||||
return NULL;
|
return NULL;
|
||||||
/* Ownership: refuse the whole transfer up front (a clear failure). */
|
/* Ownership: refuse the whole transfer up front (a clear failure) for a
|
||||||
if (identity_ownership_requested(config)) {
|
* client-CHOSEN owner/group request. A plain -o/-g/-a preserve-source
|
||||||
|
* request is deliberately not in this narrow set: it falls through to the
|
||||||
|
* super-mode override below, which forces all ownership activity off for this
|
||||||
|
* connection so no chown happens (the transfer itself still succeeds). */
|
||||||
|
if (identity_explicit_ownership_requested(config)) {
|
||||||
log_message(LOG_LEVEL_ERROR,
|
log_message(LOG_LEVEL_ERROR,
|
||||||
"daemon module '%s' refuses client-chosen ownership/super-user activities "
|
"daemon module '%s' refuses client-chosen ownership/super-user activities "
|
||||||
"(no `client owner = yes` opt-in); refusing",
|
"(no `client owner = yes` opt-in); refusing",
|
||||||
@@ -737,13 +741,26 @@ void handler(int file_descriptor) {
|
|||||||
* received config. */
|
* received config. */
|
||||||
if (gate_ctx.super_mode_override != -1)
|
if (gate_ctx.super_mode_override != -1)
|
||||||
config->super_mode = (SuperMode)gate_ctx.super_mode_override;
|
config->super_mode = (SuperMode)gate_ctx.super_mode_override;
|
||||||
|
/* Install the codec this connection negotiated before the receiver/writer
|
||||||
|
* threads start (the server forks per connection, so the process-global
|
||||||
|
* codec is private to this session). */
|
||||||
|
compression_set_algo((CompressionAlgo)config->compression_algo);
|
||||||
|
/* If the client requested ownership but the effective super mode forbids it
|
||||||
|
* (operator --no-super, a privileged standalone receiver's secure default, or
|
||||||
|
* a daemon module without `client owner = yes`), say so ONCE per connection so
|
||||||
|
* a successful -a/-o/-g transfer is not mistaken for preserved ownership. */
|
||||||
|
if (config->super_mode == SUPER_MODE_OFF && identity_ownership_requested(config))
|
||||||
|
log_message(LOG_LEVEL_WARNING,
|
||||||
|
"requested ownership will NOT be applied: super-user activities are disabled "
|
||||||
|
"for this connection (operator veto, or module without `client owner = yes`)");
|
||||||
protocol_set_8_bit_output(config->eight_bit_output);
|
protocol_set_8_bit_output(config->eight_bit_output);
|
||||||
/* Server-side per-message protocol deadline for every frame from here on.
|
/* Server-side per-message protocol deadline for every frame from here on.
|
||||||
* `timeout` is not serialized, so this is the server's own config (the server
|
* `timeout` is not serialized, so this is the server's own config (the server
|
||||||
* has no --timeout CLI and defaults it to 0): the built-in 60 s window stays
|
* has no --timeout CLI and defaults it to 0). A client's --timeout tightens
|
||||||
* in effect. A client's --timeout tightens only that client's own protocol
|
* only that client's own protocol I/O; the server floors its own deadline at
|
||||||
* I/O and the server's socket read/write timeout is the transport default. */
|
* SERVER_IO_TIMEOUT_SEC so a silent peer can never hold a session slot
|
||||||
protocol_session_set_io_timeout(&session, config->timeout);
|
* forever (the socket layer gets the same floor at startup). */
|
||||||
|
protocol_session_set_io_timeout(&session, protocol_server_io_timeout_sec(config->timeout));
|
||||||
const char* authorized_root = utils_get_authorized_root_path();
|
const char* authorized_root = utils_get_authorized_root_path();
|
||||||
if (!authorized_root) {
|
if (!authorized_root) {
|
||||||
log_message(LOG_LEVEL_ERROR, "No server-side destination root configured");
|
log_message(LOG_LEVEL_ERROR, "No server-side destination root configured");
|
||||||
@@ -892,7 +909,8 @@ void handler(int file_descriptor) {
|
|||||||
goto done;
|
goto done;
|
||||||
}
|
}
|
||||||
protocol_session_set_max_alloc(&context->session, config->max_alloc);
|
protocol_session_set_max_alloc(&context->session, config->max_alloc);
|
||||||
protocol_session_set_io_timeout(&context->session, config->timeout);
|
protocol_session_set_io_timeout(&context->session,
|
||||||
|
protocol_server_io_timeout_sec(config->timeout));
|
||||||
atomic_store(&context->session.total_allocated_bytes,
|
atomic_store(&context->session.total_allocated_bytes,
|
||||||
atomic_load(&session.total_allocated_bytes));
|
atomic_load(&session.total_allocated_bytes));
|
||||||
pipeline_context_receiver_set_queue_byte_limit(context, RECEIVER_QUEUE_MAX_BYTES);
|
pipeline_context_receiver_set_queue_byte_limit(context, RECEIVER_QUEUE_MAX_BYTES);
|
||||||
@@ -937,12 +955,38 @@ void handler(int file_descriptor) {
|
|||||||
--delay-updates run; the walker skips the staging directory. A
|
--delay-updates run; the walker skips the staging directory. A
|
||||||
server-contacting --dry-run deletes nothing (no manifest is sent). */
|
server-contacting --dry-run deletes nothing (no manifest is sent). */
|
||||||
if (context->deferred_manifest) {
|
if (context->deferred_manifest) {
|
||||||
if (!manifest_delete_all(config, context->deferred_manifest)) {
|
size_t deleted = 0;
|
||||||
|
DeleteCommitResult deletion =
|
||||||
|
manifest_delete_all_counted(config, context->deferred_manifest, &deleted);
|
||||||
|
context->stats.deleted_files += deleted;
|
||||||
|
if (deletion == DELETE_COMMIT_ERROR) {
|
||||||
transfer_ok = false;
|
transfer_ok = false;
|
||||||
|
} else if (deletion == DELETE_COMMIT_LIMIT_REACHED) {
|
||||||
|
/* The transfer still succeeds; the terminal frame reports the capped
|
||||||
|
deletion so the sender exits 25 like rsync. */
|
||||||
|
context->delete_limit_reached = true;
|
||||||
}
|
}
|
||||||
delete_manifest_free(context->deferred_manifest);
|
delete_manifest_free(context->deferred_manifest);
|
||||||
context->deferred_manifest = NULL;
|
context->deferred_manifest = NULL;
|
||||||
}
|
}
|
||||||
|
/* --delete-delay: receive_thread snapshotted each plan's extras as it
|
||||||
|
arrived; with the disk writer drained, commit the deferred removals.
|
||||||
|
--delete-during already applied its plans on the receive thread. */
|
||||||
|
if (context->deferred_plans) {
|
||||||
|
/* Defence in depth (the enclosing block already excludes dry-run): a
|
||||||
|
-n run never commits a deletion. */
|
||||||
|
DeleteCommitResult deletion =
|
||||||
|
config->dry_run ? DELETE_COMMIT_OK
|
||||||
|
: delete_plan_session_commit(context->deferred_plans, config);
|
||||||
|
context->stats.deleted_files += delete_plan_session_deleted(context->deferred_plans);
|
||||||
|
if (deletion == DELETE_COMMIT_ERROR) {
|
||||||
|
transfer_ok = false;
|
||||||
|
} else if (deletion == DELETE_COMMIT_LIMIT_REACHED) {
|
||||||
|
context->delete_limit_reached = true;
|
||||||
|
}
|
||||||
|
delete_plan_session_destroy(context->deferred_plans);
|
||||||
|
context->deferred_plans = NULL;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if (transfer_ok && !config->dry_run) {
|
if (transfer_ok && !config->dry_run) {
|
||||||
/* --delay-updates: receive_thread has finished the whole protocol stream
|
/* --delay-updates: receive_thread has finished the whole protocol stream
|
||||||
@@ -959,10 +1003,15 @@ void handler(int file_descriptor) {
|
|||||||
to stamp directory times; a directory's mtime must not be clobbered by
|
to stamp directory times; a directory's mtime must not be clobbered by
|
||||||
its children or by an extra removal. */
|
its children or by an extra removal. */
|
||||||
if (transfer_ok)
|
if (transfer_ok)
|
||||||
dir_time_list_apply(&context->dir_times, config->receive_root_directory);
|
dir_metadata_list_apply(&context->dir_times, config->receive_root_directory, config);
|
||||||
}
|
}
|
||||||
if (transfer_ok) {
|
if (transfer_ok) {
|
||||||
if (!receiver_send_final_success(file_descriptor, config, &context->outcomes))
|
Status final_status = context->delete_limit_reached ? STATUS_DELETE_LIMIT : STATUS_OK;
|
||||||
|
/* Emit the optional wire-stats record first (protocol 2.25.0), then the
|
||||||
|
success/outcome frame, exactly like the single-threaded receiver. */
|
||||||
|
if (!receiver_send_stats_frame(file_descriptor, config, &context->stats,
|
||||||
|
context->would_delete) ||
|
||||||
|
!receiver_send_final_success(file_descriptor, config, &context->outcomes, final_status))
|
||||||
transfer_ok = false;
|
transfer_ok = false;
|
||||||
} else {
|
} else {
|
||||||
send_error_detail(file_descriptor, "transfer failed on receiver");
|
send_error_detail(file_descriptor, "transfer failed on receiver");
|
||||||
@@ -1029,8 +1078,9 @@ static void print_server_usage(void) {
|
|||||||
printf(" hosts allow, hosts deny)\n");
|
printf(" hosts allow, hosts deny)\n");
|
||||||
printf(" --no-detach Stay in the foreground (default detaches to\n");
|
printf(" --no-detach Stay in the foreground (default detaches to\n");
|
||||||
printf(" background when running --daemon)\n");
|
printf(" background when running --daemon)\n");
|
||||||
printf(" --password-file=FILE Credential store for modules that declare\n");
|
printf(" --password-file=FILE FastSync-native SCRAM/PBKDF2 credential store (NOT\n");
|
||||||
printf(" 'auth users' (line format:\n");
|
printf(" rsync's auth scheme) for modules that declare 'auth\n");
|
||||||
|
printf(" users' (line format:\n");
|
||||||
printf(" user:$fastsync$1$pbkdf2-sha256$iters$salt$stored$server,\n");
|
printf(" user:$fastsync$1$pbkdf2-sha256$iters$salt$stored$server,\n");
|
||||||
printf(" generated by --hash-credentials). Legacy\n");
|
printf(" generated by --hash-credentials). Legacy\n");
|
||||||
printf(" user:SHA256HEX lines are rejected. Requires\n");
|
printf(" user:SHA256HEX lines are rejected. Requires\n");
|
||||||
@@ -1039,7 +1089,7 @@ static void print_server_usage(void) {
|
|||||||
printf(" --early-input=FILE Second credential store layered over\n");
|
printf(" --early-input=FILE Second credential store layered over\n");
|
||||||
printf(" --password-file (same format); usually a secrets-\n");
|
printf(" --password-file (same format); usually a secrets-\n");
|
||||||
printf(" manager/process-substitution file. Requires --daemon\n");
|
printf(" manager/process-substitution file. Requires --daemon\n");
|
||||||
printf(" -p <port> TCP port (default: 8080, range: 1-65535)\n");
|
printf(" -p, --port <port> TCP port (default: 8080, range: 1-65535)\n");
|
||||||
printf(" --tls Enable TLS encryption\n");
|
printf(" --tls Enable TLS encryption\n");
|
||||||
printf(" --cert <path> TLS certificate file (PEM)\n");
|
printf(" --cert <path> TLS certificate file (PEM)\n");
|
||||||
printf(" --key <path> TLS private key file (PEM)\n");
|
printf(" --key <path> TLS private key file (PEM)\n");
|
||||||
@@ -1050,7 +1100,9 @@ static void print_server_usage(void) {
|
|||||||
printf(" -4, --ipv4 Bind an IPv4 socket (default)\n");
|
printf(" -4, --ipv4 Bind an IPv4 socket (default)\n");
|
||||||
printf(" -6, --ipv6 Bind an IPv6 socket\n");
|
printf(" -6, --ipv6 Bind an IPv6 socket\n");
|
||||||
printf(" --allow-delete Permit manifest deletion\n");
|
printf(" --allow-delete Permit manifest deletion\n");
|
||||||
printf(" --trust-sender Trust the remote sender's file list\n");
|
printf(" --trust-sender Trust the remote sender's file list (receiver-local;\n");
|
||||||
|
printf(" this server-side flag is the only one that matters -- a\n");
|
||||||
|
printf(" client --trust-sender is never sent to the server)\n");
|
||||||
printf(" --no-super Operator veto: never attempt super-user activities\n");
|
printf(" --no-super Operator veto: never attempt super-user activities\n");
|
||||||
printf(" (ownership, device nodes) even as root, and refuse\n");
|
printf(" (ownership, device nodes) even as root, and refuse\n");
|
||||||
printf(" any client --copy-as/--super request\n");
|
printf(" any client --copy-as/--super request\n");
|
||||||
@@ -1127,10 +1179,18 @@ static bool daemonize(void) {
|
|||||||
if (chdir("/") != 0)
|
if (chdir("/") != 0)
|
||||||
log_message(LOG_LEVEL_WARNING, "daemon: chdir to / failed: %s", strerror(errno));
|
log_message(LOG_LEVEL_WARNING, "daemon: chdir to / failed: %s", strerror(errno));
|
||||||
umask(0);
|
umask(0);
|
||||||
|
/* Refresh the cached umask: main() captured the launch umask before this
|
||||||
|
* (single-threaded) umask(0), and file_mode_base() must see the daemon's
|
||||||
|
* actual umask. */
|
||||||
|
file_umask_capture();
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
int main(int argc, char* argv[]) {
|
int main(int argc, char* argv[]) {
|
||||||
|
/* Capture the process umask now, while still single-threaded: the cached
|
||||||
|
* value is what file_mode_base() uses, and reading it later would race with
|
||||||
|
* receiver threads creating files. */
|
||||||
|
file_umask_capture();
|
||||||
ServerCliOptions opts;
|
ServerCliOptions opts;
|
||||||
char cli_err[512];
|
char cli_err[512];
|
||||||
int parse_result = server_cli_parse(argc, argv, &opts, cli_err, sizeof(cli_err));
|
int parse_result = server_cli_parse(argc, argv, &opts, cli_err, sizeof(cli_err));
|
||||||
@@ -1191,6 +1251,10 @@ int main(int argc, char* argv[]) {
|
|||||||
server_iconv_spec = opts.iconv_spec;
|
server_iconv_spec = opts.iconv_spec;
|
||||||
signal(SIGINT, cleanup);
|
signal(SIGINT, cleanup);
|
||||||
signal(SIGTERM, cleanup);
|
signal(SIGTERM, cleanup);
|
||||||
|
/* Server-owned socket deadline floor: the client default --timeout=0 would
|
||||||
|
* otherwise leave accepted sockets without SO_RCVTIMEO/SO_SNDTIMEO and let a
|
||||||
|
* silent peer hold a connection (and its process slot) forever. */
|
||||||
|
tcp_set_timeouts(SERVER_IO_TIMEOUT_SEC, SERVER_IO_TIMEOUT_SEC);
|
||||||
|
|
||||||
if (opts.stdio_mode) {
|
if (opts.stdio_mode) {
|
||||||
/* SSH authenticates the stdio transport outside of FastSync. */
|
/* SSH authenticates the stdio transport outside of FastSync. */
|
||||||
|
|||||||
@@ -192,14 +192,20 @@ int server_cli_parse(int argc, char* argv[], ServerCliOptions* opts, char* err,
|
|||||||
inline_value = argv[++i];
|
inline_value = argv[++i];
|
||||||
}
|
}
|
||||||
opts->iconv_spec = inline_value;
|
opts->iconv_spec = inline_value;
|
||||||
} else if (arg_is(argv[i], "-p")) {
|
} else if (arg_is(argv[i], "-p") || arg_has_value(argv[i], "--port", &inline_value)) {
|
||||||
|
if (inline_value) {
|
||||||
|
opts->port_set = true;
|
||||||
|
if (parse_port_arg(inline_value, &opts->port, err, err_size) != 0)
|
||||||
|
return -1;
|
||||||
|
} else {
|
||||||
if (i + 1 >= argc) {
|
if (i + 1 >= argc) {
|
||||||
set_error(err, err_size, "missing argument for -p");
|
set_error(err, err_size, "missing argument for %s", argv[i]);
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
opts->port_set = true;
|
opts->port_set = true;
|
||||||
if (parse_port_arg(argv[++i], &opts->port, err, err_size) != 0)
|
if (parse_port_arg(argv[++i], &opts->port, err, err_size) != 0)
|
||||||
return -1;
|
return -1;
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
if (arg_has_value(argv[i], "--config", &inline_value)) {
|
if (arg_has_value(argv[i], "--config", &inline_value)) {
|
||||||
if (!inline_value) {
|
if (!inline_value) {
|
||||||
|
|||||||
+53
-18
@@ -2,6 +2,7 @@
|
|||||||
#include "data.h"
|
#include "data.h"
|
||||||
#include "file.h"
|
#include "file.h"
|
||||||
#include "file_receive.h"
|
#include "file_receive.h"
|
||||||
|
#include "identity.h"
|
||||||
#include "log.h"
|
#include "log.h"
|
||||||
#include <errno.h>
|
#include <errno.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
@@ -10,11 +11,15 @@
|
|||||||
|
|
||||||
/* Serialization metadata mode for the batch stream, captured from the config at
|
/* Serialization metadata mode for the batch stream, captured from the config at
|
||||||
* batch_write_header time. The header persists it into the file so a batch is
|
* batch_write_header time. The header persists it into the file so a batch is
|
||||||
* self-describing: batch_read_apply re-reads it from the file (not from the
|
* self-describing about whether per-entry metadata was CAPTURED in the stream:
|
||||||
* reading config), so a batch written with -M is applied identically by an
|
* batch_read_apply re-reads it from the file (not from the reading config) to
|
||||||
* invoking process regardless of its own -M setting. The batch driver is a
|
* decode the chunk records correctly. Which attributes are actually APPLIED,
|
||||||
* single sequential scan pass within one thread, so this module-level flag is
|
* however, comes from the INVOKING process's per-attribute config (the
|
||||||
* safe. */
|
* FileAttrPolicy and the dir-metadata gate), so a batch written with -M is NOT
|
||||||
|
* automatically applied identically by an invoking process with a different
|
||||||
|
* -p/-t/-o/-g: --read-batch must be invoked with the same -p/-t/-o/-g as the
|
||||||
|
* write side (rsync requires the same options). The batch driver is a single
|
||||||
|
* sequential scan pass within one thread, so this module-level flag is safe. */
|
||||||
static bool batch_metadata_mode = false;
|
static bool batch_metadata_mode = false;
|
||||||
|
|
||||||
static bool write_all_bytes(int fd, const void* data, size_t size) {
|
static bool write_all_bytes(int fd, const void* data, size_t size) {
|
||||||
@@ -91,22 +96,37 @@ int batch_read_apply(int fd, const Config* config, const char* dest_root) {
|
|||||||
if (fd < 0 || dest_root == NULL || dest_root[0] == '\0')
|
if (fd < 0 || dest_root == NULL || dest_root[0] == '\0')
|
||||||
return -1;
|
return -1;
|
||||||
|
|
||||||
|
/* Directory metadata is deferred to the end of the apply (a child write would
|
||||||
|
* otherwise clobber its parent's mtime/mode). The batch header's single
|
||||||
|
* metadata bit only says whether metadata is present in the stream; which
|
||||||
|
* attributes are APPLIED comes from the invoking process's config, so
|
||||||
|
* --read-batch must be invoked with the same -p/-t/-o/-g as the write side
|
||||||
|
* (rsync requires the same options). The identity snapshot is activated so
|
||||||
|
* -o/-g and the explicit ownership flags can apply. */
|
||||||
|
DirTimeList dir_times;
|
||||||
|
dir_time_list_init(&dir_times);
|
||||||
|
int result = -1;
|
||||||
|
if (!identity_set_active(config)) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "batch: could not activate the identity policy");
|
||||||
|
goto done;
|
||||||
|
}
|
||||||
|
|
||||||
char magic[BATCH_MAGIC_LEN];
|
char magic[BATCH_MAGIC_LEN];
|
||||||
bool eof = false;
|
bool eof = false;
|
||||||
if (!read_exact(fd, magic, BATCH_MAGIC_LEN, &eof) || eof ||
|
if (!read_exact(fd, magic, BATCH_MAGIC_LEN, &eof) || eof ||
|
||||||
memcmp(magic, BATCH_MAGIC, BATCH_MAGIC_LEN) != 0) {
|
memcmp(magic, BATCH_MAGIC, BATCH_MAGIC_LEN) != 0) {
|
||||||
log_message(LOG_LEVEL_ERROR, "batch: malformed header (bad magic)");
|
log_message(LOG_LEVEL_ERROR, "batch: malformed header (bad magic)");
|
||||||
return -1;
|
goto done;
|
||||||
}
|
}
|
||||||
unsigned char version;
|
unsigned char version;
|
||||||
if (!read_exact(fd, &version, 1, &eof) || eof || version != BATCH_FORMAT_VERSION) {
|
if (!read_exact(fd, &version, 1, &eof) || eof || version != BATCH_FORMAT_VERSION) {
|
||||||
log_message(LOG_LEVEL_ERROR, "batch: malformed header (bad or missing format version)");
|
log_message(LOG_LEVEL_ERROR, "batch: malformed header (bad or missing format version)");
|
||||||
return -1;
|
goto done;
|
||||||
}
|
}
|
||||||
unsigned char mode;
|
unsigned char mode;
|
||||||
if (!read_exact(fd, &mode, 1, &eof) || eof || (mode != 0 && mode != 1)) {
|
if (!read_exact(fd, &mode, 1, &eof) || eof || (mode != 0 && mode != 1)) {
|
||||||
log_message(LOG_LEVEL_ERROR, "batch: malformed header (bad metadata flag)");
|
log_message(LOG_LEVEL_ERROR, "batch: malformed header (bad metadata flag)");
|
||||||
return -1;
|
goto done;
|
||||||
}
|
}
|
||||||
bool use_metadata = mode == 1;
|
bool use_metadata = mode == 1;
|
||||||
|
|
||||||
@@ -114,47 +134,62 @@ int batch_read_apply(int fd, const Config* config, const char* dest_root) {
|
|||||||
unsigned long long length;
|
unsigned long long length;
|
||||||
if (!read_exact(fd, &length, sizeof(length), &eof)) {
|
if (!read_exact(fd, &length, sizeof(length), &eof)) {
|
||||||
log_message(LOG_LEVEL_ERROR, "batch: truncated length prefix");
|
log_message(LOG_LEVEL_ERROR, "batch: truncated length prefix");
|
||||||
return -1;
|
goto done;
|
||||||
}
|
}
|
||||||
if (eof)
|
if (eof)
|
||||||
break; /* clean end of stream */
|
break; /* clean end of stream */
|
||||||
if (length == 0 || length > BATCH_MAX_RECORD) {
|
if (length == 0 || length > BATCH_MAX_RECORD) {
|
||||||
log_message(LOG_LEVEL_ERROR, "batch: rejected record length %llu (valid range 1..%llu)",
|
log_message(LOG_LEVEL_ERROR, "batch: rejected record length %llu (valid range 1..%llu)",
|
||||||
length, (unsigned long long)BATCH_MAX_RECORD);
|
length, (unsigned long long)BATCH_MAX_RECORD);
|
||||||
return -1;
|
goto done;
|
||||||
}
|
}
|
||||||
char* record = (char*)malloc((size_t)length);
|
char* record = (char*)malloc((size_t)length);
|
||||||
if (record == NULL) {
|
if (record == NULL) {
|
||||||
log_message(LOG_LEVEL_ERROR, "batch: could not allocate a %llu-byte record", length);
|
log_message(LOG_LEVEL_ERROR, "batch: could not allocate a %llu-byte record", length);
|
||||||
return -1;
|
goto done;
|
||||||
}
|
}
|
||||||
if (!read_exact(fd, record, (size_t)length, &eof) || eof) {
|
if (!read_exact(fd, record, (size_t)length, &eof) || eof) {
|
||||||
log_message(LOG_LEVEL_ERROR, "batch: truncated chunk record");
|
log_message(LOG_LEVEL_ERROR, "batch: truncated chunk record");
|
||||||
free(record);
|
free(record);
|
||||||
return -1;
|
goto done;
|
||||||
}
|
}
|
||||||
Data* data = data_create(record, (size_t)length);
|
Data* data = data_create(record, (size_t)length);
|
||||||
if (data == NULL)
|
if (data == NULL)
|
||||||
return -1; /* data_create frees `record` on failure */
|
goto done; /* data_create frees `record` on failure */
|
||||||
Chunk* chunk = chunk_deserialize(data, use_metadata);
|
Chunk* chunk = chunk_deserialize(data, use_metadata);
|
||||||
data_destroy(data);
|
data_destroy(data);
|
||||||
if (chunk == NULL) {
|
if (chunk == NULL) {
|
||||||
log_message(LOG_LEVEL_ERROR, "batch: rejected malformed chunk record");
|
log_message(LOG_LEVEL_ERROR, "batch: rejected malformed chunk record");
|
||||||
return -1;
|
goto done;
|
||||||
}
|
}
|
||||||
for (int i = 0; i < chunk->element_count; i++) {
|
for (int i = 0; i < chunk->element_count; i++) {
|
||||||
File* file = chunk->items[i];
|
File* file = chunk->items[i];
|
||||||
chunk->items[i] = NULL;
|
chunk->items[i] = NULL;
|
||||||
if (file == NULL)
|
if (file == NULL)
|
||||||
continue;
|
continue;
|
||||||
FileSaveResult result = file_save_to_disk_full(dest_root, file, config);
|
FileSaveResult save = file_save_to_disk_full(dest_root, file, config);
|
||||||
|
/* Accumulate directory metadata (when it applies) before the File is
|
||||||
|
* destroyed; applied once the whole stream has been consumed. */
|
||||||
|
if (save != FILE_SAVE_ERROR && file->is_dir && file->metadata &&
|
||||||
|
dir_metadata_should_capture(config) &&
|
||||||
|
!dir_time_list_add(&dir_times, file->path, file->metadata, file->xattrs)) {
|
||||||
file_destroy(file);
|
file_destroy(file);
|
||||||
if (result == FILE_SAVE_ERROR) {
|
|
||||||
chunk_destroy(chunk);
|
chunk_destroy(chunk);
|
||||||
return -1;
|
goto done;
|
||||||
|
}
|
||||||
|
file_destroy(file);
|
||||||
|
if (save == FILE_SAVE_ERROR) {
|
||||||
|
chunk_destroy(chunk);
|
||||||
|
goto done;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
chunk_destroy(chunk);
|
chunk_destroy(chunk);
|
||||||
}
|
}
|
||||||
return 0;
|
dir_metadata_list_apply(&dir_times, dest_root, config);
|
||||||
|
result = 0;
|
||||||
|
|
||||||
|
done:
|
||||||
|
identity_clear_active();
|
||||||
|
dir_time_list_free(&dir_times);
|
||||||
|
return result;
|
||||||
}
|
}
|
||||||
+328
-17
@@ -1,12 +1,170 @@
|
|||||||
#include "checksum.h"
|
#include "checksum.h"
|
||||||
|
#include <fcntl.h>
|
||||||
#include <openssl/evp.h>
|
#include <openssl/evp.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
#include <strings.h>
|
#include <strings.h>
|
||||||
|
#include <unistd.h>
|
||||||
|
|
||||||
/* delta.c owns the single XXH_IMPLEMENTATION that provides the xxHash symbols
|
/* delta.c owns the single XXH_IMPLEMENTATION that provides the xxHash symbols
|
||||||
* for the whole binary; this TU only needs the declarations. */
|
* for the whole binary; this TU only needs the declarations. The streaming
|
||||||
|
* state structs and XXH3_update are exposed only with XXH_STATIC_LINKING_ONLY. */
|
||||||
|
#define XXH_STATIC_LINKING_ONLY
|
||||||
#include <xxhash.h>
|
#include <xxhash.h>
|
||||||
|
|
||||||
|
/* ---------------------------------------------------------------------------
|
||||||
|
* Self-contained MD4 (RFC 1320). OpenSSL's MD4 lives in the legacy provider
|
||||||
|
* and is not guaranteed present, so FastSync carries its own implementation to
|
||||||
|
* keep --checksum-choice=md4 working on every build.
|
||||||
|
* ------------------------------------------------------------------------- */
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
uint32_t state[4];
|
||||||
|
uint64_t bit_count;
|
||||||
|
uint8_t buffer[64];
|
||||||
|
size_t buffer_len;
|
||||||
|
} Md4Ctx;
|
||||||
|
|
||||||
|
static uint32_t md4_rotl(uint32_t x, int n) {
|
||||||
|
return (x << n) | (x >> (32 - n));
|
||||||
|
}
|
||||||
|
|
||||||
|
static void md4_transform(uint32_t state[4], const uint8_t block[64]) {
|
||||||
|
uint32_t x[16];
|
||||||
|
for (int i = 0; i < 16; i++)
|
||||||
|
x[i] = (uint32_t)block[i * 4] | ((uint32_t)block[i * 4 + 1] << 8) |
|
||||||
|
((uint32_t)block[i * 4 + 2] << 16) | ((uint32_t)block[i * 4 + 3] << 24);
|
||||||
|
|
||||||
|
uint32_t a = state[0], b = state[1], c = state[2], d = state[3];
|
||||||
|
|
||||||
|
#define F(x, y, z) (((x) & (y)) | (~(x) & (z)))
|
||||||
|
#define G(x, y, z) (((x) & (y)) | ((x) & (z)) | ((y) & (z)))
|
||||||
|
#define H(x, y, z) ((x) ^ (y) ^ (z))
|
||||||
|
#define ROUND1(a, b, c, d, k, s) a = md4_rotl(a + F(b, c, d) + x[k], s)
|
||||||
|
#define ROUND2(a, b, c, d, k, s) a = md4_rotl(a + G(b, c, d) + x[k] + 0x5a827999u, s)
|
||||||
|
#define ROUND3(a, b, c, d, k, s) a = md4_rotl(a + H(b, c, d) + x[k] + 0x6ed9eba1u, s)
|
||||||
|
|
||||||
|
ROUND1(a, b, c, d, 0, 3);
|
||||||
|
ROUND1(d, a, b, c, 1, 7);
|
||||||
|
ROUND1(c, d, a, b, 2, 11);
|
||||||
|
ROUND1(b, c, d, a, 3, 19);
|
||||||
|
ROUND1(a, b, c, d, 4, 3);
|
||||||
|
ROUND1(d, a, b, c, 5, 7);
|
||||||
|
ROUND1(c, d, a, b, 6, 11);
|
||||||
|
ROUND1(b, c, d, a, 7, 19);
|
||||||
|
ROUND1(a, b, c, d, 8, 3);
|
||||||
|
ROUND1(d, a, b, c, 9, 7);
|
||||||
|
ROUND1(c, d, a, b, 10, 11);
|
||||||
|
ROUND1(b, c, d, a, 11, 19);
|
||||||
|
ROUND1(a, b, c, d, 12, 3);
|
||||||
|
ROUND1(d, a, b, c, 13, 7);
|
||||||
|
ROUND1(c, d, a, b, 14, 11);
|
||||||
|
ROUND1(b, c, d, a, 15, 19);
|
||||||
|
|
||||||
|
ROUND2(a, b, c, d, 0, 3);
|
||||||
|
ROUND2(d, a, b, c, 4, 5);
|
||||||
|
ROUND2(c, d, a, b, 8, 9);
|
||||||
|
ROUND2(b, c, d, a, 12, 13);
|
||||||
|
ROUND2(a, b, c, d, 1, 3);
|
||||||
|
ROUND2(d, a, b, c, 5, 5);
|
||||||
|
ROUND2(c, d, a, b, 9, 9);
|
||||||
|
ROUND2(b, c, d, a, 13, 13);
|
||||||
|
ROUND2(a, b, c, d, 2, 3);
|
||||||
|
ROUND2(d, a, b, c, 6, 5);
|
||||||
|
ROUND2(c, d, a, b, 10, 9);
|
||||||
|
ROUND2(b, c, d, a, 14, 13);
|
||||||
|
ROUND2(a, b, c, d, 3, 3);
|
||||||
|
ROUND2(d, a, b, c, 7, 5);
|
||||||
|
ROUND2(c, d, a, b, 11, 9);
|
||||||
|
ROUND2(b, c, d, a, 15, 13);
|
||||||
|
|
||||||
|
ROUND3(a, b, c, d, 0, 3);
|
||||||
|
ROUND3(d, a, b, c, 8, 9);
|
||||||
|
ROUND3(c, d, a, b, 4, 11);
|
||||||
|
ROUND3(b, c, d, a, 12, 15);
|
||||||
|
ROUND3(a, b, c, d, 2, 3);
|
||||||
|
ROUND3(d, a, b, c, 10, 9);
|
||||||
|
ROUND3(c, d, a, b, 6, 11);
|
||||||
|
ROUND3(b, c, d, a, 14, 15);
|
||||||
|
ROUND3(a, b, c, d, 1, 3);
|
||||||
|
ROUND3(d, a, b, c, 9, 9);
|
||||||
|
ROUND3(c, d, a, b, 5, 11);
|
||||||
|
ROUND3(b, c, d, a, 13, 15);
|
||||||
|
ROUND3(a, b, c, d, 3, 3);
|
||||||
|
ROUND3(d, a, b, c, 11, 9);
|
||||||
|
ROUND3(c, d, a, b, 7, 11);
|
||||||
|
ROUND3(b, c, d, a, 15, 15);
|
||||||
|
|
||||||
|
#undef F
|
||||||
|
#undef G
|
||||||
|
#undef H
|
||||||
|
#undef ROUND1
|
||||||
|
#undef ROUND2
|
||||||
|
#undef ROUND3
|
||||||
|
|
||||||
|
state[0] += a;
|
||||||
|
state[1] += b;
|
||||||
|
state[2] += c;
|
||||||
|
state[3] += d;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void md4_init(Md4Ctx* ctx) {
|
||||||
|
ctx->state[0] = 0x67452301u;
|
||||||
|
ctx->state[1] = 0xefcdab89u;
|
||||||
|
ctx->state[2] = 0x98badcfeu;
|
||||||
|
ctx->state[3] = 0x10325476u;
|
||||||
|
ctx->bit_count = 0;
|
||||||
|
ctx->buffer_len = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void md4_update(Md4Ctx* ctx, const uint8_t* data, size_t len) {
|
||||||
|
ctx->bit_count += (uint64_t)len * 8;
|
||||||
|
while (len > 0) {
|
||||||
|
size_t space = sizeof(ctx->buffer) - ctx->buffer_len;
|
||||||
|
size_t take = len < space ? len : space;
|
||||||
|
memcpy(ctx->buffer + ctx->buffer_len, data, take);
|
||||||
|
ctx->buffer_len += take;
|
||||||
|
data += take;
|
||||||
|
len -= take;
|
||||||
|
if (ctx->buffer_len == sizeof(ctx->buffer)) {
|
||||||
|
md4_transform(ctx->state, ctx->buffer);
|
||||||
|
ctx->buffer_len = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static void md4_final(Md4Ctx* ctx, uint8_t out[16]) {
|
||||||
|
uint64_t bit_count = ctx->bit_count;
|
||||||
|
uint8_t pad = 0x80;
|
||||||
|
md4_update(ctx, &pad, 1);
|
||||||
|
uint8_t zero = 0;
|
||||||
|
while (ctx->buffer_len != 56)
|
||||||
|
md4_update(ctx, &zero, 1);
|
||||||
|
uint8_t length_le[8];
|
||||||
|
for (int i = 0; i < 8; i++)
|
||||||
|
length_le[i] = (uint8_t)((bit_count >> (8 * i)) & 0xff);
|
||||||
|
md4_update(ctx, length_le, sizeof(length_le));
|
||||||
|
for (int i = 0; i < 4; i++) {
|
||||||
|
out[i * 4] = (uint8_t)(ctx->state[i] & 0xff);
|
||||||
|
out[i * 4 + 1] = (uint8_t)((ctx->state[i] >> 8) & 0xff);
|
||||||
|
out[i * 4 + 2] = (uint8_t)((ctx->state[i] >> 16) & 0xff);
|
||||||
|
out[i * 4 + 3] = (uint8_t)((ctx->state[i] >> 24) & 0xff);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* One-shot EVP digest (md5/sha1). Returns false when OpenSSL refuses. */
|
||||||
|
static bool evp_digest(const EVP_MD* md, const void* data, size_t size, uint8_t* out,
|
||||||
|
size_t out_capacity, size_t* out_len) {
|
||||||
|
static const uint8_t empty = 0;
|
||||||
|
const void* input = data ? data : ∅
|
||||||
|
unsigned int digest_len = 0;
|
||||||
|
if (EVP_Digest(input, size, out, &digest_len, md, NULL) != 1)
|
||||||
|
return false;
|
||||||
|
if (digest_len > out_capacity)
|
||||||
|
return false;
|
||||||
|
*out_len = digest_len;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
bool checksum_digest(ChecksumAlgo algo, uint64_t seed, const void* data, size_t size, uint8_t* out,
|
bool checksum_digest(ChecksumAlgo algo, uint64_t seed, const void* data, size_t size, uint8_t* out,
|
||||||
size_t out_capacity, size_t* out_len) {
|
size_t out_capacity, size_t* out_len) {
|
||||||
if (!out || !out_len || out_capacity < CHECKSUM_MAX_DIGEST_LEN)
|
if (!out || !out_len || out_capacity < CHECKSUM_MAX_DIGEST_LEN)
|
||||||
@@ -14,39 +172,158 @@ bool checksum_digest(ChecksumAlgo algo, uint64_t seed, const void* data, size_t
|
|||||||
if (data == NULL && size != 0)
|
if (data == NULL && size != 0)
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
if (algo == CHECKSUM_ALGO_XXH64) {
|
switch (algo) {
|
||||||
|
case CHECKSUM_ALGO_XXH64: {
|
||||||
uint64_t digest = XXH64(data, size, seed);
|
uint64_t digest = XXH64(data, size, seed);
|
||||||
memcpy(out, &digest, sizeof(digest));
|
memcpy(out, &digest, sizeof(digest));
|
||||||
*out_len = sizeof(digest);
|
*out_len = sizeof(digest);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
case CHECKSUM_ALGO_XXH3: {
|
||||||
if (algo == CHECKSUM_ALGO_MD5) {
|
uint64_t digest = XXH3_64bits_withSeed(data, size, seed);
|
||||||
|
memcpy(out, &digest, sizeof(digest));
|
||||||
|
*out_len = sizeof(digest);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
case CHECKSUM_ALGO_XXH128: {
|
||||||
|
XXH128_hash_t digest = XXH3_128bits_withSeed(data, size, seed);
|
||||||
|
memcpy(out, &digest, sizeof(digest));
|
||||||
|
*out_len = sizeof(digest);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
case CHECKSUM_ALGO_MD5:
|
||||||
/* md5 takes no seed; the caller's seed is deliberately ignored (documented
|
/* md5 takes no seed; the caller's seed is deliberately ignored (documented
|
||||||
* in RSYNC_COMPAT.md). OpenSSL's one-shot EVP_Digest needs a non-NULL
|
* in RSYNC_COMPAT.md). */
|
||||||
* buffer even for an empty input, so map a NULL data + size==0 to an empty
|
return evp_digest(EVP_md5(), data, size, out, out_capacity, out_len);
|
||||||
* buffer. */
|
case CHECKSUM_ALGO_MD4: {
|
||||||
static const uint8_t empty = 0;
|
Md4Ctx ctx;
|
||||||
const void* input = data ? data : ∅
|
md4_init(&ctx);
|
||||||
unsigned int digest_len = 0;
|
md4_update(&ctx, (const uint8_t*)data, size);
|
||||||
if (EVP_Digest(input, size, out, &digest_len, EVP_md5(), NULL) != 1)
|
md4_final(&ctx, out);
|
||||||
return false;
|
*out_len = 16;
|
||||||
if (digest_len > out_capacity)
|
return true;
|
||||||
return false;
|
}
|
||||||
*out_len = digest_len;
|
case CHECKSUM_ALGO_SHA1:
|
||||||
|
/* sha1 takes no seed; the caller's seed is deliberately ignored. */
|
||||||
|
return evp_digest(EVP_sha1(), data, size, out, out_capacity, out_len);
|
||||||
|
case CHECKSUM_ALGO_NONE:
|
||||||
|
/* No checksum requested: an empty digest is the successful result. */
|
||||||
|
*out_len = 0;
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool checksum_digest_file(ChecksumAlgo algo, uint64_t seed, const char* path, uint8_t* out,
|
||||||
|
size_t out_capacity, size_t* out_len) {
|
||||||
|
if (!path || !out || !out_len || out_capacity < CHECKSUM_MAX_DIGEST_LEN)
|
||||||
|
return false;
|
||||||
|
|
||||||
|
int fd = open(path, O_RDONLY | O_CLOEXEC);
|
||||||
|
if (fd < 0)
|
||||||
|
return false;
|
||||||
|
|
||||||
|
uint8_t buffer[64 * 1024];
|
||||||
|
bool ok = false;
|
||||||
|
|
||||||
|
if (algo == CHECKSUM_ALGO_MD5) {
|
||||||
|
EVP_MD_CTX* ctx = EVP_MD_CTX_new();
|
||||||
|
if (!ctx) {
|
||||||
|
close(fd);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
unsigned int digest_len = 0;
|
||||||
|
if (EVP_DigestInit_ex(ctx, EVP_md5(), NULL) == 1) {
|
||||||
|
ok = true;
|
||||||
|
ssize_t got;
|
||||||
|
while ((got = read(fd, buffer, sizeof(buffer))) > 0) {
|
||||||
|
if (EVP_DigestUpdate(ctx, buffer, (size_t)got) != 1) {
|
||||||
|
ok = false;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (got < 0)
|
||||||
|
ok = false;
|
||||||
|
if (ok && EVP_DigestFinal_ex(ctx, out, &digest_len) == 1 && digest_len <= out_capacity)
|
||||||
|
*out_len = digest_len;
|
||||||
|
else
|
||||||
|
ok = false;
|
||||||
|
}
|
||||||
|
EVP_MD_CTX_free(ctx);
|
||||||
|
close(fd);
|
||||||
|
return ok;
|
||||||
|
}
|
||||||
|
|
||||||
|
XXH64_state_t xxh64;
|
||||||
|
XXH3_state_t* xxh3 = NULL;
|
||||||
|
if (algo == CHECKSUM_ALGO_XXH64) {
|
||||||
|
XXH64_reset(&xxh64, seed);
|
||||||
|
} else if (algo == CHECKSUM_ALGO_XXH3 || algo == CHECKSUM_ALGO_XXH128) {
|
||||||
|
xxh3 = XXH3_createState();
|
||||||
|
if (!xxh3) {
|
||||||
|
close(fd);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (algo == CHECKSUM_ALGO_XXH3)
|
||||||
|
XXH3_64bits_reset_withSeed(xxh3, seed);
|
||||||
|
else
|
||||||
|
XXH3_128bits_reset_withSeed(xxh3, seed);
|
||||||
|
} else {
|
||||||
|
close(fd);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
ok = true;
|
||||||
|
ssize_t got;
|
||||||
|
while ((got = read(fd, buffer, sizeof(buffer))) > 0) {
|
||||||
|
if (algo == CHECKSUM_ALGO_XXH64)
|
||||||
|
XXH64_update(&xxh64, buffer, (size_t)got);
|
||||||
|
else if (XXH3_64bits_update(xxh3, buffer, (size_t)got) == XXH_ERROR) {
|
||||||
|
ok = false;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (got < 0)
|
||||||
|
ok = false;
|
||||||
|
|
||||||
|
if (ok) {
|
||||||
|
if (algo == CHECKSUM_ALGO_XXH64) {
|
||||||
|
uint64_t digest = XXH64_digest(&xxh64);
|
||||||
|
memcpy(out, &digest, sizeof(digest));
|
||||||
|
*out_len = sizeof(digest);
|
||||||
|
} else if (algo == CHECKSUM_ALGO_XXH3) {
|
||||||
|
uint64_t digest = XXH3_64bits_digest(xxh3);
|
||||||
|
memcpy(out, &digest, sizeof(digest));
|
||||||
|
*out_len = sizeof(digest);
|
||||||
|
} else {
|
||||||
|
XXH128_hash_t digest = XXH3_128bits_digest(xxh3);
|
||||||
|
memcpy(out, &digest, sizeof(digest));
|
||||||
|
*out_len = sizeof(digest);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (xxh3)
|
||||||
|
XXH3_freeState(xxh3);
|
||||||
|
close(fd);
|
||||||
|
return ok;
|
||||||
|
}
|
||||||
|
|
||||||
int checksum_algo_from_name(const char* name) {
|
int checksum_algo_from_name(const char* name) {
|
||||||
if (!name)
|
if (!name)
|
||||||
return -1;
|
return -1;
|
||||||
if (strcasecmp(name, "xxh64") == 0 || strcasecmp(name, "xxhash") == 0)
|
if (strcasecmp(name, "xxh64") == 0 || strcasecmp(name, "xxhash") == 0)
|
||||||
return (int)CHECKSUM_ALGO_XXH64;
|
return (int)CHECKSUM_ALGO_XXH64;
|
||||||
|
if (strcasecmp(name, "xxh3") == 0)
|
||||||
|
return (int)CHECKSUM_ALGO_XXH3;
|
||||||
|
if (strcasecmp(name, "xxh128") == 0)
|
||||||
|
return (int)CHECKSUM_ALGO_XXH128;
|
||||||
if (strcasecmp(name, "md5") == 0)
|
if (strcasecmp(name, "md5") == 0)
|
||||||
return (int)CHECKSUM_ALGO_MD5;
|
return (int)CHECKSUM_ALGO_MD5;
|
||||||
|
if (strcasecmp(name, "md4") == 0)
|
||||||
|
return (int)CHECKSUM_ALGO_MD4;
|
||||||
|
if (strcasecmp(name, "sha1") == 0)
|
||||||
|
return (int)CHECKSUM_ALGO_SHA1;
|
||||||
|
if (strcasecmp(name, "none") == 0)
|
||||||
|
return (int)CHECKSUM_ALGO_NONE;
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -54,22 +331,56 @@ const char* checksum_algo_name(ChecksumAlgo algo) {
|
|||||||
switch (algo) {
|
switch (algo) {
|
||||||
case CHECKSUM_ALGO_XXH64:
|
case CHECKSUM_ALGO_XXH64:
|
||||||
return "xxh64";
|
return "xxh64";
|
||||||
|
case CHECKSUM_ALGO_XXH3:
|
||||||
|
return "xxh3";
|
||||||
|
case CHECKSUM_ALGO_XXH128:
|
||||||
|
return "xxh128";
|
||||||
case CHECKSUM_ALGO_MD5:
|
case CHECKSUM_ALGO_MD5:
|
||||||
return "md5";
|
return "md5";
|
||||||
|
case CHECKSUM_ALGO_MD4:
|
||||||
|
return "md4";
|
||||||
|
case CHECKSUM_ALGO_SHA1:
|
||||||
|
return "sha1";
|
||||||
|
case CHECKSUM_ALGO_NONE:
|
||||||
|
return "none";
|
||||||
}
|
}
|
||||||
return "<unknown>";
|
return "<unknown>";
|
||||||
}
|
}
|
||||||
|
|
||||||
bool checksum_algo_valid(int algo) {
|
bool checksum_algo_valid(int algo) {
|
||||||
return algo == (int)CHECKSUM_ALGO_XXH64 || algo == (int)CHECKSUM_ALGO_MD5;
|
return algo == (int)CHECKSUM_ALGO_XXH64 || algo == (int)CHECKSUM_ALGO_MD5 ||
|
||||||
|
algo == (int)CHECKSUM_ALGO_XXH3 || algo == (int)CHECKSUM_ALGO_XXH128 ||
|
||||||
|
algo == (int)CHECKSUM_ALGO_MD4 || algo == (int)CHECKSUM_ALGO_SHA1 ||
|
||||||
|
algo == (int)CHECKSUM_ALGO_NONE;
|
||||||
}
|
}
|
||||||
|
|
||||||
uint8_t checksum_digest_len(ChecksumAlgo algo) {
|
uint8_t checksum_digest_len(ChecksumAlgo algo) {
|
||||||
switch (algo) {
|
switch (algo) {
|
||||||
case CHECKSUM_ALGO_XXH64:
|
case CHECKSUM_ALGO_XXH64:
|
||||||
|
case CHECKSUM_ALGO_XXH3:
|
||||||
return 8;
|
return 8;
|
||||||
|
case CHECKSUM_ALGO_XXH128:
|
||||||
case CHECKSUM_ALGO_MD5:
|
case CHECKSUM_ALGO_MD5:
|
||||||
|
case CHECKSUM_ALGO_MD4:
|
||||||
return 16;
|
return 16;
|
||||||
|
case CHECKSUM_ALGO_SHA1:
|
||||||
|
return 20;
|
||||||
|
case CHECKSUM_ALGO_NONE:
|
||||||
|
return 0;
|
||||||
}
|
}
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
ChecksumAlgo checksum_negotiate_default(void) {
|
||||||
|
/* rsync 3.4.1 default preference order; every entry is compiled in, so this
|
||||||
|
* resolves to xxh128. */
|
||||||
|
static const ChecksumAlgo preference[] = {
|
||||||
|
CHECKSUM_ALGO_XXH128, CHECKSUM_ALGO_XXH3, CHECKSUM_ALGO_XXH64, CHECKSUM_ALGO_MD5,
|
||||||
|
CHECKSUM_ALGO_MD4, CHECKSUM_ALGO_SHA1, CHECKSUM_ALGO_NONE,
|
||||||
|
};
|
||||||
|
for (size_t i = 0; i < sizeof(preference) / sizeof(preference[0]); i++) {
|
||||||
|
if (checksum_algo_valid((int)preference[i]))
|
||||||
|
return preference[i];
|
||||||
|
}
|
||||||
|
return CHECKSUM_ALGO_XXH64;
|
||||||
|
}
|
||||||
|
|||||||
+40
-10
@@ -8,18 +8,35 @@
|
|||||||
/* Whole-file content-digest algorithms selectable with --checksum-choice and
|
/* Whole-file content-digest algorithms selectable with --checksum-choice and
|
||||||
* seeded with --checksum-seed. The ids are the values actually placed on the
|
* seeded with --checksum-seed. The ids are the values actually placed on the
|
||||||
* wire (config frame), so they must be kept stable and validated on receive.
|
* wire (config frame), so they must be kept stable and validated on receive.
|
||||||
* CHECKSUM_ALGO_XXH64 == 0 is the default and is byte-for-byte what FastSync
|
* CHECKSUM_ALGO_XXH64 == 0 is the historical FastSync default and its numeric
|
||||||
* computed before these options existed (xxHash64 with seed 0). */
|
* value is preserved. The full set mirrors the algorithms rsync 3.4.1 can be
|
||||||
typedef enum { CHECKSUM_ALGO_XXH64 = 0, CHECKSUM_ALGO_MD5 = 1 } ChecksumAlgo;
|
* built with; every one of them is implemented here. */
|
||||||
|
typedef enum {
|
||||||
|
CHECKSUM_ALGO_XXH64 = 0,
|
||||||
|
CHECKSUM_ALGO_MD5 = 1,
|
||||||
|
CHECKSUM_ALGO_XXH3 = 2,
|
||||||
|
CHECKSUM_ALGO_XXH128 = 3,
|
||||||
|
CHECKSUM_ALGO_MD4 = 4,
|
||||||
|
CHECKSUM_ALGO_SHA1 = 5,
|
||||||
|
CHECKSUM_ALGO_NONE = 6
|
||||||
|
} ChecksumAlgo;
|
||||||
|
|
||||||
/* md5 digest is 16 bytes, the longest supported. */
|
/* FastSync's negotiated default (rsync 3.4.1 auto-negotiates xxh128 first).
|
||||||
#define CHECKSUM_MAX_DIGEST_LEN 16
|
* The wire default for Config->checksum_algo is this value. */
|
||||||
|
#define CHECKSUM_ALGO_DEFAULT CHECKSUM_ALGO_XXH128
|
||||||
|
|
||||||
|
/* sha1 digest is 20 bytes, the longest supported. */
|
||||||
|
#define CHECKSUM_MAX_DIGEST_LEN 20
|
||||||
|
|
||||||
/* Compute the whole-file digest of the first `size` bytes of `data`.
|
/* Compute the whole-file digest of the first `size` bytes of `data`.
|
||||||
*
|
*
|
||||||
* - CHECKSUM_ALGO_XXH64: xxHash64(data, size, seed) (full 64-bit seed).
|
* - CHECKSUM_ALGO_XXH64: xxHash64(data, size, seed) (full 64-bit seed).
|
||||||
* - CHECKSUM_ALGO_MD5: md5(data, size) via OpenSSL EVP.
|
* - CHECKSUM_ALGO_XXH3: XXH3_64bits_withSeed(data, size, seed).
|
||||||
* md5 has no seed, so `seed` is ignored (documented).
|
* - CHECKSUM_ALGO_XXH128: XXH3_128bits_withSeed(data, size, seed).
|
||||||
|
* - CHECKSUM_ALGO_MD5: md5(data, size) via OpenSSL EVP (seed ignored).
|
||||||
|
* - CHECKSUM_ALGO_MD4: md4(data, size), self-contained RFC 1320 (seed ignored).
|
||||||
|
* - CHECKSUM_ALGO_SHA1: sha1(data, size) via OpenSSL EVP (seed ignored).
|
||||||
|
* - CHECKSUM_ALGO_NONE: no digest; *out_len is 0 and nothing is written.
|
||||||
* - `size == 0` hashes the empty input (plus its seed), not a NULL input.
|
* - `size == 0` hashes the empty input (plus its seed), not a NULL input.
|
||||||
*
|
*
|
||||||
* Writes up to `out_capacity` bytes into `out`, storing the digest length in
|
* Writes up to `out_capacity` bytes into `out`, storing the digest length in
|
||||||
@@ -28,9 +45,16 @@ typedef enum { CHECKSUM_ALGO_XXH64 = 0, CHECKSUM_ALGO_MD5 = 1 } ChecksumAlgo;
|
|||||||
bool checksum_digest(ChecksumAlgo algo, uint64_t seed, const void* data, size_t size, uint8_t* out,
|
bool checksum_digest(ChecksumAlgo algo, uint64_t seed, const void* data, size_t size, uint8_t* out,
|
||||||
size_t out_capacity, size_t* out_len);
|
size_t out_capacity, size_t* out_len);
|
||||||
|
|
||||||
|
/* Streaming whole-file digest: hash the contents of `path` without holding the
|
||||||
|
* whole file in memory. Same digest/capacity contract as checksum_digest.
|
||||||
|
* Returns false on open/read failure or an undersized buffer. */
|
||||||
|
bool checksum_digest_file(ChecksumAlgo algo, uint64_t seed, const char* path, uint8_t* out,
|
||||||
|
size_t out_capacity, size_t* out_len);
|
||||||
|
|
||||||
/* Resolve a --checksum-choice string (case-insensitive) to an algorithm id.
|
/* Resolve a --checksum-choice string (case-insensitive) to an algorithm id.
|
||||||
* Accepts "xxh64" and "xxhash" (both map to CHECKSUM_ALGO_XXH64, rsync's
|
* Accepts "xxh64"/"xxhash", "xxh3", "xxh128", "md5", "md4", "sha1", "none".
|
||||||
* xxhash spelling) and "md5". Returns -1 for any unsupported name. */
|
* "auto" is not an algorithm here; the caller resolves it to the negotiated
|
||||||
|
* default. Returns -1 for any unrecognized name. */
|
||||||
int checksum_algo_from_name(const char* name);
|
int checksum_algo_from_name(const char* name);
|
||||||
|
|
||||||
/* Canonical name of an algorithm (used in CLI error messages). */
|
/* Canonical name of an algorithm (used in CLI error messages). */
|
||||||
@@ -39,7 +63,13 @@ const char* checksum_algo_name(ChecksumAlgo algo);
|
|||||||
/* True when `algo` is a supported id (used by config receive validation). */
|
/* True when `algo` is a supported id (used by config receive validation). */
|
||||||
bool checksum_algo_valid(int algo);
|
bool checksum_algo_valid(int algo);
|
||||||
|
|
||||||
/* Digest length in bytes for an algorithm (xxx64 = 8, md5 = 16). */
|
/* Digest length in bytes for an algorithm (xxh64/xxh3 = 8,
|
||||||
|
* md5/md4/xxh128 = 16, sha1 = 20, none = 0). */
|
||||||
uint8_t checksum_digest_len(ChecksumAlgo algo);
|
uint8_t checksum_digest_len(ChecksumAlgo algo);
|
||||||
|
|
||||||
|
/* Pick the first algorithm from FastSync's compiled-in preference list that is
|
||||||
|
* supported on this build (rsync 3.4.1's `--version` order:
|
||||||
|
* xxh128 xxh3 xxh64 md5 md4 sha1 none). Used to resolve "auto". */
|
||||||
|
ChecksumAlgo checksum_negotiate_default(void);
|
||||||
|
|
||||||
#endif /* CHECKSUM_H */
|
#endif /* CHECKSUM_H */
|
||||||
+170
-77
@@ -1,90 +1,183 @@
|
|||||||
#include "chmod.h"
|
#include "chmod.h"
|
||||||
|
#include "file.h"
|
||||||
#include <stddef.h>
|
#include <stddef.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
|
|
||||||
static bool parse_clause(mode_t* mode, const char* begin, const char* end) {
|
/* rsync's --chmod parser (parse_chmod + tweak_mode). A single clause is
|
||||||
const char* p = begin;
|
* applied as it is completed, so repeated clauses and repeated --chmod options
|
||||||
unsigned who = 0;
|
* (joined with commas by the CLI) accumulate exactly like rsync. The D/F
|
||||||
while (p < end && strchr("ugoa", *p)) {
|
* selectors restrict a clause to directories/files; X adds execute only to
|
||||||
if (*p == 'a')
|
* directories or files that were already executable. */
|
||||||
who = 7;
|
|
||||||
else
|
#define CHMOD_BITS 07777
|
||||||
who |= *p == 'u' ? 1U : (*p == 'g' ? 2U : 4U);
|
#define CHMOD_FLAG_X_KEEP (1U << 0)
|
||||||
p++;
|
#define CHMOD_FLAG_DIRS_ONLY (1U << 1)
|
||||||
}
|
#define CHMOD_FLAG_FILES_ONLY (1U << 2)
|
||||||
if (who == 0)
|
|
||||||
who = 7;
|
enum chmod_op { CHMOD_OP_ADD = 1, CHMOD_OP_SUB, CHMOD_OP_EQ, CHMOD_OP_SET };
|
||||||
if (p == end || (*p != '+' && *p != '-' && *p != '='))
|
enum chmod_state {
|
||||||
return false;
|
CHMOD_STATE_ERROR,
|
||||||
char operation = *p++;
|
CHMOD_STATE_1ST_HALF,
|
||||||
mode_t bits = 0;
|
CHMOD_STATE_2ND_HALF,
|
||||||
while (p < end) {
|
CHMOD_STATE_OCTAL
|
||||||
mode_t bit;
|
};
|
||||||
switch (*p++) {
|
|
||||||
case 'r':
|
|
||||||
bit = 4;
|
|
||||||
break;
|
|
||||||
case 'w':
|
|
||||||
bit = 2;
|
|
||||||
break;
|
|
||||||
case 'x':
|
|
||||||
bit = 1;
|
|
||||||
break;
|
|
||||||
default:
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
bits |= bit;
|
|
||||||
}
|
|
||||||
for (unsigned class_index = 0; class_index < 3; class_index++) {
|
|
||||||
unsigned class_bit = 1U << class_index;
|
|
||||||
if (!(who & class_bit))
|
|
||||||
continue;
|
|
||||||
mode_t shift = (mode_t)((2U - class_index) * 3U);
|
|
||||||
mode_t mask = (mode_t)(7U << shift);
|
|
||||||
mode_t class_bits = (mode_t)(bits << shift);
|
|
||||||
if (operation == '+')
|
|
||||||
*mode |= class_bits;
|
|
||||||
else if (operation == '-')
|
|
||||||
*mode &= ~class_bits;
|
|
||||||
else
|
|
||||||
*mode = (*mode & ~mask) | class_bits;
|
|
||||||
}
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
bool chmod_apply(mode_t mode, const char* spec, mode_t* result) {
|
bool chmod_apply(mode_t mode, const char* spec, mode_t* result) {
|
||||||
if (!spec || !*spec || !result)
|
if (!spec || !*spec || !result)
|
||||||
return false;
|
return false;
|
||||||
bool numeric = true;
|
const mode_t nonperm = mode & ~(mode_t)CHMOD_BITS;
|
||||||
size_t length = strlen(spec);
|
const bool initially_executable = (mode & 0111) != 0;
|
||||||
if (length > 4)
|
|
||||||
numeric = false;
|
|
||||||
for (size_t i = 0; i < length && numeric; i++)
|
|
||||||
numeric = spec[i] >= '0' && spec[i] <= '7';
|
|
||||||
if (numeric) {
|
|
||||||
if (length == 0 || length > 4)
|
|
||||||
return false;
|
|
||||||
mode_t parsed = 0;
|
|
||||||
for (size_t i = 0; i < length; i++)
|
|
||||||
parsed = (mode_t)((parsed << 3) | (spec[i] - '0'));
|
|
||||||
*result = parsed;
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
mode_t changed = mode;
|
mode_t changed = mode;
|
||||||
const char* begin = spec;
|
int state = CHMOD_STATE_1ST_HALF;
|
||||||
while (*begin) {
|
unsigned where = 0;
|
||||||
const char* end = strchr(begin, ',');
|
int what = 0, op = 0, topbits = 0, topoct = 0, flags = 0;
|
||||||
if (!end)
|
const char* p = spec;
|
||||||
end = begin + strlen(begin);
|
while (state != CHMOD_STATE_ERROR) {
|
||||||
if (!parse_clause(&changed, begin, end))
|
if (*p == '\0' || *p == ',') {
|
||||||
return false;
|
int bits;
|
||||||
if (*end == '\0')
|
if (!op) {
|
||||||
|
state = CHMOD_STATE_ERROR;
|
||||||
break;
|
break;
|
||||||
begin = end + 1;
|
|
||||||
if (!*begin)
|
|
||||||
return false;
|
|
||||||
}
|
}
|
||||||
*result = changed;
|
if (where)
|
||||||
|
bits = (int)(where * (unsigned)what);
|
||||||
|
else {
|
||||||
|
where = 0111;
|
||||||
|
bits = (int)((where * (unsigned)what) & ~(unsigned)file_process_umask());
|
||||||
|
}
|
||||||
|
int mode_and, mode_or;
|
||||||
|
switch (op) {
|
||||||
|
case CHMOD_OP_ADD:
|
||||||
|
mode_and = CHMOD_BITS;
|
||||||
|
mode_or = bits + topoct;
|
||||||
|
break;
|
||||||
|
case CHMOD_OP_SUB:
|
||||||
|
mode_and = CHMOD_BITS - bits - topoct;
|
||||||
|
mode_or = 0;
|
||||||
|
break;
|
||||||
|
case CHMOD_OP_EQ:
|
||||||
|
mode_and = CHMOD_BITS - (int)(where * 7U) - (topoct ? topbits : 0);
|
||||||
|
mode_or = bits + topoct;
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
mode_and = 0;
|
||||||
|
mode_or = bits;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
bool is_dir = S_ISDIR(nonperm);
|
||||||
|
if (!((flags & CHMOD_FLAG_DIRS_ONLY) && !is_dir) &&
|
||||||
|
!((flags & CHMOD_FLAG_FILES_ONLY) && is_dir)) {
|
||||||
|
changed &= (mode_t)mode_and;
|
||||||
|
if ((flags & CHMOD_FLAG_X_KEEP) && !initially_executable && !is_dir)
|
||||||
|
changed |= (mode_t)(mode_or & ~0111);
|
||||||
|
else
|
||||||
|
changed |= (mode_t)mode_or;
|
||||||
|
}
|
||||||
|
if (*p == '\0')
|
||||||
|
break;
|
||||||
|
p++;
|
||||||
|
state = CHMOD_STATE_1ST_HALF;
|
||||||
|
where = 0;
|
||||||
|
what = op = topoct = topbits = flags = 0;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
switch (state) {
|
||||||
|
case CHMOD_STATE_1ST_HALF:
|
||||||
|
switch (*p) {
|
||||||
|
case 'D':
|
||||||
|
if (flags & CHMOD_FLAG_FILES_ONLY) {
|
||||||
|
state = CHMOD_STATE_ERROR;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
flags |= CHMOD_FLAG_DIRS_ONLY;
|
||||||
|
break;
|
||||||
|
case 'F':
|
||||||
|
if (flags & CHMOD_FLAG_DIRS_ONLY) {
|
||||||
|
state = CHMOD_STATE_ERROR;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
flags |= CHMOD_FLAG_FILES_ONLY;
|
||||||
|
break;
|
||||||
|
case 'u':
|
||||||
|
where |= 0100;
|
||||||
|
topbits |= 04000;
|
||||||
|
break;
|
||||||
|
case 'g':
|
||||||
|
where |= 0010;
|
||||||
|
topbits |= 02000;
|
||||||
|
break;
|
||||||
|
case 'o':
|
||||||
|
where |= 0001;
|
||||||
|
break;
|
||||||
|
case 'a':
|
||||||
|
where |= 0111;
|
||||||
|
break;
|
||||||
|
case '+':
|
||||||
|
op = CHMOD_OP_ADD;
|
||||||
|
state = CHMOD_STATE_2ND_HALF;
|
||||||
|
break;
|
||||||
|
case '-':
|
||||||
|
op = CHMOD_OP_SUB;
|
||||||
|
state = CHMOD_STATE_2ND_HALF;
|
||||||
|
break;
|
||||||
|
case '=':
|
||||||
|
op = CHMOD_OP_EQ;
|
||||||
|
state = CHMOD_STATE_2ND_HALF;
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
if (*p >= '0' && *p <= '7' && !where) {
|
||||||
|
op = CHMOD_OP_SET;
|
||||||
|
state = CHMOD_STATE_OCTAL;
|
||||||
|
where = 1;
|
||||||
|
what = *p - '0';
|
||||||
|
} else {
|
||||||
|
state = CHMOD_STATE_ERROR;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
case CHMOD_STATE_2ND_HALF:
|
||||||
|
switch (*p) {
|
||||||
|
case 'r':
|
||||||
|
what |= 4;
|
||||||
|
break;
|
||||||
|
case 'w':
|
||||||
|
what |= 2;
|
||||||
|
break;
|
||||||
|
case 'X':
|
||||||
|
flags |= CHMOD_FLAG_X_KEEP;
|
||||||
|
/* fall through */
|
||||||
|
case 'x':
|
||||||
|
what |= 1;
|
||||||
|
break;
|
||||||
|
case 's':
|
||||||
|
if (topbits)
|
||||||
|
topoct |= topbits;
|
||||||
|
else
|
||||||
|
topoct = 04000;
|
||||||
|
break;
|
||||||
|
case 't':
|
||||||
|
topoct |= 01000;
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
state = CHMOD_STATE_ERROR;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
if (*p >= '0' && *p <= '7') {
|
||||||
|
what = what * 8 + (*p - '0');
|
||||||
|
if (what > CHMOD_BITS)
|
||||||
|
state = CHMOD_STATE_ERROR;
|
||||||
|
} else {
|
||||||
|
state = CHMOD_STATE_ERROR;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
p++;
|
||||||
|
}
|
||||||
|
if (state == CHMOD_STATE_ERROR)
|
||||||
|
return false;
|
||||||
|
*result = (changed & (mode_t)CHMOD_BITS) | nonperm;
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|||||||
+4
-1
@@ -4,7 +4,10 @@
|
|||||||
#include <stdbool.h>
|
#include <stdbool.h>
|
||||||
#include <sys/stat.h>
|
#include <sys/stat.h>
|
||||||
|
|
||||||
/* Apply the supported rsync --chmod syntax to a permission mode. */
|
/* Apply rsync's --chmod syntax to a permission mode, including the D/F/X
|
||||||
|
* selectors and the s/t special bits. `mode` should carry the file type bits
|
||||||
|
* (S_IFDIR/S_IFREG) so D/F/X can be evaluated; the type bits are preserved in
|
||||||
|
* `result`. A spec may contain comma-separated clauses, which accumulate. */
|
||||||
bool chmod_apply(mode_t mode, const char* spec, mode_t* result);
|
bool chmod_apply(mode_t mode, const char* spec, mode_t* result);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
+335
-34
@@ -3,39 +3,145 @@
|
|||||||
#include "log.h"
|
#include "log.h"
|
||||||
#include "protocol.h"
|
#include "protocol.h"
|
||||||
#include <limits.h>
|
#include <limits.h>
|
||||||
|
#include <lz4.h>
|
||||||
|
#include <stdatomic.h>
|
||||||
#include <stdint.h>
|
#include <stdint.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
#include <strings.h>
|
#include <strings.h>
|
||||||
#include <threads.h>
|
#include <threads.h>
|
||||||
#include <unistd.h>
|
#include <unistd.h>
|
||||||
|
#include <zlib.h>
|
||||||
#include <zstd.h>
|
#include <zstd.h>
|
||||||
|
|
||||||
#define INITIAL_DECOMPRESS_BUF_SIZE (1024 * 1024)
|
#define INITIAL_DECOMPRESS_BUF_SIZE (1024 * 1024)
|
||||||
#define MAX_DECOMPRESSED_SIZE (100ULL * 1024 * 1024) /* 100 MB hard ceiling */
|
#define MAX_DECOMPRESSED_SIZE (100ULL * 1024 * 1024) /* 100 MB hard ceiling */
|
||||||
|
|
||||||
static char* SKIP_COMPRESSION_EXTENSIONS[] = {".jpg", ".jpeg", ".png", ".gif", ".mp4", ".mkv",
|
/* rsync 3.4.1's built-in skip-compress suffix list (the `--skip-compress`
|
||||||
".zip", ".gz", ".xz", ".zst", NULL};
|
* defaults, in the man page's order). rsync stores it as space-separated
|
||||||
|
* "*.suffix" globs; FastSync matches the plain suffix after the final dot, so
|
||||||
|
* the leading "*." is omitted here. A user --skip-compress list replaces this
|
||||||
|
* default entirely (matching rsync). */
|
||||||
|
#define DEFAULT_SKIP_COMPRESS_SUFFIXES \
|
||||||
|
"3g2 3gp 7z aac ace apk avi bz2 deb dmg ear f4v flac flv gpg gz iso jar jpeg jpg lrz lz lz4 " \
|
||||||
|
"lzma " \
|
||||||
|
"lzo m1a m1v m2a m2ts m2v m4a m4b m4p m4r m4v mka mkv mov mp1 mp2 mp3 mp4 mpa mpeg mpg mpv mts " \
|
||||||
|
"odb odf odg odi odm odp ods odt oga ogg ogm ogv ogx opus otg oth otp ots ott oxt png qt rar " \
|
||||||
|
"rpm " \
|
||||||
|
"rz rzip spx squashfs sxc sxd sxg sxm sxw sz tbz tbz2 tgz tlz ts txz tzo vob war webm webp xz " \
|
||||||
|
"z " \
|
||||||
|
"zip zst"
|
||||||
|
|
||||||
|
/* Self-describing compressed frames: the first byte is the CompressionAlgo id.
|
||||||
|
* zlib/lz4 store the uncompressed size as a little-endian uint32 after the
|
||||||
|
* codec byte so decompression can be exactly pre-sized and bounded. */
|
||||||
|
#define LZ4_SIZE_PREFIX_LEN 4
|
||||||
|
|
||||||
|
static _Atomic int g_compression_algo = COMPRESSION_ALGO_ZSTD;
|
||||||
|
|
||||||
|
/* Case-insensitive match of a bare suffix (no leading dot) against a
|
||||||
|
* space-separated suffix list. */
|
||||||
|
static bool suffix_in_list(const char* name, const char* list) {
|
||||||
|
size_t name_len = strlen(name);
|
||||||
|
while (*list) {
|
||||||
|
while (*list == ' ')
|
||||||
|
list++;
|
||||||
|
const char* start = list;
|
||||||
|
while (*list && *list != ' ')
|
||||||
|
list++;
|
||||||
|
size_t len = (size_t)(list - start);
|
||||||
|
if (len == name_len && strncasecmp(name, start, len) == 0)
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
bool compression_should_skip_with_suffixes(const char* path, char* const* suffixes, int count) {
|
bool compression_should_skip_with_suffixes(const char* path, char* const* suffixes, int count) {
|
||||||
if (!path)
|
if (!path)
|
||||||
return false;
|
return false;
|
||||||
const char* dot = strrchr(path, '.');
|
const char* dot = strrchr(path, '.');
|
||||||
if (!dot)
|
if (!dot || dot[1] == '\0')
|
||||||
return false;
|
return false;
|
||||||
if (count < 0) {
|
const char* name = dot + 1;
|
||||||
suffixes = SKIP_COMPRESSION_EXTENSIONS;
|
/* count < 0 (the user gave no --skip-compress) selects rsync's built-in
|
||||||
count = 0;
|
* default list; a non-negative count is the user's explicit list. */
|
||||||
while (SKIP_COMPRESSION_EXTENSIONS[count])
|
if (count < 0)
|
||||||
count++;
|
return suffix_in_list(name, DEFAULT_SKIP_COMPRESS_SUFFIXES);
|
||||||
}
|
|
||||||
for (int i = 0; i < count; i++) {
|
for (int i = 0; i < count; i++) {
|
||||||
if (strcasecmp(dot, suffixes[i]) == 0)
|
const char* suffix = suffixes[i];
|
||||||
|
if (suffix[0] == '.')
|
||||||
|
suffix++;
|
||||||
|
if (strcasecmp(name, suffix) == 0)
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
int compression_algo_from_name(const char* name) {
|
||||||
|
if (!name)
|
||||||
|
return -1;
|
||||||
|
if (strcasecmp(name, "zstd") == 0)
|
||||||
|
return (int)COMPRESSION_ALGO_ZSTD;
|
||||||
|
if (strcasecmp(name, "lz4") == 0)
|
||||||
|
return (int)COMPRESSION_ALGO_LZ4;
|
||||||
|
if (strcasecmp(name, "zlib") == 0)
|
||||||
|
return (int)COMPRESSION_ALGO_ZLIB;
|
||||||
|
if (strcasecmp(name, "zlibx") == 0)
|
||||||
|
return (int)COMPRESSION_ALGO_ZLIBX;
|
||||||
|
if (strcasecmp(name, "none") == 0)
|
||||||
|
return (int)COMPRESSION_ALGO_NONE;
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
const char* compression_algo_name(CompressionAlgo algo) {
|
||||||
|
switch (algo) {
|
||||||
|
case COMPRESSION_ALGO_NONE:
|
||||||
|
return "none";
|
||||||
|
case COMPRESSION_ALGO_ZSTD:
|
||||||
|
return "zstd";
|
||||||
|
case COMPRESSION_ALGO_LZ4:
|
||||||
|
return "lz4";
|
||||||
|
case COMPRESSION_ALGO_ZLIB:
|
||||||
|
return "zlib";
|
||||||
|
case COMPRESSION_ALGO_ZLIBX:
|
||||||
|
return "zlibx";
|
||||||
|
}
|
||||||
|
return "<unknown>";
|
||||||
|
}
|
||||||
|
|
||||||
|
bool compression_algo_valid(int algo) {
|
||||||
|
return algo == (int)COMPRESSION_ALGO_NONE || algo == (int)COMPRESSION_ALGO_ZSTD ||
|
||||||
|
algo == (int)COMPRESSION_ALGO_LZ4 || algo == (int)COMPRESSION_ALGO_ZLIB ||
|
||||||
|
algo == (int)COMPRESSION_ALGO_ZLIBX;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool compression_algo_enabled(CompressionAlgo algo) {
|
||||||
|
return algo != COMPRESSION_ALGO_NONE;
|
||||||
|
}
|
||||||
|
|
||||||
|
CompressionAlgo compression_negotiate_default(void) {
|
||||||
|
/* rsync 3.4.1 default preference order; every entry is compiled in, so this
|
||||||
|
* resolves to zstd. */
|
||||||
|
static const CompressionAlgo preference[] = {
|
||||||
|
COMPRESSION_ALGO_ZSTD, COMPRESSION_ALGO_LZ4, COMPRESSION_ALGO_ZLIBX,
|
||||||
|
COMPRESSION_ALGO_ZLIB, COMPRESSION_ALGO_NONE,
|
||||||
|
};
|
||||||
|
for (size_t i = 0; i < sizeof(preference) / sizeof(preference[0]); i++) {
|
||||||
|
if (compression_algo_valid((int)preference[i]))
|
||||||
|
return preference[i];
|
||||||
|
}
|
||||||
|
return COMPRESSION_ALGO_ZSTD;
|
||||||
|
}
|
||||||
|
|
||||||
|
void compression_set_algo(CompressionAlgo algo) {
|
||||||
|
if (compression_algo_valid((int)algo))
|
||||||
|
atomic_store(&g_compression_algo, (int)algo);
|
||||||
|
}
|
||||||
|
|
||||||
|
CompressionAlgo compression_get_algo(void) {
|
||||||
|
return (CompressionAlgo)atomic_load(&g_compression_algo);
|
||||||
|
}
|
||||||
|
|
||||||
/* Per-thread cache of zstd contexts plus the grow-only compression scratch
|
/* Per-thread cache of zstd contexts plus the grow-only compression scratch
|
||||||
* buffer. zstd contexts are stateful and not safe to share between threads,
|
* buffer. zstd contexts are stateful and not safe to share between threads,
|
||||||
* so each thread keeps its own (see compression_get_thread_ctx). The cache is
|
* so each thread keeps its own (see compression_get_thread_ctx). The cache is
|
||||||
@@ -126,17 +232,25 @@ static void compression_ctx_put(CompressionThreadCtx* ctx) {
|
|||||||
compression_ctx_free(ctx);
|
compression_ctx_free(ctx);
|
||||||
}
|
}
|
||||||
|
|
||||||
Data* data_compress(Data* data_to_compress, int compression_level) {
|
/* Build a frame consisting of a copy of `src` prefixed by `codec`. */
|
||||||
return data_compress_with_threads(data_to_compress, compression_level, 0);
|
static Data* frame_with_codec(const void* src, size_t size, CompressionAlgo codec) {
|
||||||
|
if (size > SIZE_MAX - 1)
|
||||||
|
return NULL;
|
||||||
|
Data* out = data_create_empty(size + 1);
|
||||||
|
if (!out)
|
||||||
|
return NULL;
|
||||||
|
((uint8_t*)out->data)[0] = (uint8_t)codec;
|
||||||
|
if (size > 0)
|
||||||
|
memcpy((uint8_t*)out->data + 1, src, size);
|
||||||
|
out->size = size + 1;
|
||||||
|
return out;
|
||||||
}
|
}
|
||||||
|
|
||||||
Data* data_compress_with_threads(Data* data_to_compress, int compression_level,
|
static Data* zstd_compress(Data* in, int compression_level, int compression_threads) {
|
||||||
int compression_threads) {
|
size_t dst_size = ZSTD_compressBound(in->size);
|
||||||
if (!data_to_compress || (!data_to_compress->data && data_to_compress->size != 0) ||
|
if (dst_size > SIZE_MAX - 1)
|
||||||
compression_threads < 0 || compression_threads > COMPRESSION_MAX_THREADS)
|
|
||||||
return NULL;
|
return NULL;
|
||||||
log_message(LOG_LEVEL_DEBUG, "Starting to compress data");
|
dst_size += 1; /* codec prefix */
|
||||||
size_t dst_size = ZSTD_compressBound(data_to_compress->size);
|
|
||||||
|
|
||||||
CompressionThreadCtx* ctx = compression_get_thread_ctx();
|
CompressionThreadCtx* ctx = compression_get_thread_ctx();
|
||||||
if (ctx == NULL) {
|
if (ctx == NULL) {
|
||||||
@@ -187,7 +301,7 @@ Data* data_compress_with_threads(Data* data_to_compress, int compression_level,
|
|||||||
|
|
||||||
if (available_threads > 0) {
|
if (available_threads > 0) {
|
||||||
/* Streaming compression needs the source size before threaded mode can end a frame. */
|
/* Streaming compression needs the source size before threaded mode can end a frame. */
|
||||||
size_t zret = ZSTD_CCtx_setPledgedSrcSize(ctx->cctx, data_to_compress->size);
|
size_t zret = ZSTD_CCtx_setPledgedSrcSize(ctx->cctx, in->size);
|
||||||
if (ZSTD_isError(zret)) {
|
if (ZSTD_isError(zret)) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Failed to set compression source size: %s",
|
log_message(LOG_LEVEL_ERROR, "Failed to set compression source size: %s",
|
||||||
ZSTD_getErrorName(zret));
|
ZSTD_getErrorName(zret));
|
||||||
@@ -205,8 +319,8 @@ Data* data_compress_with_threads(Data* data_to_compress, int compression_level,
|
|||||||
ctx->out_cap = dst_size;
|
ctx->out_cap = dst_size;
|
||||||
}
|
}
|
||||||
|
|
||||||
ZSTD_inBuffer input = {data_to_compress->data, data_to_compress->size, 0};
|
ZSTD_inBuffer input = {in->data, in->size, 0};
|
||||||
ZSTD_outBuffer output = {ctx->out_buf, dst_size, 0};
|
ZSTD_outBuffer output = {(uint8_t*)ctx->out_buf + 1, dst_size - 1, 0};
|
||||||
|
|
||||||
size_t ret;
|
size_t ret;
|
||||||
do {
|
do {
|
||||||
@@ -219,30 +333,192 @@ Data* data_compress_with_threads(Data* data_to_compress, int compression_level,
|
|||||||
|
|
||||||
/* Hand off an exactly-sized copy; the scratch buffer stays cached so the next
|
/* Hand off an exactly-sized copy; the scratch buffer stays cached so the next
|
||||||
* call does not reallocate a ZSTD_compressBound-sized block. */
|
* call does not reallocate a ZSTD_compressBound-sized block. */
|
||||||
compressed_data = data_create_empty(output.pos);
|
compressed_data = data_create_empty(output.pos + 1);
|
||||||
if (compressed_data == NULL) {
|
if (compressed_data == NULL) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Failed to allocate compressed data");
|
log_message(LOG_LEVEL_ERROR, "Failed to allocate compressed data");
|
||||||
goto cleanup;
|
goto cleanup;
|
||||||
}
|
}
|
||||||
|
((uint8_t*)compressed_data->data)[0] = (uint8_t)COMPRESSION_ALGO_ZSTD;
|
||||||
if (output.pos > 0)
|
if (output.pos > 0)
|
||||||
memcpy(compressed_data->data, ctx->out_buf, output.pos);
|
memcpy((uint8_t*)compressed_data->data + 1, (uint8_t*)ctx->out_buf + 1, output.pos);
|
||||||
compressed_data->size = output.pos;
|
compressed_data->size = output.pos + 1;
|
||||||
|
|
||||||
log_debug_message(LOG_DEBUG_UTIL, "Data succesfully compressed from %zu to %zu",
|
log_debug_message(LOG_DEBUG_UTIL, "Data succesfully compressed from %zu to %zu", in->size,
|
||||||
data_to_compress->size, compressed_data->size);
|
compressed_data->size);
|
||||||
|
|
||||||
cleanup:
|
cleanup:
|
||||||
compression_ctx_put(ctx);
|
compression_ctx_put(ctx);
|
||||||
return compressed_data;
|
return compressed_data;
|
||||||
}
|
}
|
||||||
|
|
||||||
Data* data_decompress_limited(Data* compressed_data, size_t maximum_size) {
|
static Data* lz4_compress(Data* in) {
|
||||||
if (!compressed_data || (!compressed_data->data && compressed_data->size != 0) ||
|
int bound = LZ4_compressBound((int)in->size);
|
||||||
maximum_size == 0)
|
if (bound < 0 || in->size > (size_t)INT_MAX)
|
||||||
return NULL;
|
return NULL;
|
||||||
|
Data* out = data_create_empty((size_t)bound + 1 + LZ4_SIZE_PREFIX_LEN);
|
||||||
|
if (!out)
|
||||||
|
return NULL;
|
||||||
|
uint32_t raw_size = (uint32_t)in->size;
|
||||||
|
uint8_t* p = (uint8_t*)out->data;
|
||||||
|
p[0] = (uint8_t)COMPRESSION_ALGO_LZ4;
|
||||||
|
for (int i = 0; i < LZ4_SIZE_PREFIX_LEN; i++)
|
||||||
|
p[1 + i] = (uint8_t)((raw_size >> (8 * i)) & 0xff);
|
||||||
|
int written = 0;
|
||||||
|
if (in->size > 0) {
|
||||||
|
written = LZ4_compress_default((const char*)in->data, (char*)p + 1 + LZ4_SIZE_PREFIX_LEN,
|
||||||
|
(int)in->size, bound);
|
||||||
|
if (written <= 0) {
|
||||||
|
data_destroy(out);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out->size = (size_t)written + 1 + LZ4_SIZE_PREFIX_LEN;
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
static Data* zlib_compress(Data* in, CompressionAlgo algo, int compression_level) {
|
||||||
|
int level = compression_level;
|
||||||
|
if (level < 1)
|
||||||
|
level = Z_DEFAULT_COMPRESSION;
|
||||||
|
if (level > 9)
|
||||||
|
level = 9;
|
||||||
|
uLong bound = compressBound((uLong)in->size);
|
||||||
|
if (in->size > (size_t)ULONG_MAX)
|
||||||
|
return NULL;
|
||||||
|
Data* out = data_create_empty((size_t)bound + 1 + LZ4_SIZE_PREFIX_LEN);
|
||||||
|
if (!out)
|
||||||
|
return NULL;
|
||||||
|
uint32_t raw_size = (uint32_t)in->size;
|
||||||
|
uint8_t* p = (uint8_t*)out->data;
|
||||||
|
p[0] = (uint8_t)algo;
|
||||||
|
for (int i = 0; i < LZ4_SIZE_PREFIX_LEN; i++)
|
||||||
|
p[1 + i] = (uint8_t)((raw_size >> (8 * i)) & 0xff);
|
||||||
|
uLongf dest_len = bound;
|
||||||
|
int rc = compress2(p + 1 + LZ4_SIZE_PREFIX_LEN, &dest_len, (const Bytef*)in->data,
|
||||||
|
(uLong)in->size, level);
|
||||||
|
if (rc != Z_OK) {
|
||||||
|
data_destroy(out);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
out->size = (size_t)dest_len + 1 + LZ4_SIZE_PREFIX_LEN;
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
Data* data_compress_codec(Data* data_to_compress, CompressionAlgo algo, int compression_level,
|
||||||
|
int compression_threads) {
|
||||||
|
if (!data_to_compress || (!data_to_compress->data && data_to_compress->size != 0) ||
|
||||||
|
compression_threads < 0 || compression_threads > COMPRESSION_MAX_THREADS)
|
||||||
|
return NULL;
|
||||||
|
if (!compression_algo_valid((int)algo))
|
||||||
|
return NULL;
|
||||||
|
log_message(LOG_LEVEL_DEBUG, "Starting to compress data");
|
||||||
|
switch (algo) {
|
||||||
|
case COMPRESSION_ALGO_NONE:
|
||||||
|
return frame_with_codec(data_to_compress->data, data_to_compress->size, COMPRESSION_ALGO_NONE);
|
||||||
|
case COMPRESSION_ALGO_ZSTD:
|
||||||
|
return zstd_compress(data_to_compress, compression_level, compression_threads);
|
||||||
|
case COMPRESSION_ALGO_LZ4:
|
||||||
|
return lz4_compress(data_to_compress);
|
||||||
|
case COMPRESSION_ALGO_ZLIB:
|
||||||
|
case COMPRESSION_ALGO_ZLIBX:
|
||||||
|
return zlib_compress(data_to_compress, algo, compression_level);
|
||||||
|
}
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
Data* data_compress_with_threads(Data* data_to_compress, int compression_level,
|
||||||
|
int compression_threads) {
|
||||||
|
return data_compress_codec(data_to_compress, compression_get_algo(), compression_level,
|
||||||
|
compression_threads);
|
||||||
|
}
|
||||||
|
|
||||||
|
Data* data_compress(Data* data_to_compress, int compression_level) {
|
||||||
|
return data_compress_codec(data_to_compress, compression_get_algo(), compression_level, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
static Data* decompress_none(const Data* compressed_data, size_t maximum_size) {
|
||||||
|
size_t size = compressed_data->size - 1;
|
||||||
|
if (size > maximum_size)
|
||||||
|
return NULL;
|
||||||
|
Data* out = data_create_empty(size);
|
||||||
|
if (!out)
|
||||||
|
return NULL;
|
||||||
|
if (size > 0)
|
||||||
|
memcpy(out->data, (const uint8_t*)compressed_data->data + 1, size);
|
||||||
|
out->size = size;
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Read the 4-byte little-endian raw size stored after the codec byte. */
|
||||||
|
static bool read_raw_size(const Data* in, uint32_t* raw_size) {
|
||||||
|
if (in->size < 1 + LZ4_SIZE_PREFIX_LEN)
|
||||||
|
return false;
|
||||||
|
const uint8_t* p = (const uint8_t*)in->data;
|
||||||
|
uint32_t v = 0;
|
||||||
|
for (int i = 0; i < LZ4_SIZE_PREFIX_LEN; i++)
|
||||||
|
v |= (uint32_t)p[1 + i] << (8 * i);
|
||||||
|
*raw_size = v;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
static Data* lz4_decompress(Data* compressed_data, size_t maximum_size, size_t hard_limit) {
|
||||||
|
uint32_t raw_size = 0;
|
||||||
|
if (!read_raw_size(compressed_data, &raw_size))
|
||||||
|
return NULL;
|
||||||
|
if (raw_size > hard_limit || raw_size > maximum_size)
|
||||||
|
return NULL;
|
||||||
|
size_t comp_size = compressed_data->size - 1 - LZ4_SIZE_PREFIX_LEN;
|
||||||
|
Data* out = data_create_empty(raw_size);
|
||||||
|
if (!out)
|
||||||
|
return NULL;
|
||||||
|
if (raw_size == 0) {
|
||||||
|
out->size = 0;
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
int rc = LZ4_decompress_safe((const char*)compressed_data->data + 1 + LZ4_SIZE_PREFIX_LEN,
|
||||||
|
(char*)out->data, (int)comp_size, (int)raw_size);
|
||||||
|
if (rc < 0 || (uint32_t)rc != raw_size) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "LZ4 decompression failed");
|
||||||
|
data_destroy(out);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
out->size = raw_size;
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
static Data* zlib_decompress(Data* compressed_data, size_t maximum_size, size_t hard_limit) {
|
||||||
|
uint32_t raw_size = 0;
|
||||||
|
if (!read_raw_size(compressed_data, &raw_size))
|
||||||
|
return NULL;
|
||||||
|
if (raw_size > hard_limit || raw_size > maximum_size)
|
||||||
|
return NULL;
|
||||||
|
size_t comp_size = compressed_data->size - 1 - LZ4_SIZE_PREFIX_LEN;
|
||||||
|
Data* out = data_create_empty(raw_size);
|
||||||
|
if (!out)
|
||||||
|
return NULL;
|
||||||
|
if (raw_size == 0) {
|
||||||
|
out->size = 0;
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
uLongf dest_len = raw_size;
|
||||||
|
int rc =
|
||||||
|
uncompress((Bytef*)out->data, &dest_len,
|
||||||
|
(const Bytef*)compressed_data->data + 1 + LZ4_SIZE_PREFIX_LEN, (uLong)comp_size);
|
||||||
|
if (rc != Z_OK || dest_len != raw_size) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "zlib decompression failed");
|
||||||
|
data_destroy(out);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
out->size = raw_size;
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
static Data* zstd_decompress(Data* compressed_data, size_t maximum_size) {
|
||||||
|
/* The zstd frame starts after the codec byte. */
|
||||||
|
const void* frame = (const uint8_t*)compressed_data->data + 1;
|
||||||
|
size_t frame_size = compressed_data->size - 1;
|
||||||
log_debug_message(LOG_DEBUG_UTIL, "Start to decompress data");
|
log_debug_message(LOG_DEBUG_UTIL, "Start to decompress data");
|
||||||
unsigned long long dst_size =
|
unsigned long long dst_size = ZSTD_getFrameContentSize(frame, frame_size);
|
||||||
ZSTD_getFrameContentSize(compressed_data->data, compressed_data->size);
|
|
||||||
/* ZSTD_isError() is also true for ZSTD_CONTENTSIZE_ERROR and
|
/* ZSTD_isError() is also true for ZSTD_CONTENTSIZE_ERROR and
|
||||||
* ZSTD_CONTENTSIZE_UNKNOWN (both are encoded near (size_t)-1), so test the
|
* ZSTD_CONTENTSIZE_UNKNOWN (both are encoded near (size_t)-1), so test the
|
||||||
* sentinels explicitly instead of blanket-rejecting every error-ish value:
|
* sentinels explicitly instead of blanket-rejecting every error-ish value:
|
||||||
@@ -256,9 +532,9 @@ Data* data_decompress_limited(Data* compressed_data, size_t maximum_size) {
|
|||||||
// ZSTD_CONTENTSIZE_UNKNOWN (~2^64) can cause massive allocation;
|
// ZSTD_CONTENTSIZE_UNKNOWN (~2^64) can cause massive allocation;
|
||||||
// fall back to a conservative estimate (3x compressed size) when unknown.
|
// fall back to a conservative estimate (3x compressed size) when unknown.
|
||||||
if (dst_size == ZSTD_CONTENTSIZE_UNKNOWN) {
|
if (dst_size == ZSTD_CONTENTSIZE_UNKNOWN) {
|
||||||
if (compressed_data->size > ULLONG_MAX / 3)
|
if (frame_size > ULLONG_MAX / 3)
|
||||||
return NULL;
|
return NULL;
|
||||||
dst_size = compressed_data->size * 3;
|
dst_size = frame_size * 3;
|
||||||
if (dst_size < INITIAL_DECOMPRESS_BUF_SIZE)
|
if (dst_size < INITIAL_DECOMPRESS_BUF_SIZE)
|
||||||
dst_size = INITIAL_DECOMPRESS_BUF_SIZE;
|
dst_size = INITIAL_DECOMPRESS_BUF_SIZE;
|
||||||
}
|
}
|
||||||
@@ -295,7 +571,7 @@ Data* data_decompress_limited(Data* compressed_data, size_t maximum_size) {
|
|||||||
goto cleanup;
|
goto cleanup;
|
||||||
}
|
}
|
||||||
|
|
||||||
ZSTD_inBuffer input = {compressed_data->data, compressed_data->size, 0};
|
ZSTD_inBuffer input = {frame, frame_size, 0};
|
||||||
ZSTD_outBuffer output = {uncompressed_data->data, buf_size, 0};
|
ZSTD_outBuffer output = {uncompressed_data->data, buf_size, 0};
|
||||||
|
|
||||||
size_t ret;
|
size_t ret;
|
||||||
@@ -354,6 +630,31 @@ cleanup:
|
|||||||
return uncompressed_data;
|
return uncompressed_data;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Data* data_decompress_limited(Data* compressed_data, size_t maximum_size) {
|
||||||
|
if (!compressed_data || (!compressed_data->data && compressed_data->size != 0) ||
|
||||||
|
maximum_size == 0)
|
||||||
|
return NULL;
|
||||||
|
if (compressed_data->size < 1)
|
||||||
|
return NULL;
|
||||||
|
unsigned long long hard_limit =
|
||||||
|
maximum_size < MAX_DECOMPRESSED_SIZE ? maximum_size : MAX_DECOMPRESSED_SIZE;
|
||||||
|
uint8_t codec = ((const uint8_t*)compressed_data->data)[0];
|
||||||
|
if (!compression_algo_valid(codec))
|
||||||
|
return NULL;
|
||||||
|
switch ((CompressionAlgo)codec) {
|
||||||
|
case COMPRESSION_ALGO_NONE:
|
||||||
|
return decompress_none(compressed_data, (size_t)hard_limit);
|
||||||
|
case COMPRESSION_ALGO_ZSTD:
|
||||||
|
return zstd_decompress(compressed_data, (size_t)hard_limit);
|
||||||
|
case COMPRESSION_ALGO_LZ4:
|
||||||
|
return lz4_decompress(compressed_data, maximum_size, (size_t)hard_limit);
|
||||||
|
case COMPRESSION_ALGO_ZLIB:
|
||||||
|
case COMPRESSION_ALGO_ZLIBX:
|
||||||
|
return zlib_decompress(compressed_data, maximum_size, (size_t)hard_limit);
|
||||||
|
}
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
Data* data_decompress(Data* compressed_data) {
|
Data* data_decompress(Data* compressed_data) {
|
||||||
return data_decompress_limited(compressed_data, MAX_DECOMPRESSED_SIZE);
|
return data_decompress_limited(compressed_data, MAX_DECOMPRESSED_SIZE);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,11 +6,59 @@
|
|||||||
|
|
||||||
#define COMPRESSION_MAX_THREADS 64
|
#define COMPRESSION_MAX_THREADS 64
|
||||||
|
|
||||||
|
/* Compression algorithms selectable with --compress-choice / -z. The ids are
|
||||||
|
* the values placed on the wire (Config->compression_algo), so they must be
|
||||||
|
* kept stable. NONE is "no compression"; ZSTD is the historical FastSync
|
||||||
|
* default and the negotiated "auto" choice. ZLIBX is rsync's zlib-without-
|
||||||
|
* matched-data variant: FastSync compresses only the delta/token bytes (it does
|
||||||
|
* not put matched file data in the compression stream), so its zlib codec is
|
||||||
|
* already the "x" form and zlib/zlibx share the same implementation, recorded
|
||||||
|
* under distinct ids. */
|
||||||
|
typedef enum {
|
||||||
|
COMPRESSION_ALGO_NONE = 0,
|
||||||
|
COMPRESSION_ALGO_ZSTD = 1,
|
||||||
|
COMPRESSION_ALGO_LZ4 = 2,
|
||||||
|
COMPRESSION_ALGO_ZLIB = 3,
|
||||||
|
COMPRESSION_ALGO_ZLIBX = 4
|
||||||
|
} CompressionAlgo;
|
||||||
|
|
||||||
|
/* Resolve a --compress-choice string (case-insensitive) to an algorithm id.
|
||||||
|
* Accepts "zstd", "lz4", "zlib", "zlibx", "none". "auto" is not an algorithm
|
||||||
|
* here; the caller resolves it to the negotiated default. Returns -1 for any
|
||||||
|
* unrecognized name. */
|
||||||
|
int compression_algo_from_name(const char* name);
|
||||||
|
const char* compression_algo_name(CompressionAlgo algo);
|
||||||
|
bool compression_algo_valid(int algo);
|
||||||
|
|
||||||
|
/* Pick the first algorithm from FastSync's compiled-in preference list
|
||||||
|
* (rsync 3.4.1's `--version` order: zstd lz4 zlibx zlib none). Resolves
|
||||||
|
* "auto". */
|
||||||
|
CompressionAlgo compression_negotiate_default(void);
|
||||||
|
|
||||||
|
/* True when the algorithm actually compresses (i.e. is not NONE). */
|
||||||
|
bool compression_algo_enabled(CompressionAlgo algo);
|
||||||
|
|
||||||
|
/* Select the process-wide codec used by the legacy wrappers below. Each
|
||||||
|
* process serves exactly one transfer config (the server forks per connection,
|
||||||
|
* the client configures itself before spawning transfer threads), so a
|
||||||
|
* process-global default is sufficient and constant for the lifetime of a
|
||||||
|
* transfer. Defaults to ZSTD when never set. Thread-safe. */
|
||||||
|
void compression_set_algo(CompressionAlgo algo);
|
||||||
|
CompressionAlgo compression_get_algo(void);
|
||||||
|
|
||||||
|
/* Codec-aware primitives. The compressed buffer is self-describing: its first
|
||||||
|
* byte is the CompressionAlgo id, so decompression never needs the codec passed
|
||||||
|
* separately (this keeps every existing Decompress call site source-compatible).
|
||||||
|
* `data_compress_codec` returns NULL on invalid input or an unsupported codec. */
|
||||||
|
Data* data_compress_codec(Data* data_to_compress, CompressionAlgo algo, int compression_level,
|
||||||
|
int compression_threads);
|
||||||
|
Data* data_decompress_limited(Data* compressed_data, size_t maximum_size);
|
||||||
|
|
||||||
|
/* Legacy zstd-default wrappers retained for existing callers/tests. */
|
||||||
Data* data_compress(Data* data_to_compress, int compression_level);
|
Data* data_compress(Data* data_to_compress, int compression_level);
|
||||||
Data* data_compress_with_threads(Data* data_to_compress, int compression_level,
|
Data* data_compress_with_threads(Data* data_to_compress, int compression_level,
|
||||||
int compression_threads);
|
int compression_threads);
|
||||||
Data* data_decompress(Data* compressed_data);
|
Data* data_decompress(Data* compressed_data);
|
||||||
Data* data_decompress_limited(Data* compressed_data, size_t maximum_size);
|
|
||||||
bool compression_should_skip_with_suffixes(const char* path, char* const* suffixes, int count);
|
bool compression_should_skip_with_suffixes(const char* path, char* const* suffixes, int count);
|
||||||
|
|
||||||
/* Release the calling thread's cached zstd contexts (compressor, decompressor
|
/* Release the calling thread's cached zstd contexts (compressor, decompressor
|
||||||
|
|||||||
+165
-37
@@ -14,12 +14,15 @@
|
|||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
|
#include <strings.h>
|
||||||
#include <limits.h>
|
#include <limits.h>
|
||||||
#include <errno.h>
|
#include <errno.h>
|
||||||
|
|
||||||
static void config_set_defaults(Config* config) {
|
static void config_set_defaults(Config* config) {
|
||||||
config->scanner_threads = 0;
|
config->scanner_threads = 0;
|
||||||
config->metadata_explicitly_disabled = false;
|
config->metadata_explicitly_disabled = false;
|
||||||
|
config->preserve_perms_explicit_off = false;
|
||||||
|
config->preserve_times_explicit_off = false;
|
||||||
config->show_progress = false;
|
config->show_progress = false;
|
||||||
config->compression_threads = 0;
|
config->compression_threads = 0;
|
||||||
config->ssh_port = 22;
|
config->ssh_port = 22;
|
||||||
@@ -43,12 +46,14 @@ static void config_set_defaults(Config* config) {
|
|||||||
config->server_port = 8080;
|
config->server_port = 8080;
|
||||||
config->server_port_set = false;
|
config->server_port_set = false;
|
||||||
config->server_host_set = false;
|
config->server_host_set = false;
|
||||||
/* 0 means "--timeout not given": the transport keeps its own built-in 30 s
|
/* rsync defaults: --timeout=0 (I/O timeouts disabled) and --contimeout=60.
|
||||||
* socket timeout (tcp_set_timeouts ignores non-positive values) and the
|
* A value of 0 disables the client's own deadline on both the socket layer
|
||||||
* protocol layer keeps its built-in 60 s per-message deadline. A positive
|
* (tcp_set_timeouts) and the protocol layer
|
||||||
* value overrides BOTH (see protocol_session_set_io_timeout). */
|
* (protocol_session_set_io_timeout); a positive value sets it. A server
|
||||||
|
* session floors the deadline at SERVER_IO_TIMEOUT_SEC so 0 can never hold a
|
||||||
|
* connection open forever. */
|
||||||
config->timeout = 0;
|
config->timeout = 0;
|
||||||
config->contimeout = 10;
|
config->contimeout = 60;
|
||||||
config->quiet = false;
|
config->quiet = false;
|
||||||
config->stats = false;
|
config->stats = false;
|
||||||
config->max_depth = 0;
|
config->max_depth = 0;
|
||||||
@@ -63,12 +68,15 @@ static void config_set_defaults(Config* config) {
|
|||||||
config->human_readable = false;
|
config->human_readable = false;
|
||||||
config->ignore_errors = false;
|
config->ignore_errors = false;
|
||||||
config->ignore_missing_args = false;
|
config->ignore_missing_args = false;
|
||||||
|
config->checksum_transfer_algo = CHECKSUM_ALGO_DEFAULT;
|
||||||
|
config->cli_exit_code = 0;
|
||||||
config->filters = NULL;
|
config->filters = NULL;
|
||||||
config->files_from = NULL;
|
config->files_from = NULL;
|
||||||
config->files_from_set = NULL;
|
config->files_from_set = NULL;
|
||||||
config->from0 = false;
|
config->from0 = false;
|
||||||
config->cvs_exclude = false;
|
config->cvs_exclude = false;
|
||||||
config->per_dir_filter = false;
|
config->per_dir_filter = false;
|
||||||
|
config->per_dir_filter_count = 0;
|
||||||
config->one_file_system = false;
|
config->one_file_system = false;
|
||||||
config->no_implied_dirs = false;
|
config->no_implied_dirs = false;
|
||||||
config->dirs = false;
|
config->dirs = false;
|
||||||
@@ -192,10 +200,13 @@ static bool validate_received_config(const Config* config) {
|
|||||||
valid_wire_bool(config->partial) && valid_wire_bool(config->delete_before) &&
|
valid_wire_bool(config->partial) && valid_wire_bool(config->delete_before) &&
|
||||||
valid_wire_bool(config->checksum) && valid_wire_bool(config->eight_bit_output) &&
|
valid_wire_bool(config->checksum) && valid_wire_bool(config->eight_bit_output) &&
|
||||||
valid_wire_bool(config->dry_run) && checksum_algo_valid(config->checksum_algo) &&
|
valid_wire_bool(config->dry_run) && checksum_algo_valid(config->checksum_algo) &&
|
||||||
identity_wire_valid(config) && valid_wire_bool(config->preserve_atimes) &&
|
compression_algo_valid(config->compression_algo) && identity_wire_valid(config) &&
|
||||||
valid_wire_bool(config->preserve_crtimes) && valid_wire_bool(config->omit_dir_times) &&
|
valid_wire_bool(config->preserve_atimes) && valid_wire_bool(config->preserve_crtimes) &&
|
||||||
valid_wire_bool(config->omit_link_times) && valid_wire_bool(config->munge_links) &&
|
valid_wire_bool(config->omit_dir_times) && valid_wire_bool(config->omit_link_times) &&
|
||||||
valid_wire_bool(config->keep_dirlinks) && valid_wire_bool(config->fake_super) &&
|
valid_wire_bool(config->preserve_perms) && valid_wire_bool(config->preserve_times) &&
|
||||||
|
valid_wire_bool(config->preserve_owner) && valid_wire_bool(config->preserve_group) &&
|
||||||
|
valid_wire_bool(config->munge_links) && valid_wire_bool(config->keep_dirlinks) &&
|
||||||
|
valid_wire_bool(config->fake_super) &&
|
||||||
(!config->copy_as_set || (config->copy_as_uid >= 0 && config->copy_as_gid >= 0)) &&
|
(!config->copy_as_set || (config->copy_as_uid >= 0 && config->copy_as_gid >= 0)) &&
|
||||||
(!config->use_compression ||
|
(!config->use_compression ||
|
||||||
(config->compression_level >= 1 && config->compression_level <= 22)) &&
|
(config->compression_level >= 1 && config->compression_level <= 22)) &&
|
||||||
@@ -203,8 +214,9 @@ static bool validate_received_config(const Config* config) {
|
|||||||
config->delta_block_size >= DELTA_BLOCK_SIZE_MIN &&
|
config->delta_block_size >= DELTA_BLOCK_SIZE_MIN &&
|
||||||
config->delta_block_size <= DELTA_BLOCK_SIZE_MAX &&
|
config->delta_block_size <= DELTA_BLOCK_SIZE_MAX &&
|
||||||
config->delta_max_file_size <= DELTA_MAX_FILE_SIZE && config->modify_window >= 0 &&
|
config->delta_max_file_size <= DELTA_MAX_FILE_SIZE && config->modify_window >= 0 &&
|
||||||
config->max_delete >= -1 && config->skip_compress_count >= 0 &&
|
config->max_delete >= -1 && config->max_alloc <= MAX_SERVER_ALLOC &&
|
||||||
config->skip_compress_count <= MAX_SKIP_COMPRESS_SUFFIXES && config->max_alloc > 0 &&
|
config->skip_compress_count >= 0 &&
|
||||||
|
config->skip_compress_count <= MAX_SKIP_COMPRESS_SUFFIXES &&
|
||||||
(!config->chmod_spec || !*config->chmod_spec ||
|
(!config->chmod_spec || !*config->chmod_spec ||
|
||||||
chmod_apply(0, config->chmod_spec, &(mode_t){0})) &&
|
chmod_apply(0, config->chmod_spec, &(mode_t){0})) &&
|
||||||
config->super_mode >= SUPER_MODE_AUTO && config->super_mode <= SUPER_MODE_OFF;
|
config->super_mode >= SUPER_MODE_AUTO && config->super_mode <= SUPER_MODE_OFF;
|
||||||
@@ -221,7 +233,13 @@ Config* config_create(void) {
|
|||||||
bool config_delete_timing_early(const Config* config) {
|
bool config_delete_timing_early(const Config* config) {
|
||||||
if (!config)
|
if (!config)
|
||||||
return false;
|
return false;
|
||||||
return config->delete_before || config->delete_during;
|
return config->delete_before;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool config_delete_timing_per_dir(const Config* config) {
|
||||||
|
if (!config)
|
||||||
|
return false;
|
||||||
|
return config->delete_during || config->delete_delay;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* A delete-timing flag is only meaningful together with --delete. At most one
|
/* A delete-timing flag is only meaningful together with --delete. At most one
|
||||||
@@ -292,40 +310,68 @@ const char* config_invariants_error(const Config* config) {
|
|||||||
"timing; at most one may be given and each implies --delete";
|
"timing; at most one may be given and each implies --delete";
|
||||||
if (config->iconv_spec && !charset_spec_valid(config->iconv_spec))
|
if (config->iconv_spec && !charset_spec_valid(config->iconv_spec))
|
||||||
return "--iconv requires LOCAL[,REMOTE] charset names supported by iconv";
|
return "--iconv requires LOCAL[,REMOTE] charset names supported by iconv";
|
||||||
|
if ((config->preserve_perms || config->preserve_times || config->preserve_owner ||
|
||||||
|
config->preserve_group || config->preserve_atimes || config->preserve_crtimes ||
|
||||||
|
config->use_executability) &&
|
||||||
|
!config->use_metadata)
|
||||||
|
return "a preservation attribute requires metadata transmission";
|
||||||
if (config->copy_as_set && !config->use_metadata)
|
if (config->copy_as_set && !config->use_metadata)
|
||||||
return "--copy-as requires metadata preservation and cannot be combined with --no-preserve";
|
return "--copy-as requires metadata preservation and cannot be combined with --no-preserve";
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool config_derived_use_metadata(const Config* config) {
|
||||||
|
if (!config)
|
||||||
|
return false;
|
||||||
|
if (config->preserve_perms || config->preserve_times || config->preserve_owner ||
|
||||||
|
config->preserve_group || config->preserve_atimes || config->preserve_crtimes ||
|
||||||
|
config->use_executability || config->preserve_xattrs || config->preserve_acls ||
|
||||||
|
config->fake_super || config->preserve_devices || config->preserve_specials ||
|
||||||
|
config->copy_devices || config->write_devices ||
|
||||||
|
(config->chmod_spec && config->chmod_spec[0]) || config->copy_as_set ||
|
||||||
|
config->chown_uid_set || config->chown_gid_set || config->usermap_count > 0 ||
|
||||||
|
config->groupmap_count > 0 || config->update)
|
||||||
|
return true;
|
||||||
|
return (config->use_incremental || config->use_delta) && !config->metadata_explicitly_disabled;
|
||||||
|
}
|
||||||
|
|
||||||
bool config_has_basis(const Config* config) {
|
bool config_has_basis(const Config* config) {
|
||||||
return config && config->basis_count > 0;
|
return config && config->basis_count > 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* A basis-dir path travels from the client to the receiver and is resolved
|
/* A basis-dir path travels from the client to the receiver and is resolved
|
||||||
* below the destination root, so it must be a non-empty relative path with no
|
* below the destination root when relative, or used verbatim when absolute
|
||||||
* "." or ".." component and no traversal: an absolute or escaping path would
|
* (matching rsync). Either form must be non-empty, traversal-free (no "..")
|
||||||
* make the receiver read or link files outside its authorized root.
|
* and free of "." components: an escaping path would make the receiver read or
|
||||||
|
* link files outside its authorized root. An absolute path is still subject to
|
||||||
|
* the receiver's root confinement at open time (file_open_secure_parent), so a
|
||||||
|
* basis outside the authorized root is simply not found rather than an escape.
|
||||||
*
|
*
|
||||||
* Returns a malloc'd CANONICAL copy of an accepted path, or NULL when the path
|
* Returns a malloc'd CANONICAL copy of an accepted path, or NULL when the path
|
||||||
* is rejected. Canonicalization collapses interior empty components ("a//b" ->
|
* is rejected. Canonicalization collapses interior empty components ("a//b" ->
|
||||||
* "a/b"), drops "." components and trailing "/"s, so validation, the delete
|
* "a/b"), drops "." components and trailing "/"s, and preserves a leading '/'
|
||||||
* walker prefix match and the receiver's basis lookup all agree on one form.
|
* for absolute paths, so validation, the delete walker prefix match and the
|
||||||
* The normalizer is the single source of truth for both config_basis_path_valid
|
* receiver's basis lookup all agree on one form. The normalizer is the single
|
||||||
* and config_basis_append. */
|
* source of truth for both config_basis_path_valid and config_basis_append. */
|
||||||
static char* basis_path_normalize(const char* path) {
|
static char* basis_path_normalize(const char* path) {
|
||||||
if (!path || path[0] == '\0' || path[0] == '/' || has_path_traversal(path))
|
if (!path || path[0] == '\0' || has_path_traversal(path))
|
||||||
return NULL;
|
return NULL;
|
||||||
if (strcmp(path, ".") == 0)
|
bool absolute = path[0] == '/';
|
||||||
|
if (!absolute && strcmp(path, ".") == 0)
|
||||||
|
return NULL;
|
||||||
|
if (absolute && strcmp(path, "/") == 0)
|
||||||
return NULL;
|
return NULL;
|
||||||
char* dup = str_dup(path);
|
char* dup = str_dup(path);
|
||||||
if (!dup)
|
if (!dup)
|
||||||
return NULL;
|
return NULL;
|
||||||
size_t out_len = 0;
|
size_t out_len = 0;
|
||||||
char* out = malloc(strlen(path) + 1);
|
char* out = malloc(strlen(path) + 2);
|
||||||
if (!out) {
|
if (!out) {
|
||||||
free(dup);
|
free(dup);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
if (absolute)
|
||||||
|
out[out_len++] = '/';
|
||||||
char* saveptr = NULL;
|
char* saveptr = NULL;
|
||||||
bool ok = true;
|
bool ok = true;
|
||||||
for (char* part = strtok_r(dup, "/", &saveptr); part; part = strtok_r(NULL, "/", &saveptr)) {
|
for (char* part = strtok_r(dup, "/", &saveptr); part; part = strtok_r(NULL, "/", &saveptr)) {
|
||||||
@@ -335,14 +381,14 @@ static char* basis_path_normalize(const char* path) {
|
|||||||
}
|
}
|
||||||
if (strcmp(part, ".") == 0)
|
if (strcmp(part, ".") == 0)
|
||||||
continue;
|
continue;
|
||||||
if (out_len > 0)
|
if (out_len > 0 && out[out_len - 1] != '/')
|
||||||
out[out_len++] = '/';
|
out[out_len++] = '/';
|
||||||
size_t len = strlen(part);
|
size_t len = strlen(part);
|
||||||
memcpy(out + out_len, part, len);
|
memcpy(out + out_len, part, len);
|
||||||
out_len += len;
|
out_len += len;
|
||||||
}
|
}
|
||||||
free(dup);
|
free(dup);
|
||||||
if (!ok || out_len == 0) {
|
if (!ok || out_len == 0 || (absolute && out_len == 1)) {
|
||||||
free(out);
|
free(out);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
@@ -725,10 +771,18 @@ void config_delete(Config* config) {
|
|||||||
free(config->skip_compress_suffixes[i]);
|
free(config->skip_compress_suffixes[i]);
|
||||||
free(config->skip_compress_suffixes);
|
free(config->skip_compress_suffixes);
|
||||||
}
|
}
|
||||||
|
if (config->usermap) {
|
||||||
|
for (int i = 0; i < config->usermap_count; i++)
|
||||||
|
free(config->usermap[i].to_name);
|
||||||
free(config->usermap);
|
free(config->usermap);
|
||||||
|
}
|
||||||
config->usermap = NULL;
|
config->usermap = NULL;
|
||||||
config->usermap_count = 0;
|
config->usermap_count = 0;
|
||||||
|
if (config->groupmap) {
|
||||||
|
for (int i = 0; i < config->groupmap_count; i++)
|
||||||
|
free(config->groupmap[i].to_name);
|
||||||
free(config->groupmap);
|
free(config->groupmap);
|
||||||
|
}
|
||||||
config->groupmap = NULL;
|
config->groupmap = NULL;
|
||||||
config->groupmap_count = 0;
|
config->groupmap_count = 0;
|
||||||
if (config->filters) {
|
if (config->filters) {
|
||||||
@@ -756,11 +810,14 @@ void config_delete(Config* config) {
|
|||||||
* ------------------------------------------------------------------------- */
|
* ------------------------------------------------------------------------- */
|
||||||
|
|
||||||
/* --max-alloc: raw 64-bit value, clamped server-side and installed as the
|
/* --max-alloc: raw 64-bit value, clamped server-side and installed as the
|
||||||
* session allocation ceiling. A zero value is rejected. */
|
* session allocation ceiling. A received 0 is rsync's "no alloc limit"; on the
|
||||||
|
* receive path it is mapped to the server ceiling so a client can never disable
|
||||||
|
* it (client-side 0 remains unlimited). Any value above the ceiling is clamped
|
||||||
|
* to it. */
|
||||||
static bool config_receive_max_alloc(int fd, unsigned long long* value) {
|
static bool config_receive_max_alloc(int fd, unsigned long long* value) {
|
||||||
if (!receive_n_data(fd, value, sizeof(*value)) || *value == 0)
|
if (!receive_n_data(fd, value, sizeof(*value)))
|
||||||
return false;
|
return false;
|
||||||
if (*value > MAX_SERVER_ALLOC)
|
if (*value == 0 || *value > MAX_SERVER_ALLOC)
|
||||||
*value = MAX_SERVER_ALLOC;
|
*value = MAX_SERVER_ALLOC;
|
||||||
protocol_session_set_max_alloc(NULL, *value);
|
protocol_session_set_max_alloc(NULL, *value);
|
||||||
return true;
|
return true;
|
||||||
@@ -842,6 +899,14 @@ static bool config_receive_checksum_algo(int fd, int* value) {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static bool config_receive_compression_algo(int fd, int* value) {
|
||||||
|
int algo;
|
||||||
|
if (!receive_int(fd, &algo) || !compression_algo_valid(algo))
|
||||||
|
return false;
|
||||||
|
*value = algo;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
static bool config_receive_super_mode(int fd, SuperMode* value) {
|
static bool config_receive_super_mode(int fd, SuperMode* value) {
|
||||||
int mode;
|
int mode;
|
||||||
if (!receive_int(fd, &mode) || mode < SUPER_MODE_AUTO || mode > SUPER_MODE_OFF)
|
if (!receive_int(fd, &mode) || mode < SUPER_MODE_AUTO || mode > SUPER_MODE_OFF)
|
||||||
@@ -960,7 +1025,8 @@ static bool receive_basis_entries(int fd, Config* c, ConfigStringBudget* budget)
|
|||||||
|
|
||||||
static bool send_identity_entries(int fd, const IdentityMap* map, int count) {
|
static bool send_identity_entries(int fd, const IdentityMap* map, int count) {
|
||||||
for (int i = 0; i < count; i++) {
|
for (int i = 0; i < count; i++) {
|
||||||
if (!send_int(fd, map[i].from) || !send_int(fd, map[i].to))
|
if (!send_int(fd, map[i].from) || !send_int(fd, map[i].from_hi) || !send_int(fd, map[i].to) ||
|
||||||
|
!send_str(fd, map[i].to_name ? map[i].to_name : ""))
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
return true;
|
return true;
|
||||||
@@ -968,20 +1034,32 @@ static bool send_identity_entries(int fd, const IdentityMap* map, int count) {
|
|||||||
|
|
||||||
static bool receive_identity_entries(int fd, ConfigStringBudget* budget, int count,
|
static bool receive_identity_entries(int fd, ConfigStringBudget* budget, int count,
|
||||||
IdentityMap** out) {
|
IdentityMap** out) {
|
||||||
(void)budget;
|
|
||||||
if (count <= 0)
|
if (count <= 0)
|
||||||
return true;
|
return true;
|
||||||
IdentityMap* map = calloc((size_t)count, sizeof(IdentityMap));
|
IdentityMap* map = calloc((size_t)count, sizeof(IdentityMap));
|
||||||
if (!map)
|
if (!map)
|
||||||
return false;
|
return false;
|
||||||
for (int i = 0; i < count; i++) {
|
for (int i = 0; i < count; i++) {
|
||||||
if (!receive_int(fd, &map[i].from) || !receive_int(fd, &map[i].to)) {
|
if (!receive_int(fd, &map[i].from) || !receive_int(fd, &map[i].from_hi) ||
|
||||||
free(map);
|
!receive_int(fd, &map[i].to))
|
||||||
return false;
|
goto fail;
|
||||||
|
char* name = config_receive_str(fd, budget);
|
||||||
|
if (!name)
|
||||||
|
goto fail;
|
||||||
|
if (name[0] == '\0') {
|
||||||
|
free(name);
|
||||||
|
map[i].to_name = NULL;
|
||||||
|
} else {
|
||||||
|
map[i].to_name = name;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
*out = map;
|
*out = map;
|
||||||
return true;
|
return true;
|
||||||
|
fail:
|
||||||
|
for (int i = 0; i < count; i++)
|
||||||
|
free(map[i].to_name);
|
||||||
|
free(map);
|
||||||
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* ---------------------------------------------------------------------------
|
/* ---------------------------------------------------------------------------
|
||||||
@@ -1030,6 +1108,9 @@ static bool receive_identity_entries(int fd, ConfigStringBudget* budget, int cou
|
|||||||
#define CONFIG_SEND_INT_CHECKSUM_ALGO(name) send_int(fd, c->name)
|
#define CONFIG_SEND_INT_CHECKSUM_ALGO(name) send_int(fd, c->name)
|
||||||
#define CONFIG_RECV_INT_CHECKSUM_ALGO(name) config_receive_checksum_algo(fd, &c->name)
|
#define CONFIG_RECV_INT_CHECKSUM_ALGO(name) config_receive_checksum_algo(fd, &c->name)
|
||||||
|
|
||||||
|
#define CONFIG_SEND_INT_COMPRESSION_ALGO(name) send_int(fd, c->name)
|
||||||
|
#define CONFIG_RECV_INT_COMPRESSION_ALGO(name) config_receive_compression_algo(fd, &c->name)
|
||||||
|
|
||||||
#define CONFIG_SEND_SUPERMODE(name) send_int(fd, (int)c->name)
|
#define CONFIG_SEND_SUPERMODE(name) send_int(fd, (int)c->name)
|
||||||
#define CONFIG_RECV_SUPERMODE(name) config_receive_super_mode(fd, &c->name)
|
#define CONFIG_RECV_SUPERMODE(name) config_receive_super_mode(fd, &c->name)
|
||||||
|
|
||||||
@@ -1104,6 +1185,8 @@ CONFIG_DEFINE_SEND(send_daemon_auth, CONFIG_WIRE_DAEMON_AUTH_FIELDS)
|
|||||||
CONFIG_DEFINE_SEND(send_iconv_spec, CONFIG_WIRE_ICONV_FIELDS)
|
CONFIG_DEFINE_SEND(send_iconv_spec, CONFIG_WIRE_ICONV_FIELDS)
|
||||||
CONFIG_DEFINE_SEND(send_privilege_options, CONFIG_WIRE_PRIVILEGE_FIELDS)
|
CONFIG_DEFINE_SEND(send_privilege_options, CONFIG_WIRE_PRIVILEGE_FIELDS)
|
||||||
CONFIG_DEFINE_SEND(send_copy_as_options, CONFIG_WIRE_COPY_AS_FIELDS)
|
CONFIG_DEFINE_SEND(send_copy_as_options, CONFIG_WIRE_COPY_AS_FIELDS)
|
||||||
|
CONFIG_DEFINE_SEND(send_output_options, CONFIG_WIRE_OUTPUT_FIELDS)
|
||||||
|
CONFIG_DEFINE_SEND(send_codec_options, CONFIG_WIRE_CODEC_FIELDS)
|
||||||
|
|
||||||
CONFIG_DEFINE_RECV(receive_core_fields, CONFIG_WIRE_CORE_FIELDS)
|
CONFIG_DEFINE_RECV(receive_core_fields, CONFIG_WIRE_CORE_FIELDS)
|
||||||
CONFIG_DEFINE_RECV(receive_delta_fields, CONFIG_WIRE_DELTA_FIELDS)
|
CONFIG_DEFINE_RECV(receive_delta_fields, CONFIG_WIRE_DELTA_FIELDS)
|
||||||
@@ -1122,6 +1205,8 @@ CONFIG_DEFINE_RECV(receive_daemon_auth, CONFIG_WIRE_DAEMON_AUTH_FIELDS)
|
|||||||
CONFIG_DEFINE_RECV(receive_iconv_spec, CONFIG_WIRE_ICONV_FIELDS)
|
CONFIG_DEFINE_RECV(receive_iconv_spec, CONFIG_WIRE_ICONV_FIELDS)
|
||||||
CONFIG_DEFINE_RECV(receive_privilege_options, CONFIG_WIRE_PRIVILEGE_FIELDS)
|
CONFIG_DEFINE_RECV(receive_privilege_options, CONFIG_WIRE_PRIVILEGE_FIELDS)
|
||||||
CONFIG_DEFINE_RECV(receive_copy_as_options, CONFIG_WIRE_COPY_AS_FIELDS)
|
CONFIG_DEFINE_RECV(receive_copy_as_options, CONFIG_WIRE_COPY_AS_FIELDS)
|
||||||
|
CONFIG_DEFINE_RECV(receive_output_options, CONFIG_WIRE_OUTPUT_FIELDS)
|
||||||
|
CONFIG_DEFINE_RECV(receive_codec_options, CONFIG_WIRE_CODEC_FIELDS)
|
||||||
|
|
||||||
#undef XSEND
|
#undef XSEND
|
||||||
#undef XRECV
|
#undef XRECV
|
||||||
@@ -1238,7 +1323,9 @@ bool config_send_wire_block(int file_descriptor, const Config* config) {
|
|||||||
send_daemon_module(file_descriptor, config) && send_daemon_auth(file_descriptor, config) &&
|
send_daemon_module(file_descriptor, config) && send_daemon_auth(file_descriptor, config) &&
|
||||||
send_iconv_spec(file_descriptor, config) &&
|
send_iconv_spec(file_descriptor, config) &&
|
||||||
send_privilege_options(file_descriptor, config) &&
|
send_privilege_options(file_descriptor, config) &&
|
||||||
send_copy_as_options(file_descriptor, config);
|
send_copy_as_options(file_descriptor, config) &&
|
||||||
|
send_output_options(file_descriptor, config) &&
|
||||||
|
send_codec_options(file_descriptor, config);
|
||||||
}
|
}
|
||||||
|
|
||||||
bool config_send(int file_descriptor, const Config* config) {
|
bool config_send(int file_descriptor, const Config* config) {
|
||||||
@@ -1308,20 +1395,61 @@ Config* config_receive_with_validate(int file_descriptor, ConfigValidateFunc val
|
|||||||
!receive_daemon_auth(file_descriptor, config, &budget) ||
|
!receive_daemon_auth(file_descriptor, config, &budget) ||
|
||||||
!receive_iconv_spec(file_descriptor, config, &budget) ||
|
!receive_iconv_spec(file_descriptor, config, &budget) ||
|
||||||
!receive_privilege_options(file_descriptor, config, &budget) ||
|
!receive_privilege_options(file_descriptor, config, &budget) ||
|
||||||
!receive_copy_as_options(file_descriptor, config, &budget))
|
!receive_copy_as_options(file_descriptor, config, &budget) ||
|
||||||
|
!receive_output_options(file_descriptor, config, &budget) ||
|
||||||
|
!receive_codec_options(file_descriptor, config, &budget))
|
||||||
goto error;
|
goto error;
|
||||||
if (config->compress_choice[0] != '\0' && strcmp(config->compress_choice, "zstd") != 0 &&
|
/* Validate/normalize the negotiated codec. compress_choice is the human
|
||||||
strcmp(config->compress_choice, "none") != 0) {
|
* spelling (NULL or "" when -z was not given); compression_algo is the
|
||||||
|
* concrete codec id the sender used. They must agree, and "auto" is
|
||||||
|
* canonicalized to FastSync's negotiated default so the stored spelling is
|
||||||
|
* always concrete (a hostile/older client may still send "auto"). */
|
||||||
|
if (config->compress_choice && config->compress_choice[0] != '\0') {
|
||||||
|
int choice_algo = compression_algo_from_name(config->compress_choice);
|
||||||
|
if (choice_algo < 0 && strcasecmp(config->compress_choice, "auto") != 0) {
|
||||||
char* escaped_choice = output_escape(config->compress_choice, config->eight_bit_output);
|
char* escaped_choice = output_escape(config->compress_choice, config->eight_bit_output);
|
||||||
log_message(LOG_LEVEL_ERROR, "Unsupported compression choice: %s",
|
log_message(LOG_LEVEL_ERROR, "Unsupported compression choice: %s",
|
||||||
escaped_choice ? escaped_choice : "<allocation failed>");
|
escaped_choice ? escaped_choice : "<allocation failed>");
|
||||||
char detail[128];
|
char detail[160];
|
||||||
snprintf(detail, sizeof(detail), "unsupported compression choice: %s",
|
snprintf(detail, sizeof(detail), "unsupported compression choice: %s",
|
||||||
escaped_choice ? escaped_choice : "<allocation failed>");
|
escaped_choice ? escaped_choice : "<allocation failed>");
|
||||||
send_error_detail(file_descriptor, detail);
|
send_error_detail(file_descriptor, detail);
|
||||||
free(escaped_choice);
|
free(escaped_choice);
|
||||||
goto error;
|
goto error;
|
||||||
}
|
}
|
||||||
|
if (choice_algo < 0)
|
||||||
|
choice_algo = (int)compression_negotiate_default();
|
||||||
|
if (strcasecmp(config->compress_choice, "auto") == 0 ||
|
||||||
|
choice_algo == (int)COMPRESSION_ALGO_NONE) {
|
||||||
|
const char* canonical = compression_algo_name((CompressionAlgo)choice_algo);
|
||||||
|
char* dup = str_dup(canonical);
|
||||||
|
if (!dup)
|
||||||
|
goto error;
|
||||||
|
free(config->compress_choice);
|
||||||
|
config->compress_choice = dup;
|
||||||
|
}
|
||||||
|
if (config->compression_algo != choice_algo) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Compression choice '%s' does not match codec id %d",
|
||||||
|
config->compress_choice, config->compression_algo);
|
||||||
|
send_error_detail(file_descriptor, "compression choice/codec mismatch");
|
||||||
|
goto error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
/* The concrete codec must exist only when compression is on. A client that
|
||||||
|
* left -z off has no codec in effect, but the field keeps whatever id it
|
||||||
|
* carried (the receiver never dispatches on it without use_compression), so
|
||||||
|
* the wire value round-trips untouched. */
|
||||||
|
if (config->use_compression && config->compression_algo == (int)COMPRESSION_ALGO_NONE) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Compression requested with the 'none' codec");
|
||||||
|
send_error_detail(file_descriptor, "compression requested with the none codec");
|
||||||
|
goto error;
|
||||||
|
}
|
||||||
|
/* rsync: "none" as the pre-transfer checksum is invalid with --checksum. */
|
||||||
|
if (config->checksum && config->checksum_algo == (int)CHECKSUM_ALGO_NONE) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Invalid checksum-choice for --checksum: none");
|
||||||
|
send_error_detail(file_descriptor, "checksum-choice 'none' cannot be used with --checksum");
|
||||||
|
goto error;
|
||||||
|
}
|
||||||
if (!validate_received_config(config)) {
|
if (!validate_received_config(config)) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Invalid configuration received from client");
|
log_message(LOG_LEVEL_ERROR, "Invalid configuration received from client");
|
||||||
send_error_detail(file_descriptor, "invalid configuration received from client");
|
send_error_detail(file_descriptor, "invalid configuration received from client");
|
||||||
|
|||||||
+233
-35
@@ -3,6 +3,7 @@
|
|||||||
|
|
||||||
#include "array_list.h"
|
#include "array_list.h"
|
||||||
#include "checksum.h"
|
#include "checksum.h"
|
||||||
|
#include "compression.h"
|
||||||
#include <stdbool.h>
|
#include <stdbool.h>
|
||||||
#include <stdint.h>
|
#include <stdint.h>
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
@@ -39,15 +40,20 @@ typedef struct BasisDest {
|
|||||||
char* path; /* relative to the destination root (receiver-confined) */
|
char* path; /* relative to the destination root (receiver-confined) */
|
||||||
} BasisDest;
|
} BasisDest;
|
||||||
|
|
||||||
/* One resolved FROM:TO identity-mapping rule (--usermap / --groupmap). Both
|
/* One FROM:TO identity-mapping rule (--usermap / --groupmap). `from`/`from_hi`
|
||||||
* fields are numeric ids. IDENTITY_MATCH_ANY (-1) in `from` is rsync's '*'
|
* describe the sender-side FROM matcher (a single id when from_hi == from, an
|
||||||
* wildcard (matches any transmitted id); IDENTITY_CURRENT (-1) in `to` makes
|
* inclusive LOW-HIGH range, IDENTITY_MATCH_ANY for rsync's '*', or
|
||||||
* the receiver resolve the receiving process's own current euid/egid at apply
|
* IDENTITY_MATCH_UNNAMED for rsync's empty FROM). `to` is the receiver-side TO
|
||||||
* time. Names are resolved to numbers at parse time on the client (see
|
* numeric id (IDENTITY_CURRENT = the receiving process's own euid/egid) UNLESS
|
||||||
* identity.h for the exact subset). */
|
* `to_name` is non-NULL, in which case the receiver resolves the name against
|
||||||
|
* its own account database at apply time (rsync resolves TO names on the
|
||||||
|
* receiver) and `to` is ignored. FROM names/ranges/globs are resolved on the
|
||||||
|
* client (the sender) exactly as rsync matches them against sender names. */
|
||||||
typedef struct {
|
typedef struct {
|
||||||
int32_t from;
|
int32_t from;
|
||||||
|
int32_t from_hi;
|
||||||
int32_t to;
|
int32_t to;
|
||||||
|
char* to_name;
|
||||||
} IdentityMap;
|
} IdentityMap;
|
||||||
|
|
||||||
/* --sockopts=OPTIONS allowlist. Only these option names are accepted; anything
|
/* --sockopts=OPTIONS allowlist. Only these option names are accepted; anything
|
||||||
@@ -76,7 +82,7 @@ typedef struct {
|
|||||||
typedef enum SuperMode { SUPER_MODE_AUTO = 0, SUPER_MODE_ON = 1, SUPER_MODE_OFF = 2 } SuperMode;
|
typedef enum SuperMode { SUPER_MODE_AUTO = 0, SUPER_MODE_ON = 1, SUPER_MODE_OFF = 2 } SuperMode;
|
||||||
|
|
||||||
/* ===========================================================================
|
/* ===========================================================================
|
||||||
* Config wire-field table (single source of truth for protocol 2.21.0).
|
* Config wire-field table (single source of truth for protocol 2.26.0).
|
||||||
*
|
*
|
||||||
* Every field below crosses the wire. The table is the ONLY place a
|
* Every field below crosses the wire. The table is the ONLY place a
|
||||||
* serialized field is named: config.h expands CONFIG_WIRE_FIELDS() to declare
|
* serialized field is named: config.h expands CONFIG_WIRE_FIELDS() to declare
|
||||||
@@ -198,7 +204,7 @@ typedef enum SuperMode { SUPER_MODE_AUTO = 0, SUPER_MODE_ON = 1, SUPER_MODE_OFF
|
|||||||
#define CONFIG_WIRE_FUZZY_FIELDS(X) X(fuzzy, bool, false, BOOL)
|
#define CONFIG_WIRE_FUZZY_FIELDS(X) X(fuzzy, bool, false, BOOL)
|
||||||
|
|
||||||
#define CONFIG_WIRE_CHECKSUM_FIELDS(X) \
|
#define CONFIG_WIRE_CHECKSUM_FIELDS(X) \
|
||||||
X(checksum_algo, int, CHECKSUM_ALGO_XXH64, INT_CHECKSUM_ALGO) \
|
X(checksum_algo, int, CHECKSUM_ALGO_DEFAULT, INT_CHECKSUM_ALGO) \
|
||||||
X(checksum_seed, uint64_t, 0, RAW)
|
X(checksum_seed, uint64_t, 0, RAW)
|
||||||
|
|
||||||
#define CONFIG_WIRE_IDENTITY_FIELDS(X) \
|
#define CONFIG_WIRE_IDENTITY_FIELDS(X) \
|
||||||
@@ -216,7 +222,11 @@ typedef enum SuperMode { SUPER_MODE_AUTO = 0, SUPER_MODE_ON = 1, SUPER_MODE_OFF
|
|||||||
X(preserve_atimes, bool, false, BOOL) \
|
X(preserve_atimes, bool, false, BOOL) \
|
||||||
X(preserve_crtimes, bool, false, BOOL) \
|
X(preserve_crtimes, bool, false, BOOL) \
|
||||||
X(omit_dir_times, bool, false, BOOL) \
|
X(omit_dir_times, bool, false, BOOL) \
|
||||||
X(omit_link_times, bool, false, BOOL)
|
X(omit_link_times, bool, false, BOOL) \
|
||||||
|
X(preserve_perms, bool, false, BOOL) \
|
||||||
|
X(preserve_times, bool, false, BOOL) \
|
||||||
|
X(preserve_owner, bool, false, BOOL) \
|
||||||
|
X(preserve_group, bool, false, BOOL)
|
||||||
|
|
||||||
#define CONFIG_WIRE_SYMLINK_TRUST_FIELDS(X) \
|
#define CONFIG_WIRE_SYMLINK_TRUST_FIELDS(X) \
|
||||||
X(munge_links, bool, false, BOOL) \
|
X(munge_links, bool, false, BOOL) \
|
||||||
@@ -237,6 +247,43 @@ typedef enum SuperMode { SUPER_MODE_AUTO = 0, SUPER_MODE_ON = 1, SUPER_MODE_OFF
|
|||||||
X(copy_as_uid, int32_t, 0, COPY_AS_ID) \
|
X(copy_as_uid, int32_t, 0, COPY_AS_ID) \
|
||||||
X(copy_as_gid, int32_t, 0, COPY_AS_ID)
|
X(copy_as_gid, int32_t, 0, COPY_AS_ID)
|
||||||
|
|
||||||
|
/* Output-parity wave (protocol 2.23.0). report_dest_info tells the receiver to
|
||||||
|
* answer every per-file STATUS_CHECK with a STATUS_DEST_INFO snapshot of the
|
||||||
|
* pre-transfer destination entry (see protocol.h). It is set by the client
|
||||||
|
* only when -i/--itemize-changes or --out-format asks for per-file change
|
||||||
|
* output; the transfer decision itself is unchanged.
|
||||||
|
*
|
||||||
|
* Wire-stats wave (protocol 2.25.0). report_stats tells the receiver to send a
|
||||||
|
* STATUS_STATS frame immediately before its terminal success status carrying
|
||||||
|
* the receiver-only counters (matched data, deleted-file count) and,
|
||||||
|
* for -n/--dry-run --delete, the destination-relative paths it WOULD have
|
||||||
|
* deleted. It is set by the client only when --stats, --progress/-P, an
|
||||||
|
* --out-format token needs a wire counter (%b/%c), or a dry-run carries
|
||||||
|
* --delete; the transfer decision itself is unchanged. */
|
||||||
|
#define CONFIG_WIRE_OUTPUT_FIELDS(X) \
|
||||||
|
X(report_dest_info, bool, false, BOOL) X(report_stats, bool, false, BOOL)
|
||||||
|
|
||||||
|
/* Codec-negotiation wave (protocol 2.26.0). compression_algo is the concrete
|
||||||
|
* codec the client selected for this transfer (a CompressionAlgo id) and is the
|
||||||
|
* value the receiver validates and installs. It is the resolved result of
|
||||||
|
* --compress-choice / the "auto" negotiation so both peers agree exactly.
|
||||||
|
*
|
||||||
|
* Negotiation model: FastSync enforces a strict same-version handshake, so both
|
||||||
|
* peers carry the identical compiled-in codec set. The client resolves the
|
||||||
|
* effective algorithm deterministically and serializes it here; "auto" picks
|
||||||
|
* the first entry of the rsync 3.4.1 preference order
|
||||||
|
* (compression: zstd lz4 zlibx zlib none; checksum: xxh128 xxh3 xxh64 md5 md4
|
||||||
|
* sha1 none), and an explicit request wins. The receiver rejects (before
|
||||||
|
* STATUS_OK) any algorithm outside its own supported set, which is rsync's
|
||||||
|
* "no common choice is an error" behavior. The same resolver runs on both
|
||||||
|
* sides (compression_negotiate_default / checksum_negotiate_default), so the
|
||||||
|
* fallback is consistent.
|
||||||
|
*
|
||||||
|
* The field is appended after the output block so every pre-2.26 field keeps
|
||||||
|
* its wire position. */
|
||||||
|
#define CONFIG_WIRE_CODEC_FIELDS(X) \
|
||||||
|
X(compression_algo, int, COMPRESSION_ALGO_ZSTD, INT_COMPRESSION_ALGO)
|
||||||
|
|
||||||
/* All serialized fields, in exact wire order. Concatenating the per-segment
|
/* All serialized fields, in exact wire order. Concatenating the per-segment
|
||||||
* lists here is what keeps the declaration order = the wire order. */
|
* lists here is what keeps the declaration order = the wire order. */
|
||||||
#define CONFIG_WIRE_FIELDS(X) \
|
#define CONFIG_WIRE_FIELDS(X) \
|
||||||
@@ -257,7 +304,9 @@ typedef enum SuperMode { SUPER_MODE_AUTO = 0, SUPER_MODE_ON = 1, SUPER_MODE_OFF
|
|||||||
CONFIG_WIRE_DAEMON_AUTH_FIELDS(X) \
|
CONFIG_WIRE_DAEMON_AUTH_FIELDS(X) \
|
||||||
CONFIG_WIRE_ICONV_FIELDS(X) \
|
CONFIG_WIRE_ICONV_FIELDS(X) \
|
||||||
CONFIG_WIRE_PRIVILEGE_FIELDS(X) \
|
CONFIG_WIRE_PRIVILEGE_FIELDS(X) \
|
||||||
CONFIG_WIRE_COPY_AS_FIELDS(X)
|
CONFIG_WIRE_COPY_AS_FIELDS(X) \
|
||||||
|
CONFIG_WIRE_OUTPUT_FIELDS(X) \
|
||||||
|
CONFIG_WIRE_CODEC_FIELDS(X)
|
||||||
|
|
||||||
typedef struct Config {
|
typedef struct Config {
|
||||||
/* -j/--threads=N: number of parallel scanner worker threads for the -m
|
/* -j/--threads=N: number of parallel scanner worker threads for the -m
|
||||||
@@ -266,6 +315,15 @@ typedef struct Config {
|
|||||||
* concern and is NEVER serialized into the wire config frame. */
|
* concern and is NEVER serialized into the wire config frame. */
|
||||||
int scanner_threads;
|
int scanner_threads;
|
||||||
bool metadata_explicitly_disabled;
|
bool metadata_explicitly_disabled;
|
||||||
|
/* CLIENT-ONLY (never serialized; not in CONFIG_WIRE_FIELDS). Set when the
|
||||||
|
* user explicitly turned an attribute off with --no-perms / --no-times (long
|
||||||
|
* or short form). --incremental/--delta historically auto-enabled mode and
|
||||||
|
* mtime preservation; these flags let cli_finalize_config restore that
|
||||||
|
* behavior while still honoring the explicit per-attribute negation. A
|
||||||
|
* later -p/-t re-enables the attribute directly, so the flag only prevents
|
||||||
|
* the incremental/delta implication, never a POSITIVE request. */
|
||||||
|
bool preserve_perms_explicit_off;
|
||||||
|
bool preserve_times_explicit_off;
|
||||||
bool show_progress;
|
bool show_progress;
|
||||||
int compression_threads;
|
int compression_threads;
|
||||||
int ssh_port;
|
int ssh_port;
|
||||||
@@ -301,12 +359,14 @@ typedef struct Config {
|
|||||||
char* tls_cert;
|
char* tls_cert;
|
||||||
char* tls_key;
|
char* tls_key;
|
||||||
char* tls_ca;
|
char* tls_ca;
|
||||||
/* --timeout: per-message I/O deadline in seconds. 0 (the default/unset
|
/* --timeout: per-message I/O deadline in seconds. 0 (rsync's default)
|
||||||
* sentinel) leaves the transport's built-in 30 s socket timeout and the
|
* disables the deadline entirely on the client's own socket and protocol
|
||||||
* protocol's built-in 60 s per-message deadline in place; a positive value
|
* layers; a positive value sets it. A server session never inherits the
|
||||||
* overrides both. See protocol_session_set_io_timeout. */
|
* disabled value: it applies the SERVER_IO_TIMEOUT_SEC floor (see
|
||||||
|
* protocol_server_io_timeout_sec and tcp_set_timeouts). */
|
||||||
int timeout;
|
int timeout;
|
||||||
/* --contimeout: connect()/accept timeout, transport layer only. */
|
/* --contimeout: connect()/accept timeout in seconds (rsync's default 60);
|
||||||
|
* 0 disables it. Transport layer only. */
|
||||||
int contimeout;
|
int contimeout;
|
||||||
bool quiet;
|
bool quiet;
|
||||||
bool stats;
|
bool stats;
|
||||||
@@ -341,6 +401,16 @@ typedef struct Config {
|
|||||||
* enters the keep-set. Implied by --delete-missing-args. */
|
* enters the keep-set. Implied by --delete-missing-args. */
|
||||||
bool ignore_missing_args;
|
bool ignore_missing_args;
|
||||||
|
|
||||||
|
/* Codec-negotiation CLI state (all client-only, never serialized). The
|
||||||
|
* effective pre-transfer checksum is Config->checksum_algo (serialized);
|
||||||
|
* checksum_transfer_algo is the rsync "transfer" half of a two-name
|
||||||
|
* --checksum-choice form (validated and used only to mirror rsync's
|
||||||
|
* whole-file forcing, since FastSync's per-block strong hash is fixed).
|
||||||
|
* cli_exit_code carries a parser-requested process exit status (rsync uses 4
|
||||||
|
* for an unsupported checksum/compress algorithm) so main() can mirror it. */
|
||||||
|
int checksum_transfer_algo;
|
||||||
|
int cli_exit_code;
|
||||||
|
|
||||||
// Issue #129: Advanced file selection. These fields are CLIENT-ONLY: they are
|
// Issue #129: Advanced file selection. These fields are CLIENT-ONLY: they are
|
||||||
// never serialized to the wire (the receiver must not learn them).
|
// never serialized to the wire (the receiver must not learn them).
|
||||||
ArrayList* filters; /* --filter=RULE rule strings, in order */
|
ArrayList* filters; /* --filter=RULE rule strings, in order */
|
||||||
@@ -349,6 +419,10 @@ typedef struct Config {
|
|||||||
bool from0; /* -0/--from0: NUL-delimited *-from files */
|
bool from0; /* -0/--from0: NUL-delimited *-from files */
|
||||||
bool cvs_exclude; /* -C/--cvs-exclude: standard CVS ignore set */
|
bool cvs_exclude; /* -C/--cvs-exclude: standard CVS ignore set */
|
||||||
bool per_dir_filter; /* -F: apply per-directory .rsync-filter files */
|
bool per_dir_filter; /* -F: apply per-directory .rsync-filter files */
|
||||||
|
/* -F click count. rsync's single -F means --filter='dir-merge
|
||||||
|
* /.rsync-filter' (the .rsync-filter files themselves are transferred); a
|
||||||
|
* repeated -F adds --filter='- .rsync-filter' so they are excluded too. */
|
||||||
|
int per_dir_filter_count;
|
||||||
bool one_file_system; /* -x/--one-file-system: do not cross filesystem boundaries */
|
bool one_file_system; /* -x/--one-file-system: do not cross filesystem boundaries */
|
||||||
/* --no-implied-dirs: client-only. With -R + --files-from, refuse to place a
|
/* --no-implied-dirs: client-only. With -R + --files-from, refuse to place a
|
||||||
* listed file whose ancestor directory is not itself explicitly listed. */
|
* listed file whose ancestor directory is not itself explicitly listed. */
|
||||||
@@ -519,13 +593,17 @@ typedef struct Config {
|
|||||||
/* rsync deletion-timing family (real from Phase 3). At most one of
|
/* rsync deletion-timing family (real from Phase 3). At most one of
|
||||||
delete_before / delete_during / delete_delay / delete_after may be set, and
|
delete_before / delete_during / delete_delay / delete_after may be set, and
|
||||||
only together with use_delete (the CLI implies --delete for each of them).
|
only together with use_delete (the CLI implies --delete for each of them).
|
||||||
delete_before and delete_during select the EARLY engine mode: the keep-set
|
delete_before selects the EARLY engine mode: the whole-tree keep-set
|
||||||
manifest is transmitted before any file data and extras are removed then,
|
manifest is transmitted before any file data and extras are removed then,
|
||||||
acknowledged, before the first data byte. delete_delay and delete_after
|
acknowledged, before the first data byte. delete_during and delete_delay
|
||||||
select the LATE commit mode: extras are removed only after the whole
|
select the per-directory delete-plan mode (protocol 2.24.0): one plan per
|
||||||
transfer has succeeded (plain --delete keeps this mode). The exact
|
source directory is streamed in directory order, and the receiver removes
|
||||||
semantics and the divergences from rsync are documented in RSYNC_COMPAT.md
|
each directory's extras when its plan arrives (during) or snapshots them
|
||||||
and in config_delete_timing_early() below. */
|
and removes them only after a successful transfer (delay). delete_after
|
||||||
|
(and plain --delete) keep the whole-tree commit mode: extras are removed
|
||||||
|
from a fresh end-of-transfer destination scan only after the whole transfer
|
||||||
|
succeeded. See config_delete_timing_early()/config_delete_timing_per_dir()
|
||||||
|
below. */
|
||||||
/* partial_dir */
|
/* partial_dir */
|
||||||
// PR #174: Partial transfer resumption
|
// PR #174: Partial transfer resumption
|
||||||
/* suffix */
|
/* suffix */
|
||||||
@@ -563,13 +641,17 @@ typedef struct Config {
|
|||||||
* targets and, with -K, follows an in-root destination symlink-to-directory);
|
* targets and, with -K, follows an in-root destination symlink-to-directory);
|
||||||
* -k/--copy-dirlinks is sender-only and is never serialized. */
|
* -k/--copy-dirlinks is sender-only and is never serialized. */
|
||||||
/* numeric_ids */
|
/* numeric_ids */
|
||||||
/* --numeric-ids: no name lookup, use the transmitted numeric ids raw. */
|
/* --numeric-ids: a mapping MODIFIER only -- no name lookup, use the
|
||||||
|
* transmitted numeric ids raw. It does NOT by itself request ownership. */
|
||||||
/* chown_uid_set */
|
/* chown_uid_set */
|
||||||
/* --chown USER (owner) override; IDENTITY_CURRENT = the receiver's euid. */
|
/* --chown USER (owner) override; IDENTITY_CURRENT = the receiver's euid. */
|
||||||
/* chown_gid_set */
|
/* chown_gid_set */
|
||||||
/* --chown :GROUP (group) override; IDENTITY_CURRENT = the receiver's egid. */
|
/* --chown :GROUP (group) override; IDENTITY_CURRENT = the receiver's egid. */
|
||||||
/* usermap */
|
/* usermap */
|
||||||
/* --usermap / --groupmap entries, in order (first match wins). */
|
/* --usermap / --groupmap entries, in order (first match wins). Each entry's
|
||||||
|
* from/from_hi are a single id, an inclusive range, IDENTITY_MATCH_ANY ('*'),
|
||||||
|
* or IDENTITY_MATCH_UNNAMED (empty FROM); to_name carries a receiver-resolved
|
||||||
|
* TO name (rsync resolves TO names on the receiving side). */
|
||||||
/* preserve_atimes */
|
/* preserve_atimes */
|
||||||
/* -U/--atimes: preserve source access times on the destination. */
|
/* -U/--atimes: preserve source access times on the destination. */
|
||||||
/* preserve_crtimes */
|
/* preserve_crtimes */
|
||||||
@@ -579,10 +661,29 @@ typedef struct Config {
|
|||||||
/* -O/--omit-dir-times: do not apply mtimes to directories. */
|
/* -O/--omit-dir-times: do not apply mtimes to directories. */
|
||||||
/* omit_link_times */
|
/* omit_link_times */
|
||||||
/* -J/--omit-link-times: do not apply times to symlinks. */
|
/* -J/--omit-link-times: do not apply times to symlinks. */
|
||||||
|
/* preserve_perms */
|
||||||
|
/* -p/--perms: preserve the source permission bits (mode). One of the four
|
||||||
|
* per-attribute preservation flags split out of the former single
|
||||||
|
* use_metadata bundle; --chmod and -A/--acls also imply it. */
|
||||||
|
/* preserve_times */
|
||||||
|
/* -t/--times: preserve source modification times. Split out of the former
|
||||||
|
* use_metadata bundle; --preserve and -a/--archive imply it. */
|
||||||
|
/* preserve_owner */
|
||||||
|
/* -o/--owner: preserve the source owner (uid). Split out of the former
|
||||||
|
* use_metadata bundle; --usermap/--chown (and, when a uid is requested,
|
||||||
|
* --copy-as) imply it. Owner application still requires receiver privilege
|
||||||
|
* and is gated separately by the identity flags. */
|
||||||
|
/* preserve_group */
|
||||||
|
/* -g/--group: preserve the source group (gid). Split out of the former
|
||||||
|
* use_metadata bundle; --groupmap/--chown (and, when a gid is requested,
|
||||||
|
* --copy-as) imply it. */
|
||||||
/* fake_super */
|
/* fake_super */
|
||||||
/* --fake-super: receiver-only. When set, each written file additionally gets
|
/* --fake-super: receiver-only. When set, each written file additionally gets
|
||||||
* a reserved user.fastsync.stat xattr recording the source uid/gid/mode/mtime
|
* a reserved user.fastsync.stat xattr recording the RESOLVED uid/gid (the
|
||||||
* so a later privileged restore could re-apply them. Crosses the wire. */
|
* source's own when no ownership request is active, else the --chown/--usermap
|
||||||
|
* result) plus mode/mtime so a later privileged restore could re-apply them.
|
||||||
|
* It NEVER real-chowns: the point is to record the source ownership on an
|
||||||
|
* unprivileged receiver. Crosses the wire. */
|
||||||
/* module */
|
/* module */
|
||||||
/* Daemon module selection (Wave A, protocol 2.15.0). Client-composed from a
|
/* Daemon module selection (Wave A, protocol 2.15.0). Client-composed from a
|
||||||
* host::module/path destination; NULL or "" means "no module" (the ordinary
|
* host::module/path destination; NULL or "" means "no module" (the ordinary
|
||||||
@@ -623,7 +724,7 @@ typedef struct Config {
|
|||||||
* fd-relative confinement (file_open_secure_parent, O_NOFOLLOW, root checks);
|
* fd-relative confinement (file_open_secure_parent, O_NOFOLLOW, root checks);
|
||||||
* --super only permits an attempt that is already confined. Crosses the wire
|
* --super only permits an attempt that is already confined. Crosses the wire
|
||||||
* as a trailing int so the receiver can enforce the policy. See
|
* as a trailing int so the receiver can enforce the policy. See
|
||||||
* privilege_super_permitted() and identity_ownership_requested() in
|
* privilege_super_permitted() and identity_explicit_ownership_requested() in
|
||||||
* identity.h. */
|
* identity.h. */
|
||||||
/* copy_as_set */
|
/* copy_as_set */
|
||||||
/* --copy-as=USER[:GROUP] (P7 Wave E, protocol 2.18.0). Safe-subset
|
/* --copy-as=USER[:GROUP] (P7 Wave E, protocol 2.18.0). Safe-subset
|
||||||
@@ -803,8 +904,91 @@ typedef struct Config {
|
|||||||
* unknown status, or the unconsumed detail body, and the strict same-version
|
* unknown status, or the unconsumed detail body, and the strict same-version
|
||||||
* handshake (config_receive rejects a mismatched version before parsing
|
* handshake (config_receive rejects a mismatched version before parsing
|
||||||
* anything else) is what keeps a 2.21 client and a 2.20 server from ever
|
* anything else) is what keeps a 2.21 client and a 2.20 server from ever
|
||||||
* reaching that state. */
|
* reaching that state.
|
||||||
#define PROTOCOL_VERSION "2.21.0"
|
*
|
||||||
|
* Preserve-Attribute Split Wave: 2.21.0 -> 2.22.0.
|
||||||
|
*
|
||||||
|
* WHY the bump, grounded in the wire: this wave splits the former single
|
||||||
|
* use_metadata bundle into four independent rsync-compatible preservation
|
||||||
|
* attributes (preserve_perms / preserve_times / preserve_owner /
|
||||||
|
* preserve_group) so -p/-t/-o/-g (and their --no-* negations) become real
|
||||||
|
* drop-in flags. The binary config frame gains four serialized bools appended
|
||||||
|
* to CONFIG_WIRE_METADATA_TIMES_FIELDS after omit_link_times, in this fixed
|
||||||
|
* order: preserve_perms, preserve_times, preserve_owner, preserve_group. Any
|
||||||
|
* config-frame layout change must bump the protocol version: a peer that does
|
||||||
|
* not parse the new trailing bytes would desynchronize on the frame boundary,
|
||||||
|
* and the strict same-version handshake (config_receive rejects a mismatched
|
||||||
|
* version before parsing anything else) is what keeps a 2.22 client and a 2.21
|
||||||
|
* server from ever reaching that state. The fixed-width FileMetadata layout is
|
||||||
|
* UNCHANGED: the receiver still gates attribute application on use_metadata,
|
||||||
|
* which is now DERIVED from these attributes by config_derived_use_metadata().
|
||||||
|
*
|
||||||
|
* Rsync-Parity Wave: 2.22.0 -> 2.23.0.
|
||||||
|
*
|
||||||
|
* WHY the bump, grounded in the wire. Several independent changes land in this
|
||||||
|
* protocol version:
|
||||||
|
*
|
||||||
|
* (1) Ownership parity (#286/#294): each --usermap/--groupmap wire entry grows
|
||||||
|
* from two int32s to [from][from_hi][to][to_name]; `from_hi` carries an
|
||||||
|
* inclusive LOW-HIGH range (== from for a single/any/unnamed matcher) and the
|
||||||
|
* trailing string carries a TO NAME for the receiver to resolve (rsync resolves
|
||||||
|
* TO names on the receiving side). The STATUS_MKDIR and STATUS_DIR_TIMES frames
|
||||||
|
* also gain a bounded per-entry xattr block when -X/-A is negotiated, so
|
||||||
|
* directory xattrs/ACLs (including default ACLs) are preserved like regular-file
|
||||||
|
* xattrs.
|
||||||
|
*
|
||||||
|
* (2) Delete semantics (#290): the delete-manifest frame gains a fourth trailing
|
||||||
|
* section -- a synchronized-directory count followed by that many
|
||||||
|
* destination-relative directory paths (the receive root is "."). The receiver
|
||||||
|
* confines its extras walk to these directories, so `--files-from` with
|
||||||
|
* `--delete` only removes inside listed directory subtrees (rsync parity)
|
||||||
|
* instead of deleting every untransmitted path under the receive root. The
|
||||||
|
* frame stream also gains STATUS_DELETE_LIMIT, the terminal success status sent
|
||||||
|
* instead of STATUS_OK when a --max-delete commit removes up to the bound and
|
||||||
|
* skips the rest (the sender then exits 25 like rsync).
|
||||||
|
*
|
||||||
|
* Any config-frame layout or frame-sequence change must bump the protocol
|
||||||
|
* version: a 2.22 peer would desynchronize on the new entry bytes, the extra
|
||||||
|
* trailing section or the unknown status, and the strict same-version handshake
|
||||||
|
* (config_receive rejects a mismatched version before parsing anything else) is
|
||||||
|
* what keeps a 2.23 client and a 2.22 server from ever reaching that state.
|
||||||
|
*
|
||||||
|
* (3) Output parity (#291/#292): -i/--itemize-changes and --out-format must
|
||||||
|
* compare the source against the PRE-TRANSFER destination entry (new vs
|
||||||
|
* modified, and which of size/time/perms/owner/group differ), but FastSync's
|
||||||
|
* push sender never sees the destination. The receiver therefore answers a
|
||||||
|
* per-file STATUS_CHECK with a new STATUS_DEST_INFO frame (a fixed-width
|
||||||
|
* snapshot of the old entry) before its ordinary verdict when the config frame
|
||||||
|
* carries the new report_dest_info bool appended after the --copy-as block.
|
||||||
|
* This is both a config-frame layout change (one trailing bool) and a frame
|
||||||
|
* sequence change (the new status).
|
||||||
|
*
|
||||||
|
* (4) Delete timing (protocol 2.24.0): the sender streams one delete plan per
|
||||||
|
* source directory so --delete-during/--delete-delay reproduce rsync's deletion
|
||||||
|
* timing (the plan fields and STATUS_DELETE_PLAN are documented at the keep-set
|
||||||
|
* / delete-plan definitions below).
|
||||||
|
*
|
||||||
|
* (5) Wire-stats parity (protocol 2.25.0): --stats, --progress/-P and the
|
||||||
|
* --out-format %b/%c tokens need receiver-only and wire counters that the push
|
||||||
|
* sender cannot observe, and -n/--dry-run --delete must report the extras it
|
||||||
|
* would have removed without deleting anything. The config frame gains one
|
||||||
|
* trailing report_stats bool and the receiver emits a new STATUS_STATS frame
|
||||||
|
* (carrying matched data, the deleted-file count and the would-delete path
|
||||||
|
* list) immediately before its terminal success status.
|
||||||
|
*
|
||||||
|
* (6) Codec breadth + negotiation (protocol 2.26.0): the config frame gains one
|
||||||
|
* trailing int, compression_algo (a CompressionAlgo id), appended after the
|
||||||
|
* output block. It is the negotiated/effective compression codec and is what
|
||||||
|
* the receiver's self-describing decompressor validates against its own
|
||||||
|
* supported set. The checksum_algo wire value now also accepts md4/sha1/none,
|
||||||
|
* and its default changes to the rsync 3.4.1 auto-negotiated xxh128.
|
||||||
|
*
|
||||||
|
* Any config-frame layout change must bump the protocol version: a peer that
|
||||||
|
* does not parse the new trailing bytes would desynchronize on the frame
|
||||||
|
* boundary, and the strict same-version handshake (config_receive rejects a
|
||||||
|
* mismatched version before parsing anything else) keeps mixed deployments from
|
||||||
|
* ever reaching that state. */
|
||||||
|
#define PROTOCOL_VERSION "2.26.0"
|
||||||
#define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024)
|
#define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024)
|
||||||
/* Upper bound on total basis-dir entries (rsync caps --link-dest at 20). */
|
/* Upper bound on total basis-dir entries (rsync caps --link-dest at 20). */
|
||||||
#define MAX_BASIS_DIRS 64
|
#define MAX_BASIS_DIRS 64
|
||||||
@@ -829,9 +1013,11 @@ typedef struct Config {
|
|||||||
|
|
||||||
/* Identity-mapping sentinels and bounds (see identity.h for semantics).
|
/* Identity-mapping sentinels and bounds (see identity.h for semantics).
|
||||||
* IDENTITY_MATCH_ANY is a usermap/groupmap FROM '*' (matches any id);
|
* IDENTITY_MATCH_ANY is a usermap/groupmap FROM '*' (matches any id);
|
||||||
* IDENTITY_CURRENT is a chown / map TO '*' (resolve to the receiver's current
|
* IDENTITY_MATCH_UNNAMED is a FROM with an empty token (rsync's "ids with no
|
||||||
* euid/egid at apply time). */
|
* name on the sender"); IDENTITY_CURRENT is a chown / map TO '*' (resolve to
|
||||||
|
* the receiver's current euid/egid at apply time). */
|
||||||
#define IDENTITY_MATCH_ANY (-1)
|
#define IDENTITY_MATCH_ANY (-1)
|
||||||
|
#define IDENTITY_MATCH_UNNAMED (-2)
|
||||||
#define IDENTITY_CURRENT (-1)
|
#define IDENTITY_CURRENT (-1)
|
||||||
#define MAX_IDENTITY_MAP 128
|
#define MAX_IDENTITY_MAP 128
|
||||||
|
|
||||||
@@ -896,13 +1082,17 @@ int config_parse_daemon_dest(Config* config);
|
|||||||
* 0. */
|
* 0. */
|
||||||
int config_parse_transport_dest(Config* config);
|
int config_parse_transport_dest(Config* config);
|
||||||
|
|
||||||
/* True when the negotiated delete timing performs the extra-file deletion
|
/* True for the whole-tree delete-before timing: a complete keep-set manifest is
|
||||||
* BEFORE the transfer data (--delete-before / --delete-during). The flag is
|
* transmitted before any data and committed (with an ack) before the first data
|
||||||
* a pure function of the config and is used identically on the sender (to pick
|
* byte. Pure function of the config, used identically on the sender (to pick
|
||||||
* the manifest-first frame order) and the receiver (to delete when the early
|
* the manifest-first frame order) and the receiver (to delete when the early
|
||||||
* manifest arrives). When false the deletion is committed only after the whole
|
* manifest arrives). */
|
||||||
* transfer succeeded (--delete / --delete-after / --delete-delay). */
|
|
||||||
bool config_delete_timing_early(const Config* config);
|
bool config_delete_timing_early(const Config* config);
|
||||||
|
/* True for the per-directory timings (--delete-during / --delete-delay). The
|
||||||
|
* sender streams a delete plan per source directory in directory order; the
|
||||||
|
* receiver applies each plan on arrival (during) or snapshots its extras and
|
||||||
|
* commits them only after a fully-successful transfer (delay). */
|
||||||
|
bool config_delete_timing_per_dir(const Config* config);
|
||||||
/* Delete-timing sanity: with deletion enabled at most one timing flag may be
|
/* Delete-timing sanity: with deletion enabled at most one timing flag may be
|
||||||
* set (none = the default delete-after commit timing); without deletion no
|
* set (none = the default delete-after commit timing); without deletion no
|
||||||
* timing flag may be set (each timing flag implies --delete). */
|
* timing flag may be set (each timing flag implies --delete). */
|
||||||
@@ -918,6 +1108,14 @@ bool config_has_valid_delete_timing(const Config* config);
|
|||||||
* validate_received_config() so the receiver enforces exactly the same
|
* validate_received_config() so the receiver enforces exactly the same
|
||||||
* invariants it relies on (the server is the trust boundary). */
|
* invariants it relies on (the server is the trust boundary). */
|
||||||
const char* config_invariants_error(const Config* config);
|
const char* config_invariants_error(const Config* config);
|
||||||
|
/* Single source of truth for the DERIVED transport bit (use_metadata): true
|
||||||
|
* when any configured preservation/ownership option requires the metadata
|
||||||
|
* frame to travel. Returns false when no such option is set (a bare run).
|
||||||
|
* This is a pure predicate over the config; the client lowers it into
|
||||||
|
* Config->use_metadata at the end of parsing so every implication (devices,
|
||||||
|
* executability, identity maps, incremental/delta, ...) is centralized here
|
||||||
|
* rather than scattered as direct writes. */
|
||||||
|
bool config_derived_use_metadata(const Config* config);
|
||||||
/* True when at least one --compare-dest/--copy-dest/--link-dest was set. */
|
/* True when at least one --compare-dest/--copy-dest/--link-dest was set. */
|
||||||
bool config_has_basis(const Config* config);
|
bool config_has_basis(const Config* config);
|
||||||
/* Append one basis-dir entry. Returns 0 on success, -1 on allocation failure. */
|
/* Append one basis-dir entry. Returns 0 on success, -1 on allocation failure. */
|
||||||
|
|||||||
@@ -264,6 +264,20 @@ static bool delay_publish_entry(DelayUpdatesContext* context, const Config* conf
|
|||||||
const StagedFileEntry* entry) {
|
const StagedFileEntry* entry) {
|
||||||
if (!delay_publish_backup(context, config, entry))
|
if (!delay_publish_backup(context, config, entry))
|
||||||
return false;
|
return false;
|
||||||
|
/* --force: an incoming regular file/symlink may replace a destination
|
||||||
|
DIRECTORY (possibly non-empty). The immediate-install path handles this in
|
||||||
|
file_receive; a --delay-updates run stages elsewhere and only discovers the
|
||||||
|
blocking directory here, so clear it before the rename (rsync's
|
||||||
|
"could not make way for new regular file" without --force). */
|
||||||
|
if (config && config->force_delete && file_directory_exists_secure(entry->final_path)) {
|
||||||
|
if (!file_remove_tree_secure(entry->final_path)) {
|
||||||
|
char* escaped = output_escape(entry->final_path, false);
|
||||||
|
log_message(LOG_LEVEL_ERROR, "could not remove destination directory blocking '%s': %s",
|
||||||
|
escaped ? escaped : "<allocation failed>", strerror(errno));
|
||||||
|
free(escaped);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
if (!file_rename_secure(entry->staged_path, entry->final_path)) {
|
if (!file_rename_secure(entry->staged_path, entry->final_path)) {
|
||||||
if (errno == EXDEV) {
|
if (errno == EXDEV) {
|
||||||
char* escaped = output_escape(entry->final_path, false);
|
char* escaped = output_escape(entry->final_path, false);
|
||||||
|
|||||||
@@ -0,0 +1,893 @@
|
|||||||
|
#include "delete_plan.h"
|
||||||
|
|
||||||
|
#include "charset.h"
|
||||||
|
#include "delay_updates.h"
|
||||||
|
#include "file.h"
|
||||||
|
#include "log.h"
|
||||||
|
#include "utils.h"
|
||||||
|
#include <dirent.h>
|
||||||
|
#include <errno.h>
|
||||||
|
#include <fcntl.h>
|
||||||
|
#include <limits.h>
|
||||||
|
#include <stdint.h>
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
|
#include <sys/stat.h>
|
||||||
|
#include <unistd.h>
|
||||||
|
|
||||||
|
/* Mirrors MAX_SERVER_DELETE_COUNT in file_receive.c: the server's hard bound on
|
||||||
|
* the number of entries one deletion commit may remove. A client
|
||||||
|
* --max-delete=NUM smaller than this replaces it for the run. */
|
||||||
|
#define DELETE_PLAN_SERVER_LIMIT 100000U
|
||||||
|
|
||||||
|
/* ------------------------------------------------------------------ */
|
||||||
|
/* Sender: plan builder */
|
||||||
|
/* ------------------------------------------------------------------ */
|
||||||
|
|
||||||
|
typedef struct PlanNode {
|
||||||
|
char* dir;
|
||||||
|
ArrayList* files; /* basenames kept directly in dir */
|
||||||
|
ArrayList* dirs; /* basenames of kept child directories */
|
||||||
|
bool sent;
|
||||||
|
struct PlanNode* hash_next;
|
||||||
|
} PlanNode;
|
||||||
|
|
||||||
|
struct DeletePlanSender {
|
||||||
|
PlanNode** buckets;
|
||||||
|
size_t capacity;
|
||||||
|
size_t count;
|
||||||
|
bool config_sent;
|
||||||
|
bool all_synced;
|
||||||
|
const ArrayList* synced_dirs;
|
||||||
|
/* Owned by the caller's synced_dirs list; non-NULL only for a general -R
|
||||||
|
transfer, where it is the destination prefix the delete walk is confined
|
||||||
|
to. NULL means the whole receive root (or a --files-from scope). */
|
||||||
|
const char* walk_root;
|
||||||
|
const ArrayList* protected_prefixes;
|
||||||
|
const ArrayList* size_skipped;
|
||||||
|
const ArrayList* missing_args;
|
||||||
|
size_t entries;
|
||||||
|
/* Transmitted FILE entries only. The caller's "empty scan" safety guard keys
|
||||||
|
off this (an I/O error that hid every file must refuse to delete even when
|
||||||
|
some directories were traversed), so directory keep entries do not count. */
|
||||||
|
size_t file_entries;
|
||||||
|
};
|
||||||
|
|
||||||
|
static size_t plan_hash(const char* key) {
|
||||||
|
size_t h = 5381;
|
||||||
|
for (const unsigned char* p = (const unsigned char*)key; *p; p++)
|
||||||
|
h = ((h << 5) + h) + *p;
|
||||||
|
return h;
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool list_contains_str(const ArrayList* list, const char* value) {
|
||||||
|
if (!list)
|
||||||
|
return false;
|
||||||
|
for (int i = 0; i < list->size; i++) {
|
||||||
|
if (strcmp((const char*)list->items[i], value) == 0)
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool list_add_str_unique(ArrayList* list, const char* value) {
|
||||||
|
if (!list || !value)
|
||||||
|
return false;
|
||||||
|
if (list_contains_str(list, value))
|
||||||
|
return true;
|
||||||
|
char* copy = str_dup(value);
|
||||||
|
if (!copy)
|
||||||
|
return false;
|
||||||
|
if (!array_list_add(list, copy)) {
|
||||||
|
free(copy);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
DeletePlanSender* delete_plan_sender_create(void) {
|
||||||
|
DeletePlanSender* sender = calloc(1, sizeof(DeletePlanSender));
|
||||||
|
if (!sender)
|
||||||
|
return NULL;
|
||||||
|
sender->capacity = 64;
|
||||||
|
sender->buckets = calloc(sender->capacity, sizeof(PlanNode*));
|
||||||
|
if (!sender->buckets) {
|
||||||
|
free(sender);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
sender->all_synced = true;
|
||||||
|
return sender;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void plan_node_destroy(PlanNode* node) {
|
||||||
|
if (!node)
|
||||||
|
return;
|
||||||
|
free(node->dir);
|
||||||
|
array_list_delete(node->files);
|
||||||
|
array_list_delete(node->dirs);
|
||||||
|
free(node);
|
||||||
|
}
|
||||||
|
|
||||||
|
void delete_plan_sender_destroy(DeletePlanSender* sender) {
|
||||||
|
if (!sender)
|
||||||
|
return;
|
||||||
|
for (size_t i = 0; i < sender->capacity; i++) {
|
||||||
|
PlanNode* node = sender->buckets[i];
|
||||||
|
while (node) {
|
||||||
|
PlanNode* next = node->hash_next;
|
||||||
|
plan_node_destroy(node);
|
||||||
|
node = next;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
free(sender->buckets);
|
||||||
|
free(sender);
|
||||||
|
}
|
||||||
|
|
||||||
|
static PlanNode* plan_find(const DeletePlanSender* sender, const char* dir) {
|
||||||
|
size_t index = plan_hash(dir) & (sender->capacity - 1);
|
||||||
|
for (PlanNode* node = sender->buckets[index]; node; node = node->hash_next) {
|
||||||
|
if (strcmp(node->dir, dir) == 0)
|
||||||
|
return node;
|
||||||
|
}
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool plan_grow(DeletePlanSender* sender) {
|
||||||
|
size_t new_capacity = sender->capacity * 2;
|
||||||
|
PlanNode** buckets = calloc(new_capacity, sizeof(PlanNode*));
|
||||||
|
if (!buckets)
|
||||||
|
return false;
|
||||||
|
for (size_t i = 0; i < sender->capacity; i++) {
|
||||||
|
PlanNode* node = sender->buckets[i];
|
||||||
|
while (node) {
|
||||||
|
PlanNode* next = node->hash_next;
|
||||||
|
size_t index = plan_hash(node->dir) & (new_capacity - 1);
|
||||||
|
node->hash_next = buckets[index];
|
||||||
|
buckets[index] = node;
|
||||||
|
node = next;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
free(sender->buckets);
|
||||||
|
sender->buckets = buckets;
|
||||||
|
sender->capacity = new_capacity;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
static PlanNode* plan_ensure(DeletePlanSender* sender, const char* dir) {
|
||||||
|
PlanNode* node = plan_find(sender, dir);
|
||||||
|
if (node)
|
||||||
|
return node;
|
||||||
|
if (sender->count + 1 > sender->capacity * 3 / 4 && !plan_grow(sender))
|
||||||
|
return NULL;
|
||||||
|
node = calloc(1, sizeof(PlanNode));
|
||||||
|
if (!node)
|
||||||
|
return NULL;
|
||||||
|
node->dir = str_dup(dir);
|
||||||
|
node->files = array_list_create(free);
|
||||||
|
node->dirs = array_list_create(free);
|
||||||
|
if (!node->dir || !node->files || !node->dirs) {
|
||||||
|
plan_node_destroy(node);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
size_t index = plan_hash(dir) & (sender->capacity - 1);
|
||||||
|
node->hash_next = sender->buckets[index];
|
||||||
|
sender->buckets[index] = node;
|
||||||
|
sender->count++;
|
||||||
|
return node;
|
||||||
|
}
|
||||||
|
|
||||||
|
static char* path_parent_dir(const char* path) {
|
||||||
|
const char* slash = strrchr(path, '/');
|
||||||
|
if (!slash)
|
||||||
|
return str_dup(".");
|
||||||
|
if (slash == path)
|
||||||
|
return str_dup(".");
|
||||||
|
size_t len = (size_t)(slash - path);
|
||||||
|
char* parent = malloc(len + 1);
|
||||||
|
if (!parent)
|
||||||
|
return NULL;
|
||||||
|
memcpy(parent, path, len);
|
||||||
|
parent[len] = '\0';
|
||||||
|
return parent;
|
||||||
|
}
|
||||||
|
|
||||||
|
static char* path_base_name(const char* path) {
|
||||||
|
const char* slash = strrchr(path, '/');
|
||||||
|
return str_dup(slash ? slash + 1 : path);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Copy `path`, stripping a leading '/' and any trailing '/'. */
|
||||||
|
static char* plan_clean_path(const char* path) {
|
||||||
|
while (*path == '/')
|
||||||
|
path++;
|
||||||
|
size_t len = strlen(path);
|
||||||
|
while (len > 0 && path[len - 1] == '/')
|
||||||
|
len--;
|
||||||
|
char* clean = malloc(len + 1);
|
||||||
|
if (!clean)
|
||||||
|
return NULL;
|
||||||
|
memcpy(clean, path, len);
|
||||||
|
clean[len] = '\0';
|
||||||
|
return clean;
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool plan_ensure_ancestors(DeletePlanSender* sender, const char* dir) {
|
||||||
|
char* current = str_dup(dir);
|
||||||
|
if (!current)
|
||||||
|
return false;
|
||||||
|
bool ok = true;
|
||||||
|
while (strcmp(current, ".") != 0) {
|
||||||
|
char* parent = path_parent_dir(current);
|
||||||
|
char* base = path_base_name(current);
|
||||||
|
PlanNode* parent_node = parent ? plan_ensure(sender, parent) : NULL;
|
||||||
|
if (!parent || !base || !parent_node || !list_add_str_unique(parent_node->dirs, base)) {
|
||||||
|
ok = false;
|
||||||
|
free(parent);
|
||||||
|
free(base);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
free(base);
|
||||||
|
free(current);
|
||||||
|
current = parent;
|
||||||
|
}
|
||||||
|
free(current);
|
||||||
|
return ok;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool delete_plan_sender_add(DeletePlanSender* sender, const char* path, bool is_dir) {
|
||||||
|
if (!sender || !path)
|
||||||
|
return false;
|
||||||
|
char* clean = plan_clean_path(path);
|
||||||
|
if (!clean)
|
||||||
|
return false;
|
||||||
|
if (*clean == '\0') {
|
||||||
|
free(clean);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
char* parent = path_parent_dir(clean);
|
||||||
|
char* base = path_base_name(clean);
|
||||||
|
PlanNode* parent_node = parent ? plan_ensure(sender, parent) : NULL;
|
||||||
|
bool ok = parent && base && parent_node;
|
||||||
|
if (ok) {
|
||||||
|
if (is_dir) {
|
||||||
|
ok = list_add_str_unique(parent_node->dirs, base) && plan_ensure(sender, clean) != NULL;
|
||||||
|
} else {
|
||||||
|
ok = list_add_str_unique(parent_node->files, base);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (ok)
|
||||||
|
ok = plan_ensure_ancestors(sender, parent);
|
||||||
|
if (ok) {
|
||||||
|
sender->entries++;
|
||||||
|
if (!is_dir)
|
||||||
|
sender->file_entries++;
|
||||||
|
}
|
||||||
|
free(clean);
|
||||||
|
free(parent);
|
||||||
|
free(base);
|
||||||
|
return ok;
|
||||||
|
}
|
||||||
|
|
||||||
|
void delete_plan_sender_finalize(DeletePlanSender* sender, const ArrayList* synced_dirs,
|
||||||
|
const char* walk_root) {
|
||||||
|
if (!sender)
|
||||||
|
return;
|
||||||
|
sender->synced_dirs = synced_dirs;
|
||||||
|
sender->all_synced = synced_dirs == NULL && walk_root == NULL;
|
||||||
|
sender->walk_root = walk_root;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool delete_plan_sender_empty(const DeletePlanSender* sender) {
|
||||||
|
return !sender || sender->file_entries == 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
void delete_plan_sender_set_config(DeletePlanSender* sender, const ArrayList* protected_prefixes,
|
||||||
|
const ArrayList* size_skipped, const ArrayList* missing_args) {
|
||||||
|
if (!sender)
|
||||||
|
return;
|
||||||
|
sender->protected_prefixes = protected_prefixes;
|
||||||
|
sender->size_skipped = size_skipped;
|
||||||
|
sender->missing_args = missing_args;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* True when `dir` is `root` itself or a descendant of it (path-component
|
||||||
|
* aware, so "foo" does not match "foobar"). */
|
||||||
|
static bool path_at_or_under(const char* dir, const char* root) {
|
||||||
|
if (!dir || !root)
|
||||||
|
return false;
|
||||||
|
size_t n = strlen(root);
|
||||||
|
return strncmp(dir, root, n) == 0 && (dir[n] == '\0' || dir[n] == '/');
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool plan_is_allowed(const DeletePlanSender* sender, const char* dir) {
|
||||||
|
if (sender->all_synced)
|
||||||
|
return true;
|
||||||
|
if (sender->walk_root)
|
||||||
|
return path_at_or_under(dir, sender->walk_root);
|
||||||
|
return list_contains_str(sender->synced_dirs, dir);
|
||||||
|
}
|
||||||
|
|
||||||
|
static int send_str_section(int fd, const ArrayList* list) {
|
||||||
|
int count = list ? list->size : 0;
|
||||||
|
if (!send_int(fd, count))
|
||||||
|
return -1;
|
||||||
|
for (int i = 0; i < count; i++) {
|
||||||
|
if (!send_wire_str(fd, (const char*)list->items[i]))
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
static int send_plan_node(int fd, DeletePlanSender* sender, PlanNode* node) {
|
||||||
|
if (!send_status(fd, STATUS_DELETE_PLAN))
|
||||||
|
return -1;
|
||||||
|
if (!send_int(fd, sender->config_sent ? 0 : 1))
|
||||||
|
return -1;
|
||||||
|
if (!sender->config_sent) {
|
||||||
|
if (send_str_section(fd, sender->protected_prefixes) != 0 ||
|
||||||
|
send_str_section(fd, sender->size_skipped) != 0 ||
|
||||||
|
send_str_section(fd, sender->missing_args) != 0)
|
||||||
|
return -1;
|
||||||
|
sender->config_sent = true;
|
||||||
|
}
|
||||||
|
if (!send_wire_str(fd, node->dir))
|
||||||
|
return -1;
|
||||||
|
if (send_str_section(fd, node->dirs) != 0 || send_str_section(fd, node->files) != 0)
|
||||||
|
return -1;
|
||||||
|
node->sent = true;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
static int send_prefix_plan(int fd, DeletePlanSender* sender, const char* dir) {
|
||||||
|
PlanNode* node = plan_find(sender, dir);
|
||||||
|
if (!node || node->sent)
|
||||||
|
return 0;
|
||||||
|
if (!plan_is_allowed(sender, dir))
|
||||||
|
return 0;
|
||||||
|
return send_plan_node(fd, sender, node);
|
||||||
|
}
|
||||||
|
|
||||||
|
int delete_plan_send_root(int fd, DeletePlanSender* sender) {
|
||||||
|
if (!sender)
|
||||||
|
return -1;
|
||||||
|
const char* root = sender->walk_root ? sender->walk_root : ".";
|
||||||
|
if (!plan_ensure(sender, root))
|
||||||
|
return -1;
|
||||||
|
return send_prefix_plan(fd, sender, root);
|
||||||
|
}
|
||||||
|
|
||||||
|
int delete_plan_send_for_path(int fd, DeletePlanSender* sender, const char* path, bool is_dir) {
|
||||||
|
if (!sender || !path)
|
||||||
|
return -1;
|
||||||
|
char* clean = plan_clean_path(path);
|
||||||
|
if (!clean)
|
||||||
|
return -1;
|
||||||
|
/* The walk root (the -R prefix, or ".") is sent up front by
|
||||||
|
delete_plan_send_root(); never emit the receive-root plan for a scoped -R
|
||||||
|
run, whose "." keep list would delete the prefix's siblings. */
|
||||||
|
int rc = sender->walk_root ? 0 : send_prefix_plan(fd, sender, ".");
|
||||||
|
if (rc == 0 && *clean != '\0') {
|
||||||
|
size_t len = strlen(clean);
|
||||||
|
size_t end = len;
|
||||||
|
if (!is_dir) {
|
||||||
|
const char* slash = strrchr(clean, '/');
|
||||||
|
end = slash ? (size_t)(slash - clean) : 0;
|
||||||
|
}
|
||||||
|
for (size_t i = 1; i <= end && rc == 0; i++) {
|
||||||
|
if (i == end || clean[i] == '/') {
|
||||||
|
char* prefix = malloc(i + 1);
|
||||||
|
if (!prefix) {
|
||||||
|
rc = -1;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
memcpy(prefix, clean, i);
|
||||||
|
prefix[i] = '\0';
|
||||||
|
rc = send_prefix_plan(fd, sender, prefix);
|
||||||
|
free(prefix);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
free(clean);
|
||||||
|
return rc;
|
||||||
|
}
|
||||||
|
|
||||||
|
int delete_plan_send_remaining(int fd, DeletePlanSender* sender, const ArrayList* dirs) {
|
||||||
|
if (!sender || !dirs)
|
||||||
|
return 0;
|
||||||
|
for (int i = 0; i < dirs->size; i++) {
|
||||||
|
const char* dir = (const char*)dirs->items[i];
|
||||||
|
if (delete_plan_send_for_path(fd, sender, dir, true) != 0)
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ------------------------------------------------------------------ */
|
||||||
|
/* Receiver: delete session */
|
||||||
|
/* ------------------------------------------------------------------ */
|
||||||
|
|
||||||
|
struct DeletePlanSession {
|
||||||
|
bool defer;
|
||||||
|
bool dry_run;
|
||||||
|
size_t max_delete;
|
||||||
|
size_t deleted;
|
||||||
|
size_t skipped;
|
||||||
|
bool limit_hit;
|
||||||
|
bool limit_logged;
|
||||||
|
bool config_seen;
|
||||||
|
bool missing_applied;
|
||||||
|
ArrayList* protected_prefixes;
|
||||||
|
ArrayList* size_skipped;
|
||||||
|
ArrayList* missing;
|
||||||
|
ArrayList* deferred;
|
||||||
|
};
|
||||||
|
|
||||||
|
DeletePlanSession* delete_plan_session_create(const Config* config) {
|
||||||
|
if (!config)
|
||||||
|
return NULL;
|
||||||
|
DeletePlanSession* session = calloc(1, sizeof(DeletePlanSession));
|
||||||
|
if (!session)
|
||||||
|
return NULL;
|
||||||
|
session->defer = config->delete_delay;
|
||||||
|
session->dry_run = config->dry_run;
|
||||||
|
bool user_limited =
|
||||||
|
config->max_delete >= 0 && (size_t)config->max_delete < DELETE_PLAN_SERVER_LIMIT;
|
||||||
|
session->max_delete =
|
||||||
|
user_limited ? (size_t)config->max_delete : (size_t)DELETE_PLAN_SERVER_LIMIT;
|
||||||
|
session->protected_prefixes = array_list_create(free);
|
||||||
|
session->size_skipped = array_list_create(free);
|
||||||
|
session->missing = array_list_create(free);
|
||||||
|
session->deferred = array_list_create(free);
|
||||||
|
if (!session->protected_prefixes || !session->size_skipped || !session->missing ||
|
||||||
|
!session->deferred) {
|
||||||
|
delete_plan_session_destroy(session);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
return session;
|
||||||
|
}
|
||||||
|
|
||||||
|
void delete_plan_session_destroy(DeletePlanSession* session) {
|
||||||
|
if (!session)
|
||||||
|
return;
|
||||||
|
array_list_delete(session->protected_prefixes);
|
||||||
|
array_list_delete(session->size_skipped);
|
||||||
|
array_list_delete(session->missing);
|
||||||
|
array_list_delete(session->deferred);
|
||||||
|
free(session);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool delete_plan_session_limit_reached(const DeletePlanSession* session) {
|
||||||
|
return session && session->limit_hit;
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t delete_plan_session_deleted(const DeletePlanSession* session) {
|
||||||
|
return session ? session->deleted : 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* True for a destination-relative path section entry (non-empty, relative,
|
||||||
|
* traversal-free). */
|
||||||
|
static bool valid_rel_path(const char* value) {
|
||||||
|
return value && value[0] != '\0' && value[0] != '/' && !has_path_traversal(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* True for a single child name (non-empty, no slash, not "."/".."). */
|
||||||
|
static bool valid_name(const char* value) {
|
||||||
|
return value && value[0] != '\0' && strcmp(value, ".") != 0 && strcmp(value, "..") != 0 &&
|
||||||
|
strchr(value, '/') == NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Read one count-prefixed section. `bytes` is the running per-frame budget,
|
||||||
|
* shared across every section of the frame so a hostile peer cannot retain more
|
||||||
|
* than MAX_MANIFEST_BYTES from one STATUS_DELETE_PLAN frame. */
|
||||||
|
static bool read_section(int fd, ArrayList* list, bool rel_path, size_t* bytes) {
|
||||||
|
int count;
|
||||||
|
if (!receive_int(fd, &count) || count < 0 || count > MAX_MANIFEST_ENTRIES)
|
||||||
|
return false;
|
||||||
|
for (int i = 0; i < count; i++) {
|
||||||
|
char* value = receive_wire_str(fd);
|
||||||
|
bool ok = value && (rel_path ? valid_rel_path(value) : valid_name(value));
|
||||||
|
if (ok) {
|
||||||
|
size_t entry_size = strlen(value) + sizeof(char*) + 16;
|
||||||
|
if (entry_size > MAX_MANIFEST_BYTES - *bytes) {
|
||||||
|
ok = false;
|
||||||
|
} else {
|
||||||
|
*bytes += entry_size;
|
||||||
|
ok = array_list_add(list, value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!ok) {
|
||||||
|
free(value);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
static int open_plan_dir(const Config* config, const char* dir) {
|
||||||
|
char* full = (strcmp(dir, ".") == 0) ? str_dup(config->receive_root_directory)
|
||||||
|
: path_cat(config->receive_root_directory, dir);
|
||||||
|
if (!full)
|
||||||
|
return -1;
|
||||||
|
int root_fd = utils_get_authorized_root_fd();
|
||||||
|
int fd = -1;
|
||||||
|
if (root_fd >= 0) {
|
||||||
|
if (utils_get_authorized_root_path())
|
||||||
|
fd = utils_open_authorized_destination(full);
|
||||||
|
else if (strcmp(dir, ".") == 0)
|
||||||
|
fd = dup(root_fd);
|
||||||
|
} else {
|
||||||
|
fd = open(full, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||||
|
}
|
||||||
|
free(full);
|
||||||
|
return fd;
|
||||||
|
}
|
||||||
|
|
||||||
|
typedef struct PlanSkips {
|
||||||
|
DeleteSkipEntry* entries;
|
||||||
|
int count;
|
||||||
|
} PlanSkips;
|
||||||
|
|
||||||
|
static bool build_plan_skips(const Config* config, const DeletePlanSession* session,
|
||||||
|
PlanSkips* out) {
|
||||||
|
out->entries = NULL;
|
||||||
|
out->count = 0;
|
||||||
|
int count = (config->delay_updates ? 1 : 0) + config->basis_count +
|
||||||
|
session->protected_prefixes->size + session->size_skipped->size;
|
||||||
|
if (count == 0)
|
||||||
|
return true;
|
||||||
|
out->entries = calloc((size_t)count, sizeof(DeleteSkipEntry));
|
||||||
|
if (!out->entries)
|
||||||
|
return false;
|
||||||
|
int idx = 0;
|
||||||
|
if (config->delay_updates) {
|
||||||
|
out->entries[idx].prefix = DELAY_UPDATES_STAGING_DIR;
|
||||||
|
out->entries[idx].top_level_only = true;
|
||||||
|
idx++;
|
||||||
|
}
|
||||||
|
for (int i = 0; i < config->basis_count; i++) {
|
||||||
|
out->entries[idx].prefix = config->basis_dirs[i].path;
|
||||||
|
out->entries[idx].top_level_only = false;
|
||||||
|
idx++;
|
||||||
|
}
|
||||||
|
for (int i = 0; i < session->protected_prefixes->size; i++) {
|
||||||
|
out->entries[idx].prefix = (const char*)session->protected_prefixes->items[i];
|
||||||
|
out->entries[idx].top_level_only = false;
|
||||||
|
idx++;
|
||||||
|
}
|
||||||
|
for (int i = 0; i < session->size_skipped->size; i++) {
|
||||||
|
out->entries[idx].prefix = (const char*)session->size_skipped->items[i];
|
||||||
|
out->entries[idx].top_level_only = false;
|
||||||
|
idx++;
|
||||||
|
}
|
||||||
|
out->count = idx;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool budget_available(const DeletePlanSession* session) {
|
||||||
|
return session->deleted < session->max_delete;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void note_skipped(DeletePlanSession* session) {
|
||||||
|
session->limit_hit = true;
|
||||||
|
session->skipped++;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void log_deleted(const char* rel) {
|
||||||
|
char* escaped = output_escape(rel, log_get_8_bit_output());
|
||||||
|
fprintf(stderr, " Deleted: %s\n", escaped ? escaped : "<allocation failed>");
|
||||||
|
free(escaped);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Append a snapshot path for --delete-delay. */
|
||||||
|
static bool defer_add(DeletePlanSession* session, const char* rel) {
|
||||||
|
char* copy = str_dup(rel);
|
||||||
|
if (!copy)
|
||||||
|
return false;
|
||||||
|
if (!array_list_add(session->deferred, copy)) {
|
||||||
|
free(copy);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
session->deleted++;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Process the direct children of one directory. `keep_dirs`/`keep_files`
|
||||||
|
* (basenames) are the source entries that must be kept; NULL means every child
|
||||||
|
* is an extra (the forced path used inside a removed extra directory tree).
|
||||||
|
* `survives` reports that at least one child remains (kept, protected, or
|
||||||
|
* skipped by the budget). `force_now` removes even in --delete-delay mode
|
||||||
|
* (type conflicts must clear before the incoming data). */
|
||||||
|
static bool process_children(int dirfd, const char* dir_rel, const ArrayList* keep_dirs,
|
||||||
|
const ArrayList* keep_files, bool at_root, bool force_now,
|
||||||
|
const PlanSkips* skips, DeletePlanSession* session, bool* survives);
|
||||||
|
|
||||||
|
static bool process_extra_dir(int dirfd, const char* name, const char* child_rel, bool force_now,
|
||||||
|
const PlanSkips* skips, DeletePlanSession* session, bool* removed) {
|
||||||
|
*removed = false;
|
||||||
|
int childfd = openat(dirfd, name, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||||
|
if (childfd < 0) {
|
||||||
|
if (errno == ENOENT) {
|
||||||
|
*removed = true;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
bool survives = false;
|
||||||
|
bool ok =
|
||||||
|
process_children(childfd, child_rel, NULL, NULL, false, force_now, skips, session, &survives);
|
||||||
|
close(childfd);
|
||||||
|
if (!ok)
|
||||||
|
return false;
|
||||||
|
if (survives)
|
||||||
|
return true;
|
||||||
|
if (!budget_available(session)) {
|
||||||
|
note_skipped(session);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (session->defer && !force_now) {
|
||||||
|
if (!defer_add(session, child_rel))
|
||||||
|
return false;
|
||||||
|
*removed = true;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (unlinkat(dirfd, name, AT_REMOVEDIR) == 0) {
|
||||||
|
session->deleted++;
|
||||||
|
log_deleted(child_rel);
|
||||||
|
*removed = true;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (errno == ENOENT) {
|
||||||
|
*removed = true;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
/* ENOTEMPTY/EEXIST: a protected entry the walker leaves behind survived, so
|
||||||
|
the directory stays; any other errno is a genuine failure. */
|
||||||
|
return errno == ENOTEMPTY || errno == EEXIST;
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool process_extra_file(int dirfd, const char* name, const char* child_rel, bool force_now,
|
||||||
|
DeletePlanSession* session) {
|
||||||
|
if (!budget_available(session)) {
|
||||||
|
note_skipped(session);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (session->defer && !force_now) {
|
||||||
|
return defer_add(session, child_rel);
|
||||||
|
}
|
||||||
|
if (unlinkat(dirfd, name, 0) == 0) {
|
||||||
|
session->deleted++;
|
||||||
|
log_deleted(child_rel);
|
||||||
|
} else if (errno != ENOENT) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool process_children(int dirfd, const char* dir_rel, const ArrayList* keep_dirs,
|
||||||
|
const ArrayList* keep_files, bool at_root, bool force_now,
|
||||||
|
const PlanSkips* skips, DeletePlanSession* session, bool* survives) {
|
||||||
|
*survives = false;
|
||||||
|
int scanfd = openat(dirfd, ".", O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||||
|
if (scanfd < 0)
|
||||||
|
return false;
|
||||||
|
DIR* dir = fdopendir(scanfd);
|
||||||
|
if (!dir) {
|
||||||
|
close(scanfd);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
bool operation_ok = true;
|
||||||
|
bool local_survives = false;
|
||||||
|
const struct dirent* entry;
|
||||||
|
while ((entry = readdir(dir)) != NULL) {
|
||||||
|
if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
|
||||||
|
continue;
|
||||||
|
char* child_rel =
|
||||||
|
(strcmp(dir_rel, ".") == 0) ? str_dup(entry->d_name) : path_cat(dir_rel, entry->d_name);
|
||||||
|
if (!child_rel) {
|
||||||
|
operation_ok = false;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (path_under_skip_prefix(child_rel, at_root, skips->entries, skips->count)) {
|
||||||
|
local_survives = true;
|
||||||
|
free(child_rel);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
struct stat st;
|
||||||
|
if (fstatat(dirfd, entry->d_name, &st, AT_SYMLINK_NOFOLLOW) != 0) {
|
||||||
|
if (errno != ENOENT)
|
||||||
|
operation_ok = false;
|
||||||
|
free(child_rel);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
bool is_dir = S_ISDIR(st.st_mode);
|
||||||
|
bool in_keep_dirs = is_dir && list_contains_str(keep_dirs, entry->d_name);
|
||||||
|
bool in_keep_files = !is_dir && list_contains_str(keep_files, entry->d_name);
|
||||||
|
if (in_keep_dirs) {
|
||||||
|
local_survives = true;
|
||||||
|
} else if (keep_dirs && !is_dir && list_contains_str(keep_dirs, entry->d_name)) {
|
||||||
|
/* Destination file blocks a source directory: clear it now, whatever the
|
||||||
|
delete timing, so the directory can be created. */
|
||||||
|
if (!process_extra_file(dirfd, entry->d_name, child_rel, true, session))
|
||||||
|
operation_ok = false;
|
||||||
|
} else if (in_keep_files) {
|
||||||
|
local_survives = true;
|
||||||
|
} else if (keep_files && is_dir && list_contains_str(keep_files, entry->d_name)) {
|
||||||
|
/* Destination directory blocks a source file: remove it now. */
|
||||||
|
bool removed = false;
|
||||||
|
if (!process_extra_dir(dirfd, entry->d_name, child_rel, true, skips, session, &removed))
|
||||||
|
operation_ok = false;
|
||||||
|
else if (!removed)
|
||||||
|
local_survives = true;
|
||||||
|
} else if (is_dir) {
|
||||||
|
bool removed = false;
|
||||||
|
if (!process_extra_dir(dirfd, entry->d_name, child_rel, force_now, skips, session, &removed))
|
||||||
|
operation_ok = false;
|
||||||
|
else if (!removed)
|
||||||
|
local_survives = true;
|
||||||
|
} else {
|
||||||
|
if (!process_extra_file(dirfd, entry->d_name, child_rel, force_now, session))
|
||||||
|
operation_ok = false;
|
||||||
|
}
|
||||||
|
free(child_rel);
|
||||||
|
}
|
||||||
|
closedir(dir);
|
||||||
|
*survives = local_survives;
|
||||||
|
return operation_ok;
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool apply_plan_dir(DeletePlanSession* session, const Config* config, const char* dir,
|
||||||
|
const ArrayList* dirs, const ArrayList* files) {
|
||||||
|
int dirfd = open_plan_dir(config, dir);
|
||||||
|
if (dirfd < 0) {
|
||||||
|
/* An absent destination directory has nothing to delete. */
|
||||||
|
return errno == ENOENT || errno == ENOTDIR;
|
||||||
|
}
|
||||||
|
PlanSkips skips;
|
||||||
|
if (!build_plan_skips(config, session, &skips)) {
|
||||||
|
close(dirfd);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
bool survives = false;
|
||||||
|
bool ok = process_children(dirfd, dir, dirs, files, strcmp(dir, ".") == 0, false, &skips, session,
|
||||||
|
&survives);
|
||||||
|
free(skips.entries);
|
||||||
|
close(dirfd);
|
||||||
|
if (!ok)
|
||||||
|
log_message(LOG_LEVEL_ERROR, "deletion failed while removing extraneous files");
|
||||||
|
return ok;
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool apply_missing(DeletePlanSession* session, const Config* config) {
|
||||||
|
if (session->missing_applied)
|
||||||
|
return true;
|
||||||
|
session->missing_applied = true;
|
||||||
|
/* The server clears delete_missing_args when its --allow-delete policy is
|
||||||
|
off; never honor the client's exact-path requests then. */
|
||||||
|
if (!config->delete_missing_args || session->missing->size == 0)
|
||||||
|
return true;
|
||||||
|
DeleteManifest manifest = {
|
||||||
|
.keeps = NULL, .protected = NULL, .missing = session->missing, .dirs = NULL};
|
||||||
|
size_t remaining = budget_available(session) ? session->max_delete - session->deleted : 0;
|
||||||
|
size_t deleted = 0;
|
||||||
|
size_t skipped = 0;
|
||||||
|
bool limit = false;
|
||||||
|
bool ok = manifest_delete_missing_args_limited(config, &manifest, remaining, &deleted, &skipped,
|
||||||
|
&limit);
|
||||||
|
session->deleted += deleted;
|
||||||
|
session->skipped += skipped;
|
||||||
|
if (limit)
|
||||||
|
session->limit_hit = true;
|
||||||
|
return ok;
|
||||||
|
}
|
||||||
|
|
||||||
|
int delete_plan_session_receive(DeletePlanSession* session, const Config* config, int fd) {
|
||||||
|
if (!session || !config) {
|
||||||
|
send_status(fd, STATUS_ERROR);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
int has_config;
|
||||||
|
if (!receive_int(fd, &has_config) || (has_config != 0 && has_config != 1)) {
|
||||||
|
send_status(fd, STATUS_ERROR);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
size_t bytes = 0;
|
||||||
|
if (has_config) {
|
||||||
|
if (session->config_seen || !read_section(fd, session->protected_prefixes, true, &bytes) ||
|
||||||
|
!read_section(fd, session->size_skipped, true, &bytes) ||
|
||||||
|
!read_section(fd, session->missing, true, &bytes)) {
|
||||||
|
send_status(fd, STATUS_ERROR);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
session->config_seen = true;
|
||||||
|
}
|
||||||
|
char* dir = receive_wire_str(fd);
|
||||||
|
ArrayList* dirs = array_list_create(free);
|
||||||
|
ArrayList* files = array_list_create(free);
|
||||||
|
bool parsed = dir && (strcmp(dir, ".") == 0 || valid_rel_path(dir)) && dirs && files &&
|
||||||
|
read_section(fd, dirs, false, &bytes) && read_section(fd, files, false, &bytes);
|
||||||
|
if (!parsed) {
|
||||||
|
free(dir);
|
||||||
|
array_list_delete(dirs);
|
||||||
|
array_list_delete(files);
|
||||||
|
send_status(fd, STATUS_ERROR);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
bool enabled = config->use_delete || config->delete_missing_args;
|
||||||
|
bool ok = true;
|
||||||
|
if (!session->dry_run && enabled) {
|
||||||
|
if (!session->defer && !apply_missing(session, config))
|
||||||
|
ok = false;
|
||||||
|
if (ok && !apply_plan_dir(session, config, dir, dirs, files))
|
||||||
|
ok = false;
|
||||||
|
}
|
||||||
|
free(dir);
|
||||||
|
array_list_delete(dirs);
|
||||||
|
array_list_delete(files);
|
||||||
|
if (!ok) {
|
||||||
|
send_status(fd, STATUS_ERROR);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
if (session->limit_hit && !session->limit_logged) {
|
||||||
|
session->limit_logged = true;
|
||||||
|
log_message(LOG_LEVEL_WARNING, "Deletions stopped due to the delete limit (%zu skipped)",
|
||||||
|
session->skipped);
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Apply one snapshotted --delete-delay path (post-order: children precede their
|
||||||
|
* parent directory). */
|
||||||
|
static bool apply_deferred_path(DeletePlanSession* session, const Config* config, const char* rel) {
|
||||||
|
(void)session;
|
||||||
|
char* full = path_cat(config->receive_root_directory, rel);
|
||||||
|
if (!full)
|
||||||
|
return false;
|
||||||
|
char* leaf = NULL;
|
||||||
|
int parent_fd = file_open_secure_parent(full, &leaf, false);
|
||||||
|
free(full);
|
||||||
|
if (parent_fd < 0) {
|
||||||
|
free(leaf);
|
||||||
|
return errno == ENOENT || errno == ENOTDIR;
|
||||||
|
}
|
||||||
|
struct stat st;
|
||||||
|
if (fstatat(parent_fd, leaf, &st, AT_SYMLINK_NOFOLLOW) != 0) {
|
||||||
|
bool absent = errno == ENOENT;
|
||||||
|
close(parent_fd);
|
||||||
|
free(leaf);
|
||||||
|
return absent;
|
||||||
|
}
|
||||||
|
int rc;
|
||||||
|
if (S_ISDIR(st.st_mode))
|
||||||
|
rc = unlinkat(parent_fd, leaf, AT_REMOVEDIR);
|
||||||
|
else
|
||||||
|
rc = unlinkat(parent_fd, leaf, 0);
|
||||||
|
bool ok = rc == 0 || errno == ENOENT || errno == ENOTEMPTY || errno == EEXIST;
|
||||||
|
if (rc == 0)
|
||||||
|
log_deleted(rel);
|
||||||
|
close(parent_fd);
|
||||||
|
free(leaf);
|
||||||
|
return ok;
|
||||||
|
}
|
||||||
|
|
||||||
|
DeleteCommitResult delete_plan_session_commit(DeletePlanSession* session, const Config* config) {
|
||||||
|
if (!session || !config)
|
||||||
|
return DELETE_COMMIT_ERROR;
|
||||||
|
/* Central no-mutation guard (mirrors manifest_delete_all): a dry-run never
|
||||||
|
deletes. The receive path already skips plan application, but a hostile or
|
||||||
|
buggy peer could still reach the commit, so treat it as a no-op. */
|
||||||
|
if (session->dry_run)
|
||||||
|
return DELETE_COMMIT_OK;
|
||||||
|
bool ok = true;
|
||||||
|
if (session->defer) {
|
||||||
|
for (int i = 0; i < session->deferred->size && ok; i++)
|
||||||
|
ok = apply_deferred_path(session, config, (const char*)session->deferred->items[i]);
|
||||||
|
}
|
||||||
|
if (ok)
|
||||||
|
ok = apply_missing(session, config);
|
||||||
|
if (!ok)
|
||||||
|
return DELETE_COMMIT_ERROR;
|
||||||
|
if (session->limit_hit)
|
||||||
|
return DELETE_COMMIT_LIMIT_REACHED;
|
||||||
|
return DELETE_COMMIT_OK;
|
||||||
|
}
|
||||||
@@ -0,0 +1,83 @@
|
|||||||
|
#ifndef DELETE_PLAN_H
|
||||||
|
#define DELETE_PLAN_H
|
||||||
|
|
||||||
|
#include "array_list.h"
|
||||||
|
#include "config.h"
|
||||||
|
#include "file_receive.h"
|
||||||
|
#include "protocol.h"
|
||||||
|
#include <stdbool.h>
|
||||||
|
|
||||||
|
/* Per-directory delete plans (protocol 2.24.0).
|
||||||
|
*
|
||||||
|
* rsync's --delete-during removes a directory's extras while the generator
|
||||||
|
* processes that directory, and --delete-delay records the deletion list during
|
||||||
|
* the scan but applies it only after a fully-successful transfer. FastSync has
|
||||||
|
* no per-directory generator pass; instead the sender streams one plan per
|
||||||
|
* source directory, in directory order, and the receiver applies it when it
|
||||||
|
* arrives (during) or snapshots its extras and commits them at the end (delay).
|
||||||
|
*
|
||||||
|
* The sender side builds a plan set from the path-only pre-scan (it needs every
|
||||||
|
* directory's complete direct-child list before the first data byte of that
|
||||||
|
* directory). The receiver side is a session that carries the global protected
|
||||||
|
* prefixes (filter-excluded and size-skipped source mirrors), the
|
||||||
|
* --delete-missing-args exact deletions, the shared --max-delete budget and,
|
||||||
|
* for --delete-delay, the snapshotted extras. */
|
||||||
|
|
||||||
|
/* ---- Sender: plan builder ---- */
|
||||||
|
|
||||||
|
typedef struct DeletePlanSender DeletePlanSender;
|
||||||
|
|
||||||
|
DeletePlanSender* delete_plan_sender_create(void);
|
||||||
|
void delete_plan_sender_destroy(DeletePlanSender* sender);
|
||||||
|
/* Record one transmitted entry. `path` is the destination-relative wire path;
|
||||||
|
* is_dir marks an explicit directory entry (--dirs, a -x mount point). */
|
||||||
|
bool delete_plan_sender_add(DeletePlanSender* sender, const char* path, bool is_dir);
|
||||||
|
/* Drop plans for directories outside `synced_dirs` (the --files-from
|
||||||
|
* synchronization scope; pass NULL when a full recursive transfer synchronized
|
||||||
|
* every directory). The receive root is the "." sentinel.
|
||||||
|
*
|
||||||
|
* `walk_root` scopes a general -R transfer: when non-NULL it is the
|
||||||
|
* reconstructed destination prefix the run actually transferred, and only the
|
||||||
|
* plan for that prefix (and directories below it) is ever transmitted, so the
|
||||||
|
* prefix's parent-directory siblings are never walked. Pass NULL for a plain
|
||||||
|
* recursive transfer and for --files-from. */
|
||||||
|
void delete_plan_sender_finalize(DeletePlanSender* sender, const ArrayList* synced_dirs,
|
||||||
|
const char* walk_root);
|
||||||
|
/* True when no transmitted FILE entry was recorded (an ambiguous empty scan).
|
||||||
|
Directory keep entries do not count, so an I/O error that hid every file
|
||||||
|
still refuses to delete. */
|
||||||
|
bool delete_plan_sender_empty(const DeletePlanSender* sender);
|
||||||
|
/* Attach the global config sections advertised on the first plan frame. */
|
||||||
|
void delete_plan_sender_set_config(DeletePlanSender* sender, const ArrayList* protected_prefixes,
|
||||||
|
const ArrayList* size_skipped, const ArrayList* missing_args);
|
||||||
|
/* Send the root plan (even before any data, so root extras are handled like
|
||||||
|
* rsync's first generator directory). Returns -1 on I/O error. */
|
||||||
|
int delete_plan_send_root(int fd, DeletePlanSender* sender);
|
||||||
|
/* Send the plans for every ancestor of `path` (root-first) and, when is_dir,
|
||||||
|
* for `path` itself; already-sent plans are skipped. */
|
||||||
|
int delete_plan_send_for_path(int fd, DeletePlanSender* sender, const char* path, bool is_dir);
|
||||||
|
/* Send the plan for every directory in `dirs` that has not been transmitted
|
||||||
|
* yet. Called after the data stream so an empty source directory's plan still
|
||||||
|
* clears its destination extras even though no file frame triggered it. */
|
||||||
|
int delete_plan_send_remaining(int fd, DeletePlanSender* sender, const ArrayList* dirs);
|
||||||
|
|
||||||
|
/* ---- Receiver: delete session ---- */
|
||||||
|
|
||||||
|
typedef struct DeletePlanSession DeletePlanSession;
|
||||||
|
|
||||||
|
DeletePlanSession* delete_plan_session_create(const Config* config);
|
||||||
|
void delete_plan_session_destroy(DeletePlanSession* session);
|
||||||
|
/* Read one STATUS_DELETE_PLAN frame (the leading status already consumed) and
|
||||||
|
* act on it. Returns 0 on success (including a dry-run/disabled no-op) and -1
|
||||||
|
* after signalling STATUS_ERROR on a malformed frame or a deletion failure. */
|
||||||
|
int delete_plan_session_receive(DeletePlanSession* session, const Config* config, int fd);
|
||||||
|
/* Apply the deferred snapshot (--delete-delay) and the missing-args deletions.
|
||||||
|
* Safe to call once; returns the commit outcome. */
|
||||||
|
DeleteCommitResult delete_plan_session_commit(DeletePlanSession* session, const Config* config);
|
||||||
|
/* True once the shared --max-delete budget stopped part of a deletion. */
|
||||||
|
bool delete_plan_session_limit_reached(const DeletePlanSession* session);
|
||||||
|
/* Number of destination entries the session's plans removed (or, for
|
||||||
|
--delete-delay, snapshotted for removal), for the end-of-transfer stats. */
|
||||||
|
size_t delete_plan_session_deleted(const DeletePlanSession* session);
|
||||||
|
|
||||||
|
#endif
|
||||||
+279
-100
@@ -6,6 +6,7 @@
|
|||||||
#include <fcntl.h>
|
#include <fcntl.h>
|
||||||
#include <libgen.h>
|
#include <libgen.h>
|
||||||
#include <limits.h>
|
#include <limits.h>
|
||||||
|
#include <pthread.h>
|
||||||
#include <stdatomic.h>
|
#include <stdatomic.h>
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
@@ -39,19 +40,27 @@ static bool write_all(int fd, const void* data, unsigned long long size) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/* Preallocate `size` bytes on `fd` before any data is written (--preallocate).
|
/* Preallocate `size` bytes on `fd` before any data is written (--preallocate).
|
||||||
* posix_fallocate reserves real disk blocks, so an out-of-space condition
|
* fallocate(2) reserves real disk blocks, so an out-of-space condition
|
||||||
* (ENOSPC/EDQUOT) surfaces up front instead of partway through a transfer;
|
* (ENOSPC/EDQUOT) surfaces up front instead of partway through a transfer;
|
||||||
* unavoidable fragmentation of a streamed file is also reduced. Some
|
* unavoidable fragmentation of a streamed file is also reduced. rsync favors
|
||||||
* filesystems (e.g. tmpfs, ZFS) do not support it and return EOPNOTSUPP/ENOSYS,
|
* the syscall over glibc posix_fallocate (whose emulation can be subtly
|
||||||
* where we fall back to ftruncate, which still extends the logical size so the
|
* different), so try fallocate(2) first and only fall back to posix_fallocate,
|
||||||
* fail-fast/contiguity intent degrades gracefully but never fails. Genuine
|
* then to ftruncate on filesystems (e.g. tmpfs, ZFS) that support neither. The
|
||||||
* allocation failures are propagated as the error code (caller fails the write).
|
* logical size is always extended, so the fail-fast/contiguity intent degrades
|
||||||
* posix_fallocate leaves the fd's file offset unchanged, so the subsequent
|
* gracefully but never fails on an unsupported filesystem; genuine allocation
|
||||||
* write_all at offset 0 is unaffected. Returns 0 on success (including the
|
* failures are propagated as the error code (caller fails the write). Neither
|
||||||
* fallback) or a nonzero error code. */
|
* leaves the fd's file offset guaranteed, so the caller seeks back to 0 before
|
||||||
|
* writing. Returns 0 on success (including the fallback) or a nonzero error
|
||||||
|
* code. */
|
||||||
static int preallocate_fd(int fd, unsigned long long size) {
|
static int preallocate_fd(int fd, unsigned long long size) {
|
||||||
if (size == 0)
|
if (size == 0)
|
||||||
return 0;
|
return 0;
|
||||||
|
#ifdef __linux__
|
||||||
|
if (fallocate(fd, 0, 0, (off_t)size) == 0)
|
||||||
|
return 0;
|
||||||
|
if (errno != EOPNOTSUPP && errno != ENOSYS && errno != EINVAL)
|
||||||
|
return errno;
|
||||||
|
#endif
|
||||||
int rc = posix_fallocate(fd, 0, (off_t)size);
|
int rc = posix_fallocate(fd, 0, (off_t)size);
|
||||||
if (rc == EOPNOTSUPP || rc == ENOSYS) {
|
if (rc == EOPNOTSUPP || rc == ENOSYS) {
|
||||||
if (ftruncate(fd, (off_t)size) == 0)
|
if (ftruncate(fd, (off_t)size) == 0)
|
||||||
@@ -72,6 +81,58 @@ static unsigned long long next_temp_sequence(void) {
|
|||||||
return atomic_fetch_add_explicit(&sequence, 1, memory_order_relaxed);
|
return atomic_fetch_add_explicit(&sequence, 1, memory_order_relaxed);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Process-wide umask, captured exactly once. Reading the umask requires a
|
||||||
|
* get+set round trip (umask(0); umask(old)); doing that per write would be racy
|
||||||
|
* in the multithreaded receiver, so the value is captured at process startup by
|
||||||
|
* file_umask_capture() (called at the top of main(), before any threads exist).
|
||||||
|
* The pthread_once fallback keeps a caller that never called the capture (e.g. a
|
||||||
|
* unit test) correct. */
|
||||||
|
static unsigned g_process_umask;
|
||||||
|
static atomic_bool g_process_umask_captured;
|
||||||
|
static pthread_once_t g_process_umask_once = PTHREAD_ONCE_INIT;
|
||||||
|
|
||||||
|
static void file_capture_umask_now(void) {
|
||||||
|
mode_t mask = umask(0);
|
||||||
|
umask(mask);
|
||||||
|
g_process_umask = (unsigned)mask;
|
||||||
|
atomic_store_explicit(&g_process_umask_captured, true, memory_order_release);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void file_capture_umask_once(void) {
|
||||||
|
if (atomic_load_explicit(&g_process_umask_captured, memory_order_acquire))
|
||||||
|
return;
|
||||||
|
file_capture_umask_now();
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Re-captures the umask. Must only be called while the process is still
|
||||||
|
* single-threaded (startup, or the daemon's post-fork setup after umask(0)),
|
||||||
|
* so a later re-capture can refresh the cached value before any receiver
|
||||||
|
* thread exists. */
|
||||||
|
void file_umask_capture(void) {
|
||||||
|
file_capture_umask_now();
|
||||||
|
}
|
||||||
|
|
||||||
|
unsigned file_process_umask(void) {
|
||||||
|
if (!atomic_load_explicit(&g_process_umask_captured, memory_order_acquire))
|
||||||
|
pthread_once(&g_process_umask_once, file_capture_umask_once);
|
||||||
|
return g_process_umask;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Base mode applied when the policy does not take the source mode wholesale
|
||||||
|
* (i.e. --perms is off). A pre-existing destination keeps its own mode; a
|
||||||
|
* brand-new file is created like rsync: source_mode & 0777 & ~umask (special
|
||||||
|
* bits are not part of a mode-preserving transfer without -p). Only when no
|
||||||
|
* metadata is available at all does the historical fixed 0644 default apply.
|
||||||
|
* The -E rule (and no-op for a plain -t) is layered on top of this base. */
|
||||||
|
static mode_t file_mode_base(const FileMetadata* metadata, bool existing_known,
|
||||||
|
mode_t existing_mode) {
|
||||||
|
if (existing_known)
|
||||||
|
return existing_mode;
|
||||||
|
if (metadata)
|
||||||
|
return metadata->mode & 0777 & ~(mode_t)file_process_umask();
|
||||||
|
return S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH;
|
||||||
|
}
|
||||||
|
|
||||||
bool file_checksum(File* file, ChecksumAlgo algo, uint64_t seed, uint8_t* out, size_t out_capacity,
|
bool file_checksum(File* file, ChecksumAlgo algo, uint64_t seed, uint8_t* out, size_t out_capacity,
|
||||||
size_t* out_len) {
|
size_t* out_len) {
|
||||||
if (!file || !out || !out_len || !file->data)
|
if (!file || !out || !out_len || !file->data)
|
||||||
@@ -124,6 +185,8 @@ File* file_create(const char* path) {
|
|||||||
file->rdev_major = 0;
|
file->rdev_major = 0;
|
||||||
file->rdev_minor = 0;
|
file->rdev_minor = 0;
|
||||||
file->xattrs = NULL;
|
file->xattrs = NULL;
|
||||||
|
file->dest_state = (OutputDestState){0};
|
||||||
|
file->matched_bytes = 0;
|
||||||
return file;
|
return file;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -357,10 +420,69 @@ bool file_get_trust_sender(void) {
|
|||||||
return file_trust_sender;
|
return file_trust_sender;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* True when `target` is a lexical symlink target that can never escape the
|
/* rsync 3.4.1 unsafe_symlink(): true when `target` (the link's destination
|
||||||
* receive root once created beneath it: relative (not absolute) and containing
|
* string) points outside the transfer tree rooted at the symlink's own
|
||||||
* no ".." path component. Used by --munge-links' sender-side containment: an
|
* location. `link_path` is the symlink's path relative to the top of the
|
||||||
* escaping target is never transmitted (the entry is skipped/contained). */
|
* transfer (including its name). This is a purely lexical test matching
|
||||||
|
* rsync's util1.c: absolute/empty targets are always unsafe; leading "../"
|
||||||
|
* components are counted against the symlink's own directory depth; a ".."
|
||||||
|
* that would climb above the transfer root is unsafe. rsync 3.4.1 additionally
|
||||||
|
* rejects any INTERNAL "/../" component and a trailing "/..". */
|
||||||
|
bool file_symlink_unsafe(const char* target, const char* link_path) {
|
||||||
|
if (!target || target[0] == '\0' || target[0] == '/')
|
||||||
|
return true;
|
||||||
|
const char* rest = target;
|
||||||
|
while (strncmp(rest, "../", 3) == 0) {
|
||||||
|
rest += 3;
|
||||||
|
while (*rest == '/')
|
||||||
|
rest++;
|
||||||
|
}
|
||||||
|
if (strstr(rest, "/../") != NULL)
|
||||||
|
return true;
|
||||||
|
size_t target_len = strlen(target);
|
||||||
|
if (target_len > 3 && strcmp(&target[target_len - 3], "/..") == 0)
|
||||||
|
return true;
|
||||||
|
|
||||||
|
int depth = 0;
|
||||||
|
const char* name;
|
||||||
|
const char* slash;
|
||||||
|
const char* src = link_path ? link_path : "";
|
||||||
|
for (name = src; (slash = strchr(name, '/')) != NULL; name = slash + 1) {
|
||||||
|
if (*name == '.' && (name[1] == '/' || (name[1] == '.' && name[2] == '/'))) {
|
||||||
|
if (name[1] == '.')
|
||||||
|
depth = 0;
|
||||||
|
} else {
|
||||||
|
depth++;
|
||||||
|
}
|
||||||
|
while (slash[1] == '/')
|
||||||
|
slash++;
|
||||||
|
}
|
||||||
|
if (*name == '.' && name[1] == '.' && name[2] == '\0')
|
||||||
|
depth = 0;
|
||||||
|
|
||||||
|
for (name = target; (slash = strchr(name, '/')) != NULL; name = slash + 1) {
|
||||||
|
if (*name == '.' && (name[1] == '/' || (name[1] == '.' && name[2] == '/'))) {
|
||||||
|
if (name[1] == '.') {
|
||||||
|
if (--depth < 0)
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
depth++;
|
||||||
|
}
|
||||||
|
while (slash[1] == '/')
|
||||||
|
slash++;
|
||||||
|
}
|
||||||
|
if (*name == '.' && name[1] == '.' && name[2] == '\0')
|
||||||
|
depth--;
|
||||||
|
return depth < 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Strict lexical helper: true when `target` is relative (not absolute) and
|
||||||
|
* contains no ".." component at all, so it can never escape the directory it
|
||||||
|
* is created in. This is stricter than rsync's unsafe_symlink() (which allows
|
||||||
|
* an in-tree ".."); the scanner/receiver use file_symlink_unsafe()/--safe-links
|
||||||
|
* for rsync parity, and this helper is retained for callers that want the
|
||||||
|
* ".."-free guarantee. */
|
||||||
bool file_symlink_target_contained(const char* target) {
|
bool file_symlink_target_contained(const char* target) {
|
||||||
if (!target || target[0] == '\0' || target[0] == '/')
|
if (!target || target[0] == '\0' || target[0] == '/')
|
||||||
return false;
|
return false;
|
||||||
@@ -391,8 +513,9 @@ bool file_symlink_unmunge(char* target) {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Owned copy of `target` prefixed with SYMLINK_MUNGE_PREFIX (the sender-side
|
/* Owned copy of `target` prefixed with SYMLINK_MUNGE_PREFIX (the receiver-side
|
||||||
* --munge-links rewriting). Returns NULL on allocation failure. */
|
* --munge-links rewriting, matching rsync's receiver). Returns NULL on
|
||||||
|
* allocation failure. */
|
||||||
char* file_symlink_munge(const char* target) {
|
char* file_symlink_munge(const char* target) {
|
||||||
if (!target)
|
if (!target)
|
||||||
return NULL;
|
return NULL;
|
||||||
@@ -413,23 +536,14 @@ char* file_symlink_munge(const char* target) {
|
|||||||
* the target is ever followed. The final component is never dereferenced: an
|
* the target is ever followed. The final component is never dereferenced: an
|
||||||
* existing non-directory entry at `path` is unlinked by name before the link is
|
* existing non-directory entry at `path` is unlinked by name before the link is
|
||||||
* placed; an existing directory there is left untouched (returns false, so a
|
* placed; an existing directory there is left untouched (returns false, so a
|
||||||
* caller can treat it as a collision). As a receiver-side trust-boundary
|
* caller can treat it as a collision). The link VALUE `target` is copied
|
||||||
* invariant, `target` must be file_symlink_target_contained() (relative and
|
* verbatim, matching rsync -l (which stores absolute and ".."-bearing targets
|
||||||
* ".."-free): an absolute or escaping target is rejected outright (returns
|
* as-is); target policy is the caller's job -- the scanner applies
|
||||||
* false) so a malicious sender can never materialize a symlink that points
|
* --safe-links/--copy-unsafe-links, and the receiver applies --munge-links.
|
||||||
* outside the receive root. */
|
* The PLACEMENT path is always confined below the authorized root. */
|
||||||
bool file_symlink_at_secure(const char* path, const char* target) {
|
bool file_symlink_at_secure(const char* path, const char* target) {
|
||||||
/* The link itself (`path`) is always kept below the authorized root. The
|
|
||||||
TARGET may point anywhere: normally only a contained (relative, ".."-free)
|
|
||||||
target is permitted so a malicious sender can never plant a symlink that
|
|
||||||
later dereferences outside the root. Under --trust-sender that target
|
|
||||||
containment check is relaxed (the receiver trusts the sender and copies the
|
|
||||||
link verbatim, matching rsync -l), but path/leaf confinement is never
|
|
||||||
disabled, so the link still cannot be placed outside the tree. */
|
|
||||||
if (!path || !target || has_path_traversal(path))
|
if (!path || !target || has_path_traversal(path))
|
||||||
return false;
|
return false;
|
||||||
if (!file_trust_sender && !file_symlink_target_contained(target))
|
|
||||||
return false;
|
|
||||||
char* leaf = NULL;
|
char* leaf = NULL;
|
||||||
int parent_fd = file_open_secure_parent(path, &leaf, true);
|
int parent_fd = file_open_secure_parent(path, &leaf, true);
|
||||||
if (parent_fd < 0)
|
if (parent_fd < 0)
|
||||||
@@ -558,7 +672,7 @@ int file_open_secure_parent(const char* path, char** leaf_out, bool create_dirs)
|
|||||||
if (strcmp(component, ".") != 0) {
|
if (strcmp(component, ".") != 0) {
|
||||||
int next = openat(fd, component, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
int next = openat(fd, component, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||||
if (next < 0 && create_dirs && errno == ENOENT) {
|
if (next < 0 && create_dirs && errno == ENOENT) {
|
||||||
bool created = mkdirat(fd, component, 0755) == 0;
|
bool created = mkdirat(fd, component, (mode_t)(0777 & ~(mode_t)file_process_umask())) == 0;
|
||||||
if (created || errno == EEXIST) {
|
if (created || errno == EEXIST) {
|
||||||
/* P7 Wave E: --copy-as owns EVERY entry, including the intermediate
|
/* P7 Wave E: --copy-as owns EVERY entry, including the intermediate
|
||||||
directories this walk creates implicitly. Its target ids are a
|
directories this walk creates implicitly. Its target ids are a
|
||||||
@@ -687,7 +801,7 @@ bool file_ensure_directory_secure(const char* path) {
|
|||||||
int dir_fd = openat(parent_fd, leaf, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
int dir_fd = openat(parent_fd, leaf, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||||
bool created = false;
|
bool created = false;
|
||||||
if (dir_fd < 0 && errno == ENOENT) {
|
if (dir_fd < 0 && errno == ENOENT) {
|
||||||
if (mkdirat(parent_fd, leaf, 0755) == 0) {
|
if (mkdirat(parent_fd, leaf, (mode_t)(0777 & ~(mode_t)file_process_umask())) == 0) {
|
||||||
created = true;
|
created = true;
|
||||||
dir_fd = openat(parent_fd, leaf, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
dir_fd = openat(parent_fd, leaf, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||||
} else if (errno == EEXIST) {
|
} else if (errno == EEXIST) {
|
||||||
@@ -855,29 +969,48 @@ int file_open_private_dir(const char* dir_path) {
|
|||||||
return fd;
|
return fd;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Open a --temp-dir scratch directory exactly as rsync does: the directory must
|
||||||
|
* already exist and is used as given (an absolute path is used verbatim, a
|
||||||
|
* relative one was already resolved against the destination root by the
|
||||||
|
* caller). Unlike file_open_private_dir this neither creates it nor confines
|
||||||
|
* it below the receive root, because rsync accepts any temp dir -- including
|
||||||
|
* one outside the destination tree or on another filesystem. Returns an
|
||||||
|
* O_DIRECTORY|O_CLOEXEC fd, or -1 on error. */
|
||||||
|
int file_open_temp_dir(const char* dir_path) {
|
||||||
|
if (!dir_path)
|
||||||
|
return -1;
|
||||||
|
return open(dir_path, O_RDONLY | O_DIRECTORY | O_CLOEXEC);
|
||||||
|
}
|
||||||
|
|
||||||
/* After the content and mode/times are restored on the just-written file, apply
|
/* After the content and mode/times are restored on the just-written file, apply
|
||||||
* the per-file xattrs (-X/-A) and, for --fake-super, park the source's
|
* the per-file xattrs (-X/-A) and, for --fake-super, park the source's
|
||||||
* uid/gid/mode/mtime in the reserved xattr. All fd-relative (confined to the
|
* uid/gid/mode/mtime in the reserved xattr. All fd-relative (confined to the
|
||||||
* destination file) and best-effort: a per-attribute or privilege failure is
|
* destination file) and best-effort: a per-attribute or privilege failure is
|
||||||
* logged and skipped, never fatal. */
|
* logged and skipped, never fatal. */
|
||||||
static void restore_extra_fd(int fd, const FileMetadata* metadata, const FileXattrList* xattrs,
|
static void restore_extra_fd(int fd, const FileMetadata* metadata, const FileXattrList* xattrs,
|
||||||
bool fake_super) {
|
bool fake_super, FileAttrPolicy policy) {
|
||||||
xattr_apply_fd(fd, xattrs);
|
xattr_apply_fd(fd, xattrs);
|
||||||
if (fake_super && metadata) {
|
if (fake_super && metadata) {
|
||||||
fake_super_store_fd(fd, (uint32_t)metadata->uid, (uint32_t)metadata->gid,
|
/* Record the ownership that WOULD have been applied: when an explicit
|
||||||
(uint32_t)metadata->mode, metadata->mtime_sec, metadata->mtime_nsec);
|
ownership request (--chown/--usermap/--groupmap/--copy-as or -o/-g) is
|
||||||
/* Replay: re-apply the recorded uid/gid/mode/mtime fd-relative so a save
|
active, the resolved mapping; otherwise the source's own id. The real
|
||||||
under --fake-super restores the attrs (when privileged) instead of only
|
chown is suppressed (identity_apply_ownership early-returns under
|
||||||
recording them. Best-effort; fake_super_restore_fd silently skips a
|
--fake-super) so recording never defeats the flag. Mode/mtime are still
|
||||||
non-root fchown EPERM/EACCES and never fatal. */
|
replayed (policy-gated) so unprivileged --fake-super keeps working. */
|
||||||
fake_super_restore_fd(fd);
|
uint32_t store_uid;
|
||||||
|
uint32_t store_gid;
|
||||||
|
identity_resolve_storage_ids((int32_t)metadata->uid, (int32_t)metadata->gid, &store_uid,
|
||||||
|
&store_gid);
|
||||||
|
fake_super_store_fd(fd, store_uid, store_gid, (uint32_t)metadata->mode, metadata->mtime_sec,
|
||||||
|
metadata->mtime_nsec);
|
||||||
|
fake_super_restore_fd(fd, policy);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static bool file_to_disk_secure_impl(const char* path, const void* data,
|
static bool file_to_disk_secure_impl(const char* path, const void* data,
|
||||||
unsigned long long data_size, bool inplace, bool sparse,
|
unsigned long long data_size, bool inplace, bool sparse,
|
||||||
bool preallocate, const FileMetadata* metadata,
|
bool preallocate, const FileMetadata* metadata,
|
||||||
bool preserve_executability, bool update, bool no_replace,
|
FileAttrPolicy policy, bool update, bool no_replace,
|
||||||
bool use_fsync, const char* temp_dir,
|
bool use_fsync, const char* temp_dir,
|
||||||
const FileXattrList* xattrs, bool fake_super,
|
const FileXattrList* xattrs, bool fake_super,
|
||||||
bool keep_partial) {
|
bool keep_partial) {
|
||||||
@@ -887,6 +1020,17 @@ static bool file_to_disk_secure_impl(const char* path, const void* data,
|
|||||||
return false;
|
return false;
|
||||||
int fd = -1;
|
int fd = -1;
|
||||||
bool ok = false;
|
bool ok = false;
|
||||||
|
/* Set when a --temp-dir install fails with EXDEV: rsync then falls back to a
|
||||||
|
* non-atomic write directly in the destination directory (see the tail of
|
||||||
|
* this function). */
|
||||||
|
bool cross_device_fallback = false;
|
||||||
|
/* The base mode applied when --perms is off (neither the source mode nor an
|
||||||
|
* exec-only change is taken wholesale): a pre-existing destination keeps its
|
||||||
|
* own mode (special bits dropped), while a brand-new file uses
|
||||||
|
* source&~umask when metadata is available (see file_mode_base) or 0644 when
|
||||||
|
* there is none. Captured from the destination probe before the write. */
|
||||||
|
mode_t existing_mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH;
|
||||||
|
bool existing_mode_known = false;
|
||||||
if (inplace) {
|
if (inplace) {
|
||||||
/* --inplace writes directly into the destination; a scratch --temp-dir
|
/* --inplace writes directly into the destination; a scratch --temp-dir
|
||||||
does not apply and must never redirect these writes. */
|
does not apply and must never redirect these writes. */
|
||||||
@@ -897,11 +1041,17 @@ static bool file_to_disk_secure_impl(const char* path, const void* data,
|
|||||||
super-mode gate (a client-controlled device write). fstatat with
|
super-mode gate (a client-controlled device write). fstatat with
|
||||||
AT_SYMLINK_NOFOLLOW does not follow a symlink and does not block. */
|
AT_SYMLINK_NOFOLLOW does not follow a symlink and does not block. */
|
||||||
struct stat pre_stat;
|
struct stat pre_stat;
|
||||||
if (fstatat(dirfd, leaf, &pre_stat, AT_SYMLINK_NOFOLLOW) == 0 && !S_ISREG(pre_stat.st_mode)) {
|
if (fstatat(dirfd, leaf, &pre_stat, AT_SYMLINK_NOFOLLOW) == 0) {
|
||||||
|
if (!S_ISREG(pre_stat.st_mode)) {
|
||||||
close(dirfd);
|
close(dirfd);
|
||||||
free(leaf);
|
free(leaf);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
/* Capture the old destination mode before the overwrite so a no--p/-E
|
||||||
|
* write can restore it (the write itself may clear setuid/setgid). */
|
||||||
|
existing_mode = pre_stat.st_mode & 0777;
|
||||||
|
existing_mode_known = true;
|
||||||
|
}
|
||||||
/* O_NONBLOCK: a no-op for a regular file, but a raced-in FIFO cannot block
|
/* O_NONBLOCK: a no-op for a regular file, but a raced-in FIFO cannot block
|
||||||
the open before the post-open S_ISREG re-check rejects it. */
|
the open before the post-open S_ISREG re-check rejects it. */
|
||||||
fd = openat(dirfd, leaf, O_WRONLY | O_CREAT | O_CLOEXEC | O_NOFOLLOW | O_NONBLOCK, 0644);
|
fd = openat(dirfd, leaf, O_WRONLY | O_CREAT | O_CLOEXEC | O_NOFOLLOW | O_NONBLOCK, 0644);
|
||||||
@@ -925,11 +1075,10 @@ static bool file_to_disk_secure_impl(const char* path, const void* data,
|
|||||||
} else {
|
} else {
|
||||||
/* Preallocate the expected payload size before writing so an
|
/* Preallocate the expected payload size before writing so an
|
||||||
out-of-space condition fails cleanly up front (--preallocate).
|
out-of-space condition fails cleanly up front (--preallocate).
|
||||||
--sparse takes precedence: posix_fallocate would allocate every
|
rsync lets --preallocate win over --sparse (the reserved blocks
|
||||||
block, defeating the holes the sparse writer would create, so the
|
survive the sparse writer's seeks), so both flags can be active. */
|
||||||
two never combine here (the ftruncate presize below stays). */
|
|
||||||
int prealloc_rc = 0;
|
int prealloc_rc = 0;
|
||||||
if (preallocate && !sparse && data_size > 0) {
|
if (preallocate && data_size > 0) {
|
||||||
prealloc_rc = preallocate_fd(fd, data_size);
|
prealloc_rc = preallocate_fd(fd, data_size);
|
||||||
if (prealloc_rc != 0) {
|
if (prealloc_rc != 0) {
|
||||||
char* escaped_path = output_escape(path, log_get_8_bit_output());
|
char* escaped_path = output_escape(path, log_get_8_bit_output());
|
||||||
@@ -953,15 +1102,25 @@ static bool file_to_disk_secure_impl(const char* path, const void* data,
|
|||||||
/* Normalize the mode: apply the metadata-derived safe mode when the
|
/* Normalize the mode: apply the metadata-derived safe mode when the
|
||||||
sender supplied metadata (setuid/setgid/sticky are never honored);
|
sender supplied metadata (setuid/setgid/sticky are never honored);
|
||||||
otherwise fall back to a safe default so dangerous bits on an
|
otherwise fall back to a safe default so dangerous bits on an
|
||||||
existing destination cannot survive an overwrite. */
|
existing destination cannot survive an overwrite. When the policy
|
||||||
|
requests neither -p nor -E the source mode is deliberately ignored
|
||||||
|
and the pre-existing destination mode (or 0644 for a new file) is
|
||||||
|
restored instead. The exec-bits-only -E change is likewise applied
|
||||||
|
on top of that destination-derived base, not the scratch file's
|
||||||
|
0600. */
|
||||||
if (ok) {
|
if (ok) {
|
||||||
if (metadata)
|
if (metadata) {
|
||||||
ok = file_restore_metadata_fd(fd, metadata, preserve_executability);
|
if (!policy.perms &&
|
||||||
else if (fchmod(fd, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH) != 0)
|
fchmod(fd, file_mode_base(metadata, existing_mode_known, existing_mode)) != 0)
|
||||||
|
ok = false;
|
||||||
|
if (ok)
|
||||||
|
ok = file_restore_metadata_fd(fd, metadata, policy);
|
||||||
|
} else if (fchmod(fd, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH) != 0) {
|
||||||
ok = false;
|
ok = false;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
if (ok)
|
if (ok)
|
||||||
restore_extra_fd(fd, metadata, xattrs, fake_super);
|
restore_extra_fd(fd, metadata, xattrs, fake_super, policy);
|
||||||
if (ok && use_fsync)
|
if (ok && use_fsync)
|
||||||
ok = fsync(fd) == 0;
|
ok = fsync(fd) == 0;
|
||||||
}
|
}
|
||||||
@@ -974,25 +1133,31 @@ static bool file_to_disk_secure_impl(const char* path, const void* data,
|
|||||||
install failure (partial data may exist, --partial may retain it) from a
|
install failure (partial data may exist, --partial may retain it) from a
|
||||||
pre-write validation failure (nothing to retain). */
|
pre-write validation failure (nothing to retain). */
|
||||||
bool write_attempted = false;
|
bool write_attempted = false;
|
||||||
if (update && metadata) {
|
/* Probe the destination ONCE up front: it both drives the --update check
|
||||||
/* This check protects the normal atomic path as far as possible. A
|
and records the pre-existing mode the no--p/-E fallback preserves. */
|
||||||
concurrent replacement can still occur before the final rename. */
|
|
||||||
struct stat destination_stat;
|
struct stat destination_stat;
|
||||||
if (fstatat(dirfd, leaf, &destination_stat, AT_SYMLINK_NOFOLLOW) == 0 &&
|
bool destination_is_regular =
|
||||||
S_ISREG(destination_stat.st_mode) && stat_is_newer(&destination_stat, metadata)) {
|
fstatat(dirfd, leaf, &destination_stat, AT_SYMLINK_NOFOLLOW) == 0 &&
|
||||||
|
S_ISREG(destination_stat.st_mode);
|
||||||
|
if (destination_is_regular) {
|
||||||
|
existing_mode = destination_stat.st_mode & 0777;
|
||||||
|
existing_mode_known = true;
|
||||||
|
}
|
||||||
|
if (update && metadata && destination_is_regular &&
|
||||||
|
stat_is_newer(&destination_stat, metadata)) {
|
||||||
close(dirfd);
|
close(dirfd);
|
||||||
free(leaf);
|
free(leaf);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
/* Scratch directory for the temporary working copy. When NULL the temp
|
/* Scratch directory for the temporary working copy. When NULL the temp
|
||||||
file is created in the destination directory, exactly as historically. */
|
file is created in the destination directory, exactly as historically. */
|
||||||
int scratch_dirfd = -1;
|
int scratch_dirfd = -1;
|
||||||
if (temp_dir) {
|
if (temp_dir) {
|
||||||
scratch_dirfd = file_open_private_dir(temp_dir);
|
scratch_dirfd = file_open_temp_dir(temp_dir);
|
||||||
if (scratch_dirfd < 0) {
|
if (scratch_dirfd < 0) {
|
||||||
int saved_errno = errno;
|
int saved_errno = errno;
|
||||||
log_message(LOG_LEVEL_ERROR, "could not open --temp-dir scratch directory '%s': %s",
|
log_message(LOG_LEVEL_ERROR,
|
||||||
|
"--temp-dir '%s' could not be opened (rsync requires it to already exist): %s",
|
||||||
temp_dir, strerror(saved_errno));
|
temp_dir, strerror(saved_errno));
|
||||||
close(dirfd);
|
close(dirfd);
|
||||||
free(leaf);
|
free(leaf);
|
||||||
@@ -1039,7 +1204,7 @@ static bool file_to_disk_secure_impl(const char* path, const void* data,
|
|||||||
if (fd < 0)
|
if (fd < 0)
|
||||||
continue; /* EEXIST (or a transient open error): try a fresh name. */
|
continue; /* EEXIST (or a transient open error): try a fresh name. */
|
||||||
int prealloc_rc = 0;
|
int prealloc_rc = 0;
|
||||||
if (preallocate && !sparse && data_size > 0) {
|
if (preallocate && data_size > 0) {
|
||||||
prealloc_rc = preallocate_fd(fd, data_size);
|
prealloc_rc = preallocate_fd(fd, data_size);
|
||||||
if (prealloc_rc != 0) {
|
if (prealloc_rc != 0) {
|
||||||
char* escaped_path = output_escape(path, log_get_8_bit_output());
|
char* escaped_path = output_escape(path, log_get_8_bit_output());
|
||||||
@@ -1061,10 +1226,19 @@ static bool file_to_disk_secure_impl(const char* path, const void* data,
|
|||||||
? file_store_write_sparse(fd, (const unsigned char*)data, data_size)
|
? file_store_write_sparse(fd, (const unsigned char*)data, data_size)
|
||||||
: write_all(fd, data, data_size);
|
: write_all(fd, data, data_size);
|
||||||
}
|
}
|
||||||
if (ok && metadata)
|
if (ok) {
|
||||||
ok = file_restore_metadata_fd(fd, metadata, preserve_executability);
|
if (metadata) {
|
||||||
|
if (!policy.perms &&
|
||||||
|
fchmod(fd, file_mode_base(metadata, existing_mode_known, existing_mode)) != 0)
|
||||||
|
ok = false;
|
||||||
if (ok)
|
if (ok)
|
||||||
restore_extra_fd(fd, metadata, xattrs, fake_super);
|
ok = file_restore_metadata_fd(fd, metadata, policy);
|
||||||
|
} else if (fchmod(fd, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH) != 0) {
|
||||||
|
ok = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (ok)
|
||||||
|
restore_extra_fd(fd, metadata, xattrs, fake_super, policy);
|
||||||
if (ok && use_fsync)
|
if (ok && use_fsync)
|
||||||
ok = fsync(fd) == 0;
|
ok = fsync(fd) == 0;
|
||||||
}
|
}
|
||||||
@@ -1081,17 +1255,16 @@ static bool file_to_disk_secure_impl(const char* path, const void* data,
|
|||||||
errno != ENOENT)
|
errno != ENOENT)
|
||||||
ok = false;
|
ok = false;
|
||||||
} else {
|
} else {
|
||||||
|
/* Cross-device (or otherwise impossible) link: rsync falls back to
|
||||||
|
writing the file directly in the destination directory. Record
|
||||||
|
it and retry below with no scratch dir. */
|
||||||
if (scratch_dirfd >= 0 && errno == EXDEV)
|
if (scratch_dirfd >= 0 && errno == EXDEV)
|
||||||
log_message(LOG_LEVEL_ERROR,
|
cross_device_fallback = true;
|
||||||
"temp dir is on a different filesystem than the destination; cannot "
|
|
||||||
"link file into place (EXDEV); no fallback copy is attempted");
|
|
||||||
ok = false;
|
ok = false;
|
||||||
}
|
}
|
||||||
} else if (renameat(scratch_dirfd >= 0 ? scratch_dirfd : dirfd, tmp, dirfd, leaf) != 0) {
|
} else if (renameat(scratch_dirfd >= 0 ? scratch_dirfd : dirfd, tmp, dirfd, leaf) != 0) {
|
||||||
if (scratch_dirfd >= 0 && errno == EXDEV)
|
if (scratch_dirfd >= 0 && errno == EXDEV)
|
||||||
log_message(LOG_LEVEL_ERROR,
|
cross_device_fallback = true;
|
||||||
"temp dir is on a different filesystem than the destination; cannot "
|
|
||||||
"atomically install file (EXDEV); no fallback copy is attempted");
|
|
||||||
ok = false;
|
ok = false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1124,43 +1297,49 @@ static bool file_to_disk_secure_impl(const char* path, const void* data,
|
|||||||
close(fd);
|
close(fd);
|
||||||
close(dirfd);
|
close(dirfd);
|
||||||
free(leaf);
|
free(leaf);
|
||||||
|
if (cross_device_fallback) {
|
||||||
|
/* rsync semantics: a --temp-dir on another filesystem must not abort the
|
||||||
|
write. Retry once with no scratch dir so the file is written and
|
||||||
|
installed non-atomically in the destination directory. */
|
||||||
|
log_message(LOG_LEVEL_WARNING,
|
||||||
|
"temp dir is on a different filesystem than the destination; falling back to a "
|
||||||
|
"non-atomic copy into the destination directory");
|
||||||
|
return file_to_disk_secure_impl(path, data, data_size, inplace, sparse, preallocate, metadata,
|
||||||
|
policy, update, no_replace, use_fsync, NULL, xattrs, fake_super,
|
||||||
|
keep_partial);
|
||||||
|
}
|
||||||
return ok;
|
return ok;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool file_to_disk_secure(const char* path, const void* data, unsigned long long data_size,
|
bool file_to_disk_secure(const char* path, const void* data, unsigned long long data_size,
|
||||||
bool inplace, bool sparse, bool preallocate, const FileMetadata* metadata,
|
bool inplace, bool sparse, bool preallocate, const FileMetadata* metadata,
|
||||||
bool preserve_executability, const char* temp_dir) {
|
FileAttrPolicy policy, const char* temp_dir) {
|
||||||
return file_to_disk_secure_impl(path, data, data_size, inplace, sparse, preallocate, metadata,
|
return file_to_disk_secure_impl(path, data, data_size, inplace, sparse, preallocate, metadata,
|
||||||
preserve_executability, false, false, false, temp_dir, NULL,
|
policy, false, false, false, temp_dir, NULL, false, false);
|
||||||
false, false);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
bool file_to_disk_secure_update(const char* path, const void* data, unsigned long long data_size,
|
bool file_to_disk_secure_update(const char* path, const void* data, unsigned long long data_size,
|
||||||
bool inplace, bool sparse, bool preallocate,
|
bool inplace, bool sparse, bool preallocate,
|
||||||
const FileMetadata* metadata, bool preserve_executability,
|
const FileMetadata* metadata, FileAttrPolicy policy,
|
||||||
const char* temp_dir) {
|
const char* temp_dir) {
|
||||||
return file_to_disk_secure_impl(path, data, data_size, inplace, sparse, preallocate, metadata,
|
return file_to_disk_secure_impl(path, data, data_size, inplace, sparse, preallocate, metadata,
|
||||||
preserve_executability, true, false, false, temp_dir, NULL, false,
|
policy, true, false, false, temp_dir, NULL, false, false);
|
||||||
false);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
bool file_to_disk_secure_with_fsync(const char* path, const void* data,
|
bool file_to_disk_secure_with_fsync(const char* path, const void* data,
|
||||||
unsigned long long data_size, bool inplace, bool sparse,
|
unsigned long long data_size, bool inplace, bool sparse,
|
||||||
bool preallocate, const FileMetadata* metadata,
|
bool preallocate, const FileMetadata* metadata,
|
||||||
bool preserve_executability, bool use_fsync,
|
FileAttrPolicy policy, bool use_fsync, const char* temp_dir) {
|
||||||
const char* temp_dir) {
|
|
||||||
return file_to_disk_secure_impl(path, data, data_size, inplace, sparse, preallocate, metadata,
|
return file_to_disk_secure_impl(path, data, data_size, inplace, sparse, preallocate, metadata,
|
||||||
preserve_executability, false, false, use_fsync, temp_dir, NULL,
|
policy, false, false, use_fsync, temp_dir, NULL, false, false);
|
||||||
false, false);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
bool file_to_disk_secure_no_replace(const char* path, const void* data,
|
bool file_to_disk_secure_no_replace(const char* path, const void* data,
|
||||||
unsigned long long data_size, bool sparse, bool preallocate,
|
unsigned long long data_size, bool sparse, bool preallocate,
|
||||||
const FileMetadata* metadata, bool preserve_executability,
|
const FileMetadata* metadata, FileAttrPolicy policy,
|
||||||
const char* temp_dir) {
|
const char* temp_dir) {
|
||||||
return file_to_disk_secure_impl(path, data, data_size, false, sparse, preallocate, metadata,
|
return file_to_disk_secure_impl(path, data, data_size, false, sparse, preallocate, metadata,
|
||||||
preserve_executability, false, true, false, temp_dir, NULL, false,
|
policy, false, true, false, temp_dir, NULL, false, false);
|
||||||
false);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Receiver write-path variant that also applies the per-file xattrs (-X/-A)
|
/* Receiver write-path variant that also applies the per-file xattrs (-X/-A)
|
||||||
@@ -1170,13 +1349,12 @@ bool file_to_disk_secure_no_replace(const char* path, const void* data,
|
|||||||
* failed write's temp. See file_to_disk_secure_impl for the semantics. */
|
* failed write's temp. See file_to_disk_secure_impl for the semantics. */
|
||||||
bool file_to_disk_secure_attrs(const char* path, const void* data, unsigned long long data_size,
|
bool file_to_disk_secure_attrs(const char* path, const void* data, unsigned long long data_size,
|
||||||
bool inplace, bool sparse, bool preallocate,
|
bool inplace, bool sparse, bool preallocate,
|
||||||
const FileMetadata* metadata, bool preserve_executability,
|
const FileMetadata* metadata, FileAttrPolicy policy, bool update,
|
||||||
bool update, bool no_replace, bool use_fsync,
|
bool no_replace, bool use_fsync, const FileXattrList* xattrs,
|
||||||
const FileXattrList* xattrs, bool fake_super, bool keep_partial,
|
bool fake_super, bool keep_partial, const char* temp_dir) {
|
||||||
const char* temp_dir) {
|
|
||||||
return file_to_disk_secure_impl(path, data, data_size, inplace, sparse, preallocate, metadata,
|
return file_to_disk_secure_impl(path, data, data_size, inplace, sparse, preallocate, metadata,
|
||||||
preserve_executability, update, no_replace, use_fsync, temp_dir,
|
policy, update, no_replace, use_fsync, temp_dir, xattrs,
|
||||||
xattrs, fake_super, keep_partial);
|
fake_super, keep_partial);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Atomic --link-dest install. The destination is replaced (via a temporary
|
/* Atomic --link-dest install. The destination is replaced (via a temporary
|
||||||
@@ -1197,7 +1375,7 @@ bool file_to_disk_secure_attrs(const char* path, const void* data, unsigned long
|
|||||||
static bool file_to_disk_secure_link_impl(const char* path, const char* basis_path,
|
static bool file_to_disk_secure_link_impl(const char* path, const char* basis_path,
|
||||||
const void* data, unsigned long long data_size,
|
const void* data, unsigned long long data_size,
|
||||||
bool preallocate, const FileMetadata* metadata,
|
bool preallocate, const FileMetadata* metadata,
|
||||||
bool preserve_executability, bool use_fsync,
|
FileAttrPolicy policy, bool use_fsync,
|
||||||
const FileXattrList* xattrs, bool fake_super,
|
const FileXattrList* xattrs, bool fake_super,
|
||||||
const char* temp_dir) {
|
const char* temp_dir) {
|
||||||
if (!path || !basis_path)
|
if (!path || !basis_path)
|
||||||
@@ -1209,11 +1387,12 @@ static bool file_to_disk_secure_link_impl(const char* path, const char* basis_pa
|
|||||||
|
|
||||||
int scratch_dirfd = -1;
|
int scratch_dirfd = -1;
|
||||||
if (temp_dir) {
|
if (temp_dir) {
|
||||||
scratch_dirfd = file_open_private_dir(temp_dir);
|
scratch_dirfd = file_open_temp_dir(temp_dir);
|
||||||
if (scratch_dirfd < 0) {
|
if (scratch_dirfd < 0) {
|
||||||
int saved_errno = errno;
|
int saved_errno = errno;
|
||||||
log_message(LOG_LEVEL_ERROR, "could not open --temp-dir scratch directory '%s': %s", temp_dir,
|
log_message(LOG_LEVEL_ERROR,
|
||||||
strerror(saved_errno));
|
"--temp-dir '%s' could not be opened (rsync requires it to already exist): %s",
|
||||||
|
temp_dir, strerror(saved_errno));
|
||||||
close(dirfd);
|
close(dirfd);
|
||||||
free(leaf);
|
free(leaf);
|
||||||
return false;
|
return false;
|
||||||
@@ -1286,8 +1465,8 @@ static bool file_to_disk_secure_link_impl(const char* path, const char* basis_pa
|
|||||||
/* The basis file could not be linked in (missing, cross-device, refused
|
/* The basis file could not be linked in (missing, cross-device, refused
|
||||||
by the filesystem). Write a byte-identical local copy instead. */
|
by the filesystem). Write a byte-identical local copy instead. */
|
||||||
return file_to_disk_secure_attrs(path, data, data_size, false, false, preallocate, metadata,
|
return file_to_disk_secure_attrs(path, data, data_size, false, false, preallocate, metadata,
|
||||||
preserve_executability, false, false, use_fsync, xattrs,
|
policy, false, false, use_fsync, xattrs, fake_super, false,
|
||||||
fake_super, false, temp_dir);
|
temp_dir);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (scratch_dirfd >= 0)
|
if (scratch_dirfd >= 0)
|
||||||
@@ -1299,25 +1478,25 @@ static bool file_to_disk_secure_link_impl(const char* path, const char* basis_pa
|
|||||||
|
|
||||||
bool file_to_disk_secure_link(const char* path, const char* basis_path, const void* data,
|
bool file_to_disk_secure_link(const char* path, const char* basis_path, const void* data,
|
||||||
unsigned long long data_size, bool preallocate,
|
unsigned long long data_size, bool preallocate,
|
||||||
const FileMetadata* metadata, bool preserve_executability,
|
const FileMetadata* metadata, FileAttrPolicy policy, bool use_fsync,
|
||||||
bool use_fsync, const char* temp_dir) {
|
const char* temp_dir) {
|
||||||
return file_to_disk_secure_link_impl(path, basis_path, data, data_size, preallocate, metadata,
|
return file_to_disk_secure_link_impl(path, basis_path, data, data_size, preallocate, metadata,
|
||||||
preserve_executability, use_fsync, NULL, false, temp_dir);
|
policy, use_fsync, NULL, false, temp_dir);
|
||||||
}
|
}
|
||||||
|
|
||||||
bool file_to_disk_secure_link_attrs(const char* path, const char* basis_path, const void* data,
|
bool file_to_disk_secure_link_attrs(const char* path, const char* basis_path, const void* data,
|
||||||
unsigned long long data_size, bool preallocate,
|
unsigned long long data_size, bool preallocate,
|
||||||
const FileMetadata* metadata, bool preserve_executability,
|
const FileMetadata* metadata, FileAttrPolicy policy,
|
||||||
bool use_fsync, const FileXattrList* xattrs, bool fake_super,
|
bool use_fsync, const FileXattrList* xattrs, bool fake_super,
|
||||||
const char* temp_dir) {
|
const char* temp_dir) {
|
||||||
return file_to_disk_secure_link_impl(path, basis_path, data, data_size, preallocate, metadata,
|
return file_to_disk_secure_link_impl(path, basis_path, data, data_size, preallocate, metadata,
|
||||||
preserve_executability, use_fsync, xattrs, fake_super,
|
policy, use_fsync, xattrs, fake_super, temp_dir);
|
||||||
temp_dir);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
bool file_write_to_disk(const char* path, const void* data, unsigned long long data_size,
|
bool file_write_to_disk(const char* path, const void* data, unsigned long long data_size,
|
||||||
bool inplace, bool sparse) {
|
bool inplace, bool sparse) {
|
||||||
if (!path || (!data && data_size != 0) || has_path_traversal(path))
|
if (!path || (!data && data_size != 0) || has_path_traversal(path))
|
||||||
return false;
|
return false;
|
||||||
return file_to_disk_secure(path, data, data_size, inplace, sparse, false, NULL, false, NULL);
|
FileAttrPolicy policy = {false, false, false, false};
|
||||||
|
return file_to_disk_secure(path, data, data_size, inplace, sparse, false, NULL, policy, NULL);
|
||||||
}
|
}
|
||||||
|
|||||||
+50
-28
@@ -28,17 +28,36 @@ void file_metadata_destroy(void* metadata);
|
|||||||
/* --open-noatime process-wide sender policy; see file.c. */
|
/* --open-noatime process-wide sender policy; see file.c. */
|
||||||
void file_set_open_noatime(bool enable);
|
void file_set_open_noatime(bool enable);
|
||||||
bool file_get_open_noatime(void);
|
bool file_get_open_noatime(void);
|
||||||
|
/* Capture the process umask ONCE, before any threads are created. Call this at
|
||||||
|
* the very top of main() in both entry points so the cached value is read while
|
||||||
|
* the process is still single-threaded: reading the umask needs a get+set round
|
||||||
|
* trip (umask(0); umask(old)), which would race against receiver threads
|
||||||
|
* creating files if it happened during the first write. Idempotent and safe to
|
||||||
|
* call more than once. */
|
||||||
|
void file_umask_capture(void);
|
||||||
|
/* Process-wide umask, captured once (thread-safe). Used to derive the mode of
|
||||||
|
* a brand-new destination like rsync: source_mode & 0777 & ~umask. Falls back
|
||||||
|
* to file_umask_capture() (behind pthread_once) if capture was never called. */
|
||||||
|
unsigned file_process_umask(void);
|
||||||
/* Open `path` read-only for transfer, honouring --open-noatime when set. */
|
/* Open `path` read-only for transfer, honouring --open-noatime when set. */
|
||||||
int file_open_for_read(const char* path);
|
int file_open_for_read(const char* path);
|
||||||
bool file_write_to_disk(const char* path, const void* data, unsigned long long data_size,
|
bool file_write_to_disk(const char* path, const void* data, unsigned long long data_size,
|
||||||
bool inplace, bool sparse);
|
bool inplace, bool sparse);
|
||||||
|
|
||||||
/* Symlink trust-boundary helpers (Phase 4, symlink wave). --munge-links
|
/* Symlink trust-boundary helpers (Phase 4, symlink wave; rsync parity).
|
||||||
* sender-side marker: every transmitted symlink target is prefixed with this
|
* --munge-links is a RECEIVER-side rewrite: rsync prefixes every stored symlink
|
||||||
* while the flag is on; the receiver strips it to restore the real target. */
|
* target with this marker, making the link unusable while the referenced
|
||||||
#define SYMLINK_MUNGE_PREFIX "#SYMLINK/"
|
* directory does not exist. A SENDER receiving a munged source strips it back
|
||||||
|
* off before transmitting (so a munged tree round-trips through the receiver's
|
||||||
|
* re-munging). */
|
||||||
|
#define SYMLINK_MUNGE_PREFIX "/rsyncd-munged/"
|
||||||
|
|
||||||
char* file_symlink_munge(const char* target);
|
char* file_symlink_munge(const char* target);
|
||||||
|
/* rsync 3.4.1 unsafe_symlink(): true when `target` escapes the transfer tree
|
||||||
|
* rooted at `link_path` (the symlink's transfer-relative path incl. its name).
|
||||||
|
* Absolute/empty targets and targets climbing above the transfer root (via
|
||||||
|
* "..") are unsafe, as are internal "/../" components and trailing "/..". */
|
||||||
|
bool file_symlink_unsafe(const char* target, const char* link_path);
|
||||||
/* True when a lexical target is relative and contains no ".." component, so it
|
/* True when a lexical target is relative and contains no ".." component, so it
|
||||||
* can never escape the receive root once created beneath it. */
|
* can never escape the receive root once created beneath it. */
|
||||||
bool file_symlink_target_contained(const char* target);
|
bool file_symlink_target_contained(const char* target);
|
||||||
@@ -46,8 +65,9 @@ bool file_symlink_target_contained(const char* target);
|
|||||||
* returns true when a marker was removed. */
|
* returns true when a marker was removed. */
|
||||||
bool file_symlink_unmunge(char* target);
|
bool file_symlink_unmunge(char* target);
|
||||||
/* Create a symlink at `path` -> `target`, confined below the authorized root
|
/* Create a symlink at `path` -> `target`, confined below the authorized root
|
||||||
* (O_NOFOLLOW parent walk, symlinkat; the target is never followed). Returns
|
* (O_NOFOLLOW parent walk, symlinkat; the target is never followed). The link
|
||||||
* false when a directory already occupies `path`. */
|
* value is copied verbatim (rsync -l); only the placement path is confined.
|
||||||
|
* Returns false when a directory already occupies `path`. */
|
||||||
bool file_symlink_at_secure(const char* path, const char* target);
|
bool file_symlink_at_secure(const char* path, const char* target);
|
||||||
/* --keep-dirlinks (-K) receiver process-wide policy: allow an in-root existing
|
/* --keep-dirlinks (-K) receiver process-wide policy: allow an in-root existing
|
||||||
* symlink-to-directory to be followed as a directory. */
|
* symlink-to-directory to be followed as a directory. */
|
||||||
@@ -75,37 +95,40 @@ bool file_rename_secure(const char* old_path, const char* new_path);
|
|||||||
regular file. See the .c for the exact success semantics. */
|
regular file. See the .c for the exact success semantics. */
|
||||||
bool file_remove_tree_secure(const char* path);
|
bool file_remove_tree_secure(const char* path);
|
||||||
/* Open a private 0700 directory (creating it on demand) that must live below
|
/* Open a private 0700 directory (creating it on demand) that must live below
|
||||||
the authorized root. Used for the --temp-dir scratch directory and the
|
the authorized root. Used for the --delay-updates staging directory. */
|
||||||
--delay-updates staging directory. */
|
|
||||||
int file_open_private_dir(const char* dir_path);
|
int file_open_private_dir(const char* dir_path);
|
||||||
|
|
||||||
|
/* Open an existing --temp-dir scratch directory as-is (absolute or relative;
|
||||||
|
no creation, no root confinement), matching rsync's --temp-dir handling. */
|
||||||
|
int file_open_temp_dir(const char* dir_path);
|
||||||
|
|
||||||
/* The file_to_disk_secure* variants write a temporary copy in the destination
|
/* The file_to_disk_secure* variants write a temporary copy in the destination
|
||||||
directory and atomically rename it over `path`. temp_dir is an absolute,
|
directory and atomically rename it over `path`. temp_dir is a scratch
|
||||||
root-confined scratch directory (already validated by the caller): when it
|
directory (an absolute path, or one the caller already resolved against the
|
||||||
is non-NULL the temporary copy is instead created there (with a name unique
|
destination root): when it is non-NULL the temporary copy is instead created
|
||||||
across the whole scratch directory) and atomically renamed into the
|
there (with a name unique across the whole scratch directory) and atomically
|
||||||
destination directory once fully written and fsynced. A rename across
|
renamed into the destination directory once fully written and fsynced. When
|
||||||
filesystems (EXDEV) fails the write with an error; the file is never
|
that rename/link fails with EXDEV (the scratch dir is on another filesystem)
|
||||||
silently copied into place. Pass NULL for the historical same-directory
|
the write falls back to a non-atomic copy directly in the destination
|
||||||
behavior. --inplace writes never use temp_dir. */
|
directory, matching rsync. Pass NULL for the same-directory behavior.
|
||||||
|
--inplace writes never use temp_dir. */
|
||||||
bool file_to_disk_secure(const char* path, const void* data, unsigned long long data_size,
|
bool file_to_disk_secure(const char* path, const void* data, unsigned long long data_size,
|
||||||
bool inplace, bool sparse, bool preallocate, const FileMetadata* metadata,
|
bool inplace, bool sparse, bool preallocate, const FileMetadata* metadata,
|
||||||
bool preserve_executability, const char* temp_dir);
|
FileAttrPolicy policy, const char* temp_dir);
|
||||||
bool file_to_disk_secure_with_fsync(const char* path, const void* data,
|
bool file_to_disk_secure_with_fsync(const char* path, const void* data,
|
||||||
unsigned long long data_size, bool inplace, bool sparse,
|
unsigned long long data_size, bool inplace, bool sparse,
|
||||||
bool preallocate, const FileMetadata* metadata,
|
bool preallocate, const FileMetadata* metadata,
|
||||||
bool preserve_executability, bool use_fsync,
|
FileAttrPolicy policy, bool use_fsync, const char* temp_dir);
|
||||||
const char* temp_dir);
|
|
||||||
/* With update enabled, an existing newer destination is left untouched. The
|
/* With update enabled, an existing newer destination is left untouched. The
|
||||||
check is descriptor-based for inplace writes; atomic replacement still has
|
check is descriptor-based for inplace writes; atomic replacement still has
|
||||||
an unavoidable final rename race without filesystem locking. */
|
an unavoidable final rename race without filesystem locking. */
|
||||||
bool file_to_disk_secure_update(const char* path, const void* data, unsigned long long data_size,
|
bool file_to_disk_secure_update(const char* path, const void* data, unsigned long long data_size,
|
||||||
bool inplace, bool sparse, bool preallocate,
|
bool inplace, bool sparse, bool preallocate,
|
||||||
const FileMetadata* metadata, bool preserve_executability,
|
const FileMetadata* metadata, FileAttrPolicy policy,
|
||||||
const char* temp_dir);
|
const char* temp_dir);
|
||||||
bool file_to_disk_secure_no_replace(const char* path, const void* data,
|
bool file_to_disk_secure_no_replace(const char* path, const void* data,
|
||||||
unsigned long long data_size, bool sparse, bool preallocate,
|
unsigned long long data_size, bool sparse, bool preallocate,
|
||||||
const FileMetadata* metadata, bool preserve_executability,
|
const FileMetadata* metadata, FileAttrPolicy policy,
|
||||||
const char* temp_dir);
|
const char* temp_dir);
|
||||||
/* Receiver write-path variant that also applies per-file xattrs (-X/-A) and the
|
/* Receiver write-path variant that also applies per-file xattrs (-X/-A) and the
|
||||||
* --fake-super stat xattr fd-relative before the final rename. `update` /
|
* --fake-super stat xattr fd-relative before the final rename. `update` /
|
||||||
@@ -113,10 +136,9 @@ bool file_to_disk_secure_no_replace(const char* path, const void* data,
|
|||||||
* enables --partial best-effort retention of a failed write's temp. */
|
* enables --partial best-effort retention of a failed write's temp. */
|
||||||
bool file_to_disk_secure_attrs(const char* path, const void* data, unsigned long long data_size,
|
bool file_to_disk_secure_attrs(const char* path, const void* data, unsigned long long data_size,
|
||||||
bool inplace, bool sparse, bool preallocate,
|
bool inplace, bool sparse, bool preallocate,
|
||||||
const FileMetadata* metadata, bool preserve_executability,
|
const FileMetadata* metadata, FileAttrPolicy policy, bool update,
|
||||||
bool update, bool no_replace, bool use_fsync,
|
bool no_replace, bool use_fsync, const FileXattrList* xattrs,
|
||||||
const FileXattrList* xattrs, bool fake_super, bool keep_partial,
|
bool fake_super, bool keep_partial, const char* temp_dir);
|
||||||
const char* temp_dir);
|
|
||||||
/* Atomic --link-dest install: replace `path` with a hard link to `basis_path`
|
/* Atomic --link-dest install: replace `path` with a hard link to `basis_path`
|
||||||
(via a temp name + rename); fall back to a byte-identical local copy from
|
(via a temp name + rename); fall back to a byte-identical local copy from
|
||||||
`data` when the link is impossible (EXDEV/EPERM/unsupported filesystem).
|
`data` when the link is impossible (EXDEV/EPERM/unsupported filesystem).
|
||||||
@@ -125,15 +147,15 @@ bool file_to_disk_secure_attrs(const char* path, const void* data, unsigned long
|
|||||||
never re-allocated). */
|
never re-allocated). */
|
||||||
bool file_to_disk_secure_link(const char* path, const char* basis_path, const void* data,
|
bool file_to_disk_secure_link(const char* path, const char* basis_path, const void* data,
|
||||||
unsigned long long data_size, bool preallocate,
|
unsigned long long data_size, bool preallocate,
|
||||||
const FileMetadata* metadata, bool preserve_executability,
|
const FileMetadata* metadata, FileAttrPolicy policy, bool use_fsync,
|
||||||
bool use_fsync, const char* temp_dir);
|
const char* temp_dir);
|
||||||
/* Like file_to_disk_secure_link, but the byte-copy fallback also applies the
|
/* Like file_to_disk_secure_link, but the byte-copy fallback also applies the
|
||||||
* per-file xattrs (-X/-A) and --fake-super stat xattr (fd-relative). On a
|
* per-file xattrs (-X/-A) and --fake-super stat xattr (fd-relative). On a
|
||||||
* successful hard link no attributes are applied (the shared inode already
|
* successful hard link no attributes are applied (the shared inode already
|
||||||
* carries the basis's). */
|
* carries the basis's). */
|
||||||
bool file_to_disk_secure_link_attrs(const char* path, const char* basis_path, const void* data,
|
bool file_to_disk_secure_link_attrs(const char* path, const char* basis_path, const void* data,
|
||||||
unsigned long long data_size, bool preallocate,
|
unsigned long long data_size, bool preallocate,
|
||||||
const FileMetadata* metadata, bool preserve_executability,
|
const FileMetadata* metadata, FileAttrPolicy policy,
|
||||||
bool use_fsync, const FileXattrList* xattrs, bool fake_super,
|
bool use_fsync, const FileXattrList* xattrs, bool fake_super,
|
||||||
const char* temp_dir);
|
const char* temp_dir);
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,38 @@
|
|||||||
|
#ifndef FILE_ATTR_H
|
||||||
|
#define FILE_ATTR_H
|
||||||
|
|
||||||
|
#include "config.h"
|
||||||
|
#include <stdbool.h>
|
||||||
|
#include <sys/stat.h>
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Per-attribute receiver policy for applying a transmitted FileMetadata. This
|
||||||
|
* is the split-out replacement for the former single use_metadata bundle: each
|
||||||
|
* flag is applied independently, matching rsync's -p/-t/-o/-g/-E/-U semantics.
|
||||||
|
* `use_metadata` remains the transport/presence gate (whether the metadata frame
|
||||||
|
* travelled at all); this struct decides which attributes are ACTUALLY applied.
|
||||||
|
*
|
||||||
|
* It lives in its own header (rather than metadata.h) because xattr.h's
|
||||||
|
* fake_super_restore_fd() takes one and metadata.h <-> file_types.h form an
|
||||||
|
* include cycle that must not be entered from xattr.h.
|
||||||
|
*
|
||||||
|
* The mode leg is: perms wins over executability; an exec-bits-only change is
|
||||||
|
* made only when perms is off; when neither is set the receiver deliberately
|
||||||
|
* sets no source mode. file.c then substitutes the pre-existing destination
|
||||||
|
* mode for a brand-new destination with metadata it uses the sanitized
|
||||||
|
* source-mode-&-umask base (S_IWGRP|S_IWOTH cleared), and the fixed 0644
|
||||||
|
* default only when no metadata is available at all, so a no--p overwrite
|
||||||
|
* does not lose the destination's perms.
|
||||||
|
*/
|
||||||
|
typedef struct FileAttrPolicy {
|
||||||
|
bool perms; /* config->preserve_perms: apply the source mode bits */
|
||||||
|
bool times; /* config->preserve_times: apply the source mtime */
|
||||||
|
bool atimes; /* config->preserve_atimes (-U): apply the source atime */
|
||||||
|
bool executability; /* config->use_executability (-E): exec-bits-only mode */
|
||||||
|
} FileAttrPolicy;
|
||||||
|
|
||||||
|
/* Build the per-attribute policy from a connection's Config. A NULL config
|
||||||
|
* yields the all-off policy (no attribute application). */
|
||||||
|
FileAttrPolicy file_attr_policy_from_config(const Config* config);
|
||||||
|
|
||||||
|
#endif
|
||||||
@@ -240,3 +240,29 @@ bool file_list_affects(const FileListSet* set, const char* rel) {
|
|||||||
entry (binary search for the first entry at or after `rel` + '/'). */
|
entry (binary search for the first entry at or after `rel` + '/'). */
|
||||||
return path_index_has_descendant(&set->index, rel);
|
return path_index_has_descendant(&set->index, rel);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool file_list_dir_in_scope(const FileListSet* set, const char* rel) {
|
||||||
|
if (!set || set->whole_tree)
|
||||||
|
return true;
|
||||||
|
if (!rel || rel[0] == '\0')
|
||||||
|
return false;
|
||||||
|
/* `rel` itself is listed, or one of its ancestor prefixes is an exact listed
|
||||||
|
directory (a listed prefix of a directory path is necessarily a
|
||||||
|
directory). */
|
||||||
|
size_t len = strlen(rel);
|
||||||
|
while (len > 0) {
|
||||||
|
const char* slash = NULL;
|
||||||
|
for (size_t i = len; i-- > 0;) {
|
||||||
|
if (rel[i] == '/') {
|
||||||
|
slash = rel + i;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!slash)
|
||||||
|
break;
|
||||||
|
len = (size_t)(slash - rel);
|
||||||
|
if (path_index_contains_n(&set->index, rel, len))
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return path_index_contains(&set->index, rel);
|
||||||
|
}
|
||||||
|
|||||||
@@ -40,4 +40,14 @@ void file_list_destroy(FileListSet* set);
|
|||||||
* this returns true, files are transferred only when it returns true. */
|
* this returns true, files are transferred only when it returns true. */
|
||||||
bool file_list_affects(const FileListSet* set, const char* rel);
|
bool file_list_affects(const FileListSet* set, const char* rel);
|
||||||
|
|
||||||
|
/* True when the DIRECTORY `rel` (path relative to the source root) is inside a
|
||||||
|
* listed directory subtree: `rel` itself is a listed entry, or one of `rel`'s
|
||||||
|
* ancestor directory prefixes is an exact listed entry. Unlike
|
||||||
|
* file_list_affects this does NOT treat an ancestor of a listed entry as
|
||||||
|
* affected, so an implied parent directory of a listed file is not synchronized
|
||||||
|
* (rsync deletes nothing in it). With no set or a whole-tree set every
|
||||||
|
* directory is in scope. This is the delete-walker's "synchronized directory"
|
||||||
|
* predicate. */
|
||||||
|
bool file_list_dir_in_scope(const FileListSet* set, const char* rel);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
+804
-267
File diff suppressed because it is too large
Load Diff
+69
-19
@@ -42,29 +42,37 @@ File* receive_incremental_check_ex(int fd, const Config* config, bool* skipped,
|
|||||||
typedef struct {
|
typedef struct {
|
||||||
char** paths; /* owned, destination-relative wire paths */
|
char** paths; /* owned, destination-relative wire paths */
|
||||||
FileMetadata* entries; /* owned, parallel to paths */
|
FileMetadata* entries; /* owned, parallel to paths */
|
||||||
|
FileXattrList** xattrs; /* owned, parallel to paths; NULL when none */
|
||||||
size_t count;
|
size_t count;
|
||||||
size_t capacity;
|
size_t capacity;
|
||||||
size_t bytes; /* cumulative strlen of every retained path */
|
size_t bytes; /* cumulative strlen of every retained path */
|
||||||
} DirTimeList;
|
} DirTimeList;
|
||||||
|
|
||||||
/* Capture gate shared by the sender-side and receiver-side sinks: directory
|
/* Capture gate shared by the sender-side and receiver-side sinks: directory
|
||||||
* metadata is accumulated only when --times/--metadata is in effect and
|
* metadata is accumulated only when a directory attribute is requested
|
||||||
* -O/--omit-dir-times does not suppress it. Kept here, next to the accumulator
|
* (-p/--perms for directory modes, or -t/--times for directory mtimes with
|
||||||
* it guards, so both call sites express the same condition. */
|
* -O/--omit-dir-times not suppressing them) and metadata rides the wire. Kept
|
||||||
bool dir_times_should_capture(const Config* config);
|
* here, next to the accumulator it guards, so both call sites express the same
|
||||||
|
* condition. */
|
||||||
|
bool dir_metadata_should_capture(const Config* config);
|
||||||
|
|
||||||
void dir_time_list_init(DirTimeList* list);
|
void dir_time_list_init(DirTimeList* list);
|
||||||
void dir_time_list_free(DirTimeList* list);
|
void dir_time_list_free(DirTimeList* list);
|
||||||
/* Deep-copy one directory's path + metadata into the list. Returns false on
|
/* Deep-copy one directory's path + metadata (and, when non-NULL, its captured
|
||||||
* allocation failure OR when the cumulative entry/byte caps would be exceeded
|
* xattr/ACL block) into the list. Returns false on allocation failure OR when
|
||||||
* (the caller fails the transfer). */
|
* the cumulative entry/byte caps would be exceeded (the caller fails the
|
||||||
bool dir_time_list_add(DirTimeList* list, const char* wire_path, const FileMetadata* metadata);
|
* transfer). */
|
||||||
/* Apply every accumulated directory's mtime (and atime when captured) beneath
|
bool dir_time_list_add(DirTimeList* list, const char* wire_path, const FileMetadata* metadata,
|
||||||
* `root_directory`, confined fd-relative. Best-effort per entry: an absent
|
const FileXattrList* xattrs);
|
||||||
* directory (an empty/pruned source dir that was deliberately not created) or a
|
/* Apply every accumulated directory's metadata beneath `root_directory`,
|
||||||
* non-directory at the path is skipped QUIETLY, an unreachable one with a
|
* confined fd-relative: ownership through the negotiated identity policy,
|
||||||
* warning, and never fatal. */
|
* times (mtime, plus atime when -U captured one under -t), the mode (through
|
||||||
void dir_time_list_apply(const DirTimeList* list, const char* root_directory);
|
* --chmod when configured, under -p), and the captured xattrs/ACLs (under
|
||||||
|
* -X/-A). Best-effort per entry: an absent directory (an empty/pruned source
|
||||||
|
* dir that was deliberately not created) or a non-directory at the path is
|
||||||
|
* skipped QUIETLY, an unreachable one with a warning, and never fatal. */
|
||||||
|
void dir_metadata_list_apply(const DirTimeList* list, const char* root_directory,
|
||||||
|
const Config* config);
|
||||||
|
|
||||||
/* A received delete-manifest frame: the keep-set (`keeps`, destination-relative
|
/* A received delete-manifest frame: the keep-set (`keeps`, destination-relative
|
||||||
paths the sender transferred/keeps) plus `protected`, destination-relative
|
paths the sender transferred/keeps) plus `protected`, destination-relative
|
||||||
@@ -80,11 +88,18 @@ typedef struct DeleteManifest {
|
|||||||
ArrayList* keeps;
|
ArrayList* keeps;
|
||||||
ArrayList* protected;
|
ArrayList* protected;
|
||||||
ArrayList* missing;
|
ArrayList* missing;
|
||||||
|
/* Destination-relative paths of the directories the sender synchronized for
|
||||||
|
this run. The extras walker only removes entries directly inside one of
|
||||||
|
these (the receive root is the "." sentinel); `--files-from` runs therefore
|
||||||
|
leave untransmitted directories and the unlisted parts of listed ones
|
||||||
|
alone, matching rsync's "delete only in synchronized directories". */
|
||||||
|
ArrayList* dirs;
|
||||||
} DeleteManifest;
|
} DeleteManifest;
|
||||||
|
|
||||||
void delete_manifest_free(DeleteManifest* manifest);
|
void delete_manifest_free(DeleteManifest* manifest);
|
||||||
/* Read a delete-manifest frame: keep count + keeps, then protected count +
|
/* Read a delete-manifest frame (protocol 2.23.0): keep count + keeps, then
|
||||||
protected prefixes, then missing count + missing paths (self-delimiting; the
|
protected count + protected prefixes, then missing count + missing paths,
|
||||||
|
then synchronized-directory count + directory paths (self-delimiting; the
|
||||||
leading STATUS_MANIFEST code has been consumed). Returns an owned
|
leading STATUS_MANIFEST code has been consumed). Returns an owned
|
||||||
DeleteManifest, or NULL after signalling STATUS_ERROR on a malformed frame. */
|
DeleteManifest, or NULL after signalling STATUS_ERROR on a malformed frame. */
|
||||||
DeleteManifest* receive_manifest_entries(int fd);
|
DeleteManifest* receive_manifest_entries(int fd);
|
||||||
@@ -103,11 +118,46 @@ bool manifest_delete_extras(const Config* config, DeleteManifest* manifest);
|
|||||||
confinement or I/O error (the run then fails); tolerated per-path cases are
|
confinement or I/O error (the run then fails); tolerated per-path cases are
|
||||||
reported and skipped. */
|
reported and skipped. */
|
||||||
bool manifest_delete_missing_args(const Config* config, DeleteManifest* manifest);
|
bool manifest_delete_missing_args(const Config* config, DeleteManifest* manifest);
|
||||||
|
/* Budgeted form of manifest_delete_missing_args for the per-directory delete
|
||||||
|
session: each removed mirror draws from `max_delete` (SIZE_MAX = unlimited)
|
||||||
|
and the tallies are accumulated into `*deleted`/`*skipped`. `*limit_hit` is set
|
||||||
|
when the budget stopped the pass with entries left over. Returns false only
|
||||||
|
on a genuine deletion error. */
|
||||||
|
bool manifest_delete_missing_args_limited(const Config* config, DeleteManifest* manifest,
|
||||||
|
size_t max_delete, size_t* deleted, size_t* skipped,
|
||||||
|
bool* limit_hit);
|
||||||
|
/* Outcome of committing a delete manifest. LIMIT_REACHED reports rsync's
|
||||||
|
partial --max-delete result: the budget allowed some deletions and the rest
|
||||||
|
were skipped (the run still stores all file data but the client exits 25). */
|
||||||
|
typedef enum {
|
||||||
|
DELETE_COMMIT_OK = 0,
|
||||||
|
DELETE_COMMIT_LIMIT_REACHED,
|
||||||
|
DELETE_COMMIT_ERROR
|
||||||
|
} DeleteCommitResult;
|
||||||
|
|
||||||
/* Run every deletion family the manifest carries: the --delete-missing-args
|
/* Run every deletion family the manifest carries: the --delete-missing-args
|
||||||
exact-path deletions first (user requests are not blocked by exclusion
|
exact-path deletions first (user requests are not blocked by exclusion
|
||||||
protection), then the ordinary extras walk when --delete is active. Returns
|
protection), then the ordinary extras walk when --delete is active. Both
|
||||||
true when nothing to do or everything committed. */
|
share one --max-delete budget. Returns DELETE_COMMIT_OK when nothing was to
|
||||||
bool manifest_delete_all(const Config* config, DeleteManifest* manifest);
|
do or everything committed, DELETE_COMMIT_LIMIT_REACHED when the budget
|
||||||
|
stopped part of the work, or DELETE_COMMIT_ERROR on a genuine failure. */
|
||||||
|
DeleteCommitResult manifest_delete_all(const Config* config, DeleteManifest* manifest);
|
||||||
|
/* Like manifest_delete_all, but reports how many destination entries the commit
|
||||||
|
removed (for the end-of-transfer wire stats). `deleted` may be NULL. */
|
||||||
|
DeleteCommitResult manifest_delete_all_counted(const Config* config, DeleteManifest* manifest,
|
||||||
|
size_t* deleted);
|
||||||
|
|
||||||
|
/* -n/--dry-run --delete would-delete reporting: walk the destination exactly as
|
||||||
|
the delete pass would and append (strdup'd) destination-relative paths that
|
||||||
|
WOULD be removed to `out`, without touching disk. Uses the same staging-dir,
|
||||||
|
basis-dir and protected-prefix skips as the real commit. Returns true on a
|
||||||
|
clean walk; `*count_out` receives the number of paths appended. */
|
||||||
|
bool manifest_would_delete_list(const Config* config, DeleteManifest* manifest, ArrayList* out,
|
||||||
|
size_t* count_out);
|
||||||
|
/* Convert one basis-directory path to the receive-root-relative protection
|
||||||
|
prefix the delete walker uses (NULL when it lies outside the root). Exposed
|
||||||
|
for unit tests of the root-of-"/" and normalization edge cases. */
|
||||||
|
char* file_receive_basis_delete_relative(const Config* config, const char* path);
|
||||||
|
|
||||||
/* Outcome of a single file_save_to_disk operation. The receiver needs to
|
/* Outcome of a single file_save_to_disk operation. The receiver needs to
|
||||||
distinguish "written" from "skipped" so --remove-source-files can be told
|
distinguish "written" from "skipped" so --remove-source-files can be told
|
||||||
|
|||||||
+11
-2
@@ -143,10 +143,17 @@ bool file_send_sendfile_with_skip(File* file, int file_descriptor, bool use_meta
|
|||||||
}
|
}
|
||||||
|
|
||||||
off_t offset = 0;
|
off_t offset = 0;
|
||||||
|
/* A non-positive --timeout disables the deadline: poll blocks until the
|
||||||
|
* socket is writable (rsync's --timeout=0 default). */
|
||||||
|
int io_timeout_sec = protocol_get_io_timeout_sec();
|
||||||
struct timespec deadline;
|
struct timespec deadline;
|
||||||
|
if (io_timeout_sec > 0) {
|
||||||
clock_gettime(CLOCK_MONOTONIC, &deadline);
|
clock_gettime(CLOCK_MONOTONIC, &deadline);
|
||||||
deadline.tv_sec += protocol_get_io_timeout_sec();
|
deadline.tv_sec += io_timeout_sec;
|
||||||
|
}
|
||||||
while ((unsigned long long)offset < file_size) {
|
while ((unsigned long long)offset < file_size) {
|
||||||
|
int timeout = -1;
|
||||||
|
if (io_timeout_sec > 0) {
|
||||||
struct timespec now;
|
struct timespec now;
|
||||||
clock_gettime(CLOCK_MONOTONIC, &now);
|
clock_gettime(CLOCK_MONOTONIC, &now);
|
||||||
long long remaining = (long long)(deadline.tv_sec - now.tv_sec) * 1000LL +
|
long long remaining = (long long)(deadline.tv_sec - now.tv_sec) * 1000LL +
|
||||||
@@ -155,8 +162,9 @@ bool file_send_sendfile_with_skip(File* file, int file_descriptor, bool use_meta
|
|||||||
close(fd);
|
close(fd);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
timeout = remaining > INT_MAX ? INT_MAX : (int)remaining;
|
||||||
|
}
|
||||||
struct pollfd pfd = {.fd = file_descriptor, .events = POLLOUT};
|
struct pollfd pfd = {.fd = file_descriptor, .events = POLLOUT};
|
||||||
int timeout = remaining > INT_MAX ? INT_MAX : (int)remaining;
|
|
||||||
int polled = poll(&pfd, 1, timeout);
|
int polled = poll(&pfd, 1, timeout);
|
||||||
if (polled <= 0 || (pfd.revents & (POLLERR | POLLHUP | POLLNVAL))) {
|
if (polled <= 0 || (pfd.revents & (POLLERR | POLLHUP | POLLNVAL))) {
|
||||||
close(fd);
|
close(fd);
|
||||||
@@ -174,6 +182,7 @@ bool file_send_sendfile_with_skip(File* file, int file_descriptor, bool use_meta
|
|||||||
close(fd);
|
close(fd);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
protocol_note_bytes_written((unsigned long long)sent);
|
||||||
}
|
}
|
||||||
|
|
||||||
close(fd);
|
close(fd);
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
#define FILE_TYPES_H
|
#define FILE_TYPES_H
|
||||||
|
|
||||||
#include "data.h"
|
#include "data.h"
|
||||||
|
#include "format.h"
|
||||||
#include "xattr.h"
|
#include "xattr.h"
|
||||||
#include <stdbool.h>
|
#include <stdbool.h>
|
||||||
#include <sys/stat.h>
|
#include <sys/stat.h>
|
||||||
@@ -86,6 +87,16 @@ typedef struct {
|
|||||||
* Receiver: parsed off the wire, attached here, and applied fd-relative on
|
* Receiver: parsed off the wire, attached here, and applied fd-relative on
|
||||||
* the written file. NULL/0 == the file carries no xattrs. */
|
* the written file. NULL/0 == the file carries no xattrs. */
|
||||||
FileXattrList* xattrs;
|
FileXattrList* xattrs;
|
||||||
|
/* Sender-side output-parity state (never serialized): the receiver-reported
|
||||||
|
* pre-transfer destination snapshot for this entry, filled by the per-file
|
||||||
|
* STATUS_CHECK exchange when report_dest_info is set. `known` is false when
|
||||||
|
* no report was requested/received, in which case -i/--out-format treats the
|
||||||
|
* entry conservatively as newly created. */
|
||||||
|
OutputDestState dest_state;
|
||||||
|
/* Receiver-only wire-stats tally: the number of bytes reconstructed from the
|
||||||
|
* basis file (matched delta blocks) for this entry. 0 when the file was sent
|
||||||
|
* whole. Accumulated into ReceiverStats.matched_data by the receiver sink. */
|
||||||
|
unsigned long long matched_bytes;
|
||||||
} File;
|
} File;
|
||||||
|
|
||||||
/* The path that should be sent on the wire and used for the receiver-side
|
/* The path that should be sent on the wire and used for the receiver-side
|
||||||
|
|||||||
+599
-224
@@ -1,163 +1,27 @@
|
|||||||
#include "filter.h"
|
#include "filter.h"
|
||||||
#include "log.h"
|
#include "log.h"
|
||||||
#include "utils.h"
|
#include "utils.h"
|
||||||
|
#include <ctype.h>
|
||||||
#include <errno.h>
|
#include <errno.h>
|
||||||
#include <limits.h>
|
#include <limits.h>
|
||||||
|
#include <stdarg.h>
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
|
|
||||||
/* ---- Single rule parsing ---- */
|
/* Write a diagnostic message into the caller's optional buffer. A NULL `err`
|
||||||
|
* (or a zero size) is a no-op, so a caller that only needs the boolean status
|
||||||
static bool rule_text_is_unsupported_word(const char* p, size_t len) {
|
* may pass NULL without the snprintf-on-NULL undefined behaviour. */
|
||||||
static const char* const words[] = {"merge", "dir-merge", "hide", "show",
|
static void filter_set_error(char* err, size_t err_size, const char* fmt, ...) {
|
||||||
"protect", "risk", "clear"};
|
if (!err || err_size == 0)
|
||||||
for (size_t i = 0; i < sizeof(words) / sizeof(words[0]); i++) {
|
return;
|
||||||
size_t wl = strlen(words[i]);
|
va_list ap;
|
||||||
if (len == wl && strncmp(p, words[i], wl) == 0)
|
va_start(ap, fmt);
|
||||||
return true;
|
vsnprintf(err, err_size, fmt, ap);
|
||||||
}
|
va_end(ap);
|
||||||
return false;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/* rsync include/exclude rule modifiers we do NOT implement. A rule whose +/- is
|
/* ---- Ordered rule lists ---- */
|
||||||
* immediately followed by one of these is rejected instead of being silently
|
|
||||||
* parsed as a literal pattern. */
|
|
||||||
static bool is_unsupported_rule_modifier(char c) {
|
|
||||||
return c == '!' || c == 'C' || c == 's' || c == 'r' || c == 'p' || c == 'x';
|
|
||||||
}
|
|
||||||
|
|
||||||
FilterRule* filter_rule_parse(const char* line, char* err, size_t err_size) {
|
|
||||||
if (err && err_size > 0)
|
|
||||||
err[0] = '\0';
|
|
||||||
if (!line)
|
|
||||||
return NULL;
|
|
||||||
char* text = str_dup(line);
|
|
||||||
if (!text) {
|
|
||||||
if (err)
|
|
||||||
snprintf(err, err_size, "memory allocation failed");
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
size_t len = strlen(text);
|
|
||||||
while (len > 0 && (text[len - 1] == '\n' || text[len - 1] == '\r'))
|
|
||||||
text[--len] = '\0';
|
|
||||||
|
|
||||||
const char* p = text;
|
|
||||||
while (*p == ' ' || *p == '\t')
|
|
||||||
p++;
|
|
||||||
if (*p == '\0') {
|
|
||||||
snprintf(err, err_size, "empty filter rule");
|
|
||||||
free(text);
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
|
|
||||||
FilterAction action = FILTER_ACTION_EXCLUDE;
|
|
||||||
if (*p == '+' || *p == '-') {
|
|
||||||
action = *p == '+' ? FILTER_ACTION_INCLUDE : FILTER_ACTION_EXCLUDE;
|
|
||||||
p++;
|
|
||||||
/* rsync attaches rule modifiers directly to the +/- (e.g. "-s foo"). Only
|
|
||||||
* the '/' anchor modifier is supported; anything else is a clear error
|
|
||||||
* rather than a silently-ignored literal. */
|
|
||||||
if (*p != ' ' && *p != '\t' && *p != '\0' && is_unsupported_rule_modifier(*p)) {
|
|
||||||
snprintf(err, err_size,
|
|
||||||
"filter rule modifier '%c' is not supported (only the '/' anchor after +/- "
|
|
||||||
"is implemented; put a space between +/- and the pattern)",
|
|
||||||
*p);
|
|
||||||
free(text);
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
while (*p == ' ' || *p == '\t')
|
|
||||||
p++;
|
|
||||||
} else {
|
|
||||||
/* ':' (dir-merge) and '.' (merge) are rsync filter-rule shorthands. At the
|
|
||||||
* start of a rule they mean "merge this file", so reject them instead of
|
|
||||||
* silently turning them into inert exclude patterns. */
|
|
||||||
if (*p == ':' || *p == '.' || *p == '!') {
|
|
||||||
snprintf(err, err_size,
|
|
||||||
"filter rule starting with '%c' is not supported (merge/dir-merge/list-clear "
|
|
||||||
"shorthands are not implemented; use +/- include/exclude rules)",
|
|
||||||
*p);
|
|
||||||
free(text);
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
const char* sp = p;
|
|
||||||
while (*sp != '\0' && *sp != ' ' && *sp != '\t')
|
|
||||||
sp++;
|
|
||||||
size_t word_len = (size_t)(sp - p);
|
|
||||||
if (rule_text_is_unsupported_word(p, word_len)) {
|
|
||||||
snprintf(err, err_size,
|
|
||||||
"'%.*s' filter directives are not supported (only +/- include/exclude rules "
|
|
||||||
"with an optional '/' anchor and trailing '/' dir marker)",
|
|
||||||
(int)word_len, p);
|
|
||||||
free(text);
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
if (word_len == strlen("include") && strncmp(p, "include", word_len) == 0) {
|
|
||||||
action = FILTER_ACTION_INCLUDE;
|
|
||||||
p = sp;
|
|
||||||
} else if (word_len == strlen("exclude") && strncmp(p, "exclude", word_len) == 0) {
|
|
||||||
action = FILTER_ACTION_EXCLUDE;
|
|
||||||
p = sp;
|
|
||||||
}
|
|
||||||
while (*p == ' ' || *p == '\t')
|
|
||||||
p++;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (*p == '\0') {
|
|
||||||
snprintf(err, err_size, "filter rule has no pattern");
|
|
||||||
free(text);
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* A pattern beginning with '/' is anchored (either as "-/foo" or "- /foo"). */
|
|
||||||
bool anchored = false;
|
|
||||||
if (*p == '/') {
|
|
||||||
anchored = true;
|
|
||||||
p++;
|
|
||||||
while (*p == ' ' || *p == '\t')
|
|
||||||
p++;
|
|
||||||
}
|
|
||||||
if (*p == '\0') {
|
|
||||||
snprintf(err, err_size, "filter rule has no pattern after '/' anchor");
|
|
||||||
free(text);
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Pattern runs to the end of the rule; a single trailing '/' marks dir-only. */
|
|
||||||
size_t pat_len = strlen(p);
|
|
||||||
bool dir_only = false;
|
|
||||||
if (pat_len > 1 && p[pat_len - 1] == '/') {
|
|
||||||
dir_only = true;
|
|
||||||
pat_len--;
|
|
||||||
} else if (pat_len == 1 && p[0] == '/') {
|
|
||||||
/* "//" anchored with nothing after: meaningless. */
|
|
||||||
snprintf(err, err_size, "filter rule has no pattern");
|
|
||||||
free(text);
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
|
|
||||||
FilterRule* rule = calloc(1, sizeof(FilterRule));
|
|
||||||
if (!rule) {
|
|
||||||
snprintf(err, err_size, "memory allocation failed");
|
|
||||||
free(text);
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
rule->pattern = malloc(pat_len + 1);
|
|
||||||
if (!rule->pattern) {
|
|
||||||
free(rule);
|
|
||||||
snprintf(err, err_size, "memory allocation failed");
|
|
||||||
free(text);
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
memcpy(rule->pattern, p, pat_len);
|
|
||||||
rule->pattern[pat_len] = '\0';
|
|
||||||
rule->action = action;
|
|
||||||
rule->anchored = anchored;
|
|
||||||
rule->dir_only = dir_only;
|
|
||||||
rule->owner = NULL;
|
|
||||||
free(text);
|
|
||||||
return rule;
|
|
||||||
}
|
|
||||||
|
|
||||||
void filter_rule_free(FilterRule* rule) {
|
void filter_rule_free(FilterRule* rule) {
|
||||||
if (!rule)
|
if (!rule)
|
||||||
@@ -167,8 +31,6 @@ void filter_rule_free(FilterRule* rule) {
|
|||||||
free(rule);
|
free(rule);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* ---- Ordered rule lists ---- */
|
|
||||||
|
|
||||||
FilterRuleList* filter_rule_list_create(void) {
|
FilterRuleList* filter_rule_list_create(void) {
|
||||||
return calloc(1, sizeof(FilterRuleList));
|
return calloc(1, sizeof(FilterRuleList));
|
||||||
}
|
}
|
||||||
@@ -190,28 +52,42 @@ bool filter_rule_list_add(FilterRuleList* list, FilterRule* rule) {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool filter_rule_list_parse_append(FilterRuleList* list, const char* line, char* err,
|
|
||||||
size_t err_size) {
|
|
||||||
FilterRule* rule = filter_rule_parse(line, err, err_size);
|
|
||||||
if (!rule)
|
|
||||||
return false;
|
|
||||||
if (!filter_rule_list_add(list, rule)) {
|
|
||||||
filter_rule_free(rule);
|
|
||||||
snprintf(err, err_size, "memory allocation failed");
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
void filter_rule_list_free(FilterRuleList* list) {
|
void filter_rule_list_free(FilterRuleList* list) {
|
||||||
if (!list)
|
if (!list)
|
||||||
return;
|
return;
|
||||||
for (int i = 0; i < list->count; i++)
|
for (int i = 0; i < list->count; i++)
|
||||||
filter_rule_free(list->items[i]);
|
filter_rule_free(list->items[i]);
|
||||||
|
for (int i = 0; i < list->dir_merge_count; i++)
|
||||||
|
free(list->dir_merge_names[i]);
|
||||||
|
free(list->dir_merge_names);
|
||||||
free(list->items);
|
free(list->items);
|
||||||
free(list);
|
free(list);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Register a per-directory merge-file basename (for "dir-merge NAME"/": NAME"
|
||||||
|
* and -F's .rsync-filter). Duplicate names are ignored. */
|
||||||
|
bool filter_rule_list_add_dir_merge(FilterRuleList* list, const char* name) {
|
||||||
|
if (!list || !name || name[0] == '\0')
|
||||||
|
return false;
|
||||||
|
for (int i = 0; i < list->dir_merge_count; i++) {
|
||||||
|
if (strcmp(list->dir_merge_names[i], name) == 0)
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (list->dir_merge_count == list->dir_merge_capacity) {
|
||||||
|
int new_cap = list->dir_merge_capacity > 0 ? list->dir_merge_capacity * 2 : 4;
|
||||||
|
char** grown = realloc(list->dir_merge_names, (size_t)new_cap * sizeof(char*));
|
||||||
|
if (!grown)
|
||||||
|
return false;
|
||||||
|
list->dir_merge_names = grown;
|
||||||
|
list->dir_merge_capacity = new_cap;
|
||||||
|
}
|
||||||
|
char* dup = str_dup(name);
|
||||||
|
if (!dup)
|
||||||
|
return false;
|
||||||
|
list->dir_merge_names[list->dir_merge_count++] = dup;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
static bool set_rule_owner(FilterRule* rule, const char* owner) {
|
static bool set_rule_owner(FilterRule* rule, const char* owner) {
|
||||||
char* dup = str_dup(owner ? owner : "");
|
char* dup = str_dup(owner ? owner : "");
|
||||||
if (!dup)
|
if (!dup)
|
||||||
@@ -221,7 +97,315 @@ static bool set_rule_owner(FilterRule* rule, const char* owner) {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* ---- CVS default excludes (-C) ---- */
|
/* ---- Rule parsing ---- */
|
||||||
|
|
||||||
|
/* A short rule prefix is a single character; a long rule name is alphabetic
|
||||||
|
* (with '-'). `is_short` distinguishes the modifier-attachment rules. */
|
||||||
|
typedef enum {
|
||||||
|
RULE_KIND_EXCLUDE,
|
||||||
|
RULE_KIND_INCLUDE,
|
||||||
|
RULE_KIND_HIDE,
|
||||||
|
RULE_KIND_SHOW,
|
||||||
|
RULE_KIND_PROTECT,
|
||||||
|
RULE_KIND_RISK,
|
||||||
|
RULE_KIND_MERGE,
|
||||||
|
RULE_KIND_DIR_MERGE,
|
||||||
|
RULE_KIND_CLEAR,
|
||||||
|
RULE_KIND_UNKNOWN,
|
||||||
|
} RuleKind;
|
||||||
|
|
||||||
|
static bool short_rule_char(char c, RuleKind* kind) {
|
||||||
|
switch (c) {
|
||||||
|
case '-':
|
||||||
|
*kind = RULE_KIND_EXCLUDE;
|
||||||
|
return true;
|
||||||
|
case '+':
|
||||||
|
*kind = RULE_KIND_INCLUDE;
|
||||||
|
return true;
|
||||||
|
case 'H':
|
||||||
|
*kind = RULE_KIND_HIDE;
|
||||||
|
return true;
|
||||||
|
case 'S':
|
||||||
|
*kind = RULE_KIND_SHOW;
|
||||||
|
return true;
|
||||||
|
case 'P':
|
||||||
|
*kind = RULE_KIND_PROTECT;
|
||||||
|
return true;
|
||||||
|
case 'R':
|
||||||
|
*kind = RULE_KIND_RISK;
|
||||||
|
return true;
|
||||||
|
case '.':
|
||||||
|
*kind = RULE_KIND_MERGE;
|
||||||
|
return true;
|
||||||
|
case ':':
|
||||||
|
*kind = RULE_KIND_DIR_MERGE;
|
||||||
|
return true;
|
||||||
|
case '!':
|
||||||
|
*kind = RULE_KIND_CLEAR;
|
||||||
|
return true;
|
||||||
|
default:
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool long_rule_name(const char* name, size_t len, RuleKind* kind) {
|
||||||
|
struct {
|
||||||
|
const char* word;
|
||||||
|
RuleKind kind;
|
||||||
|
} table[] = {
|
||||||
|
{"exclude", RULE_KIND_EXCLUDE}, {"include", RULE_KIND_INCLUDE},
|
||||||
|
{"hide", RULE_KIND_HIDE}, {"show", RULE_KIND_SHOW},
|
||||||
|
{"protect", RULE_KIND_PROTECT}, {"risk", RULE_KIND_RISK},
|
||||||
|
{"merge", RULE_KIND_MERGE}, {"dir-merge", RULE_KIND_DIR_MERGE},
|
||||||
|
{"clear", RULE_KIND_CLEAR},
|
||||||
|
};
|
||||||
|
for (size_t i = 0; i < sizeof(table) / sizeof(table[0]); i++) {
|
||||||
|
if (strlen(table[i].word) == len && strncmp(name, table[i].word, len) == 0) {
|
||||||
|
*kind = table[i].kind;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool is_modifier_char(char c) {
|
||||||
|
return c == 's' || c == 'r' || c == 'p' || c == 'x' || c == '/' || c == '!' || c == 'C';
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Parse "RULE[,MODIFIERS] [PATTERN]". On success `kind`, `sides`,
|
||||||
|
* `sides_explicit`, `negate`, `anchored_mod`, `perishable`, `xattr`,
|
||||||
|
* `cvs_inject` and the pattern span (`pat_start`/`pat_len`, possibly 0 for
|
||||||
|
* merge/clear) are filled. Returns true on success. */
|
||||||
|
static bool parse_rule_syntax(const char* text, RuleKind* kind, unsigned* sides,
|
||||||
|
bool* sides_explicit, bool* negate, bool* anchored_mod,
|
||||||
|
bool* perishable, bool* xattr, bool* cvs_inject,
|
||||||
|
const char** pat_start, size_t* pat_len) {
|
||||||
|
const char* p = text;
|
||||||
|
*sides = FILTER_SIDE_SENDER | FILTER_SIDE_RECEIVER;
|
||||||
|
*sides_explicit = false;
|
||||||
|
*negate = false;
|
||||||
|
*anchored_mod = false;
|
||||||
|
*perishable = false;
|
||||||
|
*xattr = false;
|
||||||
|
*cvs_inject = false;
|
||||||
|
*pat_start = NULL;
|
||||||
|
*pat_len = 0;
|
||||||
|
|
||||||
|
bool is_short = false;
|
||||||
|
if (short_rule_char(*p, kind)) {
|
||||||
|
is_short = true;
|
||||||
|
p++;
|
||||||
|
} else {
|
||||||
|
const char* name_start = p;
|
||||||
|
while (isalpha((unsigned char)*p) || *p == '-')
|
||||||
|
p++;
|
||||||
|
size_t name_len = (size_t)(p - name_start);
|
||||||
|
if (name_len == 0 || !long_rule_name(name_start, name_len, kind))
|
||||||
|
return false;
|
||||||
|
/* A long name must be followed by a separator, a comma or the end. */
|
||||||
|
if (*p != '\0' && *p != ',' && *p != ' ' && *p != '_')
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Modifiers: long names require a comma; short names may attach directly.
|
||||||
|
Only commit a modifier run that terminates at a separator or the end, so a
|
||||||
|
pattern such as "*.tmp" written as "-*.tmp" is not mistaken for modifiers. */
|
||||||
|
const char* mod_start = p;
|
||||||
|
const char* mod_end = p;
|
||||||
|
if (*p == ',') {
|
||||||
|
p++;
|
||||||
|
mod_start = p;
|
||||||
|
while (is_modifier_char(*p))
|
||||||
|
p++;
|
||||||
|
mod_end = p;
|
||||||
|
} else if (is_short) {
|
||||||
|
const char* scan = p;
|
||||||
|
while (is_modifier_char(*scan))
|
||||||
|
scan++;
|
||||||
|
if (*scan == '\0' || *scan == ' ' || *scan == '_') {
|
||||||
|
mod_start = p;
|
||||||
|
mod_end = scan;
|
||||||
|
p = scan;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (const char* m = mod_start; m < mod_end; m++) {
|
||||||
|
switch (*m) {
|
||||||
|
case 's':
|
||||||
|
*sides = FILTER_SIDE_SENDER;
|
||||||
|
*sides_explicit = true;
|
||||||
|
break;
|
||||||
|
case 'r':
|
||||||
|
*sides = FILTER_SIDE_RECEIVER;
|
||||||
|
*sides_explicit = true;
|
||||||
|
break;
|
||||||
|
case '!':
|
||||||
|
*negate = true;
|
||||||
|
break;
|
||||||
|
case '/':
|
||||||
|
*anchored_mod = true;
|
||||||
|
break;
|
||||||
|
case 'p':
|
||||||
|
*perishable = true;
|
||||||
|
break;
|
||||||
|
case 'x':
|
||||||
|
*xattr = true;
|
||||||
|
break;
|
||||||
|
case 'C':
|
||||||
|
*cvs_inject = true;
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* A single space or underscore separates the rule/modifiers from the
|
||||||
|
pattern; further spaces/underscores belong to the pattern. */
|
||||||
|
const char* pat = p;
|
||||||
|
if (*pat == ' ' || *pat == '_')
|
||||||
|
pat++;
|
||||||
|
/* Trim a trailing newline/CR (the caller may pass a raw file line). */
|
||||||
|
*pat_start = pat;
|
||||||
|
*pat_len = strlen(pat);
|
||||||
|
while (*pat_len > 0 && (pat[*pat_len - 1] == '\n' || pat[*pat_len - 1] == '\r'))
|
||||||
|
(*pat_len)--;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
FilterRule* filter_rule_parse(const char* line, const FilterParseOptions* opts, char* err,
|
||||||
|
size_t err_size) {
|
||||||
|
if (err && err_size > 0)
|
||||||
|
err[0] = '\0';
|
||||||
|
if (!line)
|
||||||
|
return NULL;
|
||||||
|
const char* p = line;
|
||||||
|
while (*p == ' ' || *p == '\t')
|
||||||
|
p++;
|
||||||
|
if (*p == '\0' || *p == '\n' || *p == '\r') {
|
||||||
|
filter_set_error(err, err_size, "empty filter rule");
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
RuleKind kind = RULE_KIND_UNKNOWN;
|
||||||
|
unsigned sides;
|
||||||
|
bool sides_explicit, negate, anchored_mod, perishable, xattr, cvs_inject;
|
||||||
|
const char* pat;
|
||||||
|
size_t pat_len;
|
||||||
|
if (!parse_rule_syntax(p, &kind, &sides, &sides_explicit, &negate, &anchored_mod, &perishable,
|
||||||
|
&xattr, &cvs_inject, &pat, &pat_len)) {
|
||||||
|
filter_set_error(err, err_size, "unrecognized filter rule syntax");
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
if (cvs_inject) {
|
||||||
|
/* The C modifier expands to the CVS defaults in place; the rule itself
|
||||||
|
carries no pattern and is handled by the caller. */
|
||||||
|
filter_set_error(err, err_size, "the C modifier is handled by the rule-list parser");
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
if (xattr) {
|
||||||
|
filter_set_error(err, err_size, "xattr-name filter rules (the x modifier) are not supported");
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
if (kind == RULE_KIND_MERGE || kind == RULE_KIND_DIR_MERGE) {
|
||||||
|
filter_set_error(err, err_size, "merge/dir-merge rules are handled by the rule-list parser");
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
if (kind == RULE_KIND_CLEAR) {
|
||||||
|
if (pat_len != 0) {
|
||||||
|
filter_set_error(err, err_size, "clear takes no pattern");
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
FilterRule* rule = calloc(1, sizeof(FilterRule));
|
||||||
|
if (!rule) {
|
||||||
|
filter_set_error(err, err_size, "memory allocation failed");
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
rule->action = FILTER_ACTION_NONE; /* clear marker: no pattern */
|
||||||
|
rule->sides = 0;
|
||||||
|
return rule;
|
||||||
|
}
|
||||||
|
|
||||||
|
FilterAction action;
|
||||||
|
switch (kind) {
|
||||||
|
case RULE_KIND_INCLUDE:
|
||||||
|
case RULE_KIND_SHOW:
|
||||||
|
case RULE_KIND_RISK:
|
||||||
|
action = FILTER_ACTION_INCLUDE;
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
action = FILTER_ACTION_EXCLUDE;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if (kind == RULE_KIND_HIDE)
|
||||||
|
sides = FILTER_SIDE_SENDER;
|
||||||
|
else if (kind == RULE_KIND_SHOW)
|
||||||
|
sides = FILTER_SIDE_SENDER;
|
||||||
|
else if (kind == RULE_KIND_PROTECT)
|
||||||
|
sides = FILTER_SIDE_RECEIVER;
|
||||||
|
else if (kind == RULE_KIND_RISK)
|
||||||
|
sides = FILTER_SIDE_RECEIVER;
|
||||||
|
if (kind == RULE_KIND_HIDE || kind == RULE_KIND_SHOW || kind == RULE_KIND_PROTECT ||
|
||||||
|
kind == RULE_KIND_RISK)
|
||||||
|
sides_explicit = true;
|
||||||
|
/* --delete-excluded turns an unqualified (no explicit s/r) rule into a
|
||||||
|
sender-side-only rule, so it no longer protects the receiver. */
|
||||||
|
if (opts && opts->delete_excluded && !sides_explicit)
|
||||||
|
sides = FILTER_SIDE_SENDER;
|
||||||
|
|
||||||
|
if (pat_len == 0) {
|
||||||
|
filter_set_error(err, err_size, "filter rule has no pattern");
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool anchored = anchored_mod;
|
||||||
|
const char* pat_begin = pat;
|
||||||
|
if (*pat_begin == '/') {
|
||||||
|
anchored = true;
|
||||||
|
pat_begin++;
|
||||||
|
/* Drop the spaces that could follow the anchor in the "-/ foo" form. */
|
||||||
|
while (*pat_begin == ' ' || *pat_begin == '\t')
|
||||||
|
pat_begin++;
|
||||||
|
pat_len = strlen(pat_begin);
|
||||||
|
while (pat_len > 0 && (pat_begin[pat_len - 1] == '\n' || pat_begin[pat_len - 1] == '\r'))
|
||||||
|
pat_len--;
|
||||||
|
}
|
||||||
|
if (pat_len == 0) {
|
||||||
|
filter_set_error(err, err_size, "filter rule has no pattern after '/' anchor");
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
bool dir_only = false;
|
||||||
|
if (pat_len > 1 && pat_begin[pat_len - 1] == '/') {
|
||||||
|
dir_only = true;
|
||||||
|
pat_len--;
|
||||||
|
}
|
||||||
|
if (pat_len == 0) {
|
||||||
|
filter_set_error(err, err_size, "filter rule has no pattern");
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
FilterRule* rule = calloc(1, sizeof(FilterRule));
|
||||||
|
if (!rule) {
|
||||||
|
filter_set_error(err, err_size, "memory allocation failed");
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
rule->pattern = malloc(pat_len + 1);
|
||||||
|
if (!rule->pattern) {
|
||||||
|
free(rule);
|
||||||
|
filter_set_error(err, err_size, "memory allocation failed");
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
memcpy(rule->pattern, pat_begin, pat_len);
|
||||||
|
rule->pattern[pat_len] = '\0';
|
||||||
|
rule->action = action;
|
||||||
|
rule->sides = sides;
|
||||||
|
rule->anchored = anchored;
|
||||||
|
rule->dir_only = dir_only;
|
||||||
|
rule->negate = negate;
|
||||||
|
rule->perishable = perishable;
|
||||||
|
(void)xattr; /* xattr-name rules never match file/dir names; accepted/ignored */
|
||||||
|
return rule;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ---- CVS default excludes (-C and the C modifier) ---- */
|
||||||
|
|
||||||
typedef struct {
|
typedef struct {
|
||||||
const char* pattern;
|
const char* pattern;
|
||||||
@@ -240,12 +424,13 @@ static const CvsDefaultRule CVS_DEFAULTS[] = {
|
|||||||
{".svn/", true}, {".git/", true}, {".hg/", true}, {".bzr/", true},
|
{".svn/", true}, {".git/", true}, {".hg/", true}, {".bzr/", true},
|
||||||
};
|
};
|
||||||
|
|
||||||
static bool cvs_rule_list_append(FilterRuleList* list) {
|
static bool filter_list_append_cvs(FilterRuleList* list, unsigned sides) {
|
||||||
for (size_t i = 0; i < sizeof(CVS_DEFAULTS) / sizeof(CVS_DEFAULTS[0]); i++) {
|
for (size_t i = 0; i < sizeof(CVS_DEFAULTS) / sizeof(CVS_DEFAULTS[0]); i++) {
|
||||||
FilterRule* rule = calloc(1, sizeof(FilterRule));
|
FilterRule* rule = calloc(1, sizeof(FilterRule));
|
||||||
if (!rule)
|
if (!rule)
|
||||||
return false;
|
return false;
|
||||||
rule->action = FILTER_ACTION_EXCLUDE;
|
rule->action = FILTER_ACTION_EXCLUDE;
|
||||||
|
rule->sides = sides;
|
||||||
rule->dir_only = CVS_DEFAULTS[i].dir_only;
|
rule->dir_only = CVS_DEFAULTS[i].dir_only;
|
||||||
size_t plen = strlen(CVS_DEFAULTS[i].pattern);
|
size_t plen = strlen(CVS_DEFAULTS[i].pattern);
|
||||||
if (rule->dir_only && plen > 0 && CVS_DEFAULTS[i].pattern[plen - 1] == '/')
|
if (rule->dir_only && plen > 0 && CVS_DEFAULTS[i].pattern[plen - 1] == '/')
|
||||||
@@ -269,76 +454,236 @@ static bool cvs_rule_list_append(FilterRuleList* list) {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
FilterRuleList* filter_base_build(const char* const* rule_texts, int rule_count, bool cvs_exclude,
|
#define FILTER_MAX_MERGE_DEPTH 16
|
||||||
|
|
||||||
|
static bool filter_list_parse_append_depth(FilterRuleList* list, const char* line,
|
||||||
|
const FilterParseOptions* opts, const char* base_dir,
|
||||||
|
int depth, char* err, size_t err_size);
|
||||||
|
|
||||||
|
/* Read a merge file and splice its rules into `list`. A relative path is
|
||||||
|
* resolved below `base_dir` when given, else used as-is (rsync resolves a
|
||||||
|
* command-line merge file relative to the current directory). */
|
||||||
|
static bool filter_list_merge_file(FilterRuleList* list, const char* name,
|
||||||
|
const FilterParseOptions* opts, const char* base_dir, int depth,
|
||||||
char* err, size_t err_size) {
|
char* err, size_t err_size) {
|
||||||
|
if (name[0] == '\0') {
|
||||||
|
filter_set_error(err, err_size, "merge requires a filename");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
char* path =
|
||||||
|
(base_dir && base_dir[0] && name[0] != '/') ? path_cat(base_dir, name) : str_dup(name);
|
||||||
|
if (!path) {
|
||||||
|
filter_set_error(err, err_size, "memory allocation failed");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
FILE* fp = fopen(path, "r");
|
||||||
|
if (!fp) {
|
||||||
|
filter_set_error(err, err_size, "could not read merge file '%s': %s", path, strerror(errno));
|
||||||
|
free(path);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
char* line = NULL;
|
||||||
|
size_t cap = 0;
|
||||||
|
bool ok = true;
|
||||||
|
while (true) {
|
||||||
|
ssize_t n = utils_getdelim_bounded(fp, &line, &cap, '\n', UTILS_MAX_LINE_LEN);
|
||||||
|
if (n < 0) {
|
||||||
|
filter_set_error(err, err_size, "error reading merge file '%s'", path);
|
||||||
|
ok = false;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if (n == 0)
|
||||||
|
break;
|
||||||
|
const char* lp = line;
|
||||||
|
while (*lp == ' ' || *lp == '\t')
|
||||||
|
lp++;
|
||||||
|
if (*lp == '\0' || *lp == '\n' || *lp == '\r' || *lp == '#')
|
||||||
|
continue;
|
||||||
|
if (!filter_list_parse_append_depth(list, lp, opts, base_dir, depth + 1, err, err_size)) {
|
||||||
|
ok = false;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
free(line);
|
||||||
|
fclose(fp);
|
||||||
|
free(path);
|
||||||
|
return ok;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Parse one line and append/merge it into `list`. Handles clear, merge and
|
||||||
|
* dir-merge at the list level. */
|
||||||
|
static bool filter_list_parse_append_depth(FilterRuleList* list, const char* line,
|
||||||
|
const FilterParseOptions* opts, const char* base_dir,
|
||||||
|
int depth, char* err, size_t err_size) {
|
||||||
|
if (depth > FILTER_MAX_MERGE_DEPTH) {
|
||||||
|
filter_set_error(err, err_size, "merge files nested too deeply");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
const char* p = line;
|
||||||
|
while (*p == ' ' || *p == '\t')
|
||||||
|
p++;
|
||||||
|
if (*p == '\0' || *p == '\n' || *p == '\r')
|
||||||
|
return true;
|
||||||
|
|
||||||
|
RuleKind kind = RULE_KIND_UNKNOWN;
|
||||||
|
unsigned sides;
|
||||||
|
bool sides_explicit, negate, anchored_mod, perishable, xattr, cvs_inject;
|
||||||
|
const char* pat;
|
||||||
|
size_t pat_len;
|
||||||
|
if (!parse_rule_syntax(p, &kind, &sides, &sides_explicit, &negate, &anchored_mod, &perishable,
|
||||||
|
&xattr, &cvs_inject, &pat, &pat_len)) {
|
||||||
|
filter_set_error(err, err_size, "unrecognized filter rule syntax: %s", p);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
(void)sides_explicit;
|
||||||
|
(void)negate;
|
||||||
|
(void)anchored_mod;
|
||||||
|
(void)perishable;
|
||||||
|
(void)xattr;
|
||||||
|
|
||||||
|
if (cvs_inject) {
|
||||||
|
/* "C" injects the CVS defaults in place; no pattern is expected. */
|
||||||
|
return filter_list_append_cvs(list, sides);
|
||||||
|
}
|
||||||
|
if (kind == RULE_KIND_CLEAR) {
|
||||||
|
if (pat_len != 0) {
|
||||||
|
filter_set_error(err, err_size, "clear takes no pattern");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
for (int i = 0; i < list->count; i++)
|
||||||
|
filter_rule_free(list->items[i]);
|
||||||
|
list->count = 0;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (kind == RULE_KIND_MERGE) {
|
||||||
|
if (pat_len == 0) {
|
||||||
|
filter_set_error(err, err_size, "merge requires a filename");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
char* name = malloc(pat_len + 1);
|
||||||
|
if (!name) {
|
||||||
|
filter_set_error(err, err_size, "memory allocation failed");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
memcpy(name, pat, pat_len);
|
||||||
|
name[pat_len] = '\0';
|
||||||
|
bool ok = filter_list_merge_file(list, name, opts, base_dir, depth, err, err_size);
|
||||||
|
free(name);
|
||||||
|
return ok;
|
||||||
|
}
|
||||||
|
if (kind == RULE_KIND_DIR_MERGE) {
|
||||||
|
if (pat_len == 0) {
|
||||||
|
filter_set_error(err, err_size, "dir-merge requires a filename");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
char* name = malloc(pat_len + 1);
|
||||||
|
if (!name) {
|
||||||
|
filter_set_error(err, err_size, "memory allocation failed");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
memcpy(name, pat, pat_len);
|
||||||
|
name[pat_len] = '\0';
|
||||||
|
bool ok = filter_rule_list_add_dir_merge(list, name);
|
||||||
|
free(name);
|
||||||
|
if (!ok) {
|
||||||
|
filter_set_error(err, err_size, "memory allocation failed");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
FilterRule* rule = filter_rule_parse(p, opts, err, err_size);
|
||||||
|
if (!rule)
|
||||||
|
return false;
|
||||||
|
if (!filter_rule_list_add(list, rule)) {
|
||||||
|
filter_rule_free(rule);
|
||||||
|
filter_set_error(err, err_size, "memory allocation failed");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool filter_rule_list_parse_append(FilterRuleList* list, const char* line,
|
||||||
|
const FilterParseOptions* opts, const char* merge_base_dir,
|
||||||
|
char* err, size_t err_size) {
|
||||||
|
if (err && err_size > 0)
|
||||||
|
err[0] = '\0';
|
||||||
|
if (!list)
|
||||||
|
return false;
|
||||||
|
return filter_list_parse_append_depth(list, line, opts, merge_base_dir, 0, err, err_size);
|
||||||
|
}
|
||||||
|
|
||||||
|
FilterRuleList* filter_base_build(const char* const* rule_texts, int rule_count, bool cvs_exclude,
|
||||||
|
bool delete_excluded, char* err, size_t err_size) {
|
||||||
if (err && err_size > 0)
|
if (err && err_size > 0)
|
||||||
err[0] = '\0';
|
err[0] = '\0';
|
||||||
FilterRuleList* list = filter_rule_list_create();
|
FilterRuleList* list = filter_rule_list_create();
|
||||||
if (!list) {
|
if (!list) {
|
||||||
snprintf(err, err_size, "memory allocation failed");
|
filter_set_error(err, err_size, "memory allocation failed");
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
FilterParseOptions opts = {.delete_excluded = delete_excluded, .cvs_exclude = cvs_exclude};
|
||||||
for (int i = 0; i < rule_count; i++) {
|
for (int i = 0; i < rule_count; i++) {
|
||||||
if (!rule_texts || !rule_texts[i])
|
if (!rule_texts || !rule_texts[i])
|
||||||
continue;
|
continue;
|
||||||
FilterRule* rule = filter_rule_parse(rule_texts[i], err, err_size);
|
if (!filter_rule_list_parse_append(list, rule_texts[i], &opts, NULL, err, err_size)) {
|
||||||
if (!rule) {
|
|
||||||
filter_rule_list_free(list);
|
filter_rule_list_free(list);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
if (!set_rule_owner(rule, "")) {
|
|
||||||
filter_rule_free(rule);
|
|
||||||
filter_rule_list_free(list);
|
|
||||||
snprintf(err, err_size, "memory allocation failed");
|
|
||||||
return NULL;
|
|
||||||
}
|
}
|
||||||
if (!filter_rule_list_add(list, rule)) {
|
if (cvs_exclude && !filter_list_append_cvs(list, FILTER_SIDE_SENDER | FILTER_SIDE_RECEIVER)) {
|
||||||
filter_rule_free(rule);
|
|
||||||
filter_rule_list_free(list);
|
filter_rule_list_free(list);
|
||||||
snprintf(err, err_size, "memory allocation failed");
|
filter_set_error(err, err_size, "memory allocation failed");
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (cvs_exclude && !cvs_rule_list_append(list)) {
|
|
||||||
filter_rule_list_free(list);
|
|
||||||
snprintf(err, err_size, "memory allocation failed");
|
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
return list;
|
return list;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* ---- Per-directory .rsync-filter files ---- */
|
/* ---- Per-directory merge files ---- */
|
||||||
|
|
||||||
FilterRuleList* filter_file_read(const char* dir_path, const char* owner_rel, bool* exists,
|
/* Undo the rules and dir-merge registrations that one merge file appended,
|
||||||
|
* leaving the caller's earlier content intact. A "clear" rule inside the file
|
||||||
|
* frees every rule, including the caller's; clamp to the surviving count so
|
||||||
|
* those already-freed rules are never resurrected and freed a second time. */
|
||||||
|
static void filter_file_rollback(FilterRuleList* list, int rules_before, int dir_merges_before) {
|
||||||
|
int first = rules_before < list->count ? rules_before : list->count;
|
||||||
|
for (int i = first; i < list->count; i++)
|
||||||
|
filter_rule_free(list->items[i]);
|
||||||
|
list->count = first;
|
||||||
|
for (int i = dir_merges_before; i < list->dir_merge_count; i++)
|
||||||
|
free(list->dir_merge_names[i]);
|
||||||
|
list->dir_merge_count = dir_merges_before;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool filter_file_append(FilterRuleList* list, const char* dir_path, const char* name,
|
||||||
|
const char* owner_rel, const FilterParseOptions* opts, bool* exists,
|
||||||
char* err, size_t err_size) {
|
char* err, size_t err_size) {
|
||||||
if (err && err_size > 0)
|
if (err && err_size > 0)
|
||||||
err[0] = '\0';
|
err[0] = '\0';
|
||||||
if (exists)
|
if (exists)
|
||||||
*exists = false;
|
*exists = false;
|
||||||
char* filter_path = path_cat(dir_path, ".rsync-filter");
|
if (!list)
|
||||||
|
return false;
|
||||||
|
char* filter_path = path_cat(dir_path, name);
|
||||||
if (!filter_path) {
|
if (!filter_path) {
|
||||||
snprintf(err, err_size, "memory allocation failed");
|
filter_set_error(err, err_size, "memory allocation failed");
|
||||||
return NULL;
|
return false;
|
||||||
}
|
}
|
||||||
FILE* fp = fopen(filter_path, "r");
|
FILE* fp = fopen(filter_path, "r");
|
||||||
free(filter_path);
|
free(filter_path);
|
||||||
if (!fp) {
|
if (!fp) {
|
||||||
if (errno == ENOENT || errno == ENOTDIR)
|
if (errno == ENOENT || errno == ENOTDIR)
|
||||||
return filter_rule_list_create();
|
return true;
|
||||||
char* escaped_dir = output_escape(dir_path, log_get_8_bit_output());
|
char* escaped_dir = output_escape(dir_path, log_get_8_bit_output());
|
||||||
log_message(LOG_LEVEL_WARNING, "Could not read .rsync-filter in %s: %s",
|
log_message(LOG_LEVEL_WARNING, "Could not read %s in %s: %s", name,
|
||||||
escaped_dir ? escaped_dir : "<allocation failed>", strerror(errno));
|
escaped_dir ? escaped_dir : "<allocation failed>", strerror(errno));
|
||||||
free(escaped_dir);
|
free(escaped_dir);
|
||||||
return filter_rule_list_create();
|
return true;
|
||||||
}
|
}
|
||||||
if (exists)
|
if (exists)
|
||||||
*exists = true;
|
*exists = true;
|
||||||
FilterRuleList* list = filter_rule_list_create();
|
int rules_before = list->count;
|
||||||
if (!list) {
|
int dir_merges_before = list->dir_merge_count;
|
||||||
fclose(fp);
|
|
||||||
snprintf(err, err_size, "memory allocation failed");
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
char* line = NULL;
|
char* line = NULL;
|
||||||
size_t line_cap = 0;
|
size_t line_cap = 0;
|
||||||
bool ok = true;
|
bool ok = true;
|
||||||
@@ -346,9 +691,10 @@ FilterRuleList* filter_file_read(const char* dir_path, const char* owner_rel, bo
|
|||||||
ssize_t n = utils_getdelim_bounded(fp, &line, &line_cap, '\n', UTILS_MAX_LINE_LEN);
|
ssize_t n = utils_getdelim_bounded(fp, &line, &line_cap, '\n', UTILS_MAX_LINE_LEN);
|
||||||
if (n < 0) {
|
if (n < 0) {
|
||||||
if (errno == EFBIG) {
|
if (errno == EFBIG) {
|
||||||
snprintf(err, err_size, "line in .rsync-filter exceeds %d bytes", (int)UTILS_MAX_LINE_LEN);
|
filter_set_error(err, err_size, "line in %s exceeds %d bytes", name,
|
||||||
|
(int)UTILS_MAX_LINE_LEN);
|
||||||
} else {
|
} else {
|
||||||
snprintf(err, err_size, "error reading .rsync-filter: %s", strerror(errno));
|
filter_set_error(err, err_size, "error reading %s: %s", name, strerror(errno));
|
||||||
}
|
}
|
||||||
ok = false;
|
ok = false;
|
||||||
break;
|
break;
|
||||||
@@ -360,20 +706,9 @@ FilterRuleList* filter_file_read(const char* dir_path, const char* owner_rel, bo
|
|||||||
p++;
|
p++;
|
||||||
if (*p == '\0' || *p == '\n' || *p == '\r' || *p == '#')
|
if (*p == '\0' || *p == '\n' || *p == '\r' || *p == '#')
|
||||||
continue;
|
continue;
|
||||||
FilterRule* rule = filter_rule_parse(p, err, err_size);
|
/* Merge files inside a per-directory file resolve relative to that
|
||||||
if (!rule) {
|
directory. */
|
||||||
ok = false;
|
if (!filter_list_parse_append_depth(list, p, opts, dir_path, 0, err, err_size)) {
|
||||||
break;
|
|
||||||
}
|
|
||||||
if (!set_rule_owner(rule, owner_rel)) {
|
|
||||||
filter_rule_free(rule);
|
|
||||||
snprintf(err, err_size, "memory allocation failed");
|
|
||||||
ok = false;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
if (!filter_rule_list_add(list, rule)) {
|
|
||||||
filter_rule_free(rule);
|
|
||||||
snprintf(err, err_size, "memory allocation failed");
|
|
||||||
ok = false;
|
ok = false;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
@@ -381,12 +716,40 @@ FilterRuleList* filter_file_read(const char* dir_path, const char* owner_rel, bo
|
|||||||
free(line);
|
free(line);
|
||||||
fclose(fp);
|
fclose(fp);
|
||||||
if (!ok) {
|
if (!ok) {
|
||||||
|
filter_file_rollback(list, rules_before, dir_merges_before);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
for (int i = rules_before; i < list->count; i++) {
|
||||||
|
if (!set_rule_owner(list->items[i], owner_rel)) {
|
||||||
|
filter_set_error(err, err_size, "memory allocation failed");
|
||||||
|
filter_file_rollback(list, rules_before, dir_merges_before);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
FilterRuleList* filter_file_read_named(const char* dir_path, const char* name,
|
||||||
|
const char* owner_rel, const FilterParseOptions* opts,
|
||||||
|
bool* exists, char* err, size_t err_size) {
|
||||||
|
FilterRuleList* list = filter_rule_list_create();
|
||||||
|
if (!list) {
|
||||||
|
if (err && err_size > 0)
|
||||||
|
filter_set_error(err, err_size, "memory allocation failed");
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
if (!filter_file_append(list, dir_path, name, owner_rel, opts, exists, err, err_size)) {
|
||||||
filter_rule_list_free(list);
|
filter_rule_list_free(list);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
return list;
|
return list;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
FilterRuleList* filter_file_read(const char* dir_path, const char* owner_rel, bool* exists,
|
||||||
|
char* err, size_t err_size) {
|
||||||
|
return filter_file_read_named(dir_path, ".rsync-filter", owner_rel, NULL, exists, err, err_size);
|
||||||
|
}
|
||||||
|
|
||||||
/* ---- Rule matching ---- */
|
/* ---- Rule matching ---- */
|
||||||
|
|
||||||
/* Match a pattern that contains '/' (non-anchored) against the end of the
|
/* Match a pattern that contains '/' (non-anchored) against the end of the
|
||||||
@@ -402,10 +765,10 @@ static bool glob_suffix_match(const char* pattern, const char* str) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
static FilterAction rule_matches(const FilterRule* rule, const char* rel_path, const char* leaf,
|
static FilterAction rule_matches(const FilterRule* rule, const char* rel_path, const char* leaf,
|
||||||
bool is_dir) {
|
bool is_dir, unsigned side) {
|
||||||
if (!rule || !rule->pattern)
|
if (!rule || !rule->pattern)
|
||||||
return FILTER_ACTION_NONE;
|
return FILTER_ACTION_NONE;
|
||||||
if (rule->dir_only && !is_dir)
|
if (!(rule->sides & side))
|
||||||
return FILTER_ACTION_NONE;
|
return FILTER_ACTION_NONE;
|
||||||
/* A rule applies only to entries below its owner directory. */
|
/* A rule applies only to entries below its owner directory. */
|
||||||
const char* rel2 = rel_path;
|
const char* rel2 = rel_path;
|
||||||
@@ -420,24 +783,36 @@ static FilterAction rule_matches(const FilterRule* rule, const char* rel_path, c
|
|||||||
if (rel2[0] == '\0')
|
if (rel2[0] == '\0')
|
||||||
return FILTER_ACTION_NONE;
|
return FILTER_ACTION_NONE;
|
||||||
bool matched;
|
bool matched;
|
||||||
if (rule->anchored) {
|
if (rule->dir_only && !is_dir)
|
||||||
|
matched = false;
|
||||||
|
else if (rule->anchored)
|
||||||
matched = glob_match(rule->pattern, rel2);
|
matched = glob_match(rule->pattern, rel2);
|
||||||
} else if (strchr(rule->pattern, '/') != NULL) {
|
else if (strchr(rule->pattern, '/') != NULL)
|
||||||
matched = glob_suffix_match(rule->pattern, rel2);
|
matched = glob_suffix_match(rule->pattern, rel2);
|
||||||
} else {
|
else
|
||||||
matched = glob_match(rule->pattern, leaf);
|
matched = glob_match(rule->pattern, leaf);
|
||||||
}
|
if (rule->negate)
|
||||||
return matched ? rule->action : FILTER_ACTION_NONE;
|
matched = !matched;
|
||||||
|
if (!matched)
|
||||||
|
return FILTER_ACTION_NONE;
|
||||||
|
if (side == FILTER_SIDE_RECEIVER)
|
||||||
|
return rule->action == FILTER_ACTION_EXCLUDE ? FILTER_ACTION_PROTECT : FILTER_ACTION_RISK;
|
||||||
|
return rule->action;
|
||||||
}
|
}
|
||||||
|
|
||||||
FilterAction filter_rules_apply(const FilterRuleList* list, const char* rel_path, const char* leaf,
|
FilterAction filter_rules_apply_side(const FilterRuleList* list, const char* rel_path,
|
||||||
bool is_dir) {
|
const char* leaf, bool is_dir, unsigned side) {
|
||||||
if (!list)
|
if (!list)
|
||||||
return FILTER_ACTION_NONE;
|
return FILTER_ACTION_NONE;
|
||||||
for (int i = 0; i < list->count; i++) {
|
for (int i = 0; i < list->count; i++) {
|
||||||
FilterAction action = rule_matches(list->items[i], rel_path, leaf, is_dir);
|
FilterAction action = rule_matches(list->items[i], rel_path, leaf, is_dir, side);
|
||||||
if (action != FILTER_ACTION_NONE)
|
if (action != FILTER_ACTION_NONE)
|
||||||
return action;
|
return action;
|
||||||
}
|
}
|
||||||
return FILTER_ACTION_NONE;
|
return FILTER_ACTION_NONE;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
FilterAction filter_rules_apply(const FilterRuleList* list, const char* rel_path, const char* leaf,
|
||||||
|
bool is_dir) {
|
||||||
|
return filter_rules_apply_side(list, rel_path, leaf, is_dir, FILTER_SIDE_SENDER);
|
||||||
|
}
|
||||||
|
|||||||
+88
-34
@@ -4,38 +4,51 @@
|
|||||||
#include <stdbool.h>
|
#include <stdbool.h>
|
||||||
#include <stddef.h>
|
#include <stddef.h>
|
||||||
|
|
||||||
/* rsync-style filter rule engine (client-side file selection).
|
/* rsync-style filter rule engine (client-side file selection and the
|
||||||
|
* receiver-side protection set it feeds).
|
||||||
*
|
*
|
||||||
* Supported rule syntax (documented subset):
|
* Rule syntax (see the rsync man page FILTER RULES section):
|
||||||
* [+|-] [anchored '/' prefix] pattern [trailing '/' for dir-only]
|
* RULE [PATTERN_OR_FILENAME]
|
||||||
*
|
* RULE,MODIFIERS [PATTERN_OR_FILENAME]
|
||||||
* "+ PATTERN" include rule (first match wins)
|
* Short RULE names may attach MODIFIERS directly ("-sr foo"); the long name
|
||||||
* "- PATTERN" exclude rule
|
* form requires the comma. The pattern/filename is separated from the rule by
|
||||||
* "PATTERN" implicit exclude rule (rsync default)
|
* one space or underscore. Rule names:
|
||||||
* "include PATTERN" / "exclude PATTERN" word forms
|
* exclude/- exclude (by default both sender-hide and receiver-protect)
|
||||||
* leading '/' after the +/- anchors the pattern to its owner directory
|
* include/+ include (by default both sender-show and receiver-risk)
|
||||||
* (the transfer root for command-line/-C rules, the directory that
|
* hide/H sender-only exclude
|
||||||
* contains a .rsync-filter file for per-directory rules)
|
* show/S sender-only include
|
||||||
* a trailing '/' makes the rule match directories only
|
* protect/P receiver-only exclude (protect from deletion)
|
||||||
*
|
* risk/R receiver-only include (allow deletion)
|
||||||
* Rejected explicitly (no silent no-ops): the rsync merge/dir-merge/list-clear
|
* merge/. read a client-side merge file for more rules
|
||||||
* shorthands written as a rule that starts with ':' or '.' or '!', the
|
* dir-merge/: per-directory merge file (registered for the scanner)
|
||||||
* merge/dir-merge/hide/show/protect/risk/clear words, and every include/exclude
|
* clear/! clear the current rule list (takes no argument)
|
||||||
* rule modifier other than '/' (! C s r p x). The pattern must be separated
|
* Modifiers: '/' absolute anchor, '!' negate match, 'C' inject CVS defaults,
|
||||||
* from +/- by a space (or a single '/' anchor), exactly like rsync's
|
* 's' sender side, 'r' receiver side, 'p' perishable, 'x' xattr name rule.
|
||||||
* "-s foo"/"-p ..." modifier syntax is refused.
|
* A trailing '/' makes a pattern match directories only. A leading '/' anchors
|
||||||
|
* the pattern to its owner directory.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
typedef enum {
|
typedef enum {
|
||||||
FILTER_ACTION_NONE = 0, /* no rule matched */
|
FILTER_ACTION_NONE = 0, /* no rule matched */
|
||||||
FILTER_ACTION_EXCLUDE = -1,
|
FILTER_ACTION_EXCLUDE = -1,
|
||||||
FILTER_ACTION_INCLUDE = 1
|
FILTER_ACTION_INCLUDE = 1,
|
||||||
|
/* Receiver-side-only verdicts: the entry is transferred but its destination
|
||||||
|
* mirror is protected from --delete (PROTECT) or explicitly left at risk
|
||||||
|
* (RISK). */
|
||||||
|
FILTER_ACTION_PROTECT = 2,
|
||||||
|
FILTER_ACTION_RISK = 3,
|
||||||
} FilterAction;
|
} FilterAction;
|
||||||
|
|
||||||
|
#define FILTER_SIDE_SENDER 1u
|
||||||
|
#define FILTER_SIDE_RECEIVER 2u
|
||||||
|
|
||||||
typedef struct {
|
typedef struct {
|
||||||
FilterAction action;
|
FilterAction action; /* EXCLUDE or INCLUDE (the base pattern action) */
|
||||||
|
unsigned sides; /* FILTER_SIDE_SENDER | FILTER_SIDE_RECEIVER */
|
||||||
bool anchored; /* pattern anchored to the rule's owner directory */
|
bool anchored; /* pattern anchored to the rule's owner directory */
|
||||||
bool dir_only; /* pattern had a trailing '/': matches directories only */
|
bool dir_only; /* pattern had a trailing '/': matches directories only */
|
||||||
|
bool negate; /* '!' modifier: match succeeds when the pattern does not */
|
||||||
|
bool perishable; /* 'p' modifier (ignored in deleted directories) */
|
||||||
char* owner; /* owning directory rel path ("" == transfer root) */
|
char* owner; /* owning directory rel path ("" == transfer root) */
|
||||||
char* pattern; /* cleaned glob pattern (no leading '/', no trailing '/') */
|
char* pattern; /* cleaned glob pattern (no leading '/', no trailing '/') */
|
||||||
} FilterRule;
|
} FilterRule;
|
||||||
@@ -44,19 +57,41 @@ typedef struct {
|
|||||||
FilterRule** items; /* owned array of rule pointers */
|
FilterRule** items; /* owned array of rule pointers */
|
||||||
int count;
|
int count;
|
||||||
int capacity;
|
int capacity;
|
||||||
|
/* Per-directory merge-file basenames registered by "dir-merge NAME"/": NAME"
|
||||||
|
* or by -F (.rsync-filter). Owned strings; the scanner reads each name in
|
||||||
|
* every directory it traverses. */
|
||||||
|
char** dir_merge_names;
|
||||||
|
int dir_merge_count;
|
||||||
|
int dir_merge_capacity;
|
||||||
} FilterRuleList;
|
} FilterRuleList;
|
||||||
|
|
||||||
|
/* Context needed while parsing a rule list (merge files, --delete-excluded). */
|
||||||
|
typedef struct {
|
||||||
|
bool delete_excluded; /* --delete-excluded: default sides become sender-only */
|
||||||
|
bool cvs_exclude; /* -C: expand the CVS default excludes */
|
||||||
|
} FilterParseOptions;
|
||||||
|
|
||||||
/* Parse a single filter-rule line (no trailing newline required). Returns an
|
/* Parse a single filter-rule line (no trailing newline required). Returns an
|
||||||
* owned rule, or NULL on unsupported/invalid syntax with a message in `err`. */
|
* owned rule, or NULL on unsupported/invalid syntax with a message in `err`.
|
||||||
FilterRule* filter_rule_parse(const char* line, char* err, size_t err_size);
|
* `opts` may be NULL (no merge expansion / no delete-excluded). */
|
||||||
|
FilterRule* filter_rule_parse(const char* line, const FilterParseOptions* opts, char* err,
|
||||||
|
size_t err_size);
|
||||||
void filter_rule_free(FilterRule* rule);
|
void filter_rule_free(FilterRule* rule);
|
||||||
|
|
||||||
FilterRuleList* filter_rule_list_create(void);
|
FilterRuleList* filter_rule_list_create(void);
|
||||||
/* Append a fully-parsed rule (takes ownership). Returns false on OOM. */
|
/* Append a fully-parsed rule (takes ownership). Returns false on OOM. */
|
||||||
bool filter_rule_list_add(FilterRuleList* list, FilterRule* rule);
|
bool filter_rule_list_add(FilterRuleList* list, FilterRule* rule);
|
||||||
/* Parse `line` and append it. Returns false and fills `err` on bad syntax. */
|
/* Register a per-directory merge-file basename (idempotent). Returns false on
|
||||||
bool filter_rule_list_parse_append(FilterRuleList* list, const char* line, char* err,
|
* OOM. Used by the scanner to read custom "dir-merge" files. */
|
||||||
size_t err_size);
|
bool filter_rule_list_add_dir_merge(FilterRuleList* list, const char* name);
|
||||||
|
/* Parse `line` and append it. Handles "clear"/"!" (resets the list), "merge
|
||||||
|
* FILE"/". FILE" (splices the file's rules) and "dir-merge NAME"/": NAME"
|
||||||
|
* (registers a per-directory filename). Returns false and fills `err` on bad
|
||||||
|
* syntax or an unreadable merge file. `merge_base_dir` resolves a relative
|
||||||
|
* merge-file path (NULL means the process working directory). */
|
||||||
|
bool filter_rule_list_parse_append(FilterRuleList* list, const char* line,
|
||||||
|
const FilterParseOptions* opts, const char* merge_base_dir,
|
||||||
|
char* err, size_t err_size);
|
||||||
void filter_rule_list_free(FilterRuleList* list);
|
void filter_rule_list_free(FilterRuleList* list);
|
||||||
|
|
||||||
/* Build the command-line filter set: `rule_texts` (--filter=RULE in the order
|
/* Build the command-line filter set: `rule_texts` (--filter=RULE in the order
|
||||||
@@ -64,19 +99,38 @@ void filter_rule_list_free(FilterRuleList* list);
|
|||||||
* cvs_exclude is true. All rules are owned by "" (the transfer root).
|
* cvs_exclude is true. All rules are owned by "" (the transfer root).
|
||||||
* Returns NULL on unsupported rule text (message in `err`). */
|
* Returns NULL on unsupported rule text (message in `err`). */
|
||||||
FilterRuleList* filter_base_build(const char* const* rule_texts, int rule_count, bool cvs_exclude,
|
FilterRuleList* filter_base_build(const char* const* rule_texts, int rule_count, bool cvs_exclude,
|
||||||
|
bool delete_excluded, char* err, size_t err_size);
|
||||||
|
|
||||||
|
/* Read "<dir_path>/<name>" and return its rules, each owned by `owner_rel`. A
|
||||||
|
* missing file yields an empty list with *exists=false; an unreadable file is
|
||||||
|
* treated as missing. Returns NULL only on parse or allocation failure
|
||||||
|
* (message in `err`). `opts` may be NULL. */
|
||||||
|
FilterRuleList* filter_file_read_named(const char* dir_path, const char* name,
|
||||||
|
const char* owner_rel, const FilterParseOptions* opts,
|
||||||
|
bool* exists, char* err, size_t err_size);
|
||||||
|
|
||||||
|
/* Append the rules of "<dir_path>/<name>" into an existing list (each owned by
|
||||||
|
* `owner_rel`). A missing file yields *exists=false and no error. Returns
|
||||||
|
* false only on parse/allocation failure (message in `err`). */
|
||||||
|
bool filter_file_append(FilterRuleList* list, const char* dir_path, const char* name,
|
||||||
|
const char* owner_rel, const FilterParseOptions* opts, bool* exists,
|
||||||
char* err, size_t err_size);
|
char* err, size_t err_size);
|
||||||
|
|
||||||
/* Read "<dir_path>/.rsync-filter" and return its rules, each owned by
|
/* filter_file_read_named with the default ".rsync-filter" name. */
|
||||||
* `owner_rel`. A missing file yields an empty list with *exists=false; an
|
|
||||||
* unreadable file is treated as missing. Returns NULL only on parse or
|
|
||||||
* allocation failure (message in `err`). */
|
|
||||||
FilterRuleList* filter_file_read(const char* dir_path, const char* owner_rel, bool* exists,
|
FilterRuleList* filter_file_read(const char* dir_path, const char* owner_rel, bool* exists,
|
||||||
char* err, size_t err_size);
|
char* err, size_t err_size);
|
||||||
|
|
||||||
/* Evaluate an entry against one ordered rule list. Returns FILTER_ACTION_NONE
|
/* Evaluate an entry against one ordered rule list for one side. Returns
|
||||||
* when no rule matched, otherwise the first matching rule's action.
|
* FILTER_ACTION_NONE when no rule matched, otherwise the first matching rule's
|
||||||
* `rel_path` is the entry's path relative to the transfer root ("" == root),
|
* action (for the receiver side an EXCLUDE is reported as
|
||||||
* `leaf` its final name, `is_dir` whether it is a directory. */
|
* FILTER_ACTION_PROTECT and an INCLUDE as FILTER_ACTION_RISK). `rel_path` is
|
||||||
|
* the entry's path relative to the transfer root ("" == root), `leaf` its final
|
||||||
|
* name, `is_dir` whether it is a directory. */
|
||||||
|
FilterAction filter_rules_apply_side(const FilterRuleList* list, const char* rel_path,
|
||||||
|
const char* leaf, bool is_dir, unsigned side);
|
||||||
|
|
||||||
|
/* Sender-side convenience wrapper (kept for callers/tests that only need the
|
||||||
|
* transfer decision). */
|
||||||
FilterAction filter_rules_apply(const FilterRuleList* list, const char* rel_path, const char* leaf,
|
FilterAction filter_rules_apply(const FilterRuleList* list, const char* rel_path, const char* leaf,
|
||||||
bool is_dir);
|
bool is_dir);
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,129 @@
|
|||||||
|
#include "format.h"
|
||||||
|
#include "protocol.h"
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <string.h>
|
||||||
|
|
||||||
|
bool format_human_size_decimal(unsigned long long bytes, char* buffer, size_t buffer_size) {
|
||||||
|
if (!buffer || buffer_size == 0)
|
||||||
|
return false;
|
||||||
|
if (bytes < 1000ULL) {
|
||||||
|
int written = snprintf(buffer, buffer_size, "%llu", bytes);
|
||||||
|
return written >= 0 && (size_t)written < buffer_size;
|
||||||
|
}
|
||||||
|
static const char units[] = "KMGTPE";
|
||||||
|
double value = (double)bytes;
|
||||||
|
size_t divisions = 0;
|
||||||
|
while (value >= 1000.0 && divisions < sizeof(units) - 1) {
|
||||||
|
value /= 1000.0;
|
||||||
|
divisions++;
|
||||||
|
}
|
||||||
|
int written = snprintf(buffer, buffer_size, "%.2f%c", value, units[divisions - 1]);
|
||||||
|
return written >= 0 && (size_t)written < buffer_size;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool format_big_num(unsigned long long value, bool human_readable, char* buffer,
|
||||||
|
size_t buffer_size) {
|
||||||
|
if (human_readable)
|
||||||
|
return format_human_size_decimal(value, buffer, buffer_size);
|
||||||
|
char digits[32];
|
||||||
|
int written = snprintf(digits, sizeof(digits), "%llu", value);
|
||||||
|
if (written < 0 || (size_t)written >= sizeof(digits))
|
||||||
|
return false;
|
||||||
|
size_t len = (size_t)written;
|
||||||
|
size_t separators = len > 1 ? (len - 1) / 3 : 0;
|
||||||
|
size_t total = len + separators;
|
||||||
|
if (total + 1 > buffer_size)
|
||||||
|
return false;
|
||||||
|
size_t out = total;
|
||||||
|
buffer[out] = '\0';
|
||||||
|
size_t digits_since_sep = 0;
|
||||||
|
for (size_t i = len; i > 0; i--) {
|
||||||
|
buffer[--out] = digits[i - 1];
|
||||||
|
digits_since_sep++;
|
||||||
|
if (digits_since_sep == 3 && i > 1) {
|
||||||
|
buffer[--out] = ',';
|
||||||
|
digits_since_sep = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool format_rsync_datetime(time_t when, bool dash, char* buffer, size_t buffer_size) {
|
||||||
|
if (!buffer || buffer_size == 0)
|
||||||
|
return false;
|
||||||
|
struct tm broken_down;
|
||||||
|
if (localtime_r(&when, &broken_down) == NULL)
|
||||||
|
return false;
|
||||||
|
const char* format = dash ? "%Y/%m/%d-%H:%M:%S" : "%Y/%m/%d %H:%M:%S";
|
||||||
|
return strftime(buffer, buffer_size, format, &broken_down) != 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool format_dest_state_send(int fd, const OutputDestState* state) {
|
||||||
|
if (!state)
|
||||||
|
return false;
|
||||||
|
int32_t has_old = state->existed ? 1 : 0;
|
||||||
|
uint64_t size = (uint64_t)state->size;
|
||||||
|
int64_t mtime = (int64_t)state->mtime_sec;
|
||||||
|
int64_t mtime_nsec = state->mtime_nsec;
|
||||||
|
uint32_t mode = state->mode;
|
||||||
|
int32_t uid = state->uid;
|
||||||
|
int32_t gid = state->gid;
|
||||||
|
return send_n_data(fd, &has_old, sizeof(has_old)) && send_n_data(fd, &size, sizeof(size)) &&
|
||||||
|
send_n_data(fd, &mtime, sizeof(mtime)) &&
|
||||||
|
send_n_data(fd, &mtime_nsec, sizeof(mtime_nsec)) && send_n_data(fd, &mode, sizeof(mode)) &&
|
||||||
|
send_n_data(fd, &uid, sizeof(uid)) && send_n_data(fd, &gid, sizeof(gid));
|
||||||
|
}
|
||||||
|
|
||||||
|
bool format_dest_state_receive(int fd, OutputDestState* state) {
|
||||||
|
if (!state)
|
||||||
|
return false;
|
||||||
|
int32_t has_old = 0;
|
||||||
|
uint64_t size = 0;
|
||||||
|
int64_t mtime = 0;
|
||||||
|
int64_t mtime_nsec = 0;
|
||||||
|
uint32_t mode = 0;
|
||||||
|
int32_t uid = 0;
|
||||||
|
int32_t gid = 0;
|
||||||
|
if (!receive_n_data(fd, &has_old, sizeof(has_old)) || !receive_n_data(fd, &size, sizeof(size)) ||
|
||||||
|
!receive_n_data(fd, &mtime, sizeof(mtime)) ||
|
||||||
|
!receive_n_data(fd, &mtime_nsec, sizeof(mtime_nsec)) ||
|
||||||
|
!receive_n_data(fd, &mode, sizeof(mode)) || !receive_n_data(fd, &uid, sizeof(uid)) ||
|
||||||
|
!receive_n_data(fd, &gid, sizeof(gid)))
|
||||||
|
return false;
|
||||||
|
memset(state, 0, sizeof(*state));
|
||||||
|
state->known = true;
|
||||||
|
state->existed = has_old != 0;
|
||||||
|
state->size = size;
|
||||||
|
state->mtime_sec = mtime;
|
||||||
|
state->mtime_nsec = mtime_nsec;
|
||||||
|
state->mode = mode;
|
||||||
|
state->uid = uid;
|
||||||
|
state->gid = gid;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool format_stats_send(int fd, const ReceiverStats* stats) {
|
||||||
|
if (!stats)
|
||||||
|
return false;
|
||||||
|
unsigned long long matched = stats->matched_data;
|
||||||
|
unsigned long long deleted = stats->deleted_files;
|
||||||
|
unsigned long long would = stats->would_delete_count;
|
||||||
|
return send_n_data(fd, &matched, sizeof(matched)) && send_n_data(fd, &deleted, sizeof(deleted)) &&
|
||||||
|
send_n_data(fd, &would, sizeof(would));
|
||||||
|
}
|
||||||
|
|
||||||
|
bool format_stats_receive(int fd, ReceiverStats* stats) {
|
||||||
|
if (!stats)
|
||||||
|
return false;
|
||||||
|
unsigned long long matched = 0;
|
||||||
|
unsigned long long deleted = 0;
|
||||||
|
unsigned long long would = 0;
|
||||||
|
if (!receive_n_data(fd, &matched, sizeof(matched)) ||
|
||||||
|
!receive_n_data(fd, &deleted, sizeof(deleted)) || !receive_n_data(fd, &would, sizeof(would)))
|
||||||
|
return false;
|
||||||
|
memset(stats, 0, sizeof(*stats));
|
||||||
|
stats->matched_data = matched;
|
||||||
|
stats->deleted_files = deleted;
|
||||||
|
stats->would_delete_count = would;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
@@ -0,0 +1,76 @@
|
|||||||
|
#ifndef FORMAT_H
|
||||||
|
#define FORMAT_H
|
||||||
|
|
||||||
|
#include <stdbool.h>
|
||||||
|
#include <stddef.h>
|
||||||
|
#include <stdint.h>
|
||||||
|
#include <time.h>
|
||||||
|
|
||||||
|
/* Low-level output-formatting primitives shared by the change-event model
|
||||||
|
* (change_list.c) and the transfer driver (client_send.c).
|
||||||
|
*
|
||||||
|
* The functions here are pure/string-level except for the STATUS_DEST_INFO
|
||||||
|
* codec, which lets the receiver report the pre-transfer destination entry so
|
||||||
|
* the sender can render rsync-accurate --itemize-changes / --out-format
|
||||||
|
* columns (see protocol.h). */
|
||||||
|
|
||||||
|
/* Pre-transfer destination snapshot, reported by the receiver when the wire
|
||||||
|
* config carries report_dest_info. `known` distinguishes "no report was
|
||||||
|
* requested/received" from "the destination did not exist" (`existed == false`
|
||||||
|
* with `known == true`). */
|
||||||
|
typedef struct {
|
||||||
|
bool known;
|
||||||
|
bool existed;
|
||||||
|
unsigned long long size;
|
||||||
|
long long mtime_sec;
|
||||||
|
long long mtime_nsec;
|
||||||
|
uint32_t mode;
|
||||||
|
int32_t uid;
|
||||||
|
int32_t gid;
|
||||||
|
} OutputDestState;
|
||||||
|
|
||||||
|
/* rsync's -h/--human-readable size (decimal, base 1000): integers below 1000
|
||||||
|
* print verbatim; larger values use the largest unit that keeps the value
|
||||||
|
* below 1000 (K/M/G/T/P/E) with exactly two decimals, so 1500000 -> "1.50M"
|
||||||
|
* and 999999 -> "1000.00K" (matching rsync's human_num). Returns false when
|
||||||
|
* the buffer is too small (nothing is written). */
|
||||||
|
bool format_human_size_decimal(unsigned long long bytes, char* buffer, size_t buffer_size);
|
||||||
|
|
||||||
|
/* rsync's general number formatting (big_num). When `human_readable` is true
|
||||||
|
* this is format_human_size_decimal; otherwise the integer is rendered with a
|
||||||
|
* ',' thousands separator every three digits (rsync's separator in the C
|
||||||
|
* locale). Returns false on an undersized buffer. */
|
||||||
|
bool format_big_num(unsigned long long value, bool human_readable, char* buffer,
|
||||||
|
size_t buffer_size);
|
||||||
|
|
||||||
|
/* rsync's %M/%t timestamp. When `dash` is true the separator between the date
|
||||||
|
* and the time is '-' (the %M form: "YYYY/MM/DD-HH:MM:SS"); otherwise it is a
|
||||||
|
* space (the %t form: "YYYY/MM/DD HH:MM:SS"). Local time. Returns false on a
|
||||||
|
* bad time or an undersized buffer. */
|
||||||
|
bool format_rsync_datetime(time_t when, bool dash, char* buffer, size_t buffer_size);
|
||||||
|
|
||||||
|
/* Fixed-width STATUS_DEST_INFO record codec (int32 has_old, uint64 size,
|
||||||
|
* int64 mtime, int64 mtime_nsec, uint32 mode, int32 uid, int32 gid). The
|
||||||
|
* status frame itself is sent/received by the caller. Returns false on I/O
|
||||||
|
* failure. */
|
||||||
|
bool format_dest_state_send(int fd, const OutputDestState* state);
|
||||||
|
bool format_dest_state_receive(int fd, OutputDestState* state);
|
||||||
|
|
||||||
|
/* End-of-transfer receiver counters reported through STATUS_STATS (protocol
|
||||||
|
* 2.25.0) when the wire config carries report_stats. `would_delete_count` is
|
||||||
|
* the number of destination-relative paths the receiver would have deleted in a
|
||||||
|
* -n/--dry-run --delete run; that many wire strings immediately follow the
|
||||||
|
* fixed record (sent/read by the caller). */
|
||||||
|
typedef struct {
|
||||||
|
unsigned long long matched_data;
|
||||||
|
unsigned long long deleted_files;
|
||||||
|
unsigned long long would_delete_count;
|
||||||
|
} ReceiverStats;
|
||||||
|
|
||||||
|
/* Fixed-width STATUS_STATS counter record. The status frame and the optional
|
||||||
|
* would-delete path list are sent/received by the caller. Returns false on I/O
|
||||||
|
* failure. */
|
||||||
|
bool format_stats_send(int fd, const ReceiverStats* stats);
|
||||||
|
bool format_stats_receive(int fd, ReceiverStats* stats);
|
||||||
|
|
||||||
|
#endif
|
||||||
+469
-126
@@ -36,14 +36,34 @@ typedef struct {
|
|||||||
bool copy_as_set;
|
bool copy_as_set;
|
||||||
int32_t copy_as_uid;
|
int32_t copy_as_uid;
|
||||||
int32_t copy_as_gid;
|
int32_t copy_as_gid;
|
||||||
|
/* -o/--owner and -g/--group: preserve the source owner/group through the
|
||||||
|
* normal name/identity resolution path. Split out of the former
|
||||||
|
* use_metadata bundle; unlike --numeric-ids/--chown/--usermap/--groupmap/-a
|
||||||
|
* these are a preserve-source request, not an arbitrary client-chosen owner,
|
||||||
|
* so they are tracked separately from the explicit ownership gate. */
|
||||||
|
bool preserve_owner;
|
||||||
|
bool preserve_group;
|
||||||
|
/* --fake-super: when active the receiver must only RECORD the (resolved)
|
||||||
|
* ownership in the reserved xattr, never perform a real chown. Snapshotted
|
||||||
|
* so the fd-relative ownership helpers can suppress the chown without a
|
||||||
|
* Config argument. */
|
||||||
|
bool fake_super;
|
||||||
bool set;
|
bool set;
|
||||||
} IdentityActive;
|
} IdentityActive;
|
||||||
|
|
||||||
static IdentityActive g_identity;
|
static IdentityActive g_identity;
|
||||||
|
|
||||||
static void identity_active_reset(void) {
|
static void identity_active_reset(void) {
|
||||||
|
if (g_identity.usermap) {
|
||||||
|
for (int i = 0; i < g_identity.usermap_count; i++)
|
||||||
|
free(g_identity.usermap[i].to_name);
|
||||||
free(g_identity.usermap);
|
free(g_identity.usermap);
|
||||||
|
}
|
||||||
|
if (g_identity.groupmap) {
|
||||||
|
for (int i = 0; i < g_identity.groupmap_count; i++)
|
||||||
|
free(g_identity.groupmap[i].to_name);
|
||||||
free(g_identity.groupmap);
|
free(g_identity.groupmap);
|
||||||
|
}
|
||||||
g_identity.usermap = NULL;
|
g_identity.usermap = NULL;
|
||||||
g_identity.groupmap = NULL;
|
g_identity.groupmap = NULL;
|
||||||
g_identity.usermap_count = 0;
|
g_identity.usermap_count = 0;
|
||||||
@@ -57,6 +77,9 @@ static void identity_active_reset(void) {
|
|||||||
g_identity.copy_as_set = false;
|
g_identity.copy_as_set = false;
|
||||||
g_identity.copy_as_uid = 0;
|
g_identity.copy_as_uid = 0;
|
||||||
g_identity.copy_as_gid = 0;
|
g_identity.copy_as_gid = 0;
|
||||||
|
g_identity.preserve_owner = false;
|
||||||
|
g_identity.preserve_group = false;
|
||||||
|
g_identity.fake_super = false;
|
||||||
g_identity.set = false;
|
g_identity.set = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -77,32 +100,57 @@ bool identity_set_active(const Config* config) {
|
|||||||
g_identity.copy_as_set = config->copy_as_set;
|
g_identity.copy_as_set = config->copy_as_set;
|
||||||
g_identity.copy_as_uid = config->copy_as_uid;
|
g_identity.copy_as_uid = config->copy_as_uid;
|
||||||
g_identity.copy_as_gid = config->copy_as_gid;
|
g_identity.copy_as_gid = config->copy_as_gid;
|
||||||
|
g_identity.preserve_owner = config->preserve_owner;
|
||||||
|
g_identity.preserve_group = config->preserve_group;
|
||||||
|
g_identity.fake_super = config->fake_super;
|
||||||
if (config->usermap_count > 0) {
|
if (config->usermap_count > 0) {
|
||||||
g_identity.usermap = calloc((size_t)config->usermap_count, sizeof(IdentityMap));
|
g_identity.usermap = calloc((size_t)config->usermap_count, sizeof(IdentityMap));
|
||||||
if (!g_identity.usermap)
|
if (!g_identity.usermap)
|
||||||
goto alloc_failed;
|
goto alloc_failed;
|
||||||
memcpy(g_identity.usermap, config->usermap,
|
for (int i = 0; i < config->usermap_count; i++) {
|
||||||
(size_t)config->usermap_count * sizeof(IdentityMap));
|
g_identity.usermap[i] = config->usermap[i];
|
||||||
|
g_identity.usermap[i].to_name =
|
||||||
|
config->usermap[i].to_name ? str_dup(config->usermap[i].to_name) : NULL;
|
||||||
|
if (config->usermap[i].to_name && !g_identity.usermap[i].to_name) {
|
||||||
|
g_identity.usermap_count = i; /* free only the entries already duplicated */
|
||||||
|
goto alloc_failed;
|
||||||
|
}
|
||||||
|
}
|
||||||
g_identity.usermap_count = config->usermap_count;
|
g_identity.usermap_count = config->usermap_count;
|
||||||
}
|
}
|
||||||
if (config->groupmap_count > 0) {
|
if (config->groupmap_count > 0) {
|
||||||
g_identity.groupmap = calloc((size_t)config->groupmap_count, sizeof(IdentityMap));
|
g_identity.groupmap = calloc((size_t)config->groupmap_count, sizeof(IdentityMap));
|
||||||
if (!g_identity.groupmap)
|
if (!g_identity.groupmap)
|
||||||
goto alloc_failed;
|
goto alloc_failed;
|
||||||
memcpy(g_identity.groupmap, config->groupmap,
|
for (int i = 0; i < config->groupmap_count; i++) {
|
||||||
(size_t)config->groupmap_count * sizeof(IdentityMap));
|
g_identity.groupmap[i] = config->groupmap[i];
|
||||||
|
g_identity.groupmap[i].to_name =
|
||||||
|
config->groupmap[i].to_name ? str_dup(config->groupmap[i].to_name) : NULL;
|
||||||
|
if (config->groupmap[i].to_name && !g_identity.groupmap[i].to_name) {
|
||||||
|
g_identity.groupmap_count = i;
|
||||||
|
goto alloc_failed;
|
||||||
|
}
|
||||||
|
}
|
||||||
g_identity.groupmap_count = config->groupmap_count;
|
g_identity.groupmap_count = config->groupmap_count;
|
||||||
}
|
}
|
||||||
g_identity.set = true;
|
g_identity.set = true;
|
||||||
/* A root receiver would honor any client-supplied ownership request (a
|
/* A root receiver would honor any client-supplied ownership request (a
|
||||||
--usermap/--groupmap/--chown/--copy-as, or raw ids under --numeric-ids).
|
--usermap/--groupmap/--chown/--copy-as, or raw ids under --numeric-ids)
|
||||||
Surface that prominently; a privileged daemon applying arbitrary client
|
ONLY when super-user activities are permitted. --no-super (or a daemon
|
||||||
ownership is a deliberate, opt-in choice the operator should be aware of. */
|
veto that forced SUPER_MODE_OFF) forbids the chown even for root, so do
|
||||||
if (geteuid() == 0)
|
not claim the ownership will be honored in that case. */
|
||||||
|
if (geteuid() == 0) {
|
||||||
|
if (privilege_super_mode_permitted(g_identity.super_mode))
|
||||||
log_message(LOG_LEVEL_WARNING,
|
log_message(LOG_LEVEL_WARNING,
|
||||||
"identity mapping active and running as root: client-supplied "
|
"identity mapping active and running as root: client-supplied "
|
||||||
"ownership (usermap/groupmap/chown/numeric-ids) will be honored; "
|
"ownership (usermap/groupmap/chown/numeric-ids) will be honored; "
|
||||||
"run the daemon as an unprivileged user unless intended");
|
"run the daemon as an unprivileged user unless intended");
|
||||||
|
else
|
||||||
|
log_message(LOG_LEVEL_WARNING,
|
||||||
|
"identity mapping active and running as root, but super-user activities are "
|
||||||
|
"disabled (--no-super): requested ownership will NOT be applied; run the "
|
||||||
|
"daemon as an unprivileged user unless intended");
|
||||||
|
}
|
||||||
/* --super explicitly requests super-user activities, but FastSync never
|
/* --super explicitly requests super-user activities, but FastSync never
|
||||||
elevates privileges: when the receiver is not already root the kernel will
|
elevates privileges: when the receiver is not already root the kernel will
|
||||||
refuse those confined attempts and each is skipped per entry. Warn exactly
|
refuse those confined attempts and each is skipped per entry. Warn exactly
|
||||||
@@ -138,25 +186,55 @@ bool privilege_super_mode_permitted(SuperMode mode) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
bool identity_active_enabled(void) {
|
bool identity_active_enabled(void) {
|
||||||
/* numeric_ids is included: this set only gates identity_apply_ownership,
|
/* --numeric-ids is deliberately NOT included: it is a mapping MODIFIER (use
|
||||||
which runs only when metadata is present (a -M/--preserve transfer). A
|
* the transmitted numeric id raw instead of a name lookup), not a request to
|
||||||
standalone --numeric-ids (no ownership-affecting flag) carries no
|
* change ownership. rsync's --numeric-ids on its own never chowns anything;
|
||||||
metadata, never reaches identity_apply_ownership, and therefore correctly
|
* it only changes how an already-requested -o/-g/map resolves. Ownership is
|
||||||
stays inert; combined with -M it activates raw-id application. --super /
|
* activated only by an explicit request: --chown/--usermap/--groupmap/
|
||||||
--no-super does NOT enable ownership: it only permits or forbids the
|
* --copy-as or a preserve-source -o/--owner / -g/--group. --super/--no-super
|
||||||
already-requested super-user activities, so a --super with no explicit
|
* likewise does NOT enable ownership: it only permits or forbids the
|
||||||
identity flag must never silently apply client-chosen ownership. */
|
* already-requested super-user activities. */
|
||||||
return g_identity.set &&
|
return g_identity.set &&
|
||||||
(g_identity.numeric_ids || g_identity.chown_uid_set || g_identity.chown_gid_set ||
|
(g_identity.chown_uid_set || g_identity.chown_gid_set || g_identity.usermap_count > 0 ||
|
||||||
g_identity.usermap_count > 0 || g_identity.groupmap_count > 0 || g_identity.copy_as_set);
|
g_identity.groupmap_count > 0 || g_identity.copy_as_set || g_identity.preserve_owner ||
|
||||||
|
g_identity.preserve_group);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool identity_owner_requested(void) {
|
||||||
|
return g_identity.set && (g_identity.copy_as_set || g_identity.chown_uid_set ||
|
||||||
|
g_identity.preserve_owner || g_identity.usermap_count > 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool identity_group_requested(void) {
|
||||||
|
return g_identity.set && (g_identity.copy_as_set || g_identity.chown_gid_set ||
|
||||||
|
g_identity.preserve_group || g_identity.groupmap_count > 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
bool identity_ownership_requested(const Config* config) {
|
bool identity_ownership_requested(const Config* config) {
|
||||||
if (!config)
|
if (!config)
|
||||||
return false;
|
return false;
|
||||||
/* Every value that makes the receiver act on a client-chosen owner, plus an
|
/* General-awareness predicate: every value that makes the receiver act on a
|
||||||
* explicit --super (super-user device-node activities). Pure config, so the
|
* client-chosen owner, plus an explicit --super (super-user device-node
|
||||||
* daemon gate can evaluate it before identity_set_active(). */
|
* activities) and the preserve-source -o/-g requests. Pure config, so callers
|
||||||
|
* can evaluate it before identity_set_active(). The daemon module gate uses
|
||||||
|
* the narrower identity_explicit_ownership_requested() below, which treats a
|
||||||
|
* plain -o/-g/-a as a preserve-source request rather than arbitrary
|
||||||
|
* client-chosen ownership. */
|
||||||
|
return config->numeric_ids || config->chown_uid_set || config->chown_gid_set ||
|
||||||
|
config->usermap_count > 0 || config->groupmap_count > 0 || config->copy_as_set ||
|
||||||
|
config->preserve_owner || config->preserve_group || config->fake_super ||
|
||||||
|
config->super_mode == SUPER_MODE_ON;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool identity_explicit_ownership_requested(const Config* config) {
|
||||||
|
if (!config)
|
||||||
|
return false;
|
||||||
|
/* The narrow set the daemon gate refuses for a non-opted module: a request
|
||||||
|
* that lets the CLIENT choose an arbitrary owner/group (rather than preserve
|
||||||
|
* the source's own). Deliberately EXCLUDES preserve_owner/preserve_group so a
|
||||||
|
* plain -a/-o/-g push is not refused; for those the gate instead forces
|
||||||
|
* super-user ownership activity off (no chown happens) unless the module has
|
||||||
|
* `client owner = yes`. */
|
||||||
return config->numeric_ids || config->chown_uid_set || config->chown_gid_set ||
|
return config->numeric_ids || config->chown_uid_set || config->chown_gid_set ||
|
||||||
config->usermap_count > 0 || config->groupmap_count > 0 || config->copy_as_set ||
|
config->usermap_count > 0 || config->groupmap_count > 0 || config->copy_as_set ||
|
||||||
config->fake_super || config->super_mode == SUPER_MODE_ON;
|
config->fake_super || config->super_mode == SUPER_MODE_ON;
|
||||||
@@ -177,6 +255,29 @@ bool identity_copy_as_refused(const Config* config) {
|
|||||||
return geteuid() != 0 || config->super_mode == SUPER_MODE_OFF;
|
return geteuid() != 0 || config->super_mode == SUPER_MODE_OFF;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Validate one received FROM:TO map rule. `from` is a single id, the LOW end
|
||||||
|
* of an inclusive range, IDENTITY_MATCH_ANY, or IDENTITY_MATCH_UNNAMED; a
|
||||||
|
* sentinel FROM must carry the same value in from_hi. `to` is a non-negative
|
||||||
|
* id, IDENTITY_CURRENT, or ignored when a bounded receiver-resolved `to_name`
|
||||||
|
* is present. */
|
||||||
|
static bool identity_wire_map_valid(const IdentityMap* map) {
|
||||||
|
if (!map)
|
||||||
|
return false;
|
||||||
|
if (map->from < IDENTITY_MATCH_UNNAMED)
|
||||||
|
return false;
|
||||||
|
if (map->from < 0) {
|
||||||
|
if (map->from_hi != map->from)
|
||||||
|
return false;
|
||||||
|
} else if (map->from_hi < map->from) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (map->to < IDENTITY_CURRENT)
|
||||||
|
return false;
|
||||||
|
if (map->to_name && strlen(map->to_name) > 255)
|
||||||
|
return false;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
bool identity_wire_valid(const Config* config) {
|
bool identity_wire_valid(const Config* config) {
|
||||||
if (!config)
|
if (!config)
|
||||||
return false;
|
return false;
|
||||||
@@ -188,11 +289,11 @@ bool identity_wire_valid(const Config* config) {
|
|||||||
if (config->chown_gid_set && config->chown_gid < IDENTITY_MATCH_ANY)
|
if (config->chown_gid_set && config->chown_gid < IDENTITY_MATCH_ANY)
|
||||||
return false;
|
return false;
|
||||||
for (int i = 0; i < config->usermap_count; i++) {
|
for (int i = 0; i < config->usermap_count; i++) {
|
||||||
if (config->usermap[i].from < IDENTITY_MATCH_ANY || config->usermap[i].to < IDENTITY_CURRENT)
|
if (!identity_wire_map_valid(&config->usermap[i]))
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
for (int i = 0; i < config->groupmap_count; i++) {
|
for (int i = 0; i < config->groupmap_count; i++) {
|
||||||
if (config->groupmap[i].from < IDENTITY_MATCH_ANY || config->groupmap[i].to < IDENTITY_CURRENT)
|
if (!identity_wire_map_valid(&config->groupmap[i]))
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
/* Defense-in-depth: a --copy-as block must never carry a negative (sentinel)
|
/* Defense-in-depth: a --copy-as block must never carry a negative (sentinel)
|
||||||
@@ -250,15 +351,137 @@ static int identity_resolve_token(const char* token, bool is_group, int32_t* out
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
static int identity_append_rule(IdentityMap** map, int* count, int32_t from, int32_t to) {
|
static bool identity_all_digits(const char* token) {
|
||||||
|
if (!token || *token == '\0')
|
||||||
|
return false;
|
||||||
|
for (const char* p = token; *p; p++)
|
||||||
|
if (*p < '0' || *p > '9')
|
||||||
|
return false;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool identity_token_has_glob(const char* token) {
|
||||||
|
return token && (strchr(token, '*') || strchr(token, '?') || strchr(token, '['));
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Parse a --usermap/--groupmap FROM token into a matcher (from/from_hi). rsync
|
||||||
|
* accepts a name, a numeric id, an inclusive LOW-HIGH range, '*' (any id), or an
|
||||||
|
* empty token (ids with no name on the sender). Returns 0 on success, -1 on a
|
||||||
|
* malformed token or an unresolvable sender-side name. */
|
||||||
|
static int identity_parse_from(const char* token, bool is_group, int32_t* out_from,
|
||||||
|
int32_t* out_hi) {
|
||||||
|
if (token[0] == '\0') {
|
||||||
|
*out_from = IDENTITY_MATCH_UNNAMED;
|
||||||
|
*out_hi = IDENTITY_MATCH_UNNAMED;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
if (strcmp(token, "*") == 0) {
|
||||||
|
*out_from = IDENTITY_MATCH_ANY;
|
||||||
|
*out_hi = IDENTITY_MATCH_ANY;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
const char* num = token[0] == '@' ? token + 1 : token;
|
||||||
|
if (identity_all_digits(num)) {
|
||||||
|
int32_t id;
|
||||||
|
if (identity_resolve_token(token, is_group, &id) != 0)
|
||||||
|
return -1;
|
||||||
|
*out_from = id;
|
||||||
|
*out_hi = id;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
/* An inclusive LOW-HIGH numeric range. */
|
||||||
|
const char* dash = strchr(num, '-');
|
||||||
|
if (dash && dash != num && dash[1] != '\0' && strchr(dash + 1, '-') == NULL) {
|
||||||
|
size_t lo_len = (size_t)(dash - num);
|
||||||
|
size_t hi_len = strlen(dash + 1);
|
||||||
|
char low[16];
|
||||||
|
char high[16];
|
||||||
|
if (lo_len < sizeof(low) && hi_len < sizeof(high)) {
|
||||||
|
memcpy(low, num, lo_len);
|
||||||
|
low[lo_len] = '\0';
|
||||||
|
memcpy(high, dash + 1, hi_len);
|
||||||
|
high[hi_len] = '\0';
|
||||||
|
if (identity_all_digits(low) && identity_all_digits(high)) {
|
||||||
|
char* endptr = NULL;
|
||||||
|
errno = 0;
|
||||||
|
long lo = strtol(low, &endptr, 10);
|
||||||
|
if (errno != 0 || !endptr || *endptr != '\0')
|
||||||
|
return -1;
|
||||||
|
errno = 0;
|
||||||
|
long hi = strtol(high, &endptr, 10);
|
||||||
|
if (errno != 0 || !endptr || *endptr != '\0' || hi < lo || hi > INT32_MAX)
|
||||||
|
return -1;
|
||||||
|
*out_from = (int32_t)lo;
|
||||||
|
*out_hi = (int32_t)hi;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
/* Not a numeric LOW-HIGH range: fall through and treat as a name (a
|
||||||
|
* hyphenated account name like "wayne-smith" must still resolve). */
|
||||||
|
}
|
||||||
|
/* A sender-side name. A wildcard other than the bare '*' is matched by rsync
|
||||||
|
* against the sender's names; because FastSync transmits numeric ids only, the
|
||||||
|
* receiver cannot evaluate it, so reject rather than silently mis-match. */
|
||||||
|
if (identity_token_has_glob(token)) {
|
||||||
|
log_message(LOG_LEVEL_ERROR,
|
||||||
|
"%smap FROM '%s': name wildcards other than '*' are not supported "
|
||||||
|
"(FastSync transmits numeric ids, so sender names are unavailable on the "
|
||||||
|
"receiver)",
|
||||||
|
is_group ? "--group" : "--user", token);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
int32_t id;
|
||||||
|
if (identity_resolve_token(token, is_group, &id) != 0)
|
||||||
|
return -1;
|
||||||
|
*out_from = id;
|
||||||
|
*out_hi = id;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Parse a --usermap/--groupmap TO token. '*', a bare numeric id, or an @N id is
|
||||||
|
* stored numerically; every other non-empty token is a NAME resolved on the
|
||||||
|
* RECEIVER at apply time (rsync resolves TO names against the receiving side).
|
||||||
|
* Returns 0 on success, -1 on an empty/malformed token. */
|
||||||
|
static int identity_parse_to(const char* token, bool is_group, int32_t* out_to, char** out_name) {
|
||||||
|
if (token[0] == '\0') {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "%smap TO value is missing", is_group ? "--group" : "--user");
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
if (strcmp(token, "*") == 0) {
|
||||||
|
*out_to = IDENTITY_CURRENT;
|
||||||
|
*out_name = NULL;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
const char* num = token[0] == '@' ? token + 1 : token;
|
||||||
|
if (identity_all_digits(num)) {
|
||||||
|
int32_t id;
|
||||||
|
if (identity_resolve_token(token, is_group, &id) != 0)
|
||||||
|
return -1;
|
||||||
|
*out_to = id;
|
||||||
|
*out_name = NULL;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
if (identity_token_has_glob(token)) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "%smap TO '%s' may not contain a wildcard",
|
||||||
|
is_group ? "--group" : "--user", token);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
char* name = str_dup(token);
|
||||||
|
if (!name)
|
||||||
|
return -1;
|
||||||
|
*out_to = 0;
|
||||||
|
*out_name = name;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
static int identity_append_rule(IdentityMap** map, int* count, const IdentityMap* rule) {
|
||||||
if (*count >= MAX_IDENTITY_MAP)
|
if (*count >= MAX_IDENTITY_MAP)
|
||||||
return -1;
|
return -1;
|
||||||
IdentityMap* grown = realloc(*map, (size_t)(*count + 1) * sizeof(IdentityMap));
|
IdentityMap* grown = realloc(*map, (size_t)(*count + 1) * sizeof(IdentityMap));
|
||||||
if (!grown)
|
if (!grown)
|
||||||
return -1;
|
return -1;
|
||||||
*map = grown;
|
*map = grown;
|
||||||
(*map)[*count].from = from;
|
(*map)[*count] = *rule;
|
||||||
(*map)[*count].to = to;
|
|
||||||
(*count)++;
|
(*count)++;
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
@@ -275,7 +498,7 @@ int identity_parse_map(Config* config, const char* value, bool is_group) {
|
|||||||
char* saveptr = NULL;
|
char* saveptr = NULL;
|
||||||
for (char* rule = strtok_r(list, ",", &saveptr); rule; rule = strtok_r(NULL, ",", &saveptr)) {
|
for (char* rule = strtok_r(list, ",", &saveptr); rule; rule = strtok_r(NULL, ",", &saveptr)) {
|
||||||
char* colon = strchr(rule, ':');
|
char* colon = strchr(rule, ':');
|
||||||
if (!colon || colon == rule) {
|
if (!colon) {
|
||||||
/* Log before freeing: `rule` points into the str_dup'd list. */
|
/* Log before freeing: `rule` points into the str_dup'd list. */
|
||||||
log_message(LOG_LEVEL_ERROR, "%s rules must be FROM:TO (got '%s')", optname, rule);
|
log_message(LOG_LEVEL_ERROR, "%s rules must be FROM:TO (got '%s')", optname, rule);
|
||||||
free(list);
|
free(list);
|
||||||
@@ -284,25 +507,25 @@ int identity_parse_map(Config* config, const char* value, bool is_group) {
|
|||||||
*colon = '\0';
|
*colon = '\0';
|
||||||
char* from_token = rule;
|
char* from_token = rule;
|
||||||
char* to_token = colon + 1;
|
char* to_token = colon + 1;
|
||||||
if (*to_token == '\0') {
|
IdentityMap parsed;
|
||||||
|
memset(&parsed, 0, sizeof(parsed));
|
||||||
|
if (identity_parse_from(from_token, is_group, &parsed.from, &parsed.from_hi) != 0) {
|
||||||
|
log_message(LOG_LEVEL_ERROR,
|
||||||
|
"%s could not resolve FROM '%s' in '%s' (a name must exist on the "
|
||||||
|
"source; use @N for a numeric id)",
|
||||||
|
optname, from_token, value);
|
||||||
free(list);
|
free(list);
|
||||||
log_message(LOG_LEVEL_ERROR, "%s rule 'FROM:' is missing the TO value (got '%s')", optname,
|
|
||||||
value);
|
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
int32_t from_id, to_id;
|
if (identity_parse_to(to_token, is_group, &parsed.to, &parsed.to_name) != 0) {
|
||||||
if (identity_resolve_token(from_token, is_group, &from_id) != 0 ||
|
log_message(LOG_LEVEL_ERROR, "%s could not parse TO '%s' in '%s'", optname, to_token, value);
|
||||||
identity_resolve_token(to_token, is_group, &to_id) != 0) {
|
|
||||||
free(list);
|
free(list);
|
||||||
log_message(LOG_LEVEL_ERROR,
|
|
||||||
"%s could not resolve '%s' (name must exist on the source; use "
|
|
||||||
"@N for a numeric id)",
|
|
||||||
optname, value);
|
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
if (identity_append_rule(is_group ? &config->groupmap : &config->usermap,
|
if (identity_append_rule(is_group ? &config->groupmap : &config->usermap,
|
||||||
is_group ? &config->groupmap_count : &config->usermap_count, from_id,
|
is_group ? &config->groupmap_count : &config->usermap_count,
|
||||||
to_id) != 0) {
|
&parsed) != 0) {
|
||||||
|
free(parsed.to_name);
|
||||||
free(list);
|
free(list);
|
||||||
log_message(LOG_LEVEL_ERROR, "%s has too many rules (max %d)", optname, MAX_IDENTITY_MAP);
|
log_message(LOG_LEVEL_ERROR, "%s has too many rules (max %d)", optname, MAX_IDENTITY_MAP);
|
||||||
return -1;
|
return -1;
|
||||||
@@ -366,6 +589,67 @@ static int identity_split_chown(const char* value, char** puser, char** pgroup)
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* --chown is rsync's shorthand for "--usermap=*:USER --groupmap=*:GROUP", so a
|
||||||
|
* name TO value must be resolved on the RECEIVER, not on the sender. Append the
|
||||||
|
* equivalent map rule (FROM matches every id). The numeric/'*' forms are stored
|
||||||
|
* numerically exactly as rsync's id_parse/user_to_uid would. Returns 0 on
|
||||||
|
* success, -1 on a malformed numeric token or allocation failure. */
|
||||||
|
static int identity_append_chown_rule(Config* config, bool is_group, const char* token) {
|
||||||
|
IdentityMap rule;
|
||||||
|
memset(&rule, 0, sizeof(rule));
|
||||||
|
rule.from = IDENTITY_MATCH_ANY;
|
||||||
|
rule.from_hi = IDENTITY_MATCH_ANY;
|
||||||
|
if (strcmp(token, "*") == 0) {
|
||||||
|
rule.to = IDENTITY_CURRENT;
|
||||||
|
} else if (identity_all_digits(token[0] == '@' ? token + 1 : token)) {
|
||||||
|
if (identity_resolve_token(token, is_group, &rule.to) != 0) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "--chown numeric id is out of range: %s", token);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
rule.to = 0;
|
||||||
|
rule.to_name = str_dup(token);
|
||||||
|
if (!rule.to_name)
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
if (identity_append_rule(is_group ? &config->groupmap : &config->usermap,
|
||||||
|
is_group ? &config->groupmap_count : &config->usermap_count,
|
||||||
|
&rule) != 0) {
|
||||||
|
free(rule.to_name);
|
||||||
|
log_message(LOG_LEVEL_ERROR, "--chown has too many rules (max %d)", MAX_IDENTITY_MAP);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Resolve/record one --chown side. The source-side numeric value is kept in
|
||||||
|
* chown_uid/chown_gid purely as a fallback (the appended map rule resolves the
|
||||||
|
* name on the receiver and wins); a name that does not exist on the sender is
|
||||||
|
* accepted and left to receiver-side resolution, matching rsync. */
|
||||||
|
static int identity_parse_chown_side(Config* config, bool is_group, const char* token) {
|
||||||
|
if (identity_append_chown_rule(config, is_group, token) != 0)
|
||||||
|
return -1;
|
||||||
|
bool numeric = identity_all_digits(token[0] == '@' ? token + 1 : token);
|
||||||
|
int32_t resolved;
|
||||||
|
if (identity_resolve_token(token, is_group, &resolved) == 0) {
|
||||||
|
if (is_group) {
|
||||||
|
config->chown_gid = resolved;
|
||||||
|
config->chown_gid_set = true;
|
||||||
|
} else {
|
||||||
|
config->chown_uid = resolved;
|
||||||
|
config->chown_uid_set = true;
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
if (numeric) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "--chown could not resolve numeric id '%s'", token);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
/* Unknown sender-side name: rsync accepts it and resolves it (or warns) on
|
||||||
|
* the receiver; do the same instead of failing the whole run. */
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
int identity_parse_chown(Config* config, const char* value) {
|
int identity_parse_chown(Config* config, const char* value) {
|
||||||
if (!config || !value || *value == '\0') {
|
if (!config || !value || *value == '\0') {
|
||||||
log_message(LOG_LEVEL_ERROR, "--chown requires a value (USER:GROUP, USER, or :GROUP)");
|
log_message(LOG_LEVEL_ERROR, "--chown requires a value (USER:GROUP, USER, or :GROUP)");
|
||||||
@@ -404,33 +688,19 @@ int identity_parse_chown(Config* config, const char* value) {
|
|||||||
if (*user == '\0') {
|
if (*user == '\0') {
|
||||||
log_message(LOG_LEVEL_ERROR, "--chown requires a user or group (got '%s')", value);
|
log_message(LOG_LEVEL_ERROR, "--chown requires a user or group (got '%s')", value);
|
||||||
ret = -1;
|
ret = -1;
|
||||||
} else if (identity_resolve_token(user, false, &config->chown_uid) != 0) {
|
} else if (identity_parse_chown_side(config, false, user) != 0) {
|
||||||
log_message(LOG_LEVEL_ERROR,
|
|
||||||
"--chown could not resolve user '%s' (use a name that exists "
|
|
||||||
"on the source, '*', or @N)",
|
|
||||||
value);
|
|
||||||
ret = -1;
|
ret = -1;
|
||||||
} else {
|
|
||||||
config->chown_uid_set = true;
|
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
/* --chown=USER:GROUP, --chown=:GROUP, --chown=USER: */
|
/* --chown=USER:GROUP, --chown=:GROUP, --chown=USER: */
|
||||||
if (*user != '\0') {
|
if (*user != '\0' && identity_parse_chown_side(config, false, user) != 0) {
|
||||||
if (identity_resolve_token(user, false, &config->chown_uid) != 0) {
|
|
||||||
log_message(LOG_LEVEL_ERROR, "--chown could not resolve user '%s'", value);
|
|
||||||
ret = -1;
|
ret = -1;
|
||||||
goto done;
|
goto done;
|
||||||
}
|
}
|
||||||
config->chown_uid_set = true;
|
if (*group != '\0' && identity_parse_chown_side(config, true, group) != 0) {
|
||||||
}
|
|
||||||
if (*group != '\0') {
|
|
||||||
if (identity_resolve_token(group, true, &config->chown_gid) != 0) {
|
|
||||||
log_message(LOG_LEVEL_ERROR, "--chown could not resolve group '%s'", value);
|
|
||||||
ret = -1;
|
ret = -1;
|
||||||
goto done;
|
goto done;
|
||||||
}
|
}
|
||||||
config->chown_gid_set = true;
|
|
||||||
}
|
|
||||||
if (!*user && !*group) {
|
if (!*user && !*group) {
|
||||||
log_message(LOG_LEVEL_ERROR, "--chown must set a user, a group, or both (got '%s')", value);
|
log_message(LOG_LEVEL_ERROR, "--chown must set a user, a group, or both (got '%s')", value);
|
||||||
ret = -1;
|
ret = -1;
|
||||||
@@ -567,9 +837,6 @@ int identity_parse_copy_as(Config* config, const char* value) {
|
|||||||
config->copy_as_set = true;
|
config->copy_as_set = true;
|
||||||
config->copy_as_uid = uid;
|
config->copy_as_uid = uid;
|
||||||
config->copy_as_gid = gid;
|
config->copy_as_gid = gid;
|
||||||
/* Ownership application needs the metadata path (the source uid/gid must be
|
|
||||||
* transmitted); imply it exactly like --chown/--usermap/--groupmap. */
|
|
||||||
config->use_metadata = true;
|
|
||||||
ret = 0;
|
ret = 0;
|
||||||
|
|
||||||
done:
|
done:
|
||||||
@@ -580,34 +847,120 @@ done:
|
|||||||
|
|
||||||
/* ---- Receiver-side ownership application ---- */
|
/* ---- Receiver-side ownership application ---- */
|
||||||
|
|
||||||
static bool identity_map_lookup(const IdentityMap* map, int count, int32_t source_id,
|
/* True when a map rule's FROM matcher accepts `id`. A sentinel FROM never
|
||||||
|
* carries a range. IDENTITY_MATCH_UNNAMED mirrors rsync's empty FROM: it
|
||||||
|
* matches only ids that have no name in the account database (rsync matches the
|
||||||
|
* sender's names; FastSync transmits numeric ids only, so it approximates this
|
||||||
|
* with the receiver's database -- documented in RSYNC_COMPAT.md). */
|
||||||
|
static bool identity_map_from_matches(const IdentityMap* map, int32_t id, bool is_group) {
|
||||||
|
if (map->from == IDENTITY_MATCH_ANY)
|
||||||
|
return true;
|
||||||
|
if (map->from == IDENTITY_MATCH_UNNAMED)
|
||||||
|
return is_group ? (getgrgid((gid_t)id) == NULL) : (getpwuid((uid_t)id) == NULL);
|
||||||
|
return id >= map->from && id <= map->from_hi;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* First matching rule wins. A rule whose TO is a receiver-side name resolves it
|
||||||
|
* against the receiver's account database here; an unresolvable TO name is
|
||||||
|
* skipped with a warning and the next rule is considered (rsync prints "Unknown
|
||||||
|
* --usermap name on receiver" and leaves the id unmapped rather than aborting). */
|
||||||
|
static bool identity_map_lookup(const IdentityMap* map, int count, int32_t source_id, bool is_group,
|
||||||
int32_t* out_to) {
|
int32_t* out_to) {
|
||||||
for (int i = 0; i < count; i++) {
|
for (int i = 0; i < count; i++) {
|
||||||
if (map[i].from == IDENTITY_MATCH_ANY || map[i].from == source_id) {
|
if (!identity_map_from_matches(&map[i], source_id, is_group))
|
||||||
|
continue;
|
||||||
|
if (map[i].to_name) {
|
||||||
|
if (is_group) {
|
||||||
|
struct group* gr = getgrnam(map[i].to_name);
|
||||||
|
if (!gr) {
|
||||||
|
log_message(LOG_LEVEL_WARNING, "Unknown --groupmap name on receiver: %s", map[i].to_name);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
*out_to = (int32_t)gr->gr_gid;
|
||||||
|
} else {
|
||||||
|
struct passwd* pw = getpwnam(map[i].to_name);
|
||||||
|
if (!pw) {
|
||||||
|
log_message(LOG_LEVEL_WARNING, "Unknown --usermap name on receiver: %s", map[i].to_name);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
*out_to = (int32_t)pw->pw_uid;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
*out_to = map[i].to;
|
*out_to = map[i].to;
|
||||||
|
}
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Resolve the owner side from the negotiated policy. Sets *out and returns
|
||||||
|
* true when an owner-affecting request is active (a usermap, --chown USER, or
|
||||||
|
* -o/--owner); returns false (leaving *out untouched) when the owner side is
|
||||||
|
* not requested, so callers can pass (uid_t)-1 to fchown and leave it as-is.
|
||||||
|
* --numeric-ids only changes the RESOLUTION (raw id instead of a name lookup);
|
||||||
|
* it never makes the side requested. */
|
||||||
|
static bool identity_resolve_owner(int32_t source_uid, uid_t* out) {
|
||||||
|
if (!(g_identity.chown_uid_set || g_identity.preserve_owner || g_identity.usermap_count > 0))
|
||||||
|
return false;
|
||||||
|
int32_t target;
|
||||||
|
if (identity_map_lookup(g_identity.usermap, g_identity.usermap_count, source_uid, false,
|
||||||
|
&target)) {
|
||||||
|
*out = target == IDENTITY_CURRENT ? geteuid() : (uid_t)target;
|
||||||
|
} else if (g_identity.chown_uid_set) {
|
||||||
|
*out = g_identity.chown_uid == IDENTITY_CURRENT ? geteuid() : (uid_t)g_identity.chown_uid;
|
||||||
|
} else if (g_identity.numeric_ids) {
|
||||||
|
*out = (uid_t)source_uid;
|
||||||
|
} else {
|
||||||
|
/* Best-effort name mapping against the receiver's own database. When the
|
||||||
|
* transmitted (numeric) id has no name here, fall back to the raw numeric id
|
||||||
|
* so -o still preserves the source owner. */
|
||||||
|
struct passwd* pw = getpwuid((uid_t)source_uid);
|
||||||
|
if (pw) {
|
||||||
|
const struct passwd* mapped = getpwnam(pw->pw_name);
|
||||||
|
*out = mapped ? mapped->pw_uid : (uid_t)source_uid;
|
||||||
|
} else {
|
||||||
|
*out = (uid_t)source_uid;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Group-side counterpart of identity_resolve_owner(). */
|
||||||
|
static bool identity_resolve_group(int32_t source_gid, gid_t* out) {
|
||||||
|
if (!(g_identity.chown_gid_set || g_identity.preserve_group || g_identity.groupmap_count > 0))
|
||||||
|
return false;
|
||||||
|
int32_t target;
|
||||||
|
if (identity_map_lookup(g_identity.groupmap, g_identity.groupmap_count, source_gid, true,
|
||||||
|
&target)) {
|
||||||
|
*out = target == IDENTITY_CURRENT ? getegid() : (gid_t)target;
|
||||||
|
} else if (g_identity.chown_gid_set) {
|
||||||
|
*out = g_identity.chown_gid == IDENTITY_CURRENT ? getegid() : (gid_t)g_identity.chown_gid;
|
||||||
|
} else if (g_identity.numeric_ids) {
|
||||||
|
*out = (gid_t)source_gid;
|
||||||
|
} else {
|
||||||
|
struct group* gr = getgrgid((gid_t)source_gid);
|
||||||
|
if (gr) {
|
||||||
|
const struct group* mapped = getgrnam(gr->gr_name);
|
||||||
|
*out = mapped ? mapped->gr_gid : (gid_t)source_gid;
|
||||||
|
} else {
|
||||||
|
*out = (gid_t)source_gid;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
/* Resolve the target ownership from the negotiated policy against the entry's
|
/* Resolve the target ownership from the negotiated policy against the entry's
|
||||||
* current stat. Shared by the fd (regular file) and no-follow (symlink) apply
|
* current stat. Shared by the fd (regular file) and no-follow (symlink) apply
|
||||||
* paths. Returns false when no side is to be changed. */
|
* paths. Returns false when no side is to be changed. */
|
||||||
static bool identity_resolve_targets(const struct stat* st, int32_t source_uid, int32_t source_gid,
|
static bool identity_resolve_targets(const struct stat* st, int32_t source_uid, int32_t source_gid,
|
||||||
uid_t* out_uid, gid_t* out_gid) {
|
uid_t* out_uid, gid_t* out_gid) {
|
||||||
bool set_uid = false;
|
|
||||||
bool set_gid = false;
|
|
||||||
uid_t uid = 0;
|
|
||||||
gid_t gid = 0;
|
|
||||||
|
|
||||||
/* --copy-as (P7 Wave E) has the highest priority: it forces BOTH the owner
|
/* --copy-as (P7 Wave E) has the highest priority: it forces BOTH the owner
|
||||||
* and group of every written entry to the requested ids, beating usermap /
|
* and group of every written entry to the requested ids, beating usermap /
|
||||||
* groupmap / --chown / --numeric-ids and the best-effort name lookup. Only
|
* groupmap / --chown / --numeric-ids and the best-effort name lookup. Only
|
||||||
* skip when the entry already carries exactly those ids. */
|
* skip when the entry already carries exactly those ids. */
|
||||||
if (g_identity.copy_as_set) {
|
if (g_identity.copy_as_set) {
|
||||||
uid = (uid_t)g_identity.copy_as_uid;
|
uid_t uid = (uid_t)g_identity.copy_as_uid;
|
||||||
gid = (gid_t)g_identity.copy_as_gid;
|
gid_t gid = (gid_t)g_identity.copy_as_gid;
|
||||||
if (st->st_uid == uid && st->st_gid == gid)
|
if (st->st_uid == uid && st->st_gid == gid)
|
||||||
return false;
|
return false;
|
||||||
*out_uid = uid;
|
*out_uid = uid;
|
||||||
@@ -615,67 +968,52 @@ static bool identity_resolve_targets(const struct stat* st, int32_t source_uid,
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
int32_t target;
|
/* Each side is resolved independently: -o/-g and the explicit identity flags
|
||||||
if (identity_map_lookup(g_identity.usermap, g_identity.usermap_count, source_uid, &target)) {
|
* request the owner/group respectively, and a side that is NOT requested must
|
||||||
uid = target == IDENTITY_CURRENT ? geteuid() : (uid_t)target;
|
* be left exactly as it is (`-1` to fchown on that side). This is what lets
|
||||||
set_uid = true;
|
* plain -g change only the group, or -o only the owner. */
|
||||||
} else if (g_identity.chown_uid_set) {
|
uid_t uid = (uid_t)-1;
|
||||||
uid = g_identity.chown_uid == IDENTITY_CURRENT ? geteuid() : (uid_t)g_identity.chown_uid;
|
gid_t gid = (gid_t)-1;
|
||||||
set_uid = true;
|
bool owner_requested = identity_resolve_owner(source_uid, &uid);
|
||||||
} else if (g_identity.numeric_ids) {
|
bool group_requested = identity_resolve_group(source_gid, &gid);
|
||||||
uid = (uid_t)source_uid;
|
if (!owner_requested && !group_requested)
|
||||||
set_uid = true;
|
|
||||||
} else {
|
|
||||||
/* Best-effort name mapping against the receiver's own database: if the
|
|
||||||
* transmitted (numeric) id resolves to a name present on this machine,
|
|
||||||
* re-resolve it. On a shared-account host this is the identity operation;
|
|
||||||
* when the id has no name here, the user side is left alone. */
|
|
||||||
struct passwd* pw = getpwuid((uid_t)source_uid);
|
|
||||||
if (pw) {
|
|
||||||
const struct passwd* mapped = getpwnam(pw->pw_name);
|
|
||||||
if (mapped) {
|
|
||||||
uid = mapped->pw_uid;
|
|
||||||
set_uid = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (identity_map_lookup(g_identity.groupmap, g_identity.groupmap_count, source_gid, &target)) {
|
|
||||||
gid = target == IDENTITY_CURRENT ? getegid() : (gid_t)target;
|
|
||||||
set_gid = true;
|
|
||||||
} else if (g_identity.chown_gid_set) {
|
|
||||||
gid = g_identity.chown_gid == IDENTITY_CURRENT ? getegid() : (gid_t)g_identity.chown_gid;
|
|
||||||
set_gid = true;
|
|
||||||
} else if (g_identity.numeric_ids) {
|
|
||||||
gid = (gid_t)source_gid;
|
|
||||||
set_gid = true;
|
|
||||||
} else {
|
|
||||||
struct group* gr = getgrgid((gid_t)source_gid);
|
|
||||||
if (gr) {
|
|
||||||
const struct group* mapped = getgrnam(gr->gr_name);
|
|
||||||
if (mapped) {
|
|
||||||
gid = mapped->gr_gid;
|
|
||||||
set_gid = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!set_uid && !set_gid)
|
|
||||||
return false;
|
return false;
|
||||||
/* An unset side keeps the file's current id so the other side can change. */
|
|
||||||
if (!set_uid)
|
/* Only change ownership when a requested side actually differs (avoid
|
||||||
uid = st->st_uid;
|
* needless syscalls and any chance of clearing setuid/setgid on an
|
||||||
if (!set_gid)
|
* already-correct entry). */
|
||||||
gid = st->st_gid;
|
bool changed = (owner_requested && uid != st->st_uid) || (group_requested && gid != st->st_gid);
|
||||||
/* Only change ownership when the target differs (avoid needless syscalls and
|
if (!changed)
|
||||||
* any chance of clearing setuid/setgid on an already-correct entry). */
|
|
||||||
if (st->st_uid == uid && st->st_gid == gid)
|
|
||||||
return false;
|
return false;
|
||||||
*out_uid = uid;
|
*out_uid = uid;
|
||||||
*out_gid = gid;
|
*out_gid = gid;
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* --fake-super storage resolution: the receiver records the ownership it WOULD
|
||||||
|
* have applied. A requested side uses the resolved mapping (--copy-as /
|
||||||
|
* usermap / --chown / -o/-g, with --numeric-ids as the raw-id modifier); a side
|
||||||
|
* that was not requested keeps the source's own id, so a plain --fake-super run
|
||||||
|
* records the source owner untouched. */
|
||||||
|
void identity_resolve_storage_ids(int32_t source_uid, int32_t source_gid, uint32_t* out_uid,
|
||||||
|
uint32_t* out_gid) {
|
||||||
|
if (g_identity.copy_as_set) {
|
||||||
|
*out_uid = (uint32_t)g_identity.copy_as_uid;
|
||||||
|
*out_gid = (uint32_t)g_identity.copy_as_gid;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
uid_t uid = (uid_t)source_uid;
|
||||||
|
gid_t gid = (gid_t)source_gid;
|
||||||
|
uid_t resolved_uid;
|
||||||
|
gid_t resolved_gid;
|
||||||
|
if (identity_resolve_owner(source_uid, &resolved_uid))
|
||||||
|
uid = resolved_uid;
|
||||||
|
if (identity_resolve_group(source_gid, &resolved_gid))
|
||||||
|
gid = resolved_gid;
|
||||||
|
*out_uid = (uint32_t)uid;
|
||||||
|
*out_gid = (uint32_t)gid;
|
||||||
|
}
|
||||||
|
|
||||||
static void identity_log_chown_failure(const char* what, uid_t uid, gid_t gid) {
|
static void identity_log_chown_failure(const char* what, uid_t uid, gid_t gid) {
|
||||||
/* EPERM/EACCES are expected when the receiver is not privileged (e.g. the CI
|
/* EPERM/EACCES are expected when the receiver is not privileged (e.g. the CI
|
||||||
* `nobody` user): warn and continue, never abort the transfer. Any other
|
* `nobody` user): warn and continue, never abort the transfer. Any other
|
||||||
@@ -710,8 +1048,12 @@ bool identity_apply_ownership(int fd, int32_t source_uid, int32_t source_gid) {
|
|||||||
/* Ownership application is OFF unless the client requested an identity flag.
|
/* Ownership application is OFF unless the client requested an identity flag.
|
||||||
* This is the controlled gate: a default (or plain -M) transfer never changes
|
* This is the controlled gate: a default (or plain -M) transfer never changes
|
||||||
* ownership, byte-for-byte preserving FastSync's existing behavior. --no-super
|
* ownership, byte-for-byte preserving FastSync's existing behavior. --no-super
|
||||||
* additionally forbids it even when the receiver is root. */
|
* additionally forbids it even when the receiver is root. --fake-super never
|
||||||
if (!identity_active_enabled() || !privilege_super_permitted() || fd < 0)
|
* performs a REAL chown: that would defeat the point of the flag (record the
|
||||||
|
* source ownership on an unprivileged receiver for a later privileged
|
||||||
|
* restore); the resolved ownership is stored in the reserved xattr instead by
|
||||||
|
* fake_super_store_fd(). */
|
||||||
|
if (!identity_active_enabled() || g_identity.fake_super || !privilege_super_permitted() || fd < 0)
|
||||||
return true;
|
return true;
|
||||||
struct stat st;
|
struct stat st;
|
||||||
if (fstat(fd, &st) != 0)
|
if (fstat(fd, &st) != 0)
|
||||||
@@ -731,7 +1073,8 @@ bool identity_apply_ownership(int fd, int32_t source_uid, int32_t source_gid) {
|
|||||||
|
|
||||||
bool identity_apply_ownership_link(int parent_fd, const char* leaf, int32_t source_uid,
|
bool identity_apply_ownership_link(int parent_fd, const char* leaf, int32_t source_uid,
|
||||||
int32_t source_gid) {
|
int32_t source_gid) {
|
||||||
if (!identity_active_enabled() || !privilege_super_permitted() || parent_fd < 0 || !leaf)
|
if (!identity_active_enabled() || g_identity.fake_super || !privilege_super_permitted() ||
|
||||||
|
parent_fd < 0 || !leaf)
|
||||||
return true;
|
return true;
|
||||||
struct stat st;
|
struct stat st;
|
||||||
if (fstatat(parent_fd, leaf, &st, AT_SYMLINK_NOFOLLOW) != 0)
|
if (fstatat(parent_fd, leaf, &st, AT_SYMLINK_NOFOLLOW) != 0)
|
||||||
|
|||||||
+35
-6
@@ -80,16 +80,37 @@ void identity_clear_active(void);
|
|||||||
* snapshot. Ownership stays OFF ("do not apply") for every transfer that
|
* snapshot. Ownership stays OFF ("do not apply") for every transfer that
|
||||||
* requests none of them, preserving FastSync's existing behavior. --super /
|
* requests none of them, preserving FastSync's existing behavior. --super /
|
||||||
* --no-super alone does NOT enable ownership; an explicit identity flag
|
* --no-super alone does NOT enable ownership; an explicit identity flag
|
||||||
* (--numeric-ids / --chown / --usermap / --groupmap / --copy-as) is required. */
|
* (--numeric-ids / --chown / --usermap / --groupmap / --copy-as) or a
|
||||||
|
* preserve-source -o/--owner / -g/--group request is required. */
|
||||||
bool identity_active_enabled(void);
|
bool identity_active_enabled(void);
|
||||||
|
|
||||||
/* Pure, config-only predicate: true when the client requested ANY
|
/* Per-side predicates over the ACTIVE per-connection snapshot (call
|
||||||
* client-chosen ownership or super-user activity (--numeric-ids, --chown,
|
* identity_set_active() first). They mirror the owner_requested /
|
||||||
* --usermap/--groupmap, --copy-as, --fake-super, or an explicit --super). Used
|
* group_requested conditions inside identity_resolve_targets() exactly, so
|
||||||
* by the daemon module gate to decide whether a module's per-module opt-in is
|
* callers that must apply only one side (e.g. the --fake-super owner replay)
|
||||||
* required; it never reads the per-connection snapshot. */
|
* can pass (uid_t)-1 / (gid_t)-1 for the side that was NOT requested and leave
|
||||||
|
* it untouched. The owner side is requested by --copy-as, --chown USER,
|
||||||
|
* --numeric-ids, -o/--owner, or a non-empty --usermap; the group side by
|
||||||
|
* --copy-as, --chown :GROUP, --numeric-ids, -g/--group, or a non-empty
|
||||||
|
* --groupmap. */
|
||||||
|
bool identity_owner_requested(void);
|
||||||
|
bool identity_group_requested(void);
|
||||||
|
|
||||||
|
/* Pure, config-only predicate: true when the client requested ANY client-chosen
|
||||||
|
* ownership or super-user activity (--numeric-ids, --chown, --usermap/--groupmap,
|
||||||
|
* --copy-as, --fake-super, an explicit --super, or a preserve-source -o/-g).
|
||||||
|
* General awareness only; the daemon module gate uses the narrower
|
||||||
|
* identity_explicit_ownership_requested() below. Never reads the snapshot. */
|
||||||
bool identity_ownership_requested(const Config* config);
|
bool identity_ownership_requested(const Config* config);
|
||||||
|
|
||||||
|
/* Pure, config-only predicate for the narrow set that lets the CLIENT choose an
|
||||||
|
* arbitrary owner/group: --numeric-ids, --chown, --usermap/--groupmap,
|
||||||
|
* --copy-as, --fake-super, or an explicit --super. Deliberately EXCLUDES a
|
||||||
|
* plain -o/--owner / -g/--group (or -a) preserve-source request, which the
|
||||||
|
* daemon gate handles by forcing super-user ownership activity off rather than
|
||||||
|
* refusing the whole transfer. Never reads the snapshot. */
|
||||||
|
bool identity_explicit_ownership_requested(const Config* config);
|
||||||
|
|
||||||
/* Apply the negotiated ownership to an already-written file descriptor.
|
/* Apply the negotiated ownership to an already-written file descriptor.
|
||||||
* source_uid/source_gid are the transmitted numeric ids. Resolution order:
|
* source_uid/source_gid are the transmitted numeric ids. Resolution order:
|
||||||
* --copy-as (highest priority, forces both ids), then a matching
|
* --copy-as (highest priority, forces both ids), then a matching
|
||||||
@@ -106,6 +127,14 @@ bool identity_ownership_requested(const Config* config);
|
|||||||
* is active returns true. */
|
* is active returns true. */
|
||||||
bool identity_apply_ownership(int fd, int32_t source_uid, int32_t source_gid);
|
bool identity_apply_ownership(int fd, int32_t source_uid, int32_t source_gid);
|
||||||
|
|
||||||
|
/* Resolve the ownership that --fake-super should RECORD in the reserved xattr
|
||||||
|
* (rather than chown for real). A requested side (--copy-as / usermap /
|
||||||
|
* --chown / -o / -g, with --numeric-ids as the raw-id modifier) yields the
|
||||||
|
* resolved target; a side that was not requested keeps the transmitted source
|
||||||
|
* id. Must be called after identity_set_active(). */
|
||||||
|
void identity_resolve_storage_ids(int32_t source_uid, int32_t source_gid, uint32_t* out_uid,
|
||||||
|
uint32_t* out_gid);
|
||||||
|
|
||||||
/* P7 Wave D: the no-follow (symlink) counterpart. Resolves the same
|
/* P7 Wave D: the no-follow (symlink) counterpart. Resolves the same
|
||||||
* usermap/groupmap/chown/numeric-ids/copy-as policy but applies it with
|
* usermap/groupmap/chown/numeric-ids/copy-as policy but applies it with
|
||||||
* fchownat(..., AT_SYMLINK_NOFOLLOW) so a symlink's own ownership is changed
|
* fchownat(..., AT_SYMLINK_NOFOLLOW) so a symlink's own ownership is changed
|
||||||
|
|||||||
+98
-41
@@ -209,22 +209,58 @@ FileMetadata* metadata_receive(int file_descriptor, int* ok) {
|
|||||||
return m;
|
return m;
|
||||||
}
|
}
|
||||||
|
|
||||||
static mode_t metadata_mode(const FileMetadata* metadata, mode_t current_mode,
|
bool metadata_mode_for_policy(mode_t source_mode, mode_t current_mode, FileAttrPolicy policy,
|
||||||
bool preserve_executability) {
|
mode_t* out_mode) {
|
||||||
const mode_t execute_bits = S_IXUSR | S_IXGRP | S_IXOTH;
|
const mode_t execute_bits = S_IXUSR | S_IXGRP | S_IXOTH;
|
||||||
if (preserve_executability)
|
if (policy.perms) {
|
||||||
return (current_mode & 0777 & ~execute_bits) | (metadata->mode & execute_bits);
|
/* rsync --perms copies the source's permission and special bits exactly,
|
||||||
return metadata->mode & 0777 & ~(S_IWGRP | S_IWOTH);
|
* including group/other write and setuid/setgid/sticky. The kernel may
|
||||||
|
* still clear setgid when the receiver is not in the file's group; the
|
||||||
|
* caller logs a failed chmod rather than silently masking the bits here. */
|
||||||
|
*out_mode = source_mode & (mode_t)(S_ISUID | S_ISGID | S_ISVTX | 0777);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (policy.executability) {
|
||||||
|
/* -E/--executability (rsync 3.4 rule): do NOT copy the source's execute
|
||||||
|
* bits per class. If the source is executable at all, derive the execute
|
||||||
|
* bits from the DESTINATION's own read bits (so a class that can read may
|
||||||
|
* execute); otherwise clear every execute bit. This runs on the
|
||||||
|
* destination-derived base (pre-existing dest mode, or source&~umask for a
|
||||||
|
* new file), and leaves the special bits untouched. --perms wins when both
|
||||||
|
* are set (handled above). */
|
||||||
|
mode_t base = current_mode & (mode_t)(S_ISUID | S_ISGID | S_ISVTX | 0777);
|
||||||
|
if (source_mode & 0111)
|
||||||
|
*out_mode = base | ((base & 0444) >> 2);
|
||||||
|
else
|
||||||
|
*out_mode = base & ~execute_bits;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
/* Neither requested: no source mode is applied at all. */
|
||||||
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
void file_restore_metadata(const char* path, const FileMetadata* metadata,
|
FileAttrPolicy file_attr_policy_from_config(const Config* config) {
|
||||||
bool preserve_executability) {
|
FileAttrPolicy policy = {false, false, false, false};
|
||||||
|
if (config) {
|
||||||
|
policy.perms = config->preserve_perms;
|
||||||
|
policy.times = config->preserve_times;
|
||||||
|
policy.atimes = config->preserve_atimes;
|
||||||
|
policy.executability = config->use_executability;
|
||||||
|
}
|
||||||
|
return policy;
|
||||||
|
}
|
||||||
|
|
||||||
|
void file_restore_metadata(const char* path, const FileMetadata* metadata, FileAttrPolicy policy) {
|
||||||
if (metadata == NULL)
|
if (metadata == NULL)
|
||||||
return;
|
return;
|
||||||
|
bool apply_mode = false;
|
||||||
|
mode_t safe_mode = 0;
|
||||||
|
if (policy.perms || policy.executability) {
|
||||||
struct stat current;
|
struct stat current;
|
||||||
mode_t current_mode = stat(path, ¤t) == 0 ? current.st_mode : 0;
|
mode_t current_mode = stat(path, ¤t) == 0 ? current.st_mode : 0;
|
||||||
mode_t safe_mode = metadata_mode(metadata, current_mode, preserve_executability);
|
apply_mode = metadata_mode_for_policy(metadata->mode, current_mode, policy, &safe_mode);
|
||||||
if (chmod(path, safe_mode) != 0) {
|
}
|
||||||
|
if (apply_mode && chmod(path, safe_mode) != 0) {
|
||||||
char* escaped_path = output_escape(path, log_get_8_bit_output());
|
char* escaped_path = output_escape(path, log_get_8_bit_output());
|
||||||
log_message(LOG_LEVEL_WARNING, "Failed to chmod %s: %s",
|
log_message(LOG_LEVEL_WARNING, "Failed to chmod %s: %s",
|
||||||
escaped_path ? escaped_path : "<allocation failed>", strerror(errno));
|
escaped_path ? escaped_path : "<allocation failed>", strerror(errno));
|
||||||
@@ -232,31 +268,34 @@ void file_restore_metadata(const char* path, const FileMetadata* metadata,
|
|||||||
}
|
}
|
||||||
/* Never apply client-supplied ownership. The descriptor API below is the
|
/* Never apply client-supplied ownership. The descriptor API below is the
|
||||||
receiver write path; retain this legacy API only for compatibility. */
|
receiver write path; retain this legacy API only for compatibility. */
|
||||||
struct timespec times[2];
|
if (policy.times || (policy.atimes && metadata->atime_valid)) {
|
||||||
times[0].tv_sec = 0;
|
struct timespec times[2] = {{.tv_sec = 0, .tv_nsec = UTIME_OMIT},
|
||||||
times[0].tv_nsec = UTIME_OMIT;
|
{.tv_sec = 0, .tv_nsec = UTIME_OMIT}};
|
||||||
|
if (policy.times) {
|
||||||
times[1].tv_sec = metadata->mtime_sec;
|
times[1].tv_sec = metadata->mtime_sec;
|
||||||
times[1].tv_nsec = metadata->mtime_nsec;
|
times[1].tv_nsec = metadata->mtime_nsec;
|
||||||
if (metadata->atime_valid) {
|
}
|
||||||
|
if (policy.atimes && metadata->atime_valid) {
|
||||||
times[0].tv_sec = metadata->atime_sec;
|
times[0].tv_sec = metadata->atime_sec;
|
||||||
times[0].tv_nsec = metadata->atime_nsec;
|
times[0].tv_nsec = metadata->atime_nsec;
|
||||||
}
|
}
|
||||||
if (metadata->crtime_valid) {
|
|
||||||
log_message(LOG_LEVEL_DEBUG,
|
|
||||||
"crtime (birth time) %lld.%09ld transmitted for %s but not applied: no portable "
|
|
||||||
"setter exists",
|
|
||||||
(long long)metadata->crtime_sec, metadata->crtime_nsec, path);
|
|
||||||
}
|
|
||||||
if (utimensat(AT_FDCWD, path, times, 0) != 0) {
|
if (utimensat(AT_FDCWD, path, times, 0) != 0) {
|
||||||
char* escaped_path = output_escape(path, log_get_8_bit_output());
|
char* escaped_path = output_escape(path, log_get_8_bit_output());
|
||||||
log_message(LOG_LEVEL_WARNING, "Failed to set timestamps on %s: %s",
|
log_message(LOG_LEVEL_WARNING, "Failed to set timestamps on %s: %s",
|
||||||
escaped_path ? escaped_path : "<allocation failed>", strerror(errno));
|
escaped_path ? escaped_path : "<allocation failed>", strerror(errno));
|
||||||
free(escaped_path);
|
free(escaped_path);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
if (metadata->crtime_valid) {
|
||||||
|
log_message(LOG_LEVEL_DEBUG,
|
||||||
|
"crtime (birth time) %lld.%09ld transmitted for %s but not applied: no portable "
|
||||||
|
"setter exists",
|
||||||
|
(long long)metadata->crtime_sec, metadata->crtime_nsec, path);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
bool file_restore_symlink_metadata(const char* path, const FileMetadata* metadata,
|
bool file_restore_symlink_metadata(const char* path, const FileMetadata* metadata,
|
||||||
bool omit_link_times) {
|
FileAttrPolicy policy, bool omit_link_times) {
|
||||||
if (path == NULL || metadata == NULL)
|
if (path == NULL || metadata == NULL)
|
||||||
return !identity_copy_as_active();
|
return !identity_copy_as_active();
|
||||||
char* leaf = NULL;
|
char* leaf = NULL;
|
||||||
@@ -269,18 +308,25 @@ bool file_restore_symlink_metadata(const char* path, const FileMetadata* metadat
|
|||||||
best-effort. */
|
best-effort. */
|
||||||
bool owned = identity_apply_ownership_link(parent_fd, leaf, (int32_t)metadata->uid,
|
bool owned = identity_apply_ownership_link(parent_fd, leaf, (int32_t)metadata->uid,
|
||||||
(int32_t)metadata->gid);
|
(int32_t)metadata->gid);
|
||||||
/* Symlink mode: not settable on Linux (fchmodat AT_SYMLINK_NOFOLLOW returns
|
/* Symlink mode: only when -p is in effect. It is not settable on Linux
|
||||||
EOPNOTSUPP/ENOTSUP); attempt it for platforms that support it and quietly
|
(fchmodat AT_SYMLINK_NOFOLLOW returns EOPNOTSUPP/ENOTSUP); attempt it for
|
||||||
ignore the unsupported case so the transfer never fails over it. */
|
platforms that support it and quietly ignore the unsupported case so the
|
||||||
mode_t link_mode = metadata->mode & 0777;
|
transfer never fails over it. */
|
||||||
|
if (policy.perms) {
|
||||||
|
mode_t link_mode = metadata->mode & (mode_t)(S_ISUID | S_ISGID | S_ISVTX | 0777);
|
||||||
if (fchmodat(parent_fd, leaf, link_mode, AT_SYMLINK_NOFOLLOW) != 0 && errno != EOPNOTSUPP &&
|
if (fchmodat(parent_fd, leaf, link_mode, AT_SYMLINK_NOFOLLOW) != 0 && errno != EOPNOTSUPP &&
|
||||||
errno != ENOTSUP && errno != ENOSYS) {
|
errno != ENOTSUP && errno != ENOSYS) {
|
||||||
log_message(LOG_LEVEL_DEBUG, "Could not set symlink mode on %s: %s", path, strerror(errno));
|
log_message(LOG_LEVEL_DEBUG, "Could not set symlink mode on %s: %s", path, strerror(errno));
|
||||||
}
|
}
|
||||||
if (!omit_link_times) {
|
}
|
||||||
|
if (!omit_link_times && (policy.times || (policy.atimes && metadata->atime_valid))) {
|
||||||
struct timespec times[2] = {{.tv_sec = 0, .tv_nsec = UTIME_OMIT},
|
struct timespec times[2] = {{.tv_sec = 0, .tv_nsec = UTIME_OMIT},
|
||||||
{.tv_sec = metadata->mtime_sec, .tv_nsec = metadata->mtime_nsec}};
|
{.tv_sec = 0, .tv_nsec = UTIME_OMIT}};
|
||||||
if (metadata->atime_valid) {
|
if (policy.times) {
|
||||||
|
times[1].tv_sec = metadata->mtime_sec;
|
||||||
|
times[1].tv_nsec = metadata->mtime_nsec;
|
||||||
|
}
|
||||||
|
if (policy.atimes && metadata->atime_valid) {
|
||||||
times[0].tv_sec = metadata->atime_sec;
|
times[0].tv_sec = metadata->atime_sec;
|
||||||
times[0].tv_nsec = metadata->atime_nsec;
|
times[0].tv_nsec = metadata->atime_nsec;
|
||||||
}
|
}
|
||||||
@@ -296,19 +342,13 @@ bool file_restore_symlink_metadata(const char* path, const FileMetadata* metadat
|
|||||||
return owned;
|
return owned;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool file_restore_metadata_fd(int fd, const FileMetadata* metadata, bool preserve_executability) {
|
bool file_restore_metadata_fd(int fd, const FileMetadata* metadata, FileAttrPolicy policy) {
|
||||||
if (fd < 0 || metadata == NULL)
|
if (fd < 0 || metadata == NULL)
|
||||||
return metadata == NULL;
|
return metadata == NULL;
|
||||||
bool ok = true;
|
bool ok = true;
|
||||||
struct stat current;
|
|
||||||
if (fstat(fd, ¤t) != 0)
|
|
||||||
return false;
|
|
||||||
mode_t safe_mode = metadata_mode(metadata, current.st_mode, preserve_executability);
|
|
||||||
if (fchmod(fd, safe_mode) != 0)
|
|
||||||
ok = false;
|
|
||||||
/* Client uid/gid values are deliberately not authoritative UNLESS the client
|
/* Client uid/gid values are deliberately not authoritative UNLESS the client
|
||||||
explicitly opted in with an identity flag (--numeric-ids / --usermap /
|
explicitly opted in with an identity flag (--numeric-ids / --usermap /
|
||||||
--groupmap / --chown). identity_apply_ownership is the controlled,
|
--groupmap / --chown / -o/-g). identity_apply_ownership is the controlled,
|
||||||
privilege-gated path: it consults the negotiated policy, resolves the
|
privilege-gated path: it consults the negotiated policy, resolves the
|
||||||
target ids, and applies them via an fd-relative fchown() that is confined
|
target ids, and applies them via an fd-relative fchown() that is confined
|
||||||
to the just-written file (EPERM/EACCES are logged, never fatal) -- EXCEPT
|
to the just-written file (EPERM/EACCES are logged, never fatal) -- EXCEPT
|
||||||
@@ -316,14 +356,19 @@ bool file_restore_metadata_fd(int fd, const FileMetadata* metadata, bool preserv
|
|||||||
marks this entry as failed instead of reporting a wrong-owner write as
|
marks this entry as failed instead of reporting a wrong-owner write as
|
||||||
success. With no identity flag set it is a no-op, so a default or plain -M
|
success. With no identity flag set it is a no-op, so a default or plain -M
|
||||||
transfer keeps FastSync's existing behavior of never applying client
|
transfer keeps FastSync's existing behavior of never applying client
|
||||||
ownership. */
|
ownership. Ownership runs BEFORE the mode because a chown clears
|
||||||
|
setuid/setgid; rsync likewise chowns first and then restores the source
|
||||||
|
mode (including its special bits). */
|
||||||
if (!identity_apply_ownership(fd, (int32_t)metadata->uid, (int32_t)metadata->gid))
|
if (!identity_apply_ownership(fd, (int32_t)metadata->uid, (int32_t)metadata->gid))
|
||||||
ok = false;
|
ok = false;
|
||||||
struct timespec times[2] = {{.tv_sec = 0, .tv_nsec = UTIME_OMIT},
|
if (policy.perms || policy.executability) {
|
||||||
{.tv_sec = metadata->mtime_sec, .tv_nsec = metadata->mtime_nsec}};
|
struct stat current;
|
||||||
if (metadata->atime_valid) {
|
if (fstat(fd, ¤t) != 0)
|
||||||
times[0].tv_sec = metadata->atime_sec;
|
return false;
|
||||||
times[0].tv_nsec = metadata->atime_nsec;
|
mode_t safe_mode = 0;
|
||||||
|
bool apply_mode = metadata_mode_for_policy(metadata->mode, current.st_mode, policy, &safe_mode);
|
||||||
|
if (apply_mode && fchmod(fd, safe_mode) != 0)
|
||||||
|
ok = false;
|
||||||
}
|
}
|
||||||
/* --crtimes captures and transmits the source birth time, but there is no
|
/* --crtimes captures and transmits the source birth time, but there is no
|
||||||
* portable way to set a birth time (utimensat can only set atime/mtime), so
|
* portable way to set a birth time (utimensat can only set atime/mtime), so
|
||||||
@@ -335,7 +380,19 @@ bool file_restore_metadata_fd(int fd, const FileMetadata* metadata, bool preserv
|
|||||||
"crtime (birth time) %lld.%09ld transmitted but not applied: no portable setter",
|
"crtime (birth time) %lld.%09ld transmitted but not applied: no portable setter",
|
||||||
(long long)metadata->crtime_sec, metadata->crtime_nsec);
|
(long long)metadata->crtime_sec, metadata->crtime_nsec);
|
||||||
}
|
}
|
||||||
|
if (policy.times || (policy.atimes && metadata->atime_valid)) {
|
||||||
|
struct timespec times[2] = {{.tv_sec = 0, .tv_nsec = UTIME_OMIT},
|
||||||
|
{.tv_sec = 0, .tv_nsec = UTIME_OMIT}};
|
||||||
|
if (policy.times) {
|
||||||
|
times[1].tv_sec = metadata->mtime_sec;
|
||||||
|
times[1].tv_nsec = metadata->mtime_nsec;
|
||||||
|
}
|
||||||
|
if (policy.atimes && metadata->atime_valid) {
|
||||||
|
times[0].tv_sec = metadata->atime_sec;
|
||||||
|
times[0].tv_nsec = metadata->atime_nsec;
|
||||||
|
}
|
||||||
if (futimens(fd, times) != 0)
|
if (futimens(fd, times) != 0)
|
||||||
ok = false;
|
ok = false;
|
||||||
|
}
|
||||||
return ok;
|
return ok;
|
||||||
}
|
}
|
||||||
|
|||||||
+20
-7
@@ -2,6 +2,7 @@
|
|||||||
#define METADATA_H
|
#define METADATA_H
|
||||||
|
|
||||||
#include "file.h"
|
#include "file.h"
|
||||||
|
#include "file_attr.h"
|
||||||
#include <stdbool.h>
|
#include <stdbool.h>
|
||||||
#include <stddef.h>
|
#include <stddef.h>
|
||||||
#include <stdint.h>
|
#include <stdint.h>
|
||||||
@@ -49,19 +50,31 @@ void metadata_to_buf(char** buf, const FileMetadata* m);
|
|||||||
FileMetadata* metadata_from_buf(const uint8_t* buf, size_t len);
|
FileMetadata* metadata_from_buf(const uint8_t* buf, size_t len);
|
||||||
bool metadata_send(int file_descriptor, const FileMetadata* m);
|
bool metadata_send(int file_descriptor, const FileMetadata* m);
|
||||||
FileMetadata* metadata_receive(int file_descriptor, int* ok);
|
FileMetadata* metadata_receive(int file_descriptor, int* ok);
|
||||||
void file_restore_metadata(const char* path, const FileMetadata* metadata,
|
void file_restore_metadata(const char* path, const FileMetadata* metadata, FileAttrPolicy policy);
|
||||||
bool preserve_executability);
|
bool file_restore_metadata_fd(int fd, const FileMetadata* metadata, FileAttrPolicy policy);
|
||||||
bool file_restore_metadata_fd(int fd, const FileMetadata* metadata, bool preserve_executability);
|
|
||||||
|
/* Shared mode-policy helper: the single source of truth for the receiver's
|
||||||
|
* mode rule. Given a source mode and the destination's CURRENT mode, returns
|
||||||
|
* true and stores the exact mode to apply in *out_mode when `policy` requests
|
||||||
|
* a change, or false when it requests neither --perms nor --executability (the
|
||||||
|
* caller then leaves the destination mode alone). --perms wins over -E; the
|
||||||
|
* -E rule derives exec bits from the destination's read bits (rsync 3.4);
|
||||||
|
* group/other write is never granted from a client-supplied mode. Shared by
|
||||||
|
* file_restore_metadata_fd() and the --fake-super replay so the two cannot
|
||||||
|
* diverge. */
|
||||||
|
bool metadata_mode_for_policy(mode_t source_mode, mode_t current_mode, FileAttrPolicy policy,
|
||||||
|
mode_t* out_mode);
|
||||||
/* P7 Wave D: apply a SYMLINK's own metadata using no-follow primitives only
|
/* P7 Wave D: apply a SYMLINK's own metadata using no-follow primitives only
|
||||||
* (utimensat/lchown/fchmodat with AT_SYMLINK_NOFOLLOW), confined fd-relative
|
* (utimensat/lchown/fchmodat with AT_SYMLINK_NOFOLLOW), confined fd-relative
|
||||||
* under the authorized root. `omit_link_times` (-J/--omit-link-times)
|
* under the authorized root. The link's mode is applied only when policy.perms;
|
||||||
* suppresses the timestamps; the link's mode/ownership are still attempted
|
* policy.times (further suppressed by `omit_link_times` for -J) applies the
|
||||||
* (ownership stays gated by the identity policy and by default is not applied).
|
* mtime with policy.atimes controlling the atime slot; ownership stays gated by
|
||||||
|
* the identity policy and by default is not applied.
|
||||||
* A null metadata or an unfollowable parent is a harmless no-op. Returns false
|
* A null metadata or an unfollowable parent is a harmless no-op. Returns false
|
||||||
* only when a REQUIRED --copy-as ownership application failed, so the caller can
|
* only when a REQUIRED --copy-as ownership application failed, so the caller can
|
||||||
* report the entry as failed instead of claiming a wrong-owner success. */
|
* report the entry as failed instead of claiming a wrong-owner success. */
|
||||||
bool file_restore_symlink_metadata(const char* path, const FileMetadata* metadata,
|
bool file_restore_symlink_metadata(const char* path, const FileMetadata* metadata,
|
||||||
bool omit_link_times);
|
FileAttrPolicy policy, bool omit_link_times);
|
||||||
|
|
||||||
/* Compare timestamps using rsync's whole-second modification window. */
|
/* Compare timestamps using rsync's whole-second modification window. */
|
||||||
bool metadata_mtime_matches(time_t left_sec, long left_nsec, time_t right_sec, long right_nsec,
|
bool metadata_mtime_matches(time_t left_sec, long left_nsec, time_t right_sec, long right_nsec,
|
||||||
|
|||||||
@@ -30,10 +30,14 @@ PipelineContextSender* pipeline_context_sender_create(Config* config, Queue* que
|
|||||||
context->max_queue_bytes = 0;
|
context->max_queue_bytes = 0;
|
||||||
context->manifest = NULL;
|
context->manifest = NULL;
|
||||||
context->excluded_paths = NULL;
|
context->excluded_paths = NULL;
|
||||||
|
context->size_skipped_paths = NULL;
|
||||||
|
context->synced_dirs = NULL;
|
||||||
|
context->plan_dirs = NULL;
|
||||||
context->missing_args = NULL;
|
context->missing_args = NULL;
|
||||||
context->scan_had_io_error = false;
|
context->scan_had_io_error = false;
|
||||||
context->remove_source_files = NULL;
|
context->remove_source_files = NULL;
|
||||||
context->early_delete = false;
|
context->early_delete = false;
|
||||||
|
context->delete_plans = NULL;
|
||||||
context->scan_stopped_early = false;
|
context->scan_stopped_early = false;
|
||||||
context->total_files = 0;
|
context->total_files = 0;
|
||||||
context->progress_bytes = 0;
|
context->progress_bytes = 0;
|
||||||
@@ -44,6 +48,7 @@ PipelineContextSender* pipeline_context_sender_create(Config* config, Queue* que
|
|||||||
protocol_session_set_max_alloc(&context->allocation_session, config->max_alloc);
|
protocol_session_set_max_alloc(&context->allocation_session, config->max_alloc);
|
||||||
context->dir_entries = NULL;
|
context->dir_entries = NULL;
|
||||||
context->dir_entries_mutex_init = false;
|
context->dir_entries_mutex_init = false;
|
||||||
|
context->delete_limit = false;
|
||||||
int init = 0;
|
int init = 0;
|
||||||
if (config->use_metadata) {
|
if (config->use_metadata) {
|
||||||
context->dir_entries = array_list_create(file_destroy);
|
context->dir_entries = array_list_create(file_destroy);
|
||||||
@@ -184,8 +189,16 @@ void pipeline_context_sender_destroy(PipelineContextSender* context) {
|
|||||||
if (context->manifest) {
|
if (context->manifest) {
|
||||||
array_list_delete(context->manifest);
|
array_list_delete(context->manifest);
|
||||||
}
|
}
|
||||||
|
if (context->delete_plans)
|
||||||
|
delete_plan_sender_destroy(context->delete_plans);
|
||||||
if (context->excluded_paths)
|
if (context->excluded_paths)
|
||||||
array_list_delete(context->excluded_paths);
|
array_list_delete(context->excluded_paths);
|
||||||
|
if (context->size_skipped_paths)
|
||||||
|
array_list_delete(context->size_skipped_paths);
|
||||||
|
if (context->synced_dirs)
|
||||||
|
array_list_delete(context->synced_dirs);
|
||||||
|
if (context->plan_dirs)
|
||||||
|
array_list_delete(context->plan_dirs);
|
||||||
if (context->missing_args)
|
if (context->missing_args)
|
||||||
array_list_delete(context->missing_args);
|
array_list_delete(context->missing_args);
|
||||||
if (context->remove_source_files)
|
if (context->remove_source_files)
|
||||||
|
|||||||
@@ -7,6 +7,7 @@
|
|||||||
#include "array_list.h"
|
#include "array_list.h"
|
||||||
#include "chunk.h"
|
#include "chunk.h"
|
||||||
#include "config.h"
|
#include "config.h"
|
||||||
|
#include "delete_plan.h"
|
||||||
#include "file.h"
|
#include "file.h"
|
||||||
#include "protocol.h"
|
#include "protocol.h"
|
||||||
#include "queue.h"
|
#include "queue.h"
|
||||||
@@ -42,6 +43,23 @@ typedef struct {
|
|||||||
scanner's exclusion sink) or, in the early modes, by the path-only pre-scan
|
scanner's exclusion sink) or, in the early modes, by the path-only pre-scan
|
||||||
on the calling thread before the pipeline starts. */
|
on the calling thread before the pipeline starts. */
|
||||||
ArrayList* excluded_paths;
|
ArrayList* excluded_paths;
|
||||||
|
/* --max-size/--min-size pruned source paths. These are ALWAYS sent as
|
||||||
|
protected prefixes (even with --delete-excluded), so the destination
|
||||||
|
mirrors of size-skipped files survive --delete like rsync. Populated by
|
||||||
|
the scanner thread (workers append under mutex_scanner) or, in the early
|
||||||
|
modes, by the path-only pre-scan on the calling thread. */
|
||||||
|
ArrayList* size_skipped_paths;
|
||||||
|
/* Destination-relative paths of the directories the source scan synchronized
|
||||||
|
for this run (the receive root is the "." sentinel). Sent with the
|
||||||
|
manifest so the receiver confines its extras walk to them, matching rsync's
|
||||||
|
"delete only in synchronized directories" (notably for --files-from).
|
||||||
|
Populated by the scanner thread or the early pre-scan. */
|
||||||
|
ArrayList* synced_dirs;
|
||||||
|
/* Destination-relative paths of every traversed source directory, for the
|
||||||
|
per-directory delete plan keep set (so an empty source directory survives
|
||||||
|
--delete rather than being removed as an extra). Prebuilt by the path-only
|
||||||
|
pre-scan on the calling thread. */
|
||||||
|
ArrayList* plan_dirs;
|
||||||
/* --delete-missing-args: the destination-relative mirrors of the --files-from
|
/* --delete-missing-args: the destination-relative mirrors of the --files-from
|
||||||
entries that are missing under the source. Computed by the preflight on
|
entries that are missing under the source. Computed by the preflight on
|
||||||
the calling thread before the pipeline starts; the sender thread transmits
|
the calling thread before the pipeline starts; the sender thread transmits
|
||||||
@@ -54,11 +72,16 @@ typedef struct {
|
|||||||
--ignore-errors kept the run going. */
|
--ignore-errors kept the run going. */
|
||||||
bool scan_had_io_error;
|
bool scan_had_io_error;
|
||||||
ArrayList* remove_source_files;
|
ArrayList* remove_source_files;
|
||||||
/* True when --delete-before/--delete-during require the keep-set manifest to
|
/* True when --delete-before requires the whole-tree keep-set manifest to be
|
||||||
be transmitted before any file data: context->manifest is then prebuilt by
|
transmitted before any file data: context->manifest is then prebuilt by a
|
||||||
a path-only pre-scan on the calling thread and the pipeline scanner must
|
path-only pre-scan on the calling thread and the pipeline scanner must not
|
||||||
not append to it. Set once before the worker threads start. */
|
append to it. Set once before the worker threads start. */
|
||||||
bool early_delete;
|
bool early_delete;
|
||||||
|
/* Non-NULL for --delete-during/--delete-delay: the per-directory plan set
|
||||||
|
prebuilt by the path-only pre-scan on the calling thread. The sender
|
||||||
|
thread transmits the root plan before any data and the remaining plans
|
||||||
|
alongside the chunks. Set once before the worker threads start. */
|
||||||
|
DeletePlanSender* delete_plans;
|
||||||
mtx_t mutex_progress;
|
mtx_t mutex_progress;
|
||||||
int total_files;
|
int total_files;
|
||||||
unsigned long long progress_bytes;
|
unsigned long long progress_bytes;
|
||||||
@@ -83,6 +106,10 @@ typedef struct {
|
|||||||
ArrayList* dir_entries;
|
ArrayList* dir_entries;
|
||||||
mtx_t dir_entries_mutex;
|
mtx_t dir_entries_mutex;
|
||||||
bool dir_entries_mutex_init;
|
bool dir_entries_mutex_init;
|
||||||
|
/* Set by the sender thread when the receiver reported a --max-delete-capped
|
||||||
|
deletion (STATUS_DELETE_LIMIT): the transfer succeeded and the process must
|
||||||
|
exit 25 like rsync. Read by the caller after the sender thread is joined. */
|
||||||
|
bool delete_limit;
|
||||||
} PipelineContextSender;
|
} PipelineContextSender;
|
||||||
|
|
||||||
/* `config` is borrowed and must outlive the context: destroy does NOT free it,
|
/* `config` is borrowed and must outlive the context: destroy does NOT free it,
|
||||||
|
|||||||
+67
-19
@@ -12,8 +12,7 @@
|
|||||||
#include <time.h>
|
#include <time.h>
|
||||||
#include <unistd.h>
|
#include <unistd.h>
|
||||||
|
|
||||||
#define RECEIVE_TIMEOUT_SEC 60 /* 60 second per-message timeout */
|
#define RECEIVE_TIMEOUT_SEC 60 /* built-in fallback for explicit -timed calls only */
|
||||||
#define SEND_TIMEOUT_SEC 60
|
|
||||||
|
|
||||||
static __thread int io_read_fd = -1;
|
static __thread int io_read_fd = -1;
|
||||||
static __thread int io_write_fd = -1;
|
static __thread int io_write_fd = -1;
|
||||||
@@ -30,6 +29,13 @@ static unsigned long long io_bwlimit = 0;
|
|||||||
static mtx_t bw_mutex;
|
static mtx_t bw_mutex;
|
||||||
static once_flag bw_mutex_once = ONCE_FLAG_INIT;
|
static once_flag bw_mutex_once = ONCE_FLAG_INIT;
|
||||||
|
|
||||||
|
/* Process-wide wire byte counters, used by the client to render rsync's
|
||||||
|
* --stats/--progress totals and the --out-format %b/%c tokens. The zero-copy
|
||||||
|
* sendfile path bypasses protocol_send_n_data, so it reports its bytes through
|
||||||
|
* protocol_note_bytes_written. */
|
||||||
|
static atomic_ullong io_bytes_written = 0;
|
||||||
|
static atomic_ullong io_bytes_read = 0;
|
||||||
|
|
||||||
static unsigned long long global_bwlimit(void);
|
static unsigned long long global_bwlimit(void);
|
||||||
|
|
||||||
static bool protocol_reserve_memory(ProtocolSession* session, size_t charge) {
|
static bool protocol_reserve_memory(ProtocolSession* session, size_t charge) {
|
||||||
@@ -99,8 +105,14 @@ void protocol_session_set_io_timeout(ProtocolSession* session, int sec) {
|
|||||||
|
|
||||||
int protocol_get_io_timeout_sec(void) {
|
int protocol_get_io_timeout_sec(void) {
|
||||||
const ProtocolSession* session = bound_session ? bound_session : &legacy_io_session;
|
const ProtocolSession* session = bound_session ? bound_session : &legacy_io_session;
|
||||||
int sec = session->io_timeout_sec;
|
/* 0 (or negative) means the session timeout is disabled, matching rsync's
|
||||||
return sec > 0 ? sec : RECEIVE_TIMEOUT_SEC;
|
* --timeout=0 default. Callers must treat a non-positive result as "wait
|
||||||
|
* without a deadline" instead of substituting a built-in window. */
|
||||||
|
return session->io_timeout_sec > 0 ? session->io_timeout_sec : 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
int protocol_server_io_timeout_sec(int client_timeout) {
|
||||||
|
return client_timeout > 0 ? client_timeout : SERVER_IO_TIMEOUT_SEC;
|
||||||
}
|
}
|
||||||
|
|
||||||
void protocol_session_set_max_alloc(ProtocolSession* session, unsigned long long max_alloc) {
|
void protocol_session_set_max_alloc(ProtocolSession* session, unsigned long long max_alloc) {
|
||||||
@@ -110,7 +122,8 @@ void protocol_session_set_max_alloc(ProtocolSession* session, unsigned long long
|
|||||||
}
|
}
|
||||||
|
|
||||||
static bool allocation_allowed(const ProtocolSession* session, size_t size) {
|
static bool allocation_allowed(const ProtocolSession* session, size_t size) {
|
||||||
return (unsigned long long)size <= session->max_alloc;
|
/* max_alloc == 0 is rsync's --max-alloc=0 "no limit". */
|
||||||
|
return session->max_alloc == 0 || (unsigned long long)size <= session->max_alloc;
|
||||||
}
|
}
|
||||||
|
|
||||||
static void* protocol_alloc_for_session(const ProtocolSession* session, size_t size) {
|
static void* protocol_alloc_for_session(const ProtocolSession* session, size_t size) {
|
||||||
@@ -236,6 +249,18 @@ SSL* io_get_ssl(void) {
|
|||||||
return io_ssl;
|
return io_ssl;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
unsigned long long protocol_bytes_written(void) {
|
||||||
|
return atomic_load(&io_bytes_written);
|
||||||
|
}
|
||||||
|
|
||||||
|
unsigned long long protocol_bytes_read(void) {
|
||||||
|
return atomic_load(&io_bytes_read);
|
||||||
|
}
|
||||||
|
|
||||||
|
void protocol_note_bytes_written(unsigned long long bytes) {
|
||||||
|
atomic_fetch_add(&io_bytes_written, bytes);
|
||||||
|
}
|
||||||
|
|
||||||
static ProtocolSession* legacy_session(int read_fd, int write_fd) {
|
static ProtocolSession* legacy_session(int read_fd, int write_fd) {
|
||||||
if (bound_session)
|
if (bound_session)
|
||||||
return bound_session;
|
return bound_session;
|
||||||
@@ -280,11 +305,15 @@ bool protocol_send_n_data(ProtocolSession* session, const void* data, size_t dat
|
|||||||
log_debug_message(LOG_DEBUG_IO, " Sending n Data: %zu", data_size);
|
log_debug_message(LOG_DEBUG_IO, " Sending n Data: %zu", data_size);
|
||||||
if (!session)
|
if (!session)
|
||||||
return false;
|
return false;
|
||||||
int timeout_sec = session->io_timeout_sec > 0 ? session->io_timeout_sec : SEND_TIMEOUT_SEC;
|
/* A non-positive session timeout disables the deadline entirely (rsync's
|
||||||
|
* --timeout=0 default); poll then blocks until the socket becomes writable. */
|
||||||
|
int timeout_sec = session->io_timeout_sec > 0 ? session->io_timeout_sec : 0;
|
||||||
int fd = session->write_fd;
|
int fd = session->write_fd;
|
||||||
struct timespec deadline;
|
struct timespec deadline;
|
||||||
|
if (timeout_sec > 0) {
|
||||||
clock_gettime(CLOCK_MONOTONIC, &deadline);
|
clock_gettime(CLOCK_MONOTONIC, &deadline);
|
||||||
deadline.tv_sec += timeout_sec;
|
deadline.tv_sec += timeout_sec;
|
||||||
|
}
|
||||||
short wait_events = POLLOUT;
|
short wait_events = POLLOUT;
|
||||||
ssize_t total_bytes_send = 0;
|
ssize_t total_bytes_send = 0;
|
||||||
while ((size_t)total_bytes_send < data_size) {
|
while ((size_t)total_bytes_send < data_size) {
|
||||||
@@ -292,7 +321,7 @@ bool protocol_send_n_data(ProtocolSession* session, const void* data, size_t dat
|
|||||||
if (session->bwlimit > 0 && chunk > 65536)
|
if (session->bwlimit > 0 && chunk > 65536)
|
||||||
chunk = 65536;
|
chunk = 65536;
|
||||||
struct pollfd pfd = {.fd = fd, .events = wait_events};
|
struct pollfd pfd = {.fd = fd, .events = wait_events};
|
||||||
int poll_result = poll(&pfd, 1, deadline_remaining_ms(&deadline));
|
int poll_result = poll(&pfd, 1, timeout_sec > 0 ? deadline_remaining_ms(&deadline) : -1);
|
||||||
if (poll_result == 0 || (poll_result < 0 && errno != EINTR)) {
|
if (poll_result == 0 || (poll_result < 0 && errno != EINTR)) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Send timeout or poll failure");
|
log_message(LOG_LEVEL_ERROR, "Send timeout or poll failure");
|
||||||
return false;
|
return false;
|
||||||
@@ -333,17 +362,27 @@ bool protocol_send_n_data(ProtocolSession* session, const void* data, size_t dat
|
|||||||
wait_events = POLLOUT;
|
wait_events = POLLOUT;
|
||||||
}
|
}
|
||||||
log_debug_message(LOG_DEBUG_IO, " Send n Data: %zu", total_bytes_send);
|
log_debug_message(LOG_DEBUG_IO, " Send n Data: %zu", total_bytes_send);
|
||||||
|
atomic_fetch_add(&io_bytes_written, (unsigned long long)total_bytes_send);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool protocol_receive_n_data_timed(ProtocolSession* session, void* data, size_t data_size,
|
bool protocol_receive_n_data_timed(ProtocolSession* session, void* data, size_t data_size,
|
||||||
int timeout_sec);
|
int timeout_sec);
|
||||||
|
static bool protocol_receive_n_data_until(ProtocolSession* session, void* data, size_t data_size,
|
||||||
|
const struct timespec* deadline);
|
||||||
|
|
||||||
bool protocol_receive_n_data(ProtocolSession* session, void* data, size_t data_size) {
|
bool protocol_receive_n_data(ProtocolSession* session, void* data, size_t data_size) {
|
||||||
/* Honor the session's configured deadline; protocol_receive_n_data_timed
|
/* Honor the session's configured deadline. A non-positive value disables the
|
||||||
* re-applies the built-in 60 s default when the value is <= 0. */
|
* deadline (rsync's --timeout=0 default): wait without a poll timeout. The
|
||||||
int timeout_sec = session ? session->io_timeout_sec : 0;
|
* explicit _timed variants keep their own 0 -> built-in-default contract. */
|
||||||
return protocol_receive_n_data_timed(session, data, data_size, timeout_sec);
|
if (!session)
|
||||||
|
return false;
|
||||||
|
if (session->io_timeout_sec <= 0)
|
||||||
|
return protocol_receive_n_data_until(session, data, data_size, NULL);
|
||||||
|
struct timespec deadline;
|
||||||
|
clock_gettime(CLOCK_MONOTONIC, &deadline);
|
||||||
|
deadline.tv_sec += session->io_timeout_sec;
|
||||||
|
return protocol_receive_n_data_until(session, data, data_size, &deadline);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Read exactly `data_size` bytes from `session` before `deadline` elapses
|
/* Read exactly `data_size` bytes from `session` before `deadline` elapses
|
||||||
@@ -353,7 +392,7 @@ bool protocol_receive_n_data(ProtocolSession* session, void* data, size_t data_s
|
|||||||
static bool protocol_receive_n_data_until(ProtocolSession* session, void* data, size_t data_size,
|
static bool protocol_receive_n_data_until(ProtocolSession* session, void* data, size_t data_size,
|
||||||
const struct timespec* deadline) {
|
const struct timespec* deadline) {
|
||||||
log_debug_message(LOG_DEBUG_IO, " Receiving n Data: %zu", data_size);
|
log_debug_message(LOG_DEBUG_IO, " Receiving n Data: %zu", data_size);
|
||||||
if (!session || !deadline)
|
if (!session)
|
||||||
return false;
|
return false;
|
||||||
int fd = session->read_fd;
|
int fd = session->read_fd;
|
||||||
|
|
||||||
@@ -362,7 +401,8 @@ static bool protocol_receive_n_data_until(ProtocolSession* session, void* data,
|
|||||||
while (total_bytes_received < data_size) {
|
while (total_bytes_received < data_size) {
|
||||||
if (!session->ssl || SSL_pending(session->ssl) == 0) {
|
if (!session->ssl || SSL_pending(session->ssl) == 0) {
|
||||||
struct pollfd pfd = {.fd = fd, .events = wait_events};
|
struct pollfd pfd = {.fd = fd, .events = wait_events};
|
||||||
int poll_result = poll(&pfd, 1, deadline_remaining_ms(deadline));
|
/* A NULL deadline means "wait indefinitely" (timeout disabled). */
|
||||||
|
int poll_result = poll(&pfd, 1, deadline ? deadline_remaining_ms(deadline) : -1);
|
||||||
if (poll_result == 0) {
|
if (poll_result == 0) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Receive timeout");
|
log_message(LOG_LEVEL_ERROR, "Receive timeout");
|
||||||
return false;
|
return false;
|
||||||
@@ -410,6 +450,7 @@ static bool protocol_receive_n_data_until(ProtocolSession* session, void* data,
|
|||||||
wait_events = POLLIN;
|
wait_events = POLLIN;
|
||||||
}
|
}
|
||||||
log_debug_message(LOG_DEBUG_IO, " Received n Data: %zu", total_bytes_received);
|
log_debug_message(LOG_DEBUG_IO, " Received n Data: %zu", total_bytes_received);
|
||||||
|
atomic_fetch_add(&io_bytes_read, (unsigned long long)total_bytes_received);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -479,6 +520,10 @@ static const char* status_to_string(Status status) {
|
|||||||
return "ERROR_DETAIL";
|
return "ERROR_DETAIL";
|
||||||
case STATUS_DRY_RUN_TRANSFER:
|
case STATUS_DRY_RUN_TRANSFER:
|
||||||
return "DRY_RUN_TRANSFER";
|
return "DRY_RUN_TRANSFER";
|
||||||
|
case STATUS_DELETE_LIMIT:
|
||||||
|
return "DELETE_LIMIT";
|
||||||
|
case STATUS_DEST_INFO:
|
||||||
|
return "DEST_INFO";
|
||||||
default:
|
default:
|
||||||
return "UNKNOWN";
|
return "UNKNOWN";
|
||||||
}
|
}
|
||||||
@@ -702,13 +747,16 @@ static bool protocol_capture_error_detail(ProtocolSession* session, Status* stat
|
|||||||
bool protocol_receive_status(ProtocolSession* session, Status* status) {
|
bool protocol_receive_status(ProtocolSession* session, Status* status) {
|
||||||
if (!session || !status)
|
if (!session || !status)
|
||||||
return false;
|
return false;
|
||||||
int timeout_sec = session->io_timeout_sec > 0 ? session->io_timeout_sec : RECEIVE_TIMEOUT_SEC;
|
|
||||||
struct timespec deadline;
|
struct timespec deadline;
|
||||||
|
const struct timespec* deadline_ptr = NULL;
|
||||||
|
if (session->io_timeout_sec > 0) {
|
||||||
clock_gettime(CLOCK_MONOTONIC, &deadline);
|
clock_gettime(CLOCK_MONOTONIC, &deadline);
|
||||||
deadline.tv_sec += timeout_sec;
|
deadline.tv_sec += session->io_timeout_sec;
|
||||||
if (!protocol_receive_n_data_until(session, status, sizeof(Status), &deadline))
|
deadline_ptr = &deadline;
|
||||||
|
}
|
||||||
|
if (!protocol_receive_n_data_until(session, status, sizeof(Status), deadline_ptr))
|
||||||
return false;
|
return false;
|
||||||
if (!protocol_capture_error_detail(session, status, &deadline, NULL))
|
if (!protocol_capture_error_detail(session, status, deadline_ptr, NULL))
|
||||||
return false;
|
return false;
|
||||||
log_debug_message(LOG_DEBUG_PROTO, "Received Status: %s", status_to_string(*status));
|
log_debug_message(LOG_DEBUG_PROTO, "Received Status: %s", status_to_string(*status));
|
||||||
return true;
|
return true;
|
||||||
@@ -747,8 +795,8 @@ static bool protocol_read_status_until(ProtocolSession* session, Status* status,
|
|||||||
short wait_events = POLLIN;
|
short wait_events = POLLIN;
|
||||||
while (got < sizeof(Status)) {
|
while (got < sizeof(Status)) {
|
||||||
if (!session->ssl || SSL_pending(session->ssl) == 0) {
|
if (!session->ssl || SSL_pending(session->ssl) == 0) {
|
||||||
int remaining_ms = deadline_remaining_ms(deadline);
|
int remaining_ms = deadline ? deadline_remaining_ms(deadline) : -1;
|
||||||
if (remaining_ms <= 0) {
|
if (remaining_ms == 0) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Receive timeout while reading status");
|
log_message(LOG_LEVEL_ERROR, "Receive timeout while reading status");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|||||||
+72
-12
@@ -34,6 +34,11 @@
|
|||||||
#define DEFAULT_MAX_ALLOC (1ULL * 1024 * 1024 * 1024)
|
#define DEFAULT_MAX_ALLOC (1ULL * 1024 * 1024 * 1024)
|
||||||
/* Server policy ceiling for a client-provided allocation limit. */
|
/* Server policy ceiling for a client-provided allocation limit. */
|
||||||
#define MAX_SERVER_ALLOC (256ULL * 1024 * 1024)
|
#define MAX_SERVER_ALLOC (256ULL * 1024 * 1024)
|
||||||
|
/* Server-owned floor for the per-message I/O deadline. A client --timeout=0
|
||||||
|
(rsync's default) disables the client's own deadlines, but a server session
|
||||||
|
must never be held open forever by a silent peer (slow-loris), so the server
|
||||||
|
floors the effective deadline at this value. */
|
||||||
|
#define SERVER_IO_TIMEOUT_SEC 60
|
||||||
/* Bounded cumulative per-connection receive budget. In-flight wire buffers,
|
/* Bounded cumulative per-connection receive budget. In-flight wire buffers,
|
||||||
decompression buffers and queued (not yet written) file payloads for a
|
decompression buffers and queued (not yet written) file payloads for a
|
||||||
connection must stay within this ceiling. */
|
connection must stay within this ceiling. */
|
||||||
@@ -59,10 +64,12 @@ typedef struct ProtocolSession {
|
|||||||
bool eight_bit_output;
|
bool eight_bit_output;
|
||||||
unsigned long long max_alloc;
|
unsigned long long max_alloc;
|
||||||
/* Per-session deadline (seconds) applied to every protocol send/receive by
|
/* Per-session deadline (seconds) applied to every protocol send/receive by
|
||||||
* protocol_send_n_data / protocol_receive_n_data. Defaults to the built-in
|
* protocol_send_n_data / protocol_receive_n_data. The initialized default is
|
||||||
* 60 s window; a value <= 0 falls back to that default. Set from the
|
* the built-in 60 s window; a value <= 0 disables the deadline (rsync's
|
||||||
* negotiated Config->timeout so --timeout is honored by the poll()-driven
|
* --timeout=0). Set from the negotiated Config->timeout so --timeout is
|
||||||
* protocol I/O, not just the socket SO_RCVTIMEO/SO_SNDTIMEO. */
|
* honored by the poll()-driven protocol I/O, not just the socket
|
||||||
|
* SO_RCVTIMEO/SO_SNDTIMEO. The server does not propagate a client 0 here: it
|
||||||
|
* installs protocol_server_io_timeout_sec() so its sessions keep a floor. */
|
||||||
int io_timeout_sec;
|
int io_timeout_sec;
|
||||||
} ProtocolSession;
|
} ProtocolSession;
|
||||||
|
|
||||||
@@ -155,7 +162,47 @@ enum NET_STATUS {
|
|||||||
* (the receiver reads none in dry-run). STATUS_OK keeps its meaning in this
|
* (the receiver reads none in dry-run). STATUS_OK keeps its meaning in this
|
||||||
* path ("already up to date / nothing to do"). Appended after
|
* path ("already up to date / nothing to do"). Appended after
|
||||||
* STATUS_ERROR_DETAIL so no existing status is renumbered. */
|
* STATUS_ERROR_DETAIL so no existing status is renumbered. */
|
||||||
STATUS_DRY_RUN_TRANSFER
|
STATUS_DRY_RUN_TRANSFER,
|
||||||
|
/* --max-delete budget exhausted (protocol 2.23.0). Sent by the receiver as
|
||||||
|
* the terminal success status INSTEAD of STATUS_OK when a --delete/
|
||||||
|
* --delete-missing-args commit removed up to the --max-delete bound but had
|
||||||
|
* to skip further extras. The transfer itself succeeded and all file data is
|
||||||
|
* stored; the sender maps this to rsync's exit code 25 ("the --max-delete
|
||||||
|
* limit stopped deletions"). Appended after STATUS_DRY_RUN_TRANSFER so no
|
||||||
|
* existing status is renumbered. */
|
||||||
|
STATUS_DELETE_LIMIT,
|
||||||
|
/* Destination-state report for output parity (protocol 2.23.0). When the
|
||||||
|
* wire config carries report_dest_info=true, the receiver answers every
|
||||||
|
* per-file STATUS_CHECK request with STATUS_DEST_INFO FIRST, followed by a
|
||||||
|
* fixed record describing the pre-transfer destination entry
|
||||||
|
* (int32 has_old; uint64 size; int64 mtime; int64 mtime_nsec; uint32 mode;
|
||||||
|
* int32 uid; int32 gid). The ordinary STATUS_OK/STATUS_NEXT/... verdict
|
||||||
|
* follows, so the sender can render rsync-accurate -i/--out-format columns
|
||||||
|
* (new vs modified, and which of size/time/perms/owner/group differ) without
|
||||||
|
* changing the transfer decision itself. Appended after
|
||||||
|
* STATUS_DELETE_LIMIT so no existing status is renumbered. */
|
||||||
|
STATUS_DEST_INFO,
|
||||||
|
/* Per-directory delete plan (protocol 2.24.0). The sender of a
|
||||||
|
* --delete-during/--delete-delay transfer streams one frame per source
|
||||||
|
* directory in directory order instead of a single whole-tree keep-set
|
||||||
|
* manifest. The receiver applies the plan when it arrives
|
||||||
|
* (--delete-during removes that directory's extras immediately) or records
|
||||||
|
* the extras and applies them only after the whole transfer succeeded
|
||||||
|
* (--delete-delay). Payload: an int32 has_config flag (1 on the first plan
|
||||||
|
* of the run, 0 afterwards); when set, the three global config sections
|
||||||
|
* (protected-prefix count+paths, size-skipped count+paths, missing-args
|
||||||
|
* count+paths); then the destination-relative directory path wire string
|
||||||
|
* ("." for the receive root); then the child-directory count + names and the
|
||||||
|
* child-file count + names that must be kept. Appended after
|
||||||
|
* STATUS_DEST_INFO so no existing status is renumbered. */
|
||||||
|
STATUS_DELETE_PLAN,
|
||||||
|
/* End-of-transfer receiver counter report (protocol 2.25.0). When the wire
|
||||||
|
* config carries report_stats=true, the receiver sends this status once,
|
||||||
|
* immediately before its terminal success status, followed by a fixed stats
|
||||||
|
* record (see format_stats_send/receive in format.h) and, when the run is a
|
||||||
|
* --dry-run with --delete, the would-delete path list. Appended after
|
||||||
|
* STATUS_DELETE_PLAN so no existing status is renumbered. */
|
||||||
|
STATUS_STATS
|
||||||
};
|
};
|
||||||
|
|
||||||
void io_set_fds(int read_fd, int write_fd);
|
void io_set_fds(int read_fd, int write_fd);
|
||||||
@@ -163,6 +210,14 @@ void io_set_bwlimit(unsigned long long bytes_per_sec);
|
|||||||
void io_set_ssl(SSL* ssl);
|
void io_set_ssl(SSL* ssl);
|
||||||
SSL* io_get_ssl(void);
|
SSL* io_get_ssl(void);
|
||||||
|
|
||||||
|
/* Process-wide wire byte counters. protocol_send_n_data/protocol_receive_n_data
|
||||||
|
* update them; the zero-copy sendfile path reports through
|
||||||
|
* protocol_note_bytes_written. Used by the client to render rsync's
|
||||||
|
* --stats/--progress totals and the --out-format %b/%c tokens. */
|
||||||
|
unsigned long long protocol_bytes_written(void);
|
||||||
|
unsigned long long protocol_bytes_read(void);
|
||||||
|
void protocol_note_bytes_written(unsigned long long bytes);
|
||||||
|
|
||||||
void protocol_session_init(ProtocolSession* session, int read_fd, int write_fd);
|
void protocol_session_init(ProtocolSession* session, int read_fd, int write_fd);
|
||||||
/* Transitional bridge for helpers whose signatures still carry only an fd. */
|
/* Transitional bridge for helpers whose signatures still carry only an fd. */
|
||||||
void protocol_session_bind(ProtocolSession* session);
|
void protocol_session_bind(ProtocolSession* session);
|
||||||
@@ -170,15 +225,20 @@ void protocol_session_unbind(void);
|
|||||||
void protocol_session_set_ssl(ProtocolSession* session, SSL* ssl);
|
void protocol_session_set_ssl(ProtocolSession* session, SSL* ssl);
|
||||||
void protocol_session_set_bwlimit(ProtocolSession* session, unsigned long long bytes_per_sec);
|
void protocol_session_set_bwlimit(ProtocolSession* session, unsigned long long bytes_per_sec);
|
||||||
void protocol_session_set_max_alloc(ProtocolSession* session, unsigned long long max_alloc);
|
void protocol_session_set_max_alloc(ProtocolSession* session, unsigned long long max_alloc);
|
||||||
/* Override the per-message send/receive deadline for this session.
|
/* Override the per-message send/receive deadline for this session. The value
|
||||||
* `sec` <= 0 restores the built-in 60 s default (used for --timeout=0/unset).
|
* is stored verbatim: a positive value sets the deadline, `sec` <= 0 disables
|
||||||
* An explicit long deadline (e.g. the delete-ack wait) is applied per-call by
|
* it (rsync's --timeout=0). An explicit long deadline (e.g. the delete-ack
|
||||||
* protocol_receive_status_timed and is unaffected by this setter. */
|
* wait) is applied per-call by protocol_receive_status_timed and is unaffected
|
||||||
|
* by this setter. */
|
||||||
void protocol_session_set_io_timeout(ProtocolSession* session, int sec);
|
void protocol_session_set_io_timeout(ProtocolSession* session, int sec);
|
||||||
/* Effective per-message I/O deadline (seconds) for the currently-bound session,
|
/* Effective per-message I/O deadline (seconds) for the currently-bound session.
|
||||||
* falling back to the built-in default. Used by the plaintext sendfile path
|
* Zero means the deadline is disabled (rsync's --timeout=0). Used by the
|
||||||
* which bypasses the protocol send primitive. */
|
* plaintext sendfile path which bypasses the protocol send primitive. */
|
||||||
int protocol_get_io_timeout_sec(void);
|
int protocol_get_io_timeout_sec(void);
|
||||||
|
/* The server-side effective deadline for a client-requested timeout: a positive
|
||||||
|
* client value is honored, otherwise the SERVER_IO_TIMEOUT_SEC floor applies so
|
||||||
|
* a silent peer can never hold a session open forever. */
|
||||||
|
int protocol_server_io_timeout_sec(int client_timeout);
|
||||||
void* protocol_alloc(size_t size);
|
void* protocol_alloc(size_t size);
|
||||||
void* protocol_realloc(void* ptr, size_t size);
|
void* protocol_realloc(void* ptr, size_t size);
|
||||||
void protocol_session_set_8_bit_output(ProtocolSession* session, bool enabled);
|
void protocol_session_set_8_bit_output(ProtocolSession* session, bool enabled);
|
||||||
|
|||||||
+181
-21
@@ -44,6 +44,181 @@ static bool parse_two_digits(const char* s, int* out) {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* True when the current character of the cursor is a decimal digit. */
|
||||||
|
static bool is_digit(const char* cp) {
|
||||||
|
return *cp >= '0' && *cp <= '9';
|
||||||
|
}
|
||||||
|
|
||||||
|
/* rsync 3.4.1's flexible --stop-at date parser (ported from
|
||||||
|
* options.c:parse_time). Returns a time_t, or (time_t)-1 on a malformed value.
|
||||||
|
* Accepted forms include Y-M-DTh:m, Y/M/DTh:m, Y-M-D, M-D, D, h:m, :m and
|
||||||
|
* "T h:m"; a 1- or 2-digit year and omitted fields are resolved to the next
|
||||||
|
* matching point in time in the local timezone. Seconds are NOT accepted
|
||||||
|
* (rsync rejects them too); FastSync keeps its own HH:MM:SS spelling as an
|
||||||
|
* extension handled by the caller. `now` is passed in so tests are
|
||||||
|
* deterministic; production passes time(NULL). */
|
||||||
|
static time_t parse_time_rsync(const char* value, time_t now) {
|
||||||
|
const char* cp;
|
||||||
|
time_t val;
|
||||||
|
struct tm today;
|
||||||
|
if (!localtime_r(&now, &today))
|
||||||
|
return (time_t)-1;
|
||||||
|
struct tm t;
|
||||||
|
int in_date, old_mday, n;
|
||||||
|
|
||||||
|
memset(&t, 0, sizeof t);
|
||||||
|
t.tm_year = t.tm_mon = t.tm_mday = -1;
|
||||||
|
t.tm_hour = t.tm_min = t.tm_isdst = -1;
|
||||||
|
cp = value;
|
||||||
|
if (*cp == 'T' || *cp == 't' || *cp == ':') {
|
||||||
|
in_date = *cp == ':' ? 0 : -1;
|
||||||
|
cp++;
|
||||||
|
} else
|
||||||
|
in_date = 1;
|
||||||
|
for (;; cp++) {
|
||||||
|
if (!is_digit(cp))
|
||||||
|
return (time_t)-1;
|
||||||
|
n = 0;
|
||||||
|
do {
|
||||||
|
n = n * 10 + *cp++ - '0';
|
||||||
|
} while (is_digit(cp));
|
||||||
|
if (*cp == ':')
|
||||||
|
in_date = 0;
|
||||||
|
if (in_date > 0) {
|
||||||
|
if (t.tm_year != -1)
|
||||||
|
return (time_t)-1;
|
||||||
|
t.tm_year = t.tm_mon;
|
||||||
|
t.tm_mon = t.tm_mday;
|
||||||
|
t.tm_mday = n;
|
||||||
|
if (!*cp)
|
||||||
|
break;
|
||||||
|
if (*cp == 'T' || *cp == 't') {
|
||||||
|
if (!cp[1])
|
||||||
|
break;
|
||||||
|
in_date = -1;
|
||||||
|
} else if (*cp != '-' && *cp != '/')
|
||||||
|
return (time_t)-1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (t.tm_hour != -1)
|
||||||
|
return (time_t)-1;
|
||||||
|
t.tm_hour = t.tm_min;
|
||||||
|
t.tm_min = n;
|
||||||
|
if (!*cp) {
|
||||||
|
if (in_date < 0)
|
||||||
|
return (time_t)-1;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if (*cp != ':')
|
||||||
|
return (time_t)-1;
|
||||||
|
in_date = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
in_date = 0;
|
||||||
|
if (t.tm_year < 0) {
|
||||||
|
t.tm_year = today.tm_year;
|
||||||
|
in_date = 1;
|
||||||
|
} else if (t.tm_year < 100) {
|
||||||
|
while (t.tm_year < today.tm_year)
|
||||||
|
t.tm_year += 100;
|
||||||
|
} else
|
||||||
|
t.tm_year -= 1900;
|
||||||
|
if (t.tm_mon < 0) {
|
||||||
|
t.tm_mon = today.tm_mon;
|
||||||
|
in_date = 2;
|
||||||
|
} else
|
||||||
|
t.tm_mon--;
|
||||||
|
if (t.tm_mday < 0) {
|
||||||
|
t.tm_mday = today.tm_mday;
|
||||||
|
in_date = 3;
|
||||||
|
}
|
||||||
|
|
||||||
|
n = 0;
|
||||||
|
if (t.tm_min < 0) {
|
||||||
|
t.tm_hour = t.tm_min = 0;
|
||||||
|
} else if (t.tm_hour < 0) {
|
||||||
|
if (in_date != 3)
|
||||||
|
return (time_t)-1;
|
||||||
|
in_date = 0;
|
||||||
|
t.tm_hour = today.tm_hour;
|
||||||
|
n = 60 * 60;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* mktime() may roll a too-large tm_mday into the following month; undo that
|
||||||
|
* in the "next match" loop below. */
|
||||||
|
old_mday = t.tm_mday;
|
||||||
|
if (t.tm_hour > 23 || t.tm_min > 59 || t.tm_mon < 0 || t.tm_mon >= 12 || t.tm_mday < 1 ||
|
||||||
|
t.tm_mday > 31 || (val = mktime(&t)) == (time_t)-1)
|
||||||
|
return (time_t)-1;
|
||||||
|
|
||||||
|
while (in_date && (val <= now || t.tm_mday < old_mday)) {
|
||||||
|
switch (in_date) {
|
||||||
|
case 3:
|
||||||
|
old_mday = ++t.tm_mday;
|
||||||
|
break;
|
||||||
|
case 2:
|
||||||
|
if (t.tm_mday < old_mday)
|
||||||
|
t.tm_mday = old_mday; /* the month already got bumped forward */
|
||||||
|
else if (++t.tm_mon == 12) {
|
||||||
|
t.tm_mon = 0;
|
||||||
|
t.tm_year++;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
case 1:
|
||||||
|
if (t.tm_mday < old_mday) {
|
||||||
|
/* mon==1 mday==29 got bumped to mon==2 */
|
||||||
|
if (t.tm_mon != 2 || old_mday != 29)
|
||||||
|
return (time_t)-1;
|
||||||
|
t.tm_mon = 1;
|
||||||
|
t.tm_mday = 29;
|
||||||
|
}
|
||||||
|
t.tm_year++;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if ((val = mktime(&t)) == (time_t)-1) {
|
||||||
|
if (in_date != 3 || t.tm_mday <= 28)
|
||||||
|
return (time_t)-1;
|
||||||
|
t.tm_mday = old_mday = 1;
|
||||||
|
in_date = 2;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (n) {
|
||||||
|
while (val <= now)
|
||||||
|
val += n;
|
||||||
|
}
|
||||||
|
return val;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* FastSync's HH:MM or HH:MM:SS spelling on the current local day. rsync's own
|
||||||
|
* --stop-at accepts only HH:MM, so this is a strict superset extension. */
|
||||||
|
static bool parse_clock_time(const char* value, time_t now, time_t* out_deadline) {
|
||||||
|
size_t len = strlen(value);
|
||||||
|
if (len != 5 && len != 8)
|
||||||
|
return false;
|
||||||
|
if (value[2] != ':' || (len == 8 && value[5] != ':'))
|
||||||
|
return false;
|
||||||
|
int hh, mm, ss = 0;
|
||||||
|
if (!parse_two_digits(value, &hh) || !parse_two_digits(value + 3, &mm))
|
||||||
|
return false;
|
||||||
|
if (len == 8 && !parse_two_digits(value + 6, &ss))
|
||||||
|
return false;
|
||||||
|
if (hh > 23 || mm > 59 || ss > 59)
|
||||||
|
return false;
|
||||||
|
|
||||||
|
struct tm today;
|
||||||
|
if (!localtime_r(&now, &today))
|
||||||
|
return false;
|
||||||
|
today.tm_hour = hh;
|
||||||
|
today.tm_min = mm;
|
||||||
|
today.tm_sec = ss;
|
||||||
|
today.tm_isdst = -1;
|
||||||
|
time_t deadline = mktime(&today);
|
||||||
|
if (deadline == (time_t)-1)
|
||||||
|
return false;
|
||||||
|
*out_deadline = deadline;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
bool stop_parse_at_time(const char* value, time_t now, time_t* out_deadline) {
|
bool stop_parse_at_time(const char* value, time_t now, time_t* out_deadline) {
|
||||||
if (!value || !out_deadline)
|
if (!value || !out_deadline)
|
||||||
return false;
|
return false;
|
||||||
@@ -91,28 +266,13 @@ bool stop_parse_at_time(const char* value, time_t now, time_t* out_deadline) {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* HH:MM or HH:MM:SS on the current local day. */
|
/* HH:MM or HH:MM:SS on the current local day (FastSync extension). */
|
||||||
size_t len = strlen(value);
|
if (parse_clock_time(value, now, out_deadline))
|
||||||
if (len != 5 && len != 8)
|
return true;
|
||||||
return false;
|
|
||||||
if (value[2] != ':' || (len == 8 && value[5] != ':'))
|
|
||||||
return false;
|
|
||||||
int hh, mm, ss = 0;
|
|
||||||
if (!parse_two_digits(value, &hh) || !parse_two_digits(value + 3, &mm))
|
|
||||||
return false;
|
|
||||||
if (len == 8 && !parse_two_digits(value + 6, &ss))
|
|
||||||
return false;
|
|
||||||
if (hh > 23 || mm > 59 || ss > 59)
|
|
||||||
return false;
|
|
||||||
|
|
||||||
struct tm today;
|
/* rsync's full/partial date-and-time form (e.g. 2000-12-31T23:59, 12-31,
|
||||||
if (!localtime_r(&now, &today))
|
* 14:00, :59, 1, 1-30). */
|
||||||
return false;
|
time_t deadline = parse_time_rsync(value, now);
|
||||||
today.tm_hour = hh;
|
|
||||||
today.tm_min = mm;
|
|
||||||
today.tm_sec = ss;
|
|
||||||
today.tm_isdst = -1;
|
|
||||||
time_t deadline = mktime(&today);
|
|
||||||
if (deadline == (time_t)-1)
|
if (deadline == (time_t)-1)
|
||||||
return false;
|
return false;
|
||||||
*out_deadline = deadline;
|
*out_deadline = deadline;
|
||||||
|
|||||||
@@ -289,14 +289,14 @@ void server_accept_loop(Server* server, void (*child_fn)(int, void*), void* chil
|
|||||||
accept_loop(server, child_fn, child_ctx, log_fmt);
|
accept_loop(server, child_fn, child_ctx, log_fmt);
|
||||||
}
|
}
|
||||||
|
|
||||||
static int g_timeout_sec = 30;
|
/* rsync defaults: --timeout=0 (disabled) and --contimeout=60. A non-positive
|
||||||
static int g_contimeout_sec = 10;
|
* value means "no timeout" rather than "leave the built-in value in place". */
|
||||||
|
static int g_timeout_sec = 0;
|
||||||
|
static int g_contimeout_sec = 60;
|
||||||
|
|
||||||
void tcp_set_timeouts(int timeout_sec, int contimeout_sec) {
|
void tcp_set_timeouts(int timeout_sec, int contimeout_sec) {
|
||||||
if (timeout_sec > 0)
|
g_timeout_sec = timeout_sec > 0 ? timeout_sec : 0;
|
||||||
g_timeout_sec = timeout_sec;
|
g_contimeout_sec = contimeout_sec > 0 ? contimeout_sec : 0;
|
||||||
if (contimeout_sec > 0)
|
|
||||||
g_contimeout_sec = contimeout_sec;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
int tcp_get_contimeout_sec(void) {
|
int tcp_get_contimeout_sec(void) {
|
||||||
@@ -308,6 +308,10 @@ int tcp_get_timeout_sec(void) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
static void tcp_apply_socket_timeout(int fd) {
|
static void tcp_apply_socket_timeout(int fd) {
|
||||||
|
/* timeout 0 means no timeout: leave the socket in its default (blocking)
|
||||||
|
* mode instead of installing a zero SO_RCVTIMEO/SO_SNDTIMEO. */
|
||||||
|
if (g_timeout_sec <= 0)
|
||||||
|
return;
|
||||||
struct timeval tv;
|
struct timeval tv;
|
||||||
tv.tv_sec = g_timeout_sec;
|
tv.tv_sec = g_timeout_sec;
|
||||||
tv.tv_usec = 0;
|
tv.tv_usec = 0;
|
||||||
@@ -483,11 +487,15 @@ bool tcp_connect_socket_ex(Client* client, const char* host, int port,
|
|||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* --contimeout=0 disables the connect timeout: skip the pre-connect socket
|
||||||
|
* timeouts entirely. */
|
||||||
|
if (g_contimeout_sec > 0) {
|
||||||
struct timeval ct;
|
struct timeval ct;
|
||||||
ct.tv_sec = g_contimeout_sec;
|
ct.tv_sec = g_contimeout_sec;
|
||||||
ct.tv_usec = 0;
|
ct.tv_usec = 0;
|
||||||
setsockopt(client->file_descriptor, SOL_SOCKET, SO_RCVTIMEO, &ct, sizeof(ct));
|
setsockopt(client->file_descriptor, SOL_SOCKET, SO_RCVTIMEO, &ct, sizeof(ct));
|
||||||
setsockopt(client->file_descriptor, SOL_SOCKET, SO_SNDTIMEO, &ct, sizeof(ct));
|
setsockopt(client->file_descriptor, SOL_SOCKET, SO_SNDTIMEO, &ct, sizeof(ct));
|
||||||
|
}
|
||||||
|
|
||||||
if (bind_addr_family != 0) {
|
if (bind_addr_family != 0) {
|
||||||
if (rp->ai_family != bind_addr_family) {
|
if (rp->ai_family != bind_addr_family) {
|
||||||
|
|||||||
+251
-165
@@ -60,7 +60,7 @@ bool path_is_within_root(const char* root, const char* path) {
|
|||||||
* two differ in create-vs-no-create, in what path component they stop at, and
|
* two differ in create-vs-no-create, in what path component they stop at, and
|
||||||
* in the extra receiver policies they apply, so they are intentionally kept
|
* in the extra receiver policies they apply, so they are intentionally kept
|
||||||
* separate. Both rely on the shared lexical path_is_within_root check. */
|
* separate. Both rely on the shared lexical path_is_within_root check. */
|
||||||
static int open_authorized_destination(const char* dest_root) {
|
int utils_open_authorized_destination(const char* dest_root) {
|
||||||
int root_fd = utils_get_authorized_root_fd();
|
int root_fd = utils_get_authorized_root_fd();
|
||||||
const char* root_path = utils_get_authorized_root_path();
|
const char* root_path = utils_get_authorized_root_path();
|
||||||
if (root_fd < 0 || !root_path || !dest_root || !path_is_within_root(root_path, dest_root))
|
if (root_fd < 0 || !root_path || !dest_root || !path_is_within_root(root_path, dest_root))
|
||||||
@@ -584,22 +584,41 @@ bool path_under_skip_prefix(const char* child_rel, bool at_root, const DeleteSki
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* All-or-nothing max-delete needs to know BEFORE any unlink whether the run
|
/* Per-run deletion budget and tallies. `max_delete` is the cap on the number
|
||||||
would delete more than max_delete entries. This rehearsal pass walks the
|
of entries the walker may remove (SIZE_MAX = unlimited); once it is reached
|
||||||
destination with the same decisions as the delete pass but never touches the
|
the remaining extras are counted in `skipped` and left in place, matching
|
||||||
filesystem: it counts every regular file the delete pass would unlink and
|
rsync's partial --max-delete behavior. */
|
||||||
every directory it would rmdir (a directory is removed only once every entry
|
typedef struct {
|
||||||
below it has been removed and nothing the walker leaves in place survives).
|
size_t max_delete;
|
||||||
Entries the walker never removes (symlinks, manifest-listed files, protected
|
size_t deleted;
|
||||||
prefixes) mark the enclosing directory as surviving, exactly as they would
|
size_t skipped;
|
||||||
make a real rmdir fail with ENOTEMPTY. Stops early once *count reaches the
|
bool limit_hit;
|
||||||
cap (sets *exceeds). Returns false on a traversal error. */
|
} DeleteBudget;
|
||||||
static bool count_extras_fd(int dirfd, const char* rel_path, const PathIndex* keep, size_t cap,
|
|
||||||
size_t* count, bool* exceeds, const DeleteSkipEntry* skips,
|
/* True when direct children of the directory named by `rel` may be removed.
|
||||||
int skip_count, bool* survives) {
|
With no synchronization info (dirs == NULL) the whole tree is deletable; when
|
||||||
|
a dirs index is supplied only its exact entries are (the receive root is the
|
||||||
|
"." sentinel). */
|
||||||
|
static bool is_synced_dir(const PathIndex* dirs, const char* rel) {
|
||||||
|
if (!dirs)
|
||||||
|
return true;
|
||||||
|
return path_index_contains(dirs, rel[0] == '\0' ? "." : rel);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Remove the extras directly inside the directory open on `dirfd`, recursing
|
||||||
|
into every child directory so kept content below a synchronized prefix is
|
||||||
|
reached. `all_removed` reports whether every child entry was removed (so the
|
||||||
|
caller may rmdir this directory). A child directory is never removed when it
|
||||||
|
is itself a synchronized directory or holds kept content; with a dirs index
|
||||||
|
supplied, direct children of a non-synchronized directory are never extras at
|
||||||
|
all (they are left in place but still descended into). Symlinks are unlinked
|
||||||
|
like any other non-directory extra (never followed). */
|
||||||
|
static bool delete_extras_fd(int dirfd, const char* rel_path, const PathIndex* keep,
|
||||||
|
const PathIndex* dirs, DeleteBudget* budget,
|
||||||
|
const DeleteSkipEntry* skips, int skip_count, bool parent_deletable,
|
||||||
|
bool* all_removed) {
|
||||||
/* openat(dirfd, ".") opens an independent file description: a dup() would
|
/* openat(dirfd, ".") opens an independent file description: a dup() would
|
||||||
share dirfd's file offset, and a prior rehearsal pass must not have drained
|
share dirfd's file offset and a prior pass could leave the stream drained. */
|
||||||
this directory's stream before the delete pass reads it again. */
|
|
||||||
int scanfd = openat(dirfd, ".", O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
int scanfd = openat(dirfd, ".", O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||||
if (scanfd < 0)
|
if (scanfd < 0)
|
||||||
return false;
|
return false;
|
||||||
@@ -610,93 +629,10 @@ static bool count_extras_fd(int dirfd, const char* rel_path, const PathIndex* ke
|
|||||||
}
|
}
|
||||||
bool operation_ok = true;
|
bool operation_ok = true;
|
||||||
bool local_survives = false;
|
bool local_survives = false;
|
||||||
bool at_root = rel_path[0] == '\0';
|
/* A directory is deletable when it or ANY ancestor is synchronized; the
|
||||||
const struct dirent* entry;
|
`parent_deletable` flag carries that down the recursion so dest-only
|
||||||
while ((entry = readdir(dir)) != NULL) {
|
directories below a synchronized root are removed wholesale. */
|
||||||
if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
|
bool deletable = parent_deletable || is_synced_dir(dirs, rel_path);
|
||||||
continue;
|
|
||||||
if (*exceeds)
|
|
||||||
break;
|
|
||||||
char* child_rel = path_cat((char*)rel_path, entry->d_name);
|
|
||||||
if (!child_rel) {
|
|
||||||
operation_ok = false;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (path_under_skip_prefix(child_rel, at_root, skips, skip_count)) {
|
|
||||||
local_survives = true;
|
|
||||||
free(child_rel);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
struct stat st;
|
|
||||||
if (fstatat(dirfd, entry->d_name, &st, AT_SYMLINK_NOFOLLOW) != 0) {
|
|
||||||
if (errno != ENOENT)
|
|
||||||
operation_ok = false;
|
|
||||||
free(child_rel);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (S_ISLNK(st.st_mode)) {
|
|
||||||
local_survives = true;
|
|
||||||
free(child_rel);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (S_ISDIR(st.st_mode)) {
|
|
||||||
int childfd = openat(dirfd, entry->d_name, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
|
||||||
bool child_ok = true;
|
|
||||||
bool child_survives = true;
|
|
||||||
if (childfd >= 0) {
|
|
||||||
child_ok = count_extras_fd(childfd, child_rel, keep, cap, count, exceeds, skips, skip_count,
|
|
||||||
&child_survives);
|
|
||||||
close(childfd);
|
|
||||||
} else if (errno != ENOENT) {
|
|
||||||
operation_ok = false;
|
|
||||||
}
|
|
||||||
if (!child_ok)
|
|
||||||
operation_ok = false;
|
|
||||||
if (keep_is_dir(keep, child_rel)) {
|
|
||||||
/* A directory with kept content below it is never removed. */
|
|
||||||
local_survives = true;
|
|
||||||
} else if (child_survives) {
|
|
||||||
/* The directory still holds entries the walker leaves in place, so an
|
|
||||||
rmdir would fail with ENOTEMPTY; the delete pass leaves it behind
|
|
||||||
rather than reporting an error (matching rsync). */
|
|
||||||
local_survives = true;
|
|
||||||
} else {
|
|
||||||
if (*count >= cap) {
|
|
||||||
*exceeds = true;
|
|
||||||
} else {
|
|
||||||
(*count)++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
bool found = keep_is_file(keep, child_rel);
|
|
||||||
if (!found) {
|
|
||||||
if (*count >= cap) {
|
|
||||||
*exceeds = true;
|
|
||||||
} else {
|
|
||||||
(*count)++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
free(child_rel);
|
|
||||||
}
|
|
||||||
closedir(dir);
|
|
||||||
*survives = local_survives;
|
|
||||||
return operation_ok;
|
|
||||||
}
|
|
||||||
|
|
||||||
static bool delete_extras_fd(int dirfd, const char* rel_path, const PathIndex* keep,
|
|
||||||
size_t max_delete, size_t* deleted_count, const DeleteSkipEntry* skips,
|
|
||||||
int skip_count) {
|
|
||||||
/* Independent file description (see count_extras_fd). */
|
|
||||||
int scanfd = openat(dirfd, ".", O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
|
||||||
if (scanfd < 0)
|
|
||||||
return false;
|
|
||||||
DIR* dir = fdopendir(scanfd);
|
|
||||||
if (!dir) {
|
|
||||||
close(scanfd);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
bool operation_ok = true;
|
|
||||||
const struct dirent* entry;
|
const struct dirent* entry;
|
||||||
while ((entry = readdir(dir)) != NULL) {
|
while ((entry = readdir(dir)) != NULL) {
|
||||||
if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
|
if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
|
||||||
@@ -714,6 +650,7 @@ static bool delete_extras_fd(int dirfd, const char* rel_path, const PathIndex* k
|
|||||||
destination directory that happens to be called .fastsync-stage is
|
destination directory that happens to be called .fastsync-stage is
|
||||||
ordinary content. */
|
ordinary content. */
|
||||||
if (path_under_skip_prefix(child_rel, rel_path[0] == '\0', skips, skip_count)) {
|
if (path_under_skip_prefix(child_rel, rel_path[0] == '\0', skips, skip_count)) {
|
||||||
|
local_survives = true;
|
||||||
free(child_rel);
|
free(child_rel);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -724,28 +661,28 @@ static bool delete_extras_fd(int dirfd, const char* rel_path, const PathIndex* k
|
|||||||
free(child_rel);
|
free(child_rel);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
// Skip symlinks to prevent following them outside the destination tree
|
|
||||||
if (S_ISLNK(st.st_mode)) {
|
|
||||||
free(child_rel);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (S_ISDIR(st.st_mode)) {
|
if (S_ISDIR(st.st_mode)) {
|
||||||
int childfd = openat(dirfd, entry->d_name, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
int childfd = openat(dirfd, entry->d_name, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||||
bool child_removed = false;
|
bool child_all_removed = false;
|
||||||
if (childfd >= 0) {
|
if (childfd >= 0) {
|
||||||
child_removed = delete_extras_fd(childfd, child_rel, keep, max_delete, deleted_count, skips,
|
if (!delete_extras_fd(childfd, child_rel, keep, dirs, budget, skips, skip_count, deletable,
|
||||||
skip_count);
|
&child_all_removed))
|
||||||
if (!child_removed)
|
|
||||||
operation_ok = false;
|
operation_ok = false;
|
||||||
close(childfd);
|
close(childfd);
|
||||||
} else if (errno != ENOENT) {
|
} else if (errno != ENOENT) {
|
||||||
operation_ok = false;
|
operation_ok = false;
|
||||||
}
|
}
|
||||||
if (child_removed && !keep_is_dir(keep, child_rel)) {
|
bool child_synced = dirs && path_index_contains(dirs, child_rel);
|
||||||
if (*deleted_count >= max_delete) {
|
if (child_synced || keep_is_dir(keep, child_rel)) {
|
||||||
operation_ok = false;
|
/* A synchronized directory and a directory holding kept content are
|
||||||
} else {
|
never removed. */
|
||||||
if (unlinkat(dirfd, entry->d_name, AT_REMOVEDIR) != 0) {
|
local_survives = true;
|
||||||
|
} else if (child_all_removed && deletable) {
|
||||||
|
if (budget->deleted >= budget->max_delete) {
|
||||||
|
budget->limit_hit = true;
|
||||||
|
budget->skipped++;
|
||||||
|
local_survives = true;
|
||||||
|
} else if (unlinkat(dirfd, entry->d_name, AT_REMOVEDIR) != 0) {
|
||||||
/* ENOENT: already gone (fine). ENOTEMPTY/EEXIST: the directory
|
/* ENOENT: already gone (fine). ENOTEMPTY/EEXIST: the directory
|
||||||
still holds entries the walker leaves in place (a protected
|
still holds entries the walker leaves in place (a protected
|
||||||
excluded prefix, a kept file the manifest protects, a symlink);
|
excluded prefix, a kept file the manifest protects, a symlink);
|
||||||
@@ -753,26 +690,29 @@ static bool delete_extras_fd(int dirfd, const char* rel_path, const PathIndex* k
|
|||||||
Only genuine I/O failures abort the deletion. */
|
Only genuine I/O failures abort the deletion. */
|
||||||
if (errno != ENOENT && errno != ENOTEMPTY && errno != EEXIST)
|
if (errno != ENOENT && errno != ENOTEMPTY && errno != EEXIST)
|
||||||
operation_ok = false;
|
operation_ok = false;
|
||||||
|
local_survives = true;
|
||||||
} else {
|
} else {
|
||||||
(*deleted_count)++;
|
budget->deleted++;
|
||||||
}
|
}
|
||||||
}
|
} else {
|
||||||
|
local_survives = true;
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// Check if relative path is in manifest
|
|
||||||
bool found = keep_is_file(keep, child_rel);
|
bool found = keep_is_file(keep, child_rel);
|
||||||
if (!found) {
|
if (found || !deletable) {
|
||||||
if (*deleted_count >= max_delete) {
|
/* Kept file, or a child of a directory that is not synchronized: never
|
||||||
operation_ok = false;
|
an extra for this run. */
|
||||||
free(child_rel);
|
local_survives = true;
|
||||||
continue;
|
} else if (budget->deleted >= budget->max_delete) {
|
||||||
}
|
budget->limit_hit = true;
|
||||||
if (unlinkat(dirfd, entry->d_name, 0) != 0) {
|
budget->skipped++;
|
||||||
|
local_survives = true;
|
||||||
|
} else if (unlinkat(dirfd, entry->d_name, 0) != 0) {
|
||||||
if (errno != ENOENT)
|
if (errno != ENOENT)
|
||||||
operation_ok = false;
|
operation_ok = false;
|
||||||
|
local_survives = true;
|
||||||
} else {
|
} else {
|
||||||
(*deleted_count)++;
|
budget->deleted++;
|
||||||
}
|
|
||||||
char* escaped_path = output_escape(child_rel, log_get_8_bit_output());
|
char* escaped_path = output_escape(child_rel, log_get_8_bit_output());
|
||||||
fprintf(stderr, " Deleted: %s\n", escaped_path ? escaped_path : "<allocation failed>");
|
fprintf(stderr, " Deleted: %s\n", escaped_path ? escaped_path : "<allocation failed>");
|
||||||
free(escaped_path);
|
free(escaped_path);
|
||||||
@@ -781,26 +721,125 @@ static bool delete_extras_fd(int dirfd, const char* rel_path, const PathIndex* k
|
|||||||
free(child_rel);
|
free(child_rel);
|
||||||
}
|
}
|
||||||
closedir(dir);
|
closedir(dir);
|
||||||
|
*all_removed = !local_survives;
|
||||||
return operation_ok;
|
return operation_ok;
|
||||||
}
|
}
|
||||||
|
|
||||||
DeleteWalkResult delete_extras_limited(const char* dest_root, const ArrayList* manifest,
|
/* Read-only mirror of delete_extras_fd: records the paths that WOULD be removed
|
||||||
size_t max_delete, const DeleteSkipEntry* skips,
|
without unlinking anything. A child directory is reported after its own
|
||||||
int skip_count, size_t* deleted_out) {
|
reportable children (depth-first), matching the delete pass's ordering. */
|
||||||
if (deleted_out)
|
static bool list_extras_fd(int dirfd, const char* rel_path, const PathIndex* keep,
|
||||||
*deleted_out = 0;
|
const PathIndex* dirs, ArrayList* out, size_t* recorded,
|
||||||
if (!manifest)
|
const DeleteSkipEntry* skips, int skip_count, bool parent_deletable,
|
||||||
return DELETE_WALK_ERROR;
|
bool* all_removed) {
|
||||||
/* Index the keep-set once so both passes answer membership in O(path length)
|
int scanfd = openat(dirfd, ".", O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||||
instead of scanning every manifest entry for every destination entry. */
|
if (scanfd < 0)
|
||||||
|
return false;
|
||||||
|
DIR* dir = fdopendir(scanfd);
|
||||||
|
if (!dir) {
|
||||||
|
close(scanfd);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
bool operation_ok = true;
|
||||||
|
bool local_survives = false;
|
||||||
|
bool deletable = parent_deletable || is_synced_dir(dirs, rel_path);
|
||||||
|
const struct dirent* entry;
|
||||||
|
while ((entry = readdir(dir)) != NULL) {
|
||||||
|
if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
|
||||||
|
continue;
|
||||||
|
char* child_rel = path_cat((char*)rel_path, entry->d_name);
|
||||||
|
if (!child_rel) {
|
||||||
|
operation_ok = false;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (path_under_skip_prefix(child_rel, rel_path[0] == '\0', skips, skip_count)) {
|
||||||
|
local_survives = true;
|
||||||
|
free(child_rel);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
struct stat st;
|
||||||
|
if (fstatat(dirfd, entry->d_name, &st, AT_SYMLINK_NOFOLLOW) != 0) {
|
||||||
|
if (errno != ENOENT)
|
||||||
|
operation_ok = false;
|
||||||
|
free(child_rel);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (S_ISDIR(st.st_mode)) {
|
||||||
|
int childfd = openat(dirfd, entry->d_name, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||||
|
bool child_all_removed = false;
|
||||||
|
if (childfd >= 0) {
|
||||||
|
if (!list_extras_fd(childfd, child_rel, keep, dirs, out, recorded, skips, skip_count,
|
||||||
|
deletable, &child_all_removed))
|
||||||
|
operation_ok = false;
|
||||||
|
close(childfd);
|
||||||
|
} else if (errno != ENOENT) {
|
||||||
|
operation_ok = false;
|
||||||
|
}
|
||||||
|
bool child_synced = dirs && path_index_contains(dirs, child_rel);
|
||||||
|
if (child_synced || keep_is_dir(keep, child_rel)) {
|
||||||
|
local_survives = true;
|
||||||
|
} else if (child_all_removed && deletable) {
|
||||||
|
size_t len = strlen(child_rel);
|
||||||
|
char* copy = malloc(len + 2);
|
||||||
|
if (!copy) {
|
||||||
|
operation_ok = false;
|
||||||
|
} else {
|
||||||
|
memcpy(copy, child_rel, len);
|
||||||
|
copy[len] = '/';
|
||||||
|
copy[len + 1] = '\0';
|
||||||
|
if (!array_list_add(out, copy)) {
|
||||||
|
free(copy);
|
||||||
|
operation_ok = false;
|
||||||
|
} else {
|
||||||
|
(*recorded)++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
local_survives = true;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
bool found = keep_is_file(keep, child_rel);
|
||||||
|
if (found || !deletable) {
|
||||||
|
local_survives = true;
|
||||||
|
} else {
|
||||||
|
char* copy = str_dup(child_rel);
|
||||||
|
if (!copy || !array_list_add(out, copy)) {
|
||||||
|
free(copy);
|
||||||
|
operation_ok = false;
|
||||||
|
} else {
|
||||||
|
(*recorded)++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
free(child_rel);
|
||||||
|
}
|
||||||
|
closedir(dir);
|
||||||
|
*all_removed = !local_survives;
|
||||||
|
return operation_ok;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool delete_extras_list(const char* dest_root, const ArrayList* manifest,
|
||||||
|
const ArrayList* synced_dirs, const DeleteSkipEntry* skips, int skip_count,
|
||||||
|
ArrayList* out, size_t* count_out) {
|
||||||
|
if (count_out)
|
||||||
|
*count_out = 0;
|
||||||
|
if (!manifest || !out)
|
||||||
|
return false;
|
||||||
PathIndex keep;
|
PathIndex keep;
|
||||||
if (!build_keep_index(manifest, &keep))
|
if (!build_keep_index(manifest, &keep))
|
||||||
return DELETE_WALK_ERROR;
|
return false;
|
||||||
|
PathIndex dirs;
|
||||||
|
bool have_dirs = synced_dirs != NULL;
|
||||||
|
if (have_dirs &&
|
||||||
|
!path_index_build(&dirs, (const char* const*)synced_dirs->items, (size_t)synced_dirs->size)) {
|
||||||
|
path_index_free(&keep);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
int rootfd;
|
int rootfd;
|
||||||
int root_fd = utils_get_authorized_root_fd();
|
int root_fd = utils_get_authorized_root_fd();
|
||||||
if (root_fd >= 0) {
|
if (root_fd >= 0) {
|
||||||
if (utils_get_authorized_root_path())
|
if (utils_get_authorized_root_path())
|
||||||
rootfd = open_authorized_destination(dest_root);
|
rootfd = utils_open_authorized_destination(dest_root);
|
||||||
else if (dest_root == NULL)
|
else if (dest_root == NULL)
|
||||||
rootfd = dup(root_fd);
|
rootfd = dup(root_fd);
|
||||||
else
|
else
|
||||||
@@ -810,39 +849,86 @@ DeleteWalkResult delete_extras_limited(const char* dest_root, const ArrayList* m
|
|||||||
}
|
}
|
||||||
if (rootfd < 0) {
|
if (rootfd < 0) {
|
||||||
path_index_free(&keep);
|
path_index_free(&keep);
|
||||||
return DELETE_WALK_ERROR;
|
if (have_dirs)
|
||||||
|
path_index_free(&dirs);
|
||||||
|
return false;
|
||||||
}
|
}
|
||||||
if (max_delete != SIZE_MAX) {
|
bool all_removed = false;
|
||||||
/* Rehearse the deletion first so a run that would exceed the cap removes
|
size_t recorded = 0;
|
||||||
nothing (rsync's all-or-nothing --max-delete contract). */
|
bool ok = list_extras_fd(rootfd, "", &keep, have_dirs ? &dirs : NULL, out, &recorded, skips,
|
||||||
size_t count = 0;
|
skip_count, false, &all_removed);
|
||||||
bool exceeds = false;
|
|
||||||
bool survives = false;
|
|
||||||
bool counted_ok = count_extras_fd(rootfd, "", &keep, max_delete, &count, &exceeds, skips,
|
|
||||||
skip_count, &survives);
|
|
||||||
if (!counted_ok) {
|
|
||||||
close(rootfd);
|
|
||||||
path_index_free(&keep);
|
|
||||||
return DELETE_WALK_ERROR;
|
|
||||||
}
|
|
||||||
if (exceeds) {
|
|
||||||
close(rootfd);
|
|
||||||
path_index_free(&keep);
|
|
||||||
return DELETE_WALK_LIMIT_EXCEEDED;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
size_t deleted_count = 0;
|
|
||||||
bool ok = delete_extras_fd(rootfd, "", &keep, max_delete, &deleted_count, skips, skip_count);
|
|
||||||
if (close(rootfd) != 0)
|
if (close(rootfd) != 0)
|
||||||
ok = false;
|
ok = false;
|
||||||
path_index_free(&keep);
|
path_index_free(&keep);
|
||||||
|
if (have_dirs)
|
||||||
|
path_index_free(&dirs);
|
||||||
|
if (count_out)
|
||||||
|
*count_out = recorded;
|
||||||
|
return ok;
|
||||||
|
}
|
||||||
|
|
||||||
|
DeleteWalkResult delete_extras_limited(const char* dest_root, const ArrayList* manifest,
|
||||||
|
const ArrayList* synced_dirs, size_t max_delete,
|
||||||
|
const DeleteSkipEntry* skips, int skip_count,
|
||||||
|
size_t* deleted_out, size_t* skipped_out) {
|
||||||
if (deleted_out)
|
if (deleted_out)
|
||||||
*deleted_out = deleted_count;
|
*deleted_out = 0;
|
||||||
return ok ? DELETE_WALK_OK : DELETE_WALK_ERROR;
|
if (skipped_out)
|
||||||
|
*skipped_out = 0;
|
||||||
|
if (!manifest)
|
||||||
|
return DELETE_WALK_ERROR;
|
||||||
|
/* Index the keep-set (and the synchronized-dir set, when supplied) once so
|
||||||
|
membership is answered in O(path length) instead of scanning every entry
|
||||||
|
for every destination entry. */
|
||||||
|
PathIndex keep;
|
||||||
|
if (!build_keep_index(manifest, &keep))
|
||||||
|
return DELETE_WALK_ERROR;
|
||||||
|
PathIndex dirs;
|
||||||
|
bool have_dirs = synced_dirs != NULL;
|
||||||
|
if (have_dirs &&
|
||||||
|
!path_index_build(&dirs, (const char* const*)synced_dirs->items, (size_t)synced_dirs->size)) {
|
||||||
|
path_index_free(&keep);
|
||||||
|
return DELETE_WALK_ERROR;
|
||||||
|
}
|
||||||
|
int rootfd;
|
||||||
|
int root_fd = utils_get_authorized_root_fd();
|
||||||
|
if (root_fd >= 0) {
|
||||||
|
if (utils_get_authorized_root_path())
|
||||||
|
rootfd = utils_open_authorized_destination(dest_root);
|
||||||
|
else if (dest_root == NULL)
|
||||||
|
rootfd = dup(root_fd);
|
||||||
|
else
|
||||||
|
rootfd = -1;
|
||||||
|
} else {
|
||||||
|
rootfd = open(dest_root, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||||
|
}
|
||||||
|
if (rootfd < 0) {
|
||||||
|
path_index_free(&keep);
|
||||||
|
if (have_dirs)
|
||||||
|
path_index_free(&dirs);
|
||||||
|
return DELETE_WALK_ERROR;
|
||||||
|
}
|
||||||
|
DeleteBudget budget = {.max_delete = max_delete, .deleted = 0, .skipped = 0, .limit_hit = false};
|
||||||
|
bool all_removed = false;
|
||||||
|
bool ok = delete_extras_fd(rootfd, "", &keep, have_dirs ? &dirs : NULL, &budget, skips,
|
||||||
|
skip_count, false, &all_removed);
|
||||||
|
if (close(rootfd) != 0)
|
||||||
|
ok = false;
|
||||||
|
path_index_free(&keep);
|
||||||
|
if (have_dirs)
|
||||||
|
path_index_free(&dirs);
|
||||||
|
if (deleted_out)
|
||||||
|
*deleted_out = budget.deleted;
|
||||||
|
if (skipped_out)
|
||||||
|
*skipped_out = budget.skipped;
|
||||||
|
if (!ok)
|
||||||
|
return DELETE_WALK_ERROR;
|
||||||
|
return budget.limit_hit ? DELETE_WALK_LIMIT_REACHED : DELETE_WALK_OK;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool delete_extras(const char* dest_root, const ArrayList* manifest) {
|
bool delete_extras(const char* dest_root, const ArrayList* manifest) {
|
||||||
return delete_extras_limited(dest_root, manifest, SIZE_MAX, NULL, 0, NULL) == DELETE_WALK_OK;
|
return delete_extras_limited(dest_root, manifest, NULL, SIZE_MAX, NULL, 0, NULL, NULL) ==
|
||||||
|
DELETE_WALK_OK;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool has_path_traversal(const char* path) {
|
bool has_path_traversal(const char* path) {
|
||||||
|
|||||||
+32
-16
@@ -98,10 +98,10 @@ bool glob_match(const char* pattern, const char* str);
|
|||||||
typedef enum {
|
typedef enum {
|
||||||
/* Every extra entry was removed (or there were none). */
|
/* Every extra entry was removed (or there were none). */
|
||||||
DELETE_WALK_OK = 0,
|
DELETE_WALK_OK = 0,
|
||||||
/* The destination holds more extras than the numeric cap for this run. With
|
/* The numeric cap for this run was reached before every extra was removed.
|
||||||
the all-or-nothing max-delete semantics NOTHING was removed (the walker
|
The walker removed exactly the entries the cap allowed and skipped (without
|
||||||
counts first and refuses to start when the run would exceed the limit). */
|
removing) the rest, matching rsync's partial --max-delete behavior. */
|
||||||
DELETE_WALK_LIMIT_EXCEEDED,
|
DELETE_WALK_LIMIT_REACHED,
|
||||||
/* A traversal or unlink failure aborted the deletion (partial removal is
|
/* A traversal or unlink failure aborted the deletion (partial removal is
|
||||||
possible, mirroring the delete pass). */
|
possible, mirroring the delete pass). */
|
||||||
DELETE_WALK_ERROR
|
DELETE_WALK_ERROR
|
||||||
@@ -122,20 +122,36 @@ typedef struct {
|
|||||||
only DIRECT children of the destination root, i.e. child_rel has no '/'). */
|
only DIRECT children of the destination root, i.e. child_rel has no '/'). */
|
||||||
bool path_under_skip_prefix(const char* child_rel, bool at_root, const DeleteSkipEntry* skips,
|
bool path_under_skip_prefix(const char* child_rel, bool at_root, const DeleteSkipEntry* skips,
|
||||||
int skip_count);
|
int skip_count);
|
||||||
/* Remove files/dirs under dest_root that are not listed in manifest without
|
/* Remove files/dirs/symlinks under dest_root that are not listed in manifest
|
||||||
ever descending into a protected prefix (see DeleteSkipEntry). When
|
without ever descending into a protected prefix (see DeleteSkipEntry). When
|
||||||
max_delete is not SIZE_MAX the run is all-or-nothing: extras are counted
|
`synced_dirs` is non-NULL, extras are only removed directly inside a directory
|
||||||
first and DELETE_WALK_LIMIT_EXCEEDED is returned (with nothing removed) when
|
whose destination-relative path is an exact entry in that list (the receive
|
||||||
the count would exceed the cap. `deleted_out` optionally receives the number
|
root is the "." sentinel); directories outside the synchronized set are still
|
||||||
of entries actually removed. The all-or-nothing guarantee holds only while
|
descended into so kept content below a listed directory is preserved, but
|
||||||
the destination tree is not being concurrently modified: the rehearsal pass
|
nothing in them is removed. A NULL `synced_dirs` keeps the legacy behavior of
|
||||||
and the delete pass are two separate walks, so a concurrent change between
|
treating the whole destination tree as deletable. `max_delete` caps the
|
||||||
them (another process adding/removing entries) can make the second pass
|
number of removed entries (SIZE_MAX = unlimited): the walker removes up to the
|
||||||
delete a different set than the first one counted. */
|
cap and returns DELETE_WALK_LIMIT_REACHED when more extras remained.
|
||||||
|
`deleted_out`/`skipped_out` optionally receive the number of entries removed
|
||||||
|
and the number skipped because of the cap. */
|
||||||
DeleteWalkResult delete_extras_limited(const char* dest_root, const ArrayList* manifest,
|
DeleteWalkResult delete_extras_limited(const char* dest_root, const ArrayList* manifest,
|
||||||
size_t max_delete, const DeleteSkipEntry* skips,
|
const ArrayList* synced_dirs, size_t max_delete,
|
||||||
int skip_count, size_t* deleted_out);
|
const DeleteSkipEntry* skips, int skip_count,
|
||||||
|
size_t* deleted_out, size_t* skipped_out);
|
||||||
|
/* Read-only companion to delete_extras_limited: walk the destination exactly as
|
||||||
|
the delete pass would and APPEND (strdup'd) destination-relative paths that
|
||||||
|
WOULD be removed, without touching disk. Used for -n/--dry-run --delete
|
||||||
|
would-delete reporting. Returns true on a clean walk; the caller owns the
|
||||||
|
strings appended to `out` and receives their count in *count_out. */
|
||||||
|
bool delete_extras_list(const char* dest_root, const ArrayList* manifest,
|
||||||
|
const ArrayList* synced_dirs, const DeleteSkipEntry* skips, int skip_count,
|
||||||
|
ArrayList* out, size_t* count_out);
|
||||||
bool delete_extras(const char* dest_root, const ArrayList* manifest);
|
bool delete_extras(const char* dest_root, const ArrayList* manifest);
|
||||||
|
/* Open the existing destination directory at `dest_root`, confined to the
|
||||||
|
authorized root with an O_NOFOLLOW component walk (the same confinement the
|
||||||
|
deletion walker uses for its root). Returns a new fd the caller owns, or -1
|
||||||
|
on error (including a destination that does not exist). */
|
||||||
|
int utils_open_authorized_destination(const char* dest_root);
|
||||||
bool utils_set_authorized_root(int fd, const char* canonical_path);
|
bool utils_set_authorized_root(int fd, const char* canonical_path);
|
||||||
/* The fd-only compatibility form is fail-closed for path-based operations;
|
/* The fd-only compatibility form is fail-closed for path-based operations;
|
||||||
* callers should use utils_set_authorized_root with the canonical identity. */
|
* callers should use utils_set_authorized_root with the canonical identity. */
|
||||||
|
|||||||
+50
-38
@@ -2,6 +2,7 @@
|
|||||||
#include "xattr.h"
|
#include "xattr.h"
|
||||||
#include "identity.h"
|
#include "identity.h"
|
||||||
#include "log.h"
|
#include "log.h"
|
||||||
|
#include "metadata.h"
|
||||||
#include "protocol.h"
|
#include "protocol.h"
|
||||||
#include "utils.h"
|
#include "utils.h"
|
||||||
#include "file_types.h"
|
#include "file_types.h"
|
||||||
@@ -37,6 +38,22 @@ void xattr_list_free(FileXattrList* list) {
|
|||||||
free(list);
|
free(list);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
FileXattrList* xattr_list_clone(const FileXattrList* list) {
|
||||||
|
if (!list)
|
||||||
|
return NULL;
|
||||||
|
FileXattrList* clone = xattr_list_new();
|
||||||
|
if (!clone)
|
||||||
|
return NULL;
|
||||||
|
for (int i = 0; i < list->count; i++) {
|
||||||
|
if (!xattr_list_append(clone, list->items[i].name, list->items[i].value,
|
||||||
|
list->items[i].value_len)) {
|
||||||
|
xattr_list_free(clone);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return clone;
|
||||||
|
}
|
||||||
|
|
||||||
bool xattr_list_append(FileXattrList* list, const char* name, const void* value, size_t value_len) {
|
bool xattr_list_append(FileXattrList* list, const char* name, const void* value, size_t value_len) {
|
||||||
if (!list || !name || (!value && value_len != 0))
|
if (!list || !name || (!value && value_len != 0))
|
||||||
return false;
|
return false;
|
||||||
@@ -364,26 +381,12 @@ void fake_super_store_fd(int fd, uint32_t uid, uint32_t gid, uint32_t mode, int6
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/* --fake-super replay: read the freshly-stored record and re-apply the source
|
/* --fake-super replay: read the freshly-stored record and re-apply mode/mtime
|
||||||
* stat fd-relative. A privileged (root) run can actually change the owner;
|
* fd-relative. The recorded uid/gid are retained for a later privileged
|
||||||
* a non-root run silently skips the fchown on EPERM/EACCES (never fatal,
|
* restore but are NEVER chowned here: --fake-super only RECORDS ownership, it
|
||||||
* mirroring the normal metadata identity path; other errors are logged) and
|
* must not real-chown the recorded (resolved) owner. Mode/mtime still apply so
|
||||||
* still applies mode/mtime where permitted.
|
* unprivileged --fake-super keeps working. */
|
||||||
*
|
bool fake_super_restore_fd(int fd, FileAttrPolicy policy) {
|
||||||
* The OWNER leg additionally honors three policies:
|
|
||||||
* - an explicit ownership identity policy must be active (numeric-ids /
|
|
||||||
* chown / usermap / groupmap / copy-as). --fake-super on its own only
|
|
||||||
* RECORDS the source owner; replaying that owner as a live chown without an
|
|
||||||
* explicit ownership opt-in would be an un-gated client-chosen-ownership
|
|
||||||
* primitive.
|
|
||||||
* - --no-super (privilege_super_permitted() false) suppresses it even for a
|
|
||||||
* root receiver, exactly like the normal metadata identity path.
|
|
||||||
* - an active --copy-as is AUTHORITATIVE: the identity path already forced the
|
|
||||||
* target owner, so replaying the recorded source owner here would silently
|
|
||||||
* override it. The xattr record is still stored/replayed for a later
|
|
||||||
* privileged restore; only the live chown is skipped. Mode/mtime remain
|
|
||||||
* applied either way so unprivileged --fake-super still works. */
|
|
||||||
bool fake_super_restore_fd(int fd) {
|
|
||||||
if (fd < 0)
|
if (fd < 0)
|
||||||
return false;
|
return false;
|
||||||
char record[128];
|
char record[128];
|
||||||
@@ -398,28 +401,37 @@ bool fake_super_restore_fd(int fd) {
|
|||||||
5)
|
5)
|
||||||
return false; /* malformed record: skip, never fatal */
|
return false; /* malformed record: skip, never fatal */
|
||||||
|
|
||||||
/* Owner is applied best-effort only: a non-root process cannot chown and
|
/* --fake-super NEVER performs a real chown: that would defeat the whole
|
||||||
must not abort the transfer for that reason (FastSync identity philosophy).
|
point of the flag (record privileged ownership on an unprivileged receiver
|
||||||
EPERM/EACCES (expected for a non-root receiver) are skipped silently; a
|
for a later privileged restore). The uid/gid parsed above are retained in
|
||||||
genuine EINVAL (an impossible stored id) is logged so the corruption is
|
the record for that later restore, but no ownership change happens here. */
|
||||||
not hidden. --no-super suppresses the owner leg even for root, and an
|
(void)ul_uid;
|
||||||
active --copy-as is authoritative so its forced owner must not be
|
(void)ul_gid;
|
||||||
overwritten by the recorded source owner. */
|
/* Mode is applied only when the per-attribute policy asks for it, through the
|
||||||
if (identity_active_enabled() && privilege_super_permitted() && !identity_copy_as_active() &&
|
SAME shared helper the normal metadata path uses (metadata_mode_for_policy):
|
||||||
fchown(fd, (uid_t)ul_uid, (gid_t)ul_gid) != 0 && errno != EPERM && errno != EACCES)
|
under --perms the recorded source mode is copied exactly, including
|
||||||
log_message(LOG_LEVEL_WARNING, "--fake-super: could not restore owner on destination file: %s",
|
group/other write and setuid/setgid/sticky bits (rsync parity), and the -E
|
||||||
|
rule derives exec bits from the destination's read bits exactly like
|
||||||
|
file_restore_metadata_fd. */
|
||||||
|
if (policy.perms || policy.executability) {
|
||||||
|
struct stat cur;
|
||||||
|
mode_t want = 0;
|
||||||
|
if (fstat(fd, &cur) != 0) {
|
||||||
|
log_message(LOG_LEVEL_WARNING, "--fake-super: could not read destination mode: %s",
|
||||||
strerror(errno));
|
strerror(errno));
|
||||||
/* Mode is applied through the same sanitization the normal metadata path
|
} else if (metadata_mode_for_policy((mode_t)ul_mode, cur.st_mode, policy, &want)) {
|
||||||
uses (metadata_mode): group/other write bits are never granted, so a
|
if (fchmod(fd, want) != 0)
|
||||||
recorded source mode of 0666 restores as 0644 — identical to a non-fake-
|
log_message(LOG_LEVEL_WARNING,
|
||||||
super --preserve run, never a privilege-granting regression. */
|
"--fake-super: could not restore mode on destination file: %s",
|
||||||
if (fchmod(fd, (mode_t)(ul_mode & 0777U & ~(S_IWGRP | S_IWOTH))) != 0)
|
|
||||||
log_message(LOG_LEVEL_WARNING, "--fake-super: could not restore mode on destination file: %s",
|
|
||||||
strerror(errno));
|
strerror(errno));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (policy.times) {
|
||||||
struct timespec times[2] = {{.tv_sec = 0, .tv_nsec = UTIME_OMIT},
|
struct timespec times[2] = {{.tv_sec = 0, .tv_nsec = UTIME_OMIT},
|
||||||
{.tv_sec = (time_t)mtime_sec, .tv_nsec = mtime_nsec}};
|
{.tv_sec = (time_t)mtime_sec, .tv_nsec = mtime_nsec}};
|
||||||
if (futimens(fd, times) != 0)
|
if (futimens(fd, times) != 0)
|
||||||
log_message(LOG_LEVEL_WARNING, "--fake-super: could not restore mtime on destination file: %s",
|
log_message(LOG_LEVEL_WARNING,
|
||||||
strerror(errno));
|
"--fake-super: could not restore mtime on destination file: %s", strerror(errno));
|
||||||
|
}
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
+15
-10
@@ -1,6 +1,7 @@
|
|||||||
#ifndef XATTR_H
|
#ifndef XATTR_H
|
||||||
#define XATTR_H
|
#define XATTR_H
|
||||||
|
|
||||||
|
#include "file_attr.h"
|
||||||
#include <stdbool.h>
|
#include <stdbool.h>
|
||||||
#include <stddef.h>
|
#include <stddef.h>
|
||||||
#include <stdint.h>
|
#include <stdint.h>
|
||||||
@@ -55,6 +56,8 @@ typedef struct {
|
|||||||
|
|
||||||
FileXattrList* xattr_list_new(void);
|
FileXattrList* xattr_list_new(void);
|
||||||
void xattr_list_free(FileXattrList* list);
|
void xattr_list_free(FileXattrList* list);
|
||||||
|
/* Deep-copy `list` (NULL in, NULL out). Returns NULL on allocation failure. */
|
||||||
|
FileXattrList* xattr_list_clone(const FileXattrList* list);
|
||||||
/* Append one entry (deep copy). Returns false on allocation failure. */
|
/* Append one entry (deep copy). Returns false on allocation failure. */
|
||||||
bool xattr_list_append(FileXattrList* list, const char* name, const void* value, size_t value_len);
|
bool xattr_list_append(FileXattrList* list, const char* name, const void* value, size_t value_len);
|
||||||
|
|
||||||
@@ -95,15 +98,17 @@ void fake_super_store_fd(int fd, uint32_t uid, uint32_t gid, uint32_t mode, int6
|
|||||||
int64_t mtime_nsec);
|
int64_t mtime_nsec);
|
||||||
|
|
||||||
/* --fake-super replay: parse the FAKESUPER_XATTR record previously written on
|
/* --fake-super replay: parse the FAKESUPER_XATTR record previously written on
|
||||||
* `fd` by fake_super_store_fd and re-apply uid/gid/mode/mtime fd-relative.
|
* `fd` by fake_super_store_fd and re-apply mode/mtime fd-relative. The
|
||||||
* Best-effort: absence of the xattr or a malformed record is a silent no-op
|
* recorded uid/gid are deliberately NOT chowned for real: --fake-super only
|
||||||
* that never fails the transfer. The OWNER leg is applied only when an explicit
|
* RECORDS ownership (the caller stores the resolved mapping via
|
||||||
* ownership identity policy is active (numeric-ids/chown/usermap/groupmap/
|
* identity_resolve_storage_ids), it never performs a real chown. Best-effort:
|
||||||
* copy-as), when super-user activities are permitted, and when --copy-as is not
|
* absence of the xattr or a malformed record is a silent no-op that never fails
|
||||||
* authoritative; a non-root EPERM/EACCES is skipped silently, matching
|
* the transfer. The MODE leg is applied only when policy.perms||policy.
|
||||||
* FastSync's identity philosophy. The mode is sanitized exactly like the normal
|
* executability and the MTIME leg only when policy.times, so the fake-super
|
||||||
* metadata path (group/other write bits never granted). Returns true when the
|
* replay cannot bypass the per-attribute split; the mode follows the normal
|
||||||
* xattr was present and parsed. */
|
* metadata path exactly (under --perms the source mode is copied verbatim,
|
||||||
bool fake_super_restore_fd(int fd);
|
* special and group/other write bits included).
|
||||||
|
* Returns true when the xattr was present and parsed. */
|
||||||
|
bool fake_super_restore_fd(int fd, FileAttrPolicy policy);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
@@ -115,7 +115,9 @@ static void build_canonical_frame(void) {
|
|||||||
if (cfg->usermap) {
|
if (cfg->usermap) {
|
||||||
cfg->usermap_count = 1;
|
cfg->usermap_count = 1;
|
||||||
cfg->usermap[0].from = MAP_FROM;
|
cfg->usermap[0].from = MAP_FROM;
|
||||||
|
cfg->usermap[0].from_hi = MAP_FROM;
|
||||||
cfg->usermap[0].to = MAP_TO;
|
cfg->usermap[0].to = MAP_TO;
|
||||||
|
cfg->usermap[0].to_name = NULL;
|
||||||
}
|
}
|
||||||
if (!cfg->send_directory || !cfg->receive_root_directory || !cfg->usermap) {
|
if (!cfg->send_directory || !cfg->receive_root_directory || !cfg->usermap) {
|
||||||
config_delete(cfg);
|
config_delete(cfg);
|
||||||
|
|||||||
@@ -101,9 +101,15 @@ class CountingProxy:
|
|||||||
return
|
return
|
||||||
counter[0] += len(data)
|
counter[0] += len(data)
|
||||||
|
|
||||||
def run(self, cmd):
|
def run(self, cmd, join_timeout=20):
|
||||||
"""Forward one client run (the full command list) to the real server and
|
"""Forward one client run (the full command list) to the real server and
|
||||||
return the CompletedProcess after the counts have settled."""
|
return the CompletedProcess after the counts have settled.
|
||||||
|
|
||||||
|
``join_timeout`` bounds how long to wait for the forwarding threads. The
|
||||||
|
client->server count is published as soon as the client side reaches EOF
|
||||||
|
(i.e. once the client process has exited), so callers that only need that
|
||||||
|
count can pass a small value instead of waiting for the server to close
|
||||||
|
its idle socket."""
|
||||||
|
|
||||||
def serve():
|
def serve():
|
||||||
try:
|
try:
|
||||||
@@ -119,15 +125,15 @@ class CountingProxy:
|
|||||||
a.start()
|
a.start()
|
||||||
b.start()
|
b.start()
|
||||||
a.join()
|
a.join()
|
||||||
b.join()
|
|
||||||
self.client_to_server = c2s[0]
|
self.client_to_server = c2s[0]
|
||||||
|
b.join()
|
||||||
self.server_to_client = s2c[0]
|
self.server_to_client = s2c[0]
|
||||||
self._listener.close()
|
self._listener.close()
|
||||||
|
|
||||||
thread = threading.Thread(target=serve)
|
thread = threading.Thread(target=serve, daemon=True)
|
||||||
thread.start()
|
thread.start()
|
||||||
result = subprocess.run(cmd, capture_output=True, text=True, timeout=180)
|
result = subprocess.run(cmd, capture_output=True, text=True, timeout=180)
|
||||||
thread.join(20)
|
thread.join(join_timeout)
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
@@ -254,6 +260,13 @@ def _wait_for_port(port, timeout=5):
|
|||||||
|
|
||||||
|
|
||||||
def _wait_proc(proc, timeout=5):
|
def _wait_proc(proc, timeout=5):
|
||||||
|
"""Stop a long-lived subprocess promptly. The server installs a SIGTERM
|
||||||
|
handler, so signal first and only escalate to SIGKILL if it does not exit;
|
||||||
|
waiting without signalling would burn the full timeout on every stop."""
|
||||||
|
if proc.poll() is not None:
|
||||||
|
proc.wait()
|
||||||
|
return
|
||||||
|
proc.terminate()
|
||||||
try:
|
try:
|
||||||
proc.wait(timeout=timeout)
|
proc.wait(timeout=timeout)
|
||||||
except subprocess.TimeoutExpired:
|
except subprocess.TimeoutExpired:
|
||||||
|
|||||||
@@ -0,0 +1,218 @@
|
|||||||
|
"""Differential tests for --checksum-choice / --compress-choice against rsync 3.4.1.
|
||||||
|
|
||||||
|
These pin the accepted/rejected algorithm matrix and exit codes to real rsync,
|
||||||
|
and verify that every codec FastSync now offers still transfers byte-exactly.
|
||||||
|
The rsync-based tests skip cleanly when rsync is not installed.
|
||||||
|
|
||||||
|
The FastSync server confines transfers to its authorized root (the project
|
||||||
|
directory when the shared test server is launched), so every scratch tree lives
|
||||||
|
under ``TEST_DATA_DIR`` rather than pytest's ``tmp_path``.
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
sys.path.insert(0, os.path.dirname(__file__))
|
||||||
|
from common import (
|
||||||
|
TEST_DATA_DIR,
|
||||||
|
run_client,
|
||||||
|
clean_dir,
|
||||||
|
get_dest_received_dir,
|
||||||
|
)
|
||||||
|
|
||||||
|
RSYNC = shutil.which("rsync")
|
||||||
|
requires_rsync = pytest.mark.skipif(RSYNC is None, reason="rsync 3.4.1 not installed")
|
||||||
|
|
||||||
|
CHECKSUM_NAMES = ["xxh128", "xxh3", "xxh64", "md5", "md4", "sha1"]
|
||||||
|
COMPRESS_NAMES = ["zstd", "lz4", "zlib", "zlibx"]
|
||||||
|
|
||||||
|
CODEC_ROOT = os.path.join(TEST_DATA_DIR, "codec_differential")
|
||||||
|
|
||||||
|
|
||||||
|
def _rsync(args):
|
||||||
|
env = dict(os.environ, LC_ALL="C")
|
||||||
|
return subprocess.run([RSYNC] + args, capture_output=True, text=True, env=env, timeout=120)
|
||||||
|
|
||||||
|
|
||||||
|
def _scratch(tag):
|
||||||
|
"""A confined, uniquely named scratch directory under the project tree."""
|
||||||
|
path = os.path.join(CODEC_ROOT, tag)
|
||||||
|
clean_dir(path)
|
||||||
|
os.makedirs(path, exist_ok=True)
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def _make_corpus(root):
|
||||||
|
clean_dir(root)
|
||||||
|
os.makedirs(os.path.join(root, "sub"), exist_ok=True)
|
||||||
|
# Highly compressible payload so each codec is actually exercised.
|
||||||
|
with open(os.path.join(root, "big.bin"), "wb") as fh:
|
||||||
|
fh.write(b"FastSync codec payload " * 4096)
|
||||||
|
with open(os.path.join(root, "sub", "text.txt"), "wb") as fh:
|
||||||
|
fh.write(b"hello codec world\n" * 128)
|
||||||
|
with open(os.path.join(root, "empty"), "wb"):
|
||||||
|
pass
|
||||||
|
return root
|
||||||
|
|
||||||
|
|
||||||
|
def _tree_bytes(root):
|
||||||
|
out = {}
|
||||||
|
for dirpath, _dirs, files in os.walk(root):
|
||||||
|
for name in files:
|
||||||
|
path = os.path.join(dirpath, name)
|
||||||
|
with open(path, "rb") as fh:
|
||||||
|
out[os.path.relpath(path, root)] = fh.read()
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
class TestCodecChoiceMatrix:
|
||||||
|
"""The CLI accept/reject set and exit codes must match rsync 3.4.1."""
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
@pytest.mark.ci
|
||||||
|
@pytest.mark.parametrize("name", CHECKSUM_NAMES)
|
||||||
|
def test_checksum_names_accepted_by_both(self, name, shared_server):
|
||||||
|
src = _make_corpus(_scratch(f"cc_src_{name}"))
|
||||||
|
rdst = _scratch(f"cc_rsync_{name}")
|
||||||
|
rsync_result = _rsync(["-a", f"--cc={name}", src + "/", rdst + "/"])
|
||||||
|
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||||
|
|
||||||
|
fdst = _scratch(f"cc_fs_{name}")
|
||||||
|
result, _ = run_client(src, fdst, flags=[f"--cc={name}"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
@pytest.mark.ci
|
||||||
|
@pytest.mark.parametrize("name", COMPRESS_NAMES)
|
||||||
|
def test_compress_names_accepted_by_both(self, name, shared_server):
|
||||||
|
src = _make_corpus(_scratch(f"zc_src_{name}"))
|
||||||
|
rdst = _scratch(f"zc_rsync_{name}")
|
||||||
|
rsync_result = _rsync(["-az", f"--zc={name}", src + "/", rdst + "/"])
|
||||||
|
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||||
|
|
||||||
|
fdst = _scratch(f"zc_fs_{name}")
|
||||||
|
result, _ = run_client(src, fdst, flags=["-z", f"--zc={name}"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
@pytest.mark.ci
|
||||||
|
@pytest.mark.parametrize("choice", ["md4,sha1", "sha1,md4", "auto,md5", "none,md5"])
|
||||||
|
def test_checksum_two_name_accepted_by_both(self, choice, shared_server):
|
||||||
|
tag = choice.replace(",", "_")
|
||||||
|
src = _make_corpus(_scratch(f"two_src_{tag}"))
|
||||||
|
rdst = _scratch(f"two_rsync_{tag}")
|
||||||
|
rsync_result = _rsync(["-a", "--checksum", f"--cc={choice}", src + "/", rdst + "/"])
|
||||||
|
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||||
|
|
||||||
|
fdst = _scratch(f"two_fs_{tag}")
|
||||||
|
result, _ = run_client(src, fdst, flags=["--checksum", f"--cc={choice}"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, (result.stderr or result.stdout)[:200]
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
@pytest.mark.ci
|
||||||
|
@pytest.mark.parametrize("name", ["sha256", "crc32", "md5,", "md4,md5,sha1"])
|
||||||
|
def test_unknown_checksum_rejected_exit_4_both(self, name, shared_server):
|
||||||
|
src = _make_corpus(_scratch(f"badcc_src_{name.replace(',', '_').replace(':', '_')}"))
|
||||||
|
rdst = _scratch(f"badcc_rsync_{name.replace(',', '_').replace(':', '_')}")
|
||||||
|
rsync_result = _rsync(["-a", f"--cc={name}", src + "/", rdst + "/"])
|
||||||
|
assert rsync_result.returncode == 4, rsync_result.stderr
|
||||||
|
|
||||||
|
fdst = _scratch(f"badcc_fs_{name.replace(',', '_').replace(':', '_')}")
|
||||||
|
result, _ = run_client(src, fdst, flags=[f"--cc={name}"], port=shared_server.port)
|
||||||
|
assert result.returncode == 4, (result.stderr or result.stdout)[:200]
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
@pytest.mark.ci
|
||||||
|
@pytest.mark.parametrize("choice", ["none", "md5,none"])
|
||||||
|
def test_checksum_none_with_checksum_rejected_exit_4_both(self, choice, shared_server):
|
||||||
|
tag = choice.replace(",", "_")
|
||||||
|
src = _make_corpus(_scratch(f"nonecc_src_{tag}"))
|
||||||
|
rdst = _scratch(f"nonecc_rsync_{tag}")
|
||||||
|
rsync_result = _rsync(["-a", "--checksum", f"--cc={choice}", src + "/", rdst + "/"])
|
||||||
|
assert rsync_result.returncode == 4, rsync_result.stderr
|
||||||
|
|
||||||
|
fdst = _scratch(f"nonecc_fs_{tag}")
|
||||||
|
result, _ = run_client(src, fdst, flags=["--checksum", f"--cc={choice}"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 4, (result.stderr or result.stdout)[:200]
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
@pytest.mark.ci
|
||||||
|
@pytest.mark.parametrize("name", ["bogus", "zstd,lz4"])
|
||||||
|
def test_unknown_compress_rejected_exit_4_both(self, name, shared_server):
|
||||||
|
tag = name.replace(",", "_")
|
||||||
|
src = _make_corpus(_scratch(f"badzc_src_{tag}"))
|
||||||
|
rdst = _scratch(f"badzc_rsync_{tag}")
|
||||||
|
rsync_result = _rsync(["-az", f"--zc={name}", src + "/", rdst + "/"])
|
||||||
|
assert rsync_result.returncode == 4, rsync_result.stderr
|
||||||
|
|
||||||
|
fdst = _scratch(f"badzc_fs_{tag}")
|
||||||
|
result, _ = run_client(src, fdst, flags=["-z", f"--zc={name}"], port=shared_server.port)
|
||||||
|
assert result.returncode == 4, (result.stderr or result.stdout)[:200]
|
||||||
|
|
||||||
|
|
||||||
|
class TestCodecTransferDifferential:
|
||||||
|
"""Each codec lands the same bytes rsync lands."""
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
@pytest.mark.ci
|
||||||
|
@pytest.mark.parametrize("name", COMPRESS_NAMES + ["none"])
|
||||||
|
def test_compress_codec_matches_rsync_bytes(self, name, shared_server):
|
||||||
|
src = _make_corpus(_scratch(f"byteszc_src_{name}"))
|
||||||
|
rsync_dst = _scratch(f"byteszc_rsync_{name}")
|
||||||
|
rsync_result = _rsync(["-a", "-z", f"--zc={name}", src + "/", rsync_dst + "/"])
|
||||||
|
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||||
|
|
||||||
|
fs_dst = _scratch(f"byteszc_fs_{name}")
|
||||||
|
result, _ = run_client(src, fs_dst, flags=["-a", "-z", f"--zc={name}"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||||
|
received = get_dest_received_dir(fs_dst, src)
|
||||||
|
assert _tree_bytes(received) == _tree_bytes(rsync_dst)
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
@pytest.mark.ci
|
||||||
|
@pytest.mark.parametrize("name", CHECKSUM_NAMES)
|
||||||
|
def test_checksum_codec_matches_rsync_bytes(self, name, shared_server):
|
||||||
|
src = _make_corpus(_scratch(f"bytescc_src_{name}"))
|
||||||
|
rsync_dst = _scratch(f"bytescc_rsync_{name}")
|
||||||
|
rsync_result = _rsync(["-a", "--checksum", f"--cc={name}", src + "/", rsync_dst + "/"])
|
||||||
|
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||||
|
|
||||||
|
fs_dst = _scratch(f"bytescc_fs_{name}")
|
||||||
|
result, _ = run_client(src, fs_dst, flags=["-a", "--checksum", f"--cc={name}"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||||
|
received = get_dest_received_dir(fs_dst, src)
|
||||||
|
assert _tree_bytes(received) == _tree_bytes(rsync_dst)
|
||||||
|
|
||||||
|
|
||||||
|
class TestCodecNegotiationFallback:
|
||||||
|
"""FastSync's auto negotiation and deterministic fallback order."""
|
||||||
|
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_default_checksum_and_compression_agree(self, shared_server):
|
||||||
|
"""A default transfer (auto on both peers) succeeds; the negotiated
|
||||||
|
default is xxh128 + zstd."""
|
||||||
|
src = _make_corpus(_scratch("auto_src"))
|
||||||
|
fdst = _scratch("auto_fs")
|
||||||
|
result, _ = run_client(src, fdst, flags=["-a", "-z"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||||
|
received = get_dest_received_dir(fdst, src)
|
||||||
|
assert _tree_bytes(received) == _tree_bytes(src)
|
||||||
|
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_explicit_choice_overrides_auto(self, shared_server):
|
||||||
|
"""An explicit --zc/--cc wins over the negotiated default on both ends,
|
||||||
|
so the receiver decodes with the sender's codec."""
|
||||||
|
src = _make_corpus(_scratch("explicit_src"))
|
||||||
|
fdst = _scratch("explicit_fs")
|
||||||
|
result, _ = run_client(src, fdst, flags=["-a", "-z", "--zc=lz4", "--cc=sha1"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||||
|
received = get_dest_received_dir(fdst, src)
|
||||||
|
assert _tree_bytes(received) == _tree_bytes(src)
|
||||||
@@ -0,0 +1,358 @@
|
|||||||
|
"""Differential + regression coverage for rsync's delete timing.
|
||||||
|
|
||||||
|
``--delete-during``/``--delete-delay`` stream a per-directory delete plan instead
|
||||||
|
of one whole-tree manifest, so the timing is observable:
|
||||||
|
|
||||||
|
* ``--delete-during`` removes a directory's extras as it processes that
|
||||||
|
directory (so an interrupted transfer has already removed the extras of the
|
||||||
|
directories it reached);
|
||||||
|
* ``--delete-delay`` snapshots those extras while scanning and commits the
|
||||||
|
removals only after a fully-successful transfer (so an extra created in the
|
||||||
|
destination after its directory's plan survives, and a failed transfer
|
||||||
|
removes nothing);
|
||||||
|
* ``--delete-after`` re-scans the destination at the end (so that same
|
||||||
|
late-created extra is removed).
|
||||||
|
|
||||||
|
The final-state tests compare against real ``rsync 3.4.1`` where a deterministic
|
||||||
|
comparison exists; the timing tests use a byte-slicing proxy to force a
|
||||||
|
mid-transfer failure or to create a destination entry while the transfer is in
|
||||||
|
flight.
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
import select
|
||||||
|
import shutil
|
||||||
|
import socket
|
||||||
|
import struct
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
sys.path.insert(0, os.path.dirname(__file__))
|
||||||
|
from common import ( # noqa: E402
|
||||||
|
BUILD_DIR,
|
||||||
|
TEST_DATA_DIR,
|
||||||
|
ServerManager,
|
||||||
|
clean_dir,
|
||||||
|
get_dest_received_dir,
|
||||||
|
run_client,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Every test here is deterministic (the proxy throttles until the delete-plan
|
||||||
|
# frames are processed), so the PR gate runs the whole module.
|
||||||
|
pytestmark = pytest.mark.ci
|
||||||
|
|
||||||
|
RSYNC = shutil.which("rsync")
|
||||||
|
requires_rsync = pytest.mark.skipif(RSYNC is None, reason="rsync 3.4.1 not installed")
|
||||||
|
|
||||||
|
BIG_BYTES = 8 * 1024 * 1024
|
||||||
|
# Forward/cut this far into the stream: past the (small) delete-plan frames and
|
||||||
|
# well into the big payload, so the receiver has already processed the plan.
|
||||||
|
MID_TRANSFER_BYTES = 256 * 1024
|
||||||
|
# Throttle the proxy so the receiver keeps up with the (fast) client and the
|
||||||
|
# plan frames are provably processed before the hook/cut offset is reached.
|
||||||
|
PROXY_THROTTLE = 0.001
|
||||||
|
|
||||||
|
|
||||||
|
def _write(path, content):
|
||||||
|
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||||
|
with open(path, "wb") as fh:
|
||||||
|
fh.write(content)
|
||||||
|
|
||||||
|
|
||||||
|
def _seed_pair(tag, big=False):
|
||||||
|
"""Create a source tree and a destination mirror seeded with extras.
|
||||||
|
|
||||||
|
The tree is a single directory ``d`` containing the transferred files plus,
|
||||||
|
in the destination, an extra ``d/old_extra``.
|
||||||
|
"""
|
||||||
|
source = os.path.join(TEST_DATA_DIR, f"dtp_{tag}_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, f"dtp_{tag}_dst")
|
||||||
|
clean_dir(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
_write(os.path.join(source, "d", "keep.txt"), b"kept payload\n")
|
||||||
|
if big:
|
||||||
|
_write(os.path.join(source, "d", "big.bin"), b"B" * BIG_BYTES)
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
os.makedirs(os.path.join(received, "d"), exist_ok=True)
|
||||||
|
_write(os.path.join(received, "d", "old_extra"), b"stale extra\n")
|
||||||
|
return source, dest, received
|
||||||
|
|
||||||
|
|
||||||
|
def _tree(root):
|
||||||
|
"""Sorted relative paths of every entry below root (files and dirs)."""
|
||||||
|
out = []
|
||||||
|
for dirpath, dirs, files in os.walk(root):
|
||||||
|
for name in dirs:
|
||||||
|
out.append(os.path.relpath(os.path.join(dirpath, name), root))
|
||||||
|
for name in files:
|
||||||
|
out.append(os.path.relpath(os.path.join(dirpath, name), root))
|
||||||
|
return sorted(out)
|
||||||
|
|
||||||
|
|
||||||
|
def _rsync(args):
|
||||||
|
env = dict(os.environ, LC_ALL="C")
|
||||||
|
return subprocess.run([RSYNC] + args, capture_output=True, text=True, env=env, timeout=120)
|
||||||
|
|
||||||
|
|
||||||
|
class _SlicingProxy:
|
||||||
|
"""Forward the client stream to a server, optionally cutting it or invoking a
|
||||||
|
hook after a byte threshold. ``forward_limit`` mode resets both ends after
|
||||||
|
that many client bytes (a mid-transfer failure). ``hook`` mode calls the
|
||||||
|
hook once and keeps forwarding to completion.
|
||||||
|
|
||||||
|
With ``wait_for_reply`` the hook is a real barrier, not a timing guess: it
|
||||||
|
fires only after the server has sent *any* reply, which the receiver does
|
||||||
|
only after it has consumed the frames that precede the payload (the
|
||||||
|
per-directory delete plan for ``--delete-delay``). The caller pairs it with
|
||||||
|
``--incremental`` so a per-file handshake reply is guaranteed mid-transfer.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, target_port, forward_limit=None, hook=None, hook_after=0,
|
||||||
|
throttle=0.0, wait_for_reply=False):
|
||||||
|
self.target = ("127.0.0.1", target_port)
|
||||||
|
self.forward_limit = forward_limit
|
||||||
|
self.hook = hook
|
||||||
|
self.hook_after = hook_after
|
||||||
|
self.throttle = throttle
|
||||||
|
self.wait_for_reply = wait_for_reply
|
||||||
|
self.server_replied = threading.Event()
|
||||||
|
self.hook_called = threading.Event()
|
||||||
|
self.listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||||
|
self.listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||||
|
self.listener.bind(("127.0.0.1", 0))
|
||||||
|
self.listener.listen(1)
|
||||||
|
self.listener.settimeout(20)
|
||||||
|
self.port = self.listener.getsockname()[1]
|
||||||
|
self._thread = threading.Thread(target=self._serve, daemon=True)
|
||||||
|
self._thread.start()
|
||||||
|
|
||||||
|
def _serve(self):
|
||||||
|
try:
|
||||||
|
client, _ = self.listener.accept()
|
||||||
|
except OSError:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
backend = socket.create_connection(self.target, timeout=10)
|
||||||
|
except OSError:
|
||||||
|
client.close()
|
||||||
|
return
|
||||||
|
client.settimeout(20)
|
||||||
|
backend.settimeout(20)
|
||||||
|
forwarded = 0
|
||||||
|
socks = [client, backend]
|
||||||
|
try:
|
||||||
|
while socks:
|
||||||
|
ready, _, _ = select.select(socks, [], [], 20)
|
||||||
|
if not ready:
|
||||||
|
break
|
||||||
|
for sock in ready:
|
||||||
|
data = sock.recv(65536)
|
||||||
|
if not data:
|
||||||
|
socks.remove(sock)
|
||||||
|
peer = backend if sock is client else client
|
||||||
|
try:
|
||||||
|
peer.shutdown(socket.SHUT_WR)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
continue
|
||||||
|
if sock is client:
|
||||||
|
if self.forward_limit is not None:
|
||||||
|
room = self.forward_limit - forwarded
|
||||||
|
if room <= 0:
|
||||||
|
socks = []
|
||||||
|
break
|
||||||
|
data = data[:room]
|
||||||
|
backend.sendall(data)
|
||||||
|
forwarded += len(data)
|
||||||
|
self._maybe_hook(forwarded)
|
||||||
|
if self.forward_limit is not None and forwarded >= self.forward_limit:
|
||||||
|
socks = []
|
||||||
|
break
|
||||||
|
if self.throttle > 0:
|
||||||
|
time.sleep(self.throttle)
|
||||||
|
else:
|
||||||
|
client.sendall(data)
|
||||||
|
# Any server reply proves the receiver consumed the
|
||||||
|
# frames that precede it, so the hook barrier is met.
|
||||||
|
self.server_replied.set()
|
||||||
|
self._maybe_hook(forwarded)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
for sock in (client, backend):
|
||||||
|
try:
|
||||||
|
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack("ii", 1, 0))
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
try:
|
||||||
|
sock.close()
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
try:
|
||||||
|
self.listener.close()
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def _maybe_hook(self, forwarded):
|
||||||
|
"""Fire the one-shot hook once its barrier is satisfied: enough client
|
||||||
|
bytes have been forwarded and, when ``wait_for_reply`` is set, the
|
||||||
|
server has sent a reply proving it processed the preceding frames."""
|
||||||
|
if self.hook is None or self.hook_called.is_set():
|
||||||
|
return
|
||||||
|
if forwarded < self.hook_after:
|
||||||
|
return
|
||||||
|
if self.wait_for_reply and not self.server_replied.is_set():
|
||||||
|
return
|
||||||
|
self.hook()
|
||||||
|
self.hook_called.set()
|
||||||
|
|
||||||
|
def finish(self):
|
||||||
|
self._thread.join(30)
|
||||||
|
try:
|
||||||
|
self.listener.close()
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class TestDeleteTimingFinalStateParity:
|
||||||
|
"""On a successful transfer the per-directory timings match rsync's result."""
|
||||||
|
|
||||||
|
def _run_fastsync(self, tag, timing):
|
||||||
|
source, dest, received = _seed_pair(tag)
|
||||||
|
with ServerManager() as server:
|
||||||
|
server.start(extra_args=["--allow-delete"])
|
||||||
|
result, _ = run_client(source, dest, flags=[timing], port=server.port)
|
||||||
|
return result, received
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("timing", ["--delete-during", "--delete-delay"])
|
||||||
|
@requires_rsync
|
||||||
|
def test_success_final_state_matches_rsync(self, timing):
|
||||||
|
# Build the rsync fixture from the same seed so both sides start equal.
|
||||||
|
source, dest, received = _seed_pair("parity_rsync")
|
||||||
|
source2 = source
|
||||||
|
rsync_dst = os.path.join(TEST_DATA_DIR, "dtp_parity_rsync_dst")
|
||||||
|
clean_dir(rsync_dst)
|
||||||
|
# rsync mirrors src/ into dst/; seed the same extra.
|
||||||
|
_write(os.path.join(rsync_dst, "d", "old_extra"), b"stale extra\n")
|
||||||
|
|
||||||
|
rsync_result = _rsync(["-a", timing, source2 + "/", rsync_dst + "/"])
|
||||||
|
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||||
|
rsync_tree = _tree(rsync_dst)
|
||||||
|
|
||||||
|
with ServerManager() as server:
|
||||||
|
server.start(extra_args=["--allow-delete"])
|
||||||
|
result, _ = run_client(source, dest, flags=[timing], port=server.port)
|
||||||
|
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||||
|
fastsync_tree = _tree(received)
|
||||||
|
assert fastsync_tree == rsync_tree, (
|
||||||
|
f"{timing}: fastsync tree {fastsync_tree} != rsync tree {rsync_tree}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestDeleteTimingTypeConflictParity:
|
||||||
|
"""A destination entry whose type differs from the source is replaced, in
|
||||||
|
both per-directory timings and in both directions, exactly like rsync."""
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("timing", ["--delete-during", "--delete-delay"])
|
||||||
|
@requires_rsync
|
||||||
|
def test_type_conflicts_match_rsync(self, timing):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "dtc_src")
|
||||||
|
clean_dir(source)
|
||||||
|
_write(os.path.join(source, "foo"), b"now a file\n")
|
||||||
|
_write(os.path.join(source, "bar", "inner.txt"), b"now a dir\n")
|
||||||
|
|
||||||
|
def seed_dest(root):
|
||||||
|
clean_dir(root)
|
||||||
|
_write(os.path.join(root, "foo", "inner.txt"), b"was a dir\n")
|
||||||
|
_write(os.path.join(root, "bar"), b"was a file\n")
|
||||||
|
|
||||||
|
rsync_dst = os.path.join(TEST_DATA_DIR, "dtc_rsync_dst")
|
||||||
|
seed_dest(rsync_dst)
|
||||||
|
rsync_result = _rsync(["-a", timing, source + "/", rsync_dst + "/"])
|
||||||
|
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||||
|
rsync_tree = _tree(rsync_dst)
|
||||||
|
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "dtc_dst")
|
||||||
|
clean_dir(dest)
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
seed_dest(received)
|
||||||
|
with ServerManager() as server:
|
||||||
|
server.start(extra_args=["--allow-delete"])
|
||||||
|
result, _ = run_client(source, dest, flags=[timing], port=server.port)
|
||||||
|
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||||
|
assert _tree(received) == rsync_tree, (
|
||||||
|
f"{timing}: fastsync tree {_tree(received)} != rsync tree {rsync_tree}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestDeleteTimingFailure:
|
||||||
|
"""A mid-transfer failure distinguishes during from delay."""
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("mt", [False, True])
|
||||||
|
def test_during_removes_delay_preserves_on_failure(self, mt):
|
||||||
|
source, dest, received = _seed_pair("failure", big=True)
|
||||||
|
extra = os.path.join(received, "d", "old_extra")
|
||||||
|
assert os.path.exists(extra)
|
||||||
|
with ServerManager() as server:
|
||||||
|
server.start(extra_args=["--allow-delete"])
|
||||||
|
for timing, expect_removed in (("--delete-during", True),
|
||||||
|
("--delete-delay", False)):
|
||||||
|
# Re-seed the extra before each run.
|
||||||
|
_write(extra, b"stale extra\n")
|
||||||
|
proxy = _SlicingProxy(server.port, forward_limit=MID_TRANSFER_BYTES, throttle=PROXY_THROTTLE)
|
||||||
|
flags = [timing] + (["--threads"] if mt else [])
|
||||||
|
result, _ = run_client(source, dest, flags=flags, port=proxy.port)
|
||||||
|
proxy.finish()
|
||||||
|
assert result.returncode != 0, f"{timing}: truncated transfer succeeded"
|
||||||
|
present = os.path.exists(extra)
|
||||||
|
assert present != expect_removed, (
|
||||||
|
f"{timing} (mt={mt}): extra present={present}, expected "
|
||||||
|
f"removed={expect_removed}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestDeleteDelayVsAfterSnapshot:
|
||||||
|
"""A destination entry created after its directory's scan survives under
|
||||||
|
--delete-delay but is removed by --delete-after's fresh end scan."""
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("mt", [False, True])
|
||||||
|
def test_late_created_extra_survives_delay_not_after(self, mt):
|
||||||
|
source, dest, received = _seed_pair("latecreate", big=True)
|
||||||
|
old_extra = os.path.join(received, "d", "old_extra")
|
||||||
|
new_extra = os.path.join(received, "d", "new_extra")
|
||||||
|
with ServerManager() as server:
|
||||||
|
server.start(extra_args=["--allow-delete"])
|
||||||
|
for timing, new_survives in (("--delete-delay", True),
|
||||||
|
("--delete-after", False)):
|
||||||
|
_write(old_extra, b"stale extra\n")
|
||||||
|
if os.path.exists(new_extra):
|
||||||
|
os.unlink(new_extra)
|
||||||
|
|
||||||
|
def hook():
|
||||||
|
# Runs on the proxy thread while the big file is in flight,
|
||||||
|
# after the directory's plan (delay) has been processed.
|
||||||
|
_write(new_extra, b"created mid-transfer\n")
|
||||||
|
|
||||||
|
# --incremental gives the receiver a mid-transfer handshake
|
||||||
|
# reply; the proxy waits for it (wait_for_reply) so the hook is
|
||||||
|
# causally after the plan frame, never a timing guess.
|
||||||
|
# --ignore-times forces the big file to transfer on the second
|
||||||
|
# timing too (the first run already installed it), keeping the
|
||||||
|
# mid-transfer reply present in both iterations.
|
||||||
|
proxy = _SlicingProxy(server.port, hook=hook,
|
||||||
|
hook_after=MID_TRANSFER_BYTES,
|
||||||
|
throttle=PROXY_THROTTLE, wait_for_reply=True)
|
||||||
|
flags = [timing, "--incremental", "--ignore-times"] + (["--threads"] if mt else [])
|
||||||
|
result, _ = run_client(source, dest, flags=flags, port=proxy.port)
|
||||||
|
proxy.finish()
|
||||||
|
assert result.returncode == 0, (
|
||||||
|
f"{timing}: {(result.stderr or result.stdout)[:300]}"
|
||||||
|
)
|
||||||
|
assert proxy.hook_called.is_set(), f"{timing}: hook never fired"
|
||||||
|
assert not os.path.exists(old_extra), f"{timing}: old extra survived"
|
||||||
|
assert os.path.exists(new_extra) == new_survives, (
|
||||||
|
f"{timing} (mt={mt}): new_extra present="
|
||||||
|
f"{os.path.exists(new_extra)}, expected survives={new_survives}"
|
||||||
|
)
|
||||||
@@ -36,7 +36,7 @@ from common import ( # noqa: E402
|
|||||||
verify_transfer,
|
verify_transfer,
|
||||||
)
|
)
|
||||||
|
|
||||||
PROTOCOL_VERSION = b"2.21.0"
|
PROTOCOL_VERSION = b"2.26.0"
|
||||||
STATUS_MANIFEST = 5
|
STATUS_MANIFEST = 5
|
||||||
STATUS_OK = 0
|
STATUS_OK = 0
|
||||||
|
|
||||||
@@ -59,6 +59,10 @@ def _seed_source():
|
|||||||
if os.path.exists(SOURCE_DIR):
|
if os.path.exists(SOURCE_DIR):
|
||||||
shutil.rmtree(SOURCE_DIR)
|
shutil.rmtree(SOURCE_DIR)
|
||||||
os.makedirs(os.path.join(SOURCE_DIR, "nested"))
|
os.makedirs(os.path.join(SOURCE_DIR, "nested"))
|
||||||
|
# The receiver rejects a destination root that does not exist, and the
|
||||||
|
# capture fixture can run before any test that creates it, so create it
|
||||||
|
# here (test order/distribution must not matter).
|
||||||
|
os.makedirs(DEST_DIR, exist_ok=True)
|
||||||
with open(os.path.join(SOURCE_DIR, "hello.txt"), "wb") as fh:
|
with open(os.path.join(SOURCE_DIR, "hello.txt"), "wb") as fh:
|
||||||
fh.write(b"fault injection payload\n" * 64)
|
fh.write(b"fault injection payload\n" * 64)
|
||||||
with open(os.path.join(SOURCE_DIR, "nested", "deep.bin"), "wb") as fh:
|
with open(os.path.join(SOURCE_DIR, "nested", "deep.bin"), "wb") as fh:
|
||||||
|
|||||||
+1255
-180
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,584 @@
|
|||||||
|
"""Output-parity tests (#291 selection/output, #292 output formatting).
|
||||||
|
|
||||||
|
These tests exercise rsync-style selection ordering and output formatting. The
|
||||||
|
differential tests run the SAME transfer with real ``rsync 3.4.1`` and with
|
||||||
|
fastsync and compare stdout, so they are skipped when rsync is unavailable.
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
sys.path.insert(0, os.path.dirname(__file__))
|
||||||
|
from common import TEST_DATA_DIR, run_client, clean_dir, get_dest_received_dir, ServerManager
|
||||||
|
|
||||||
|
RSYNC = shutil.which("rsync")
|
||||||
|
requires_rsync = pytest.mark.skipif(RSYNC is None, reason="rsync 3.4.1 not installed")
|
||||||
|
|
||||||
|
|
||||||
|
def _rsync(args):
|
||||||
|
env = dict(os.environ, LC_ALL="C")
|
||||||
|
return subprocess.run(
|
||||||
|
[RSYNC] + args, capture_output=True, text=True, env=env, timeout=120
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _make_selection_tree(root):
|
||||||
|
clean_dir(root)
|
||||||
|
os.makedirs(os.path.join(root, "sub"))
|
||||||
|
with open(os.path.join(root, "a.txt"), "wb") as fh:
|
||||||
|
fh.write(b"top text\n")
|
||||||
|
with open(os.path.join(root, "b.log"), "wb") as fh:
|
||||||
|
fh.write(b"log data\n")
|
||||||
|
with open(os.path.join(root, "sub", "c.txt"), "wb") as fh:
|
||||||
|
fh.write(b"nested text\n")
|
||||||
|
with open(os.path.join(root, "sub", "d.log"), "wb") as fh:
|
||||||
|
fh.write(b"nested log\n")
|
||||||
|
|
||||||
|
|
||||||
|
class TestSelectionOrdering:
|
||||||
|
"""#291: --include/--exclude compile into one ordered rule list."""
|
||||||
|
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_include_then_exclude_keeps_only_matching(self, shared_server):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "out_inc_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "out_inc_dst")
|
||||||
|
_make_selection_tree(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
result, _ = run_client(
|
||||||
|
source, dest,
|
||||||
|
flags=["--preserve", "--include=*.txt", "--exclude=*"],
|
||||||
|
port=shared_server.port,
|
||||||
|
)
|
||||||
|
assert result.returncode == 0, f"include/exclude failed: {result.stderr[:300]}"
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
assert os.path.exists(os.path.join(received, "a.txt"))
|
||||||
|
# `*` also excludes the directory, so nothing below sub/ is sent.
|
||||||
|
assert not os.path.exists(os.path.join(received, "b.log"))
|
||||||
|
assert not os.path.exists(os.path.join(received, "sub", "c.txt"))
|
||||||
|
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_include_dirs_then_files_idiom(self, shared_server):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "out_inc2_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "out_inc2_dst")
|
||||||
|
_make_selection_tree(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
result, _ = run_client(
|
||||||
|
source, dest,
|
||||||
|
flags=["--preserve", "--include=*/", "--include=*.txt", "--exclude=*"],
|
||||||
|
port=shared_server.port,
|
||||||
|
)
|
||||||
|
assert result.returncode == 0, f"include/exclude failed: {result.stderr[:300]}"
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
assert os.path.exists(os.path.join(received, "a.txt"))
|
||||||
|
assert os.path.exists(os.path.join(received, "sub", "c.txt"))
|
||||||
|
assert not os.path.exists(os.path.join(received, "b.log"))
|
||||||
|
assert not os.path.exists(os.path.join(received, "sub", "d.log"))
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
def test_include_idiom_matches_rsync_selection(self, shared_server):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "out_inc3_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "out_inc3_dst")
|
||||||
|
rdst = os.path.join(TEST_DATA_DIR, "out_inc3_rdst")
|
||||||
|
_make_selection_tree(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
clean_dir(rdst)
|
||||||
|
flags = ["--include=*/", "--include=*.txt", "--exclude=*"]
|
||||||
|
rsync_result = _rsync(["-a"] + flags + [source + "/", rdst + "/"])
|
||||||
|
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||||
|
result, _ = run_client(source, dest, flags=["--preserve"] + flags,
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
assert os.path.exists(os.path.join(received, "a.txt"))
|
||||||
|
assert os.path.exists(os.path.join(received, "sub", "c.txt"))
|
||||||
|
assert not os.path.exists(os.path.join(received, "b.log"))
|
||||||
|
# rsync -a src/ dst/ writes directly into dst/
|
||||||
|
assert os.path.exists(os.path.join(rdst, "a.txt"))
|
||||||
|
assert os.path.exists(os.path.join(rdst, "sub", "c.txt"))
|
||||||
|
assert not os.path.exists(os.path.join(rdst, "b.log"))
|
||||||
|
|
||||||
|
|
||||||
|
class TestOneFileSystem:
|
||||||
|
"""#291: -x emits the mount-point directory but not its contents."""
|
||||||
|
|
||||||
|
def test_one_file_system_emits_mount_point_dir(self, shared_server):
|
||||||
|
local = os.stat(".")
|
||||||
|
shm = "/dev/shm"
|
||||||
|
try:
|
||||||
|
shm_stat = os.stat(shm)
|
||||||
|
except OSError:
|
||||||
|
pytest.skip("/dev/shm not available")
|
||||||
|
if shm_stat.st_dev == local.st_dev:
|
||||||
|
pytest.skip("no cross-device filesystem available")
|
||||||
|
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "out_ofs_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "out_ofs_dst")
|
||||||
|
clean_dir(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
os.makedirs(os.path.join(source, "nested"))
|
||||||
|
os.makedirs(os.path.join(shm, "fastsync_ofs_probe"), exist_ok=True)
|
||||||
|
with open(os.path.join(source, "keep.txt"), "wb") as fh:
|
||||||
|
fh.write(b"keep\n")
|
||||||
|
with open(os.path.join(shm, "fastsync_ofs_probe", "inside.txt"), "wb") as fh:
|
||||||
|
fh.write(b"cross\n")
|
||||||
|
link = os.path.join(source, "nested", "link")
|
||||||
|
try:
|
||||||
|
os.symlink(os.path.join(shm, "fastsync_ofs_probe"), link)
|
||||||
|
except OSError:
|
||||||
|
pytest.skip("cannot create symlink")
|
||||||
|
|
||||||
|
try:
|
||||||
|
result, _ = run_client(
|
||||||
|
source, dest,
|
||||||
|
flags=["--preserve", "--copy-links", "-x"],
|
||||||
|
port=shared_server.port,
|
||||||
|
)
|
||||||
|
assert result.returncode == 0, f"-x failed: {result.stderr[:300]}"
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
assert os.path.exists(os.path.join(received, "keep.txt"))
|
||||||
|
# The mount-point directory entry is created but its contents are not.
|
||||||
|
assert os.path.isdir(os.path.join(received, "nested", "link"))
|
||||||
|
assert not os.path.exists(os.path.join(received, "nested", "link", "inside.txt"))
|
||||||
|
finally:
|
||||||
|
shutil.rmtree(os.path.join(shm, "fastsync_ofs_probe"), ignore_errors=True)
|
||||||
|
|
||||||
|
|
||||||
|
def _make_output_tree(root):
|
||||||
|
clean_dir(root)
|
||||||
|
os.makedirs(os.path.join(root, "sub"))
|
||||||
|
with open(os.path.join(root, "a.txt"), "wb") as fh:
|
||||||
|
fh.write(b"hello\n")
|
||||||
|
with open(os.path.join(root, "sub", "b.txt"), "wb") as fh:
|
||||||
|
fh.write("wörld\n".encode("utf-8"))
|
||||||
|
os.symlink("a.txt", os.path.join(root, "link"))
|
||||||
|
|
||||||
|
|
||||||
|
class TestItemizeParity:
|
||||||
|
"""#292: -i output matches rsync 3.4.1 for the cases fastsync can observe."""
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_itemize_first_transfer_matches_rsync(self, shared_server):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "out_item_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "out_item_dst")
|
||||||
|
rdst = os.path.join(TEST_DATA_DIR, "out_item_rdst")
|
||||||
|
_make_output_tree(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
clean_dir(rdst)
|
||||||
|
rsync_result = _rsync(["-a", "-i", source + "/", rdst + "/"])
|
||||||
|
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||||
|
rsync_lines = sorted(
|
||||||
|
line for line in rsync_result.stdout.splitlines()
|
||||||
|
if line.startswith(">f") or line.startswith("cL")
|
||||||
|
)
|
||||||
|
result, _ = run_client(source, dest, flags=["-a", "-i"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, result.stderr[:300]
|
||||||
|
fast_lines = sorted(
|
||||||
|
line for line in result.stdout.splitlines()
|
||||||
|
if line.startswith(">f") or line.startswith("cL")
|
||||||
|
)
|
||||||
|
assert fast_lines == rsync_lines, f"rsync={rsync_lines} fastsync={fast_lines}"
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_itemize_modified_file_matches_rsync(self, shared_server):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "out_item2_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "out_item2_dst")
|
||||||
|
rdst = os.path.join(TEST_DATA_DIR, "out_item2_rdst")
|
||||||
|
_make_output_tree(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
clean_dir(rdst)
|
||||||
|
seed = run_client(source, dest, flags=["-a"], port=shared_server.port)
|
||||||
|
assert seed[0].returncode == 0, seed[0].stderr[:300]
|
||||||
|
assert _rsync(["-a", source + "/", rdst + "/"]).returncode == 0
|
||||||
|
|
||||||
|
with open(os.path.join(source, "a.txt"), "wb") as fh:
|
||||||
|
fh.write(b"hello changed and longer\n")
|
||||||
|
# Pin the source mtime so rsync's `t` column is deterministic (a write
|
||||||
|
# that lands in the same whole second as the seed would not show `t`).
|
||||||
|
os.utime(os.path.join(source, "a.txt"), (1000000000, 1000000000))
|
||||||
|
|
||||||
|
rsync_result = _rsync(["-a", "-i", source + "/", rdst + "/"])
|
||||||
|
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||||
|
rsync_lines = sorted(
|
||||||
|
line for line in rsync_result.stdout.splitlines() if line.startswith(">f")
|
||||||
|
)
|
||||||
|
result, _ = run_client(source, dest,
|
||||||
|
flags=["-a", "-i", "--incremental"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, result.stderr[:300]
|
||||||
|
fast_lines = sorted(
|
||||||
|
line for line in result.stdout.splitlines() if line.startswith(">f")
|
||||||
|
)
|
||||||
|
assert fast_lines == rsync_lines, f"rsync={rsync_lines} fastsync={fast_lines}"
|
||||||
|
|
||||||
|
|
||||||
|
class TestOutFormatParity:
|
||||||
|
@requires_rsync
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_out_format_n_l_matches_rsync(self, shared_server):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "out_fmt_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "out_fmt_dst")
|
||||||
|
rdst = os.path.join(TEST_DATA_DIR, "out_fmt_rdst")
|
||||||
|
_make_output_tree(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
clean_dir(rdst)
|
||||||
|
fmt = "%n %l"
|
||||||
|
rsync_result = _rsync(["-a", "--out-format=" + fmt, source + "/", rdst + "/"])
|
||||||
|
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||||
|
rsync_lines = sorted(
|
||||||
|
line for line in rsync_result.stdout.splitlines()
|
||||||
|
if line and not line.split(" ", 1)[0].endswith("/")
|
||||||
|
)
|
||||||
|
result, _ = run_client(source, dest,
|
||||||
|
flags=["-a", "--out-format=" + fmt],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, result.stderr[:300]
|
||||||
|
fast_lines = sorted(
|
||||||
|
line for line in result.stdout.splitlines()
|
||||||
|
if line and not line.split(" ", 1)[0].endswith("/")
|
||||||
|
)
|
||||||
|
assert fast_lines == rsync_lines, f"rsync={rsync_lines} fastsync={fast_lines}"
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_out_format_M_datetime_shape(self, shared_server):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "out_M_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "out_M_dst")
|
||||||
|
_make_output_tree(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
result, _ = run_client(source, dest,
|
||||||
|
flags=["-a", "--out-format=%M %f"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, result.stderr[:300]
|
||||||
|
import re
|
||||||
|
pattern = re.compile(r"^\d{4}/\d{2}/\d{2}-\d{2}:\d{2}:\d{2} ")
|
||||||
|
for line in result.stdout.splitlines():
|
||||||
|
if line:
|
||||||
|
assert pattern.match(line), f"bad %M format: {line!r}"
|
||||||
|
|
||||||
|
|
||||||
|
class TestListOnlyParity:
|
||||||
|
@requires_rsync
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_list_only_matches_rsync(self, shared_server):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "out_list_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "out_list_dst")
|
||||||
|
_make_output_tree(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
rsync_result = _rsync(["-r", "--list-only", source + "/"])
|
||||||
|
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||||
|
rsync_lines = sorted(rsync_result.stdout.splitlines())
|
||||||
|
result, _ = run_client(source, dest, flags=["--list-only", "-l"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, result.stderr[:300]
|
||||||
|
fast_lines = sorted(result.stdout.splitlines())
|
||||||
|
assert fast_lines == rsync_lines, (
|
||||||
|
f"rsync={rsync_lines}\nfastsync={fast_lines}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _make_one_file(root, name="f.bin", size=100):
|
||||||
|
clean_dir(root)
|
||||||
|
with open(os.path.join(root, name), "wb") as fh:
|
||||||
|
fh.write(bytes((i * 7 + 3) & 0xFF for i in range(size)))
|
||||||
|
|
||||||
|
|
||||||
|
class TestWireStatsParity:
|
||||||
|
"""Wire-counter output parity: --out-format %b/%c/%C, --progress and
|
||||||
|
--stats versus real rsync 3.4.1."""
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_out_format_checksum_matches_rsync(self, shared_server):
|
||||||
|
"""%C (whole-file xxh128, seed 0) is protocol-independent, so the full
|
||||||
|
`%C %l %n` line must be byte-identical to rsync."""
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "wire_ck_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "wire_ck_dst")
|
||||||
|
rdst = os.path.join(TEST_DATA_DIR, "wire_ck_rdst")
|
||||||
|
_make_one_file(source, "f.bin", 200000)
|
||||||
|
clean_dir(dest)
|
||||||
|
clean_dir(rdst)
|
||||||
|
fmt = "%C %l %n"
|
||||||
|
rsync_result = _rsync(["-a", "--out-format=" + fmt, source + "/", rdst + "/"])
|
||||||
|
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||||
|
result, _ = run_client(source, dest, flags=["-a", "--out-format=" + fmt],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, result.stderr[:300]
|
||||||
|
|
||||||
|
def file_lines(text):
|
||||||
|
# Ignore the root directory entry: fastsync does not transfer the
|
||||||
|
# source-root dir itself (a separate pre-existing divergence).
|
||||||
|
return [
|
||||||
|
line for line in text.splitlines() if not line.rsplit(" ", 1)[-1].endswith("/")
|
||||||
|
]
|
||||||
|
|
||||||
|
assert file_lines(result.stdout) == file_lines(rsync_result.stdout), (
|
||||||
|
f"rsync={rsync_result.stdout!r} fastsync={result.stdout!r}"
|
||||||
|
)
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_out_format_b_is_wire_bytes(self, shared_server):
|
||||||
|
"""%b is the bytes actually transferred (wire), not the source length.
|
||||||
|
|
||||||
|
A differential run against rsync confirms both implementations report a
|
||||||
|
framed value greater than %l. The exact numbers are not compared: each
|
||||||
|
counts its own protocol framing and checksum trailer, so the two are
|
||||||
|
protocol-specific and cannot be numerically equal (documented
|
||||||
|
divergence)."""
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "wire_b_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "wire_b_dst")
|
||||||
|
rdst = os.path.join(TEST_DATA_DIR, "wire_b_rdst")
|
||||||
|
_make_one_file(source, "f.bin", 5000)
|
||||||
|
clean_dir(dest)
|
||||||
|
clean_dir(rdst)
|
||||||
|
fmt = "%b %l"
|
||||||
|
rsync_result = _rsync(["-a", "--out-format=" + fmt, source + "/", rdst + "/"])
|
||||||
|
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||||
|
result, _ = run_client(source, dest, flags=["-a", "--out-format=" + fmt],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, result.stderr[:300]
|
||||||
|
rb, rl = (int(x) for x in rsync_result.stdout.split()[:2])
|
||||||
|
fb, fl = (int(x) for x in result.stdout.split()[:2])
|
||||||
|
assert rl == fl == 5000, (rsync_result.stdout, result.stdout)
|
||||||
|
assert rb > rl, f"rsync %b must include framing: {rsync_result.stdout!r}"
|
||||||
|
assert fb > fl, f"fastsync %b must include framing: {result.stdout!r}"
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_out_format_c_whole_file_matches_rsync(self, shared_server):
|
||||||
|
"""%c is the block-checksum bytes received. rsync reports its 16-byte
|
||||||
|
sum header even for a whole-file transfer (no basis), so `%c` must match
|
||||||
|
rsync exactly for the whole-file case."""
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "wire_c_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "wire_c_dst")
|
||||||
|
rdst = os.path.join(TEST_DATA_DIR, "wire_c_rdst")
|
||||||
|
_make_one_file(source, "f.bin", 5000)
|
||||||
|
clean_dir(dest)
|
||||||
|
clean_dir(rdst)
|
||||||
|
fmt = "%c %l %n"
|
||||||
|
rsync_result = _rsync(["-a", "--out-format=" + fmt, source + "/", rdst + "/"])
|
||||||
|
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||||
|
result, _ = run_client(source, dest, flags=["-a", "--out-format=" + fmt],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, result.stderr[:300]
|
||||||
|
|
||||||
|
def file_lines(text):
|
||||||
|
return [
|
||||||
|
line for line in text.splitlines()
|
||||||
|
if line and not line.rsplit(" ", 1)[-1].endswith("/")
|
||||||
|
]
|
||||||
|
|
||||||
|
assert file_lines(result.stdout) == file_lines(rsync_result.stdout), (
|
||||||
|
f"rsync={rsync_result.stdout!r} fastsync={result.stdout!r}"
|
||||||
|
)
|
||||||
|
assert result.stdout.split()[0] == rsync_result.stdout.split()[0] == "16", (
|
||||||
|
f"%c must be rsync's 16-byte sum header: {result.stdout!r}"
|
||||||
|
)
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_out_format_c_delta_mode_divergence(self, shared_server):
|
||||||
|
"""Documented residual: with delta enabled, rsync's %c is its 16-byte sum
|
||||||
|
header plus one checksum entry per block (protocol-specific, so it grows
|
||||||
|
with the basis size), while FastSync's %c is the bytes of its own delta
|
||||||
|
handshake. FastSync's delta %c therefore cannot match rsync numerically;
|
||||||
|
only the whole-file case is aligned. Pinned here so a future change is
|
||||||
|
noticed."""
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "wire_cd_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "wire_cd_dst")
|
||||||
|
rdst = os.path.join(TEST_DATA_DIR, "wire_cd_rdst")
|
||||||
|
_make_one_file(source, "f.bin", 5000)
|
||||||
|
clean_dir(dest)
|
||||||
|
clean_dir(rdst)
|
||||||
|
fmt = "%c %l"
|
||||||
|
# rsync local default is whole-file; force the block-delta path.
|
||||||
|
rsync_result = _rsync(["-a", "--no-whole-file", "--out-format=" + fmt,
|
||||||
|
source + "/", rdst + "/"])
|
||||||
|
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||||
|
result, _ = run_client(source, dest,
|
||||||
|
flags=["-a", "--incremental", "--delta",
|
||||||
|
"--out-format=" + fmt],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, result.stderr[:300]
|
||||||
|
rs_c = int(rsync_result.stdout.split()[0])
|
||||||
|
fs_c = int(result.stdout.split()[0])
|
||||||
|
# No basis exists, so rsync still reports only its sum header.
|
||||||
|
assert rs_c == 16, rsync_result.stdout
|
||||||
|
# FastSync reports its own handshake bytes and is not aligned.
|
||||||
|
assert fs_c > 16, (
|
||||||
|
f"FastSync delta %c changed to {fs_c}; the documented divergence "
|
||||||
|
"may be closable now"
|
||||||
|
)
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
@pytest.mark.ci
|
||||||
|
@pytest.mark.parametrize("mt", [False, True])
|
||||||
|
@pytest.mark.parametrize("progress_flag", ["--progress", "-P"])
|
||||||
|
def test_progress_first_frame_matches_rsync(self, shared_server, progress_flag, mt):
|
||||||
|
"""For a sub-32 KiB file the first --progress/-P frame is deterministic
|
||||||
|
(0.00 kB/s, 0:00:00) and must be byte-identical to rsync's, in both the
|
||||||
|
single-threaded and --threads send paths."""
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "wire_pg_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "wire_pg_dst")
|
||||||
|
rdst = os.path.join(TEST_DATA_DIR, "wire_pg_rdst")
|
||||||
|
_make_one_file(source, "f.bin", 100)
|
||||||
|
clean_dir(dest)
|
||||||
|
clean_dir(rdst)
|
||||||
|
rsync_result = _rsync(["-a", progress_flag, source + "/", rdst + "/"])
|
||||||
|
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||||
|
flags = ["-a", progress_flag] + (["--threads"] if mt else [])
|
||||||
|
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
|
||||||
|
assert result.returncode == 0, result.stderr[:300]
|
||||||
|
|
||||||
|
def frames(text):
|
||||||
|
# subprocess text mode normalizes \r to \n (universal newlines).
|
||||||
|
return [p for p in text.split("\n") if "%" in p]
|
||||||
|
|
||||||
|
rsync_frames = frames(rsync_result.stdout)
|
||||||
|
fast_frames = frames(result.stdout)
|
||||||
|
assert rsync_frames and fast_frames, (rsync_result.stdout, result.stdout)
|
||||||
|
assert fast_frames[0] == rsync_frames[0], (rsync_frames[0], fast_frames[0])
|
||||||
|
assert "(xfr#1," in fast_frames[-1], fast_frames[-1]
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
@pytest.mark.ci
|
||||||
|
@pytest.mark.parametrize("mt", [False, True])
|
||||||
|
def test_stats_selected_lines_match_rsync(self, shared_server, mt):
|
||||||
|
"""The protocol-independent --stats lines must match rsync exactly, in
|
||||||
|
both the single-threaded and --threads (multithreaded) send paths."""
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "wire_st_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "wire_st_dst")
|
||||||
|
rdst = os.path.join(TEST_DATA_DIR, "wire_st_rdst")
|
||||||
|
_make_one_file(source, "f.bin", 6000)
|
||||||
|
clean_dir(dest)
|
||||||
|
clean_dir(rdst)
|
||||||
|
rsync_result = _rsync(["-a", "--stats", source + "/", rdst + "/"])
|
||||||
|
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||||
|
flags = ["-a", "--stats"] + (["--threads"] if mt else [])
|
||||||
|
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
|
||||||
|
assert result.returncode == 0, result.stderr[:300]
|
||||||
|
keys = (
|
||||||
|
"Number of regular files transferred",
|
||||||
|
"Total file size",
|
||||||
|
"Total transferred file size",
|
||||||
|
"Literal data",
|
||||||
|
"Matched data",
|
||||||
|
"Number of deleted files",
|
||||||
|
"File list size",
|
||||||
|
)
|
||||||
|
|
||||||
|
def pick(text):
|
||||||
|
out = {}
|
||||||
|
for line in text.splitlines():
|
||||||
|
for key in keys:
|
||||||
|
if line.startswith(key + ":"):
|
||||||
|
out[key] = line
|
||||||
|
return out
|
||||||
|
|
||||||
|
assert pick(result.stdout) == pick(rsync_result.stdout), (
|
||||||
|
f"rsync={pick(rsync_result.stdout)} fastsync={pick(result.stdout)}"
|
||||||
|
)
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_stats_file_count_breakdown_residual(self, shared_server):
|
||||||
|
"""Residual (row #3): rsync prints the `Number of files` and
|
||||||
|
`Number of created files` lines with a per-type breakdown
|
||||||
|
(`(reg: X, dir: Y, link: Z)`).
|
||||||
|
|
||||||
|
FastSync cannot reproduce it from what the sender currently knows: the
|
||||||
|
scanner does not put directory entries in the transfer list (directories
|
||||||
|
are created implicitly), and without a per-entry destination-probe the
|
||||||
|
sender cannot tell which entries the receiver newly created. So FastSync
|
||||||
|
prints the bare transferred-entry count. This test pins the divergence
|
||||||
|
explicitly -- the row must not be marked ✅.
|
||||||
|
"""
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "wire_stc_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "wire_stc_dst")
|
||||||
|
rdst = os.path.join(TEST_DATA_DIR, "wire_stc_rdst")
|
||||||
|
_make_one_file(source, "f.bin", 6000)
|
||||||
|
clean_dir(dest)
|
||||||
|
clean_dir(rdst)
|
||||||
|
rsync_result = _rsync(["-a", "--stats", source + "/", rdst + "/"])
|
||||||
|
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||||
|
result, _ = run_client(source, dest, flags=["-a", "--stats"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, result.stderr[:300]
|
||||||
|
|
||||||
|
def stats_line(text, key):
|
||||||
|
for line in text.splitlines():
|
||||||
|
if line.startswith(key + ":"):
|
||||||
|
return line
|
||||||
|
return None
|
||||||
|
|
||||||
|
r_files = stats_line(rsync_result.stdout, "Number of files")
|
||||||
|
r_created = stats_line(rsync_result.stdout, "Number of created files")
|
||||||
|
f_files = stats_line(result.stdout, "Number of files")
|
||||||
|
f_created = stats_line(result.stdout, "Number of created files")
|
||||||
|
|
||||||
|
# rsync always carries the type breakdown (the source root counts as a
|
||||||
|
# directory; the single regular file as reg).
|
||||||
|
assert re.match(r"Number of files: 2 \(reg: 1, dir: 1\)$", r_files), r_files
|
||||||
|
assert re.match(r"Number of created files: 1 \(reg: 1\)$", r_created), r_created
|
||||||
|
# FastSync prints only the bare count: no directory accounting and no
|
||||||
|
# per-entry "created" knowledge.
|
||||||
|
assert re.fullmatch(r"Number of files: 1", f_files), f_files
|
||||||
|
assert re.fullmatch(r"Number of created files: 1", f_created), f_created
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
@pytest.mark.ci
|
||||||
|
@pytest.mark.parametrize("mt", [False, True])
|
||||||
|
def test_dry_run_delete_lines_match_rsync(self, mt):
|
||||||
|
"""-n --delete emits transfer-relative `*deleting` lines like rsync
|
||||||
|
(single-threaded and --threads)."""
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "wire_del_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "wire_del_dst")
|
||||||
|
rdst = os.path.join(TEST_DATA_DIR, "wire_del_rdst")
|
||||||
|
clean_dir(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
clean_dir(rdst)
|
||||||
|
with open(os.path.join(source, "a.txt"), "wb") as fh:
|
||||||
|
fh.write(b"a\n")
|
||||||
|
for root, entries in (
|
||||||
|
(rdst, {"extra.txt": b"x\n"}),
|
||||||
|
(rdst, {"sub/y.txt": b"y\n", "extradir/z.txt": b"z\n"}),
|
||||||
|
):
|
||||||
|
for rel, data in entries.items():
|
||||||
|
full = os.path.join(root, rel)
|
||||||
|
os.makedirs(os.path.dirname(full), exist_ok=True)
|
||||||
|
with open(full, "wb") as fh:
|
||||||
|
fh.write(data)
|
||||||
|
# FastSync mirrors the source's absolute path under dest.
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
for rel, data in (
|
||||||
|
("extra.txt", b"x\n"),
|
||||||
|
("sub/y.txt", b"y\n"),
|
||||||
|
("extradir/z.txt", b"z\n"),
|
||||||
|
):
|
||||||
|
full = os.path.join(received, rel)
|
||||||
|
os.makedirs(os.path.dirname(full), exist_ok=True)
|
||||||
|
with open(full, "wb") as fh:
|
||||||
|
fh.write(data)
|
||||||
|
|
||||||
|
rsync_result = _rsync(["-a", "-n", "--delete", "-i", source + "/", rdst + "/"])
|
||||||
|
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||||
|
rsync_del = sorted(
|
||||||
|
line for line in rsync_result.stdout.splitlines() if line.startswith("*deleting")
|
||||||
|
)
|
||||||
|
# The shared session server refuses deletion; start one that allows it.
|
||||||
|
flags = ["-a", "-n", "--delete", "-i"] + (["--threads"] if mt else [])
|
||||||
|
with ServerManager() as server:
|
||||||
|
server.start(extra_args=["--allow-delete"])
|
||||||
|
result, _ = run_client(source, dest, flags=flags, port=server.port)
|
||||||
|
assert result.returncode == 0, result.stderr[:300]
|
||||||
|
fast_del = sorted(
|
||||||
|
line for line in result.stdout.splitlines() if line.startswith("*deleting")
|
||||||
|
)
|
||||||
|
assert fast_del == rsync_del, f"rsync={rsync_del}\nfastsync={fast_del}"
|
||||||
@@ -0,0 +1,299 @@
|
|||||||
|
"""Differential/regression coverage for the parity-completion review blockers.
|
||||||
|
|
||||||
|
Each test pins a fix against real ``rsync 3.4.1`` where a deterministic
|
||||||
|
comparison exists; the differential tests skip cleanly when rsync is absent.
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
sys.path.insert(0, os.path.dirname(__file__))
|
||||||
|
from common import ( # noqa: E402
|
||||||
|
TEST_DATA_DIR,
|
||||||
|
ServerManager,
|
||||||
|
clean_dir,
|
||||||
|
get_dest_received_dir,
|
||||||
|
run_client,
|
||||||
|
)
|
||||||
|
|
||||||
|
RSYNC = shutil.which("rsync")
|
||||||
|
requires_rsync = pytest.mark.skipif(RSYNC is None, reason="rsync 3.4.1 not installed")
|
||||||
|
|
||||||
|
|
||||||
|
def _write(path, content):
|
||||||
|
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||||
|
with open(path, "wb") as fh:
|
||||||
|
fh.write(content)
|
||||||
|
|
||||||
|
|
||||||
|
def _tree(root):
|
||||||
|
"""Sorted relative paths of every entry below root (files and dirs)."""
|
||||||
|
out = []
|
||||||
|
for dirpath, dirs, files in os.walk(root):
|
||||||
|
for name in dirs:
|
||||||
|
out.append(os.path.relpath(os.path.join(dirpath, name), root))
|
||||||
|
for name in files:
|
||||||
|
out.append(os.path.relpath(os.path.join(dirpath, name), root))
|
||||||
|
return sorted(out)
|
||||||
|
|
||||||
|
|
||||||
|
def _rsync(args):
|
||||||
|
env = dict(os.environ, LC_ALL="C")
|
||||||
|
return subprocess.run([RSYNC] + args, capture_output=True, text=True, env=env, timeout=120)
|
||||||
|
|
||||||
|
|
||||||
|
class TestRelativePerDirDeleteScope:
|
||||||
|
"""Blocker #1: -R --delete-during/--delete-delay must not delete destination
|
||||||
|
content outside the transferred prefix (rsync keeps sibling directories)."""
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
@pytest.mark.ci
|
||||||
|
@pytest.mark.parametrize("mt", [False, True])
|
||||||
|
@pytest.mark.parametrize("timing", ["--delete-during", "--delete-delay"])
|
||||||
|
def test_prefix_scoped_delete_matches_rsync(self, timing, mt):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, f"delblk_src{int(mt)}")
|
||||||
|
clean_dir(source)
|
||||||
|
_write(os.path.join(source, "foo", "a.txt"), b"payload\n")
|
||||||
|
spec = source + "/./foo"
|
||||||
|
|
||||||
|
def seed(root):
|
||||||
|
clean_dir(root)
|
||||||
|
_write(os.path.join(root, "foo", "extra.txt"), b"stale\n")
|
||||||
|
_write(os.path.join(root, "unrelated", "keep.txt"), b"keep\n")
|
||||||
|
|
||||||
|
rdst = os.path.join(TEST_DATA_DIR, f"delblk_rdst{int(mt)}")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, f"delblk_dst{int(mt)}")
|
||||||
|
seed(rdst)
|
||||||
|
seed(dest)
|
||||||
|
r = _rsync(["-aR", timing, spec, rdst + "/"])
|
||||||
|
assert r.returncode == 0, r.stderr
|
||||||
|
with ServerManager() as server:
|
||||||
|
server.start(extra_args=["--allow-delete"])
|
||||||
|
flags = ["-a", "-R", timing] + (["--threads"] if mt else [])
|
||||||
|
result, _ = run_client(spec, dest, flags=flags, port=server.port)
|
||||||
|
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||||
|
#The prefix's parent-directory sibling survives on both sides.
|
||||||
|
assert os.path.isfile(os.path.join(dest, "unrelated", "keep.txt"))
|
||||||
|
assert os.path.isfile(os.path.join(rdst, "unrelated", "keep.txt"))
|
||||||
|
#The in - scope extra is removed on both sides.
|
||||||
|
assert not os.path.exists(os.path.join(dest, "foo", "extra.txt"))
|
||||||
|
assert not os.path.exists(os.path.join(rdst, "foo", "extra.txt"))
|
||||||
|
assert _tree(dest) == _tree(rdst)
|
||||||
|
|
||||||
|
|
||||||
|
def _stats_value(text, label):
|
||||||
|
for line in text.splitlines():
|
||||||
|
if line.startswith(label + ":"):
|
||||||
|
return int(line.split(":", 1)[1].strip().split()[0].replace(",", ""))
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _seed_delta_pair(tag):
|
||||||
|
"""Source file plus a same-size/basis destination file whose mtime differs,
|
||||||
|
and an extra destination file to be deleted."""
|
||||||
|
source = os.path.join(TEST_DATA_DIR, f"stats_{tag}_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, f"stats_{tag}_dst")
|
||||||
|
rdst = os.path.join(TEST_DATA_DIR, f"stats_{tag}_rdst")
|
||||||
|
clean_dir(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
clean_dir(rdst)
|
||||||
|
payload = (b"0123456789abcdef" * 16384)[:200000]
|
||||||
|
_write(os.path.join(source, "f.bin"), payload)
|
||||||
|
#Destination basis : same length, one byte changed, deliberately older.
|
||||||
|
basis = bytearray(payload)
|
||||||
|
basis[100000] ^= 0xFF
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
for root in (rdst, received):
|
||||||
|
_write(os.path.join(root, "f.bin"), bytes(basis))
|
||||||
|
_write(os.path.join(root, "extra.txt"), b"delete me\n")
|
||||||
|
old = 1000000
|
||||||
|
os.utime(os.path.join(root, "f.bin"), (old, old))
|
||||||
|
return source, dest, rdst
|
||||||
|
|
||||||
|
|
||||||
|
class TestReceiverWireStats:
|
||||||
|
"""Blocker #3/#4: the receiver must populate the STATUS_STATS counters
|
||||||
|
(matched data, deleted files) on both the single-threaded and -m paths."""
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
@pytest.mark.ci
|
||||||
|
@pytest.mark.parametrize("threads", [False, True])
|
||||||
|
def test_stats_reports_matched_and_deleted(self, threads):
|
||||||
|
source, dest, rdst = _seed_delta_pair(f"mt{int(threads)}")
|
||||||
|
rsync_result = _rsync(["-a", "--stats", "--delete", "--no-whole-file", source + "/",
|
||||||
|
rdst + "/"])
|
||||||
|
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||||
|
assert _stats_value(rsync_result.stdout, "Matched data") > 0
|
||||||
|
assert _stats_value(rsync_result.stdout, "Number of deleted files") == 1
|
||||||
|
|
||||||
|
with ServerManager() as server:
|
||||||
|
server.start(extra_args=["--allow-delete"])
|
||||||
|
flags = ["-a", "--stats", "--delete", "--delta", "--incremental"]
|
||||||
|
if threads:
|
||||||
|
flags.append("--threads")
|
||||||
|
result, _ = run_client(source, dest, flags=flags, port=server.port)
|
||||||
|
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||||
|
assert _stats_value(result.stdout, "Matched data") > 0, result.stdout
|
||||||
|
assert _stats_value(result.stdout, "Number of deleted files") == 1, result.stdout
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_threads_dry_run_delete_lines_match_rsync(self):
|
||||||
|
"""-n --delete --threads must emit transfer-relative `*deleting` lines."""
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "stats_drydel_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "stats_drydel_dst")
|
||||||
|
rdst = os.path.join(TEST_DATA_DIR, "stats_drydel_rdst")
|
||||||
|
clean_dir(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
clean_dir(rdst)
|
||||||
|
_write(os.path.join(source, "a.txt"), b"a\n")
|
||||||
|
for root in (rdst, get_dest_received_dir(dest, source)):
|
||||||
|
_write(os.path.join(root, "extra.txt"), b"x\n")
|
||||||
|
_write(os.path.join(root, "sub", "y.txt"), b"y\n")
|
||||||
|
rsync_result = _rsync(["-a", "-n", "--delete", "-i", source + "/", rdst + "/"])
|
||||||
|
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||||
|
rsync_del = sorted(
|
||||||
|
line for line in rsync_result.stdout.splitlines() if line.startswith("*deleting")
|
||||||
|
)
|
||||||
|
with ServerManager() as server:
|
||||||
|
server.start(extra_args=["--allow-delete"])
|
||||||
|
result, _ = run_client(source, dest,
|
||||||
|
flags=["-a", "-n", "--delete", "-i", "--threads"],
|
||||||
|
port=server.port)
|
||||||
|
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||||
|
fast_del = sorted(
|
||||||
|
line for line in result.stdout.splitlines() if line.startswith("*deleting")
|
||||||
|
)
|
||||||
|
assert fast_del and fast_del == rsync_del, f"rsync={rsync_del}\nfastsync={fast_del}"
|
||||||
|
|
||||||
|
|
||||||
|
class TestRelativeFilesFromProtect:
|
||||||
|
"""Blocker #9: a -R + --files-from receiver-protect rule must record the bare
|
||||||
|
relative wire path so the protected destination mirror survives --delete."""
|
||||||
|
|
||||||
|
@pytest.mark.ci
|
||||||
|
@pytest.mark.parametrize("mt", [False, True])
|
||||||
|
def test_hidden_protected_mirror_survives_delete(self, mt):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "rfprot_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "rfprot_dst")
|
||||||
|
clean_dir(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
#Root - level entry exercises the parallel root scanner; the nested one
|
||||||
|
#exercises the sequential worker scanner.
|
||||||
|
_write(os.path.join(source, "root_secret.tmp"), b"root\n")
|
||||||
|
_write(os.path.join(source, "sub", "nested_secret.tmp"), b"nested\n")
|
||||||
|
_write(os.path.join(source, "sub", "keep.txt"), b"keep\n")
|
||||||
|
listfile = os.path.join(TEST_DATA_DIR, "rfprot.list")
|
||||||
|
with open(listfile, "w") as fh:
|
||||||
|
fh.write(".\n")
|
||||||
|
#H hides from the sender, P protects the receiver mirror from-- delete.
|
||||||
|
filters = ["--filter=H root_secret.tmp", "--filter=P root_secret.tmp",
|
||||||
|
"--filter=H sub/nested_secret.tmp", "--filter=P sub/nested_secret.tmp"]
|
||||||
|
with ServerManager() as server:
|
||||||
|
server.start(extra_args=["--allow-delete"])
|
||||||
|
seed = ["--files-from", listfile, "-R"] + (["--threads"] if mt else [])
|
||||||
|
result, _ = run_client(source, dest, flags=seed, port=server.port)
|
||||||
|
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||||
|
assert os.path.isfile(os.path.join(dest, "root_secret.tmp"))
|
||||||
|
assert os.path.isfile(os.path.join(dest, "sub", "nested_secret.tmp"))
|
||||||
|
_write(os.path.join(dest, "extra.txt"), b"extra\n")
|
||||||
|
_write(os.path.join(dest, "sub", "extra.txt"), b"extra\n")
|
||||||
|
flags = seed + ["--delete"] + filters
|
||||||
|
result, _ = run_client(source, dest, flags=flags, port=server.port)
|
||||||
|
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||||
|
assert os.path.isfile(os.path.join(dest, "root_secret.tmp")), \
|
||||||
|
"root-level protected mirror was deleted"
|
||||||
|
assert os.path.isfile(os.path.join(dest, "sub", "nested_secret.tmp")), \
|
||||||
|
"nested protected mirror was deleted"
|
||||||
|
assert not os.path.exists(os.path.join(dest, "extra.txt"))
|
||||||
|
assert not os.path.exists(os.path.join(dest, "sub", "extra.txt"))
|
||||||
|
|
||||||
|
|
||||||
|
class TestInvalidPerDirFilter:
|
||||||
|
"""Blocker #8: a per-directory filter file that fails to parse must fail the
|
||||||
|
scan even when an earlier merge file in the same directory existed."""
|
||||||
|
|
||||||
|
@pytest.mark.ci
|
||||||
|
@pytest.mark.parametrize("mt", [False, True])
|
||||||
|
def test_invalid_dir_filter_fails_scan(self, mt):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "badfilter_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "badfilter_dst")
|
||||||
|
clean_dir(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
#A valid.rsync - filter makes any_exists true for the directory; the
|
||||||
|
#invalid.rules must not then be silently ignored.
|
||||||
|
_write(os.path.join(source, ".rsync-filter"), b"- *.bak\n")
|
||||||
|
_write(os.path.join(source, ".rules"), b"protect\n")
|
||||||
|
_write(os.path.join(source, "a.txt"), b"a\n")
|
||||||
|
flags = ["-a", "-F", "--filter=: .rules"]
|
||||||
|
if mt:
|
||||||
|
flags.append("--threads")
|
||||||
|
with ServerManager() as server:
|
||||||
|
result, _ = run_client(source, dest, flags=flags, port=server.port)
|
||||||
|
assert result.returncode != 0, "invalid per-directory filter was silently ignored"
|
||||||
|
assert "invalid per-directory filter" in (result.stderr + result.stdout)
|
||||||
|
|
||||||
|
|
||||||
|
class TestWouldDeleteEscaping:
|
||||||
|
"""Blocker #5: -n --delete --out-format must escape control bytes in a
|
||||||
|
peer-supplied would-delete path so it cannot forge output lines."""
|
||||||
|
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_out_format_escapes_control_chars(self):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "esc_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "esc_dst")
|
||||||
|
clean_dir(source)
|
||||||
|
_write(os.path.join(source, "a.txt"), b"a\n")
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
clean_dir(received)
|
||||||
|
_write(os.path.join(received, "a.txt"), b"a\n")
|
||||||
|
#A newline in a destination filename must not split the printed line.
|
||||||
|
with open(os.path.join(received, "evil\nname.txt"), "wb") as fh:
|
||||||
|
fh.write(b"x\n")
|
||||||
|
with ServerManager() as server:
|
||||||
|
server.start(extra_args=["--allow-delete"])
|
||||||
|
result, _ = run_client(source, dest,
|
||||||
|
flags=["-a", "-n", "--delete", "--out-format=%n"],
|
||||||
|
port=server.port)
|
||||||
|
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||||
|
assert "\\#012" in result.stdout, result.stdout
|
||||||
|
assert "evil\nname.txt" not in result.stdout, result.stdout
|
||||||
|
|
||||||
|
|
||||||
|
class TestEmptySourceDirectoryDelete:
|
||||||
|
"""Blocker #10: an empty in-scope source directory must survive
|
||||||
|
--delete-during/--delete-delay (rsync keeps it) while its extras are still
|
||||||
|
removed."""
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
@pytest.mark.ci
|
||||||
|
@pytest.mark.parametrize("mt", [False, True])
|
||||||
|
@pytest.mark.parametrize("timing", ["--delete-during", "--delete-delay"])
|
||||||
|
def test_empty_source_dir_survives_matches_rsync(self, timing, mt):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, f"emptydir_src{int(mt)}")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, f"emptydir_dst{int(mt)}")
|
||||||
|
rdst = os.path.join(TEST_DATA_DIR, f"emptydir_rdst{int(mt)}")
|
||||||
|
clean_dir(source)
|
||||||
|
os.makedirs(os.path.join(source, "empty"))
|
||||||
|
_write(os.path.join(source, "keep.txt"), b"keep\n")
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
for root in (rdst, received):
|
||||||
|
clean_dir(root)
|
||||||
|
_write(os.path.join(root, "keep.txt"), b"keep\n")
|
||||||
|
_write(os.path.join(root, "empty", "extra.txt"), b"extra\n")
|
||||||
|
rsync_result = _rsync(["-a", timing, source + "/", rdst + "/"])
|
||||||
|
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||||
|
assert os.path.isdir(os.path.join(rdst, "empty"))
|
||||||
|
assert not os.path.exists(os.path.join(rdst, "empty", "extra.txt"))
|
||||||
|
with ServerManager() as server:
|
||||||
|
server.start(extra_args=["--allow-delete"])
|
||||||
|
flags = ["-a", timing] + (["--threads"] if mt else [])
|
||||||
|
result, _ = run_client(source, dest, flags=flags, port=server.port)
|
||||||
|
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||||
|
assert os.path.isdir(os.path.join(received, "empty")), \
|
||||||
|
"empty source directory was removed"
|
||||||
|
assert not os.path.exists(os.path.join(received, "empty", "extra.txt"))
|
||||||
|
assert _tree(received) == _tree(rdst)
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,287 @@
|
|||||||
|
"""rsync 3.4.1 parity for selection/path semantics and client option aliases.
|
||||||
|
|
||||||
|
Each test pins behaviour against real ``rsync 3.4.1``; the differential tests
|
||||||
|
skip cleanly when rsync is not installed.
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
sys.path.insert(0, os.path.dirname(__file__))
|
||||||
|
from common import (
|
||||||
|
TEST_DATA_DIR,
|
||||||
|
CLIENT_CMD,
|
||||||
|
ServerManager,
|
||||||
|
run_client,
|
||||||
|
clean_dir,
|
||||||
|
get_dest_received_dir,
|
||||||
|
)
|
||||||
|
|
||||||
|
RSYNC = shutil.which("rsync")
|
||||||
|
requires_rsync = pytest.mark.skipif(RSYNC is None, reason="rsync 3.4.1 not installed")
|
||||||
|
|
||||||
|
|
||||||
|
def _tree(root):
|
||||||
|
"""Sorted relative paths of directories (``D ``) and files (``F ``)."""
|
||||||
|
out = []
|
||||||
|
for dirpath, dirs, files in os.walk(root):
|
||||||
|
rel = os.path.relpath(dirpath, root)
|
||||||
|
for d in dirs:
|
||||||
|
out.append("D " + (d if rel == "." else os.path.join(rel, d)))
|
||||||
|
for f in files:
|
||||||
|
out.append("F " + (f if rel == "." else os.path.join(rel, f)))
|
||||||
|
return sorted(out)
|
||||||
|
|
||||||
|
|
||||||
|
def _rsync(args):
|
||||||
|
env = dict(os.environ, LC_ALL="C")
|
||||||
|
return subprocess.run([RSYNC] + args, capture_output=True, text=True, env=env, timeout=120)
|
||||||
|
|
||||||
|
|
||||||
|
def _make_tree(root):
|
||||||
|
clean_dir(root)
|
||||||
|
for rel, content in {
|
||||||
|
"top.txt": b"top\n",
|
||||||
|
"foo/bar/baz/f.txt": b"deep\n",
|
||||||
|
"sub/x.txt": b"x\n",
|
||||||
|
}.items():
|
||||||
|
full = os.path.join(root, rel)
|
||||||
|
os.makedirs(os.path.dirname(full), exist_ok=True)
|
||||||
|
with open(full, "wb") as fh:
|
||||||
|
fh.write(content)
|
||||||
|
return root
|
||||||
|
|
||||||
|
|
||||||
|
class TestRelativeGeneral:
|
||||||
|
"""#11: -R without --files-from uses rsync's '/./' cut and relative
|
||||||
|
reconstruction instead of always mirroring the full source path."""
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
@pytest.mark.ci
|
||||||
|
@pytest.mark.parametrize("suffix", ["", "/./foo", "/./foo/bar", "/./"])
|
||||||
|
def test_relative_cut_matches_rsync(self, shared_server, suffix):
|
||||||
|
source = _make_tree(os.path.join(TEST_DATA_DIR, "sel_rel_src"))
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "sel_rel_dst")
|
||||||
|
rdst = os.path.join(TEST_DATA_DIR, "sel_rel_rdst")
|
||||||
|
clean_dir(dest)
|
||||||
|
clean_dir(rdst)
|
||||||
|
spec = source + suffix
|
||||||
|
r = _rsync(["-aR", spec, rdst + "/"])
|
||||||
|
assert r.returncode == 0, r.stderr
|
||||||
|
result, _ = run_client(spec, dest, flags=["-R"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, result.stderr[:300]
|
||||||
|
assert _tree(rdst) == _tree(dest), f"layout mismatch for {spec!r}"
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_no_implied_dirs_matches_rsync(self, shared_server):
|
||||||
|
source = _make_tree(os.path.join(TEST_DATA_DIR, "sel_nid_src"))
|
||||||
|
a = os.path.join(source, "foo")
|
||||||
|
b = os.path.join(source, "foo", "bar")
|
||||||
|
os.chmod(a, 0o700)
|
||||||
|
os.chmod(b, 0o711)
|
||||||
|
os.utime(a, (978307200, 978307200))
|
||||||
|
os.utime(b, (978307200, 978307200))
|
||||||
|
spec = source + "/./foo/bar"
|
||||||
|
for extra in ([], ["--no-implied-dirs"]):
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "sel_nid_dst")
|
||||||
|
rdst = os.path.join(TEST_DATA_DIR, "sel_nid_rdst")
|
||||||
|
clean_dir(dest)
|
||||||
|
clean_dir(rdst)
|
||||||
|
r = _rsync(["-aR"] + extra + [spec, rdst + "/"])
|
||||||
|
assert r.returncode == 0, r.stderr
|
||||||
|
result, _ = run_client(spec, dest, flags=["-a", "-R"] + extra,
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, result.stderr[:300]
|
||||||
|
for rel in ("foo", "foo/bar"):
|
||||||
|
rs = os.stat(os.path.join(rdst, rel))
|
||||||
|
fs = os.stat(os.path.join(dest, rel))
|
||||||
|
assert (rs.st_mode & 0o7777) == (fs.st_mode & 0o7777), \
|
||||||
|
f"mode mismatch for {rel} with {extra}"
|
||||||
|
if extra == ["--no-implied-dirs"]:
|
||||||
|
# The implied parent directory is created at run time (no
|
||||||
|
# metadata applied), so rsync's and FastSync's separate runs
|
||||||
|
# can differ by a second; compare with a tolerance.
|
||||||
|
assert abs(rs.st_mtime - fs.st_mtime) <= 2, \
|
||||||
|
f"mtime mismatch for {rel} with {extra}"
|
||||||
|
else:
|
||||||
|
assert int(rs.st_mtime) == int(fs.st_mtime), \
|
||||||
|
f"mtime mismatch for {rel} with {extra}"
|
||||||
|
|
||||||
|
|
||||||
|
class TestDirsOneLevel:
|
||||||
|
"""#13: -d with a trailing slash (or '.') lists the source's immediate
|
||||||
|
contents; FastSync mirrors them below the source-root mirror, so compare
|
||||||
|
rsync's destination tree against that mirror."""
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_dirs_trailing_slash_matches_rsync(self, shared_server):
|
||||||
|
source = _make_tree(os.path.join(TEST_DATA_DIR, "sel_dirs_src"))
|
||||||
|
os.makedirs(os.path.join(source, "empty"), exist_ok=True)
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "sel_dirs_dst")
|
||||||
|
rdst = os.path.join(TEST_DATA_DIR, "sel_dirs_rdst")
|
||||||
|
clean_dir(dest)
|
||||||
|
clean_dir(rdst)
|
||||||
|
r = _rsync(["-d", source + "/", rdst + "/"])
|
||||||
|
assert r.returncode == 0, r.stderr
|
||||||
|
result, _ = run_client(source + "/", dest, flags=["-d"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, result.stderr[:300]
|
||||||
|
mirror = get_dest_received_dir(dest, source)
|
||||||
|
assert _tree(rdst) == _tree(mirror)
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_dirs_relative_matches_rsync(self, shared_server):
|
||||||
|
source = _make_tree(os.path.join(TEST_DATA_DIR, "sel_dirsr_src"))
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "sel_dirsr_dst")
|
||||||
|
rdst = os.path.join(TEST_DATA_DIR, "sel_dirsr_rdst")
|
||||||
|
clean_dir(dest)
|
||||||
|
clean_dir(rdst)
|
||||||
|
spec = source + "/./foo"
|
||||||
|
r = _rsync(["-d", "-R", spec, rdst + "/"])
|
||||||
|
assert r.returncode == 0, r.stderr
|
||||||
|
result, _ = run_client(spec, dest, flags=["-d", "-R"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, result.stderr[:300]
|
||||||
|
assert _tree(rdst) == _tree(dest)
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_relative_delete_scope_matches_rsync(self):
|
||||||
|
"""-R --delete must be confined to the transferred prefix subtree so a
|
||||||
|
sibling destination directory survives (rsync parity)."""
|
||||||
|
source = _make_tree(os.path.join(TEST_DATA_DIR, "sel_delscope_src"))
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "sel_delscope_dst")
|
||||||
|
rdst = os.path.join(TEST_DATA_DIR, "sel_delscope_rdst")
|
||||||
|
spec = source + "/./foo"
|
||||||
|
for root in (dest, rdst):
|
||||||
|
clean_dir(root)
|
||||||
|
os.makedirs(os.path.join(root, "foo"))
|
||||||
|
with open(os.path.join(root, "foo", "extra.txt"), "wb") as fh:
|
||||||
|
fh.write(b"extra\n")
|
||||||
|
os.makedirs(os.path.join(root, "unrelated"))
|
||||||
|
with open(os.path.join(root, "unrelated", "keep.txt"), "wb") as fh:
|
||||||
|
fh.write(b"keep\n")
|
||||||
|
with ServerManager() as server:
|
||||||
|
server.start(extra_args=["--allow-delete"])
|
||||||
|
r = _rsync(["-aR", "--delete", spec, rdst + "/"])
|
||||||
|
assert r.returncode == 0, r.stderr
|
||||||
|
result, _ = run_client(spec, dest, flags=["-a", "-R", "--delete"],
|
||||||
|
port=server.port)
|
||||||
|
assert result.returncode == 0, result.stderr[:300]
|
||||||
|
assert (os.path.isfile(os.path.join(dest, "unrelated", "keep.txt"))
|
||||||
|
== os.path.isfile(os.path.join(rdst, "unrelated", "keep.txt")))
|
||||||
|
assert _tree(dest) == _tree(rdst)
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
@pytest.mark.ci
|
||||||
|
@pytest.mark.parametrize("mt", [False, True])
|
||||||
|
def test_relative_delete_protects_excluded_mirror(self, mt):
|
||||||
|
"""-R --delete with --exclude must protect the destination mirror of an
|
||||||
|
excluded source path (recorded as a prefix-relative wire path)."""
|
||||||
|
source = _make_tree(os.path.join(TEST_DATA_DIR, "sel_delexc_src"))
|
||||||
|
with open(os.path.join(source, "foo", "secret.tmp"), "wb") as fh:
|
||||||
|
fh.write(b"secret\n")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "sel_delexc_dst")
|
||||||
|
rdst = os.path.join(TEST_DATA_DIR, "sel_delexc_rdst")
|
||||||
|
for root in (dest, rdst):
|
||||||
|
clean_dir(root)
|
||||||
|
os.makedirs(os.path.join(root, "foo"))
|
||||||
|
with open(os.path.join(root, "foo", "secret.tmp"), "wb") as fh:
|
||||||
|
fh.write(b"secret\n")
|
||||||
|
with open(os.path.join(root, "foo", "extra.txt"), "wb") as fh:
|
||||||
|
fh.write(b"extra\n")
|
||||||
|
spec = source + "/./foo"
|
||||||
|
with ServerManager() as server:
|
||||||
|
server.start(extra_args=["--allow-delete"])
|
||||||
|
r = _rsync(["-aR", "--delete", "--exclude=*.tmp", spec, rdst + "/"])
|
||||||
|
assert r.returncode == 0, r.stderr
|
||||||
|
flags = ["-a", "-R", "--delete", "--exclude=*.tmp"] + (["--threads"] if mt else [])
|
||||||
|
result, _ = run_client(spec, dest, flags=flags, port=server.port)
|
||||||
|
assert result.returncode == 0, result.stderr[:300]
|
||||||
|
assert _tree(dest) == _tree(rdst)
|
||||||
|
assert os.path.isfile(os.path.join(dest, "foo", "secret.tmp"))
|
||||||
|
assert not os.path.exists(os.path.join(dest, "foo", "extra.txt"))
|
||||||
|
|
||||||
|
|
||||||
|
class TestClientAliases:
|
||||||
|
"""#5: safe rsync option aliases accepted client-side."""
|
||||||
|
|
||||||
|
def _seed(self):
|
||||||
|
source = _make_tree(os.path.join(TEST_DATA_DIR, "sel_alias_src"))
|
||||||
|
return source
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"flag",
|
||||||
|
[
|
||||||
|
"--ignore-non-existing",
|
||||||
|
"--protect-args",
|
||||||
|
"--msgs2stderr",
|
||||||
|
"--no-msgs2stderr",
|
||||||
|
"--no-iconv",
|
||||||
|
"--iconv=.",
|
||||||
|
"--iconv=-",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_alias_accepted(self, shared_server, flag):
|
||||||
|
source = self._seed()
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "sel_alias_dst")
|
||||||
|
clean_dir(dest)
|
||||||
|
result, _ = run_client(source, dest, flags=[flag], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"{flag} rejected: {result.stderr[:300]}"
|
||||||
|
|
||||||
|
def test_lone_h_prints_help(self):
|
||||||
|
result = subprocess.run([CLIENT_CMD[0], "-h"], capture_output=True, text=True,
|
||||||
|
timeout=30)
|
||||||
|
assert result.returncode == 0, result.stderr
|
||||||
|
assert "Usage" in (result.stdout + result.stderr)
|
||||||
|
|
||||||
|
def test_h_with_args_still_human_readable(self, shared_server):
|
||||||
|
source = self._seed()
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "sel_h_dst")
|
||||||
|
clean_dir(dest)
|
||||||
|
result, _ = run_client(source, dest, flags=["-h"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, result.stderr[:300]
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
assert os.path.isfile(os.path.join(received, "top.txt"))
|
||||||
|
|
||||||
|
|
||||||
|
class TestFilesFromEdges:
|
||||||
|
"""#8/#58: --files-from empty list succeeds; rsync 3.4.1 rejects the
|
||||||
|
--no-ignore-missing-args negation, so FastSync must reject it too."""
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_empty_files_from_list_succeeds(self, shared_server):
|
||||||
|
source = _make_tree(os.path.join(TEST_DATA_DIR, "sel_ff_src"))
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "sel_ff_dst")
|
||||||
|
rdst = os.path.join(TEST_DATA_DIR, "sel_ff_rdst")
|
||||||
|
clean_dir(dest)
|
||||||
|
clean_dir(rdst)
|
||||||
|
lst = os.path.join(TEST_DATA_DIR, "sel_ff_empty")
|
||||||
|
with open(lst, "w") as fh:
|
||||||
|
fh.write("")
|
||||||
|
r = _rsync(["-a", "--files-from=" + lst, source + "/", rdst + "/"])
|
||||||
|
assert r.returncode == 0, r.stderr
|
||||||
|
result, _ = run_client(source, dest, flags=["--files-from", lst],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, result.stderr[:300]
|
||||||
|
assert _tree(rdst) == []
|
||||||
|
assert _tree(dest) == []
|
||||||
|
|
||||||
|
@requires_rsync
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_no_ignore_missing_args_rejected_like_rsync(self):
|
||||||
|
source = _make_tree(os.path.join(TEST_DATA_DIR, "sel_nima_src"))
|
||||||
|
r = _rsync(["-a", "--no-ignore-missing-args", source + "/",
|
||||||
|
os.path.join(TEST_DATA_DIR, "sel_nima_rdst") + "/"])
|
||||||
|
assert r.returncode != 0, "rsync unexpectedly accepted --no-ignore-missing-args"
|
||||||
|
|
||||||
|
cmd = CLIENT_CMD + ["--source-dir", source, "--dest-dir",
|
||||||
|
os.path.join(TEST_DATA_DIR, "sel_nima_dst"), "--save-to-disk",
|
||||||
|
"--no-ignore-missing-args"]
|
||||||
|
result = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
|
||||||
|
assert result.returncode != 0, "FastSync unexpectedly accepted the negation"
|
||||||
@@ -94,14 +94,14 @@ def _seed_protocol_source(source):
|
|||||||
class TestProtocol:
|
class TestProtocol:
|
||||||
@pytest.mark.ci
|
@pytest.mark.ci
|
||||||
def test_protocol_current_version_accepted(self, shared_server):
|
def test_protocol_current_version_accepted(self, shared_server):
|
||||||
"""--protocol=2.21.0 (the current PROTOCOL_VERSION) is accepted and the
|
"""--protocol=2.26.0 (the current PROTOCOL_VERSION) is accepted and the
|
||||||
transfer completes normally."""
|
transfer completes normally."""
|
||||||
source = os.path.join(TEST_DATA_DIR, "proto_ok_src")
|
source = os.path.join(TEST_DATA_DIR, "proto_ok_src")
|
||||||
dest = os.path.join(TEST_DATA_DIR, "proto_ok_dst")
|
dest = os.path.join(TEST_DATA_DIR, "proto_ok_dst")
|
||||||
shutil.rmtree(dest, ignore_errors=True)
|
shutil.rmtree(dest, ignore_errors=True)
|
||||||
os.makedirs(dest)
|
os.makedirs(dest)
|
||||||
_seed_protocol_source(source)
|
_seed_protocol_source(source)
|
||||||
result, _ = run_client(source, dest, flags=["--protocol=2.21.0"],
|
result, _ = run_client(source, dest, flags=["--protocol=2.26.0"],
|
||||||
port=shared_server.port)
|
port=shared_server.port)
|
||||||
assert result.returncode == 0, \
|
assert result.returncode == 0, \
|
||||||
f"--protocol current run failed: {(result.stderr or result.stdout)[:400]}"
|
f"--protocol current run failed: {(result.stderr or result.stdout)[:400]}"
|
||||||
@@ -118,7 +118,8 @@ class TestProtocol:
|
|||||||
shutil.rmtree(dest, ignore_errors=True)
|
shutil.rmtree(dest, ignore_errors=True)
|
||||||
os.makedirs(dest)
|
os.makedirs(dest)
|
||||||
_seed_protocol_source(source)
|
_seed_protocol_source(source)
|
||||||
for bad in ("2.20.0", "2.19.0", "2.18.0", "2.17.0", "2.15.0", "2.16.0", "216", "31"):
|
for bad in ("2.22.0", "2.21.0", "2.20.0", "2.19.0", "2.18.0", "2.17.0", "2.15.0", "2.16.0",
|
||||||
|
"216", "31"):
|
||||||
result, _ = run_client(source, dest, flags=[f"--protocol={bad}"],
|
result, _ = run_client(source, dest, flags=[f"--protocol={bad}"],
|
||||||
port=shared_server.port)
|
port=shared_server.port)
|
||||||
assert result.returncode != 0, f"--protocol={bad} should be rejected"
|
assert result.returncode != 0, f"--protocol={bad} should be rejected"
|
||||||
|
|||||||
@@ -0,0 +1,489 @@
|
|||||||
|
"""Wave 2b: per-attribute preservation split (-p/-t/-o/-g and their negations).
|
||||||
|
|
||||||
|
The receiver applies each attribute independently (see src/shared/file_attr.h).
|
||||||
|
These tests cover the per-flag behavior end-to-end, the CLI negations, directory
|
||||||
|
modes, and the unprivileged best-effort / root-only ownership paths. They reuse
|
||||||
|
the established helpers from common.py.
|
||||||
|
|
||||||
|
The `-s` spelling is rsync's --secluded-args no-op in FastSync; chunk
|
||||||
|
serialization is the long-form --chunk-serialization, which is what the feature
|
||||||
|
matrix below exercises.
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
import stat
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
sys.path.insert(0, os.path.dirname(__file__))
|
||||||
|
from common import (
|
||||||
|
TEST_DATA_DIR,
|
||||||
|
run_client,
|
||||||
|
clean_dir,
|
||||||
|
get_dest_received_dir,
|
||||||
|
ServerManager,
|
||||||
|
)
|
||||||
|
|
||||||
|
DISTINCT_MTIME = 1_000_000_000 # 2001-09-09T01:46:40Z, a whole second
|
||||||
|
|
||||||
|
|
||||||
|
def _process_umask():
|
||||||
|
current = os.umask(0)
|
||||||
|
os.umask(current)
|
||||||
|
return current
|
||||||
|
|
||||||
|
|
||||||
|
def _seed_file(source, dest, name, content, mode, mtime=None):
|
||||||
|
"""Create a one-file source tree at an explicit mode (and mtime), and a
|
||||||
|
clean destination. Returns the source file path."""
|
||||||
|
clean_dir(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
path = os.path.join(source, name)
|
||||||
|
with open(path, "wb") as fh:
|
||||||
|
fh.write(content)
|
||||||
|
os.chmod(path, mode)
|
||||||
|
if mtime is not None:
|
||||||
|
os.utime(path, (mtime, mtime))
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def _received(dest, source, name):
|
||||||
|
return os.path.join(get_dest_received_dir(dest, source), name)
|
||||||
|
|
||||||
|
|
||||||
|
class TestPreservePerms:
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_p_applies_source_mode(self, shared_server):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "perms_p_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "perms_p_dst")
|
||||||
|
_seed_file(source, dest, "f.txt", b"perms\n", 0o750)
|
||||||
|
|
||||||
|
result, _ = run_client(source, dest, flags=["-p"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, \
|
||||||
|
f"-p failed: {(result.stderr or result.stdout)[:300]}"
|
||||||
|
got = os.stat(_received(dest, source, "f.txt")).st_mode & 0o777
|
||||||
|
assert got == 0o750, f"-p must apply the source mode, got {oct(got)}"
|
||||||
|
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_without_p_preexisting_dest_keeps_mode(self, shared_server):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "perms_nop_exist_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "perms_nop_exist_dst")
|
||||||
|
src_file = _seed_file(source, dest, "f.txt", b"one\n", 0o750)
|
||||||
|
|
||||||
|
# Seed the destination.
|
||||||
|
result, _ = run_client(source, dest, flags=["-p"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"seed failed: {(result.stderr or '')[:200]}"
|
||||||
|
|
||||||
|
# Give the destination a distinguishable mode, then re-transfer without
|
||||||
|
# -p (but with -t so metadata still travels).
|
||||||
|
dst_file = _received(dest, source, "f.txt")
|
||||||
|
os.chmod(dst_file, 0o600)
|
||||||
|
with open(src_file, "wb") as fh:
|
||||||
|
fh.write(b"two, changed content\n")
|
||||||
|
|
||||||
|
result, _ = run_client(source, dest, flags=["-t"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"re-run failed: {(result.stderr or '')[:200]}"
|
||||||
|
got = os.stat(dst_file).st_mode & 0o777
|
||||||
|
assert got == 0o600, \
|
||||||
|
f"without -p a pre-existing destination must keep its mode, got {oct(got)}"
|
||||||
|
with open(dst_file, "rb") as fh:
|
||||||
|
assert fh.read() == b"two, changed content\n"
|
||||||
|
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_without_p_new_dest_gets_source_and_umask(self, shared_server):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "perms_nop_new_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "perms_nop_new_dst")
|
||||||
|
# 0664 has group/other bits that the umask strips, so the result is not
|
||||||
|
# just the source mode. Under strict rsync parity the source mode is
|
||||||
|
# masked only by the umask (group/other write is no longer force-cleared
|
||||||
|
# on top of it).
|
||||||
|
_seed_file(source, dest, "f.txt", b"new\n", 0o664)
|
||||||
|
|
||||||
|
result, _ = run_client(source, dest, flags=["-t"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"-t failed: {(result.stderr or result.stdout)[:300]}"
|
||||||
|
want = 0o664 & ~_process_umask()
|
||||||
|
got = os.stat(_received(dest, source, "f.txt")).st_mode & 0o777
|
||||||
|
assert got == want, \
|
||||||
|
f"new no--p destination mode: want {oct(want)}, got {oct(got)}"
|
||||||
|
|
||||||
|
|
||||||
|
class TestPreserveTimes:
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_t_applies_mtime(self, shared_server):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "times_t_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "times_t_dst")
|
||||||
|
_seed_file(source, dest, "f.txt", b"times\n", 0o644, mtime=DISTINCT_MTIME)
|
||||||
|
|
||||||
|
result, _ = run_client(source, dest, flags=["-t"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"-t failed: {(result.stderr or '')[:300]}"
|
||||||
|
dst_m = os.stat(_received(dest, source, "f.txt")).st_mtime
|
||||||
|
assert abs(dst_m - DISTINCT_MTIME) < 2, \
|
||||||
|
f"-t must apply the source mtime, got {dst_m}"
|
||||||
|
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_without_t_dest_mtime_differs(self, shared_server):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "times_not_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "times_not_dst")
|
||||||
|
_seed_file(source, dest, "f.txt", b"times\n", 0o644, mtime=DISTINCT_MTIME)
|
||||||
|
|
||||||
|
# -p transmits metadata but must not apply the source mtime.
|
||||||
|
result, _ = run_client(source, dest, flags=["-p"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"-p failed: {(result.stderr or '')[:300]}"
|
||||||
|
dst_m = os.stat(_received(dest, source, "f.txt")).st_mtime
|
||||||
|
assert abs(dst_m - DISTINCT_MTIME) > 24 * 3600, \
|
||||||
|
f"without -t the destination mtime must not be the source mtime ({dst_m})"
|
||||||
|
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_incremental_t_retransfers_after_no_t(self, shared_server):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "times_incr_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "times_incr_dst")
|
||||||
|
_seed_file(source, dest, "f.txt", b"retransfer\n", 0o644, mtime=DISTINCT_MTIME)
|
||||||
|
|
||||||
|
# First run without -t: the destination mtime becomes "now", differing
|
||||||
|
# from the pinned source mtime.
|
||||||
|
result, _ = run_client(source, dest, flags=["-p"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"seed failed: {(result.stderr or '')[:200]}"
|
||||||
|
dst_file = _received(dest, source, "f.txt")
|
||||||
|
assert abs(os.stat(dst_file).st_mtime - DISTINCT_MTIME) > 24 * 3600
|
||||||
|
|
||||||
|
# The incremental quick-check now sees a mtime mismatch, so the file is
|
||||||
|
# re-transferred and -t stamps the source time.
|
||||||
|
result, _ = run_client(source, dest, flags=["--incremental", "-t"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"incremental -t failed: {(result.stderr or '')[:300]}"
|
||||||
|
dst_m = os.stat(dst_file).st_mtime
|
||||||
|
assert abs(dst_m - DISTINCT_MTIME) < 2, \
|
||||||
|
f"second --incremental -t run must re-transfer and stamp the mtime, got {dst_m}"
|
||||||
|
|
||||||
|
|
||||||
|
class TestPreserveNegations:
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_a_no_owner_no_group_keeps_perms_and_times(self, shared_server):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "neg_owner_group_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "neg_owner_group_dst")
|
||||||
|
_seed_file(source, dest, "f.txt", b"neg\n", 0o750, mtime=DISTINCT_MTIME)
|
||||||
|
|
||||||
|
result, _ = run_client(source, dest, flags=["-a", "--no-owner", "--no-group"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"-a --no-owner --no-group: {(result.stderr or '')[:300]}"
|
||||||
|
st = os.stat(_received(dest, source, "f.txt"))
|
||||||
|
assert st.st_mode & 0o777 == 0o750, "perms must survive the owner/group negation"
|
||||||
|
assert abs(st.st_mtime - DISTINCT_MTIME) < 2, "times must survive the owner/group negation"
|
||||||
|
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_a_no_perms_keeps_times_and_dest_mode(self, shared_server):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "neg_perms_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "neg_perms_dst")
|
||||||
|
src_file = _seed_file(source, dest, "f.txt", b"one\n", 0o750, mtime=DISTINCT_MTIME)
|
||||||
|
|
||||||
|
result, _ = run_client(source, dest, flags=["-a"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"seed failed: {(result.stderr or '')[:200]}"
|
||||||
|
dst_file = _received(dest, source, "f.txt")
|
||||||
|
os.chmod(dst_file, 0o600)
|
||||||
|
with open(src_file, "wb") as fh:
|
||||||
|
fh.write(b"changed\n")
|
||||||
|
# Rewriting the source bumped its mtime; restore the pinned value so the
|
||||||
|
# --no-perms run still has a distinct source time to apply.
|
||||||
|
os.utime(src_file, (DISTINCT_MTIME, DISTINCT_MTIME))
|
||||||
|
|
||||||
|
result, _ = run_client(source, dest, flags=["-a", "--no-perms"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"-a --no-perms: {(result.stderr or '')[:300]}"
|
||||||
|
st = os.stat(dst_file)
|
||||||
|
assert st.st_mode & 0o777 == 0o600, \
|
||||||
|
f"--no-perms must keep the destination mode, got {oct(st.st_mode & 0o777)}"
|
||||||
|
assert abs(st.st_mtime - DISTINCT_MTIME) < 2, "--no-perms must not disable times"
|
||||||
|
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_a_no_times_keeps_perms_but_not_mtime(self, shared_server):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "neg_times_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "neg_times_dst")
|
||||||
|
_seed_file(source, dest, "f.txt", b"neg times\n", 0o750, mtime=DISTINCT_MTIME)
|
||||||
|
|
||||||
|
result, _ = run_client(source, dest, flags=["-a", "--no-times"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"-a --no-times: {(result.stderr or '')[:300]}"
|
||||||
|
st = os.stat(_received(dest, source, "f.txt"))
|
||||||
|
assert st.st_mode & 0o777 == 0o750, "--no-times must not disable perms"
|
||||||
|
assert abs(st.st_mtime - DISTINCT_MTIME) > 24 * 3600, \
|
||||||
|
"--no-times must not apply the source mtime"
|
||||||
|
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_preserve_no_preserve_clears_all(self, shared_server):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "neg_bundle_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "neg_bundle_dst")
|
||||||
|
_seed_file(source, dest, "f.txt", b"bundle\n", 0o750, mtime=DISTINCT_MTIME)
|
||||||
|
|
||||||
|
result, _ = run_client(source, dest, flags=["--preserve", "--no-preserve"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"--preserve --no-preserve: {(result.stderr or '')[:300]}"
|
||||||
|
dst_file = _received(dest, source, "f.txt")
|
||||||
|
with open(dst_file, "rb") as fh:
|
||||||
|
assert fh.read() == b"bundle\n"
|
||||||
|
st = os.stat(dst_file)
|
||||||
|
# No metadata travels at all: a new file gets the fixed safe 0644 and
|
||||||
|
# the source mtime is not applied.
|
||||||
|
assert st.st_mode & 0o777 == 0o644, \
|
||||||
|
f"--no-preserve must not apply the source mode, got {oct(st.st_mode & 0o777)}"
|
||||||
|
assert abs(st.st_mtime - DISTINCT_MTIME) > 24 * 3600, \
|
||||||
|
"--no-preserve must not apply the source mtime"
|
||||||
|
|
||||||
|
|
||||||
|
class TestDirectoryModes:
|
||||||
|
def _tree(self, name, dir_mode, pin_mtime):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, name + "_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, name + "_dst")
|
||||||
|
clean_dir(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
sub = os.path.join(source, "sub")
|
||||||
|
os.makedirs(sub)
|
||||||
|
with open(os.path.join(sub, "file.txt"), "wb") as fh:
|
||||||
|
fh.write(b"dir mode content\n")
|
||||||
|
os.chmod(sub, dir_mode)
|
||||||
|
if pin_mtime:
|
||||||
|
os.utime(sub, (DISTINCT_MTIME, DISTINCT_MTIME))
|
||||||
|
return source, dest, sub
|
||||||
|
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_p_applies_directory_mode(self, shared_server):
|
||||||
|
source, dest, _ = self._tree("dirmode_p", 0o750, pin_mtime=False)
|
||||||
|
result, _ = run_client(source, dest, flags=["-p"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"-p failed: {(result.stderr or result.stdout)[:300]}"
|
||||||
|
got = os.stat(os.path.join(get_dest_received_dir(dest, source), "sub")).st_mode & 0o777
|
||||||
|
assert got == 0o750, f"-p must apply the source directory mode, got {oct(got)}"
|
||||||
|
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_p_preserves_directory_group_other_write(self, shared_server):
|
||||||
|
# Strict rsync parity: -p copies the source directory mode exactly,
|
||||||
|
# including group/other write (the old sanitization is gone).
|
||||||
|
source, dest, _ = self._tree("dirmode_go_write", 0o777, pin_mtime=False)
|
||||||
|
result, _ = run_client(source, dest, flags=["-p"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"-p failed: {(result.stderr or result.stdout)[:300]}"
|
||||||
|
mode = os.stat(os.path.join(get_dest_received_dir(dest, source), "sub")).st_mode & 0o777
|
||||||
|
assert mode == 0o777, \
|
||||||
|
f"-p must preserve the source directory mode exactly, got {oct(mode)}"
|
||||||
|
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_omit_dir_times_suppresses_times_not_modes(self, shared_server):
|
||||||
|
source, dest, _ = self._tree("dirmode_omit", 0o750, pin_mtime=True)
|
||||||
|
result, _ = run_client(source, dest, flags=["-a", "-O"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"-a -O failed: {(result.stderr or result.stdout)[:300]}"
|
||||||
|
st = os.stat(os.path.join(get_dest_received_dir(dest, source), "sub"))
|
||||||
|
assert st.st_mode & 0o777 == 0o750, \
|
||||||
|
f"-O must suppress only dir times, not dir modes (got {oct(st.st_mode & 0o777)})"
|
||||||
|
assert abs(st.st_mtime - DISTINCT_MTIME) > 5, \
|
||||||
|
f"-O must not apply the directory mtime (got {st.st_mtime})"
|
||||||
|
|
||||||
|
|
||||||
|
class TestOwnershipBestEffort:
|
||||||
|
"""-o/-g/-a must succeed with correct content even when the receiver cannot
|
||||||
|
chown (the unprivileged CI case). Ownership is deliberately not asserted."""
|
||||||
|
|
||||||
|
@pytest.mark.ci
|
||||||
|
@pytest.mark.parametrize("flags", [["-o"], ["-g"], ["-a"]])
|
||||||
|
def test_ownership_flags_succeed_unprivileged(self, shared_server, flags):
|
||||||
|
tag = flags[0].strip("-")
|
||||||
|
source = os.path.join(TEST_DATA_DIR, f"best_effort_{tag}_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, f"best_effort_{tag}_dst")
|
||||||
|
_seed_file(source, dest, "f.txt", b"best effort ownership\n", 0o640)
|
||||||
|
|
||||||
|
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
|
||||||
|
assert result.returncode == 0, \
|
||||||
|
f"{flags} exit {result.returncode}: {(result.stderr or result.stdout)[:300]}"
|
||||||
|
with open(_received(dest, source, "f.txt"), "rb") as fh:
|
||||||
|
assert fh.read() == b"best effort ownership\n"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.skipif(os.geteuid() != 0, reason="only root can change ownership")
|
||||||
|
class TestOwnershipRoot:
|
||||||
|
"""Root-only per-attribute ownership application. Not marked ci: the PR
|
||||||
|
gate runs as an unprivileged user."""
|
||||||
|
|
||||||
|
def _seed_owned(self, tag, uid, gid):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, f"root_owner_{tag}_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, f"root_owner_{tag}_dst")
|
||||||
|
path = _seed_file(source, dest, "f.txt", b"root ownership\n", 0o644)
|
||||||
|
os.chown(path, uid, gid)
|
||||||
|
return source, dest
|
||||||
|
|
||||||
|
def test_o_applies_owner_only(self, shared_server):
|
||||||
|
source, dest = self._seed_owned("o", 12345, 12346)
|
||||||
|
result, _ = run_client(source, dest, flags=["-o"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"-o failed: {(result.stderr or '')[:300]}"
|
||||||
|
st = os.stat(_received(dest, source, "f.txt"))
|
||||||
|
assert st.st_uid == 12345, f"-o must apply the owner, got uid={st.st_uid}"
|
||||||
|
assert st.st_gid != 12346, "-o must not change the group"
|
||||||
|
|
||||||
|
def test_g_applies_group_only(self, shared_server):
|
||||||
|
source, dest = self._seed_owned("g", 12345, 54321)
|
||||||
|
result, _ = run_client(source, dest, flags=["-g"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"-g failed: {(result.stderr or '')[:300]}"
|
||||||
|
st = os.stat(_received(dest, source, "f.txt"))
|
||||||
|
assert st.st_gid == 54321, f"-g must apply the group, got gid={st.st_gid}"
|
||||||
|
assert st.st_uid != 12345, "-g must not change the owner"
|
||||||
|
|
||||||
|
def test_a_applies_owner_and_group(self, shared_server):
|
||||||
|
source, dest = self._seed_owned("a", 12345, 54321)
|
||||||
|
result, _ = run_client(source, dest, flags=["-a"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"-a failed: {(result.stderr or '')[:300]}"
|
||||||
|
st = os.stat(_received(dest, source, "f.txt"))
|
||||||
|
assert (st.st_uid, st.st_gid) == (12345, 54321), \
|
||||||
|
f"-a must apply owner+group, got uid={st.st_uid} gid={st.st_gid}"
|
||||||
|
|
||||||
|
def test_chown_overrides_o(self, shared_server):
|
||||||
|
source, dest = self._seed_owned("chown", 11111, 22222)
|
||||||
|
result, _ = run_client(source, dest, flags=["-o", "--chown=@33333:@44444"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"-o --chown failed: {(result.stderr or '')[:300]}"
|
||||||
|
st = os.stat(_received(dest, source, "f.txt"))
|
||||||
|
assert (st.st_uid, st.st_gid) == (33333, 44444), \
|
||||||
|
f"--chown must override -o, got uid={st.st_uid} gid={st.st_gid}"
|
||||||
|
|
||||||
|
def test_fake_super_o_does_not_real_chown(self, shared_server):
|
||||||
|
# #294: --fake-super only RECORDS ownership; it must never real-chown the
|
||||||
|
# recorded source owner (that defeats the point of the flag). With -o the
|
||||||
|
# resolved owner is parked in the reserved xattr and the on-disk owner is
|
||||||
|
# left as the receiver's.
|
||||||
|
source, dest = self._seed_owned("fake_o", 12345, 54321)
|
||||||
|
result, _ = run_client(source, dest, flags=["--fake-super", "-o"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"--fake-super -o failed: {(result.stderr or '')[:300]}"
|
||||||
|
dst = _received(dest, source, "f.txt")
|
||||||
|
st = os.stat(dst)
|
||||||
|
assert st.st_uid != 12345, \
|
||||||
|
f"--fake-super -o must NOT real-chown the source owner, got uid={st.st_uid}"
|
||||||
|
record = os.getxattr(dst, "user.fastsync.stat").decode()
|
||||||
|
fields = record.split(":")
|
||||||
|
assert fields[0] == "12345", \
|
||||||
|
f"--fake-super must record the resolved owner, got {fields[0]}"
|
||||||
|
|
||||||
|
def test_o_applies_directory_owner(self, shared_server):
|
||||||
|
"""#286.2: -o must apply the source owner to DIRECTORIES too (the
|
||||||
|
deferred directory-metadata application now runs the identity path)."""
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "root_dir_o_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "root_dir_o_dst")
|
||||||
|
clean_dir(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
os.makedirs(os.path.join(source, "sub", "deep"))
|
||||||
|
with open(os.path.join(source, "sub", "deep", "f.txt"), "wb") as fh:
|
||||||
|
fh.write(b"dir owner\n")
|
||||||
|
os.chown(os.path.join(source, "sub"), 12345, 12346)
|
||||||
|
os.chown(os.path.join(source, "sub", "deep"), 23456, 34567)
|
||||||
|
|
||||||
|
result, _ = run_client(source, dest, flags=["-o", "-t"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"-o dir failed: {(result.stderr or '')[:300]}"
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
sub = os.stat(os.path.join(received, "sub"))
|
||||||
|
deep = os.stat(os.path.join(received, "sub", "deep"))
|
||||||
|
assert sub.st_uid == 12345, f"dir 'sub' owner not applied: {sub.st_uid}"
|
||||||
|
assert deep.st_uid == 23456, f"dir 'sub/deep' owner not applied: {deep.st_uid}"
|
||||||
|
# -o alone must not change the group.
|
||||||
|
assert sub.st_gid != 12346
|
||||||
|
|
||||||
|
def test_a_applies_directory_owner_and_group(self, shared_server):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "root_dir_a_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "root_dir_a_dst")
|
||||||
|
clean_dir(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
os.makedirs(os.path.join(source, "sub"))
|
||||||
|
with open(os.path.join(source, "sub", "f.txt"), "wb") as fh:
|
||||||
|
fh.write(b"dir owner group\n")
|
||||||
|
os.chown(os.path.join(source, "sub"), 12345, 54321)
|
||||||
|
|
||||||
|
result, _ = run_client(source, dest, flags=["-a"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"-a dir failed: {(result.stderr or '')[:300]}"
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
st = os.stat(os.path.join(received, "sub"))
|
||||||
|
assert (st.st_uid, st.st_gid) == (12345, 54321), \
|
||||||
|
f"-a must apply dir owner+group, got uid={st.st_uid} gid={st.st_gid}"
|
||||||
|
|
||||||
|
def test_numeric_ids_alone_does_not_chown(self, shared_server):
|
||||||
|
"""#286.1: --numeric-ids is a mapping modifier, not an ownership request.
|
||||||
|
`-t --numeric-ids` must leave the receiver's ownership untouched."""
|
||||||
|
source, dest = self._seed_owned("num_only", 12345, 54321)
|
||||||
|
result, _ = run_client(source, dest, flags=["-t", "--numeric-ids"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, \
|
||||||
|
f"-t --numeric-ids failed: {(result.stderr or '')[:300]}"
|
||||||
|
st = os.stat(_received(dest, source, "f.txt"))
|
||||||
|
assert st.st_uid != 12345, \
|
||||||
|
f"--numeric-ids alone must not chown, got uid={st.st_uid}"
|
||||||
|
|
||||||
|
def test_numeric_ids_with_o_uses_raw_id(self, shared_server):
|
||||||
|
source, dest = self._seed_owned("num_o", 12345, 54321)
|
||||||
|
result, _ = run_client(source, dest, flags=["-o", "-t", "--numeric-ids"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, \
|
||||||
|
f"-o --numeric-ids failed: {(result.stderr or '')[:300]}"
|
||||||
|
st = os.stat(_received(dest, source, "f.txt"))
|
||||||
|
assert st.st_uid == 12345, \
|
||||||
|
f"-o --numeric-ids must apply the raw id, got uid={st.st_uid}"
|
||||||
|
|
||||||
|
|
||||||
|
class TestPreserveFeatureMatrix:
|
||||||
|
"""A representative per-attribute check under the alternate transfer engines
|
||||||
|
(chunk serialization, --delay-updates, and the multithreaded scanner)."""
|
||||||
|
|
||||||
|
@pytest.mark.ci
|
||||||
|
@pytest.mark.parametrize("extra", ["--chunk-serialization", "--delay-updates", "--threads"])
|
||||||
|
def test_p_and_t_hold_under_engine(self, shared_server, extra):
|
||||||
|
tag = extra.strip("-").replace("-", "_")
|
||||||
|
source = os.path.join(TEST_DATA_DIR, f"matrix_{tag}_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, f"matrix_{tag}_dst")
|
||||||
|
_seed_file(source, dest, "f.txt", b"matrix\n", 0o750, mtime=DISTINCT_MTIME)
|
||||||
|
|
||||||
|
result, _ = run_client(source, dest, flags=["-p", "-t", extra],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, \
|
||||||
|
f"-p -t {extra} failed: {(result.stderr or result.stdout)[:300]}"
|
||||||
|
st = os.stat(_received(dest, source, "f.txt"))
|
||||||
|
assert st.st_mode & 0o777 == 0o750, f"mode lost under {extra}"
|
||||||
|
assert abs(st.st_mtime - DISTINCT_MTIME) < 2, f"mtime lost under {extra}"
|
||||||
|
|
||||||
|
|
||||||
|
class TestSpecialNodeModes:
|
||||||
|
"""Strict rsync parity: with -p the source FIFO mode is copied exactly,
|
||||||
|
including group/other write. Without -p the node follows the same
|
||||||
|
source & ~umask base as any other new entry. FIFOs are created
|
||||||
|
unprivileged via mkfifo."""
|
||||||
|
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_specials_p_preserves_fifo_mode(self):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "specialmode_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "specialmode_dst")
|
||||||
|
clean_dir(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
|
||||||
|
src_fifo = os.path.join(source, "world.fifo")
|
||||||
|
os.mkfifo(src_fifo)
|
||||||
|
os.chmod(src_fifo, 0o777)
|
||||||
|
assert os.stat(src_fifo).st_mode & 0o777 == 0o777
|
||||||
|
|
||||||
|
# Production daemonizes with umask(0) (server.c) so the source mode is
|
||||||
|
# what reaches mkfifo. The session server runs in the foreground and
|
||||||
|
# would inherit the runner's umask, which alone would strip the write
|
||||||
|
# bits and mask a regression. Start a dedicated foreground server under
|
||||||
|
# umask(0) to exercise the real path.
|
||||||
|
server = ServerManager()
|
||||||
|
saved_umask = os.umask(0)
|
||||||
|
try:
|
||||||
|
server.start(extra_args=["--allow-super"])
|
||||||
|
finally:
|
||||||
|
os.umask(saved_umask)
|
||||||
|
try:
|
||||||
|
result, _ = run_client(source, dest, flags=["--specials", "-p"],
|
||||||
|
port=server.port)
|
||||||
|
finally:
|
||||||
|
server.stop()
|
||||||
|
|
||||||
|
assert result.returncode == 0, \
|
||||||
|
f"--specials -p failed: {(result.stderr or result.stdout)[:300]}"
|
||||||
|
|
||||||
|
received = _received(dest, source, "world.fifo")
|
||||||
|
assert os.path.lexists(received), "source FIFO was not recreated on the destination"
|
||||||
|
st = os.lstat(received)
|
||||||
|
assert stat.S_ISFIFO(st.st_mode), f"received entry is not a FIFO: {oct(st.st_mode)}"
|
||||||
|
mode = st.st_mode & 0o777
|
||||||
|
assert mode == 0o777, \
|
||||||
|
f"-p must preserve the source FIFO mode exactly (want 0o777), got {oct(mode)}"
|
||||||
@@ -0,0 +1,262 @@
|
|||||||
|
"""Guard the README against drifting from the real CLI.
|
||||||
|
|
||||||
|
This test parses README.md and checks it against the actual sources of truth
|
||||||
|
instead of against a hand-maintained copy:
|
||||||
|
|
||||||
|
* client ``--help`` output -> ``src/client/usage.c`` (``print_usage``)
|
||||||
|
* server ``--help`` output -> ``src/server/server.c`` (``print_server_usage``)
|
||||||
|
* ``FASTSYNC_*`` env vars -> ``getenv("...")`` call sites under ``src/``
|
||||||
|
|
||||||
|
It is deliberately offline and read-only: no server is started, no transfer is
|
||||||
|
performed. Each binary is invoked at most once per test session and the result
|
||||||
|
is cached.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import functools
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
sys.path.insert(0, os.path.dirname(__file__))
|
||||||
|
from common import BUILD_DIR, PROJECT_ROOT
|
||||||
|
|
||||||
|
pytestmark = pytest.mark.ci
|
||||||
|
|
||||||
|
README_PATH = os.path.join(PROJECT_ROOT, "README.md")
|
||||||
|
SRC_DIR = os.path.join(PROJECT_ROOT, "src")
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Markdown helpers
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
_HEADING_RE = re.compile(r"^(#+)\s+(.*?)\s*$")
|
||||||
|
_BACKTICK_RE = re.compile(r"`([^`]*)`")
|
||||||
|
# A documented option may carry an argument annotation that is not part of the
|
||||||
|
# option name itself: ``--out=FILE``, ``--exclude <pattern>``, ``--threads[=N]``,
|
||||||
|
# ``--copy-as=USER[:GROUP]``. Cut the name loose from the first such marker.
|
||||||
|
_OPTION_SUFFIX_RE = re.compile(r"[=\s<\[(].*$")
|
||||||
|
_OPTION_TOKEN_RE = re.compile(r"^--?[A-Za-z][A-Za-z0-9-]*$")
|
||||||
|
|
||||||
|
|
||||||
|
def _readme_lines():
|
||||||
|
with open(README_PATH, encoding="utf-8") as fh:
|
||||||
|
return fh.read().splitlines()
|
||||||
|
|
||||||
|
|
||||||
|
def _heading_level(line):
|
||||||
|
match = _HEADING_RE.match(line)
|
||||||
|
return len(match.group(1)) if match else 0
|
||||||
|
|
||||||
|
|
||||||
|
def _section(lines, heading):
|
||||||
|
"""Return ``(lineno, line)`` pairs under the first exact ``heading``.
|
||||||
|
|
||||||
|
The section runs until the next heading of the same or higher level, so a
|
||||||
|
``##`` section includes its ``###`` subsections. Line numbers are 1-based
|
||||||
|
to match what a reader sees in an editor.
|
||||||
|
"""
|
||||||
|
target_level = _heading_level(heading)
|
||||||
|
for index, line in enumerate(lines):
|
||||||
|
if line.strip() != heading:
|
||||||
|
continue
|
||||||
|
start = index + 1
|
||||||
|
for end in range(start, len(lines)):
|
||||||
|
level = _heading_level(lines[end])
|
||||||
|
if level and level <= target_level:
|
||||||
|
return [(n + 1, lines[n]) for n in range(start, end)]
|
||||||
|
return [(n + 1, lines[n]) for n in range(start, len(lines))]
|
||||||
|
raise AssertionError(
|
||||||
|
f"README heading not found (has the README been restructured?): {heading!r}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _first_column_spans(section_lines):
|
||||||
|
"""Backticked spans from the first column of every markdown table row."""
|
||||||
|
spans = []
|
||||||
|
for lineno, line in section_lines:
|
||||||
|
stripped = line.strip()
|
||||||
|
if not stripped.startswith("|"):
|
||||||
|
continue
|
||||||
|
cells = stripped.split("|")
|
||||||
|
if len(cells) < 2:
|
||||||
|
continue
|
||||||
|
first = cells[1]
|
||||||
|
if set(first.strip()) <= set("-: "):
|
||||||
|
continue # header separator row, e.g. |---|---|
|
||||||
|
for match in _BACKTICK_RE.finditer(first):
|
||||||
|
spans.append((match.group(1), lineno))
|
||||||
|
return spans
|
||||||
|
|
||||||
|
|
||||||
|
def _documented_option_tokens(section_lines):
|
||||||
|
"""``(token, lineno, raw_cell)`` for each CLI option in a section's tables."""
|
||||||
|
found = []
|
||||||
|
for raw, lineno in _first_column_spans(section_lines):
|
||||||
|
for piece in re.split(r"[,\s]+", raw):
|
||||||
|
name = _OPTION_SUFFIX_RE.sub("", piece).strip()
|
||||||
|
if _OPTION_TOKEN_RE.match(name):
|
||||||
|
found.append((name, lineno, raw))
|
||||||
|
return found
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Sources of truth
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
_GETENV_RE = re.compile(r'getenv\s*\(\s*"([^"]+)"\s*\)')
|
||||||
|
_FASTSYNC_ENV_RE = re.compile(r"FASTSYNC_[A-Z0-9_]+")
|
||||||
|
|
||||||
|
|
||||||
|
def _getenv_names():
|
||||||
|
"""Every string literal passed to ``getenv()`` anywhere under ``src/``."""
|
||||||
|
names = set()
|
||||||
|
for root, _dirs, files in os.walk(SRC_DIR):
|
||||||
|
for filename in files:
|
||||||
|
if not filename.endswith((".c", ".h")):
|
||||||
|
continue
|
||||||
|
path = os.path.join(root, filename)
|
||||||
|
with open(path, encoding="utf-8", errors="replace") as fh:
|
||||||
|
names.update(_GETENV_RE.findall(fh.read()))
|
||||||
|
return names
|
||||||
|
|
||||||
|
|
||||||
|
@functools.lru_cache(maxsize=None)
|
||||||
|
def _help_stdout(binary_name):
|
||||||
|
"""Cached ``<binary> --help`` stdout; skipped (not failed) if unbuilt."""
|
||||||
|
binary = os.path.join(BUILD_DIR, binary_name)
|
||||||
|
if not (os.path.isfile(binary) and os.access(binary, os.X_OK)):
|
||||||
|
pytest.skip(
|
||||||
|
f"{binary} is not built; run "
|
||||||
|
"`cmake -B build -S . && cmake --build build` first"
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
result = subprocess.run(
|
||||||
|
[binary, "--help"], capture_output=True, text=True, timeout=30
|
||||||
|
)
|
||||||
|
except OSError as exc:
|
||||||
|
pytest.skip(f"could not execute {binary}: {exc}")
|
||||||
|
assert result.returncode == 0, (
|
||||||
|
f"{binary} --help exited {result.returncode}: "
|
||||||
|
f"{(result.stderr or result.stdout).strip()[:200]}"
|
||||||
|
)
|
||||||
|
return result.stdout
|
||||||
|
|
||||||
|
|
||||||
|
def _mentions_option(help_text, token):
|
||||||
|
"""True when ``token`` appears as a standalone option in ``help_text``.
|
||||||
|
|
||||||
|
A plain substring test would let a removed token hide behind a longer one
|
||||||
|
(``--del`` inside ``--delete``); requiring a non-word boundary on both sides
|
||||||
|
keeps every documented token individually accountable.
|
||||||
|
"""
|
||||||
|
return (
|
||||||
|
re.search(r"(?<![\w-])" + re.escape(token) + r"(?![\w-])", help_text)
|
||||||
|
is not None
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# Options the help text intentionally expresses only as a family (for example
|
||||||
|
# the generic ``--no-OPTION`` entry) rather than by spelling every member out.
|
||||||
|
# Add an entry here only with a one-line justification; prefer fixing the
|
||||||
|
# extractor first. Currently empty: every option the README documents is
|
||||||
|
# printed verbatim by the matching ``--help`` (including ``--no-super`` and
|
||||||
|
# ``--no-detach``).
|
||||||
|
_FAMILY_FORM_ALLOWLIST = frozenset()
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Tests
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_documented_env_vars_exist():
|
||||||
|
"""Every ``FASTSYNC_*`` in the README env table has a ``getenv()`` site.
|
||||||
|
|
||||||
|
Both directions are checked so the documented set and the source set stay
|
||||||
|
identical: a documented variable with no call site is a README defect, and a
|
||||||
|
new ``FASTSYNC_*`` call site without documentation is a README gap.
|
||||||
|
"""
|
||||||
|
documented = _documented_env_vars()
|
||||||
|
assert documented, "no FASTSYNC_* variables found in the README env table"
|
||||||
|
|
||||||
|
getenv_names = _getenv_names()
|
||||||
|
documented_names = {name for name, _lineno in documented}
|
||||||
|
|
||||||
|
missing_in_source = [
|
||||||
|
(name, lineno) for name, lineno in documented if name not in getenv_names
|
||||||
|
]
|
||||||
|
if missing_in_source:
|
||||||
|
details = "; ".join(
|
||||||
|
f"`{name}` (README.md line {lineno})"
|
||||||
|
for name, lineno in sorted(missing_in_source, key=lambda item: item[1])
|
||||||
|
)
|
||||||
|
pytest.fail(
|
||||||
|
"README documents environment variable(s) with no getenv() call "
|
||||||
|
f"site under src/: {details}"
|
||||||
|
)
|
||||||
|
|
||||||
|
fastsync_getenv = {name for name in getenv_names if name.startswith("FASTSYNC_")}
|
||||||
|
undocumented = sorted(fastsync_getenv - documented_names)
|
||||||
|
assert not undocumented, (
|
||||||
|
"src/ reads FASTSYNC_* environment variable(s) that the README does not "
|
||||||
|
f"document in '## Environment Variables': {', '.join(undocumented)}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_documented_client_flags_exist_in_help():
|
||||||
|
"""Client options in README tables must appear in ``client --help``."""
|
||||||
|
_assert_documented_flags(
|
||||||
|
"client",
|
||||||
|
["### Client", "## Client Options", "## FastSync Extensions"],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_documented_server_flags_exist_in_help():
|
||||||
|
"""Server options in README tables must appear in ``server --help``."""
|
||||||
|
_assert_documented_flags("server", ["### Server", "## Server Options"])
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Implementation helpers for the tests above
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _documented_env_vars():
|
||||||
|
"""``(name, lineno)`` for each ``FASTSYNC_*`` token in the env table."""
|
||||||
|
lines = _readme_lines()
|
||||||
|
section = _section(lines, "## Environment Variables")
|
||||||
|
found = []
|
||||||
|
for raw, lineno in _first_column_spans(section):
|
||||||
|
for match in _FASTSYNC_ENV_RE.finditer(raw):
|
||||||
|
found.append((match.group(0), lineno))
|
||||||
|
return found
|
||||||
|
|
||||||
|
|
||||||
|
def _assert_documented_flags(binary_name, headings):
|
||||||
|
help_text = _help_stdout(binary_name)
|
||||||
|
readme_lines = _readme_lines()
|
||||||
|
|
||||||
|
failures = []
|
||||||
|
for heading in headings:
|
||||||
|
for token, lineno, raw in _documented_option_tokens(
|
||||||
|
_section(readme_lines, heading)
|
||||||
|
):
|
||||||
|
if token in _FAMILY_FORM_ALLOWLIST:
|
||||||
|
continue
|
||||||
|
if not _mentions_option(help_text, token):
|
||||||
|
failures.append((lineno, token, heading, raw))
|
||||||
|
|
||||||
|
if failures:
|
||||||
|
failures.sort()
|
||||||
|
shown = "\n".join(
|
||||||
|
f" {token} (README.md line {lineno}, section {heading!r}, "
|
||||||
|
f"table cell `{raw}`)"
|
||||||
|
for lineno, token, heading, raw in failures
|
||||||
|
)
|
||||||
|
pytest.fail(
|
||||||
|
f"README documents option(s) missing from `{binary_name} --help`:\n"
|
||||||
|
f"{shown}"
|
||||||
|
)
|
||||||
@@ -150,13 +150,36 @@ class TestStopAt:
|
|||||||
assert _received_files(received) == [], \
|
assert _received_files(received) == [], \
|
||||||
f"expected nothing transferred, got {_received_files(received)}"
|
f"expected nothing transferred, got {_received_files(received)}"
|
||||||
|
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_stop_at_rsync_date_form(self, shared_server):
|
||||||
|
"""rsync's full date form (Y-M-DTh:m) is accepted; a deadline well in the
|
||||||
|
future lets the transfer complete normally."""
|
||||||
|
source, dest = _make("dateform")
|
||||||
|
_seed_source(source)
|
||||||
|
stamp = time.strftime("%Y-%m-%dT%H:%M", time.localtime(time.time() + 3600))
|
||||||
|
result, _ = run_client(source, dest, flags=[f"--stop-at={stamp}"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, \
|
||||||
|
f"--stop-at={stamp} should be accepted: " \
|
||||||
|
f"{(result.stderr or result.stdout)[:400]}"
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
mismatches, missing = verify_transfer(source, received)
|
||||||
|
assert not mismatches and not missing
|
||||||
|
|
||||||
|
# The slash-separated date spelling is accepted too.
|
||||||
|
slash = time.strftime("%Y/%m/%dT%H:%M", time.localtime(time.time() + 3600))
|
||||||
|
result, _ = run_client(source, dest, flags=[f"--stop-at={slash}"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"--stop-at={slash} should be accepted"
|
||||||
|
|
||||||
@pytest.mark.ci
|
@pytest.mark.ci
|
||||||
def test_stop_rejects_garbage(self, shared_server):
|
def test_stop_rejects_garbage(self, shared_server):
|
||||||
"""Malformed --stop-at/--stop-after values are rejected up front."""
|
"""Malformed --stop-at/--stop-after values are rejected up front."""
|
||||||
source, dest = _make("garbage")
|
source, dest = _make("garbage")
|
||||||
_seed_source(source)
|
_seed_source(source)
|
||||||
for flag in ("--stop-after=abc", "--stop-at=12:99", "--stop-at=12",
|
for flag in ("--stop-after=abc", "--stop-at=12:99", "--stop-at=1234",
|
||||||
"--stop-at=now+5x", "--stop-at=now-5s"):
|
"--stop-at=now+5x", "--stop-at=now-5s",
|
||||||
|
"--stop-at=2000-13-45", "--stop-at=2030-12-31T23:59:59"):
|
||||||
result, _ = run_client(source, dest, flags=[flag],
|
result, _ = run_client(source, dest, flags=[flag],
|
||||||
port=shared_server.port)
|
port=shared_server.port)
|
||||||
assert result.returncode != 0, f"{flag} should be rejected"
|
assert result.returncode != 0, f"{flag} should be rejected"
|
||||||
|
|||||||
@@ -15,6 +15,7 @@
|
|||||||
#include "test_file.h"
|
#include "test_file.h"
|
||||||
#include "test_file_list.h"
|
#include "test_file_list.h"
|
||||||
#include "test_file_sendfile.h"
|
#include "test_file_sendfile.h"
|
||||||
|
#include "test_format.h"
|
||||||
#include "test_fuzz_smoke.h"
|
#include "test_fuzz_smoke.h"
|
||||||
#include "test_glob.h"
|
#include "test_glob.h"
|
||||||
#include "test_hardlink.h"
|
#include "test_hardlink.h"
|
||||||
@@ -58,6 +59,7 @@ int main() {
|
|||||||
RUN_TEST(test_chunk);
|
RUN_TEST(test_chunk);
|
||||||
RUN_TEST(test_batch);
|
RUN_TEST(test_batch);
|
||||||
RUN_TEST(test_change_list);
|
RUN_TEST(test_change_list);
|
||||||
|
RUN_TEST(test_format);
|
||||||
RUN_TEST(test_config);
|
RUN_TEST(test_config);
|
||||||
RUN_TEST(test_credentials);
|
RUN_TEST(test_credentials);
|
||||||
RUN_TEST(test_compression);
|
RUN_TEST(test_compression);
|
||||||
|
|||||||
+106
-16
@@ -1,5 +1,6 @@
|
|||||||
#include "test_change_list.h"
|
#include "test_change_list.h"
|
||||||
#include "change_list.h"
|
#include "change_list.h"
|
||||||
|
#include "config.h"
|
||||||
#include "test_utils.h"
|
#include "test_utils.h"
|
||||||
#include "utils.h"
|
#include "utils.h"
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
@@ -9,64 +10,150 @@
|
|||||||
static ChangeEvent sample_event(void) {
|
static ChangeEvent sample_event(void) {
|
||||||
ChangeEvent event;
|
ChangeEvent event;
|
||||||
memset(&event, 0, sizeof(event));
|
memset(&event, 0, sizeof(event));
|
||||||
event.path = "/srv/root/sub/file.txt";
|
event.path = "src/sub/file.txt";
|
||||||
|
event.name = "sub/file.txt";
|
||||||
event.decision = CHANGE_SENT;
|
event.decision = CHANGE_SENT;
|
||||||
event.is_directory = false;
|
event.is_directory = false;
|
||||||
event.size = 12345;
|
event.size = 12345;
|
||||||
event.bytes_sent = 999;
|
event.bytes_sent = 999;
|
||||||
event.mtime_sec = 1700000000;
|
event.mtime_sec = 1700000000;
|
||||||
|
event.mtime_nsec = 0;
|
||||||
|
event.mode = 0100644;
|
||||||
|
event.uid = 1000;
|
||||||
|
event.gid = 1000;
|
||||||
return event;
|
return event;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Expected %M expansion computed independently with localtime_r. */
|
||||||
|
static void expected_mtime(time_t when, char out[32]) {
|
||||||
|
struct tm broken_down;
|
||||||
|
localtime_r(&when, &broken_down);
|
||||||
|
strftime(out, 32, "%Y/%m/%d-%H:%M:%S", &broken_down);
|
||||||
|
}
|
||||||
|
|
||||||
static void test_format_tokens() {
|
static void test_format_tokens() {
|
||||||
ChangeEvent event = sample_event();
|
ChangeEvent event = sample_event();
|
||||||
char* line = change_render_format("%f %n %l %b %M %%", &event);
|
Config* config = config_create();
|
||||||
|
char when[32];
|
||||||
|
expected_mtime(event.mtime_sec, when);
|
||||||
|
char* line = change_render_format("%f %n %l %b %M %%", config, &event);
|
||||||
EXPECT_NOT_NULL(line);
|
EXPECT_NOT_NULL(line);
|
||||||
EXPECT_EQ_STR(line, "/srv/root/sub/file.txt file.txt 12345 999 1700000000 %");
|
char expected[256];
|
||||||
|
snprintf(expected, sizeof(expected), "src/sub/file.txt sub/file.txt 12345 999 %s %%", when);
|
||||||
|
EXPECT_EQ_STR(line, expected);
|
||||||
free(line);
|
free(line);
|
||||||
|
config_delete(config);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void test_format_unknown_tokens_preserved() {
|
static void test_format_unknown_tokens_preserved() {
|
||||||
ChangeEvent event = sample_event();
|
ChangeEvent event = sample_event();
|
||||||
char* line = change_render_format("x%q=%f%z", &event);
|
Config* config = config_create();
|
||||||
|
char* line = change_render_format("x%q=%f%z", config, &event);
|
||||||
EXPECT_NOT_NULL(line);
|
EXPECT_NOT_NULL(line);
|
||||||
EXPECT_EQ_STR(line, "x%q=/srv/root/sub/file.txt%z");
|
EXPECT_EQ_STR(line, "x%q=src/sub/file.txt%z");
|
||||||
free(line);
|
free(line);
|
||||||
|
config_delete(config);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void test_format_leaf_name() {
|
static void test_format_directory_name_has_trailing_slash() {
|
||||||
ChangeEvent event = sample_event();
|
ChangeEvent event = sample_event();
|
||||||
event.path = "bare.txt";
|
event.is_directory = true;
|
||||||
char* line = change_render_format("%n|%f", &event);
|
event.path = "src/sub";
|
||||||
|
event.name = "sub";
|
||||||
|
Config* config = config_create();
|
||||||
|
char* line = change_render_format("%n|%f", config, &event);
|
||||||
EXPECT_NOT_NULL(line);
|
EXPECT_NOT_NULL(line);
|
||||||
EXPECT_EQ_STR(line, "bare.txt|bare.txt");
|
EXPECT_EQ_STR(line, "sub/|src/sub");
|
||||||
free(line);
|
free(line);
|
||||||
|
config_delete(config);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void test_render_itemize_sent_file() {
|
static void test_render_itemize_sent_file() {
|
||||||
ChangeEvent event = sample_event();
|
ChangeEvent event = sample_event();
|
||||||
char* line = change_render_itemize(&event);
|
Config* config = config_create();
|
||||||
|
char* line = change_render_itemize(config, &event);
|
||||||
EXPECT_NOT_NULL(line);
|
EXPECT_NOT_NULL(line);
|
||||||
EXPECT_EQ_STR(line, ">f+++++++++ /srv/root/sub/file.txt");
|
EXPECT_EQ_STR(line, ">f+++++++++ sub/file.txt");
|
||||||
free(line);
|
free(line);
|
||||||
|
config_delete(config);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_render_itemize_directory() {
|
||||||
|
ChangeEvent event = sample_event();
|
||||||
|
event.is_directory = true;
|
||||||
|
event.path = "src/sub";
|
||||||
|
event.name = "sub";
|
||||||
|
Config* config = config_create();
|
||||||
|
char* line = change_render_itemize(config, &event);
|
||||||
|
EXPECT_NOT_NULL(line);
|
||||||
|
EXPECT_EQ_STR(line, "cd+++++++++ sub/");
|
||||||
|
free(line);
|
||||||
|
config_delete(config);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_render_itemize_symlink() {
|
||||||
|
ChangeEvent event = sample_event();
|
||||||
|
event.is_symlink = true;
|
||||||
|
event.path = "src/link";
|
||||||
|
event.name = "link";
|
||||||
|
event.symlink_target = "a.txt";
|
||||||
|
Config* config = config_create();
|
||||||
|
char* line = change_render_itemize(config, &event);
|
||||||
|
EXPECT_NOT_NULL(line);
|
||||||
|
EXPECT_EQ_STR(line, "cL+++++++++ link -> a.txt");
|
||||||
|
free(line);
|
||||||
|
config_delete(config);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_render_itemize_compares_destination() {
|
||||||
|
ChangeEvent event = sample_event();
|
||||||
|
Config* config = config_create();
|
||||||
|
config->preserve_perms = true;
|
||||||
|
config->preserve_owner = true;
|
||||||
|
config->preserve_group = true;
|
||||||
|
event.dest.known = true;
|
||||||
|
event.dest.existed = true;
|
||||||
|
event.dest.size = 1;
|
||||||
|
event.dest.mtime_sec = 1700000000;
|
||||||
|
event.dest.mtime_nsec = 0;
|
||||||
|
event.dest.mode = 0100600;
|
||||||
|
event.dest.uid = 1;
|
||||||
|
event.dest.gid = 2;
|
||||||
|
char* line = change_render_itemize(config, &event);
|
||||||
|
EXPECT_NOT_NULL(line);
|
||||||
|
/* size, perms, owner and group differ; time matches. */
|
||||||
|
EXPECT_EQ_STR(line, ">f.s.pog... sub/file.txt");
|
||||||
|
free(line);
|
||||||
|
config_delete(config);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void test_render_itemize_up_to_date_is_empty() {
|
static void test_render_itemize_up_to_date_is_empty() {
|
||||||
ChangeEvent event = sample_event();
|
ChangeEvent event = sample_event();
|
||||||
|
Config* config = config_create();
|
||||||
event.decision = CHANGE_UP_TO_DATE;
|
event.decision = CHANGE_UP_TO_DATE;
|
||||||
char* line = change_render_itemize(&event);
|
char* line = change_render_itemize(config, &event);
|
||||||
EXPECT_NOT_NULL(line);
|
EXPECT_NOT_NULL(line);
|
||||||
EXPECT_EQ_STR(line, "");
|
EXPECT_EQ_STR(line, "");
|
||||||
free(line);
|
free(line);
|
||||||
|
config_delete(config);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void test_render_list_line() {
|
static void test_render_list_line() {
|
||||||
char* line = change_render_list_line(0100644, 4096, 1700000000, "/srv/x.txt");
|
ChangeEvent event;
|
||||||
|
memset(&event, 0, sizeof(event));
|
||||||
|
Config* config = config_create();
|
||||||
|
event.name = "sub/x.txt";
|
||||||
|
event.path = "sub/x.txt";
|
||||||
|
event.mode = 0100644;
|
||||||
|
event.size = 4096;
|
||||||
|
event.mtime_sec = 1700000000;
|
||||||
|
char* line = change_render_list_line(config, &event);
|
||||||
EXPECT_NOT_NULL(line);
|
EXPECT_NOT_NULL(line);
|
||||||
EXPECT_TRUE(strncmp(line, "-rw-r--r--", 10) == 0);
|
EXPECT_TRUE(strncmp(line, "-rw-r--r--", 10) == 0);
|
||||||
EXPECT_TRUE(strstr(line, "4096") != NULL);
|
EXPECT_TRUE(strstr(line, "4,096") != NULL);
|
||||||
EXPECT_TRUE(strstr(line, "/srv/x.txt") != NULL);
|
EXPECT_TRUE(strstr(line, "sub/x.txt") != NULL);
|
||||||
free(line);
|
free(line);
|
||||||
|
config_delete(config);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void test_change_list_enabled() {
|
static void test_change_list_enabled() {
|
||||||
@@ -93,8 +180,11 @@ static void test_change_list_enabled() {
|
|||||||
void test_change_list() {
|
void test_change_list() {
|
||||||
test_format_tokens();
|
test_format_tokens();
|
||||||
test_format_unknown_tokens_preserved();
|
test_format_unknown_tokens_preserved();
|
||||||
test_format_leaf_name();
|
test_format_directory_name_has_trailing_slash();
|
||||||
test_render_itemize_sent_file();
|
test_render_itemize_sent_file();
|
||||||
|
test_render_itemize_directory();
|
||||||
|
test_render_itemize_symlink();
|
||||||
|
test_render_itemize_compares_destination();
|
||||||
test_render_itemize_up_to_date_is_empty();
|
test_render_itemize_up_to_date_is_empty();
|
||||||
test_render_list_line();
|
test_render_list_line();
|
||||||
test_change_list_enabled();
|
test_change_list_enabled();
|
||||||
|
|||||||
+103
-2
@@ -91,6 +91,63 @@ static void test_checksum_md5_seed_ignored() {
|
|||||||
EXPECT_TRUE(memcmp(a, b, alen) == 0);
|
EXPECT_TRUE(memcmp(a, b, alen) == 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void test_checksum_md4_vectors() {
|
||||||
|
uint8_t out[CHECKSUM_MAX_DIGEST_LEN];
|
||||||
|
size_t len = 0;
|
||||||
|
/* RFC 1320 / RFC 1321 test vectors. */
|
||||||
|
EXPECT_TRUE(checksum_digest(CHECKSUM_ALGO_MD4, 0, "", 0, out, sizeof(out), &len));
|
||||||
|
EXPECT_TRUE(len == (size_t)16);
|
||||||
|
const uint8_t expect_empty[16] = {0x31, 0xd6, 0xcf, 0xe0, 0xd1, 0x6a, 0xe9, 0x31,
|
||||||
|
0xb7, 0x3c, 0x59, 0xd7, 0xe0, 0xc0, 0x89, 0xc0};
|
||||||
|
EXPECT_TRUE(memcmp(out, expect_empty, 16) == 0);
|
||||||
|
|
||||||
|
EXPECT_TRUE(checksum_digest(CHECKSUM_ALGO_MD4, 0, "abc", 3, out, sizeof(out), &len));
|
||||||
|
const uint8_t expect_abc[16] = {0xa4, 0x48, 0x01, 0x7a, 0xaf, 0x21, 0xd8, 0x52,
|
||||||
|
0x5f, 0xc1, 0x0a, 0xe8, 0x7a, 0xa6, 0x72, 0x9d};
|
||||||
|
EXPECT_TRUE(memcmp(out, expect_abc, 16) == 0);
|
||||||
|
|
||||||
|
/* A longer input exercises the block loop and the padding boundary. */
|
||||||
|
const char* msg =
|
||||||
|
"12345678901234567890123456789012345678901234567890123456789012345678901234567890";
|
||||||
|
EXPECT_TRUE(checksum_digest(CHECKSUM_ALGO_MD4, 0, msg, strlen(msg), out, sizeof(out), &len));
|
||||||
|
const uint8_t expect_long[16] = {0xe3, 0x3b, 0x4d, 0xdc, 0x9c, 0x38, 0xf2, 0x19,
|
||||||
|
0x9c, 0x3e, 0x7b, 0x16, 0x4f, 0xcc, 0x05, 0x36};
|
||||||
|
EXPECT_TRUE(memcmp(out, expect_long, 16) == 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_checksum_sha1_vectors() {
|
||||||
|
uint8_t out[CHECKSUM_MAX_DIGEST_LEN];
|
||||||
|
size_t len = 0;
|
||||||
|
EXPECT_TRUE(checksum_digest(CHECKSUM_ALGO_SHA1, 0, "abc", 3, out, sizeof(out), &len));
|
||||||
|
EXPECT_TRUE(len == (size_t)20);
|
||||||
|
const uint8_t expect_abc[20] = {0xa9, 0x99, 0x3e, 0x36, 0x47, 0x06, 0x81, 0x6a, 0xba, 0x3e,
|
||||||
|
0x25, 0x71, 0x78, 0x50, 0xc2, 0x6c, 0x9c, 0xd0, 0xd8, 0x9d};
|
||||||
|
EXPECT_TRUE(memcmp(out, expect_abc, 20) == 0);
|
||||||
|
|
||||||
|
EXPECT_TRUE(checksum_digest(CHECKSUM_ALGO_SHA1, 0, "", 0, out, sizeof(out), &len));
|
||||||
|
EXPECT_TRUE(len == (size_t)20);
|
||||||
|
const uint8_t expect_empty[20] = {0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, 0x32, 0x55,
|
||||||
|
0xbf, 0xef, 0x95, 0x60, 0x18, 0x90, 0xaf, 0xd8, 0x07, 0x09};
|
||||||
|
EXPECT_TRUE(memcmp(out, expect_empty, 20) == 0);
|
||||||
|
|
||||||
|
/* sha1 has no seed: the digest is seed-independent (documented). */
|
||||||
|
uint8_t seeded[CHECKSUM_MAX_DIGEST_LEN];
|
||||||
|
size_t seeded_len = 0;
|
||||||
|
EXPECT_TRUE(
|
||||||
|
checksum_digest(CHECKSUM_ALGO_SHA1, 12345, "abc", 3, seeded, sizeof(seeded), &seeded_len));
|
||||||
|
EXPECT_TRUE(seeded_len == (size_t)20);
|
||||||
|
EXPECT_TRUE(memcmp(expect_abc, seeded, 20) == 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* "none" is a successful no-digest: length 0, nothing written. */
|
||||||
|
static void test_checksum_none_digest() {
|
||||||
|
uint8_t out[CHECKSUM_MAX_DIGEST_LEN];
|
||||||
|
size_t len = 99;
|
||||||
|
EXPECT_TRUE(checksum_digest(CHECKSUM_ALGO_NONE, 0, "data", 4, out, sizeof(out), &len));
|
||||||
|
EXPECT_EQ_INT((int)len, 0);
|
||||||
|
EXPECT_EQ_INT((int)checksum_digest_len(CHECKSUM_ALGO_NONE), 0);
|
||||||
|
}
|
||||||
|
|
||||||
static void test_checksum_algo_name_mapping() {
|
static void test_checksum_algo_name_mapping() {
|
||||||
EXPECT_EQ_INT(checksum_algo_from_name("xxh64"), (int)CHECKSUM_ALGO_XXH64);
|
EXPECT_EQ_INT(checksum_algo_from_name("xxh64"), (int)CHECKSUM_ALGO_XXH64);
|
||||||
EXPECT_EQ_INT(checksum_algo_from_name("XXH64"), (int)CHECKSUM_ALGO_XXH64);
|
EXPECT_EQ_INT(checksum_algo_from_name("XXH64"), (int)CHECKSUM_ALGO_XXH64);
|
||||||
@@ -98,18 +155,58 @@ static void test_checksum_algo_name_mapping() {
|
|||||||
EXPECT_EQ_INT(checksum_algo_from_name("XXHASH"), (int)CHECKSUM_ALGO_XXH64);
|
EXPECT_EQ_INT(checksum_algo_from_name("XXHASH"), (int)CHECKSUM_ALGO_XXH64);
|
||||||
EXPECT_EQ_INT(checksum_algo_from_name("md5"), (int)CHECKSUM_ALGO_MD5);
|
EXPECT_EQ_INT(checksum_algo_from_name("md5"), (int)CHECKSUM_ALGO_MD5);
|
||||||
EXPECT_EQ_INT(checksum_algo_from_name("MD5"), (int)CHECKSUM_ALGO_MD5);
|
EXPECT_EQ_INT(checksum_algo_from_name("MD5"), (int)CHECKSUM_ALGO_MD5);
|
||||||
|
EXPECT_EQ_INT(checksum_algo_from_name("xxh3"), (int)CHECKSUM_ALGO_XXH3);
|
||||||
|
EXPECT_EQ_INT(checksum_algo_from_name("XXH3"), (int)CHECKSUM_ALGO_XXH3);
|
||||||
|
EXPECT_EQ_INT(checksum_algo_from_name("xxh128"), (int)CHECKSUM_ALGO_XXH128);
|
||||||
|
EXPECT_EQ_INT(checksum_algo_from_name("XXH128"), (int)CHECKSUM_ALGO_XXH128);
|
||||||
|
EXPECT_EQ_INT(checksum_algo_from_name("md4"), (int)CHECKSUM_ALGO_MD4);
|
||||||
|
EXPECT_EQ_INT(checksum_algo_from_name("MD4"), (int)CHECKSUM_ALGO_MD4);
|
||||||
|
EXPECT_EQ_INT(checksum_algo_from_name("sha1"), (int)CHECKSUM_ALGO_SHA1);
|
||||||
|
EXPECT_EQ_INT(checksum_algo_from_name("SHA1"), (int)CHECKSUM_ALGO_SHA1);
|
||||||
|
EXPECT_EQ_INT(checksum_algo_from_name("none"), (int)CHECKSUM_ALGO_NONE);
|
||||||
|
/* Names rsync does not offer (or FastSync cannot compute) are rejected. */
|
||||||
EXPECT_TRUE(checksum_algo_from_name("sha256") < 0);
|
EXPECT_TRUE(checksum_algo_from_name("sha256") < 0);
|
||||||
EXPECT_TRUE(checksum_algo_from_name("crc32") < 0);
|
EXPECT_TRUE(checksum_algo_from_name("crc32") < 0);
|
||||||
EXPECT_TRUE(checksum_algo_from_name("none") < 0);
|
|
||||||
EXPECT_TRUE(checksum_algo_from_name("xxh3") < 0);
|
|
||||||
EXPECT_TRUE(checksum_algo_from_name("") < 0);
|
EXPECT_TRUE(checksum_algo_from_name("") < 0);
|
||||||
EXPECT_TRUE(checksum_algo_from_name(NULL) < 0);
|
EXPECT_TRUE(checksum_algo_from_name(NULL) < 0);
|
||||||
|
|
||||||
EXPECT_TRUE(checksum_algo_valid((int)CHECKSUM_ALGO_XXH64));
|
EXPECT_TRUE(checksum_algo_valid((int)CHECKSUM_ALGO_XXH64));
|
||||||
EXPECT_TRUE(checksum_algo_valid((int)CHECKSUM_ALGO_MD5));
|
EXPECT_TRUE(checksum_algo_valid((int)CHECKSUM_ALGO_MD5));
|
||||||
|
EXPECT_TRUE(checksum_algo_valid((int)CHECKSUM_ALGO_XXH3));
|
||||||
|
EXPECT_TRUE(checksum_algo_valid((int)CHECKSUM_ALGO_XXH128));
|
||||||
|
EXPECT_TRUE(checksum_algo_valid((int)CHECKSUM_ALGO_MD4));
|
||||||
|
EXPECT_TRUE(checksum_algo_valid((int)CHECKSUM_ALGO_SHA1));
|
||||||
|
EXPECT_TRUE(checksum_algo_valid((int)CHECKSUM_ALGO_NONE));
|
||||||
EXPECT_FALSE(checksum_algo_valid(99));
|
EXPECT_FALSE(checksum_algo_valid(99));
|
||||||
EXPECT_EQ_STR(checksum_algo_name(CHECKSUM_ALGO_XXH64), "xxh64");
|
EXPECT_EQ_STR(checksum_algo_name(CHECKSUM_ALGO_XXH64), "xxh64");
|
||||||
EXPECT_EQ_STR(checksum_algo_name(CHECKSUM_ALGO_MD5), "md5");
|
EXPECT_EQ_STR(checksum_algo_name(CHECKSUM_ALGO_MD5), "md5");
|
||||||
|
EXPECT_EQ_STR(checksum_algo_name(CHECKSUM_ALGO_XXH3), "xxh3");
|
||||||
|
EXPECT_EQ_STR(checksum_algo_name(CHECKSUM_ALGO_XXH128), "xxh128");
|
||||||
|
EXPECT_EQ_STR(checksum_algo_name(CHECKSUM_ALGO_MD4), "md4");
|
||||||
|
EXPECT_EQ_STR(checksum_algo_name(CHECKSUM_ALGO_SHA1), "sha1");
|
||||||
|
EXPECT_EQ_STR(checksum_algo_name(CHECKSUM_ALGO_NONE), "none");
|
||||||
|
|
||||||
|
/* rsync 3.4.1 auto-negotiates xxh128 first. */
|
||||||
|
EXPECT_EQ_INT((int)checksum_negotiate_default(), (int)CHECKSUM_ALGO_XXH128);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* xxh3 is 8 bytes and seed-aware; xxh128 is 16 bytes and differs from both
|
||||||
|
* xxh64 and md5 for the same input. */
|
||||||
|
static void test_checksum_xxh3_xxh128() {
|
||||||
|
EXPECT_EQ_INT((int)checksum_digest_len(CHECKSUM_ALGO_XXH3), 8);
|
||||||
|
EXPECT_EQ_INT((int)checksum_digest_len(CHECKSUM_ALGO_XXH128), 16);
|
||||||
|
|
||||||
|
uint8_t a[CHECKSUM_MAX_DIGEST_LEN], b[CHECKSUM_MAX_DIGEST_LEN];
|
||||||
|
size_t alen = 0, blen = 0;
|
||||||
|
EXPECT_TRUE(checksum_digest(CHECKSUM_ALGO_XXH3, 0, "payload", 7, a, sizeof(a), &alen));
|
||||||
|
EXPECT_TRUE(alen == (size_t)8);
|
||||||
|
EXPECT_TRUE(checksum_digest(CHECKSUM_ALGO_XXH3, 5, "payload", 7, b, sizeof(b), &blen));
|
||||||
|
EXPECT_TRUE(memcmp(a, b, alen) != 0);
|
||||||
|
|
||||||
|
EXPECT_TRUE(checksum_digest(CHECKSUM_ALGO_XXH128, 0, "payload", 7, a, sizeof(a), &alen));
|
||||||
|
EXPECT_TRUE(alen == (size_t)16);
|
||||||
|
EXPECT_TRUE(checksum_digest(CHECKSUM_ALGO_XXH128, 0, "payload", 7, b, sizeof(b), &blen));
|
||||||
|
EXPECT_TRUE(memcmp(a, b, blen) == 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void test_checksum_truncated_buffer_rejected() {
|
static void test_checksum_truncated_buffer_rejected() {
|
||||||
@@ -139,9 +236,13 @@ void test_checksum(void) {
|
|||||||
test_checksum_xxh64_seed_changes_digest();
|
test_checksum_xxh64_seed_changes_digest();
|
||||||
test_checksum_xxh64_seed_deterministic();
|
test_checksum_xxh64_seed_deterministic();
|
||||||
test_checksum_md5_vectors();
|
test_checksum_md5_vectors();
|
||||||
|
test_checksum_md4_vectors();
|
||||||
|
test_checksum_sha1_vectors();
|
||||||
|
test_checksum_none_digest();
|
||||||
test_checksum_algo_lengths_distinct();
|
test_checksum_algo_lengths_distinct();
|
||||||
test_checksum_md5_seed_ignored();
|
test_checksum_md5_seed_ignored();
|
||||||
test_checksum_algo_name_mapping();
|
test_checksum_algo_name_mapping();
|
||||||
|
test_checksum_xxh3_xxh128();
|
||||||
test_checksum_truncated_buffer_rejected();
|
test_checksum_truncated_buffer_rejected();
|
||||||
test_checksum_null_empty_digest();
|
test_checksum_null_empty_digest();
|
||||||
}
|
}
|
||||||
+1132
-32
File diff suppressed because it is too large
Load Diff
+110
-6
@@ -64,6 +64,21 @@ static void test_skip_compress_suffix_matching() {
|
|||||||
EXPECT_TRUE(compression_should_skip_with_suffixes("backup.TAR.GZ", suffixes, 2));
|
EXPECT_TRUE(compression_should_skip_with_suffixes("backup.TAR.GZ", suffixes, 2));
|
||||||
EXPECT_FALSE(compression_should_skip_with_suffixes("notes.txt", suffixes, 2));
|
EXPECT_FALSE(compression_should_skip_with_suffixes("notes.txt", suffixes, 2));
|
||||||
EXPECT_FALSE(compression_should_skip_with_suffixes("archive.zip", suffixes, 0));
|
EXPECT_FALSE(compression_should_skip_with_suffixes("archive.zip", suffixes, 0));
|
||||||
|
|
||||||
|
/* A user suffix may omit the leading dot (rsync's spelling). */
|
||||||
|
char* bare[] = {"zip", "gz"};
|
||||||
|
EXPECT_TRUE(compression_should_skip_with_suffixes("archive.zip", bare, 2));
|
||||||
|
EXPECT_TRUE(compression_should_skip_with_suffixes("x.GZ", bare, 2));
|
||||||
|
|
||||||
|
/* No user list (count < 0) selects rsync 3.4.1's built-in default list. */
|
||||||
|
EXPECT_TRUE(compression_should_skip_with_suffixes("movie.mp4", NULL, -1));
|
||||||
|
EXPECT_TRUE(compression_should_skip_with_suffixes("archive.TAR.GZ", NULL, -1));
|
||||||
|
EXPECT_TRUE(compression_should_skip_with_suffixes("photo.jpeg", NULL, -1));
|
||||||
|
EXPECT_TRUE(compression_should_skip_with_suffixes("disk.squashfs", NULL, -1));
|
||||||
|
EXPECT_TRUE(compression_should_skip_with_suffixes("data.7z", NULL, -1));
|
||||||
|
EXPECT_FALSE(compression_should_skip_with_suffixes("notes.txt", NULL, -1));
|
||||||
|
EXPECT_FALSE(compression_should_skip_with_suffixes("program", NULL, -1));
|
||||||
|
EXPECT_FALSE(compression_should_skip_with_suffixes("trailing.", NULL, -1));
|
||||||
}
|
}
|
||||||
|
|
||||||
static void test_data_compress_with_threads_roundtrip() {
|
static void test_data_compress_with_threads_roundtrip() {
|
||||||
@@ -142,20 +157,22 @@ static void test_chunk_compress_decompress_roundtrip() {
|
|||||||
|
|
||||||
/* Build a zstd frame whose header omits the content size (the content size
|
/* Build a zstd frame whose header omits the content size (the content size
|
||||||
* flag is cleared), which ZSTD_getFrameContentSize reports as
|
* flag is cleared), which ZSTD_getFrameContentSize reports as
|
||||||
* ZSTD_CONTENTSIZE_UNKNOWN. */
|
* ZSTD_CONTENTSIZE_UNKNOWN. The frame carries the codec-id prefix the
|
||||||
|
* decompressor dispatches on. */
|
||||||
static Data* make_unknown_size_frame(const void* src, size_t len) {
|
static Data* make_unknown_size_frame(const void* src, size_t len) {
|
||||||
ZSTD_CCtx* cctx = ZSTD_createCCtx();
|
ZSTD_CCtx* cctx = ZSTD_createCCtx();
|
||||||
if (!cctx)
|
if (!cctx)
|
||||||
return NULL;
|
return NULL;
|
||||||
ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0);
|
ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0);
|
||||||
size_t cap = ZSTD_compressBound(len);
|
size_t cap = ZSTD_compressBound(len);
|
||||||
Data* out = data_create_empty(cap);
|
Data* out = data_create_empty(cap + 1);
|
||||||
if (!out) {
|
if (!out) {
|
||||||
ZSTD_freeCCtx(cctx);
|
ZSTD_freeCCtx(cctx);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
((uint8_t*)out->data)[0] = (uint8_t)COMPRESSION_ALGO_ZSTD;
|
||||||
ZSTD_inBuffer in = {src, len, 0};
|
ZSTD_inBuffer in = {src, len, 0};
|
||||||
ZSTD_outBuffer ob = {out->data, cap, 0};
|
ZSTD_outBuffer ob = {(uint8_t*)out->data + 1, cap, 0};
|
||||||
size_t ret;
|
size_t ret;
|
||||||
do {
|
do {
|
||||||
ret = ZSTD_compressStream2(cctx, &ob, &in, ZSTD_e_end);
|
ret = ZSTD_compressStream2(cctx, &ob, &in, ZSTD_e_end);
|
||||||
@@ -165,7 +182,7 @@ static Data* make_unknown_size_frame(const void* src, size_t len) {
|
|||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
} while (ret > 0);
|
} while (ret > 0);
|
||||||
out->size = ob.pos;
|
out->size = ob.pos + 1;
|
||||||
ZSTD_freeCCtx(cctx);
|
ZSTD_freeCCtx(cctx);
|
||||||
return out;
|
return out;
|
||||||
}
|
}
|
||||||
@@ -184,8 +201,9 @@ static void test_data_decompress_unknown_size_frame() {
|
|||||||
Data* frame = make_unknown_size_frame(buf, len);
|
Data* frame = make_unknown_size_frame(buf, len);
|
||||||
free(buf);
|
free(buf);
|
||||||
EXPECT_NOT_NULL(frame);
|
EXPECT_NOT_NULL(frame);
|
||||||
/* Guard the premise of the test: the frame really has no stored size. */
|
/* Guard the premise of the test: the frame (after the codec byte) really has
|
||||||
EXPECT_EQ_INT((int)ZSTD_getFrameContentSize(frame->data, frame->size),
|
* no stored size. */
|
||||||
|
EXPECT_EQ_INT((int)ZSTD_getFrameContentSize((uint8_t*)frame->data + 1, frame->size - 1),
|
||||||
(int)ZSTD_CONTENTSIZE_UNKNOWN);
|
(int)ZSTD_CONTENTSIZE_UNKNOWN);
|
||||||
|
|
||||||
Data* decompressed = data_decompress(frame);
|
Data* decompressed = data_decompress(frame);
|
||||||
@@ -311,6 +329,89 @@ static void test_data_decompress_truncated_frame_fails() {
|
|||||||
data_destroy(input);
|
data_destroy(input);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Every codec must round-trip byte-exactly through the self-describing frame,
|
||||||
|
* including the empty and a highly compressible large payload. */
|
||||||
|
static void codec_roundtrip(CompressionAlgo algo) {
|
||||||
|
const char* samples[] = {
|
||||||
|
"",
|
||||||
|
"Hello, World! This is test data for compression round-trip!",
|
||||||
|
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||||
|
};
|
||||||
|
for (size_t s = 0; s < sizeof(samples) / sizeof(samples[0]); s++) {
|
||||||
|
size_t len = strlen(samples[s]);
|
||||||
|
Data* original = data_create_empty(len);
|
||||||
|
EXPECT_NOT_NULL(original);
|
||||||
|
if (len > 0)
|
||||||
|
memcpy(original->data, samples[s], len);
|
||||||
|
original->size = len;
|
||||||
|
|
||||||
|
Data* compressed = data_compress_codec(original, algo, 3, 0);
|
||||||
|
EXPECT_NOT_NULL(compressed);
|
||||||
|
EXPECT_EQ_INT((int)((uint8_t*)compressed->data)[0], (int)algo);
|
||||||
|
Data* decompressed = data_decompress(compressed);
|
||||||
|
EXPECT_NOT_NULL(decompressed);
|
||||||
|
EXPECT_EQ_INT((int)decompressed->size, (int)len);
|
||||||
|
EXPECT_EQ_INT(memcmp(decompressed->data, original->data, len), 0);
|
||||||
|
data_destroy(decompressed);
|
||||||
|
data_destroy(compressed);
|
||||||
|
data_destroy(original);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_codec_roundtrips() {
|
||||||
|
codec_roundtrip(COMPRESSION_ALGO_NONE);
|
||||||
|
codec_roundtrip(COMPRESSION_ALGO_ZSTD);
|
||||||
|
codec_roundtrip(COMPRESSION_ALGO_LZ4);
|
||||||
|
codec_roundtrip(COMPRESSION_ALGO_ZLIB);
|
||||||
|
codec_roundtrip(COMPRESSION_ALGO_ZLIBX);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_codec_name_mapping() {
|
||||||
|
EXPECT_EQ_INT(compression_algo_from_name("zstd"), (int)COMPRESSION_ALGO_ZSTD);
|
||||||
|
EXPECT_EQ_INT(compression_algo_from_name("ZSTD"), (int)COMPRESSION_ALGO_ZSTD);
|
||||||
|
EXPECT_EQ_INT(compression_algo_from_name("lz4"), (int)COMPRESSION_ALGO_LZ4);
|
||||||
|
EXPECT_EQ_INT(compression_algo_from_name("zlib"), (int)COMPRESSION_ALGO_ZLIB);
|
||||||
|
EXPECT_EQ_INT(compression_algo_from_name("zlibx"), (int)COMPRESSION_ALGO_ZLIBX);
|
||||||
|
EXPECT_EQ_INT(compression_algo_from_name("none"), (int)COMPRESSION_ALGO_NONE);
|
||||||
|
EXPECT_TRUE(compression_algo_from_name("bogus") < 0);
|
||||||
|
EXPECT_TRUE(compression_algo_from_name(NULL) < 0);
|
||||||
|
EXPECT_TRUE(compression_algo_valid((int)COMPRESSION_ALGO_LZ4));
|
||||||
|
EXPECT_TRUE(compression_algo_valid((int)COMPRESSION_ALGO_ZLIB));
|
||||||
|
EXPECT_TRUE(compression_algo_valid((int)COMPRESSION_ALGO_ZLIBX));
|
||||||
|
EXPECT_FALSE(compression_algo_valid(99));
|
||||||
|
EXPECT_EQ_STR(compression_algo_name(COMPRESSION_ALGO_ZSTD), "zstd");
|
||||||
|
EXPECT_EQ_STR(compression_algo_name(COMPRESSION_ALGO_LZ4), "lz4");
|
||||||
|
EXPECT_EQ_STR(compression_algo_name(COMPRESSION_ALGO_ZLIB), "zlib");
|
||||||
|
EXPECT_EQ_STR(compression_algo_name(COMPRESSION_ALGO_ZLIBX), "zlibx");
|
||||||
|
EXPECT_EQ_STR(compression_algo_name(COMPRESSION_ALGO_NONE), "none");
|
||||||
|
/* rsync 3.4.1 auto-negotiates zstd first. */
|
||||||
|
EXPECT_EQ_INT((int)compression_negotiate_default(), (int)COMPRESSION_ALGO_ZSTD);
|
||||||
|
EXPECT_FALSE(compression_algo_enabled(COMPRESSION_ALGO_NONE));
|
||||||
|
EXPECT_TRUE(compression_algo_enabled(COMPRESSION_ALGO_ZSTD));
|
||||||
|
}
|
||||||
|
|
||||||
|
/* The process-global codec selects what the legacy wrappers produce. */
|
||||||
|
static void test_codec_global_selection() {
|
||||||
|
Data* original = data_create_empty(64);
|
||||||
|
EXPECT_NOT_NULL(original);
|
||||||
|
memset(original->data, 'q', 64);
|
||||||
|
original->size = 64;
|
||||||
|
|
||||||
|
compression_set_algo(COMPRESSION_ALGO_LZ4);
|
||||||
|
Data* compressed = data_compress(original, 3);
|
||||||
|
EXPECT_NOT_NULL(compressed);
|
||||||
|
EXPECT_EQ_INT((int)((uint8_t*)compressed->data)[0], (int)COMPRESSION_ALGO_LZ4);
|
||||||
|
Data* decompressed = data_decompress(compressed);
|
||||||
|
EXPECT_NOT_NULL(decompressed);
|
||||||
|
EXPECT_TRUE(memcmp(decompressed->data, original->data, 64) == 0);
|
||||||
|
data_destroy(decompressed);
|
||||||
|
data_destroy(compressed);
|
||||||
|
|
||||||
|
/* Restore the default so later tests are unaffected. */
|
||||||
|
compression_set_algo(COMPRESSION_ALGO_ZSTD);
|
||||||
|
data_destroy(original);
|
||||||
|
}
|
||||||
|
|
||||||
void test_compression() {
|
void test_compression() {
|
||||||
test_data_compress_decompress_roundtrip();
|
test_data_compress_decompress_roundtrip();
|
||||||
test_data_compress_decompress_large();
|
test_data_compress_decompress_large();
|
||||||
@@ -320,4 +421,7 @@ void test_compression() {
|
|||||||
test_data_compress_with_threads_roundtrip();
|
test_data_compress_with_threads_roundtrip();
|
||||||
test_data_compress_reused_contexts_multithreaded();
|
test_data_compress_reused_contexts_multithreaded();
|
||||||
test_chunk_compress_decompress_roundtrip();
|
test_chunk_compress_decompress_roundtrip();
|
||||||
|
test_codec_roundtrips();
|
||||||
|
test_codec_name_mapping();
|
||||||
|
test_codec_global_selection();
|
||||||
}
|
}
|
||||||
|
|||||||
+398
-20
@@ -9,6 +9,7 @@
|
|||||||
#include "test_utils.h"
|
#include "test_utils.h"
|
||||||
#include "utils.h"
|
#include "utils.h"
|
||||||
#include <signal.h>
|
#include <signal.h>
|
||||||
|
#include <stddef.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <sys/socket.h>
|
#include <sys/socket.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
@@ -551,6 +552,83 @@ static void test_config_send_receive() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* #5: a received --max-alloc=0 (rsync's "no limit") is floored to the server
|
||||||
|
* ceiling on the receive path, so a client cannot disable it. */
|
||||||
|
static void test_config_receive_max_alloc_zero_floored() {
|
||||||
|
Config* send_cfg = config_create();
|
||||||
|
EXPECT_NOT_NULL(send_cfg);
|
||||||
|
send_cfg->send_directory = str_dup("/send/src");
|
||||||
|
send_cfg->receive_root_directory = str_dup("/send/dst");
|
||||||
|
send_cfg->max_alloc = 0;
|
||||||
|
|
||||||
|
int p[2];
|
||||||
|
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
|
||||||
|
io_set_fds(p[0], p[1]);
|
||||||
|
io_set_bwlimit(0);
|
||||||
|
|
||||||
|
pid_t pid = fork();
|
||||||
|
if (pid == 0) {
|
||||||
|
close(p[1]);
|
||||||
|
io_set_fds(p[0], p[0]);
|
||||||
|
Config* recv_cfg = config_receive(p[0]);
|
||||||
|
bool ok = recv_cfg != NULL && recv_cfg->max_alloc == MAX_SERVER_ALLOC;
|
||||||
|
config_delete(recv_cfg);
|
||||||
|
close(p[0]);
|
||||||
|
close(p[1]);
|
||||||
|
_exit(ok ? 0 : 1);
|
||||||
|
} else {
|
||||||
|
close(p[0]);
|
||||||
|
io_set_fds(p[1], p[1]);
|
||||||
|
bool sent = config_send(p[1], send_cfg);
|
||||||
|
int status;
|
||||||
|
waitpid(pid, &status, 0);
|
||||||
|
close(p[0]);
|
||||||
|
close(p[1]);
|
||||||
|
config_delete(send_cfg);
|
||||||
|
EXPECT_TRUE(sent);
|
||||||
|
EXPECT_TRUE(WIFEXITED(status) && WEXITSTATUS(status) == 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* #4: a hostile/older client that still sends compress_choice=auto must be
|
||||||
|
* accepted (as zstd) rather than failing the whole transfer. */
|
||||||
|
static void test_config_receive_compress_choice_auto_canonicalized() {
|
||||||
|
Config* send_cfg = config_create();
|
||||||
|
EXPECT_NOT_NULL(send_cfg);
|
||||||
|
send_cfg->send_directory = str_dup("/send/src");
|
||||||
|
send_cfg->receive_root_directory = str_dup("/send/dst");
|
||||||
|
free(send_cfg->compress_choice);
|
||||||
|
send_cfg->compress_choice = str_dup("auto");
|
||||||
|
|
||||||
|
int p[2];
|
||||||
|
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
|
||||||
|
io_set_fds(p[0], p[1]);
|
||||||
|
io_set_bwlimit(0);
|
||||||
|
|
||||||
|
pid_t pid = fork();
|
||||||
|
if (pid == 0) {
|
||||||
|
close(p[1]);
|
||||||
|
io_set_fds(p[0], p[0]);
|
||||||
|
Config* recv_cfg = config_receive(p[0]);
|
||||||
|
bool ok = recv_cfg != NULL && strcmp(recv_cfg->compress_choice, "zstd") == 0;
|
||||||
|
config_delete(recv_cfg);
|
||||||
|
close(p[0]);
|
||||||
|
close(p[1]);
|
||||||
|
_exit(ok ? 0 : 1);
|
||||||
|
} else {
|
||||||
|
close(p[0]);
|
||||||
|
io_set_fds(p[1], p[1]);
|
||||||
|
bool sent = config_send(p[1], send_cfg);
|
||||||
|
int status;
|
||||||
|
waitpid(pid, &status, 0);
|
||||||
|
close(p[0]);
|
||||||
|
close(p[1]);
|
||||||
|
config_delete(send_cfg);
|
||||||
|
EXPECT_TRUE(sent);
|
||||||
|
EXPECT_TRUE(WIFEXITED(status) && WEXITSTATUS(status) == 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
static void test_config_send_receive_version_mismatch() {
|
static void test_config_send_receive_version_mismatch() {
|
||||||
/* A peer using the previous wire format must be rejected. */
|
/* A peer using the previous wire format must be rejected. */
|
||||||
Config* cfg = config_create();
|
Config* cfg = config_create();
|
||||||
@@ -774,13 +852,15 @@ static void test_config_delete_timing_early_helper() {
|
|||||||
cfg->use_delete = true;
|
cfg->use_delete = true;
|
||||||
cfg->delete_before = true;
|
cfg->delete_before = true;
|
||||||
EXPECT_TRUE(config_delete_timing_early(cfg));
|
EXPECT_TRUE(config_delete_timing_early(cfg));
|
||||||
|
EXPECT_FALSE(config_delete_timing_per_dir(cfg));
|
||||||
EXPECT_TRUE(config_has_valid_delete_timing(cfg));
|
EXPECT_TRUE(config_has_valid_delete_timing(cfg));
|
||||||
config_delete(cfg);
|
config_delete(cfg);
|
||||||
|
|
||||||
cfg = config_create();
|
cfg = config_create();
|
||||||
cfg->use_delete = true;
|
cfg->use_delete = true;
|
||||||
cfg->delete_during = true;
|
cfg->delete_during = true;
|
||||||
EXPECT_TRUE(config_delete_timing_early(cfg));
|
EXPECT_FALSE(config_delete_timing_early(cfg));
|
||||||
|
EXPECT_TRUE(config_delete_timing_per_dir(cfg));
|
||||||
EXPECT_TRUE(config_has_valid_delete_timing(cfg));
|
EXPECT_TRUE(config_has_valid_delete_timing(cfg));
|
||||||
config_delete(cfg);
|
config_delete(cfg);
|
||||||
|
|
||||||
@@ -788,6 +868,7 @@ static void test_config_delete_timing_early_helper() {
|
|||||||
cfg->use_delete = true;
|
cfg->use_delete = true;
|
||||||
cfg->delete_delay = true;
|
cfg->delete_delay = true;
|
||||||
EXPECT_FALSE(config_delete_timing_early(cfg));
|
EXPECT_FALSE(config_delete_timing_early(cfg));
|
||||||
|
EXPECT_TRUE(config_delete_timing_per_dir(cfg));
|
||||||
EXPECT_TRUE(config_has_valid_delete_timing(cfg));
|
EXPECT_TRUE(config_has_valid_delete_timing(cfg));
|
||||||
config_delete(cfg);
|
config_delete(cfg);
|
||||||
|
|
||||||
@@ -795,6 +876,7 @@ static void test_config_delete_timing_early_helper() {
|
|||||||
cfg->use_delete = true;
|
cfg->use_delete = true;
|
||||||
cfg->delete_after = true;
|
cfg->delete_after = true;
|
||||||
EXPECT_FALSE(config_delete_timing_early(cfg));
|
EXPECT_FALSE(config_delete_timing_early(cfg));
|
||||||
|
EXPECT_FALSE(config_delete_timing_per_dir(cfg));
|
||||||
EXPECT_TRUE(config_has_valid_delete_timing(cfg));
|
EXPECT_TRUE(config_has_valid_delete_timing(cfg));
|
||||||
config_delete(cfg);
|
config_delete(cfg);
|
||||||
|
|
||||||
@@ -1114,7 +1196,10 @@ static void test_config_basis_wire_rejects_escaping() {
|
|||||||
c->basis_dirs = calloc(1, sizeof(BasisDest));
|
c->basis_dirs = calloc(1, sizeof(BasisDest));
|
||||||
c->basis_dirs[0].type = BASIS_DEST_LINK;
|
c->basis_dirs[0].type = BASIS_DEST_LINK;
|
||||||
c->basis_dirs[0].path = str_dup("/abs");
|
c->basis_dirs[0].path = str_dup("/abs");
|
||||||
EXPECT_FALSE(roundtrip_config_ok(c));
|
/* An absolute basis dir is accepted (rsync parity); it is only usable when it
|
||||||
|
lies within the receiver's authorized root, which file_open_secure_parent
|
||||||
|
enforces at lookup time. */
|
||||||
|
EXPECT_TRUE(roundtrip_config_ok(c));
|
||||||
config_delete(c);
|
config_delete(c);
|
||||||
|
|
||||||
/* A well-formed list still round-trips even with a manually built struct. */
|
/* A well-formed list still round-trips even with a manually built struct. */
|
||||||
@@ -1147,8 +1232,13 @@ static void test_config_basis_normalization() {
|
|||||||
/* Degenerate values that normalize away to nothing stay rejected. */
|
/* Degenerate values that normalize away to nothing stay rejected. */
|
||||||
EXPECT_EQ_INT(config_basis_append(c, BASIS_DEST_LINK, "."), -1);
|
EXPECT_EQ_INT(config_basis_append(c, BASIS_DEST_LINK, "."), -1);
|
||||||
EXPECT_EQ_INT(config_basis_append(c, BASIS_DEST_LINK, ".."), -1);
|
EXPECT_EQ_INT(config_basis_append(c, BASIS_DEST_LINK, ".."), -1);
|
||||||
EXPECT_EQ_INT(config_basis_append(c, BASIS_DEST_LINK, "/abs"), -1);
|
/* An absolute path is canonicalized (leading '/' preserved) and accepted. */
|
||||||
|
EXPECT_EQ_INT(config_basis_append(c, BASIS_DEST_LINK, "/abs"), 0);
|
||||||
|
EXPECT_EQ_STR(c->basis_dirs[c->basis_count - 1].path, "/abs");
|
||||||
|
EXPECT_EQ_INT(config_basis_append(c, BASIS_DEST_LINK, "/a//b/"), 0);
|
||||||
|
EXPECT_EQ_STR(c->basis_dirs[c->basis_count - 1].path, "/a/b");
|
||||||
EXPECT_EQ_INT(config_basis_append(c, BASIS_DEST_LINK, "a/../b"), -1);
|
EXPECT_EQ_INT(config_basis_append(c, BASIS_DEST_LINK, "a/../b"), -1);
|
||||||
|
EXPECT_EQ_INT(config_basis_append(c, BASIS_DEST_LINK, "/"), -1);
|
||||||
EXPECT_EQ_INT(config_basis_append(c, BASIS_DEST_LINK, ""), -1);
|
EXPECT_EQ_INT(config_basis_append(c, BASIS_DEST_LINK, ""), -1);
|
||||||
config_delete(c);
|
config_delete(c);
|
||||||
}
|
}
|
||||||
@@ -1278,6 +1368,61 @@ static void test_config_receive_rejects_invalid_checksum_algo() {
|
|||||||
EXPECT_FALSE(roundtrip_config_ok(c));
|
EXPECT_FALSE(roundtrip_config_ok(c));
|
||||||
config_delete(c);
|
config_delete(c);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* The negotiated codec id and the human --compress-choice spelling must agree,
|
||||||
|
* and the id itself must be a known codec. */
|
||||||
|
static void test_config_receive_rejects_invalid_compression_algo() {
|
||||||
|
if (is_running_under_valgrind())
|
||||||
|
return;
|
||||||
|
Config* c = config_create();
|
||||||
|
EXPECT_NOT_NULL(c);
|
||||||
|
c->send_directory = str_dup("/src");
|
||||||
|
c->receive_root_directory = str_dup("/dst");
|
||||||
|
c->compression_algo = 99;
|
||||||
|
EXPECT_FALSE(roundtrip_config_ok(c));
|
||||||
|
config_delete(c);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_config_receive_rejects_codec_mismatch() {
|
||||||
|
if (is_running_under_valgrind())
|
||||||
|
return;
|
||||||
|
Config* c = config_create();
|
||||||
|
EXPECT_NOT_NULL(c);
|
||||||
|
c->send_directory = str_dup("/src");
|
||||||
|
c->receive_root_directory = str_dup("/dst");
|
||||||
|
free(c->compress_choice);
|
||||||
|
c->compress_choice = str_dup("lz4");
|
||||||
|
c->use_compression = true;
|
||||||
|
c->compression_algo = (int)COMPRESSION_ALGO_ZSTD; /* does not match lz4 */
|
||||||
|
EXPECT_FALSE(roundtrip_config_ok(c));
|
||||||
|
config_delete(c);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_config_receive_rejects_none_codec_with_compression() {
|
||||||
|
if (is_running_under_valgrind())
|
||||||
|
return;
|
||||||
|
Config* c = config_create();
|
||||||
|
EXPECT_NOT_NULL(c);
|
||||||
|
c->send_directory = str_dup("/src");
|
||||||
|
c->receive_root_directory = str_dup("/dst");
|
||||||
|
c->use_compression = true;
|
||||||
|
c->compression_algo = (int)COMPRESSION_ALGO_NONE;
|
||||||
|
EXPECT_FALSE(roundtrip_config_ok(c));
|
||||||
|
config_delete(c);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_config_receive_rejects_checksum_none_with_checksum() {
|
||||||
|
if (is_running_under_valgrind())
|
||||||
|
return;
|
||||||
|
Config* c = config_create();
|
||||||
|
EXPECT_NOT_NULL(c);
|
||||||
|
c->send_directory = str_dup("/src");
|
||||||
|
c->receive_root_directory = str_dup("/dst");
|
||||||
|
c->checksum = true;
|
||||||
|
c->checksum_algo = (int)CHECKSUM_ALGO_NONE;
|
||||||
|
EXPECT_FALSE(roundtrip_config_ok(c));
|
||||||
|
config_delete(c);
|
||||||
|
}
|
||||||
/* The identity-mapping fields (--numeric-ids / --usermap / --groupmap /
|
/* The identity-mapping fields (--numeric-ids / --usermap / --groupmap /
|
||||||
--chown) cross the config wire unchanged: the receiver needs them to apply
|
--chown) cross the config wire unchanged: the receiver needs them to apply
|
||||||
ownership with the same policy the client requested. */
|
ownership with the same policy the client requested. */
|
||||||
@@ -1292,6 +1437,9 @@ static void test_config_metadata_times_wire_roundtrip() {
|
|||||||
send_cfg->preserve_crtimes = true;
|
send_cfg->preserve_crtimes = true;
|
||||||
send_cfg->omit_dir_times = true;
|
send_cfg->omit_dir_times = true;
|
||||||
send_cfg->omit_link_times = true;
|
send_cfg->omit_link_times = true;
|
||||||
|
/* The preservation attributes now require the metadata frame to travel
|
||||||
|
* (config_invariants_error rejects them otherwise). */
|
||||||
|
send_cfg->use_metadata = true;
|
||||||
/* --open-noatime is client-only and must NOT cross the wire. */
|
/* --open-noatime is client-only and must NOT cross the wire. */
|
||||||
send_cfg->open_noatime = true;
|
send_cfg->open_noatime = true;
|
||||||
|
|
||||||
@@ -1341,13 +1489,19 @@ static void test_config_identity_wire_roundtrip() {
|
|||||||
send_cfg->usermap_count = 2;
|
send_cfg->usermap_count = 2;
|
||||||
send_cfg->usermap = calloc(2, sizeof(IdentityMap));
|
send_cfg->usermap = calloc(2, sizeof(IdentityMap));
|
||||||
send_cfg->usermap[0].from = IDENTITY_MATCH_ANY;
|
send_cfg->usermap[0].from = IDENTITY_MATCH_ANY;
|
||||||
|
send_cfg->usermap[0].from_hi = IDENTITY_MATCH_ANY;
|
||||||
send_cfg->usermap[0].to = 65534;
|
send_cfg->usermap[0].to = 65534;
|
||||||
|
send_cfg->usermap[0].to_name = NULL;
|
||||||
send_cfg->usermap[1].from = 1000;
|
send_cfg->usermap[1].from = 1000;
|
||||||
|
send_cfg->usermap[1].from_hi = 1000;
|
||||||
send_cfg->usermap[1].to = 1000;
|
send_cfg->usermap[1].to = 1000;
|
||||||
|
send_cfg->usermap[1].to_name = NULL;
|
||||||
send_cfg->groupmap_count = 1;
|
send_cfg->groupmap_count = 1;
|
||||||
send_cfg->groupmap = calloc(1, sizeof(IdentityMap));
|
send_cfg->groupmap = calloc(1, sizeof(IdentityMap));
|
||||||
send_cfg->groupmap[0].from = 0;
|
send_cfg->groupmap[0].from = 0;
|
||||||
|
send_cfg->groupmap[0].from_hi = 0;
|
||||||
send_cfg->groupmap[0].to = IDENTITY_CURRENT;
|
send_cfg->groupmap[0].to = IDENTITY_CURRENT;
|
||||||
|
send_cfg->groupmap[0].to_name = str_dup("root");
|
||||||
|
|
||||||
int p[2];
|
int p[2];
|
||||||
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
|
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
|
||||||
@@ -1363,9 +1517,12 @@ static void test_config_identity_wire_roundtrip() {
|
|||||||
ok = recv->numeric_ids && recv->chown_uid_set && recv->chown_uid == 1001 &&
|
ok = recv->numeric_ids && recv->chown_uid_set && recv->chown_uid == 1001 &&
|
||||||
recv->chown_gid_set && recv->chown_gid == IDENTITY_CURRENT && recv->usermap_count == 2 &&
|
recv->chown_gid_set && recv->chown_gid == IDENTITY_CURRENT && recv->usermap_count == 2 &&
|
||||||
recv->groupmap_count == 1 && recv->usermap[0].from == IDENTITY_MATCH_ANY &&
|
recv->groupmap_count == 1 && recv->usermap[0].from == IDENTITY_MATCH_ANY &&
|
||||||
recv->usermap[0].to == 65534 && recv->usermap[1].from == 1000 &&
|
recv->usermap[0].from_hi == IDENTITY_MATCH_ANY && recv->usermap[0].to == 65534 &&
|
||||||
recv->usermap[1].to == 1000 && recv->groupmap[0].from == 0 &&
|
recv->usermap[0].to_name == NULL && recv->usermap[1].from == 1000 &&
|
||||||
recv->groupmap[0].to == IDENTITY_CURRENT;
|
recv->usermap[1].from_hi == 1000 && recv->usermap[1].to == 1000 &&
|
||||||
|
recv->groupmap[0].from == 0 && recv->groupmap[0].from_hi == 0 &&
|
||||||
|
recv->groupmap[0].to == IDENTITY_CURRENT && recv->groupmap[0].to_name != NULL &&
|
||||||
|
strcmp(recv->groupmap[0].to_name, "root") == 0;
|
||||||
}
|
}
|
||||||
config_delete(recv);
|
config_delete(recv);
|
||||||
close(p[0]);
|
close(p[0]);
|
||||||
@@ -1395,7 +1552,8 @@ static void test_config_receive_rejects_invalid_identity() {
|
|||||||
c->receive_root_directory = str_dup("/dst");
|
c->receive_root_directory = str_dup("/dst");
|
||||||
c->usermap_count = 1;
|
c->usermap_count = 1;
|
||||||
c->usermap = calloc(1, sizeof(IdentityMap));
|
c->usermap = calloc(1, sizeof(IdentityMap));
|
||||||
c->usermap[0].from = -2; /* below IDENTITY_MATCH_ANY */
|
c->usermap[0].from = -3; /* below IDENTITY_MATCH_UNNAMED */
|
||||||
|
c->usermap[0].from_hi = -3;
|
||||||
c->usermap[0].to = 0;
|
c->usermap[0].to = 0;
|
||||||
EXPECT_FALSE(roundtrip_config_ok(c));
|
EXPECT_FALSE(roundtrip_config_ok(c));
|
||||||
config_delete(c);
|
config_delete(c);
|
||||||
@@ -2056,9 +2214,51 @@ static void test_identity_ownership_requested() {
|
|||||||
config_delete(gm);
|
config_delete(gm);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* The narrow client-CHOSEN ownership predicate the daemon module gate refuses:
|
||||||
|
* a preserve-source -o/-g (or -a) must NOT be in it (it is handled by forcing
|
||||||
|
* super-user activity off instead), while every explicit identity flag is. */
|
||||||
|
static void test_identity_explicit_ownership_requested() {
|
||||||
|
EXPECT_FALSE(identity_explicit_ownership_requested(NULL));
|
||||||
|
|
||||||
|
Config* c = config_create();
|
||||||
|
EXPECT_NOT_NULL(c);
|
||||||
|
EXPECT_FALSE(identity_explicit_ownership_requested(c));
|
||||||
|
c->preserve_owner = true;
|
||||||
|
EXPECT_FALSE(identity_explicit_ownership_requested(c));
|
||||||
|
EXPECT_TRUE(identity_ownership_requested(c)); /* general awareness does see -o */
|
||||||
|
c->preserve_group = true;
|
||||||
|
EXPECT_FALSE(identity_explicit_ownership_requested(c));
|
||||||
|
c->preserve_owner = false;
|
||||||
|
c->preserve_group = false;
|
||||||
|
|
||||||
|
c->numeric_ids = true;
|
||||||
|
EXPECT_TRUE(identity_explicit_ownership_requested(c));
|
||||||
|
c->numeric_ids = false;
|
||||||
|
c->chown_uid_set = true;
|
||||||
|
EXPECT_TRUE(identity_explicit_ownership_requested(c));
|
||||||
|
c->chown_uid_set = false;
|
||||||
|
c->chown_gid_set = true;
|
||||||
|
EXPECT_TRUE(identity_explicit_ownership_requested(c));
|
||||||
|
c->chown_gid_set = false;
|
||||||
|
c->copy_as_set = true;
|
||||||
|
EXPECT_TRUE(identity_explicit_ownership_requested(c));
|
||||||
|
c->copy_as_set = false;
|
||||||
|
c->fake_super = true;
|
||||||
|
EXPECT_TRUE(identity_explicit_ownership_requested(c));
|
||||||
|
c->fake_super = false;
|
||||||
|
c->super_mode = SUPER_MODE_ON;
|
||||||
|
EXPECT_TRUE(identity_explicit_ownership_requested(c));
|
||||||
|
c->super_mode = SUPER_MODE_AUTO;
|
||||||
|
EXPECT_EQ_INT(identity_parse_map(c, "@1:@2", false), 0);
|
||||||
|
EXPECT_TRUE(identity_explicit_ownership_requested(c));
|
||||||
|
config_delete(c);
|
||||||
|
}
|
||||||
|
|
||||||
/* P7 Wave E hardening (A3): --super no longer implies raw numeric-id
|
/* P7 Wave E hardening (A3): --super no longer implies raw numeric-id
|
||||||
preservation, so it must never enable ownership application on its own; an
|
preservation, so it must never enable ownership application on its own.
|
||||||
explicit identity flag is required. */
|
#286: --numeric-ids is a mapping MODIFIER only and is likewise inert on its
|
||||||
|
own; a real ownership request (-o/-g or an explicit identity flag) is
|
||||||
|
required to activate chown. */
|
||||||
static void test_super_does_not_imply_numeric() {
|
static void test_super_does_not_imply_numeric() {
|
||||||
Config* c = config_create();
|
Config* c = config_create();
|
||||||
EXPECT_NOT_NULL(c);
|
EXPECT_NOT_NULL(c);
|
||||||
@@ -2068,11 +2268,40 @@ static void test_super_does_not_imply_numeric() {
|
|||||||
EXPECT_FALSE(identity_active_enabled());
|
EXPECT_FALSE(identity_active_enabled());
|
||||||
c->numeric_ids = true;
|
c->numeric_ids = true;
|
||||||
EXPECT_TRUE(identity_set_active(c));
|
EXPECT_TRUE(identity_set_active(c));
|
||||||
|
EXPECT_FALSE(identity_active_enabled()); /* mapping modifier only */
|
||||||
|
c->preserve_owner = true;
|
||||||
|
EXPECT_TRUE(identity_set_active(c));
|
||||||
EXPECT_TRUE(identity_active_enabled());
|
EXPECT_TRUE(identity_active_enabled());
|
||||||
identity_clear_active();
|
identity_clear_active();
|
||||||
config_delete(c);
|
config_delete(c);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* The preserve-source -o/-g requests enable ownership application through the
|
||||||
|
* active snapshot (identity_active_enabled) even though they are deliberately
|
||||||
|
* absent from the narrow client-chosen identity_explicit_ownership_requested()
|
||||||
|
* gate. */
|
||||||
|
static void test_identity_active_enabled_includes_preserve_attrs() {
|
||||||
|
Config* c = config_create();
|
||||||
|
EXPECT_NOT_NULL(c);
|
||||||
|
c->use_metadata = true;
|
||||||
|
EXPECT_TRUE(identity_set_active(c));
|
||||||
|
EXPECT_FALSE(identity_active_enabled());
|
||||||
|
|
||||||
|
c->preserve_owner = true;
|
||||||
|
EXPECT_TRUE(identity_set_active(c));
|
||||||
|
EXPECT_TRUE(identity_active_enabled());
|
||||||
|
EXPECT_FALSE(identity_explicit_ownership_requested(c));
|
||||||
|
|
||||||
|
c->preserve_owner = false;
|
||||||
|
c->preserve_group = true;
|
||||||
|
EXPECT_TRUE(identity_set_active(c));
|
||||||
|
EXPECT_TRUE(identity_active_enabled());
|
||||||
|
EXPECT_FALSE(identity_explicit_ownership_requested(c));
|
||||||
|
|
||||||
|
identity_clear_active();
|
||||||
|
config_delete(c);
|
||||||
|
}
|
||||||
|
|
||||||
/* The single shared predicate must reject every cross-field combination the
|
/* The single shared predicate must reject every cross-field combination the
|
||||||
client/server enforce and accept a plain valid config. Because both
|
client/server enforce and accept a plain valid config. Because both
|
||||||
validate_config() (client) and validate_received_config() (server) call it,
|
validate_config() (client) and validate_received_config() (server) call it,
|
||||||
@@ -2193,6 +2422,95 @@ static void test_config_invariants_error_all_combinations() {
|
|||||||
config_delete(c);
|
config_delete(c);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Every per-attribute preservation flag requires the metadata frame to travel:
|
||||||
|
* the invariant rejects any of them while use_metadata is false, and setting
|
||||||
|
* use_metadata clears the violation. */
|
||||||
|
static void test_config_preservation_requires_metadata() {
|
||||||
|
static const size_t attrs[] = {
|
||||||
|
offsetof(Config, preserve_perms), offsetof(Config, preserve_times),
|
||||||
|
offsetof(Config, preserve_owner), offsetof(Config, preserve_group),
|
||||||
|
offsetof(Config, preserve_atimes), offsetof(Config, preserve_crtimes),
|
||||||
|
offsetof(Config, use_executability),
|
||||||
|
};
|
||||||
|
Config* c = config_create();
|
||||||
|
EXPECT_NOT_NULL(c);
|
||||||
|
EXPECT_NULL(config_invariants_error(c));
|
||||||
|
for (size_t i = 0; i < sizeof(attrs) / sizeof(attrs[0]); i++) {
|
||||||
|
bool* field = (bool*)((char*)c + attrs[i]);
|
||||||
|
*field = true;
|
||||||
|
EXPECT_NOT_NULL(config_invariants_error(c));
|
||||||
|
c->use_metadata = true;
|
||||||
|
EXPECT_NULL(config_invariants_error(c));
|
||||||
|
c->use_metadata = false;
|
||||||
|
*field = false;
|
||||||
|
}
|
||||||
|
config_delete(c);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* config_derived_use_metadata is the single source of truth for the derived
|
||||||
|
* transport bit: each representative flag turns it on, and it stays off for a
|
||||||
|
* bare config (numeric_ids alone, omit flags, whole-file, ...). */
|
||||||
|
static void test_config_derived_use_metadata() {
|
||||||
|
static const size_t true_flags[] = {
|
||||||
|
offsetof(Config, preserve_perms), offsetof(Config, preserve_times),
|
||||||
|
offsetof(Config, preserve_owner), offsetof(Config, preserve_group),
|
||||||
|
offsetof(Config, preserve_atimes), offsetof(Config, preserve_crtimes),
|
||||||
|
offsetof(Config, use_executability), offsetof(Config, preserve_xattrs),
|
||||||
|
offsetof(Config, preserve_acls), offsetof(Config, fake_super),
|
||||||
|
offsetof(Config, preserve_devices), offsetof(Config, preserve_specials),
|
||||||
|
offsetof(Config, copy_devices), offsetof(Config, write_devices),
|
||||||
|
offsetof(Config, copy_as_set), offsetof(Config, chown_uid_set),
|
||||||
|
offsetof(Config, chown_gid_set), offsetof(Config, update),
|
||||||
|
};
|
||||||
|
Config* c = config_create();
|
||||||
|
EXPECT_NOT_NULL(c);
|
||||||
|
EXPECT_FALSE(config_derived_use_metadata(c));
|
||||||
|
EXPECT_FALSE(config_derived_use_metadata(NULL));
|
||||||
|
for (size_t i = 0; i < sizeof(true_flags) / sizeof(true_flags[0]); i++) {
|
||||||
|
bool* field = (bool*)((char*)c + true_flags[i]);
|
||||||
|
*field = true;
|
||||||
|
EXPECT_TRUE(config_derived_use_metadata(c));
|
||||||
|
*field = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* A non-empty --chmod spec. */
|
||||||
|
c->chmod_spec = str_dup("u=rw");
|
||||||
|
EXPECT_TRUE(config_derived_use_metadata(c));
|
||||||
|
free(c->chmod_spec);
|
||||||
|
c->chmod_spec = NULL;
|
||||||
|
|
||||||
|
/* Identity-map counts. */
|
||||||
|
c->usermap_count = 1;
|
||||||
|
EXPECT_TRUE(config_derived_use_metadata(c));
|
||||||
|
c->usermap_count = 0;
|
||||||
|
c->groupmap_count = 1;
|
||||||
|
EXPECT_TRUE(config_derived_use_metadata(c));
|
||||||
|
c->groupmap_count = 0;
|
||||||
|
|
||||||
|
/* Incremental/delta imply metadata unless --no-preserve disabled it. */
|
||||||
|
c->use_incremental = true;
|
||||||
|
EXPECT_TRUE(config_derived_use_metadata(c));
|
||||||
|
c->metadata_explicitly_disabled = true;
|
||||||
|
EXPECT_FALSE(config_derived_use_metadata(c));
|
||||||
|
c->metadata_explicitly_disabled = false;
|
||||||
|
c->use_incremental = false;
|
||||||
|
c->use_delta = true;
|
||||||
|
EXPECT_TRUE(config_derived_use_metadata(c));
|
||||||
|
c->metadata_explicitly_disabled = true;
|
||||||
|
EXPECT_FALSE(config_derived_use_metadata(c));
|
||||||
|
c->metadata_explicitly_disabled = false;
|
||||||
|
c->use_delta = false;
|
||||||
|
|
||||||
|
/* Flags that must NOT imply metadata on their own. */
|
||||||
|
c->numeric_ids = true;
|
||||||
|
c->omit_dir_times = true;
|
||||||
|
c->omit_link_times = true;
|
||||||
|
c->whole_file = true;
|
||||||
|
c->ignore_times = true;
|
||||||
|
EXPECT_FALSE(config_derived_use_metadata(c));
|
||||||
|
config_delete(c);
|
||||||
|
}
|
||||||
|
|
||||||
/* The receiver previously missed several of these; a forged frame that sets
|
/* The receiver previously missed several of these; a forged frame that sets
|
||||||
the offending serialized fields must now be refused at the config
|
the offending serialized fields must now be refused at the config
|
||||||
handshake. (whole_file is client-only, so its rules cannot appear here.) */
|
handshake. (whole_file is client-only, so its rules cannot appear here.) */
|
||||||
@@ -2286,6 +2604,7 @@ static bool basis_equal(const Config* a, const Config* b) {
|
|||||||
#define CONFIG_CMP_STR_MODULE(a, b, name) str_opt_equal((a)->name, (b)->name)
|
#define CONFIG_CMP_STR_MODULE(a, b, name) str_opt_equal((a)->name, (b)->name)
|
||||||
#define CONFIG_CMP_STR_REDACTED_AUTH(a, b, name) str_opt_equal((a)->name, (b)->name)
|
#define CONFIG_CMP_STR_REDACTED_AUTH(a, b, name) str_opt_equal((a)->name, (b)->name)
|
||||||
#define CONFIG_CMP_INT_CHECKSUM_ALGO(a, b, name) ((a)->name == (b)->name)
|
#define CONFIG_CMP_INT_CHECKSUM_ALGO(a, b, name) ((a)->name == (b)->name)
|
||||||
|
#define CONFIG_CMP_INT_COMPRESSION_ALGO(a, b, name) ((a)->name == (b)->name)
|
||||||
#define CONFIG_CMP_SUPERMODE(a, b, name) ((a)->name == (b)->name)
|
#define CONFIG_CMP_SUPERMODE(a, b, name) ((a)->name == (b)->name)
|
||||||
#define CONFIG_CMP_INT_IDENTITY(a, b, name) ((a)->name == (b)->name)
|
#define CONFIG_CMP_INT_IDENTITY(a, b, name) ((a)->name == (b)->name)
|
||||||
#define CONFIG_CMP_INT_SKIPCOUNT(a, b, name) ((a)->name == (b)->name)
|
#define CONFIG_CMP_INT_SKIPCOUNT(a, b, name) ((a)->name == (b)->name)
|
||||||
@@ -2355,6 +2674,34 @@ static void test_config_wire_roundtrip_all_fields() {
|
|||||||
config_delete(populated);
|
config_delete(populated);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Each of the four split-out preservation bools must survive a frame
|
||||||
|
* round-trip on its own. The all-fields golden sets an alternating
|
||||||
|
* true/false pattern precisely because a run of identical adjacent booleans
|
||||||
|
* would let a same-KIND field swap produce the same bytes; isolating one true
|
||||||
|
* bit at a time pins each new field's position and width independently. */
|
||||||
|
static void test_config_preserve_attribute_wire_roundtrip() {
|
||||||
|
if (is_running_under_valgrind())
|
||||||
|
return;
|
||||||
|
static const size_t attrs[] = {
|
||||||
|
offsetof(Config, preserve_perms),
|
||||||
|
offsetof(Config, preserve_times),
|
||||||
|
offsetof(Config, preserve_owner),
|
||||||
|
offsetof(Config, preserve_group),
|
||||||
|
};
|
||||||
|
for (size_t i = 0; i < sizeof(attrs) / sizeof(attrs[0]); i++) {
|
||||||
|
Config* c = config_create();
|
||||||
|
EXPECT_NOT_NULL(c);
|
||||||
|
c->send_directory = str_dup("/src");
|
||||||
|
c->receive_root_directory = str_dup("/dst");
|
||||||
|
/* The preservation invariant requires the metadata frame to travel, so set
|
||||||
|
* the transport bit; otherwise config_receive() legitimately refuses. */
|
||||||
|
c->use_metadata = true;
|
||||||
|
*(bool*)((char*)c + attrs[i]) = true;
|
||||||
|
EXPECT_TRUE(roundtrip_and_compare(c));
|
||||||
|
config_delete(c);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/* Populate every serialized field with a non-default value so the wire frame
|
/* Populate every serialized field with a non-default value so the wire frame
|
||||||
* exercises each table entry. Boolean runs deliberately alternate true/false:
|
* exercises each table entry. Boolean runs deliberately alternate true/false:
|
||||||
* a run of identical booleans would make an adjacent swap (same KIND) produce
|
* a run of identical booleans would make an adjacent swap (same KIND) produce
|
||||||
@@ -2427,7 +2774,7 @@ static void golden_config_populate(Config* c) {
|
|||||||
c->modify_window = 3;
|
c->modify_window = 3;
|
||||||
c->compress_choice = str_dup("zstd");
|
c->compress_choice = str_dup("zstd");
|
||||||
/* "u=rwx,go=rx" is the same 11 bytes as the original "u=rwX,go=rX" (so the
|
/* "u=rwx,go=rx" is the same 11 bytes as the original "u=rwX,go=rX" (so the
|
||||||
* frame stays 637 bytes) but X is not in FastSync's chmod grammar, and the
|
* frame stays 653 bytes) but X is not in FastSync's chmod grammar, and the
|
||||||
* receive-side golden validates the frame. */
|
* receive-side golden validates the frame. */
|
||||||
c->chmod_spec = str_dup("u=rwx,go=rx");
|
c->chmod_spec = str_dup("u=rwx,go=rx");
|
||||||
c->skip_compress_set = true;
|
c->skip_compress_set = true;
|
||||||
@@ -2448,17 +2795,29 @@ static void golden_config_populate(Config* c) {
|
|||||||
c->usermap_count = 2;
|
c->usermap_count = 2;
|
||||||
c->usermap = calloc(2, sizeof(IdentityMap));
|
c->usermap = calloc(2, sizeof(IdentityMap));
|
||||||
c->usermap[0].from = IDENTITY_MATCH_ANY;
|
c->usermap[0].from = IDENTITY_MATCH_ANY;
|
||||||
|
c->usermap[0].from_hi = IDENTITY_MATCH_ANY;
|
||||||
c->usermap[0].to = 1000;
|
c->usermap[0].to = 1000;
|
||||||
|
c->usermap[0].to_name = NULL;
|
||||||
c->usermap[1].from = 5;
|
c->usermap[1].from = 5;
|
||||||
|
c->usermap[1].from_hi = 9;
|
||||||
c->usermap[1].to = 6;
|
c->usermap[1].to = 6;
|
||||||
|
c->usermap[1].to_name = NULL;
|
||||||
c->groupmap_count = 1;
|
c->groupmap_count = 1;
|
||||||
c->groupmap = calloc(1, sizeof(IdentityMap));
|
c->groupmap = calloc(1, sizeof(IdentityMap));
|
||||||
c->groupmap[0].from = 7;
|
c->groupmap[0].from = 7;
|
||||||
|
c->groupmap[0].from_hi = 7;
|
||||||
c->groupmap[0].to = 8;
|
c->groupmap[0].to = 8;
|
||||||
|
c->groupmap[0].to_name = str_dup("root");
|
||||||
c->preserve_atimes = true;
|
c->preserve_atimes = true;
|
||||||
c->preserve_crtimes = false;
|
c->preserve_crtimes = false;
|
||||||
c->omit_dir_times = true;
|
c->omit_dir_times = true;
|
||||||
c->omit_link_times = false;
|
c->omit_link_times = false;
|
||||||
|
/* Mixed true/false so a field reorder or a dropped attribute changes the
|
||||||
|
* pinned hash rather than passing silently. */
|
||||||
|
c->preserve_perms = true;
|
||||||
|
c->preserve_times = false;
|
||||||
|
c->preserve_owner = true;
|
||||||
|
c->preserve_group = false;
|
||||||
c->munge_links = true;
|
c->munge_links = true;
|
||||||
c->keep_dirlinks = false;
|
c->keep_dirlinks = false;
|
||||||
c->fake_super = true;
|
c->fake_super = true;
|
||||||
@@ -2472,13 +2831,14 @@ static void golden_config_populate(Config* c) {
|
|||||||
c->copy_as_gid = 222;
|
c->copy_as_gid = 222;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* The pinned golden frame (protocol 2.21.0). The values below are the only
|
/* The pinned golden frame (protocol 2.26.0). The values below are the only
|
||||||
* thing that ties the generated table to the historical wire format; update
|
* thing that ties the generated table to the historical wire format; update
|
||||||
* them ONLY with a PROTOCOL_VERSION bump and a documented reason. The combined
|
* them ONLY with a PROTOCOL_VERSION bump and a documented reason. The 2.24.0
|
||||||
* 2.21.0 wave appends the serialized dry_run bool to CONFIG_WIRE_CORE_FIELDS
|
* delete-plan wave changed only the version string; 2.25.0 appended the
|
||||||
* and keeps the protocol version string at 2.21.0. */
|
* report_stats bool and 2.26.0 appended the compression_algo int. The
|
||||||
#define GOLDEN_WIRE_LEN 637
|
* byte-exact values are recomputed for the merged layout. */
|
||||||
#define GOLDEN_WIRE_HASH 13228626061067899189ULL
|
#define GOLDEN_WIRE_LEN 705
|
||||||
|
#define GOLDEN_WIRE_HASH 4673424031554175633ULL
|
||||||
|
|
||||||
static unsigned long long fnv1a_64(const unsigned char* buf, size_t len) {
|
static unsigned long long fnv1a_64(const unsigned char* buf, size_t len) {
|
||||||
unsigned long long h = 1469598103934665603ULL;
|
unsigned long long h = 1469598103934665603ULL;
|
||||||
@@ -2560,7 +2920,7 @@ static unsigned long long capture_wire_hash(const Config* cfg, size_t* out_len)
|
|||||||
return h;
|
return h;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Byte-for-byte wire compatibility guard (protocol 2.21.0). The expected hash
|
/* Byte-for-byte wire compatibility guard (protocol 2.26.0). The expected hash
|
||||||
* pins the pre-X-macro byte stream; the refactor MUST NOT change it. */
|
* pins the pre-X-macro byte stream; the refactor MUST NOT change it. */
|
||||||
static void test_config_wire_golden() {
|
static void test_config_wire_golden() {
|
||||||
if (is_running_under_valgrind())
|
if (is_running_under_valgrind())
|
||||||
@@ -2613,12 +2973,18 @@ static void test_config_wire_golden_receive() {
|
|||||||
!recv->use_multithreading;
|
!recv->use_multithreading;
|
||||||
ok = ok && recv->compression_level == 7 && recv->chunk_size == 65536;
|
ok = ok && recv->compression_level == 7 && recv->chunk_size == 65536;
|
||||||
ok = ok && recv->use_delta && !recv->whole_file && recv->use_xattrs;
|
ok = ok && recv->use_delta && !recv->whole_file && recv->use_xattrs;
|
||||||
|
/* Per-attribute preservation split decoded from the pinned bytes. */
|
||||||
|
ok = ok && recv->preserve_perms && !recv->preserve_times && recv->preserve_owner &&
|
||||||
|
!recv->preserve_group;
|
||||||
/* Bounded/validated KINDs decoded from the pinned bytes. */
|
/* Bounded/validated KINDs decoded from the pinned bytes. */
|
||||||
ok = ok && recv->checksum_algo == CHECKSUM_ALGO_MD5;
|
ok = ok && recv->checksum_algo == CHECKSUM_ALGO_MD5;
|
||||||
ok = ok && recv->super_mode == SUPER_MODE_ON;
|
ok = ok && recv->super_mode == SUPER_MODE_ON;
|
||||||
ok = ok && recv->chown_uid == 1234 && recv->chown_gid == 5678;
|
ok = ok && recv->chown_uid == 1234 && recv->chown_gid == 5678;
|
||||||
ok = ok && recv->usermap_count == 2 && recv->usermap[0].from == IDENTITY_MATCH_ANY &&
|
ok = ok && recv->usermap_count == 2 && recv->usermap[0].from == IDENTITY_MATCH_ANY &&
|
||||||
recv->usermap[0].to == 1000 && recv->usermap[1].from == 5 && recv->usermap[1].to == 6;
|
recv->usermap[0].from_hi == IDENTITY_MATCH_ANY && recv->usermap[0].to == 1000 &&
|
||||||
|
recv->usermap[1].from == 5 && recv->usermap[1].from_hi == 9 && recv->usermap[1].to == 6;
|
||||||
|
ok = ok && recv->groupmap_count == 1 && recv->groupmap[0].from == 7 &&
|
||||||
|
recv->groupmap[0].to_name != NULL && strcmp(recv->groupmap[0].to_name, "root") == 0;
|
||||||
ok = ok && recv->basis_count == 2 && recv->basis_dirs[0].type == BASIS_DEST_COMPARE &&
|
ok = ok && recv->basis_count == 2 && recv->basis_dirs[0].type == BASIS_DEST_COMPARE &&
|
||||||
recv->basis_dirs[1].type == BASIS_DEST_LINK;
|
recv->basis_dirs[1].type == BASIS_DEST_LINK;
|
||||||
ok = ok && recv->module != NULL && strcmp(recv->module, "goldenmod") == 0;
|
ok = ok && recv->module != NULL && strcmp(recv->module, "goldenmod") == 0;
|
||||||
@@ -2702,13 +3068,14 @@ static void test_config_wire_receive_bounds() {
|
|||||||
/* BOOL: only 0/1 is a legal wire value. */
|
/* BOOL: only 0/1 is a legal wire value. */
|
||||||
EXPECT_TRUE(receive_hand_built_frame_rejected(write_frame_with_invalid_bool));
|
EXPECT_TRUE(receive_hand_built_frame_rejected(write_frame_with_invalid_bool));
|
||||||
|
|
||||||
/* RAW_MAXALLOC: zero is rejected before it can become the session ceiling. */
|
/* RAW_MAXALLOC: zero is rsync's --max-alloc=0 "no limit" and round-trips;
|
||||||
|
* only the over-ceiling clamp is applied server-side. */
|
||||||
Config* c = config_create();
|
Config* c = config_create();
|
||||||
EXPECT_NOT_NULL(c);
|
EXPECT_NOT_NULL(c);
|
||||||
c->send_directory = str_dup("/src");
|
c->send_directory = str_dup("/src");
|
||||||
c->receive_root_directory = str_dup("/dst");
|
c->receive_root_directory = str_dup("/dst");
|
||||||
c->max_alloc = 0;
|
c->max_alloc = 0;
|
||||||
EXPECT_TRUE(roundtrip_config_rejected(c));
|
EXPECT_FALSE(roundtrip_config_rejected(c));
|
||||||
config_delete(c);
|
config_delete(c);
|
||||||
|
|
||||||
/* STR_MODULE: a name outside [A-Za-z0-9._-] is refused. */
|
/* STR_MODULE: a name outside [A-Za-z0-9._-] is refused. */
|
||||||
@@ -2829,6 +3196,8 @@ void test_config() {
|
|||||||
test_pipeline_receiver_lifecycle();
|
test_pipeline_receiver_lifecycle();
|
||||||
if (!is_running_under_valgrind()) {
|
if (!is_running_under_valgrind()) {
|
||||||
test_config_send_receive();
|
test_config_send_receive();
|
||||||
|
test_config_receive_max_alloc_zero_floored();
|
||||||
|
test_config_receive_compress_choice_auto_canonicalized();
|
||||||
test_config_local_only_fields_not_serialized();
|
test_config_local_only_fields_not_serialized();
|
||||||
test_config_send_receive_version_mismatch();
|
test_config_send_receive_version_mismatch();
|
||||||
test_config_receive_truncated();
|
test_config_receive_truncated();
|
||||||
@@ -2846,6 +3215,10 @@ void test_config() {
|
|||||||
test_config_basis_normalization();
|
test_config_basis_normalization();
|
||||||
test_config_checksum_options_wire_roundtrip();
|
test_config_checksum_options_wire_roundtrip();
|
||||||
test_config_receive_rejects_invalid_checksum_algo();
|
test_config_receive_rejects_invalid_checksum_algo();
|
||||||
|
test_config_receive_rejects_invalid_compression_algo();
|
||||||
|
test_config_receive_rejects_codec_mismatch();
|
||||||
|
test_config_receive_rejects_none_codec_with_compression();
|
||||||
|
test_config_receive_rejects_checksum_none_with_checksum();
|
||||||
test_config_identity_wire_roundtrip();
|
test_config_identity_wire_roundtrip();
|
||||||
test_config_receive_rejects_invalid_identity();
|
test_config_receive_rejects_invalid_identity();
|
||||||
test_config_metadata_times_wire_roundtrip();
|
test_config_metadata_times_wire_roundtrip();
|
||||||
@@ -2867,15 +3240,20 @@ void test_config() {
|
|||||||
test_config_receive_rejects_oversized_string_budget();
|
test_config_receive_rejects_oversized_string_budget();
|
||||||
test_config_receive_with_validate_rejects();
|
test_config_receive_with_validate_rejects();
|
||||||
test_config_invariants_error_all_combinations();
|
test_config_invariants_error_all_combinations();
|
||||||
|
test_config_preservation_requires_metadata();
|
||||||
|
test_config_derived_use_metadata();
|
||||||
test_config_receive_rejects_unified_invariants();
|
test_config_receive_rejects_unified_invariants();
|
||||||
test_config_wire_golden();
|
test_config_wire_golden();
|
||||||
test_config_wire_golden_receive();
|
test_config_wire_golden_receive();
|
||||||
test_config_wire_receive_bounds();
|
test_config_wire_receive_bounds();
|
||||||
test_config_receive_rejects_overcap_counts();
|
test_config_receive_rejects_overcap_counts();
|
||||||
test_config_wire_roundtrip_all_fields();
|
test_config_wire_roundtrip_all_fields();
|
||||||
|
test_config_preserve_attribute_wire_roundtrip();
|
||||||
}
|
}
|
||||||
test_identity_copy_as_refused();
|
test_identity_copy_as_refused();
|
||||||
test_identity_ownership_requested();
|
test_identity_ownership_requested();
|
||||||
|
test_identity_explicit_ownership_requested();
|
||||||
|
test_identity_active_enabled_includes_preserve_attrs();
|
||||||
test_super_does_not_imply_numeric();
|
test_super_does_not_imply_numeric();
|
||||||
test_privilege_super_permitted_modes();
|
test_privilege_super_permitted_modes();
|
||||||
test_config_delete_timing_early_helper();
|
test_config_delete_timing_early_helper();
|
||||||
|
|||||||
+498
-42
@@ -28,6 +28,10 @@ static void test_file_create() {
|
|||||||
EXPECT_NULL(f->data->data);
|
EXPECT_NULL(f->data->data);
|
||||||
EXPECT_EQ_INT((int)f->data->size, 0);
|
EXPECT_EQ_INT((int)f->data->size, 0);
|
||||||
EXPECT_NULL(f->metadata);
|
EXPECT_NULL(f->metadata);
|
||||||
|
/* An unset destination snapshot must read as known == false, never
|
||||||
|
indeterminate bytes (-i/--out-format without --incremental). */
|
||||||
|
EXPECT_FALSE(f->dest_state.known);
|
||||||
|
EXPECT_FALSE(f->dest_state.existed);
|
||||||
file_destroy(f);
|
file_destroy(f);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -326,6 +330,52 @@ static void test_file_save_to_disk_partial_install() {
|
|||||||
rmdir(root);
|
rmdir(root);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* --temp-dir is a client-controlled wire value that must be confined below the
|
||||||
|
* receive root: an absolute or `..`-escaping value is rejected (a client must
|
||||||
|
* never make the receiver write scratch files in an arbitrary directory), while
|
||||||
|
* a relative one resolves under the root and is used for the atomic install. */
|
||||||
|
static void test_file_save_to_disk_temp_dir_confined() {
|
||||||
|
const char* root = "test_temp_confine_tmp";
|
||||||
|
const char* dest_file = "test_temp_confine_tmp/file.txt";
|
||||||
|
char outside[PATH_MAX];
|
||||||
|
snprintf(outside, sizeof(outside), "/tmp/fastsync_temp_outside_%d", (int)getpid());
|
||||||
|
unlink(dest_file);
|
||||||
|
rmdir("test_temp_confine_tmp/scratch");
|
||||||
|
rmdir(root);
|
||||||
|
mkdir(root, 0755);
|
||||||
|
mkdir("test_temp_confine_tmp/scratch", 0755);
|
||||||
|
mkdir(outside, 0755);
|
||||||
|
|
||||||
|
File* f = file_create("file.txt");
|
||||||
|
EXPECT_NOT_NULL(f);
|
||||||
|
const char* content = "confined temp dir";
|
||||||
|
f->data->data = malloc(strlen(content));
|
||||||
|
EXPECT_NOT_NULL(f->data->data);
|
||||||
|
memcpy(f->data->data, content, strlen(content));
|
||||||
|
f->data->size = strlen(content);
|
||||||
|
|
||||||
|
Config* config = config_create();
|
||||||
|
EXPECT_NOT_NULL(config);
|
||||||
|
config->temp_dir = str_dup(outside);
|
||||||
|
EXPECT_EQ_INT(file_save_to_disk_full(root, f, config), FILE_SAVE_ERROR);
|
||||||
|
EXPECT_EQ_INT(access(dest_file, F_OK), -1);
|
||||||
|
free(config->temp_dir);
|
||||||
|
config->temp_dir = str_dup("../escape");
|
||||||
|
EXPECT_EQ_INT(file_save_to_disk_full(root, f, config), FILE_SAVE_ERROR);
|
||||||
|
EXPECT_EQ_INT(access(dest_file, F_OK), -1);
|
||||||
|
free(config->temp_dir);
|
||||||
|
config->temp_dir = str_dup("scratch");
|
||||||
|
EXPECT_EQ_INT(file_save_to_disk_full(root, f, config), FILE_SAVE_WRITTEN);
|
||||||
|
EXPECT_EQ_INT(access(dest_file, F_OK), 0);
|
||||||
|
|
||||||
|
file_destroy(f);
|
||||||
|
config_delete(config);
|
||||||
|
unlink(dest_file);
|
||||||
|
rmdir("test_temp_confine_tmp/scratch");
|
||||||
|
rmdir(root);
|
||||||
|
rmdir(outside);
|
||||||
|
}
|
||||||
|
|
||||||
/* Issue #251: file_save_to_disk_full must distinguish receiver-side skips
|
/* Issue #251: file_save_to_disk_full must distinguish receiver-side skips
|
||||||
(--existing/--ignore-existing/--update) from real writes so the sender can
|
(--existing/--ignore-existing/--update) from real writes so the sender can
|
||||||
decide whether --remove-source-files may unlink its source. */
|
decide whether --remove-source-files may unlink its source. */
|
||||||
@@ -410,7 +460,7 @@ static void test_file_write_to_disk_with_fsync() {
|
|||||||
const char* path = "test_file_write_to_disk_fsync.txt";
|
const char* path = "test_file_write_to_disk_fsync.txt";
|
||||||
const char* content = "fsync file content";
|
const char* content = "fsync file content";
|
||||||
EXPECT_TRUE(file_to_disk_secure_with_fsync(path, content, strlen(content), false, false, false,
|
EXPECT_TRUE(file_to_disk_secure_with_fsync(path, content, strlen(content), false, false, false,
|
||||||
NULL, false, true, NULL));
|
NULL, (FileAttrPolicy){0}, true, NULL));
|
||||||
struct stat st;
|
struct stat st;
|
||||||
EXPECT_EQ_INT(stat(path, &st), 0);
|
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||||
EXPECT_EQ_INT((int)st.st_size, (int)strlen(content));
|
EXPECT_EQ_INT((int)st.st_size, (int)strlen(content));
|
||||||
@@ -420,8 +470,8 @@ static void test_file_write_to_disk_with_fsync() {
|
|||||||
static void test_file_write_to_disk_preallocate_atomic() {
|
static void test_file_write_to_disk_preallocate_atomic() {
|
||||||
const char* path = "test_file_write_prealloc_atomic.txt";
|
const char* path = "test_file_write_prealloc_atomic.txt";
|
||||||
const char* content = "prealloc atomic content";
|
const char* content = "prealloc atomic content";
|
||||||
EXPECT_TRUE(
|
EXPECT_TRUE(file_to_disk_secure(path, content, strlen(content), false, false, true, NULL,
|
||||||
file_to_disk_secure(path, content, strlen(content), false, false, true, NULL, false, NULL));
|
(FileAttrPolicy){0}, NULL));
|
||||||
struct stat st;
|
struct stat st;
|
||||||
EXPECT_EQ_INT(stat(path, &st), 0);
|
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||||
EXPECT_EQ_INT((int)st.st_size, (int)strlen(content));
|
EXPECT_EQ_INT((int)st.st_size, (int)strlen(content));
|
||||||
@@ -438,8 +488,8 @@ static void test_file_write_to_disk_preallocate_atomic() {
|
|||||||
static void test_file_write_to_disk_preallocate_inplace() {
|
static void test_file_write_to_disk_preallocate_inplace() {
|
||||||
const char* path = "test_file_write_prealloc_inplace.txt";
|
const char* path = "test_file_write_prealloc_inplace.txt";
|
||||||
const char* content = "prealloc inplace content";
|
const char* content = "prealloc inplace content";
|
||||||
EXPECT_TRUE(
|
EXPECT_TRUE(file_to_disk_secure(path, content, strlen(content), true, false, true, NULL,
|
||||||
file_to_disk_secure(path, content, strlen(content), true, false, true, NULL, false, NULL));
|
(FileAttrPolicy){0}, NULL));
|
||||||
struct stat st;
|
struct stat st;
|
||||||
EXPECT_EQ_INT(stat(path, &st), 0);
|
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||||
EXPECT_EQ_INT((int)st.st_size, (int)strlen(content));
|
EXPECT_EQ_INT((int)st.st_size, (int)strlen(content));
|
||||||
@@ -518,6 +568,20 @@ static void test_file_symlink_helpers() {
|
|||||||
EXPECT_FALSE(file_symlink_target_contained("../escape"));
|
EXPECT_FALSE(file_symlink_target_contained("../escape"));
|
||||||
EXPECT_FALSE(file_symlink_target_contained("a/../b"));
|
EXPECT_FALSE(file_symlink_target_contained("a/../b"));
|
||||||
EXPECT_FALSE(file_symlink_target_contained(""));
|
EXPECT_FALSE(file_symlink_target_contained(""));
|
||||||
|
|
||||||
|
/* rsync 3.4.1 unsafe_symlink(): absolute/empty are unsafe; ".." is measured
|
||||||
|
against the symlink's own transfer-relative directory depth. */
|
||||||
|
EXPECT_TRUE(file_symlink_unsafe("/etc/passwd", "link"));
|
||||||
|
EXPECT_TRUE(file_symlink_unsafe("", "link"));
|
||||||
|
EXPECT_FALSE(file_symlink_unsafe("a.txt", "link"));
|
||||||
|
EXPECT_FALSE(file_symlink_unsafe("./a.txt", "link"));
|
||||||
|
EXPECT_FALSE(file_symlink_unsafe("../real.txt", "a/up1"));
|
||||||
|
EXPECT_FALSE(file_symlink_unsafe("../../real.txt", "a/b/up3"));
|
||||||
|
EXPECT_TRUE(file_symlink_unsafe("../../../outside", "a/b/esc"));
|
||||||
|
EXPECT_TRUE(file_symlink_unsafe("../outside", "esc"));
|
||||||
|
/* Internal /../ and a trailing /.. are rejected by rsync 3.4.1. */
|
||||||
|
EXPECT_TRUE(file_symlink_unsafe("a/b/../real.txt", "norm"));
|
||||||
|
EXPECT_TRUE(file_symlink_unsafe("dir/..", "link"));
|
||||||
}
|
}
|
||||||
|
|
||||||
static void test_file_symlink_at_secure() {
|
static void test_file_symlink_at_secure() {
|
||||||
@@ -943,7 +1007,8 @@ static void test_inplace_overwrite_metadata_strips_special_bits() {
|
|||||||
|
|
||||||
struct stat st;
|
struct stat st;
|
||||||
EXPECT_EQ_INT(stat(path, &st), 0);
|
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||||
/* Metadata-derived mode is applied and never includes setuid/setgid/sticky. */
|
/* No -p: the pre-existing destination mode (without its special bits) is
|
||||||
|
* restored; the source mode is not applied. */
|
||||||
EXPECT_EQ_INT((int)(st.st_mode & (S_ISUID | S_ISGID | S_ISVTX)), 0);
|
EXPECT_EQ_INT((int)(st.st_mode & (S_ISUID | S_ISGID | S_ISVTX)), 0);
|
||||||
EXPECT_EQ_INT((int)(st.st_mode & 0777), 0755);
|
EXPECT_EQ_INT((int)(st.st_mode & 0777), 0755);
|
||||||
|
|
||||||
@@ -951,6 +1016,212 @@ static void test_inplace_overwrite_metadata_strips_special_bits() {
|
|||||||
rmdir(root);
|
rmdir(root);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* The per-attribute split: with no -p/-E the atomic (inode-replacing) write
|
||||||
|
* must restore the PRE-EXISTING destination mode instead of the source mode; a
|
||||||
|
* brand-new file keeps the historical 0644 default; -p applies the source. */
|
||||||
|
static void test_atomic_no_perms_preserves_destination_mode() {
|
||||||
|
const char* path = "test_attr_split_mode.txt";
|
||||||
|
unlink(path);
|
||||||
|
int fd = open(path, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0640);
|
||||||
|
EXPECT_TRUE(fd >= 0);
|
||||||
|
/* cppcheck-suppress knownConditionTrueFalse */
|
||||||
|
if (fd < 0)
|
||||||
|
return;
|
||||||
|
EXPECT_EQ_INT(fchmod(fd, 0640), 0);
|
||||||
|
EXPECT_EQ_INT(close(fd), 0);
|
||||||
|
|
||||||
|
FileMetadata m;
|
||||||
|
memset(&m, 0, sizeof(m));
|
||||||
|
m.mode = 0755;
|
||||||
|
m.uid = geteuid();
|
||||||
|
m.gid = getegid();
|
||||||
|
m.mtime_sec = 1700000000;
|
||||||
|
|
||||||
|
/* No -p/-E: the pre-existing 0640 survives the atomic overwrite. */
|
||||||
|
bool ok = file_to_disk_secure_attrs(path, "data", 4, false, false, false, &m,
|
||||||
|
(FileAttrPolicy){false, false, false, false}, false, false,
|
||||||
|
false, NULL, false, false, NULL);
|
||||||
|
EXPECT_TRUE(ok);
|
||||||
|
struct stat st;
|
||||||
|
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||||
|
EXPECT_EQ_INT((int)(st.st_mode & 0777), 0640);
|
||||||
|
|
||||||
|
/* -p: the source mode wins. */
|
||||||
|
ok = file_to_disk_secure_attrs(path, "data2", 5, false, false, false, &m,
|
||||||
|
(FileAttrPolicy){true, true, false, false}, false, false, false,
|
||||||
|
NULL, false, false, NULL);
|
||||||
|
EXPECT_TRUE(ok);
|
||||||
|
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||||
|
EXPECT_EQ_INT((int)(st.st_mode & 0777), 0755);
|
||||||
|
|
||||||
|
/* -E only (rsync rule): an executable source derives exec from the
|
||||||
|
pre-existing destination's read bits. Dest 0640 (owner+group read) with a
|
||||||
|
source 0755 gives 0750, not 0751 and not the scratch 0711. */
|
||||||
|
EXPECT_EQ_INT(chmod(path, 0640), 0);
|
||||||
|
ok = file_to_disk_secure_attrs(path, "data3", 6, false, false, false, &m,
|
||||||
|
(FileAttrPolicy){false, false, false, true}, false, false, false,
|
||||||
|
NULL, false, false, NULL);
|
||||||
|
EXPECT_TRUE(ok);
|
||||||
|
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||||
|
EXPECT_EQ_INT((int)(st.st_mode & 0777), 0750);
|
||||||
|
|
||||||
|
unlink(path);
|
||||||
|
|
||||||
|
/* A brand-new file with no -p uses rsync's source&~umask base when metadata
|
||||||
|
is available (m.mode is 0755 here). */
|
||||||
|
const char* fresh = "test_attr_split_fresh.txt";
|
||||||
|
unlink(fresh);
|
||||||
|
ok = file_to_disk_secure_attrs(fresh, "data", 4, false, false, false, &m,
|
||||||
|
(FileAttrPolicy){false, false, false, false}, false, false, false,
|
||||||
|
NULL, false, false, NULL);
|
||||||
|
EXPECT_TRUE(ok);
|
||||||
|
EXPECT_EQ_INT(stat(fresh, &st), 0);
|
||||||
|
EXPECT_EQ_INT((int)(st.st_mode & 0777), (int)(m.mode & 0777 & ~(mode_t)file_process_umask()));
|
||||||
|
unlink(fresh);
|
||||||
|
|
||||||
|
/* Without any metadata the historical fixed 0644 default still applies. */
|
||||||
|
unlink(fresh);
|
||||||
|
ok = file_to_disk_secure_attrs(fresh, "data", 4, false, false, false, NULL,
|
||||||
|
(FileAttrPolicy){false, false, false, false}, false, false, false,
|
||||||
|
NULL, false, false, NULL);
|
||||||
|
EXPECT_TRUE(ok);
|
||||||
|
EXPECT_EQ_INT(stat(fresh, &st), 0);
|
||||||
|
EXPECT_EQ_INT((int)(st.st_mode & 0777), 0644);
|
||||||
|
unlink(fresh);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Strict rsync parity: a brand-new destination file with no -p follows
|
||||||
|
* rsync's source_mode & ~umask base, so group/other write in the source mode is
|
||||||
|
* honored exactly as the umask allows (it is no longer force-cleared). */
|
||||||
|
static void test_new_file_mode_honors_source_and_umask() {
|
||||||
|
const char* path = "test_new_file_mode.bin";
|
||||||
|
unlink(path);
|
||||||
|
FileMetadata m;
|
||||||
|
memset(&m, 0, sizeof(m));
|
||||||
|
m.mode = 0666; /* maximal group/other write in the source mode */
|
||||||
|
m.uid = geteuid();
|
||||||
|
m.gid = getegid();
|
||||||
|
|
||||||
|
bool ok = file_to_disk_secure_attrs(path, "x", 1, false, false, false, &m,
|
||||||
|
(FileAttrPolicy){false, false, false, false}, false, false,
|
||||||
|
false, NULL, false, false, NULL);
|
||||||
|
EXPECT_TRUE(ok);
|
||||||
|
struct stat st;
|
||||||
|
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||||
|
EXPECT_EQ_INT((int)(st.st_mode & 0777), (int)(0666 & ~(mode_t)file_process_umask()));
|
||||||
|
unlink(path);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Strict rsync parity for recreated special nodes: with -p the source mode is
|
||||||
|
* copied exactly (0777 -> 0777), and without -p the same source & ~umask base
|
||||||
|
* as any other new entry applies. The process umask is cleared so the source
|
||||||
|
* bits are what reaches mkfifo. */
|
||||||
|
static void test_special_fifo_mode_honors_source_and_umask_impl() {
|
||||||
|
const char* root = "test_special_mode_tmp";
|
||||||
|
const char* with_p = "test_special_mode_tmp/with_p.fifo";
|
||||||
|
const char* no_p = "test_special_mode_tmp/no_p.fifo";
|
||||||
|
unlink(with_p);
|
||||||
|
unlink(no_p);
|
||||||
|
rmdir(root);
|
||||||
|
EXPECT_EQ_INT(mkdir(root, 0700), 0);
|
||||||
|
|
||||||
|
Config* cfg = config_create();
|
||||||
|
EXPECT_NOT_NULL(cfg);
|
||||||
|
|
||||||
|
FileMetadata meta;
|
||||||
|
memset(&meta, 0, sizeof(meta));
|
||||||
|
meta.mode = S_IFIFO | 0777;
|
||||||
|
meta.uid = geteuid();
|
||||||
|
meta.gid = getegid();
|
||||||
|
meta.mtime_sec = 1000000000;
|
||||||
|
|
||||||
|
/* -p: the source mode (including group/other write) is copied exactly. */
|
||||||
|
File* f = file_create("with_p.fifo");
|
||||||
|
EXPECT_NOT_NULL(f);
|
||||||
|
f->is_special = true;
|
||||||
|
f->metadata = &meta;
|
||||||
|
cfg->preserve_specials = true;
|
||||||
|
cfg->preserve_perms = true;
|
||||||
|
cfg->use_metadata = true;
|
||||||
|
EXPECT_EQ_INT(file_save_to_disk_full(root, f, cfg), FILE_SAVE_WRITTEN);
|
||||||
|
struct stat st;
|
||||||
|
EXPECT_EQ_INT(lstat(with_p, &st), 0);
|
||||||
|
EXPECT_TRUE(S_ISFIFO(st.st_mode));
|
||||||
|
EXPECT_EQ_INT((int)(st.st_mode & 0777), 0777);
|
||||||
|
f->metadata = NULL;
|
||||||
|
file_destroy(f);
|
||||||
|
|
||||||
|
/* No -p: source & ~umask (umask is cleared, so 0777). */
|
||||||
|
f = file_create("no_p.fifo");
|
||||||
|
EXPECT_NOT_NULL(f);
|
||||||
|
f->is_special = true;
|
||||||
|
f->metadata = &meta;
|
||||||
|
cfg->preserve_perms = false;
|
||||||
|
EXPECT_EQ_INT(file_save_to_disk_full(root, f, cfg), FILE_SAVE_WRITTEN);
|
||||||
|
EXPECT_EQ_INT(lstat(no_p, &st), 0);
|
||||||
|
EXPECT_TRUE(S_ISFIFO(st.st_mode));
|
||||||
|
EXPECT_EQ_INT((int)(st.st_mode & 0777), 0777);
|
||||||
|
f->metadata = NULL;
|
||||||
|
file_destroy(f);
|
||||||
|
|
||||||
|
config_delete(cfg);
|
||||||
|
unlink(with_p);
|
||||||
|
unlink(no_p);
|
||||||
|
rmdir(root);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* The receiver daemon runs umask(0), so the source mode reaches mkfifo
|
||||||
|
* unmasked. Run the body with umask(0) and refresh the cached process umask so
|
||||||
|
* file_process_umask() agrees, then restore both. */
|
||||||
|
static void test_special_fifo_mode_honors_source_and_umask() {
|
||||||
|
mode_t saved_umask = umask(0);
|
||||||
|
file_umask_capture();
|
||||||
|
test_special_fifo_mode_honors_source_and_umask_impl();
|
||||||
|
umask(saved_umask);
|
||||||
|
file_umask_capture();
|
||||||
|
}
|
||||||
|
|
||||||
|
/* --specials recreates a unix-domain socket via mknod(S_IFSOCK), which Linux
|
||||||
|
* permits unprivileged. Without --specials the entry is skipped. */
|
||||||
|
static void test_special_socket_recreated() {
|
||||||
|
const char* root = "test_special_sock_tmp";
|
||||||
|
const char* sock = "test_special_sock_tmp/source.sock";
|
||||||
|
unlink(sock);
|
||||||
|
rmdir(root);
|
||||||
|
EXPECT_EQ_INT(mkdir(root, 0700), 0);
|
||||||
|
|
||||||
|
Config* cfg = config_create();
|
||||||
|
EXPECT_NOT_NULL(cfg);
|
||||||
|
FileMetadata meta;
|
||||||
|
memset(&meta, 0, sizeof(meta));
|
||||||
|
meta.mode = S_IFSOCK | 0600;
|
||||||
|
meta.uid = geteuid();
|
||||||
|
meta.gid = getegid();
|
||||||
|
|
||||||
|
File* f = file_create("source.sock");
|
||||||
|
EXPECT_NOT_NULL(f);
|
||||||
|
f->is_special = true;
|
||||||
|
f->metadata = &meta;
|
||||||
|
cfg->preserve_specials = true;
|
||||||
|
cfg->use_metadata = true;
|
||||||
|
EXPECT_EQ_INT(file_save_to_disk_full(root, f, cfg), FILE_SAVE_WRITTEN);
|
||||||
|
struct stat st;
|
||||||
|
EXPECT_EQ_INT(lstat(sock, &st), 0);
|
||||||
|
EXPECT_TRUE(S_ISSOCK(st.st_mode));
|
||||||
|
|
||||||
|
/* Without --specials the same entry is skipped, never a regular file. */
|
||||||
|
unlink(sock);
|
||||||
|
cfg->preserve_specials = false;
|
||||||
|
EXPECT_EQ_INT(file_save_to_disk_full(root, f, cfg), FILE_SAVE_SKIPPED);
|
||||||
|
EXPECT_EQ_INT(lstat(sock, &st), -1);
|
||||||
|
|
||||||
|
f->metadata = NULL;
|
||||||
|
file_destroy(f);
|
||||||
|
config_delete(cfg);
|
||||||
|
unlink(sock);
|
||||||
|
rmdir(root);
|
||||||
|
}
|
||||||
|
|
||||||
static void test_inplace_overwrite_truncates_shorter_payload() {
|
static void test_inplace_overwrite_truncates_shorter_payload() {
|
||||||
const char* root = "test_inplace_trunc_tmp";
|
const char* root = "test_inplace_trunc_tmp";
|
||||||
const char* path = "test_inplace_trunc_tmp/big.txt";
|
const char* path = "test_inplace_trunc_tmp/big.txt";
|
||||||
@@ -1125,34 +1396,27 @@ static void test_dir_entry_save_to_disk() {
|
|||||||
* receiver enables it from its own process (the standalone server's --trust-
|
* receiver enables it from its own process (the standalone server's --trust-
|
||||||
* sender CLI switch, which a client forwards as --remote-option=--trust-sender),
|
* sender CLI switch, which a client forwards as --remote-option=--trust-sender),
|
||||||
* so these tests force file_set_trust_sender(true) directly. Trust must RELAX
|
* so these tests force file_set_trust_sender(true) directly. Trust must RELAX
|
||||||
* only the redundant list-level re-validation (an escaping symlink TARGET is
|
* only the redundant list-level re-validation and must NEVER disable the
|
||||||
* copied verbatim, rsync -l parity) and must NEVER disable the low-level
|
* low-level fd-relative confinement floor: file_open_secure_parent's ".."
|
||||||
* fd-relative confinement floor: file_open_secure_parent's ".." rejection, the
|
* rejection, the O_NOFOLLOW parent walk, leaf/destination confinement, and the
|
||||||
* O_NOFOLLOW parent walk, leaf/destination confinement, and the ungated
|
* ungated has_path_traversal on the link's own placement path in
|
||||||
* has_path_traversal on the link's own placement path in file_symlink_at_secure
|
* file_symlink_at_secure stay hard. A hostile sender therefore still cannot
|
||||||
* stay hard. A hostile sender therefore still cannot place a file, directory
|
* place a file, directory or symlink outside the receive root. */
|
||||||
* or symlink outside the receive root even with trust on. */
|
|
||||||
|
|
||||||
static void test_trust_sender_relaxes_symlink_target() {
|
static void test_symlink_target_verbatim() {
|
||||||
const char* root = "test_trust_sender_root";
|
const char* root = "test_symlink_verbatim_root";
|
||||||
const char* link = "test_trust_sender_root/escape_link";
|
const char* link = "test_symlink_verbatim_root/escape_link";
|
||||||
unlink(link);
|
unlink(link);
|
||||||
rmdir(root);
|
rmdir(root);
|
||||||
EXPECT_EQ_INT(mkdir(root, 0755), 0);
|
EXPECT_EQ_INT(mkdir(root, 0755), 0);
|
||||||
|
|
||||||
/* Control: without trust an absolute (escaping) target is refused and the
|
/* rsync -l parity: a symlink target is stored verbatim, absolute or not; the
|
||||||
link is never placed. */
|
scanner's --safe-links/--copy-unsafe-links is what filters links. */
|
||||||
file_set_trust_sender(false);
|
file_set_trust_sender(false);
|
||||||
EXPECT_FALSE(file_symlink_at_secure(link, "/etc/passwd"));
|
|
||||||
struct stat st;
|
|
||||||
EXPECT_EQ_INT(lstat(link, &st), -1);
|
|
||||||
|
|
||||||
/* Trust ON: the escaping target is copied verbatim (rsync -l parity) ... */
|
|
||||||
file_set_trust_sender(true);
|
|
||||||
EXPECT_TRUE(file_symlink_at_secure(link, "/etc/passwd"));
|
EXPECT_TRUE(file_symlink_at_secure(link, "/etc/passwd"));
|
||||||
|
struct stat st;
|
||||||
EXPECT_EQ_INT(lstat(link, &st), 0);
|
EXPECT_EQ_INT(lstat(link, &st), 0);
|
||||||
EXPECT_TRUE(S_ISLNK(st.st_mode));
|
EXPECT_TRUE(S_ISLNK(st.st_mode));
|
||||||
/* ...but the link itself still lands beneath the receive root. */
|
|
||||||
char target[128];
|
char target[128];
|
||||||
ssize_t target_len = readlink(link, target, sizeof(target) - 1);
|
ssize_t target_len = readlink(link, target, sizeof(target) - 1);
|
||||||
EXPECT_TRUE(target_len > 0);
|
EXPECT_TRUE(target_len > 0);
|
||||||
@@ -1163,10 +1427,10 @@ static void test_trust_sender_relaxes_symlink_target() {
|
|||||||
}
|
}
|
||||||
unlink(link);
|
unlink(link);
|
||||||
|
|
||||||
/* Same relaxation through the real save funnel (file_save_to_disk_full). */
|
/* The same through the real save funnel: verbatim by default. */
|
||||||
Config* config = config_create();
|
Config* config = config_create();
|
||||||
EXPECT_NOT_NULL(config);
|
EXPECT_NOT_NULL(config);
|
||||||
const char* save_link = "test_trust_sender_root/save_link";
|
const char* save_link = "test_symlink_verbatim_root/save_link";
|
||||||
unlink(save_link);
|
unlink(save_link);
|
||||||
|
|
||||||
File* sym = file_create("save_link");
|
File* sym = file_create("save_link");
|
||||||
@@ -1176,10 +1440,6 @@ static void test_trust_sender_relaxes_symlink_target() {
|
|||||||
EXPECT_NOT_NULL(sym->symlink_target);
|
EXPECT_NOT_NULL(sym->symlink_target);
|
||||||
|
|
||||||
file_set_trust_sender(false);
|
file_set_trust_sender(false);
|
||||||
EXPECT_EQ_INT(file_save_to_disk_full(root, sym, config), FILE_SAVE_SKIPPED);
|
|
||||||
EXPECT_EQ_INT(lstat(save_link, &st), -1);
|
|
||||||
|
|
||||||
file_set_trust_sender(true);
|
|
||||||
EXPECT_EQ_INT(file_save_to_disk_full(root, sym, config), FILE_SAVE_WRITTEN);
|
EXPECT_EQ_INT(file_save_to_disk_full(root, sym, config), FILE_SAVE_WRITTEN);
|
||||||
EXPECT_EQ_INT(lstat(save_link, &st), 0);
|
EXPECT_EQ_INT(lstat(save_link, &st), 0);
|
||||||
EXPECT_TRUE(S_ISLNK(st.st_mode));
|
EXPECT_TRUE(S_ISLNK(st.st_mode));
|
||||||
@@ -1305,7 +1565,7 @@ void test_trust_sender() {
|
|||||||
helper), so a later group never inherits a stray trust/authorized-root
|
helper), so a later group never inherits a stray trust/authorized-root
|
||||||
policy. */
|
policy. */
|
||||||
file_set_trust_sender(false);
|
file_set_trust_sender(false);
|
||||||
test_trust_sender_relaxes_symlink_target();
|
test_symlink_target_verbatim();
|
||||||
test_trust_sender_confines_hostile_paths();
|
test_trust_sender_confines_hostile_paths();
|
||||||
test_trust_sender_authorized_root_confinement();
|
test_trust_sender_authorized_root_confinement();
|
||||||
file_set_trust_sender(false);
|
file_set_trust_sender(false);
|
||||||
@@ -1335,7 +1595,8 @@ static void test_file_write_to_disk_sparse_preserves_holes() {
|
|||||||
buf[size - 1 - i] = (unsigned char)((i * 7) % 253);
|
buf[size - 1 - i] = (unsigned char)((i * 7) % 253);
|
||||||
}
|
}
|
||||||
|
|
||||||
EXPECT_TRUE(file_to_disk_secure(path, buf, size, false, true, false, NULL, false, NULL));
|
EXPECT_TRUE(
|
||||||
|
file_to_disk_secure(path, buf, size, false, true, false, NULL, (FileAttrPolicy){0}, NULL));
|
||||||
|
|
||||||
/* Logical size must equal data_size exactly. */
|
/* Logical size must equal data_size exactly. */
|
||||||
struct stat st;
|
struct stat st;
|
||||||
@@ -1400,8 +1661,9 @@ static void test_file_write_to_disk_partial_retention() {
|
|||||||
m.mtime_nsec = 2000000000; /* invalid: forces futimens EINVAL after the write */
|
m.mtime_nsec = 2000000000; /* invalid: forces futimens EINVAL after the write */
|
||||||
m.atime_valid = false;
|
m.atime_valid = false;
|
||||||
m.crtime_valid = false;
|
m.crtime_valid = false;
|
||||||
bool ok = file_to_disk_secure_attrs(path, content, strlen(content), false, false, true, &m, false,
|
bool ok = file_to_disk_secure_attrs(path, content, strlen(content), false, false, true, &m,
|
||||||
false, false, false, NULL, false, true, NULL);
|
(FileAttrPolicy){true, true, false, false}, false, false,
|
||||||
|
false, NULL, false, true, NULL);
|
||||||
EXPECT_FALSE(ok); /* the write itself succeeded, but metadata restore failed */
|
EXPECT_FALSE(ok); /* the write itself succeeded, but metadata restore failed */
|
||||||
/* Retained: the already-written temp now sits at the destination path. */
|
/* Retained: the already-written temp now sits at the destination path. */
|
||||||
int fd = open(path, O_RDONLY);
|
int fd = open(path, O_RDONLY);
|
||||||
@@ -1419,8 +1681,9 @@ static void test_file_write_to_disk_partial_retention() {
|
|||||||
unlink(path);
|
unlink(path);
|
||||||
|
|
||||||
/* Same failure with keep_partial=false: temp is unlinked, nothing retained. */
|
/* Same failure with keep_partial=false: temp is unlinked, nothing retained. */
|
||||||
ok = file_to_disk_secure_attrs(path, content, strlen(content), false, false, true, &m, false,
|
ok = file_to_disk_secure_attrs(path, content, strlen(content), false, false, true, &m,
|
||||||
false, false, false, NULL, false, false, NULL);
|
(FileAttrPolicy){true, true, false, false}, false, false, false,
|
||||||
|
NULL, false, false, NULL);
|
||||||
EXPECT_FALSE(ok);
|
EXPECT_FALSE(ok);
|
||||||
EXPECT_TRUE(access(path, F_OK) == -1);
|
EXPECT_TRUE(access(path, F_OK) == -1);
|
||||||
}
|
}
|
||||||
@@ -1440,19 +1703,35 @@ static void test_dir_time_list() {
|
|||||||
dir_time_list_init(&list);
|
dir_time_list_init(&list);
|
||||||
EXPECT_EQ_INT((int)list.count, 0);
|
EXPECT_EQ_INT((int)list.count, 0);
|
||||||
FileMetadata metadata = {.mtime_sec = 1000000000, .mtime_nsec = 0};
|
FileMetadata metadata = {.mtime_sec = 1000000000, .mtime_nsec = 0};
|
||||||
EXPECT_TRUE(dir_time_list_add(&list, "sub", &metadata));
|
EXPECT_TRUE(dir_time_list_add(&list, "sub", &metadata, NULL));
|
||||||
EXPECT_TRUE(dir_time_list_add(&list, "sub", &metadata));
|
/* A captured xattr block is deep-copied into the list. */
|
||||||
|
FileXattrList* xl = xattr_list_new();
|
||||||
|
EXPECT_NOT_NULL(xl);
|
||||||
|
EXPECT_TRUE(xattr_list_append(xl, "user.dir", "v", 1));
|
||||||
|
EXPECT_TRUE(dir_time_list_add(&list, "sub", &metadata, xl));
|
||||||
|
xattr_list_free(xl); /* the list owns its own copy now */
|
||||||
EXPECT_EQ_INT((int)list.count, 2);
|
EXPECT_EQ_INT((int)list.count, 2);
|
||||||
|
EXPECT_NOT_NULL(list.xattrs);
|
||||||
|
EXPECT_NOT_NULL(list.xattrs[1]);
|
||||||
|
EXPECT_EQ_INT(list.xattrs[1]->count, 1);
|
||||||
|
EXPECT_EQ_STR(list.xattrs[1]->items[0].name, "user.dir");
|
||||||
|
EXPECT_NULL(list.xattrs[0]);
|
||||||
|
|
||||||
dir_time_list_apply(&list, root);
|
Config* cfg = config_create();
|
||||||
|
EXPECT_NOT_NULL(cfg);
|
||||||
|
cfg->use_metadata = true;
|
||||||
|
cfg->preserve_times = true;
|
||||||
|
dir_metadata_list_apply(&list, root, cfg);
|
||||||
struct stat st;
|
struct stat st;
|
||||||
EXPECT_EQ_INT(stat(sub, &st), 0);
|
EXPECT_EQ_INT(stat(sub, &st), 0);
|
||||||
EXPECT_EQ_INT((int)st.st_mtime, 1000000000);
|
EXPECT_EQ_INT((int)st.st_mtime, 1000000000);
|
||||||
|
config_delete(cfg);
|
||||||
|
|
||||||
dir_time_list_free(&list);
|
dir_time_list_free(&list);
|
||||||
EXPECT_EQ_INT((int)list.count, 0);
|
EXPECT_EQ_INT((int)list.count, 0);
|
||||||
EXPECT_NULL(list.paths);
|
EXPECT_NULL(list.paths);
|
||||||
EXPECT_NULL(list.entries);
|
EXPECT_NULL(list.entries);
|
||||||
|
EXPECT_NULL(list.xattrs);
|
||||||
|
|
||||||
rmdir(sub);
|
rmdir(sub);
|
||||||
rmdir(root);
|
rmdir(root);
|
||||||
@@ -1477,14 +1756,14 @@ static void test_dir_time_list_cap() {
|
|||||||
for (size_t i = 0; i < MAX_DIR_TIME_ENTRIES + 1 && !rejected; i++) {
|
for (size_t i = 0; i < MAX_DIR_TIME_ENTRIES + 1 && !rejected; i++) {
|
||||||
size_t before_count = list.count;
|
size_t before_count = list.count;
|
||||||
size_t before_bytes = list.bytes;
|
size_t before_bytes = list.bytes;
|
||||||
if (!dir_time_list_add(&list, path, &metadata)) {
|
if (!dir_time_list_add(&list, path, &metadata, NULL)) {
|
||||||
rejected = true;
|
rejected = true;
|
||||||
/* The rejected add must not have partially mutated the list. */
|
/* The rejected add must not have partially mutated the list. */
|
||||||
EXPECT_TRUE(list.count == before_count);
|
EXPECT_TRUE(list.count == before_count);
|
||||||
EXPECT_TRUE(list.bytes == before_bytes);
|
EXPECT_TRUE(list.bytes == before_bytes);
|
||||||
} else {
|
} else {
|
||||||
EXPECT_TRUE(list.count == before_count + 1);
|
EXPECT_TRUE(list.count == before_count + 1);
|
||||||
EXPECT_TRUE(list.bytes == before_bytes + path_len + sizeof(FileMetadata) + sizeof(char*));
|
EXPECT_TRUE(list.bytes == before_bytes + path_len + sizeof(FileMetadata) + 2 * sizeof(char*));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
EXPECT_TRUE(rejected);
|
EXPECT_TRUE(rejected);
|
||||||
@@ -1648,6 +1927,175 @@ static void test_keep_dirlinks_secure_open() {
|
|||||||
file_set_keep_dirlinks(false);
|
file_set_keep_dirlinks(false);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Build an ArrayList of str_dup'd strings (NULL on allocation failure). */
|
||||||
|
static ArrayList* make_manifest_string_list(const char* const* entries, int count) {
|
||||||
|
ArrayList* list = array_list_create(free);
|
||||||
|
if (!list)
|
||||||
|
return NULL;
|
||||||
|
for (int i = 0; i < count; i++) {
|
||||||
|
char* dup = str_dup(entries[i]);
|
||||||
|
if (!dup || !array_list_add(list, dup)) {
|
||||||
|
free(dup);
|
||||||
|
array_list_delete(list);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return list;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Regression (#3): a non-empty --delete-missing-args directory charges each
|
||||||
|
* removed entry exactly once. The directory itself must not be counted twice;
|
||||||
|
* if it were, `deleted` would exceed --max-delete and the extras walk would
|
||||||
|
* underflow its remaining budget and delete past the user's cap. */
|
||||||
|
static void test_manifest_delete_missing_dir_budget_double_count() {
|
||||||
|
char root[PATH_MAX];
|
||||||
|
snprintf(root, sizeof(root), "/tmp/fastsync_mgdir_%d", (int)getpid());
|
||||||
|
char* gone = path_cat(root, "gone");
|
||||||
|
char* gone_file = path_cat(gone, "f0");
|
||||||
|
char* extra = path_cat(root, "extra.txt");
|
||||||
|
EXPECT_NOT_NULL(gone);
|
||||||
|
EXPECT_NOT_NULL(gone_file);
|
||||||
|
EXPECT_NOT_NULL(extra);
|
||||||
|
mkdir(root, 0755);
|
||||||
|
mkdir(gone, 0755);
|
||||||
|
EXPECT_EQ_INT(access(extra, F_OK), -1);
|
||||||
|
EXPECT_TRUE(file_write_to_disk(extra, "extra", 5, false, false));
|
||||||
|
/* The missing-arg directory holds N-1 == 2 entries; with the directory itself
|
||||||
|
that is exactly --max-delete=3. */
|
||||||
|
EXPECT_TRUE(file_write_to_disk(gone_file, "x", 1, false, false));
|
||||||
|
char* gone_file2 = path_cat(gone, "f1");
|
||||||
|
EXPECT_TRUE(gone_file2 != NULL && file_write_to_disk(gone_file2, "x", 1, false, false));
|
||||||
|
|
||||||
|
Config* cfg = config_create();
|
||||||
|
EXPECT_NOT_NULL(cfg);
|
||||||
|
cfg->receive_root_directory = str_dup(root);
|
||||||
|
cfg->use_delete = true;
|
||||||
|
cfg->delete_missing_args = true;
|
||||||
|
cfg->max_delete = 3;
|
||||||
|
|
||||||
|
const char* missing_names[] = {"gone"};
|
||||||
|
const char* synced[] = {"."};
|
||||||
|
DeleteManifest manifest = {0};
|
||||||
|
manifest.keeps = make_manifest_string_list(NULL, 0);
|
||||||
|
manifest.missing = make_manifest_string_list(missing_names, 1);
|
||||||
|
manifest.dirs = make_manifest_string_list(synced, 1);
|
||||||
|
EXPECT_NOT_NULL(manifest.keeps);
|
||||||
|
EXPECT_NOT_NULL(manifest.missing);
|
||||||
|
EXPECT_NOT_NULL(manifest.dirs);
|
||||||
|
|
||||||
|
DeleteCommitResult result = manifest_delete_all(cfg, &manifest);
|
||||||
|
EXPECT_EQ_INT((int)result, (int)DELETE_COMMIT_LIMIT_REACHED);
|
||||||
|
/* The whole missing-arg directory is gone (dir + its 2 entries == 3). */
|
||||||
|
EXPECT_EQ_INT(access(gone, F_OK), -1);
|
||||||
|
/* The saturated budget must leave the in-scope extra untouched. */
|
||||||
|
EXPECT_EQ_INT(access(extra, F_OK), 0);
|
||||||
|
|
||||||
|
array_list_delete(manifest.keeps);
|
||||||
|
array_list_delete(manifest.missing);
|
||||||
|
array_list_delete(manifest.dirs);
|
||||||
|
config_delete(cfg);
|
||||||
|
unlink(extra);
|
||||||
|
free(gone);
|
||||||
|
free(gone_file);
|
||||||
|
free(gone_file2);
|
||||||
|
free(extra);
|
||||||
|
rmdir(root);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Blocker #7: when the receive root is "/", every absolute basis path is below
|
||||||
|
it and its child relative form must drop only the single leading slash. */
|
||||||
|
static void test_basis_delete_relative_root_slash() {
|
||||||
|
Config* cfg = config_create();
|
||||||
|
EXPECT_NOT_NULL(cfg);
|
||||||
|
cfg->receive_root_directory = str_dup("/");
|
||||||
|
|
||||||
|
char* rel = file_receive_basis_delete_relative(cfg, "/a");
|
||||||
|
EXPECT_NOT_NULL(rel);
|
||||||
|
EXPECT_EQ_STR(rel, "a");
|
||||||
|
free(rel);
|
||||||
|
rel = file_receive_basis_delete_relative(cfg, "/a/b");
|
||||||
|
EXPECT_NOT_NULL(rel);
|
||||||
|
EXPECT_EQ_STR(rel, "a/b");
|
||||||
|
free(rel);
|
||||||
|
/* The root itself is not a child. */
|
||||||
|
EXPECT_NULL(file_receive_basis_delete_relative(cfg, "/"));
|
||||||
|
/* A relative entry is already root-relative. */
|
||||||
|
rel = file_receive_basis_delete_relative(cfg, "x/y");
|
||||||
|
EXPECT_NOT_NULL(rel);
|
||||||
|
EXPECT_EQ_STR(rel, "x/y");
|
||||||
|
free(rel);
|
||||||
|
/* An absolute path outside a non-"/" root is unreachable. */
|
||||||
|
free(cfg->receive_root_directory);
|
||||||
|
cfg->receive_root_directory = str_dup("/root");
|
||||||
|
EXPECT_NULL(file_receive_basis_delete_relative(cfg, "/other/a"));
|
||||||
|
rel = file_receive_basis_delete_relative(cfg, "/root/a");
|
||||||
|
EXPECT_NOT_NULL(rel);
|
||||||
|
EXPECT_EQ_STR(rel, "a");
|
||||||
|
free(rel);
|
||||||
|
config_delete(cfg);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Blocker #6: -n --delete would-delete enumeration must normalize an absolute
|
||||||
|
basis directory under the receive root exactly like the real commit path, so
|
||||||
|
the basis snapshot is protected rather than reported as a deletable extra. */
|
||||||
|
static void test_manifest_would_delete_protects_absolute_basis() {
|
||||||
|
char root[PATH_MAX];
|
||||||
|
snprintf(root, sizeof(root), "/tmp/fastsync_wdbasis_%d", (int)getpid());
|
||||||
|
char* basis = path_cat(root, "basis");
|
||||||
|
char* basis_file = path_cat(basis, "snapshot.bin");
|
||||||
|
char* extra = path_cat(root, "extra.txt");
|
||||||
|
EXPECT_NOT_NULL(basis);
|
||||||
|
EXPECT_NOT_NULL(basis_file);
|
||||||
|
EXPECT_NOT_NULL(extra);
|
||||||
|
mkdir(root, 0755);
|
||||||
|
mkdir(basis, 0755);
|
||||||
|
EXPECT_TRUE(file_write_to_disk(basis_file, "x", 1, false, false));
|
||||||
|
EXPECT_TRUE(file_write_to_disk(extra, "e", 1, false, false));
|
||||||
|
|
||||||
|
Config* cfg = config_create();
|
||||||
|
EXPECT_NOT_NULL(cfg);
|
||||||
|
cfg->receive_root_directory = str_dup(root);
|
||||||
|
cfg->use_delete = true;
|
||||||
|
EXPECT_EQ_INT(config_basis_append(cfg, BASIS_DEST_COMPARE, basis), 0);
|
||||||
|
|
||||||
|
const char* synced[] = {"."};
|
||||||
|
DeleteManifest manifest = {0};
|
||||||
|
manifest.keeps = make_manifest_string_list(NULL, 0);
|
||||||
|
manifest.protected = make_manifest_string_list(NULL, 0);
|
||||||
|
manifest.dirs = make_manifest_string_list(synced, 1);
|
||||||
|
EXPECT_NOT_NULL(manifest.keeps);
|
||||||
|
EXPECT_NOT_NULL(manifest.protected);
|
||||||
|
EXPECT_NOT_NULL(manifest.dirs);
|
||||||
|
ArrayList* out = array_list_create(free);
|
||||||
|
EXPECT_NOT_NULL(out);
|
||||||
|
size_t count = 0;
|
||||||
|
EXPECT_TRUE(manifest_would_delete_list(cfg, &manifest, out, &count));
|
||||||
|
bool saw_basis = false;
|
||||||
|
bool saw_extra = false;
|
||||||
|
for (int i = 0; i < out->size; i++) {
|
||||||
|
const char* p = (const char*)out->items[i];
|
||||||
|
if (strcmp(p, "basis") == 0 || strncmp(p, "basis/", 6) == 0)
|
||||||
|
saw_basis = true;
|
||||||
|
if (strcmp(p, "extra.txt") == 0)
|
||||||
|
saw_extra = true;
|
||||||
|
}
|
||||||
|
EXPECT_FALSE(saw_basis);
|
||||||
|
EXPECT_TRUE(saw_extra);
|
||||||
|
|
||||||
|
array_list_delete(out);
|
||||||
|
array_list_delete(manifest.keeps);
|
||||||
|
array_list_delete(manifest.protected);
|
||||||
|
array_list_delete(manifest.dirs);
|
||||||
|
config_delete(cfg);
|
||||||
|
unlink(basis_file);
|
||||||
|
rmdir(basis);
|
||||||
|
unlink(extra);
|
||||||
|
rmdir(root);
|
||||||
|
free(basis);
|
||||||
|
free(basis_file);
|
||||||
|
free(extra);
|
||||||
|
}
|
||||||
|
|
||||||
void test_file() {
|
void test_file() {
|
||||||
test_file_create();
|
test_file_create();
|
||||||
test_file_special_rdev_valid();
|
test_file_special_rdev_valid();
|
||||||
@@ -1661,6 +2109,7 @@ void test_file() {
|
|||||||
test_file_save_to_disk_ignore_existing();
|
test_file_save_to_disk_ignore_existing();
|
||||||
test_file_save_to_disk_ignore_existing_entry_types();
|
test_file_save_to_disk_ignore_existing_entry_types();
|
||||||
test_file_save_to_disk_partial_install();
|
test_file_save_to_disk_partial_install();
|
||||||
|
test_file_save_to_disk_temp_dir_confined();
|
||||||
test_file_save_to_disk_reports_skips();
|
test_file_save_to_disk_reports_skips();
|
||||||
test_file_write_to_disk_sparse_preserves_holes();
|
test_file_write_to_disk_sparse_preserves_holes();
|
||||||
test_file_write_to_disk_partial_retention();
|
test_file_write_to_disk_partial_retention();
|
||||||
@@ -1694,7 +2143,14 @@ void test_file() {
|
|||||||
test_keep_dirlinks_secure_open();
|
test_keep_dirlinks_secure_open();
|
||||||
test_inplace_overwrite_clears_special_mode_bits();
|
test_inplace_overwrite_clears_special_mode_bits();
|
||||||
test_inplace_overwrite_metadata_strips_special_bits();
|
test_inplace_overwrite_metadata_strips_special_bits();
|
||||||
|
test_atomic_no_perms_preserves_destination_mode();
|
||||||
|
test_new_file_mode_honors_source_and_umask();
|
||||||
|
test_special_fifo_mode_honors_source_and_umask();
|
||||||
|
test_special_socket_recreated();
|
||||||
test_inplace_overwrite_truncates_shorter_payload();
|
test_inplace_overwrite_truncates_shorter_payload();
|
||||||
test_inplace_refuses_fifo_destination();
|
test_inplace_refuses_fifo_destination();
|
||||||
test_inplace_refuses_device_destination();
|
test_inplace_refuses_device_destination();
|
||||||
|
test_manifest_delete_missing_dir_budget_double_count();
|
||||||
|
test_basis_delete_relative_root_slash();
|
||||||
|
test_manifest_would_delete_protects_absolute_basis();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -148,6 +148,50 @@ static void test_ancestor_and_descendant_queries() {
|
|||||||
remove(path);
|
remove(path);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* The delete-walker's synchronized-directory predicate: a directory is in scope
|
||||||
|
only when it is a listed directory or lies below one, NOT when it is merely an
|
||||||
|
implied parent of a listed file. */
|
||||||
|
static void test_dir_in_scope() {
|
||||||
|
char err[160];
|
||||||
|
|
||||||
|
/* NULL set / empty list semantics. */
|
||||||
|
EXPECT_TRUE(file_list_dir_in_scope(NULL, "anything"));
|
||||||
|
|
||||||
|
const char* path = "test_file_list_dirscope.txt";
|
||||||
|
write_list(path, "d1/leaf.txt\n");
|
||||||
|
FileListSet* set = file_list_load(path, false, err, sizeof(err));
|
||||||
|
EXPECT_NOT_NULL(set);
|
||||||
|
/* d1 is only an implied parent of a listed FILE: not synchronized. */
|
||||||
|
EXPECT_FALSE(file_list_dir_in_scope(set, "d1"));
|
||||||
|
EXPECT_FALSE(file_list_dir_in_scope(set, "d1/sub"));
|
||||||
|
EXPECT_FALSE(file_list_dir_in_scope(set, "other"));
|
||||||
|
file_list_destroy(set);
|
||||||
|
remove(path);
|
||||||
|
|
||||||
|
/* A listed DIRECTORY synchronizes itself and its whole subtree. */
|
||||||
|
write_list(path, "d1/\nother\n");
|
||||||
|
set = file_list_load(path, false, err, sizeof(err));
|
||||||
|
EXPECT_NOT_NULL(set);
|
||||||
|
EXPECT_TRUE(file_list_dir_in_scope(set, "d1"));
|
||||||
|
EXPECT_TRUE(file_list_dir_in_scope(set, "d1/sub/deep"));
|
||||||
|
EXPECT_TRUE(file_list_dir_in_scope(set, "other"));
|
||||||
|
EXPECT_TRUE(file_list_dir_in_scope(set, "other/x"));
|
||||||
|
EXPECT_FALSE(file_list_dir_in_scope(set, "d2"));
|
||||||
|
EXPECT_FALSE(file_list_dir_in_scope(set, "d1x")); /* component boundary */
|
||||||
|
EXPECT_FALSE(file_list_dir_in_scope(set, ""));
|
||||||
|
file_list_destroy(set);
|
||||||
|
remove(path);
|
||||||
|
|
||||||
|
/* "." lists the whole tree. */
|
||||||
|
write_list(path, ".\n");
|
||||||
|
set = file_list_load(path, false, err, sizeof(err));
|
||||||
|
EXPECT_NOT_NULL(set);
|
||||||
|
EXPECT_TRUE(file_list_dir_in_scope(set, ""));
|
||||||
|
EXPECT_TRUE(file_list_dir_in_scope(set, "anything/at/all"));
|
||||||
|
file_list_destroy(set);
|
||||||
|
remove(path);
|
||||||
|
}
|
||||||
|
|
||||||
/* Regression for the remote OOM: an adversarial --files-from entry made of a
|
/* Regression for the remote OOM: an adversarial --files-from entry made of a
|
||||||
very deep chain of repeated components must be indexed with memory
|
very deep chain of repeated components must be indexed with memory
|
||||||
proportional to the entry count. The old implementation stored one copied
|
proportional to the entry count. The old implementation stored one copied
|
||||||
@@ -219,6 +263,7 @@ static void test_oversized_entry_rejected() {
|
|||||||
void test_file_list() {
|
void test_file_list() {
|
||||||
test_membership_matches_reference();
|
test_membership_matches_reference();
|
||||||
test_ancestor_and_descendant_queries();
|
test_ancestor_and_descendant_queries();
|
||||||
|
test_dir_in_scope();
|
||||||
test_deep_paths_are_bounded();
|
test_deep_paths_are_bounded();
|
||||||
test_oversized_entry_rejected();
|
test_oversized_entry_rejected();
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,94 @@
|
|||||||
|
#include "test_format.h"
|
||||||
|
#include "format.h"
|
||||||
|
#include "test_utils.h"
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <string.h>
|
||||||
|
#include <sys/socket.h>
|
||||||
|
#include <time.h>
|
||||||
|
#include <unistd.h>
|
||||||
|
|
||||||
|
static void expect_big_num(unsigned long long value, bool human, const char* expected) {
|
||||||
|
char buffer[64];
|
||||||
|
EXPECT_TRUE(format_big_num(value, human, buffer, sizeof(buffer)));
|
||||||
|
EXPECT_EQ_STR(buffer, expected);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_human_size_decimal() {
|
||||||
|
/* Values below 1000 print verbatim; larger values use the largest unit that
|
||||||
|
* keeps the value below 1000 and exactly two decimals (rsync human_num). */
|
||||||
|
expect_big_num(0, true, "0");
|
||||||
|
expect_big_num(999, true, "999");
|
||||||
|
expect_big_num(1000, true, "1.00K");
|
||||||
|
expect_big_num(1500, true, "1.50K");
|
||||||
|
expect_big_num(9999, true, "10.00K");
|
||||||
|
expect_big_num(999999, true, "1000.00K");
|
||||||
|
expect_big_num(1000000, true, "1.00M");
|
||||||
|
expect_big_num(1500000, true, "1.50M");
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_big_num_grouping() {
|
||||||
|
/* Non-human numbers are comma-grouped every three digits (rsync big_num). */
|
||||||
|
expect_big_num(0, false, "0");
|
||||||
|
expect_big_num(1, false, "1");
|
||||||
|
expect_big_num(999, false, "999");
|
||||||
|
expect_big_num(1000, false, "1,000");
|
||||||
|
expect_big_num(4096, false, "4,096");
|
||||||
|
expect_big_num(1234567, false, "1,234,567");
|
||||||
|
expect_big_num(1000000000ULL, false, "1,000,000,000");
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_datetime_format() {
|
||||||
|
char buffer[32];
|
||||||
|
time_t when = 1700000000;
|
||||||
|
EXPECT_TRUE(format_rsync_datetime(when, true, buffer, sizeof(buffer)));
|
||||||
|
/* %M shape: YYYY/MM/DD-HH:MM:SS */
|
||||||
|
EXPECT_EQ_INT(strlen(buffer), 19);
|
||||||
|
EXPECT_EQ_INT(buffer[4], '/');
|
||||||
|
EXPECT_EQ_INT(buffer[7], '/');
|
||||||
|
EXPECT_EQ_INT(buffer[10], '-');
|
||||||
|
EXPECT_EQ_INT(buffer[13], ':');
|
||||||
|
EXPECT_EQ_INT(buffer[16], ':');
|
||||||
|
|
||||||
|
char space_form[32];
|
||||||
|
EXPECT_TRUE(format_rsync_datetime(when, false, space_form, sizeof(space_form)));
|
||||||
|
EXPECT_EQ_INT(space_form[10], ' ');
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_dest_state_roundtrip() {
|
||||||
|
/* The wire codec is exercised over a socketpair so the real send/receive
|
||||||
|
* primitives run. */
|
||||||
|
int fds[2];
|
||||||
|
if (socketpair(AF_UNIX, SOCK_STREAM, 0, fds) != 0)
|
||||||
|
return;
|
||||||
|
OutputDestState out;
|
||||||
|
memset(&out, 0, sizeof(out));
|
||||||
|
out.known = true;
|
||||||
|
out.existed = true;
|
||||||
|
out.size = 123456789ULL;
|
||||||
|
out.mtime_sec = 1700000000;
|
||||||
|
out.mtime_nsec = 123456789;
|
||||||
|
out.mode = 0100644;
|
||||||
|
out.uid = 1000;
|
||||||
|
out.gid = 1000;
|
||||||
|
OutputDestState in;
|
||||||
|
memset(&in, 0, sizeof(in));
|
||||||
|
EXPECT_TRUE(format_dest_state_send(fds[0], &out));
|
||||||
|
EXPECT_TRUE(format_dest_state_receive(fds[1], &in));
|
||||||
|
EXPECT_TRUE(in.known);
|
||||||
|
EXPECT_TRUE(in.existed);
|
||||||
|
EXPECT_TRUE(in.size == out.size);
|
||||||
|
EXPECT_TRUE(in.mtime_sec == out.mtime_sec);
|
||||||
|
EXPECT_TRUE(in.mtime_nsec == out.mtime_nsec);
|
||||||
|
EXPECT_TRUE(in.mode == out.mode);
|
||||||
|
EXPECT_TRUE(in.uid == out.uid);
|
||||||
|
EXPECT_TRUE(in.gid == out.gid);
|
||||||
|
close(fds[0]);
|
||||||
|
close(fds[1]);
|
||||||
|
}
|
||||||
|
|
||||||
|
void test_format(void) {
|
||||||
|
test_human_size_decimal();
|
||||||
|
test_big_num_grouping();
|
||||||
|
test_datetime_format();
|
||||||
|
test_dest_state_roundtrip();
|
||||||
|
}
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
#ifndef TEST_FORMAT_H
|
||||||
|
#define TEST_FORMAT_H
|
||||||
|
|
||||||
|
void test_format(void);
|
||||||
|
|
||||||
|
#endif
|
||||||
+23
-14
@@ -18,6 +18,10 @@
|
|||||||
|
|
||||||
/* P8 config-frame tail: super_mode (4) + copy-as presence (4) + uid (4) + gid (4). */
|
/* P8 config-frame tail: super_mode (4) + copy-as presence (4) + uid (4) + gid (4). */
|
||||||
#define P8_TAIL_BYTES 16
|
#define P8_TAIL_BYTES 16
|
||||||
|
/* Bytes after the P8 tail: report_dest_info (4), report_stats (4, wire-stats
|
||||||
|
* wave) and compression_algo (4, codec wave). The P8 fields sit this many
|
||||||
|
* bytes before the end of the frame. */
|
||||||
|
#define POST_P8_TAIL_BYTES 12
|
||||||
|
|
||||||
/* Smoke test for chunk_deserialize fuzz target */
|
/* Smoke test for chunk_deserialize fuzz target */
|
||||||
static void test_fuzz_chunk_deserialize() {
|
static void test_fuzz_chunk_deserialize() {
|
||||||
@@ -320,7 +324,7 @@ static void test_fuzz_config_receive_p8_tail() {
|
|||||||
size_t len = 0;
|
size_t len = 0;
|
||||||
bool captured = capture_config_frame(c, &frame, &len);
|
bool captured = capture_config_frame(c, &frame, &len);
|
||||||
config_delete(c);
|
config_delete(c);
|
||||||
if (!captured || len <= P8_TAIL_BYTES) {
|
if (!captured || len <= P8_TAIL_BYTES + POST_P8_TAIL_BYTES) {
|
||||||
free(frame);
|
free(frame);
|
||||||
EXPECT_TRUE(false);
|
EXPECT_TRUE(false);
|
||||||
return;
|
return;
|
||||||
@@ -334,31 +338,31 @@ static void test_fuzz_config_receive_p8_tail() {
|
|||||||
|
|
||||||
/* super_mode outside the 0..2 tri-state is refused. */
|
/* super_mode outside the 0..2 tri-state is refused. */
|
||||||
memcpy(mut, frame, len);
|
memcpy(mut, frame, len);
|
||||||
put_i32(mut, len - P8_TAIL_BYTES, 99);
|
put_i32(mut, len - POST_P8_TAIL_BYTES - P8_TAIL_BYTES, 99);
|
||||||
EXPECT_FALSE(receive_config_frame(mut, len));
|
EXPECT_FALSE(receive_config_frame(mut, len));
|
||||||
put_i32(mut, len - P8_TAIL_BYTES, -1);
|
put_i32(mut, len - POST_P8_TAIL_BYTES - P8_TAIL_BYTES, -1);
|
||||||
EXPECT_FALSE(receive_config_frame(mut, len));
|
EXPECT_FALSE(receive_config_frame(mut, len));
|
||||||
|
|
||||||
/* A negative (sentinel) and an extreme copy-as uid/gid are refused. */
|
/* A negative (sentinel) and an extreme copy-as uid/gid are refused. */
|
||||||
memcpy(mut, frame, len);
|
memcpy(mut, frame, len);
|
||||||
put_i32(mut, len - P8_TAIL_BYTES, SUPER_MODE_AUTO);
|
put_i32(mut, len - POST_P8_TAIL_BYTES - P8_TAIL_BYTES, SUPER_MODE_AUTO);
|
||||||
put_i32(mut, len - P8_TAIL_BYTES + 4, 1);
|
put_i32(mut, len - POST_P8_TAIL_BYTES - P8_TAIL_BYTES + 4, 1);
|
||||||
put_i32(mut, len - P8_TAIL_BYTES + 8, -1);
|
put_i32(mut, len - POST_P8_TAIL_BYTES - P8_TAIL_BYTES + 8, -1);
|
||||||
put_i32(mut, len - P8_TAIL_BYTES + 12, 0);
|
put_i32(mut, len - POST_P8_TAIL_BYTES - P8_TAIL_BYTES + 12, 0);
|
||||||
EXPECT_FALSE(receive_config_frame(mut, len));
|
EXPECT_FALSE(receive_config_frame(mut, len));
|
||||||
put_i32(mut, len - P8_TAIL_BYTES + 8, 0);
|
put_i32(mut, len - POST_P8_TAIL_BYTES - P8_TAIL_BYTES + 8, 0);
|
||||||
put_i32(mut, len - P8_TAIL_BYTES + 12, INT32_MIN);
|
put_i32(mut, len - POST_P8_TAIL_BYTES - P8_TAIL_BYTES + 12, INT32_MIN);
|
||||||
EXPECT_FALSE(receive_config_frame(mut, len));
|
EXPECT_FALSE(receive_config_frame(mut, len));
|
||||||
|
|
||||||
/* A presence int that is not a wire bool is refused. */
|
/* A presence int that is not a wire bool is refused. */
|
||||||
memcpy(mut, frame, len);
|
memcpy(mut, frame, len);
|
||||||
put_i32(mut, len - P8_TAIL_BYTES, SUPER_MODE_AUTO);
|
put_i32(mut, len - POST_P8_TAIL_BYTES - P8_TAIL_BYTES, SUPER_MODE_AUTO);
|
||||||
put_i32(mut, len - P8_TAIL_BYTES + 4, 2);
|
put_i32(mut, len - POST_P8_TAIL_BYTES - P8_TAIL_BYTES + 4, 2);
|
||||||
EXPECT_FALSE(receive_config_frame(mut, len));
|
EXPECT_FALSE(receive_config_frame(mut, len));
|
||||||
|
|
||||||
/* Truncating anywhere inside the P8 tail is refused. */
|
/* Truncating anywhere inside the P8 tail is refused. */
|
||||||
EXPECT_FALSE(receive_config_frame(frame, len - 2));
|
EXPECT_FALSE(receive_config_frame(frame, len - 2));
|
||||||
EXPECT_FALSE(receive_config_frame(frame, len - P8_TAIL_BYTES));
|
EXPECT_FALSE(receive_config_frame(frame, len - POST_P8_TAIL_BYTES - P8_TAIL_BYTES));
|
||||||
|
|
||||||
free(mut);
|
free(mut);
|
||||||
free(frame);
|
free(frame);
|
||||||
@@ -379,7 +383,9 @@ static void test_fuzz_config_receive_huge_map_count() {
|
|||||||
}
|
}
|
||||||
c->usermap_count = 1;
|
c->usermap_count = 1;
|
||||||
c->usermap[0].from = sentinel_from;
|
c->usermap[0].from = sentinel_from;
|
||||||
|
c->usermap[0].from_hi = sentinel_from;
|
||||||
c->usermap[0].to = sentinel_to;
|
c->usermap[0].to = sentinel_to;
|
||||||
|
c->usermap[0].to_name = NULL;
|
||||||
|
|
||||||
unsigned char* frame = NULL;
|
unsigned char* frame = NULL;
|
||||||
size_t len = 0;
|
size_t len = 0;
|
||||||
@@ -390,9 +396,12 @@ static void test_fuzz_config_receive_huge_map_count() {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
unsigned char pattern[8];
|
/* One wire entry is [from][from_hi][to][to_name]; search the fixed-width
|
||||||
|
prefix (the to_name length-prefixed string follows). */
|
||||||
|
unsigned char pattern[12];
|
||||||
memcpy(pattern, &sentinel_from, sizeof(sentinel_from));
|
memcpy(pattern, &sentinel_from, sizeof(sentinel_from));
|
||||||
memcpy(pattern + sizeof(sentinel_from), &sentinel_to, sizeof(sentinel_to));
|
memcpy(pattern + sizeof(sentinel_from), &sentinel_from, sizeof(sentinel_from));
|
||||||
|
memcpy(pattern + 2 * sizeof(sentinel_from), &sentinel_to, sizeof(sentinel_to));
|
||||||
size_t entry_off = find_bytes(frame, len, pattern, sizeof(pattern));
|
size_t entry_off = find_bytes(frame, len, pattern, sizeof(pattern));
|
||||||
if (entry_off == SIZE_MAX || entry_off < sizeof(int32_t)) {
|
if (entry_off == SIZE_MAX || entry_off < sizeof(int32_t)) {
|
||||||
free(frame);
|
free(frame);
|
||||||
|
|||||||
@@ -213,5 +213,8 @@ void test_iconv() {
|
|||||||
test_iconv_wire_sender_converts_local_to_remote();
|
test_iconv_wire_sender_converts_local_to_remote();
|
||||||
test_iconv_wire_receiver_converts_remote_to_local();
|
test_iconv_wire_receiver_converts_remote_to_local();
|
||||||
test_iconv_wire_disabled_passthrough();
|
test_iconv_wire_disabled_passthrough();
|
||||||
|
// This subtest forks to exercise the wire string handshake; the instrumented
|
||||||
|
// parent is too slow under valgrind for the child's blocking reads.
|
||||||
|
if (!is_running_under_valgrind())
|
||||||
test_iconv_wire_str_roundtrip();
|
test_iconv_wire_str_roundtrip();
|
||||||
}
|
}
|
||||||
+303
-8
@@ -1,8 +1,10 @@
|
|||||||
#include "test_metadata.h"
|
#include "test_metadata.h"
|
||||||
#include "chmod.h"
|
#include "chmod.h"
|
||||||
|
#include "identity.h"
|
||||||
#include "metadata.h"
|
#include "metadata.h"
|
||||||
#include "protocol.h"
|
#include "protocol.h"
|
||||||
#include "test_utils.h"
|
#include "test_utils.h"
|
||||||
|
#include <fcntl.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
#include <sys/ioctl.h>
|
#include <sys/ioctl.h>
|
||||||
@@ -337,7 +339,7 @@ static void test_file_restore_metadata_applies_atime() {
|
|||||||
m.crtime_sec = 0;
|
m.crtime_sec = 0;
|
||||||
m.crtime_nsec = 0;
|
m.crtime_nsec = 0;
|
||||||
|
|
||||||
file_restore_metadata(path, &m, false);
|
file_restore_metadata(path, &m, (FileAttrPolicy){true, true, true, false});
|
||||||
|
|
||||||
struct stat st;
|
struct stat st;
|
||||||
EXPECT_EQ_INT(stat(path, &st), 0);
|
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||||
@@ -376,7 +378,7 @@ static void test_file_restore_metadata() {
|
|||||||
.atime_valid = false,
|
.atime_valid = false,
|
||||||
.crtime_valid = false};
|
.crtime_valid = false};
|
||||||
|
|
||||||
file_restore_metadata(path, &m, false);
|
file_restore_metadata(path, &m, (FileAttrPolicy){true, true, false, false});
|
||||||
|
|
||||||
struct stat st;
|
struct stat st;
|
||||||
EXPECT_EQ_INT(stat(path, &st), 0);
|
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||||
@@ -393,7 +395,7 @@ static void test_file_restore_executability_only() {
|
|||||||
|
|
||||||
FileMetadata m = {
|
FileMetadata m = {
|
||||||
.mode = 0751, .uid = getuid(), .gid = getgid(), .mtime_sec = 0, .mtime_nsec = 0};
|
.mode = 0751, .uid = getuid(), .gid = getgid(), .mtime_sec = 0, .mtime_nsec = 0};
|
||||||
file_restore_metadata(path, &m, true);
|
file_restore_metadata(path, &m, (FileAttrPolicy){false, false, false, true});
|
||||||
|
|
||||||
struct stat st;
|
struct stat st;
|
||||||
EXPECT_EQ_INT(stat(path, &st), 0);
|
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||||
@@ -407,14 +409,190 @@ static void test_directory_restore_executability_only() {
|
|||||||
|
|
||||||
FileMetadata m = {
|
FileMetadata m = {
|
||||||
.mode = 0755, .uid = getuid(), .gid = getgid(), .mtime_sec = 0, .mtime_nsec = 0};
|
.mode = 0755, .uid = getuid(), .gid = getgid(), .mtime_sec = 0, .mtime_nsec = 0};
|
||||||
file_restore_metadata(path, &m, true);
|
file_restore_metadata(path, &m, (FileAttrPolicy){false, false, false, true});
|
||||||
|
|
||||||
struct stat st;
|
struct stat st;
|
||||||
EXPECT_EQ_INT(stat(path, &st), 0);
|
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||||
EXPECT_EQ_INT(st.st_mode & 0777, 0711);
|
/* rsync -E: an executable source derives exec from the DESTINATION's read
|
||||||
|
* bits. A 0700 directory has read only for the owner, so only the owner
|
||||||
|
* gains exec -- the result stays 0700 (not 0711, the old per-class copy). */
|
||||||
|
EXPECT_EQ_INT(st.st_mode & 0777, 0700);
|
||||||
rmdir(path);
|
rmdir(path);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* rsync 3.4 -E truth table (preserve_perms off), verified against rsync 3.4.1:
|
||||||
|
* (src,dest) -> result. A non-executable source clears every execute bit; an
|
||||||
|
* executable source sets a class's execute bit iff that class can read. */
|
||||||
|
static void test_file_restore_executability_rsync_rule() {
|
||||||
|
static const struct {
|
||||||
|
mode_t src;
|
||||||
|
mode_t dest;
|
||||||
|
mode_t want;
|
||||||
|
} cases[] = {
|
||||||
|
{0755, 0644, 0755}, {0755, 0600, 0700}, {0755, 0640, 0750}, {0755, 0666, 0777},
|
||||||
|
{0700, 0640, 0750}, {0111, 0644, 0755}, {0644, 0755, 0644}, {0644, 0600, 0600},
|
||||||
|
};
|
||||||
|
const char* path = "temp_exec_rsync_rule.txt";
|
||||||
|
for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
|
||||||
|
EXPECT_TRUE(file_write_to_disk(path, "x", 1, false, false));
|
||||||
|
EXPECT_EQ_INT(chmod(path, cases[i].dest), 0);
|
||||||
|
FileMetadata m = {.mode = cases[i].src, .uid = getuid(), .gid = getgid()};
|
||||||
|
file_restore_metadata(path, &m, (FileAttrPolicy){false, false, false, true});
|
||||||
|
struct stat st;
|
||||||
|
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||||
|
EXPECT_EQ_INT(st.st_mode & 0777, cases[i].want);
|
||||||
|
unlink(path);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* The shared metadata_mode_for_policy() helper is the single source of truth
|
||||||
|
* used by both the normal metadata path and the --fake-super replay. It must
|
||||||
|
* reproduce the per-attribute split: no mode change when neither -p nor -E is
|
||||||
|
* set; -p applies the source mode exactly (including group/other write and the
|
||||||
|
* setuid/setgid/sticky bits) regardless of the destination; -E derives exec
|
||||||
|
* bits from the destination and --perms wins when both are set. */
|
||||||
|
static void test_metadata_mode_for_policy() {
|
||||||
|
mode_t out = 0xdead;
|
||||||
|
EXPECT_FALSE(
|
||||||
|
metadata_mode_for_policy(0777, 0644, (FileAttrPolicy){false, false, false, false}, &out));
|
||||||
|
EXPECT_EQ_INT((int)out, 0xdead); /* untouched when no change is requested */
|
||||||
|
|
||||||
|
EXPECT_TRUE(
|
||||||
|
metadata_mode_for_policy(0777, 0644, (FileAttrPolicy){true, false, false, false}, &out));
|
||||||
|
EXPECT_EQ_INT((int)(out & 0777), 0777); /* group/other write is preserved */
|
||||||
|
|
||||||
|
mode_t specials = (mode_t)(S_ISUID | S_ISGID | S_ISVTX | 0672);
|
||||||
|
EXPECT_TRUE(
|
||||||
|
metadata_mode_for_policy(specials, 0644, (FileAttrPolicy){true, false, false, false}, &out));
|
||||||
|
EXPECT_EQ_INT((int)(out & (S_ISUID | S_ISGID | S_ISVTX | 0777)),
|
||||||
|
(int)(S_ISUID | S_ISGID | S_ISVTX | 0672));
|
||||||
|
|
||||||
|
/* -E: exec bits derive from the DESTINATION's read bits. */
|
||||||
|
EXPECT_TRUE(
|
||||||
|
metadata_mode_for_policy(0755, 0644, (FileAttrPolicy){false, false, false, true}, &out));
|
||||||
|
EXPECT_EQ_INT((int)(out & 0777), 0755);
|
||||||
|
EXPECT_TRUE(
|
||||||
|
metadata_mode_for_policy(0644, 0755, (FileAttrPolicy){false, false, false, true}, &out));
|
||||||
|
EXPECT_EQ_INT((int)(out & 0777), 0644);
|
||||||
|
EXPECT_TRUE(
|
||||||
|
metadata_mode_for_policy(0755, 0600, (FileAttrPolicy){false, false, false, true}, &out));
|
||||||
|
EXPECT_EQ_INT((int)(out & 0777), 0700);
|
||||||
|
|
||||||
|
/* --perms wins over -E when both are set. */
|
||||||
|
EXPECT_TRUE(
|
||||||
|
metadata_mode_for_policy(0700, 0644, (FileAttrPolicy){true, false, false, true}, &out));
|
||||||
|
EXPECT_EQ_INT((int)(out & 0777), 0700);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* A brand-new file with -p off is created like rsync: source_mode & 0777 &
|
||||||
|
* ~umask (when metadata is available). The -E rule is then layered on top. */
|
||||||
|
static void test_new_file_mode_from_source_and_umask() {
|
||||||
|
const char* path = "temp_new_file_base.txt";
|
||||||
|
unlink(path);
|
||||||
|
FileMetadata m = {.mode = 0751, .uid = getuid(), .gid = getgid()};
|
||||||
|
mode_t want = (mode_t)(0751 & 0777 & ~(mode_t)file_process_umask());
|
||||||
|
|
||||||
|
bool ok = file_to_disk_secure_attrs(path, "x", 1, false, false, false, &m,
|
||||||
|
(FileAttrPolicy){false, false, false, false}, false, false,
|
||||||
|
false, NULL, false, false, NULL);
|
||||||
|
EXPECT_TRUE(ok);
|
||||||
|
struct stat st;
|
||||||
|
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||||
|
EXPECT_EQ_INT(st.st_mode & 0777, want);
|
||||||
|
unlink(path);
|
||||||
|
|
||||||
|
/* -E on top of the source&~umask base (src 0751, umask 022 -> 0751). */
|
||||||
|
ok = file_to_disk_secure_attrs(path, "x", 1, false, false, false, &m,
|
||||||
|
(FileAttrPolicy){false, false, false, true}, false, false, false,
|
||||||
|
NULL, false, false, NULL);
|
||||||
|
EXPECT_TRUE(ok);
|
||||||
|
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||||
|
mode_t want_e =
|
||||||
|
(want & 0444) ? (mode_t)(want | ((want & 0444) >> 2)) : (mode_t)(want & ~(mode_t)0111);
|
||||||
|
EXPECT_EQ_INT(st.st_mode & 0777, want_e);
|
||||||
|
unlink(path);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* The per-attribute split: -t/-U apply times without touching the mode; an
|
||||||
|
* all-off policy applies neither mode nor times. */
|
||||||
|
static void test_file_restore_attribute_split() {
|
||||||
|
const char* path = "temp_meta_split_test.txt";
|
||||||
|
EXPECT_TRUE(file_write_to_disk(path, "x", 1, false, false));
|
||||||
|
EXPECT_EQ_INT(chmod(path, 0640), 0);
|
||||||
|
|
||||||
|
FileMetadata m = {.mode = 0755,
|
||||||
|
.uid = getuid(),
|
||||||
|
.gid = getgid(),
|
||||||
|
.mtime_sec = 1234567890,
|
||||||
|
.mtime_nsec = 0,
|
||||||
|
.atime_valid = false,
|
||||||
|
.crtime_valid = false};
|
||||||
|
|
||||||
|
/* times only: mtime changes, mode stays 0640. */
|
||||||
|
file_restore_metadata(path, &m, (FileAttrPolicy){false, true, false, false});
|
||||||
|
struct stat st;
|
||||||
|
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||||
|
EXPECT_EQ_INT(st.st_mode & 0777, 0640);
|
||||||
|
EXPECT_EQ_INT((int)st.st_mtime, 1234567890);
|
||||||
|
|
||||||
|
/* no attributes: neither mode nor mtime changes. */
|
||||||
|
EXPECT_EQ_INT(chmod(path, 0640), 0);
|
||||||
|
struct timespec ts[2] = {{.tv_sec = 1000000000, .tv_nsec = 0},
|
||||||
|
{.tv_sec = 1000000000, .tv_nsec = 0}};
|
||||||
|
EXPECT_EQ_INT(utimensat(AT_FDCWD, path, ts, 0), 0);
|
||||||
|
FileMetadata m2 = m;
|
||||||
|
m2.mode = 0700;
|
||||||
|
m2.mtime_sec = 1600000000;
|
||||||
|
file_restore_metadata(path, &m2, (FileAttrPolicy){false, false, false, false});
|
||||||
|
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||||
|
EXPECT_EQ_INT(st.st_mode & 0777, 0640);
|
||||||
|
EXPECT_EQ_INT((int)st.st_mtime, 1000000000);
|
||||||
|
|
||||||
|
unlink(path);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_file_attr_policy_from_config() {
|
||||||
|
FileAttrPolicy none = file_attr_policy_from_config(NULL);
|
||||||
|
EXPECT_FALSE(none.perms);
|
||||||
|
EXPECT_FALSE(none.times);
|
||||||
|
EXPECT_FALSE(none.atimes);
|
||||||
|
EXPECT_FALSE(none.executability);
|
||||||
|
|
||||||
|
Config* c = config_create();
|
||||||
|
EXPECT_NOT_NULL(c);
|
||||||
|
c->preserve_perms = true;
|
||||||
|
c->preserve_times = true;
|
||||||
|
c->preserve_atimes = true;
|
||||||
|
c->use_executability = true;
|
||||||
|
FileAttrPolicy p = file_attr_policy_from_config(c);
|
||||||
|
EXPECT_TRUE(p.perms);
|
||||||
|
EXPECT_TRUE(p.times);
|
||||||
|
EXPECT_TRUE(p.atimes);
|
||||||
|
EXPECT_TRUE(p.executability);
|
||||||
|
config_delete(c);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Strict rsync parity: -p copies the source's setuid/setgid/sticky bits (they
|
||||||
|
* are attempted, not masked away). On Linux these are settable on a file the
|
||||||
|
* receiving user owns; a mount that denies them would log a chmod failure. */
|
||||||
|
static void test_perms_preserves_special_bits() {
|
||||||
|
const char* path = "temp_special_bits.txt";
|
||||||
|
unlink(path);
|
||||||
|
FileMetadata m = {
|
||||||
|
.mode = (mode_t)(S_ISUID | S_ISGID | S_ISVTX | 0755), .uid = getuid(), .gid = getgid()};
|
||||||
|
|
||||||
|
bool ok = file_to_disk_secure_attrs(path, "x", 1, false, false, false, &m,
|
||||||
|
(FileAttrPolicy){true, false, false, false}, false, false,
|
||||||
|
false, NULL, false, false, NULL);
|
||||||
|
EXPECT_TRUE(ok);
|
||||||
|
struct stat st;
|
||||||
|
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||||
|
EXPECT_EQ_INT((int)(st.st_mode & 0777), 0755);
|
||||||
|
EXPECT_EQ_INT((int)(st.st_mode & (S_ISUID | S_ISGID | S_ISVTX)),
|
||||||
|
(int)(S_ISUID | S_ISGID | S_ISVTX));
|
||||||
|
unlink(path);
|
||||||
|
}
|
||||||
|
|
||||||
static void test_chmod_changes() {
|
static void test_chmod_changes() {
|
||||||
mode_t result;
|
mode_t result;
|
||||||
EXPECT_TRUE(chmod_apply(0777, "u=rw,go=r", &result));
|
EXPECT_TRUE(chmod_apply(0777, "u=rw,go=r", &result));
|
||||||
@@ -432,8 +610,45 @@ static void test_chmod_changes() {
|
|||||||
EXPECT_EQ_INT(result, 0755);
|
EXPECT_EQ_INT(result, 0755);
|
||||||
EXPECT_FALSE(chmod_apply(0777, "888", &result));
|
EXPECT_FALSE(chmod_apply(0777, "888", &result));
|
||||||
EXPECT_FALSE(chmod_apply(0777, "10000", &result));
|
EXPECT_FALSE(chmod_apply(0777, "10000", &result));
|
||||||
EXPECT_FALSE(chmod_apply(0777, "a+X", &result));
|
|
||||||
EXPECT_FALSE(chmod_apply(0777, "a+r,", &result));
|
EXPECT_FALSE(chmod_apply(0777, "a+r,", &result));
|
||||||
|
|
||||||
|
/* go+w is honored (rsync gives 0666 from a 0644 file). */
|
||||||
|
EXPECT_TRUE(chmod_apply(0644, "go+w", &result));
|
||||||
|
EXPECT_EQ_INT(result, 0666);
|
||||||
|
|
||||||
|
/* X only sets execute on directories or already-executable files. */
|
||||||
|
EXPECT_TRUE(chmod_apply(0644, "a+X", &result));
|
||||||
|
EXPECT_EQ_INT(result, 0644);
|
||||||
|
EXPECT_TRUE(chmod_apply(0755, "a+X", &result));
|
||||||
|
EXPECT_EQ_INT(result, 0755);
|
||||||
|
EXPECT_TRUE(chmod_apply((mode_t)(S_IFDIR | 0644), "a+X", &result));
|
||||||
|
EXPECT_EQ_INT((int)(result & 0777), 0755);
|
||||||
|
EXPECT_TRUE(S_ISDIR(result));
|
||||||
|
|
||||||
|
/* D/F selectors restrict a clause to directories/files. */
|
||||||
|
EXPECT_TRUE(chmod_apply((mode_t)(S_IFDIR | 0700), "Dg+s", &result));
|
||||||
|
EXPECT_EQ_INT((int)(result & 07777), 02700);
|
||||||
|
EXPECT_TRUE(chmod_apply((mode_t)(S_IFREG | 0644), "Dg+s", &result));
|
||||||
|
EXPECT_EQ_INT((int)(result & 07777), 0644);
|
||||||
|
EXPECT_TRUE(chmod_apply((mode_t)(S_IFREG | 0644), "Fo-w", &result));
|
||||||
|
EXPECT_EQ_INT((int)(result & 07777), 0644);
|
||||||
|
EXPECT_TRUE(chmod_apply((mode_t)(S_IFREG | 0666), "Fo-w", &result));
|
||||||
|
EXPECT_EQ_INT((int)(result & 07777), 0664);
|
||||||
|
EXPECT_TRUE(chmod_apply((mode_t)(S_IFDIR | 0666), "Fo-w", &result));
|
||||||
|
EXPECT_EQ_INT((int)(result & 07777), 0666);
|
||||||
|
EXPECT_FALSE(chmod_apply(0644, "DFu+w", &result));
|
||||||
|
|
||||||
|
/* Special bits: s/t map to setuid/setgid/sticky like rsync. */
|
||||||
|
EXPECT_TRUE(chmod_apply(0755, "u+s", &result));
|
||||||
|
EXPECT_EQ_INT((int)(result & 07777), 04755);
|
||||||
|
EXPECT_TRUE(chmod_apply(0755, "g+s", &result));
|
||||||
|
EXPECT_EQ_INT((int)(result & 07777), 02755);
|
||||||
|
EXPECT_TRUE(chmod_apply(0755, "a+t", &result));
|
||||||
|
EXPECT_EQ_INT((int)(result & 07777), 01755);
|
||||||
|
|
||||||
|
/* Comma-separated clauses accumulate (the CLI joins repeated options). */
|
||||||
|
EXPECT_TRUE(chmod_apply(0644, "g+w,u+x", &result));
|
||||||
|
EXPECT_EQ_INT((int)(result & 07777), 0764);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* P7 Wave D: symlink metadata is applied with no-follow primitives, and -J
|
/* P7 Wave D: symlink metadata is applied with no-follow primitives, and -J
|
||||||
@@ -455,7 +670,7 @@ static void test_file_restore_symlink_metadata() {
|
|||||||
|
|
||||||
/* Positive path: a non-omitted apply stamps the link's own mtime. */
|
/* Positive path: a non-omitted apply stamps the link's own mtime. */
|
||||||
FileMetadata applied = {.mtime_sec = 1000000000, .mtime_nsec = 0};
|
FileMetadata applied = {.mtime_sec = 1000000000, .mtime_nsec = 0};
|
||||||
file_restore_symlink_metadata(link, &applied, false);
|
file_restore_symlink_metadata(link, &applied, (FileAttrPolicy){false, true, false, false}, false);
|
||||||
struct stat st;
|
struct stat st;
|
||||||
EXPECT_EQ_INT(lstat(link, &st), 0);
|
EXPECT_EQ_INT(lstat(link, &st), 0);
|
||||||
EXPECT_TRUE(S_ISLNK(st.st_mode));
|
EXPECT_TRUE(S_ISLNK(st.st_mode));
|
||||||
@@ -464,7 +679,7 @@ static void test_file_restore_symlink_metadata() {
|
|||||||
|
|
||||||
/* -J: a different time must be left untouched. */
|
/* -J: a different time must be left untouched. */
|
||||||
FileMetadata newer = {.mtime_sec = 1234567890, .mtime_nsec = 0};
|
FileMetadata newer = {.mtime_sec = 1234567890, .mtime_nsec = 0};
|
||||||
file_restore_symlink_metadata(link, &newer, true);
|
file_restore_symlink_metadata(link, &newer, (FileAttrPolicy){false, true, false, false}, true);
|
||||||
EXPECT_EQ_INT(lstat(link, &st), 0);
|
EXPECT_EQ_INT(lstat(link, &st), 0);
|
||||||
EXPECT_EQ_INT((int)st.st_mtime, (int)t1);
|
EXPECT_EQ_INT((int)st.st_mtime, (int)t1);
|
||||||
if (symlink_times_supported)
|
if (symlink_times_supported)
|
||||||
@@ -475,6 +690,79 @@ static void test_file_restore_symlink_metadata() {
|
|||||||
rmdir(dir);
|
rmdir(dir);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Per-attribute gating of the descriptor restore path: -p alone applies the
|
||||||
|
* mode, -t alone the mtime, -U alone the atime, and an all-off policy leaves
|
||||||
|
* the destination's mode and times exactly as they are. This pins the fd API
|
||||||
|
* the receiver actually uses (the path-based file_restore_metadata has its own
|
||||||
|
* split test). */
|
||||||
|
static void test_file_restore_metadata_fd_attribute_split() {
|
||||||
|
identity_clear_active(); /* no ownership policy leaking from a previous test */
|
||||||
|
const char* path = "temp_meta_fd_split_test.txt";
|
||||||
|
EXPECT_TRUE(file_write_to_disk(path, "x", 1, false, false));
|
||||||
|
EXPECT_EQ_INT(chmod(path, 0640), 0);
|
||||||
|
int fd = open(path, O_RDWR);
|
||||||
|
EXPECT_TRUE(fd >= 0);
|
||||||
|
/* cppcheck-suppress knownConditionTrueFalse -- EXPECT_TRUE above asserts,
|
||||||
|
but cppcheck cannot see through the macro; the guard is defensive. */
|
||||||
|
if (fd < 0) {
|
||||||
|
unlink(path);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
FileMetadata m = {.mode = 0755,
|
||||||
|
.uid = getuid(),
|
||||||
|
.gid = getgid(),
|
||||||
|
.mtime_sec = 1234567890,
|
||||||
|
.mtime_nsec = 0,
|
||||||
|
.atime_valid = true,
|
||||||
|
.atime_sec = 999999999,
|
||||||
|
.atime_nsec = 0,
|
||||||
|
.crtime_valid = false};
|
||||||
|
struct stat st;
|
||||||
|
struct stat before;
|
||||||
|
|
||||||
|
/* perms-only: mode applied, mtime untouched. */
|
||||||
|
EXPECT_EQ_INT(fstat(fd, &before), 0);
|
||||||
|
EXPECT_TRUE(file_restore_metadata_fd(fd, &m, (FileAttrPolicy){true, false, false, false}));
|
||||||
|
EXPECT_EQ_INT(fstat(fd, &st), 0);
|
||||||
|
EXPECT_EQ_INT(st.st_mode & 0777, 0755);
|
||||||
|
EXPECT_EQ_INT((int)st.st_mtime, (int)before.st_mtime);
|
||||||
|
|
||||||
|
/* times-only: mtime applied, mode untouched. */
|
||||||
|
EXPECT_EQ_INT(chmod(path, 0600), 0);
|
||||||
|
EXPECT_TRUE(file_restore_metadata_fd(fd, &m, (FileAttrPolicy){false, true, false, false}));
|
||||||
|
EXPECT_EQ_INT(fstat(fd, &st), 0);
|
||||||
|
EXPECT_EQ_INT(st.st_mode & 0777, 0600);
|
||||||
|
EXPECT_EQ_INT((int)st.st_mtime, 1234567890);
|
||||||
|
|
||||||
|
/* atime-only: atime applied, mtime and mode untouched. */
|
||||||
|
struct timespec reset[2] = {{.tv_sec = 1000000000, .tv_nsec = 0},
|
||||||
|
{.tv_sec = 1000000000, .tv_nsec = 0}};
|
||||||
|
EXPECT_EQ_INT(futimens(fd, reset), 0);
|
||||||
|
EXPECT_EQ_INT(fstat(fd, &before), 0);
|
||||||
|
EXPECT_TRUE(file_restore_metadata_fd(fd, &m, (FileAttrPolicy){false, false, true, false}));
|
||||||
|
EXPECT_EQ_INT(fstat(fd, &st), 0);
|
||||||
|
EXPECT_EQ_INT((int)st.st_atime, 999999999);
|
||||||
|
EXPECT_EQ_INT((int)st.st_mtime, (int)before.st_mtime);
|
||||||
|
EXPECT_EQ_INT(st.st_mode & 0777, 0600);
|
||||||
|
|
||||||
|
/* all-off: neither mode nor either time is touched. */
|
||||||
|
EXPECT_EQ_INT(chmod(path, 0640), 0);
|
||||||
|
EXPECT_EQ_INT(futimens(fd, reset), 0);
|
||||||
|
FileMetadata m2 = m;
|
||||||
|
m2.mode = 0700;
|
||||||
|
m2.mtime_sec = 1600000000;
|
||||||
|
m2.atime_sec = 1700000000;
|
||||||
|
EXPECT_TRUE(file_restore_metadata_fd(fd, &m2, (FileAttrPolicy){false, false, false, false}));
|
||||||
|
EXPECT_EQ_INT(fstat(fd, &st), 0);
|
||||||
|
EXPECT_EQ_INT(st.st_mode & 0777, 0640);
|
||||||
|
EXPECT_EQ_INT((int)st.st_mtime, 1000000000);
|
||||||
|
EXPECT_EQ_INT((int)st.st_atime, 1000000000);
|
||||||
|
|
||||||
|
close(fd);
|
||||||
|
unlink(path);
|
||||||
|
}
|
||||||
|
|
||||||
void test_metadata() {
|
void test_metadata() {
|
||||||
test_metadata_to_from_buf_roundtrip();
|
test_metadata_to_from_buf_roundtrip();
|
||||||
test_metadata_to_buf_null();
|
test_metadata_to_buf_null();
|
||||||
@@ -491,6 +779,13 @@ void test_metadata() {
|
|||||||
test_file_restore_metadata_applies_atime();
|
test_file_restore_metadata_applies_atime();
|
||||||
test_file_restore_executability_only();
|
test_file_restore_executability_only();
|
||||||
test_directory_restore_executability_only();
|
test_directory_restore_executability_only();
|
||||||
|
test_file_restore_executability_rsync_rule();
|
||||||
|
test_metadata_mode_for_policy();
|
||||||
|
test_new_file_mode_from_source_and_umask();
|
||||||
|
test_file_restore_attribute_split();
|
||||||
|
test_file_restore_metadata_fd_attribute_split();
|
||||||
|
test_file_attr_policy_from_config();
|
||||||
test_file_restore_symlink_metadata();
|
test_file_restore_symlink_metadata();
|
||||||
|
test_perms_preserves_special_bits();
|
||||||
test_chmod_changes();
|
test_chmod_changes();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -243,6 +243,7 @@ static void test_write_thread_done() {
|
|||||||
* However, pipeline_context_receiver_destroy will call config_delete
|
* However, pipeline_context_receiver_destroy will call config_delete
|
||||||
* and queue_destroy which would double-free since we created them
|
* and queue_destroy which would double-free since we created them
|
||||||
* in this test. Let me just free the context directly. */
|
* in this test. Let me just free the context directly. */
|
||||||
|
array_list_delete(ctx->would_delete);
|
||||||
mtx_destroy(&ctx->mutex);
|
mtx_destroy(&ctx->mutex);
|
||||||
cnd_destroy(&ctx->condition_not_full);
|
cnd_destroy(&ctx->condition_not_full);
|
||||||
cnd_destroy(&ctx->condition_not_empty);
|
cnd_destroy(&ctx->condition_not_empty);
|
||||||
@@ -327,6 +328,7 @@ static void test_receiver_enqueue_byte_budget() {
|
|||||||
EXPECT_EQ_INT((int)ctx->queued_bytes, 2000); /* second payload now in flight */
|
EXPECT_EQ_INT((int)ctx->queued_bytes, 2000); /* second payload now in flight */
|
||||||
|
|
||||||
/* Tear down: the second file is still queued and is freed by queue_destroy. */
|
/* Tear down: the second file is still queued and is freed by queue_destroy. */
|
||||||
|
array_list_delete(ctx->would_delete);
|
||||||
mtx_destroy(&ctx->mutex);
|
mtx_destroy(&ctx->mutex);
|
||||||
cnd_destroy(&ctx->condition_not_full);
|
cnd_destroy(&ctx->condition_not_full);
|
||||||
cnd_destroy(&ctx->condition_not_empty);
|
cnd_destroy(&ctx->condition_not_empty);
|
||||||
|
|||||||
+44
-3
@@ -291,6 +291,35 @@ static void test_max_alloc_allows_configured_buffer() {
|
|||||||
protocol_session_unbind();
|
protocol_session_unbind();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* max_alloc == 0 is rsync's --max-alloc=0 "no limit": allocations of any size
|
||||||
|
* are permitted. */
|
||||||
|
static void test_max_alloc_zero_means_unlimited() {
|
||||||
|
ProtocolSession session;
|
||||||
|
protocol_session_init(&session, -1, -1);
|
||||||
|
protocol_session_set_max_alloc(&session, 0);
|
||||||
|
protocol_session_bind(&session);
|
||||||
|
void* first = protocol_alloc(1024 * 1024);
|
||||||
|
void* second = protocol_alloc(8 * 1024 * 1024);
|
||||||
|
EXPECT_NOT_NULL(first);
|
||||||
|
EXPECT_NOT_NULL(second);
|
||||||
|
free(first);
|
||||||
|
free(second);
|
||||||
|
protocol_session_unbind();
|
||||||
|
}
|
||||||
|
|
||||||
|
/* A non-positive session io timeout disables the deadline: the getter reports 0
|
||||||
|
* (not the built-in 60 s fallback) so callers know to wait indefinitely. */
|
||||||
|
static void test_protocol_get_io_timeout_zero_disables() {
|
||||||
|
ProtocolSession session;
|
||||||
|
protocol_session_init(&session, -1, -1);
|
||||||
|
protocol_session_bind(&session);
|
||||||
|
protocol_session_set_io_timeout(&session, 0);
|
||||||
|
EXPECT_EQ_INT(protocol_get_io_timeout_sec(), 0);
|
||||||
|
protocol_session_set_io_timeout(&session, 45);
|
||||||
|
EXPECT_EQ_INT(protocol_get_io_timeout_sec(), 45);
|
||||||
|
protocol_session_unbind();
|
||||||
|
}
|
||||||
|
|
||||||
static void test_max_alloc_is_bound_in_worker_threads() {
|
static void test_max_alloc_is_bound_in_worker_threads() {
|
||||||
enum { WORKER_COUNT = 4 };
|
enum { WORKER_COUNT = 4 };
|
||||||
ProtocolSession sessions[WORKER_COUNT];
|
ProtocolSession sessions[WORKER_COUNT];
|
||||||
@@ -493,10 +522,19 @@ static void test_data_create_starts_uncharged_and_unowned() {
|
|||||||
data_destroy(reserved);
|
data_destroy(reserved);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* The server floors a client --timeout=0 at SERVER_IO_TIMEOUT_SEC so a silent
|
||||||
|
* peer can never hold a session slot forever (slow-loris). */
|
||||||
|
static void test_protocol_server_io_timeout_floor() {
|
||||||
|
EXPECT_EQ_INT(protocol_server_io_timeout_sec(0), SERVER_IO_TIMEOUT_SEC);
|
||||||
|
EXPECT_EQ_INT(protocol_server_io_timeout_sec(-7), SERVER_IO_TIMEOUT_SEC);
|
||||||
|
EXPECT_EQ_INT(protocol_server_io_timeout_sec(30), 30);
|
||||||
|
EXPECT_TRUE(SERVER_IO_TIMEOUT_SEC > 0);
|
||||||
|
}
|
||||||
|
|
||||||
static void test_protocol_session_io_timeout() {
|
static void test_protocol_session_io_timeout() {
|
||||||
/* Default is the built-in 60 s window; the setter stores exactly what it is
|
/* The default is the built-in 60 s window; the setter stores exactly what it
|
||||||
* given (<= 0 means "fall back to the default") so callers can propagate
|
* is given (<= 0 disables the deadline, matching rsync's --timeout=0) so
|
||||||
* --timeout without special-casing 0. */
|
* callers can propagate --timeout without special-casing 0. */
|
||||||
ProtocolSession session;
|
ProtocolSession session;
|
||||||
protocol_session_init(&session, -1, -1);
|
protocol_session_init(&session, -1, -1);
|
||||||
EXPECT_EQ_INT(session.io_timeout_sec, 60);
|
EXPECT_EQ_INT(session.io_timeout_sec, 60);
|
||||||
@@ -641,6 +679,7 @@ void test_protocol() {
|
|||||||
test_send_receive_int();
|
test_send_receive_int();
|
||||||
test_send_receive_status();
|
test_send_receive_status();
|
||||||
test_protocol_session_io_timeout();
|
test_protocol_session_io_timeout();
|
||||||
|
test_protocol_server_io_timeout_floor();
|
||||||
test_send_receive_status_timed();
|
test_send_receive_status_timed();
|
||||||
test_receive_status_keepalive_skips_reply();
|
test_receive_status_keepalive_skips_reply();
|
||||||
test_receive_status_keepalive_aborts();
|
test_receive_status_keepalive_aborts();
|
||||||
@@ -650,6 +689,8 @@ void test_protocol() {
|
|||||||
test_max_alloc_rejects_single_buffer();
|
test_max_alloc_rejects_single_buffer();
|
||||||
test_explicit_session_max_alloc_cannot_be_bypassed();
|
test_explicit_session_max_alloc_cannot_be_bypassed();
|
||||||
test_max_alloc_allows_configured_buffer();
|
test_max_alloc_allows_configured_buffer();
|
||||||
|
test_max_alloc_zero_means_unlimited();
|
||||||
|
test_protocol_get_io_timeout_zero_disables();
|
||||||
test_max_alloc_is_bound_in_worker_threads();
|
test_max_alloc_is_bound_in_worker_threads();
|
||||||
test_protocol_accounting_is_released_in_worker_threads();
|
test_protocol_accounting_is_released_in_worker_threads();
|
||||||
test_protocol_accounting_reservation_is_atomic();
|
test_protocol_accounting_reservation_is_atomic();
|
||||||
|
|||||||
@@ -65,7 +65,7 @@ static void test_receiver_aborts_idle_keepalive() {
|
|||||||
ssize_t wrote = write(sv[0], &keepalive, sizeof(keepalive));
|
ssize_t wrote = write(sv[0], &keepalive, sizeof(keepalive));
|
||||||
int result = -2;
|
int result = -2;
|
||||||
if (wrote == (ssize_t)sizeof(keepalive))
|
if (wrote == (ssize_t)sizeof(keepalive))
|
||||||
result = receiver_process_pending(config, sv[1], &sink, NULL);
|
result = receiver_process_pending(config, sv[1], &sink, NULL, NULL);
|
||||||
Status reply = STATUS_OK;
|
Status reply = STATUS_OK;
|
||||||
ssize_t got = -1;
|
ssize_t got = -1;
|
||||||
if (result == -1)
|
if (result == -1)
|
||||||
|
|||||||
+138
-7
@@ -644,17 +644,19 @@ static void test_scanner_one_file_system_cross_device() {
|
|||||||
EXPECT_EQ_INT(seq_off_rc, 0);
|
EXPECT_EQ_INT(seq_off_rc, 0);
|
||||||
EXPECT_TRUE(seq_off_found);
|
EXPECT_TRUE(seq_off_found);
|
||||||
EXPECT_EQ_INT(seq_off_total, 2);
|
EXPECT_EQ_INT(seq_off_total, 2);
|
||||||
/* Sequential: with -x the cross-device subtree is dropped, keep.txt remains. */
|
/* Sequential: with -x the cross-device subtree is not descended into, but
|
||||||
|
* rsync-compatible behavior still emits the mount-point directory entry as an
|
||||||
|
* empty directory File, so keep.txt plus that entry are present. */
|
||||||
EXPECT_EQ_INT(seq_on_rc, 0);
|
EXPECT_EQ_INT(seq_on_rc, 0);
|
||||||
EXPECT_FALSE(seq_on_found);
|
EXPECT_FALSE(seq_on_found);
|
||||||
EXPECT_EQ_INT(seq_on_total, 1);
|
EXPECT_EQ_INT(seq_on_total, 2);
|
||||||
/* Parallel: same behavior, worker path (depth > 1). */
|
/* Parallel: same behavior, worker path (depth > 1). */
|
||||||
EXPECT_EQ_INT(par_off_rc, 0);
|
EXPECT_EQ_INT(par_off_rc, 0);
|
||||||
EXPECT_TRUE(par_off_found);
|
EXPECT_TRUE(par_off_found);
|
||||||
EXPECT_EQ_INT(par_off_total, 2);
|
EXPECT_EQ_INT(par_off_total, 2);
|
||||||
EXPECT_EQ_INT(par_on_rc, 0);
|
EXPECT_EQ_INT(par_on_rc, 0);
|
||||||
EXPECT_FALSE(par_on_found);
|
EXPECT_FALSE(par_on_found);
|
||||||
EXPECT_EQ_INT(par_on_total, 1);
|
EXPECT_EQ_INT(par_on_total, 2);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Collect emitted file paths (relative to `root`) from a sequential scan.
|
/* Collect emitted file paths (relative to `root`) from a sequential scan.
|
||||||
@@ -853,7 +855,7 @@ static void test_filter_rules(bool parallel) {
|
|||||||
/* - *.tmp excludes only the tmp file; other files remain (default include). */
|
/* - *.tmp excludes only the tmp file; other files remain (default include). */
|
||||||
const char* exclude_only[] = {"- *.tmp"};
|
const char* exclude_only[] = {"- *.tmp"};
|
||||||
char err[160];
|
char err[160];
|
||||||
FilterRuleList* base = filter_base_build(exclude_only, 1, false, err, sizeof(err));
|
FilterRuleList* base = filter_base_build(exclude_only, 1, false, false, err, sizeof(err));
|
||||||
EXPECT_NOT_NULL(base);
|
EXPECT_NOT_NULL(base);
|
||||||
ScannerOptions options = {0};
|
ScannerOptions options = {0};
|
||||||
options.base_filters = base;
|
options.base_filters = base;
|
||||||
@@ -873,7 +875,7 @@ static void test_filter_rules(bool parallel) {
|
|||||||
|
|
||||||
/* Anchored include then exclude-all: only root-level keep* survives. */
|
/* Anchored include then exclude-all: only root-level keep* survives. */
|
||||||
const char* anchored[] = {"+ /a.txt", "- *"};
|
const char* anchored[] = {"+ /a.txt", "- *"};
|
||||||
base = filter_base_build(anchored, 2, false, err, sizeof(err));
|
base = filter_base_build(anchored, 2, false, false, err, sizeof(err));
|
||||||
EXPECT_NOT_NULL(base);
|
EXPECT_NOT_NULL(base);
|
||||||
options.base_filters = base;
|
options.base_filters = base;
|
||||||
rc = parallel ? collect_files_parallel(root, &options, &paths, &count)
|
rc = parallel ? collect_files_parallel(root, &options, &paths, &count)
|
||||||
@@ -884,6 +886,35 @@ static void test_filter_rules(bool parallel) {
|
|||||||
free_paths(paths, count);
|
free_paths(paths, count);
|
||||||
filter_rule_list_free(base);
|
filter_rule_list_free(base);
|
||||||
|
|
||||||
|
/* The common include idiom (the exact rule order the CLI compiles from
|
||||||
|
* --include='*.txt' --exclude='*'): only .txt files survive. */
|
||||||
|
const char* idiom[] = {"+ *.txt", "- *"};
|
||||||
|
base = filter_base_build(idiom, 2, false, false, err, sizeof(err));
|
||||||
|
EXPECT_NOT_NULL(base);
|
||||||
|
options.base_filters = base;
|
||||||
|
rc = parallel ? collect_files_parallel(root, &options, &paths, &count)
|
||||||
|
: collect_files(root, &options, &paths, &count);
|
||||||
|
EXPECT_EQ_INT(rc, 0);
|
||||||
|
EXPECT_EQ_INT(count, 2);
|
||||||
|
EXPECT_TRUE(has_path(paths, count, "a.txt"));
|
||||||
|
EXPECT_TRUE(has_path(paths, count, "c.txt"));
|
||||||
|
EXPECT_FALSE(has_path(paths, count, "b.tmp"));
|
||||||
|
free_paths(paths, count);
|
||||||
|
filter_rule_list_free(base);
|
||||||
|
|
||||||
|
/* An include rule alone is NOT a mandatory whitelist (rsync semantics): only
|
||||||
|
* the matching file is affected, everything else is still transferred. */
|
||||||
|
const char* include_alone[] = {"+ *.txt"};
|
||||||
|
base = filter_base_build(include_alone, 1, false, false, err, sizeof(err));
|
||||||
|
EXPECT_NOT_NULL(base);
|
||||||
|
options.base_filters = base;
|
||||||
|
rc = parallel ? collect_files_parallel(root, &options, &paths, &count)
|
||||||
|
: collect_files(root, &options, &paths, &count);
|
||||||
|
EXPECT_EQ_INT(rc, 0);
|
||||||
|
EXPECT_EQ_INT(count, 3);
|
||||||
|
free_paths(paths, count);
|
||||||
|
filter_rule_list_free(base);
|
||||||
|
|
||||||
unlink("test_scan_filter/a.txt");
|
unlink("test_scan_filter/a.txt");
|
||||||
unlink("test_scan_filter/b.tmp");
|
unlink("test_scan_filter/b.tmp");
|
||||||
unlink("test_scan_filter/c.txt");
|
unlink("test_scan_filter/c.txt");
|
||||||
@@ -901,7 +932,7 @@ static void test_filter_dir_only_and_anchored(bool parallel) {
|
|||||||
|
|
||||||
const char* rules[] = {"- /sub/"};
|
const char* rules[] = {"- /sub/"};
|
||||||
char err[160];
|
char err[160];
|
||||||
FilterRuleList* base = filter_base_build(rules, 1, false, err, sizeof(err));
|
FilterRuleList* base = filter_base_build(rules, 1, false, false, err, sizeof(err));
|
||||||
EXPECT_NOT_NULL(base);
|
EXPECT_NOT_NULL(base);
|
||||||
ScannerOptions options = {0};
|
ScannerOptions options = {0};
|
||||||
options.base_filters = base;
|
options.base_filters = base;
|
||||||
@@ -935,7 +966,7 @@ static void test_cvs_defaults(bool parallel) {
|
|||||||
create_test_file("test_scan_cvs/keep.txt", "keep");
|
create_test_file("test_scan_cvs/keep.txt", "keep");
|
||||||
|
|
||||||
char err[160];
|
char err[160];
|
||||||
FilterRuleList* base = filter_base_build(NULL, 0, true, err, sizeof(err));
|
FilterRuleList* base = filter_base_build(NULL, 0, true, false, err, sizeof(err));
|
||||||
EXPECT_NOT_NULL(base);
|
EXPECT_NOT_NULL(base);
|
||||||
ScannerOptions options = {0};
|
ScannerOptions options = {0};
|
||||||
options.base_filters = base;
|
options.base_filters = base;
|
||||||
@@ -974,6 +1005,7 @@ static void test_per_dir_filter(bool parallel) {
|
|||||||
|
|
||||||
ScannerOptions options = {0};
|
ScannerOptions options = {0};
|
||||||
options.per_dir_filters = true;
|
options.per_dir_filters = true;
|
||||||
|
options.exclude_per_dir_filter_files = true; /* -FF */
|
||||||
if (parallel)
|
if (parallel)
|
||||||
options.num_threads = 2;
|
options.num_threads = 2;
|
||||||
char** paths = NULL;
|
char** paths = NULL;
|
||||||
@@ -1038,6 +1070,59 @@ static void test_scanner_path_relative() {
|
|||||||
EXPECT_NULL(scanner_path_relative("/tmp/foo", "/tmp/foobar"));
|
EXPECT_NULL(scanner_path_relative("/tmp/foo", "/tmp/foobar"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* -R/--relative destination prefix: the '/./' cut point and normalization. */
|
||||||
|
static void test_scanner_relative_prefix() {
|
||||||
|
char* p = NULL;
|
||||||
|
|
||||||
|
/* No cut: the whole spec with leading/trailing slashes removed. */
|
||||||
|
p = scanner_relative_prefix("/tmp/src/foo/");
|
||||||
|
EXPECT_NOT_NULL(p);
|
||||||
|
EXPECT_EQ_STR(p, "tmp/src/foo");
|
||||||
|
free(p);
|
||||||
|
|
||||||
|
p = scanner_relative_prefix("src/foo");
|
||||||
|
EXPECT_NOT_NULL(p);
|
||||||
|
EXPECT_EQ_STR(p, "src/foo");
|
||||||
|
free(p);
|
||||||
|
|
||||||
|
/* Trailing "/." is the directory itself, not a cut. */
|
||||||
|
p = scanner_relative_prefix("src/foo/.");
|
||||||
|
EXPECT_NOT_NULL(p);
|
||||||
|
EXPECT_EQ_STR(p, "src/foo");
|
||||||
|
free(p);
|
||||||
|
|
||||||
|
/* The first "/./" cuts everything before it. */
|
||||||
|
p = scanner_relative_prefix("/a/./b/c");
|
||||||
|
EXPECT_NOT_NULL(p);
|
||||||
|
EXPECT_EQ_STR(p, "b/c");
|
||||||
|
free(p);
|
||||||
|
|
||||||
|
p = scanner_relative_prefix("src/./");
|
||||||
|
EXPECT_NOT_NULL(p);
|
||||||
|
EXPECT_EQ_STR(p, "");
|
||||||
|
free(p);
|
||||||
|
|
||||||
|
/* A later "." component is normalized away. */
|
||||||
|
p = scanner_relative_prefix("a/./b/./c");
|
||||||
|
EXPECT_NOT_NULL(p);
|
||||||
|
EXPECT_EQ_STR(p, "b/c");
|
||||||
|
free(p);
|
||||||
|
|
||||||
|
/* A leading "./" is the cut at the start. */
|
||||||
|
p = scanner_relative_prefix("./s2");
|
||||||
|
EXPECT_NOT_NULL(p);
|
||||||
|
EXPECT_EQ_STR(p, "s2");
|
||||||
|
free(p);
|
||||||
|
|
||||||
|
p = scanner_relative_prefix(".");
|
||||||
|
EXPECT_NOT_NULL(p);
|
||||||
|
EXPECT_EQ_STR(p, "");
|
||||||
|
free(p);
|
||||||
|
|
||||||
|
EXPECT_NULL(scanner_relative_prefix(NULL));
|
||||||
|
EXPECT_NULL(scanner_relative_prefix(""));
|
||||||
|
}
|
||||||
|
|
||||||
/* rsync precedence: a deeper .rsync-filter overrides a shallower one, so an
|
/* rsync precedence: a deeper .rsync-filter overrides a shallower one, so an
|
||||||
* inner "+ *.tmp" re-includes what the outer "- *.tmp" excluded. */
|
* inner "+ *.tmp" re-includes what the outer "- *.tmp" excluded. */
|
||||||
static void test_per_dir_filter_override(bool parallel) {
|
static void test_per_dir_filter_override(bool parallel) {
|
||||||
@@ -1053,6 +1138,7 @@ static void test_per_dir_filter_override(bool parallel) {
|
|||||||
|
|
||||||
ScannerOptions options = {0};
|
ScannerOptions options = {0};
|
||||||
options.per_dir_filters = true;
|
options.per_dir_filters = true;
|
||||||
|
options.exclude_per_dir_filter_files = true; /* -FF */
|
||||||
if (parallel)
|
if (parallel)
|
||||||
options.num_threads = 2;
|
options.num_threads = 2;
|
||||||
char** paths = NULL;
|
char** paths = NULL;
|
||||||
@@ -1502,6 +1588,49 @@ static void test_scanner_entry_classification() {
|
|||||||
rmdir(root);
|
rmdir(root);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* A dereferenced symlink with no referent (broken/unreadable) must record a
|
||||||
|
* non-fatal I/O error so the run can exit 23 like rsync, without aborting the
|
||||||
|
* scan or treating the condition as a fatal failure. */
|
||||||
|
static void test_scanner_broken_referent_io_error(void) {
|
||||||
|
const char* root = "test_scan_broken_ref";
|
||||||
|
const char* good = "test_scan_broken_ref/good.txt";
|
||||||
|
const char* broken = "test_scan_broken_ref/broken";
|
||||||
|
|
||||||
|
EXPECT_EQ_INT(mkdir(root, 0755), 0);
|
||||||
|
create_test_file(good, "hello");
|
||||||
|
EXPECT_EQ_INT(symlink("/nonexistent/quickwins/target", broken), 0);
|
||||||
|
|
||||||
|
{
|
||||||
|
ScannerOptions options = {0};
|
||||||
|
options.copy_links = true;
|
||||||
|
DirectoryScanner* scanner = directory_scanner_create_with_options(root, &options);
|
||||||
|
EXPECT_NOT_NULL(scanner);
|
||||||
|
Chunk* chunk;
|
||||||
|
while ((chunk = directory_scanner_next(scanner)) != NULL)
|
||||||
|
chunk_destroy(chunk);
|
||||||
|
EXPECT_FALSE(directory_scanner_failed(scanner));
|
||||||
|
EXPECT_TRUE(directory_scanner_had_io_error(scanner));
|
||||||
|
directory_scanner_destroy(scanner);
|
||||||
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
ScannerOptions options = {0};
|
||||||
|
options.copy_links = true;
|
||||||
|
ParallelScanner* scanner = parallel_scanner_create_with_options(root, &options, NULL);
|
||||||
|
EXPECT_NOT_NULL(scanner);
|
||||||
|
Chunk* chunk;
|
||||||
|
while ((chunk = parallel_scanner_next(scanner)) != NULL)
|
||||||
|
chunk_destroy(chunk);
|
||||||
|
EXPECT_FALSE(parallel_scanner_failed(scanner));
|
||||||
|
EXPECT_TRUE(parallel_scanner_had_io_error(scanner));
|
||||||
|
parallel_scanner_destroy(scanner);
|
||||||
|
}
|
||||||
|
|
||||||
|
unlink(broken);
|
||||||
|
unlink(good);
|
||||||
|
rmdir(root);
|
||||||
|
}
|
||||||
|
|
||||||
void test_scanner() {
|
void test_scanner() {
|
||||||
test_scanner_single_file();
|
test_scanner_single_file();
|
||||||
test_scanner_multiple_files();
|
test_scanner_multiple_files();
|
||||||
@@ -1520,6 +1649,7 @@ void test_scanner() {
|
|||||||
test_scanner_one_file_system_decision();
|
test_scanner_one_file_system_decision();
|
||||||
test_scanner_one_file_system_same_device();
|
test_scanner_one_file_system_same_device();
|
||||||
test_parallel_scanner_one_file_system_same_device();
|
test_parallel_scanner_one_file_system_same_device();
|
||||||
|
test_scanner_broken_referent_io_error();
|
||||||
test_scanner_one_file_system_cross_device();
|
test_scanner_one_file_system_cross_device();
|
||||||
test_files_from_subset(false);
|
test_files_from_subset(false);
|
||||||
test_files_from_subset(true);
|
test_files_from_subset(true);
|
||||||
@@ -1532,6 +1662,7 @@ void test_scanner() {
|
|||||||
test_per_dir_filter(false);
|
test_per_dir_filter(false);
|
||||||
test_per_dir_filter(true);
|
test_per_dir_filter(true);
|
||||||
test_scanner_path_relative();
|
test_scanner_path_relative();
|
||||||
|
test_scanner_relative_prefix();
|
||||||
test_per_dir_filter_override(false);
|
test_per_dir_filter_override(false);
|
||||||
test_per_dir_filter_override(true);
|
test_per_dir_filter_override(true);
|
||||||
test_dirs_no_descent();
|
test_dirs_no_descent();
|
||||||
|
|||||||
+93
-11
@@ -566,7 +566,7 @@ static Config* make_late_delete_config(const char* root) {
|
|||||||
|
|
||||||
static int run_pending_receiver(Config* cfg, int fd, DeleteManifest** pending) {
|
static int run_pending_receiver(Config* cfg, int fd, DeleteManifest** pending) {
|
||||||
ReceiverSink sink = {0};
|
ReceiverSink sink = {0};
|
||||||
return receiver_process_pending(cfg, fd, &sink, pending);
|
return receiver_process_pending(cfg, fd, &sink, pending, NULL);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void test_late_manifest_abort_frees_keepset() {
|
static void test_late_manifest_abort_frees_keepset() {
|
||||||
@@ -582,6 +582,7 @@ static void test_late_manifest_abort_frees_keepset() {
|
|||||||
EXPECT_TRUE(send_str(p[1], "keep.txt"));
|
EXPECT_TRUE(send_str(p[1], "keep.txt"));
|
||||||
EXPECT_TRUE(send_int(p[1], 0)); /* protected-prefix section is empty */
|
EXPECT_TRUE(send_int(p[1], 0)); /* protected-prefix section is empty */
|
||||||
EXPECT_TRUE(send_int(p[1], 0)); /* missing-args section is empty */
|
EXPECT_TRUE(send_int(p[1], 0)); /* missing-args section is empty */
|
||||||
|
EXPECT_TRUE(send_int(p[1], 0)); /* synchronized-directories section is empty */
|
||||||
EXPECT_TRUE(send_status(p[1], STATUS_ABORT));
|
EXPECT_TRUE(send_status(p[1], STATUS_ABORT));
|
||||||
|
|
||||||
DeleteManifest* pending = NULL;
|
DeleteManifest* pending = NULL;
|
||||||
@@ -606,6 +607,7 @@ static void test_late_manifest_eof_frees_keepset() {
|
|||||||
EXPECT_TRUE(send_str(p[1], "keep.txt"));
|
EXPECT_TRUE(send_str(p[1], "keep.txt"));
|
||||||
EXPECT_TRUE(send_int(p[1], 0)); /* protected-prefix section is empty */
|
EXPECT_TRUE(send_int(p[1], 0)); /* protected-prefix section is empty */
|
||||||
EXPECT_TRUE(send_int(p[1], 0)); /* missing-args section is empty */
|
EXPECT_TRUE(send_int(p[1], 0)); /* missing-args section is empty */
|
||||||
|
EXPECT_TRUE(send_int(p[1], 0)); /* synchronized-directories section is empty */
|
||||||
shutdown(p[1], SHUT_WR);
|
shutdown(p[1], SHUT_WR);
|
||||||
|
|
||||||
DeleteManifest* pending = NULL;
|
DeleteManifest* pending = NULL;
|
||||||
@@ -630,11 +632,13 @@ static void test_late_second_manifest_frees_both() {
|
|||||||
EXPECT_TRUE(send_str(p[1], "first.txt"));
|
EXPECT_TRUE(send_str(p[1], "first.txt"));
|
||||||
EXPECT_TRUE(send_int(p[1], 0)); /* protected-prefix section is empty */
|
EXPECT_TRUE(send_int(p[1], 0)); /* protected-prefix section is empty */
|
||||||
EXPECT_TRUE(send_int(p[1], 0)); /* missing-args section is empty */
|
EXPECT_TRUE(send_int(p[1], 0)); /* missing-args section is empty */
|
||||||
|
EXPECT_TRUE(send_int(p[1], 0)); /* synchronized-directories section is empty */
|
||||||
EXPECT_TRUE(send_status(p[1], STATUS_MANIFEST));
|
EXPECT_TRUE(send_status(p[1], STATUS_MANIFEST));
|
||||||
EXPECT_TRUE(send_int(p[1], 1));
|
EXPECT_TRUE(send_int(p[1], 1));
|
||||||
EXPECT_TRUE(send_str(p[1], "second.txt"));
|
EXPECT_TRUE(send_str(p[1], "second.txt"));
|
||||||
EXPECT_TRUE(send_int(p[1], 0)); /* protected-prefix section is empty */
|
EXPECT_TRUE(send_int(p[1], 0)); /* protected-prefix section is empty */
|
||||||
EXPECT_TRUE(send_int(p[1], 0)); /* missing-args section is empty */
|
EXPECT_TRUE(send_int(p[1], 0)); /* missing-args section is empty */
|
||||||
|
EXPECT_TRUE(send_int(p[1], 0)); /* synchronized-directories section is empty */
|
||||||
|
|
||||||
DeleteManifest* pending = NULL;
|
DeleteManifest* pending = NULL;
|
||||||
EXPECT_EQ_INT(run_pending_receiver(cfg, p[0], &pending), -1);
|
EXPECT_EQ_INT(run_pending_receiver(cfg, p[0], &pending), -1);
|
||||||
@@ -645,9 +649,10 @@ static void test_late_second_manifest_frees_both() {
|
|||||||
config_delete(cfg);
|
config_delete(cfg);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* A delete-manifest frame with a third (missing-args) section round-trips: the
|
/* A delete-manifest frame with all four sections round-trips: the receiver
|
||||||
receiver keeps all three sections and the missing paths are confined exactly
|
keeps the keep-set, protected prefixes, missing-args paths and synchronized
|
||||||
like the keep-set (a traversal entry in the missing section is rejected).
|
directories, and every section is confined exactly like the keep-set (a
|
||||||
|
traversal entry in the missing section is rejected).
|
||||||
receive_manifest_entries() reads the counts directly (the leading
|
receive_manifest_entries() reads the counts directly (the leading
|
||||||
STATUS_MANIFEST code is consumed by the caller, so these frames do not send
|
STATUS_MANIFEST code is consumed by the caller, so these frames do not send
|
||||||
it). */
|
it). */
|
||||||
@@ -666,6 +671,9 @@ static void test_receive_manifest_three_sections() {
|
|||||||
EXPECT_TRUE(send_int(p[1], 2));
|
EXPECT_TRUE(send_int(p[1], 2));
|
||||||
EXPECT_TRUE(send_str(p[1], "gone.txt"));
|
EXPECT_TRUE(send_str(p[1], "gone.txt"));
|
||||||
EXPECT_TRUE(send_str(p[1], "dir/gone.bin"));
|
EXPECT_TRUE(send_str(p[1], "dir/gone.bin"));
|
||||||
|
EXPECT_TRUE(send_int(p[1], 2));
|
||||||
|
EXPECT_TRUE(send_str(p[1], "."));
|
||||||
|
EXPECT_TRUE(send_str(p[1], "dir"));
|
||||||
|
|
||||||
DeleteManifest* manifest = receive_manifest_entries(p[0]);
|
DeleteManifest* manifest = receive_manifest_entries(p[0]);
|
||||||
EXPECT_NOT_NULL(manifest);
|
EXPECT_NOT_NULL(manifest);
|
||||||
@@ -676,9 +684,12 @@ static void test_receive_manifest_three_sections() {
|
|||||||
EXPECT_EQ_INT(manifest->missing->size, 2);
|
EXPECT_EQ_INT(manifest->missing->size, 2);
|
||||||
EXPECT_EQ_STR((char*)manifest->missing->items[0], "gone.txt");
|
EXPECT_EQ_STR((char*)manifest->missing->items[0], "gone.txt");
|
||||||
EXPECT_EQ_STR((char*)manifest->missing->items[1], "dir/gone.bin");
|
EXPECT_EQ_STR((char*)manifest->missing->items[1], "dir/gone.bin");
|
||||||
|
EXPECT_EQ_INT(manifest->dirs->size, 2);
|
||||||
|
EXPECT_EQ_STR((char*)manifest->dirs->items[0], ".");
|
||||||
|
EXPECT_EQ_STR((char*)manifest->dirs->items[1], "dir");
|
||||||
delete_manifest_free(manifest);
|
delete_manifest_free(manifest);
|
||||||
|
|
||||||
/* A traversal entry in the third section is rejected like every other. */
|
/* A traversal entry in the missing section is rejected like every other. */
|
||||||
EXPECT_TRUE(send_int(p[1], 0));
|
EXPECT_TRUE(send_int(p[1], 0));
|
||||||
EXPECT_TRUE(send_int(p[1], 0));
|
EXPECT_TRUE(send_int(p[1], 0));
|
||||||
EXPECT_TRUE(send_int(p[1], 1));
|
EXPECT_TRUE(send_int(p[1], 1));
|
||||||
@@ -780,12 +791,13 @@ static void test_receiver_pending_commits_missing_args() {
|
|||||||
EXPECT_TRUE(send_int(p[1], 2));
|
EXPECT_TRUE(send_int(p[1], 2));
|
||||||
EXPECT_TRUE(send_str(p[1], "gone.txt"));
|
EXPECT_TRUE(send_str(p[1], "gone.txt"));
|
||||||
EXPECT_TRUE(send_str(p[1], "never_here.txt"));
|
EXPECT_TRUE(send_str(p[1], "never_here.txt"));
|
||||||
|
EXPECT_TRUE(send_int(p[1], 0)); /* no synchronized directories */
|
||||||
EXPECT_TRUE(send_status(p[1], STATUS_FINISHED));
|
EXPECT_TRUE(send_status(p[1], STATUS_FINISHED));
|
||||||
|
|
||||||
/* NULL pending: the single-threaded commit path deletes at FINISHED. The
|
/* NULL pending: the single-threaded commit path deletes at FINISHED. The
|
||||||
sink sends the terminal STATUS_OK success frame. */
|
sink sends the terminal STATUS_OK success frame. */
|
||||||
ReceiverSink sink = {.send_success = true};
|
ReceiverSink sink = {.send_success = true};
|
||||||
EXPECT_EQ_INT(receiver_process_pending(cfg, p[0], &sink, NULL), 0);
|
EXPECT_EQ_INT(receiver_process_pending(cfg, p[0], &sink, NULL, NULL), 0);
|
||||||
Status ack;
|
Status ack;
|
||||||
EXPECT_TRUE(receive_status(p[1], &ack));
|
EXPECT_TRUE(receive_status(p[1], &ack));
|
||||||
EXPECT_EQ_INT(ack, STATUS_OK);
|
EXPECT_EQ_INT(ack, STATUS_OK);
|
||||||
@@ -818,10 +830,24 @@ static void test_special_socket_path_log_escaped() {
|
|||||||
set_log_level(LOG_LEVEL_WARNING);
|
set_log_level(LOG_LEVEL_WARNING);
|
||||||
log_set_8_bit_output(false);
|
log_set_8_bit_output(false);
|
||||||
|
|
||||||
|
const char* root = "test_special_sock_escape_root";
|
||||||
|
const char* existing = "test_special_sock_escape_root/evil\npath";
|
||||||
|
unlink(existing);
|
||||||
|
rmdir(root);
|
||||||
|
EXPECT_EQ_INT(mkdir(root, 0700), 0);
|
||||||
|
FILE* planted = fopen(existing, "wb");
|
||||||
|
EXPECT_NOT_NULL(planted);
|
||||||
|
fclose(planted);
|
||||||
|
|
||||||
FILE* capture = tmpfile();
|
FILE* capture = tmpfile();
|
||||||
EXPECT_NOT_NULL(capture);
|
EXPECT_NOT_NULL(capture);
|
||||||
log_set_file(capture);
|
log_set_file(capture);
|
||||||
|
|
||||||
|
Config* cfg = config_create();
|
||||||
|
EXPECT_NOT_NULL(cfg);
|
||||||
|
cfg->preserve_specials = true;
|
||||||
|
cfg->use_metadata = true;
|
||||||
|
|
||||||
File* file = file_create("evil\npath");
|
File* file = file_create("evil\npath");
|
||||||
EXPECT_NOT_NULL(file);
|
EXPECT_NOT_NULL(file);
|
||||||
file->is_special = true;
|
file->is_special = true;
|
||||||
@@ -829,7 +855,9 @@ static void test_special_socket_path_log_escaped() {
|
|||||||
EXPECT_NOT_NULL(file->metadata);
|
EXPECT_NOT_NULL(file->metadata);
|
||||||
file->metadata->mode = S_IFSOCK | 0644;
|
file->metadata->mode = S_IFSOCK | 0644;
|
||||||
|
|
||||||
FileSaveResult result = file_save_to_disk_full("/tmp/dst", file, NULL);
|
/* A non-matching entry already occupies the path: the socket creation is
|
||||||
|
refused and the warning must escape the path's control byte. */
|
||||||
|
FileSaveResult result = file_save_to_disk_full(root, file, cfg);
|
||||||
EXPECT_EQ_INT(result, FILE_SAVE_SKIPPED);
|
EXPECT_EQ_INT(result, FILE_SAVE_SKIPPED);
|
||||||
|
|
||||||
fflush(capture);
|
fflush(capture);
|
||||||
@@ -841,8 +869,11 @@ static void test_special_socket_path_log_escaped() {
|
|||||||
log_set_file(NULL);
|
log_set_file(NULL);
|
||||||
fclose(capture);
|
fclose(capture);
|
||||||
file_destroy(file);
|
file_destroy(file);
|
||||||
|
config_delete(cfg);
|
||||||
|
unlink(existing);
|
||||||
|
rmdir(root);
|
||||||
|
|
||||||
EXPECT_NOT_NULL(strstr(output, "socket not recreated: evil\\#012path"));
|
EXPECT_NOT_NULL(strstr(output, "refusing to replace existing entry with socket: evil\\#012path"));
|
||||||
}
|
}
|
||||||
|
|
||||||
/* B1: a client-planted FIFO at the destination must not block the receiver's
|
/* B1: a client-planted FIFO at the destination must not block the receiver's
|
||||||
@@ -1041,10 +1072,10 @@ static void test_incremental_check_basis_fifo_does_not_hang() {
|
|||||||
EXPECT_TRUE(send_n_data(p[1], &mtime, sizeof(mtime)));
|
EXPECT_TRUE(send_n_data(p[1], &mtime, sizeof(mtime)));
|
||||||
EXPECT_TRUE(send_n_data(p[1], &mtime_nsec, sizeof(mtime_nsec)));
|
EXPECT_TRUE(send_n_data(p[1], &mtime_nsec, sizeof(mtime_nsec)));
|
||||||
/* config_has_basis() makes the request carry the source digest. */
|
/* config_has_basis() makes the request carry the source digest. */
|
||||||
uint8_t wire_len = 8;
|
uint8_t wire_len = checksum_digest_len((ChecksumAlgo)cfg->checksum_algo);
|
||||||
uint8_t digest[8] = {0};
|
uint8_t digest[CHECKSUM_MAX_DIGEST_LEN] = {0};
|
||||||
EXPECT_TRUE(send_n_data(p[1], &wire_len, sizeof(wire_len)));
|
EXPECT_TRUE(send_n_data(p[1], &wire_len, sizeof(wire_len)));
|
||||||
EXPECT_TRUE(send_n_data(p[1], digest, sizeof(digest)));
|
EXPECT_TRUE(send_n_data(p[1], digest, wire_len));
|
||||||
Status s;
|
Status s;
|
||||||
EXPECT_TRUE(receive_status(p[1], &s));
|
EXPECT_TRUE(receive_status(p[1], &s));
|
||||||
EXPECT_EQ_INT(s, STATUS_NEXT);
|
EXPECT_EQ_INT(s, STATUS_NEXT);
|
||||||
@@ -1097,6 +1128,56 @@ static void test_receive_manifest_total_entry_cap() {
|
|||||||
config_delete(cfg);
|
config_delete(cfg);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* A server-contacting --dry-run must never delete, even on the per-directory
|
||||||
|
(--delete-during/--delete-delay) commit path. The receive path already skips
|
||||||
|
plan application under -n, but a plan frame carrying --delete-missing-args
|
||||||
|
exact deletions used to be honored by delete_plan_session_commit(). Seed a
|
||||||
|
destination mirror, stream a plan naming it, and prove it survives. */
|
||||||
|
static void test_dry_run_delete_plan_commit_does_not_delete() {
|
||||||
|
char* root = make_check_root("drydelplan");
|
||||||
|
EXPECT_NOT_NULL(root);
|
||||||
|
write_check_file(root, "victim.txt", "must survive");
|
||||||
|
|
||||||
|
Config* cfg = config_create();
|
||||||
|
EXPECT_NOT_NULL(cfg);
|
||||||
|
cfg->send_directory = str_dup("/src");
|
||||||
|
cfg->receive_root_directory = str_dup(root);
|
||||||
|
cfg->use_delete = true;
|
||||||
|
cfg->delete_during = true;
|
||||||
|
cfg->delete_missing_args = true;
|
||||||
|
cfg->dry_run = true;
|
||||||
|
|
||||||
|
int p[2];
|
||||||
|
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
|
||||||
|
io_set_fds(p[0], p[1]);
|
||||||
|
io_set_bwlimit(0);
|
||||||
|
|
||||||
|
EXPECT_TRUE(send_status(p[1], STATUS_DELETE_PLAN));
|
||||||
|
EXPECT_TRUE(send_int(p[1], 1)); /* first frame carries the config sections */
|
||||||
|
EXPECT_TRUE(send_int(p[1], 0)); /* protected prefixes */
|
||||||
|
EXPECT_TRUE(send_int(p[1], 0)); /* size-skipped prefixes */
|
||||||
|
EXPECT_TRUE(send_int(p[1], 1)); /* missing-args exact deletions */
|
||||||
|
EXPECT_TRUE(send_str(p[1], "victim.txt"));
|
||||||
|
EXPECT_TRUE(send_str(p[1], ".")); /* receive root plan */
|
||||||
|
EXPECT_TRUE(send_int(p[1], 0)); /* kept child directories */
|
||||||
|
EXPECT_TRUE(send_int(p[1], 0)); /* kept child files */
|
||||||
|
EXPECT_TRUE(send_status(p[1], STATUS_FINISHED));
|
||||||
|
|
||||||
|
ReceiverSink sink = {.send_success = true};
|
||||||
|
EXPECT_EQ_INT(receiver_process_pending(cfg, p[0], &sink, NULL, NULL), 0);
|
||||||
|
|
||||||
|
char path[1024];
|
||||||
|
snprintf(path, sizeof(path), "%s/victim.txt", root);
|
||||||
|
EXPECT_EQ_INT(access(path, F_OK), 0);
|
||||||
|
|
||||||
|
close(p[0]);
|
||||||
|
close(p[1]);
|
||||||
|
config_delete(cfg);
|
||||||
|
remove(path);
|
||||||
|
rmdir(root);
|
||||||
|
free(root);
|
||||||
|
}
|
||||||
|
|
||||||
void test_server() {
|
void test_server() {
|
||||||
test_special_socket_path_log_escaped();
|
test_special_socket_path_log_escaped();
|
||||||
if (!is_running_under_valgrind()) {
|
if (!is_running_under_valgrind()) {
|
||||||
@@ -1119,5 +1200,6 @@ void test_server() {
|
|||||||
test_receive_manifest_three_sections();
|
test_receive_manifest_three_sections();
|
||||||
test_manifest_delete_missing_args();
|
test_manifest_delete_missing_args();
|
||||||
test_receiver_pending_commits_missing_args();
|
test_receiver_pending_commits_missing_args();
|
||||||
|
test_dry_run_delete_plan_commit_does_not_delete();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -234,6 +234,31 @@ static void test_server_cli_password_requires_daemon() {
|
|||||||
server_cli_options_free(&opts);
|
server_cli_options_free(&opts);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* --port is the rsync-style alias for -p, in both the separate and =value
|
||||||
|
* spellings; an invalid value is still validated. */
|
||||||
|
static void test_server_cli_port_alias() {
|
||||||
|
const char* a1[] = {"fastsync-server", "--port", "9000"};
|
||||||
|
ServerCliOptions opts;
|
||||||
|
EXPECT_EQ_INT(parse_ok(a1, 3, &opts), 0);
|
||||||
|
EXPECT_EQ_INT(opts.port, 9000);
|
||||||
|
EXPECT_TRUE(opts.port_set);
|
||||||
|
server_cli_options_free(&opts);
|
||||||
|
|
||||||
|
const char* a2[] = {"fastsync-server", "--port=9001"};
|
||||||
|
ServerCliOptions opts2;
|
||||||
|
EXPECT_EQ_INT(parse_ok(a2, 2, &opts2), 0);
|
||||||
|
EXPECT_EQ_INT(opts2.port, 9001);
|
||||||
|
EXPECT_TRUE(opts2.port_set);
|
||||||
|
server_cli_options_free(&opts2);
|
||||||
|
|
||||||
|
char err[128];
|
||||||
|
ServerCliOptions opts3;
|
||||||
|
const char* a3[] = {"fastsync-server", "--port", "notaport"};
|
||||||
|
EXPECT_EQ_INT(server_cli_parse(3, (char**)a3, &opts3, err, sizeof(err)), -1);
|
||||||
|
EXPECT_TRUE(strstr(err, "invalid port") != NULL);
|
||||||
|
server_cli_options_free(&opts3);
|
||||||
|
}
|
||||||
|
|
||||||
static void test_server_cli_help() {
|
static void test_server_cli_help() {
|
||||||
char err[256];
|
char err[256];
|
||||||
const char* a1[] = {"s", "--help"};
|
const char* a1[] = {"s", "--help"};
|
||||||
@@ -254,5 +279,6 @@ void test_server_cli() {
|
|||||||
test_server_cli_password_requires_daemon();
|
test_server_cli_password_requires_daemon();
|
||||||
test_server_cli_no_super();
|
test_server_cli_no_super();
|
||||||
test_server_cli_allow_super();
|
test_server_cli_allow_super();
|
||||||
|
test_server_cli_port_alias();
|
||||||
test_server_cli_help();
|
test_server_cli_help();
|
||||||
}
|
}
|
||||||
|
|||||||
+91
-59
@@ -136,7 +136,8 @@ static void test_walker_removes_extras_keeps_manifest_and_protected() {
|
|||||||
EXPECT_NOT_NULL(manifest);
|
EXPECT_NOT_NULL(manifest);
|
||||||
DeleteSkipEntry skip = {"prot", false};
|
DeleteSkipEntry skip = {"prot", false};
|
||||||
size_t deleted = 0;
|
size_t deleted = 0;
|
||||||
DeleteWalkResult result = delete_extras_limited(root, manifest, 100000, &skip, 1, &deleted);
|
DeleteWalkResult result =
|
||||||
|
delete_extras_limited(root, manifest, NULL, 100000, &skip, 1, &deleted, NULL);
|
||||||
EXPECT_EQ_INT((int)result, (int)DELETE_WALK_OK);
|
EXPECT_EQ_INT((int)result, (int)DELETE_WALK_OK);
|
||||||
EXPECT_FALSE(file_exists(root, "a.txt"));
|
EXPECT_FALSE(file_exists(root, "a.txt"));
|
||||||
EXPECT_TRUE(file_exists(root, "keep.txt"));
|
EXPECT_TRUE(file_exists(root, "keep.txt"));
|
||||||
@@ -170,7 +171,8 @@ static void test_walker_keeps_nested_manifest_dirs() {
|
|||||||
ArrayList* manifest = make_manifest_strings(keeps, 3);
|
ArrayList* manifest = make_manifest_strings(keeps, 3);
|
||||||
EXPECT_NOT_NULL(manifest);
|
EXPECT_NOT_NULL(manifest);
|
||||||
size_t deleted = 0;
|
size_t deleted = 0;
|
||||||
DeleteWalkResult result = delete_extras_limited(root, manifest, 100000, NULL, 0, &deleted);
|
DeleteWalkResult result =
|
||||||
|
delete_extras_limited(root, manifest, NULL, 100000, NULL, 0, &deleted, NULL);
|
||||||
EXPECT_EQ_INT((int)result, (int)DELETE_WALK_OK);
|
EXPECT_EQ_INT((int)result, (int)DELETE_WALK_OK);
|
||||||
EXPECT_FALSE(file_exists(root, "extra.txt"));
|
EXPECT_FALSE(file_exists(root, "extra.txt"));
|
||||||
EXPECT_TRUE(file_exists(root, "keepdir/deep/keep.txt"));
|
EXPECT_TRUE(file_exists(root, "keepdir/deep/keep.txt"));
|
||||||
@@ -187,7 +189,9 @@ static void test_walker_keeps_nested_manifest_dirs() {
|
|||||||
free(root);
|
free(root);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void test_walker_max_delete_exceeded_deletes_nothing() {
|
/* --max-delete is a partial cap (rsync parity): delete up to the limit, skip
|
||||||
|
the rest, and report DELETE_WALK_LIMIT_REACHED. */
|
||||||
|
static void test_walker_max_delete_partial_deletes_up_to_cap() {
|
||||||
char* root = make_walk_root("maxdel");
|
char* root = make_walk_root("maxdel");
|
||||||
EXPECT_NOT_NULL(root);
|
EXPECT_NOT_NULL(root);
|
||||||
EXPECT_TRUE(write_file_at(root, "a.txt", "extra"));
|
EXPECT_TRUE(write_file_at(root, "a.txt", "extra"));
|
||||||
@@ -197,12 +201,15 @@ static void test_walker_max_delete_exceeded_deletes_nothing() {
|
|||||||
ArrayList* manifest = make_manifest_strings(keeps, 0);
|
ArrayList* manifest = make_manifest_strings(keeps, 0);
|
||||||
EXPECT_NOT_NULL(manifest);
|
EXPECT_NOT_NULL(manifest);
|
||||||
size_t deleted = 999;
|
size_t deleted = 999;
|
||||||
DeleteWalkResult result = delete_extras_limited(root, manifest, 2, NULL, 0, &deleted);
|
size_t skipped = 0;
|
||||||
EXPECT_EQ_INT((int)result, (int)DELETE_WALK_LIMIT_EXCEEDED);
|
DeleteWalkResult result =
|
||||||
EXPECT_EQ_INT((int)deleted, 0);
|
delete_extras_limited(root, manifest, NULL, 2, NULL, 0, &deleted, &skipped);
|
||||||
EXPECT_TRUE(file_exists(root, "a.txt"));
|
EXPECT_EQ_INT((int)result, (int)DELETE_WALK_LIMIT_REACHED);
|
||||||
EXPECT_TRUE(file_exists(root, "b.txt"));
|
EXPECT_EQ_INT((int)deleted, 2);
|
||||||
EXPECT_TRUE(file_exists(root, "c.txt"));
|
EXPECT_EQ_INT((int)skipped, 1);
|
||||||
|
int remaining = (file_exists(root, "a.txt") ? 1 : 0) + (file_exists(root, "b.txt") ? 1 : 0) +
|
||||||
|
(file_exists(root, "c.txt") ? 1 : 0);
|
||||||
|
EXPECT_EQ_INT(remaining, 1);
|
||||||
array_list_delete(manifest);
|
array_list_delete(manifest);
|
||||||
remove_walk_tree(root);
|
remove_walk_tree(root);
|
||||||
free(root);
|
free(root);
|
||||||
@@ -217,7 +224,7 @@ static void test_walker_max_delete_exact_bound_deletes() {
|
|||||||
ArrayList* manifest = make_manifest_strings(keeps, 0);
|
ArrayList* manifest = make_manifest_strings(keeps, 0);
|
||||||
EXPECT_NOT_NULL(manifest);
|
EXPECT_NOT_NULL(manifest);
|
||||||
size_t deleted = 0;
|
size_t deleted = 0;
|
||||||
DeleteWalkResult result = delete_extras_limited(root, manifest, 2, NULL, 0, &deleted);
|
DeleteWalkResult result = delete_extras_limited(root, manifest, NULL, 2, NULL, 0, &deleted, NULL);
|
||||||
EXPECT_EQ_INT((int)result, (int)DELETE_WALK_OK);
|
EXPECT_EQ_INT((int)result, (int)DELETE_WALK_OK);
|
||||||
EXPECT_EQ_INT((int)deleted, 2);
|
EXPECT_EQ_INT((int)deleted, 2);
|
||||||
EXPECT_FALSE(file_exists(root, "a.txt"));
|
EXPECT_FALSE(file_exists(root, "a.txt"));
|
||||||
@@ -227,6 +234,77 @@ static void test_walker_max_delete_exact_bound_deletes() {
|
|||||||
free(root);
|
free(root);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Extraneous destination symlinks (including one pointing at a directory) must
|
||||||
|
be unlinked, never followed, so their targets survive. */
|
||||||
|
static void test_walker_removes_extraneous_symlinks() {
|
||||||
|
char* root = make_walk_root("symlink");
|
||||||
|
char* outside = make_walk_root("symlink_out");
|
||||||
|
EXPECT_NOT_NULL(root);
|
||||||
|
EXPECT_NOT_NULL(outside);
|
||||||
|
EXPECT_TRUE(write_file_at(outside, "secret.txt", "keep"));
|
||||||
|
EXPECT_TRUE(write_file_at(root, "keep.txt", "kept"));
|
||||||
|
char* link_file = path_cat(root, "link_file");
|
||||||
|
char* link_dir = path_cat(root, "link_dir");
|
||||||
|
char* link_broken = path_cat(root, "link_broken");
|
||||||
|
EXPECT_NOT_NULL(link_file);
|
||||||
|
EXPECT_NOT_NULL(link_dir);
|
||||||
|
EXPECT_NOT_NULL(link_broken);
|
||||||
|
EXPECT_EQ_INT(symlink("keep.txt", link_file), 0);
|
||||||
|
EXPECT_EQ_INT(symlink(outside, link_dir), 0);
|
||||||
|
EXPECT_EQ_INT(symlink("/nonexistent-target", link_broken), 0);
|
||||||
|
const char* keeps[] = {"keep.txt"};
|
||||||
|
ArrayList* manifest = make_manifest_strings(keeps, 1);
|
||||||
|
EXPECT_NOT_NULL(manifest);
|
||||||
|
size_t deleted = 0;
|
||||||
|
DeleteWalkResult result =
|
||||||
|
delete_extras_limited(root, manifest, NULL, 100000, NULL, 0, &deleted, NULL);
|
||||||
|
EXPECT_EQ_INT((int)result, (int)DELETE_WALK_OK);
|
||||||
|
EXPECT_FALSE(file_exists(root, "link_file"));
|
||||||
|
EXPECT_FALSE(file_exists(root, "link_dir"));
|
||||||
|
EXPECT_FALSE(file_exists(root, "link_broken"));
|
||||||
|
EXPECT_TRUE(file_exists(root, "keep.txt"));
|
||||||
|
EXPECT_TRUE(file_exists(outside, "secret.txt"));
|
||||||
|
free(link_file);
|
||||||
|
free(link_dir);
|
||||||
|
free(link_broken);
|
||||||
|
array_list_delete(manifest);
|
||||||
|
remove_walk_tree(root);
|
||||||
|
remove_walk_tree(outside);
|
||||||
|
free(root);
|
||||||
|
free(outside);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* With a synchronized-dir set, extras outside it survive while extras directly
|
||||||
|
inside a listed directory are removed; the receive root is the "." sentinel. */
|
||||||
|
static void test_walker_confines_deletion_to_synced_dirs() {
|
||||||
|
char* root = make_walk_root("synced");
|
||||||
|
EXPECT_NOT_NULL(root);
|
||||||
|
EXPECT_TRUE(write_file_at(root, "rootextra.txt", "keep"));
|
||||||
|
EXPECT_EQ_INT(make_subdir(root, "inscope"), 0);
|
||||||
|
EXPECT_TRUE(write_file_at(root, "inscope/extra.txt", "delete"));
|
||||||
|
EXPECT_TRUE(write_file_at(root, "inscope/keep.txt", "kept"));
|
||||||
|
EXPECT_EQ_INT(make_subdir(root, "outscope"), 0);
|
||||||
|
EXPECT_TRUE(write_file_at(root, "outscope/extra.txt", "keep"));
|
||||||
|
const char* keeps[] = {"inscope/keep.txt"};
|
||||||
|
ArrayList* manifest = make_manifest_strings(keeps, 1);
|
||||||
|
ArrayList* dirs = array_list_create(free);
|
||||||
|
EXPECT_NOT_NULL(manifest);
|
||||||
|
EXPECT_NOT_NULL(dirs);
|
||||||
|
EXPECT_TRUE(array_list_add(dirs, str_dup("inscope")));
|
||||||
|
size_t deleted = 0;
|
||||||
|
DeleteWalkResult result =
|
||||||
|
delete_extras_limited(root, manifest, dirs, 100000, NULL, 0, &deleted, NULL);
|
||||||
|
EXPECT_EQ_INT((int)result, (int)DELETE_WALK_OK);
|
||||||
|
EXPECT_TRUE(file_exists(root, "rootextra.txt"));
|
||||||
|
EXPECT_FALSE(file_exists(root, "inscope/extra.txt"));
|
||||||
|
EXPECT_TRUE(file_exists(root, "inscope/keep.txt"));
|
||||||
|
EXPECT_TRUE(file_exists(root, "outscope/extra.txt"));
|
||||||
|
array_list_delete(manifest);
|
||||||
|
array_list_delete(dirs);
|
||||||
|
remove_walk_tree(root);
|
||||||
|
free(root);
|
||||||
|
}
|
||||||
|
|
||||||
static void test_walker_unlimited_deletes_all() {
|
static void test_walker_unlimited_deletes_all() {
|
||||||
char* root = make_walk_root("unlim");
|
char* root = make_walk_root("unlim");
|
||||||
EXPECT_NOT_NULL(root);
|
EXPECT_NOT_NULL(root);
|
||||||
@@ -245,53 +323,6 @@ static void test_walker_unlimited_deletes_all() {
|
|||||||
free(root);
|
free(root);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* The 100000-entry server hard bound (MAX_SERVER_DELETE_COUNT, which this test
|
|
||||||
exercises through a literal to avoid reaching into file_receive.c) is also
|
|
||||||
all-or-nothing: a destination holding more extras than the bound must be left
|
|
||||||
completely untouched. Skipped under valgrind: 100k file creations would be
|
|
||||||
far too slow under instrumentation. */
|
|
||||||
static void test_walker_hard_bound_all_or_nothing() {
|
|
||||||
if (is_running_under_valgrind())
|
|
||||||
return;
|
|
||||||
enum { HARD_BOUND = 100000 };
|
|
||||||
char* root = make_walk_root("hardbound");
|
|
||||||
EXPECT_NOT_NULL(root);
|
|
||||||
int rootfd = open(root, O_RDONLY | O_DIRECTORY | O_CLOEXEC);
|
|
||||||
EXPECT_TRUE(rootfd >= 0);
|
|
||||||
bool created = true;
|
|
||||||
for (int i = 0; created && i < HARD_BOUND + 1; i++) {
|
|
||||||
char name[32];
|
|
||||||
snprintf(name, sizeof(name), "f%d", i);
|
|
||||||
int fd = openat(rootfd, name, O_WRONLY | O_CREAT | O_TRUNC, 0644);
|
|
||||||
if (fd < 0)
|
|
||||||
created = false;
|
|
||||||
else
|
|
||||||
close(fd);
|
|
||||||
}
|
|
||||||
EXPECT_TRUE(created);
|
|
||||||
const char* keeps[1] = {NULL};
|
|
||||||
ArrayList* manifest = make_manifest_strings(keeps, 0);
|
|
||||||
EXPECT_NOT_NULL(manifest);
|
|
||||||
size_t deleted = 999;
|
|
||||||
DeleteWalkResult result = delete_extras_limited(root, manifest, HARD_BOUND, NULL, 0, &deleted);
|
|
||||||
EXPECT_EQ_INT((int)result, (int)DELETE_WALK_LIMIT_EXCEEDED);
|
|
||||||
EXPECT_EQ_INT((int)deleted, 0);
|
|
||||||
EXPECT_TRUE(file_exists(root, "f0"));
|
|
||||||
EXPECT_TRUE(file_exists(root, "f100000"));
|
|
||||||
array_list_delete(manifest);
|
|
||||||
/* Fast cleanup: unlink every created name through the still-open root fd. */
|
|
||||||
if (rootfd >= 0) {
|
|
||||||
for (int i = 0; i < HARD_BOUND + 1; i++) {
|
|
||||||
char name[32];
|
|
||||||
snprintf(name, sizeof(name), "f%d", i);
|
|
||||||
(void)unlinkat(rootfd, name, 0);
|
|
||||||
}
|
|
||||||
close(rootfd);
|
|
||||||
}
|
|
||||||
rmdir(root);
|
|
||||||
free(root);
|
|
||||||
}
|
|
||||||
|
|
||||||
typedef struct {
|
typedef struct {
|
||||||
bool eight_bit_output;
|
bool eight_bit_output;
|
||||||
const char* expected;
|
const char* expected;
|
||||||
@@ -553,10 +584,11 @@ void test_shared_utils() {
|
|||||||
test_getdelim_bounded();
|
test_getdelim_bounded();
|
||||||
test_walker_removes_extras_keeps_manifest_and_protected();
|
test_walker_removes_extras_keeps_manifest_and_protected();
|
||||||
test_walker_keeps_nested_manifest_dirs();
|
test_walker_keeps_nested_manifest_dirs();
|
||||||
test_walker_max_delete_exceeded_deletes_nothing();
|
test_walker_max_delete_partial_deletes_up_to_cap();
|
||||||
test_walker_max_delete_exact_bound_deletes();
|
test_walker_max_delete_exact_bound_deletes();
|
||||||
|
test_walker_removes_extraneous_symlinks();
|
||||||
|
test_walker_confines_deletion_to_synced_dirs();
|
||||||
test_walker_unlimited_deletes_all();
|
test_walker_unlimited_deletes_all();
|
||||||
test_walker_hard_bound_all_or_nothing();
|
|
||||||
test_loopback_helpers();
|
test_loopback_helpers();
|
||||||
test_fd_peer_ip();
|
test_fd_peer_ip();
|
||||||
|
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user