Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7585f46eb4 | ||
|
|
00197102bf | ||
|
|
13b257d1dd | ||
|
|
f3d7672694 | ||
|
|
18b821d32c | ||
|
|
6ad065925f | ||
|
|
46dcefe218 | ||
|
|
b88acdbd3c | ||
|
|
c29bce54fc | ||
|
|
d98e971fcc | ||
|
|
492ce0ce89 | ||
|
|
ec6692ac41 | ||
|
|
1f8d60e30d | ||
|
|
6db9827b87 | ||
|
|
a07cc00bb0 | ||
|
|
3ae7685655 | ||
|
|
4f945a8e39 | ||
|
|
15f38f5b76 | ||
|
|
787967d3ce | ||
|
|
06e7aef5c9 | ||
|
|
ee265d78ba | ||
|
|
47b1b9b915 | ||
|
|
bb6c788cf9 | ||
|
|
0b40c47d6a | ||
|
|
6f81005094 | ||
|
|
193d358c64 | ||
|
|
8792e3265a | ||
|
|
3ec0ffb644 | ||
|
|
80e8d7f450 | ||
|
|
86741725fd | ||
|
|
f96f1764af | ||
|
|
d780a4625e | ||
|
|
5d1303ffdf | ||
|
|
06b53c5b3d | ||
|
|
bf09e8893e | ||
|
|
034c27926f | ||
|
|
aa15099d22 | ||
|
|
ff4db23831 | ||
|
|
79e45441c0 | ||
|
|
6537227467 | ||
|
|
309c9aed98 | ||
|
|
84594197ee | ||
|
|
7bf25048f6 | ||
|
|
b6b5eee20e | ||
|
|
8dcd87609d | ||
|
|
19fe63bd59 | ||
|
|
1f5f8dc5a7 | ||
|
|
3787695ba6 | ||
|
|
b7cb213c8c | ||
|
|
8cc3dd993b | ||
|
|
1167e7970b | ||
|
|
e98729f00e | ||
|
|
c0020364b2 | ||
|
|
d7ac940a6b | ||
|
|
be20e836de | ||
|
|
b549887138 | ||
|
|
1ffd4744c6 | ||
|
|
494cef2a0b | ||
|
|
ed7527cc2c | ||
|
|
934defa965 | ||
|
|
ade9be8600 | ||
|
|
707bb659e8 | ||
|
|
33980bc4c8 | ||
|
|
6a9372f4f5 | ||
|
|
5e1d6b6e10 | ||
|
|
d2d1b63f44 | ||
|
|
a6659472fe | ||
|
|
4638030288 | ||
|
|
07dfec629f | ||
|
|
c062a0762e | ||
|
|
283f9f0823 | ||
|
|
5754b9a952 | ||
|
|
b8a0efef7b | ||
|
|
2e77c09447 | ||
|
|
06c4026b74 | ||
|
|
9f47b13712 | ||
|
|
b1eddf0133 | ||
|
|
338c27db73 | ||
|
|
8d46a26c04 | ||
|
|
707ba272df | ||
|
|
bc71a3c0a5 | ||
|
|
cee9b7647c | ||
|
|
3adb6dddb5 | ||
|
|
d77849774e | ||
|
|
b5c6f8b60f | ||
|
|
134dcd74cc | ||
|
|
42f2f845ac | ||
|
|
0669335ca5 | ||
|
|
391f76cd56 | ||
|
|
2021afe613 | ||
|
|
ba914e8ab3 | ||
|
|
df8fe1ae4e | ||
|
|
2c490d58b7 | ||
|
|
d55dabff2e | ||
|
|
b478a59a81 | ||
|
|
865941f850 | ||
|
|
221cefa7cc | ||
|
|
bb9b59024a | ||
|
|
5baf120243 | ||
|
|
84b7e1fd2f | ||
|
|
800a978e15 | ||
|
|
0f40e747f0 | ||
|
|
b07306d5bc | ||
|
|
f2c89b6e7c | ||
|
|
379f127859 | ||
|
|
3799200f71 | ||
|
|
91c4a967e6 | ||
|
|
88c8968b4c | ||
|
|
082b31886a | ||
|
|
423a62e691 | ||
|
|
bc18ae205b | ||
|
|
10a61c6101 | ||
|
|
bc1e1191af | ||
|
|
0fbb9de915 | ||
|
|
9b05972375 | ||
|
|
d119f35066 | ||
|
|
00829fd265 | ||
|
|
ff261bc38a | ||
|
|
b235721f8b | ||
|
|
79a28cdb96 | ||
|
|
402cae80ad | ||
|
|
9691dba6f0 | ||
|
|
558782d339 | ||
|
|
5597e74f6a | ||
|
|
b4d54504f9 | ||
|
|
ee6523afac | ||
|
|
4163caa1d3 | ||
|
|
10159dc120 | ||
|
|
cd7b96d0bb | ||
|
|
f6f49d536e | ||
|
|
38d304103c | ||
|
|
4b09213b88 | ||
|
|
36fd0774e8 | ||
|
|
711b7e50b3 | ||
|
|
67076bf218 | ||
|
|
8ec8cb7203 | ||
|
|
82959395fb | ||
|
|
c9f94ea46e | ||
|
|
eff9852038 | ||
|
|
c80098623f | ||
|
|
6119e1e75c | ||
|
|
25062352f6 | ||
|
|
00d628d4ba | ||
|
|
3545d88905 | ||
|
|
596a039454 | ||
|
|
ae037cc27b | ||
|
|
3d0672a721 | ||
|
|
b82aab72c5 | ||
|
|
8e7764007d | ||
|
|
1042d15db7 | ||
|
|
cbe37a77dd | ||
|
|
a5d45ef266 | ||
|
|
a960391b34 | ||
|
|
134f8b027a | ||
|
|
eb7e3fd2e0 | ||
|
|
6a129b54d4 | ||
|
|
c7b2c7eb2b | ||
|
|
e77dfbec70 | ||
|
|
f3ac4df4d0 | ||
|
|
91197fd7cf | ||
|
|
13708352ec | ||
|
|
d162d93570 | ||
|
|
9fa1696eff | ||
|
|
b705fb807f | ||
|
|
ef76c9034d | ||
|
|
cee281ff55 | ||
|
|
5c509831b8 | ||
|
|
ee57aeea4a | ||
|
|
803c1d3385 | ||
|
|
b8ec62beef | ||
|
|
59bfd32b9e | ||
|
|
3432a33d9a | ||
|
|
a1b081d328 | ||
|
|
bbecff9c04 | ||
|
|
c1553bd5d6 | ||
|
|
1a550bda24 | ||
|
|
902f86192d | ||
|
|
6a40ac86e5 | ||
|
|
410ba6e992 | ||
|
|
6c6f02e5dd | ||
|
|
d9006d1fda | ||
|
|
36375010d3 | ||
|
|
5efa0dba7c | ||
|
|
e32733fbf6 | ||
|
|
b02799327d | ||
|
|
946aa934cc | ||
|
|
125921c11b | ||
|
|
9dd5288381 | ||
|
|
3f2c74dd9e | ||
|
|
51e41dee2a | ||
|
|
dbf1b39d47 | ||
|
|
845f20a28d | ||
|
|
a5083776da | ||
|
|
76a81f1684 | ||
|
|
24b81c7e5a | ||
|
|
0e33f84f38 | ||
|
|
c7ac039523 | ||
|
|
e771cc9da6 | ||
|
|
7f9f82a068 | ||
|
|
695b5c8c25 | ||
|
|
5b2188d909 | ||
|
|
e5da916d54 | ||
|
|
de640bba1b | ||
|
|
f0f5719be0 | ||
|
|
6200b298ac | ||
|
|
394a9aae22 | ||
|
|
12d4af1b89 | ||
|
|
9d7c55d3c0 | ||
|
|
5a104bfd88 | ||
|
|
2ada8f9ad5 | ||
|
|
1493f1806d | ||
|
|
ea28e25535 | ||
|
|
d6295d62ce | ||
|
|
a9f416ce44 | ||
|
|
448edc0432 | ||
|
|
4a7703b06a | ||
|
|
6fc297544e | ||
|
|
583d3c8edb | ||
|
|
9883757190 | ||
|
|
a690109975 | ||
|
|
36d4d0e43e | ||
|
|
a0b9d9794b | ||
|
|
3e9f70d9ba | ||
|
|
2b5aaef409 | ||
|
|
478f80be9f | ||
|
|
e674b25213 | ||
|
|
1116da9f64 | ||
|
|
684153350a | ||
|
|
ec206b02d0 | ||
|
|
88bdfeeb58 | ||
|
|
3f5b0250f4 | ||
|
|
c41bfb2cdb | ||
|
|
1b2632f968 | ||
|
|
7dbca70a4b | ||
|
|
1a053f06e5 | ||
|
|
58b3a33e82 | ||
|
|
17b0632098 | ||
|
|
376e6500ab | ||
|
|
82a1d5e240 | ||
|
|
3eec5a4cc3 | ||
|
|
6144c7fc7f | ||
|
|
ea4ab661b4 | ||
|
|
84827ca617 | ||
|
|
23552e823d | ||
|
|
d1a567f7e3 | ||
|
|
93c1fc3c1f | ||
|
|
4815b1b281 | ||
|
|
ad7bc3348b | ||
|
|
34970b961c | ||
|
|
b3f7cad4db | ||
|
|
cd8a84c0a2 | ||
|
|
09c384d7d0 | ||
|
|
81ad313ee5 |
@@ -9,7 +9,7 @@ on:
|
|||||||
jobs:
|
jobs:
|
||||||
lint:
|
lint:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
container: gitea.tap-tap.win/taptap/fastsync-ci:v10
|
container: gitea.tap-tap.win/taptap/fastsync-ci:v11
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||||
@@ -26,7 +26,7 @@ jobs:
|
|||||||
# suite) run on merge to dev/main, so PR CI stays well under ~3 minutes.
|
# suite) run on merge to dev/main, so PR CI stays well under ~3 minutes.
|
||||||
build-and-test:
|
build-and-test:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
container: gitea.tap-tap.win/taptap/fastsync-ci:v10
|
container: gitea.tap-tap.win/taptap/fastsync-ci:v11
|
||||||
needs: lint
|
needs: lint
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
@@ -49,9 +49,50 @@ jobs:
|
|||||||
if: github.event_name == 'push'
|
if: github.event_name == 'push'
|
||||||
run: python3 -m pytest tests/integration/ -n 4 --dist=load -m "not setpriv" --durations=25 --tb=short -q
|
run: python3 -m pytest tests/integration/ -n 4 --dist=load -m "not setpriv" --durations=25 --tb=short -q
|
||||||
|
|
||||||
|
# Differential rsync-parity gate: runs real rsync 3.4.1 and FastSync over the
|
||||||
|
# same corpora and compares destinations + normalized output. The fast subset
|
||||||
|
# guards the ✅ surface on every PR; the full set (with FASTSYNC_PARITY_STRICT
|
||||||
|
# so a fixed caveat must be removed from the allowlist) burns the documented
|
||||||
|
# ⚠️/❌ residuals down on push. See tests/integration/README.md.
|
||||||
|
parity-fast:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
container: gitea.tap-tap.win/taptap/fastsync-ci:v11
|
||||||
|
needs: lint
|
||||||
|
if: github.event_name == 'pull_request'
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||||
|
|
||||||
|
- name: Configure
|
||||||
|
run: cmake -B build -S . -DSTRICT_WARNINGS=ON
|
||||||
|
|
||||||
|
- name: Build
|
||||||
|
run: cmake --build build -j$(nproc)
|
||||||
|
|
||||||
|
- name: Differential parity (fast subset)
|
||||||
|
run: python3 -m pytest tests/integration/test_differential_parity.py -n 4 --dist=load -m parity_ci -q
|
||||||
|
|
||||||
|
parity-full:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
container: gitea.tap-tap.win/taptap/fastsync-ci:v11
|
||||||
|
needs: lint
|
||||||
|
if: github.event_name == 'push'
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||||
|
|
||||||
|
- name: Configure
|
||||||
|
run: cmake -B build -S . -DSTRICT_WARNINGS=ON
|
||||||
|
|
||||||
|
- name: Build
|
||||||
|
run: cmake --build build -j$(nproc)
|
||||||
|
|
||||||
|
- name: Differential parity (full set)
|
||||||
|
run: FASTSYNC_PARITY_STRICT=1 python3 -m pytest tests/integration/test_differential_parity.py -n 4 --dist=load -m parity -q
|
||||||
|
|
||||||
sanitizers:
|
sanitizers:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
container: gitea.tap-tap.win/taptap/fastsync-ci:v10
|
container: gitea.tap-tap.win/taptap/fastsync-ci:v11
|
||||||
needs: lint
|
needs: lint
|
||||||
if: github.event_name == 'push'
|
if: github.event_name == 'push'
|
||||||
strategy:
|
strategy:
|
||||||
@@ -72,7 +113,7 @@ jobs:
|
|||||||
|
|
||||||
fuzz-build:
|
fuzz-build:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
container: gitea.tap-tap.win/taptap/fastsync-ci:v10
|
container: gitea.tap-tap.win/taptap/fastsync-ci:v11
|
||||||
needs: lint
|
needs: lint
|
||||||
if: github.event_name == 'push'
|
if: github.event_name == 'push'
|
||||||
steps:
|
steps:
|
||||||
@@ -94,7 +135,7 @@ jobs:
|
|||||||
|
|
||||||
coverage:
|
coverage:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
container: gitea.tap-tap.win/taptap/fastsync-ci:v10
|
container: gitea.tap-tap.win/taptap/fastsync-ci:v11
|
||||||
needs: lint
|
needs: lint
|
||||||
if: github.event_name == 'push'
|
if: github.event_name == 'push'
|
||||||
steps:
|
steps:
|
||||||
@@ -118,7 +159,7 @@ jobs:
|
|||||||
|
|
||||||
valgrind:
|
valgrind:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
container: gitea.tap-tap.win/taptap/fastsync-ci:v10
|
container: gitea.tap-tap.win/taptap/fastsync-ci:v11
|
||||||
needs: lint
|
needs: lint
|
||||||
if: github.event_name == 'push'
|
if: github.event_name == 'push'
|
||||||
steps:
|
steps:
|
||||||
|
|||||||
@@ -12,3 +12,12 @@ build_docker2/
|
|||||||
# Test/run artifacts
|
# Test/run artifacts
|
||||||
root/
|
root/
|
||||||
test_partial_install_tmp/
|
test_partial_install_tmp/
|
||||||
|
|
||||||
|
# Editor/tooling + test caches/artifacts
|
||||||
|
.pytest_cache/
|
||||||
|
*.gcda
|
||||||
|
*.gcno
|
||||||
|
*.gcov
|
||||||
|
di/
|
||||||
|
test_data-manual/
|
||||||
|
*.log
|
||||||
|
|||||||
@@ -55,6 +55,16 @@ if(NOT ZSTD_LIBRARY)
|
|||||||
message(FATAL_ERROR "zstd library not found. Ensure it is in your nix-shell!")
|
message(FATAL_ERROR "zstd library not found. Ensure it is in your nix-shell!")
|
||||||
endif()
|
endif()
|
||||||
|
|
||||||
|
find_library(ZLIB_LIBRARY z)
|
||||||
|
if(NOT ZLIB_LIBRARY)
|
||||||
|
message(FATAL_ERROR "zlib library not found. Ensure zlib1g-dev / nix zlib is available!")
|
||||||
|
endif()
|
||||||
|
|
||||||
|
find_library(LZ4_LIBRARY lz4)
|
||||||
|
if(NOT LZ4_LIBRARY)
|
||||||
|
message(FATAL_ERROR "lz4 library not found. Ensure liblz4-dev / nix lz4 is available!")
|
||||||
|
endif()
|
||||||
|
|
||||||
find_package(OpenSSL REQUIRED)
|
find_package(OpenSSL REQUIRED)
|
||||||
|
|
||||||
file(GLOB SHARED_SRCS "src/shared/*.c")
|
file(GLOB SHARED_SRCS "src/shared/*.c")
|
||||||
@@ -64,15 +74,15 @@ file(GLOB TEST_SRCS "tests/*.c")
|
|||||||
|
|
||||||
add_executable(server ${SERVER_SRCS} ${SHARED_SRCS})
|
add_executable(server ${SERVER_SRCS} ${SHARED_SRCS})
|
||||||
target_include_directories(server PRIVATE src/shared src/server src/client)
|
target_include_directories(server PRIVATE src/shared src/server src/client)
|
||||||
target_link_libraries(server PRIVATE Threads::Threads ${ZSTD_LIBRARY} OpenSSL::SSL OpenSSL::Crypto xxhash)
|
target_link_libraries(server PRIVATE Threads::Threads ${ZSTD_LIBRARY} ${ZLIB_LIBRARY} ${LZ4_LIBRARY} OpenSSL::SSL OpenSSL::Crypto xxhash)
|
||||||
|
|
||||||
add_executable(client ${CLIENT_SRCS} ${SHARED_SRCS})
|
add_executable(client ${CLIENT_SRCS} ${SHARED_SRCS})
|
||||||
target_include_directories(client PRIVATE src/shared src/server src/client)
|
target_include_directories(client PRIVATE src/shared src/server src/client)
|
||||||
target_link_libraries(client PRIVATE Threads::Threads ${ZSTD_LIBRARY} OpenSSL::SSL OpenSSL::Crypto xxhash)
|
target_link_libraries(client PRIVATE Threads::Threads ${ZSTD_LIBRARY} ${ZLIB_LIBRARY} ${LZ4_LIBRARY} OpenSSL::SSL OpenSSL::Crypto xxhash)
|
||||||
|
|
||||||
add_executable(tests ${TEST_SRCS} ${SHARED_SRCS} src/client/scanner.c)
|
add_executable(tests ${TEST_SRCS} ${SHARED_SRCS} src/client/scanner.c)
|
||||||
target_include_directories(tests PRIVATE tests src/shared src/server src/client)
|
target_include_directories(tests PRIVATE tests src/shared src/server src/client)
|
||||||
target_link_libraries(tests PRIVATE Threads::Threads ${ZSTD_LIBRARY} OpenSSL::SSL OpenSSL::Crypto xxhash)
|
target_link_libraries(tests PRIVATE Threads::Threads ${ZSTD_LIBRARY} ${ZLIB_LIBRARY} ${LZ4_LIBRARY} OpenSSL::SSL OpenSSL::Crypto xxhash)
|
||||||
```
|
```
|
||||||
|
|
||||||
### Source Layout
|
### Source Layout
|
||||||
@@ -85,17 +95,23 @@ tests/integration/ — Python pytest integration tests
|
|||||||
```
|
```
|
||||||
|
|
||||||
### Dependencies
|
### Dependencies
|
||||||
- **zstd** — found via `find_library(ZSTD_LIBRARY zstd)`
|
- **zstd** — found via `find_library(ZSTD_LIBRARY zstd)` (default compression codec)
|
||||||
|
- **zlib** — found via `find_library(ZLIB_LIBRARY z)` (the `zlib`/`zlibx` codecs)
|
||||||
|
- **lz4** — found via `find_library(LZ4_LIBRARY lz4)` (the `lz4` codec)
|
||||||
- **OpenSSL** — found via `find_package(OpenSSL REQUIRED)` (TLS 1.2+ transport)
|
- **OpenSSL** — found via `find_package(OpenSSL REQUIRED)` (TLS 1.2+ transport)
|
||||||
- **xxHash** — fetched via `FetchContent` from the upstream repository (delta transfer hashing, v0.8.3)
|
- **xxHash** — fetched via `FetchContent` from the upstream repository (delta transfer hashing, v0.8.3)
|
||||||
- **pthreads** — found via `find_package(Threads REQUIRED)`
|
- **pthreads** — found via `find_package(Threads REQUIRED)`
|
||||||
- **C11 standard** — required
|
- **C11 standard** — required
|
||||||
- **CMake 3.22+** — minimum version
|
- **CMake 3.22+** — minimum version
|
||||||
|
|
||||||
|
The codec matrix (protocol 2.26.0) uses zstd/zlib/lz4 for compression and
|
||||||
|
xxHash/OpenSSL for the `xxh128`/`xxh3`/`xxh64`/`md5`/`md4`/`sha1` checksums
|
||||||
|
(`none` needs no library); both codec families are negotiated per transfer.
|
||||||
|
|
||||||
## Conventions
|
## Conventions
|
||||||
|
|
||||||
- Use `file(GLOB ...)` for source collection (existing pattern).
|
- Use `file(GLOB ...)` for source collection (existing pattern).
|
||||||
- All targets link `Threads::Threads`, `${ZSTD_LIBRARY}`, `OpenSSL::SSL`, `OpenSSL::Crypto`, and `xxhash`.
|
- All targets link `Threads::Threads`, `${ZSTD_LIBRARY}`, `${ZLIB_LIBRARY}`, `${LZ4_LIBRARY}`, `OpenSSL::SSL`, `OpenSSL::Crypto`, and `xxhash`.
|
||||||
- Include directories: `src/shared`, `src/server`, `src/client`, `tests` (for test target).
|
- Include directories: `src/shared`, `src/server`, `src/client`, `tests` (for test target).
|
||||||
- Sanitizer support: pass `-DSANITIZER=address`, `-DSANITIZER=thread`, or `-DSANITIZER=undefined` to cmake (live option in CMakeLists.txt).
|
- Sanitizer support: pass `-DSANITIZER=address`, `-DSANITIZER=thread`, or `-DSANITIZER=undefined` to cmake (live option in CMakeLists.txt).
|
||||||
- Build with `cmake -B build -S . && cmake --build build -j$(nproc)`.
|
- Build with `cmake -B build -S . && cmake --build build -j$(nproc)`.
|
||||||
|
|||||||
@@ -116,7 +116,7 @@ The project uses Gitea Actions. Key jobs:
|
|||||||
jobs:
|
jobs:
|
||||||
new-job:
|
new-job:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
container: gitea.tap-tap.win/taptap/fastsync-ci:v10
|
container: gitea.tap-tap.win/taptap/fastsync-ci:v11
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
- name: Configure
|
- name: Configure
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ Ask the user or determine from context:
|
|||||||
- **Minor** (x.Y.0) — new features, backward compatible
|
- **Minor** (x.Y.0) — new features, backward compatible
|
||||||
- **Patch** (x.y.Z) — bug fixes, no protocol changes
|
- **Patch** (x.y.Z) — bug fixes, no protocol changes
|
||||||
|
|
||||||
Current version: `PROTOCOL_VERSION "2.21.0"` in `src/shared/config.h`
|
Current version: `PROTOCOL_VERSION "2.29.0"` in `src/shared/config.h`
|
||||||
|
|
||||||
### Step 2: Check Protocol Version
|
### Step 2: Check Protocol Version
|
||||||
|
|
||||||
|
|||||||
@@ -4,18 +4,19 @@ FastSync is a high-performance file synchronization system written in C11. It su
|
|||||||
|
|
||||||
## Dependency installation
|
## Dependency installation
|
||||||
|
|
||||||
**CI rule:** never add `apt-get install` / `pip install` steps to CI workflows — use the custom Docker image instead. The image is built from the repo-root `Dockerfile` and is the same image CI uses: `gitea.tap-tap.win/taptap/fastsync-ci:v10`. It contains the full toolchain: gcc/g++, CMake, libzstd-dev, libssl-dev, make, git, cppcheck, clang-format, python3 + pytest + pytest-xdist, openssh-client, and Node.js.
|
**CI rule:** never add `apt-get install` / `pip install` steps to CI workflows — use the custom Docker image instead. The image is built from the repo-root `Dockerfile` and is the same image CI uses: `gitea.tap-tap.win/taptap/fastsync-ci:v11`. It contains the full toolchain: gcc/g++, CMake, libzstd-dev, zlib1g-dev, liblz4-dev, libxxhash-dev, libssl-dev, make, git, cppcheck, clang-format, python3 + pytest + pytest-xdist, openssh-client, Node.js, plus `rsync` 3.4.1 (with zstd/xxhash/lz4), `acl` and `attr` (setfacl/getfacl, setfattr/getfattr) for drop-in parity tests. (CMake hard-requires zstd, zlib, and lz4; xxHash is fetched via `FetchContent`.)
|
||||||
|
|
||||||
**Host rule:** for local development, use `nix-shell` (see `README.md`) which provides zstd, OpenSSL, CMake, and gcc. The Docker image can also be used locally for CI parity.
|
**Host rule:** for local development, use `nix-shell` (see `README.md`) which provides zstd, zlib, lz4, OpenSSL, CMake, and gcc. The Docker image can also be used locally for CI parity.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Use the prebuilt CI image directly (faster, guaranteed CI parity)
|
# Use the prebuilt CI image directly (faster, guaranteed CI parity)
|
||||||
docker pull gitea.tap-tap.win/taptap/fastsync-ci:v10
|
docker pull gitea.tap-tap.win/taptap/fastsync-ci:v11
|
||||||
docker tag gitea.tap-tap.win/taptap/fastsync-ci:v10 fastsync-ci:local
|
docker tag gitea.tap-tap.win/taptap/fastsync-ci:v11 fastsync-ci:local
|
||||||
|
|
||||||
# Or build the image from the repo-root Dockerfile
|
# Or build the image from the repo-root Dockerfile
|
||||||
# (Note: the prebuilt :v10 image reflects the previous Dockerfile state;
|
# (Note: the prebuilt :v11 image is built from the current Dockerfile and
|
||||||
# rebuild from source to pick up any newly added packages like lcov/valgrind.)
|
# includes rsync 3.4.1 plus acl/attr; rebuild from source after changing
|
||||||
|
# the Dockerfile.)
|
||||||
docker build -t fastsync-ci:local .
|
docker build -t fastsync-ci:local .
|
||||||
|
|
||||||
# Build, run unit tests, and run integration tests inside the container
|
# Build, run unit tests, and run integration tests inside the container
|
||||||
@@ -37,11 +38,12 @@ If a dependency is missing from the CI image, add it to the `Dockerfile` (and re
|
|||||||
When configuring for CI parity, use:
|
When configuring for CI parity, use:
|
||||||
```bash
|
```bash
|
||||||
cmake -B build -S . -DSTRICT_WARNINGS=ON # -Wextra -Wpedantic -Werror
|
cmake -B build -S . -DSTRICT_WARNINGS=ON # -Wextra -Wpedantic -Werror
|
||||||
cmake -B build -S . -DSANITIZER=address # AddressSanitizer (ASan)
|
cmake -B build -S . -DSANITIZER=address # AddressSanitizer (ASan); in the CI matrix
|
||||||
cmake -B build -S . -DSANITIZER=thread # ThreadSanitizer (TSan)
|
cmake -B build -S . -DSANITIZER=undefined # UndefinedBehaviorSanitizer (UBSan); in the CI matrix
|
||||||
|
cmake -B build -S . -DSANITIZER=thread # ThreadSanitizer (TSan); local-only, NOT in CI
|
||||||
```
|
```
|
||||||
|
|
||||||
The CI workflow (`.gitea/workflows/ci.yaml`) runs lint (clang-format, cppcheck), then a **fast PR gate** — build + unit + a representative subset of integration tests marked `@pytest.mark.ci`, parallelized with pytest-xdist (`-n 4 --dist=load`). The full coverage jobs (full integration suite as `-m "not setpriv"`, sanitizer, fuzz, coverage, valgrind) run **only on push to `dev`/`main`**; pull requests skip them to keep PR CI under ~3 minutes. The two `setpriv` privilege tests are excluded from CI via a marker because their result depends on the runner/container uid and host mount permissions.
|
The CI workflow (`.gitea/workflows/ci.yaml`) runs lint (clang-format, cppcheck), then a **fast PR gate** — build + unit + a representative subset of integration tests marked `@pytest.mark.ci`, parallelized with pytest-xdist (`-n 4 --dist=load`). The full coverage jobs (full integration suite as `-m "not setpriv"`, the `address`+`undefined` sanitizer matrix, fuzz, coverage, valgrind) run **only on push to `dev`/`main`**; pull requests skip them to keep PR CI under ~3 minutes. TSan is not part of the CI matrix and is a local-only configuration. The `setpriv`-marked privilege tests (four decorated functions, collecting to eight instances because two are parametrized) are excluded from CI via a marker because their result depends on the runner/container uid and host mount permissions.
|
||||||
|
|
||||||
## Build
|
## Build
|
||||||
|
|
||||||
@@ -55,6 +57,21 @@ cmake -B build -S . && cmake --build build -j$(nproc)
|
|||||||
./build/tests # unit tests
|
./build/tests # unit tests
|
||||||
python3 -m pytest tests/integration/ -n 4 --dist=load -m "not setpriv" # full integration suite (CI excludes env-dependent privilege tests)
|
python3 -m pytest tests/integration/ -n 4 --dist=load -m "not setpriv" # full integration suite (CI excludes env-dependent privilege tests)
|
||||||
python3 -m pytest tests/integration/ -n 4 --dist=load -m ci # PR-gate subset only
|
python3 -m pytest tests/integration/ -n 4 --dist=load -m ci # PR-gate subset only
|
||||||
|
|
||||||
|
# Differential rsync-parity gate (real rsync 3.4.1 vs FastSync)
|
||||||
|
python3 -m pytest tests/integration/test_differential_parity.py -n 4 --dist=load -m parity_ci # fast PR subset
|
||||||
|
python3 -m pytest tests/integration/test_differential_parity.py -n 4 --dist=load -m parity # full set
|
||||||
|
```
|
||||||
|
|
||||||
|
See `tests/integration/README.md` for the differential parity gate and its
|
||||||
|
`parity_caveats.py` allowlist (the residual burn-down mechanism).
|
||||||
|
|
||||||
|
Unit tests under valgrind must set `FASTSYNC_UNDER_VALGRIND=1` (CI does): the
|
||||||
|
tests use it to skip fork-based tests, because valgrind 3.22 does not expose
|
||||||
|
`vgpreload` in the guest's `/proc/self/maps`.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
FASTSYNC_UNDER_VALGRIND=1 valgrind --leak-check=full --show-leak-kinds=definite --error-exitcode=1 ./build/tests
|
||||||
```
|
```
|
||||||
|
|
||||||
## CI Workflow — Waiting for Results
|
## CI Workflow — Waiting for Results
|
||||||
@@ -66,14 +83,14 @@ When running the CI workflow via `tea` (the task execution agent), always set a
|
|||||||
### If lint (clang-format) fails
|
### If lint (clang-format) fails
|
||||||
Run clang-format in the CI Docker image to match the exact CI version:
|
Run clang-format in the CI Docker image to match the exact CI version:
|
||||||
```bash
|
```bash
|
||||||
docker run --rm -v "$PWD:/workspace" -w /workspace gitea.tap-tap.win/taptap/fastsync-ci:v10 \
|
docker run --rm -v "$PWD:/workspace" -w /workspace gitea.tap-tap.win/taptap/fastsync-ci:v11 \
|
||||||
sh -c 'find src/ tests/ -name "*.c" -o -name "*.h" | xargs clang-format -i'
|
sh -c 'find src/ tests/ -name "*.c" -o -name "*.h" | xargs clang-format -i'
|
||||||
```
|
```
|
||||||
|
|
||||||
### If cppcheck fails
|
### If cppcheck fails
|
||||||
Fix reported issues locally, then verify with:
|
Fix reported issues locally, then verify with:
|
||||||
```bash
|
```bash
|
||||||
docker run --rm -v "$PWD:/workspace" -w /workspace gitea.tap-tap.win/taptap/fastsync-ci:v10 \
|
docker run --rm -v "$PWD:/workspace" -w /workspace gitea.tap-tap.win/taptap/fastsync-ci:v11 \
|
||||||
sh -c 'cppcheck --enable=warning,style,performance,portability --suppress=missingIncludeSystem --error-exitcode=1 --inline-suppr src/ tests/'
|
sh -c 'cppcheck --enable=warning,style,performance,portability --suppress=missingIncludeSystem --error-exitcode=1 --inline-suppr src/ tests/'
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -89,7 +106,7 @@ Two main branches: `dev` (integration) and `main` (stable releases).
|
|||||||
|
|
||||||
### Rules
|
### Rules
|
||||||
- **All PRs target `dev`** — never target `main` directly
|
- **All PRs target `dev`** — never target `main` directly
|
||||||
- **`dev` is the default branch** in Gitea repo settings
|
- **`dev` is intended to be the default branch** in Gitea repo settings — verify in the repo settings, since this clone's `origin/HEAD` still points at `main`
|
||||||
- **`main` is protected** — only merged from `dev` via PR with 2 approvals + full CI pass
|
- **`main` is protected** — only merged from `dev` via PR with 2 approvals + full CI pass
|
||||||
- **Feature/bug branches** branch from `dev`, PR back to `dev`
|
- **Feature/bug branches** branch from `dev`, PR back to `dev`
|
||||||
- **`dev` → `main` merges** happen on-demand or weekly, requiring full CI + review
|
- **`dev` → `main` merges** happen on-demand or weekly, requiring full CI + review
|
||||||
|
|||||||
+465
@@ -4,6 +4,471 @@ All notable changes to FastSync are documented here. Versions match
|
|||||||
`PROTOCOL_VERSION` (printed by `fastsync --version`); the client and server must
|
`PROTOCOL_VERSION` (printed by `fastsync --version`); the client and server must
|
||||||
run the same version because the handshake is strict.
|
run the same version because the handshake is strict.
|
||||||
|
|
||||||
|
## [Unreleased]
|
||||||
|
|
||||||
|
## [2.29.0] - 2026-09-23
|
||||||
|
|
||||||
|
The rsync-parity cycle 2.29 (no wire change; `PROTOCOL_VERSION` stays 2.28.0).
|
||||||
|
`RSYNC_COMPAT.md` moves from **116 ✅ / 14 ⚠️ / 27 ❌** to
|
||||||
|
**120 ✅ / 10 ⚠️ / 27 ❌** of 157 rows.
|
||||||
|
|
||||||
|
An audit cycle follows on the same wire version (`PROTOCOL_VERSION` stays
|
||||||
|
2.28.0): a security-and-correctness pass over the parity-2.29 baseline, plus a
|
||||||
|
set of audit follow-ups (filter merge modifiers, the `--inplace`/`--partial-dir`
|
||||||
|
conflict, credential-file hardening, and small leak/log/test fixes). It fixes
|
||||||
|
a `--temp-dir` symlink escape, gates client-controlled special permission bits,
|
||||||
|
corrects `--partial-dir`/`--bwlimit`/`-z` behavior, handles unsupported filter
|
||||||
|
modifiers, and tightens client and wire validation. The only parity
|
||||||
|
reclassification is `--filter=RULE` moving ✅ → ⚠️, because its merge-only
|
||||||
|
`e`/`n`/`w`/`-` modifiers are now accepted and consumed but their semantics
|
||||||
|
remain unimplemented (accepted-but-ignored); the matrix is therefore **119 ✅ /
|
||||||
|
11 ⚠️ / 27 ❌** of 157 rows. The affected rows' notes and the summary tally in
|
||||||
|
`RSYNC_COMPAT.md` were updated. A following triage-fix cycle (see **Triage
|
||||||
|
fixes** below) moves `-F` and `-i` to ⚠️, for a final **117 ✅ / 13 ⚠️ / 27 ❌**
|
||||||
|
of 157 rows.
|
||||||
|
|
||||||
|
A no-wire parity burn-down cycle follows on 2.28.0: it accepts
|
||||||
|
`--inc-recursive`/`--no-inc-recursive` as inert no-ops, accepts an absolute
|
||||||
|
`--temp-dir` that canonicalizes inside the receive root, closes the
|
||||||
|
`--delete-before` phase-0 divergence (both the single-threaded and `--threads`
|
||||||
|
data passes replay the pre-scan list), makes `--fake-super` interoperable with
|
||||||
|
rsync's `user.rsync.%stat` key/grammar (regular files and char/block devices
|
||||||
|
faked as regular files), turns a failed device `mknod` into a continuing
|
||||||
|
per-entry failure, and accepts a practical subset of rsync's `rsyncd.conf`
|
||||||
|
grammar (modules are read-only by default, and accepted-but-unenforced
|
||||||
|
access-control keys emit a startup warning). The matrix moves to **119 ✅ /
|
||||||
|
14 ⚠️ / 24 ❌** of 157 rows.
|
||||||
|
|
||||||
|
A structural cycle then lands a transport I/O vtable over TCP/TLS (fixing the
|
||||||
|
TLS-multithreaded sendfile path and making the per-thread SSL resolution
|
||||||
|
explicit) and bumps the wire to **2.29.0**: the `STATUS_SYMLINK` frame grows an
|
||||||
|
optional symlink-xattr block (captured no-follow with `llistxattr`/`lgetxattr`,
|
||||||
|
applied no-follow with `lsetxattr`). Because the handshake is strict, 2.28.0 and
|
||||||
|
2.29.0 peers are incompatible. Note: Linux refuses to associate xattrs with a
|
||||||
|
symlink at all, so the symlink-xattr block is a no-op on Linux and is carried
|
||||||
|
for correctness on platforms/filesystems that do support it; the config-frame
|
||||||
|
layout is unchanged (golden length still 886).
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- **rsync-exact traversal order.** The sequential scanner now walks each
|
||||||
|
directory's entries in rsync 3.4.1's flist order (non-directories ascending,
|
||||||
|
then directories ascending, depth-first), so `--info=name`, the
|
||||||
|
`--delete-during`/`--delete-delay`/`-n` would-delete order and the partial
|
||||||
|
`--max-delete` survivor set match rsync byte-for-byte. `--threads` has no
|
||||||
|
rsync analogue and stays unordered.
|
||||||
|
- **Delete timing.** The complete `--delete-during`/`--delete-delay`
|
||||||
|
per-directory plan set is transmitted before the first data frame, so a
|
||||||
|
mid-transfer abort has already removed every planned extra like rsync's
|
||||||
|
generator; `-d/--dirs` uses per-directory plans (shielded untraversed
|
||||||
|
subdirectories) instead of the end-of-transfer commit. `-n`, `--delete`,
|
||||||
|
`--del`/`--delete-during` and `--delete-delay` are now ✅ Parity.
|
||||||
|
- **Basis directories.** A relative `--compare-dest`/`--copy-dest`/`--link-dest`
|
||||||
|
DIR resolves against the destination directory with the transfer-relative
|
||||||
|
name appended, exactly like rsync 3.4.1.
|
||||||
|
- **`-y`/`--fuzzy`.** The candidate search no longer inherits the ordinary delta
|
||||||
|
engine's 16 KiB minimum or 10× size-ratio bound, so an oversized or
|
||||||
|
sub-16-KiB sibling is reused exactly as rsync reuses it.
|
||||||
|
- `--info=mount` prints rsync's mount-point skip line (repeated `-xx` drops the
|
||||||
|
mount-point directory); `--info=stats` enables the `--stats` block; `-x` is
|
||||||
|
repeatable. `--stats` counts traversed directories for the `Number of files`
|
||||||
|
breakdown under a plain `-r` scan. `--debug` emits real output for
|
||||||
|
`flist`/`del`/`hash`/`deltasum`/`recv`/`filter`/`send`.
|
||||||
|
|
||||||
|
### Known residuals
|
||||||
|
|
||||||
|
- `--progress` and `--info` still need a receiver→sender event channel for the
|
||||||
|
root `./` line, ancestor-directory suppression, receiver-side `skip`/`backup`
|
||||||
|
wording, and symlink/empty-directory quick-checks.
|
||||||
|
- `--delete-before`'s phase-0 late-file divergence remains (rsync's pre-scan
|
||||||
|
fixes the file list before the data pass).
|
||||||
|
- A single file larger than 256 MiB cannot be streamed in the default path
|
||||||
|
(a general whole-file limit, not basis-specific).
|
||||||
|
- `--stats` byte totals and `--msgs2stderr` stay documented divergences.
|
||||||
|
|
||||||
|
### Security
|
||||||
|
|
||||||
|
- **`--temp-dir` symlink escape fixed.** The receiver's scratch directory was
|
||||||
|
opened with a bare `open()`, so a symlink planted under the receive root could
|
||||||
|
redirect receiver scratch files outside the authorized root. The opened
|
||||||
|
directory is now judged by the real path of its fd (`/proc/self/fd` via
|
||||||
|
`realpath`) and an escaping target is refused (`EACCES`, logged); an in-root
|
||||||
|
link to another filesystem (the `EXDEV` fallback case) still works.
|
||||||
|
- **Client-controlled special bits masked when super-user activities are not
|
||||||
|
permitted.** Setuid/setgid/sticky bits (`--perms`, `--chmod`, the symlink and
|
||||||
|
special-node paths, and deferred directory modes) are now stripped when the
|
||||||
|
connection forbids super activities (`--no-super`, a non-opted daemon module,
|
||||||
|
a privileged listener without `--allow-super`); exact rsync semantics are
|
||||||
|
preserved wherever super activities are permitted.
|
||||||
|
- **Daemon umask no longer forced to `0`.** `daemonize()` now sets the
|
||||||
|
conventional `022`, so implied parent directories created without `-p` are no
|
||||||
|
longer world-writable `0777`.
|
||||||
|
- **Daemon modules are read-only by default.** A `--daemon` module is now
|
||||||
|
served read-only unless it sets `read only = no` (or rsync's `write only =
|
||||||
|
yes`), matching rsync: a real `rsyncd.conf` that omits `read only` is no
|
||||||
|
longer silently writable. A global `read only` still sets the default for
|
||||||
|
later modules, and an explicit module value wins. This is a behavior change
|
||||||
|
for existing FastSync-native configs that relied on the old writable default;
|
||||||
|
add `read only = no` to keep them writable. An rsync `write only = yes` is
|
||||||
|
mapped to writability (FastSync is push-only, so a module can never be read
|
||||||
|
from the network).
|
||||||
|
- **Accepted-but-unenforced rsync security keys now warn at startup.** The
|
||||||
|
rsync keys FastSync recognizes but does not implement — `secrets file`,
|
||||||
|
`refuse options`, `exclude`/`include`/`filter`, `max size`/`min size`,
|
||||||
|
`pre-xfer exec`/`post-xfer exec`, `incoming chmod`/`outgoing chmod`,
|
||||||
|
`name converter`, `use chroot`, `uid`/`gid`, and the rest of the
|
||||||
|
access-control set — load for migration compatibility but now emit a
|
||||||
|
`WARN` naming the key (and module) so an operator does not believe the
|
||||||
|
restriction is enforced. `auth users`/`secrets file` stay fail-closed: a
|
||||||
|
module declaring `auth users` still requires a FastSync credential store.
|
||||||
|
- **Credentials and signal handling hardened.** Secret files are opened with
|
||||||
|
`O_NOFOLLOW|O_NONBLOCK` (while allowing fd-backed store paths and bound-waiting
|
||||||
|
a FIFO read for ~3 s so a slow process substitution works but a connected-but-
|
||||||
|
silent FIFO cannot hang), and signal handlers use `sigaction` with
|
||||||
|
async-signal-safe bodies.
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
|
||||||
|
- **`-z` on 100–256 MiB files.** The decompressor's internal ceiling was 100 MiB
|
||||||
|
while the receiver advertises and the sender compresses whole files up to
|
||||||
|
`MAX_RECEIVE_WHOLE_FILE_SIZE` (256 MiB), so `-z` on a 100–256 MiB regular file
|
||||||
|
failed with `Declared decompressed size exceeds 104857600 bytes`. The ceiling
|
||||||
|
is now defined in terms of the protocol whole-file bound (still an
|
||||||
|
allocation-clamped bomb guard).
|
||||||
|
- **`--bwlimit` now paces `--sendfile`.** The plaintext-TCP `--sendfile` fast
|
||||||
|
path bypassed the protocol's token bucket, so the limit was ignored there. It
|
||||||
|
now throttles through the same per-session leaky bucket as the TLS path.
|
||||||
|
- **`--partial-dir` implies `--partial`.** Matching rsync 3.4.1 (which sets
|
||||||
|
`keep_partial` after option parsing), `--partial-dir=DIR` alone retains an
|
||||||
|
interrupted transfer's partial and wins over an explicit `--no-partial`;
|
||||||
|
`--inplace` still bypasses the partial machinery, and combining `--inplace`
|
||||||
|
with `--partial-dir` is now rejected up front with rsync's message
|
||||||
|
(`--inplace cannot be used with --partial-dir`).
|
||||||
|
- **Filter modifiers handled.** The `x` xattr-name modifier is rejected with a
|
||||||
|
clear error everywhere. The merge-only `e`/`n`/`w` and `-` modifiers are now
|
||||||
|
accepted and consumed on `merge`/`dir-merge` rules (so they no longer leak
|
||||||
|
into the merge filename) while still being rejected on non-merge rules,
|
||||||
|
matching rsync; their semantics remain unimplemented (accepted-but-ignored).
|
||||||
|
Glued patterns (`-newfile`, `-e2e`) and mixed tokens (`H,!secret`) keep their
|
||||||
|
historical parsing.
|
||||||
|
- **Credential-file reads hardened.** Secret files (`--password-file`/
|
||||||
|
`--early-input`/`--hash-credentials` input) are opened with `O_NOFOLLOW`, so a
|
||||||
|
symlinked credential path now fails closed (`ELOOP`) instead of being followed
|
||||||
|
before the owner/mode gate; literal fd-backed paths (`/dev/fd/<digits>`,
|
||||||
|
`/proc/self/fd/<digits>`) are exempt so process substitution still works. A
|
||||||
|
FIFO/process-substitution read now waits under a bounded ~3 s deadline for its
|
||||||
|
writer, so a slow producer works while a connected-but-silent FIFO fails
|
||||||
|
instead of hanging.
|
||||||
|
- **Miscellaneous correctness fixes:** `--filter` rule count is checked
|
||||||
|
client-side against `MAX_FILTER_RULES` before any network I/O (the receiver
|
||||||
|
still re-checks the expanded count); unknown wire `Status` values are rejected
|
||||||
|
as protocol errors; a mutex leak on an init-failure path, an `errno` read
|
||||||
|
after `free()` in deferred delete application, `log_perror` misuse for
|
||||||
|
non-`errno` conditions, and a `NULL` `server_host`/`ssh_destination`
|
||||||
|
allocation path were fixed (the `config_create` failure now releases through
|
||||||
|
`config_delete`); the decompression-limit log now prints the effective bound
|
||||||
|
rather than the compile-time ceiling; the daemon umask and root test fixtures
|
||||||
|
were hardened; `SSL_read` length is clamped and `sendfile` `poll()` retries on
|
||||||
|
`EINTR`.
|
||||||
|
|
||||||
|
### Refactored / Docs
|
||||||
|
|
||||||
|
- Dropped dead `filter_rules_apply` and dead `--old-args` plumbing, unified
|
||||||
|
`set_error`, deduplicated `path_is_within` and shared constants, and added
|
||||||
|
printf format attributes (fixing format mismatches). `RSYNC_COMPAT.md`,
|
||||||
|
`CHANGELOG.md` and `HANDOFF.md` were updated for the audit cycle; the
|
||||||
|
`RSYNC_COMPAT.md` summary tally was corrected to match the rows.
|
||||||
|
|
||||||
|
### Triage fixes
|
||||||
|
|
||||||
|
- **`--dirs` directory xattrs applied inline.** A `-d/--dirs` transfer now
|
||||||
|
applies captured directory `-X`/`-A` xattrs fd-relative on the directory entry
|
||||||
|
instead of dropping them, so directory xattrs survive the non-recursive path
|
||||||
|
(`src/shared/file_save.c`, `tests/test_xattr.c`).
|
||||||
|
- **Directory/root itemize and `--out-format` lines.** `-i`/`--itemize-changes`
|
||||||
|
and `--out-format` now emit the transfer-root `./` line and per-directory
|
||||||
|
`cd...`/`.d..t...` lines, rendered by the shared itemize code. This matches
|
||||||
|
rsync's fresh-transfer output; because the root line is unconditional and an
|
||||||
|
incremental re-run may itemize directories/symlinks that rsync's quick-check
|
||||||
|
leaves silent, `-i` is now a ⚠️ Caveat row.
|
||||||
|
- **FROM name globs for identity maps.** `--usermap`/`--groupmap` `FROM` tokens
|
||||||
|
now accept `*`/`?`/`[...]` globs, expanded sender-side against the passwd/group
|
||||||
|
database and collapsed into bounded numeric ranges (`MAX_IDENTITY_MAP`),
|
||||||
|
matching rsync.
|
||||||
|
- **Transport fallback unit tests.** Added unit coverage for the TCP/TLS
|
||||||
|
transport fallback paths (`tests/test_transport_tcp.c`,
|
||||||
|
`tests/test_transport_tls.c`).
|
||||||
|
- **Docs corrections.** `RSYNC_COMPAT.md`/`README.md` corrected stale parity
|
||||||
|
claims for issues #286–#297: the `-F` and `-i` reclassifications, the
|
||||||
|
`--munge-links` direction, the accepted checksum/compression name sets,
|
||||||
|
`--bwlimit` parsing, `--stop-at` grammar, `--trust-sender`, symlink xattrs, and
|
||||||
|
the native/non-interoperable batch and credential notes. The summary tally is
|
||||||
|
now **117 ✅ / 13 ⚠️ / 27 ❌** of 157 rows.
|
||||||
|
|
||||||
|
## [2.28.0] - 2026-09-20
|
||||||
|
|
||||||
|
The rsync-parity cycle. `PROTOCOL_VERSION` moves `2.26.0 → 2.27.0 → 2.28.0`;
|
||||||
|
client and server must run the same version (the handshake is strict). See
|
||||||
|
`RSYNC_COMPAT.md` for the per-option matrix, now **116 ✅ / 14 ⚠️ / 27 ❌** of
|
||||||
|
157 rows.
|
||||||
|
|
||||||
|
### Added
|
||||||
|
|
||||||
|
- **Differential rsync 3.4.1 parity gate** (`tests/integration/
|
||||||
|
test_differential_parity.py`, `parity_harness.py`, `parity_caveats.py`): runs
|
||||||
|
real `rsync` and FastSync over generated corpora and diffs the destination
|
||||||
|
tree, normalized stdout and exit code. A fast subset runs on pull requests and
|
||||||
|
the full strict set on push; the residual allowlist is empty.
|
||||||
|
- FastSync-only long option **`--verify-basis`**: require a
|
||||||
|
`--compare-dest`/`--copy-dest`/`--link-dest` hit to match the source by
|
||||||
|
whole-file digest instead of trusting the size+mtime quick-check.
|
||||||
|
- FastSync-only long option **`--delete-commit`** (implies `--delete`): the old
|
||||||
|
atomic late whole-tree commit.
|
||||||
|
- `--bwlimit` now parses rsync's units exactly and paces like rsync's leaky
|
||||||
|
bucket; `--ignore-errors` reproduces rsync's skip-unreadable-subdir and
|
||||||
|
IO-error-suppressed deletion (exit 23).
|
||||||
|
- `--info=name/flist/del/remove/nonreg/progress` emit rsync's line format,
|
||||||
|
including real-run `deleting`/`*deleting` lines carried by a new
|
||||||
|
`report_deletes` wire bool.
|
||||||
|
- Receiver-observed `--stats` counters: `Number of created files` now carries
|
||||||
|
rsync's `(reg/dir/link/special)` breakdown and `Literal data` is exact for a
|
||||||
|
delta transfer (extended `STATUS_STATS`).
|
||||||
|
- `--progress` uses an opt-in paths-only pre-count so the `to-chk` denominator
|
||||||
|
counts every entry like rsync, and emits per-directory/symlink/special names.
|
||||||
|
- Receiver-side `protect`/`risk` filter engine (new bounded filter-rule wire
|
||||||
|
block): `--filter='P ...'` now shields a destination-only entry like rsync.
|
||||||
|
- `auto` for `--compress-choice`/`--checksum-choice` honors
|
||||||
|
`RSYNC_COMPRESS_LIST`/`RSYNC_CHECKSUM_LIST`, and per-codec compression-level
|
||||||
|
defaults match rsync.
|
||||||
|
- Empty source directories are recreated recursively; `-R --no-implied-dirs
|
||||||
|
--files-from` places listed files under missing implied parents; `--iconv`
|
||||||
|
matches rsync's push direction; `--delete-delay` reports actual removals and
|
||||||
|
recursively removes a refilled deferred directory.
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- **`--delete` now defaults to delete-during (rsync `--del`) timing.** With no
|
||||||
|
explicit timing flag, a plain `--delete` removes each directory's extras as
|
||||||
|
that directory is processed instead of committing one whole-tree deletion only
|
||||||
|
after the entire transfer succeeds. This matches rsync, frees destination
|
||||||
|
space progressively, and avoids the whole-old+new-tree peak that could
|
||||||
|
`ENOSPC` a tight destination. The client maps the default onto the existing
|
||||||
|
`delete_during` wire boolean, so `PROTOCOL_VERSION` stays `2.28.0`.
|
||||||
|
- Basis directories (`--compare-dest`/`--copy-dest`/`--link-dest`) now default
|
||||||
|
to rsync's metadata quick-check (equal size and mtime; `--size-only` drops the
|
||||||
|
mtime leg) instead of FastSync's historical always-verify content hash.
|
||||||
|
`--copy-dest` re-applies the source attributes, and basis materialization is
|
||||||
|
streamed so the 256 MiB whole-file cap no longer applies to a basis hit.
|
||||||
|
- The per-directory `STATUS_DELETE_PLAN` frame gained a one-int `apply` flag:
|
||||||
|
the one-shot per-run config block (protected prefixes, size-pruned mirrors,
|
||||||
|
`--delete-missing-args` exact paths) is now always transmitted first on a
|
||||||
|
config-only carrier (`apply=false`), fixing a latent bug where a
|
||||||
|
`--delete-missing-args` run whose `--files-from` list synchronized no directory
|
||||||
|
never sent its exact deletions.
|
||||||
|
|
||||||
|
### Notes
|
||||||
|
|
||||||
|
- `--delete`/`--delete-during` remain caveats for the mid-transfer abort
|
||||||
|
boundary (rsync's generator removes all planned extras ahead of its throttled
|
||||||
|
sender; FastSync removes only reached directories — final trees agree).
|
||||||
|
`--delete-before`, `--progress`, `--stats`, `--fuzzy` and the basis rows keep
|
||||||
|
their documented residuals in `RSYNC_COMPAT.md`; `--filter` and
|
||||||
|
`--delete-excluded` are now parity, including protection of a destination-only
|
||||||
|
excluded entry under default `--delete`.
|
||||||
|
|
||||||
|
### Migration
|
||||||
|
|
||||||
|
- Scripts that relied on plain `--delete` deleting nothing until the transfer
|
||||||
|
fully succeeded must pass **`--delete-commit`** (or `--delete-after`) to keep
|
||||||
|
that behavior. Plain `--delete` now removes reached directories' extras during
|
||||||
|
the transfer, exactly like rsync's default; on a completed run the final tree
|
||||||
|
is unchanged.
|
||||||
|
- Deployments that relied on FastSync's stricter basis verification should pass
|
||||||
|
**`--verify-basis`**; the default now trusts the size+mtime quick-check like
|
||||||
|
rsync.
|
||||||
|
|
||||||
|
## [2.26.0] - 2026-09-17
|
||||||
|
|
||||||
|
|
||||||
|
### Added
|
||||||
|
|
||||||
|
- **Parity-completion wave.** Closed the remaining rsync-parity gaps against
|
||||||
|
rsync 3.4.1 and reclassified the inherently non-rsync rows. It moved the wire
|
||||||
|
protocol three times (`2.23.0 → 2.24.0 → 2.25.0 → 2.26.0`).
|
||||||
|
- **Delete timing (2.24.0):** per-directory delete plans
|
||||||
|
(`STATUS_DELETE_PLAN`) for `--delete-during`/`--delete-delay`. An interrupted
|
||||||
|
during-transfer has already removed the reached directories' extras, while a
|
||||||
|
delayed transfer commits per directory only after the whole transfer
|
||||||
|
succeeds (a late-created extra survives `--delete-delay` but not
|
||||||
|
`--delete-after`). `-R --delete` is scoped to the transferred prefix; empty
|
||||||
|
in-scope source directories survive; dry-run never deletes.
|
||||||
|
- **Wire stats (2.25.0):** `STATUS_STATS` carries the receiver counters
|
||||||
|
(matched data, deleted files) and the dry-run would-delete list. `--stats`
|
||||||
|
prints rsync's protocol-independent lines; `--progress`/`-P` print per-file
|
||||||
|
blocks; `--out-format` gains `%b` (wire bytes), `%c` (block-sum bytes) and
|
||||||
|
`%C` (whole-file digest); `-n --delete` prints escaped `*deleting` lines in
|
||||||
|
the sequential and `--threads` paths.
|
||||||
|
- **Codecs (2.26.0):** `lz4`/`zlib`/`zlibx` compression and `md4`/`sha1`/
|
||||||
|
`none` checksums, with rsync-style `auto` negotiation (default `xxh128` +
|
||||||
|
`zstd`) and exit-4 rejection of unknown names; the resolved `compression_algo`
|
||||||
|
crosses the wire.
|
||||||
|
- General `-R`/`--relative` (including the `/./` cut) and `--no-implied-dirs`;
|
||||||
|
one-level `-d`/`--dirs` listing for `dir`, `dir/` and `.`; the full filter
|
||||||
|
grammar (`merge`/`dir-merge`/`hide`/`show`/`protect`/`risk`/`clear` and
|
||||||
|
modifiers) with `-f` bound to `--filter`; a single `-F` transfers
|
||||||
|
`.rsync-filter` and `-FF` excludes it.
|
||||||
|
- Receiver-side `--chown`/`--usermap`/`--groupmap` TO-name resolution; absolute
|
||||||
|
basis directories and a `--link-dest` relink of an up-to-date destination;
|
||||||
|
a receiver-side `--ignore-existing` short-circuit before any payload;
|
||||||
|
`--preallocate` now wins over `--sparse` via `fallocate(2)`.
|
||||||
|
- Client quick wins: `--iconv=.`/`-`/`--no-iconv`, a lone `-h` prints help, an
|
||||||
|
empty `--files-from` succeeds (exit 0), a broken referent under
|
||||||
|
`-L`/`--copy-unsafe-links` exits 23, the full `--info`/`--debug`
|
||||||
|
vocabularies, and the aliases `--ignore-non-existing`, `--protect-args`,
|
||||||
|
`--msgs2stderr`.
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- `PROTOCOL_VERSION` bumped `2.23.0 → 2.24.0` (delete plans),
|
||||||
|
`2.24.0 → 2.25.0` (`STATUS_STATS` + `report_stats`), and
|
||||||
|
`2.25.0 → 2.26.0` (codec negotiation + `md4`/`sha1`/`none`).
|
||||||
|
- `--checksum-choice`/`--cc` now accepts `md4`, `sha1`, `none` and the two-name
|
||||||
|
form; the negotiated whole-file default is `xxh128`.
|
||||||
|
- `--compress-choice`/`--zc` now accepts `lz4`, `zlib`, `zlibx`.
|
||||||
|
- `RSYNC_COMPAT.md` reclassifies the matrix: 9 already-parity rows to ✅, 17
|
||||||
|
inherently non-rsync rows to ❌ (native daemon config/auth, batch, privileged
|
||||||
|
xattr namespaces, and the safe-subset device/privilege flags), and the genuine
|
||||||
|
fixes to ✅; new rows cover `--bwlimit`, `--partial`, `--partial-dir`,
|
||||||
|
`--no-whole-file`, `--inc-recursive`/`--no-inc-recursive`, `--protect-args`
|
||||||
|
and `--msgs2stderr`.
|
||||||
|
- The client `--help` `--max-delete` text now describes the implemented partial
|
||||||
|
semantics (delete up to N, skip the rest, exit 25).
|
||||||
|
|
||||||
|
### Notes
|
||||||
|
|
||||||
|
- Remaining documented divergences include the `--stats` per-type file-count
|
||||||
|
breakdown, `%b`/`%c` being FastSync wire counts, `-n --delete` line ordering,
|
||||||
|
the default `--delete` timing (delete-after, not rsync's delete-during),
|
||||||
|
destination-only exclude protection (still sender-derived), `--temp-dir`
|
||||||
|
absolute paths, basis-dir attribute re-application and the 256 MiB whole-file
|
||||||
|
cap, `--fuzzy` tie-breaking, `--bwlimit=0`/decimal rates, `zlibx`==`zlib`, and
|
||||||
|
recursive empty-directory creation.
|
||||||
|
- Build: adds zlib and lz4 as link dependencies.
|
||||||
|
|
||||||
|
## [2.23.0] - 2026-09-16
|
||||||
|
|
||||||
|
### Added
|
||||||
|
|
||||||
|
- **Rsync-parity wave.** Closed the remaining CLI, filesystem, ownership,
|
||||||
|
deletion, and output gaps against rsync 3.4.1.
|
||||||
|
- Short options `-r` (`--recursive`), `-b` (`--backup`), `-L`
|
||||||
|
(`--copy-links`), and `-B` (`--block-size`/`--delta-block`); rsync
|
||||||
|
short-option clustering (`-av`, `-aAX`, `-rlpt`) and attached/inline values
|
||||||
|
(`--opt=value`, `-B1000`, `-essh`, `-MOPT`). A value that starts with `-`
|
||||||
|
is not mistaken for a cluster.
|
||||||
|
- `-c`/`--checksum` now implies the incremental checksum quick-check (and,
|
||||||
|
like rsync, does not imply `-t`).
|
||||||
|
- `--checksum-choice`/`--cc` accepts `xxh64`/`xxhash`/`xxh3`/`xxh128`/`md5`/
|
||||||
|
`auto` and rejects `md4`/`sha1`/`none` and the two-name form by name;
|
||||||
|
`--checksum-seed=0` (the default) is randomized per transfer and the chosen
|
||||||
|
seed is sent to the receiver.
|
||||||
|
- `--compress-choice`/`--zc` accepts `zstd`/`none`/`auto` and rejects
|
||||||
|
`lz4`/`zlib`/`zlibx` by name; `--skip-compress` defaults to rsync 3.4.1's
|
||||||
|
built-in suffix list; `--no-whole-file` is accepted.
|
||||||
|
- `--timeout` defaults to 0 (disabled) and `--contimeout` to 60 s (both `0`
|
||||||
|
disables), matching rsync; `--max-alloc=0` means no local limit.
|
||||||
|
- `--temp-dir` is confined to the receive root (absolute/`..` rejected by the
|
||||||
|
receiver) and an `EXDEV` install falls back to a non-atomic copy.
|
||||||
|
- `--numeric-ids` is documented as a mapping modifier only;
|
||||||
|
`--usermap`/`--groupmap` support inclusive `LOW-HIGH` ranges, `*`,
|
||||||
|
empty-`FROM` (unnamed ids), and receiver-resolved `TO` names; `--chown`
|
||||||
|
conflicts with a map on the same side are rejected.
|
||||||
|
- `--fake-super` records the *resolved* owner (never a real chown) and replays
|
||||||
|
mode/time; directory ownership and directory xattrs/ACLs are preserved.
|
||||||
|
- `-l`/`--links` stores symlink targets verbatim (absolute and `..`-bearing
|
||||||
|
included), matching rsync; `--safe-links`/`--copy-unsafe-links` are applied
|
||||||
|
sender-side and `--munge-links` uses rsync's `/rsyncd-munged/` marker;
|
||||||
|
`--trust-sender` no longer affects symlink targets.
|
||||||
|
- `--specials` recreates unix sockets with `mknod(S_IFSOCK)` (so `-D` covers
|
||||||
|
the full rsync node set).
|
||||||
|
- Deletion: the manifest carries a synchronized-directory section so
|
||||||
|
`--files-from` subsets no longer delete untransmitted paths;
|
||||||
|
`--delete-excluded` leaves size-pruned mirrors protected; extraneous
|
||||||
|
destination symlinks are unlinked (never followed); `--max-delete=N` is
|
||||||
|
partial (delete up to N, skip the rest, exit 25) and `--delete-missing-args`
|
||||||
|
removals draw from the same budget; `--force` is honored during
|
||||||
|
`--delay-updates` publication.
|
||||||
|
- `-x`/`--one-file-system` emits the mount-point directory entry; the
|
||||||
|
`--include`/`--exclude` layers are an ordered first-match rule list.
|
||||||
|
- `--chmod` is a faithful port of rsync 3.4.1 (numeric/symbolic, `D`/`F`/`X`,
|
||||||
|
`s`/`t`, append semantics, no `-p` implication, no sanitization).
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- `PROTOCOL_VERSION` bumped `2.22.0 → 2.23.0`: the delete manifest gains a
|
||||||
|
synchronized-directory section and the terminal status gains
|
||||||
|
`STATUS_DELETE_LIMIT` (client exit 25 on a `--max-delete`-capped commit).
|
||||||
|
- **The 2.22.0 mode-masking divergence is removed.** Under `-p` the source mode
|
||||||
|
is copied exactly, including `S_IWGRP`/`S_IWOTH` and setuid/setgid/sticky;
|
||||||
|
`--chmod` no longer implies `-p`. New files without `-p` still use
|
||||||
|
`source_mode & ~umask` when metadata is present (else `0644`), and new
|
||||||
|
directories without `-p` still use the `0755` creation default.
|
||||||
|
- `--protocol=NUM` accepts only the current `2.23.0` version string.
|
||||||
|
|
||||||
|
### Notes
|
||||||
|
|
||||||
|
- The rsync-compatibility matrix (`RSYNC_COMPAT.md`) now classifies every row
|
||||||
|
as **parity**, **caveat** (works with a documented divergence), or
|
||||||
|
**divergent** (not supported/no-op/impossible), replacing the previous
|
||||||
|
misleading "N implemented / 0 divergence" summary. Durable documented
|
||||||
|
divergences remain: receiver-side symlink target containment is not enforced
|
||||||
|
by default (verbatim storage is rsync parity; use `--safe-links`),
|
||||||
|
`--temp-dir` rejects absolute/foreign-filesystem paths, `--copy-devices`
|
||||||
|
reads a bounded `st_size`, a broken referent under `--copy-links` exits 0,
|
||||||
|
new directories without `-p` use `0755`, `--stats` receiver-only counters are
|
||||||
|
0, and `--password-file`/`--early-input`/`--hash-credentials`/`--iterations`
|
||||||
|
and the batch format are FastSync-native.
|
||||||
|
|
||||||
|
## [2.22.0] - 2026-09-15
|
||||||
|
|
||||||
|
### Added
|
||||||
|
|
||||||
|
- **Per-attribute metadata preservation (protocol 2.22.0).** The former single
|
||||||
|
metadata bundle is split into four independent, rsync-compatible flags:
|
||||||
|
`-p/--perms`, `-t/--times`, `-o/--owner`, and `-g/--group`, each applied
|
||||||
|
independently on the receiver, with negations `--no-perms`/`--no-times`/
|
||||||
|
`--no-owner`/`--no-group` (short `--no-p`/`--no-t`/`--no-o`/`--no-g`) and
|
||||||
|
`--no-preserve` clearing all four. `-a/--archive` is now full rsync
|
||||||
|
`-rlptgoD` (owner and group included; their application stays
|
||||||
|
privilege-gated). `-A/--acls` and `--chmod` imply `-p`, `-X/--xattrs` does
|
||||||
|
not, `-E/--executability` sets only executability, and `-U`/`-N` do not imply
|
||||||
|
`-t`. `--incremental`/`--delta` still auto-preserve perms+times unless the
|
||||||
|
user explicitly negated them.
|
||||||
|
- Receiver applies directory modes under `-p` (at the end of the transfer,
|
||||||
|
alongside the deferred directory times) and symlink mode under `-p`; `-O`
|
||||||
|
suppresses directory times only.
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- `PROTOCOL_VERSION` bumped `2.21.0 → 2.22.0`: the binary config frame gains
|
||||||
|
four appended booleans (`preserve_perms`/`preserve_times`/`preserve_owner`/
|
||||||
|
`preserve_group`) after `omit_link_times`. The fixed-width `FileMetadata`
|
||||||
|
layout is unchanged; the receiver derives the metadata-frame gate
|
||||||
|
(`use_metadata`) from the four attributes.
|
||||||
|
|
||||||
|
### Notes
|
||||||
|
|
||||||
|
- Documented divergences from rsync: a client-supplied mode never grants
|
||||||
|
group/other write (`S_IWGRP|S_IWOTH` are stripped for files, directories,
|
||||||
|
symlinks, and specials; rsync's `-p` preserves them exactly); a brand-new file
|
||||||
|
without `-p` gets `source_mode & ~umask` (sanitized) when metadata is present,
|
||||||
|
else the historical fixed `0644`; `--chmod` implies `-p` (rsync does not);
|
||||||
|
`-o`/`-g` map by name on the receiver with a raw-numeric fallback (only
|
||||||
|
numeric ids cross the wire); and a daemon module without `client owner = yes`
|
||||||
|
does not refuse a plain `-a`/`-o`/`-g` but forces super-user activities off,
|
||||||
|
applies no ownership, and logs a warning (explicit `--chown`/`--usermap`/
|
||||||
|
`--groupmap`/`--numeric-ids`/`--copy-as`/`--super` are still refused).
|
||||||
|
|
||||||
## [2.21.0] - 2026-09-14
|
## [2.21.0] - 2026-09-14
|
||||||
|
|
||||||
### Added
|
### Added
|
||||||
|
|||||||
+31
-7
@@ -1,6 +1,6 @@
|
|||||||
cmake_minimum_required(VERSION 3.22)
|
cmake_minimum_required(VERSION 3.22)
|
||||||
|
|
||||||
project(FastFileTransfer VERSION 2.21.0)
|
project(FastFileTransfer VERSION 2.29.0)
|
||||||
|
|
||||||
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
|
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
|
||||||
set(CMAKE_C_STANDARD 11)
|
set(CMAKE_C_STANDARD 11)
|
||||||
@@ -26,9 +26,9 @@ elseif(NOT SANITIZER STREQUAL "none")
|
|||||||
endif()
|
endif()
|
||||||
|
|
||||||
# --- Strict warnings option ---
|
# --- Strict warnings option ---
|
||||||
option(STRICT_WARNINGS "Enable strict warnings (Wextra, Wpedantic, Werror)" OFF)
|
option(STRICT_WARNINGS "Enable strict warnings (Wextra, Wpedantic, Wformat-signedness, Werror)" OFF)
|
||||||
if(STRICT_WARNINGS)
|
if(STRICT_WARNINGS)
|
||||||
add_compile_options(-Wextra -Wpedantic -Werror)
|
add_compile_options(-Wextra -Wpedantic -Wformat-signedness -Werror)
|
||||||
endif()
|
endif()
|
||||||
|
|
||||||
# --- Coverage option ---
|
# --- Coverage option ---
|
||||||
@@ -68,6 +68,16 @@ if(NOT ZSTD_LIBRARY)
|
|||||||
message(FATAL_ERROR "zstd library not found. Ensure it is in your nix-shell!")
|
message(FATAL_ERROR "zstd library not found. Ensure it is in your nix-shell!")
|
||||||
endif()
|
endif()
|
||||||
|
|
||||||
|
find_library(ZLIB_LIBRARY z)
|
||||||
|
if(NOT ZLIB_LIBRARY)
|
||||||
|
message(FATAL_ERROR "zlib library not found. Ensure zlib1g-dev / nix zlib is available!")
|
||||||
|
endif()
|
||||||
|
|
||||||
|
find_library(LZ4_LIBRARY lz4)
|
||||||
|
if(NOT LZ4_LIBRARY)
|
||||||
|
message(FATAL_ERROR "lz4 library not found. Ensure liblz4-dev / nix lz4 is available!")
|
||||||
|
endif()
|
||||||
|
|
||||||
find_package(OpenSSL REQUIRED)
|
find_package(OpenSSL REQUIRED)
|
||||||
|
|
||||||
# --- Explicit source lists ---
|
# --- Explicit source lists ---
|
||||||
@@ -89,15 +99,21 @@ set(SHARED_SRCS
|
|||||||
src/shared/daemon_limits.c
|
src/shared/daemon_limits.c
|
||||||
src/shared/data.c
|
src/shared/data.c
|
||||||
src/shared/delay_updates.c
|
src/shared/delay_updates.c
|
||||||
|
src/shared/delete.c
|
||||||
|
src/shared/delete_commit.c
|
||||||
|
src/shared/delete_plan.c
|
||||||
src/shared/delta.c
|
src/shared/delta.c
|
||||||
src/shared/file.c
|
src/shared/file.c
|
||||||
src/shared/file_list.c
|
src/shared/file_list.c
|
||||||
src/shared/file_receive.c
|
src/shared/file_receive.c
|
||||||
|
src/shared/file_save.c
|
||||||
src/shared/file_send.c
|
src/shared/file_send.c
|
||||||
src/shared/file_store.c
|
src/shared/file_store.c
|
||||||
src/shared/filter.c
|
src/shared/filter.c
|
||||||
|
src/shared/format.c
|
||||||
src/shared/hardlink.c
|
src/shared/hardlink.c
|
||||||
src/shared/identity.c
|
src/shared/identity.c
|
||||||
|
src/shared/incremental_check.c
|
||||||
src/shared/log.c
|
src/shared/log.c
|
||||||
src/shared/metadata.c
|
src/shared/metadata.c
|
||||||
src/shared/motd.c
|
src/shared/motd.c
|
||||||
@@ -124,9 +140,14 @@ set(SERVER_MAIN_SRCS src/server/server.c)
|
|||||||
# Client implementation (no main): everything except the CLI entry point.
|
# Client implementation (no main): everything except the CLI entry point.
|
||||||
set(CLIENT_CORE_SRCS
|
set(CLIENT_CORE_SRCS
|
||||||
src/client/change_list.c
|
src/client/change_list.c
|
||||||
|
src/client/client_manifest.c
|
||||||
|
src/client/client_report.c
|
||||||
|
src/client/client_scan.c
|
||||||
src/client/client_send.c
|
src/client/client_send.c
|
||||||
src/client/client_validation.c
|
src/client/client_validation.c
|
||||||
src/client/scanner.c
|
src/client/scanner.c
|
||||||
|
src/client/scanner_filter.c
|
||||||
|
src/client/scanner_parallel.c
|
||||||
src/client/usage.c
|
src/client/usage.c
|
||||||
)
|
)
|
||||||
set(CLIENT_MAIN_SRCS src/client/client_cli.c)
|
set(CLIENT_MAIN_SRCS src/client/client_cli.c)
|
||||||
@@ -134,8 +155,8 @@ set(CLIENT_MAIN_SRCS src/client/client_cli.c)
|
|||||||
# --- Library targets ---
|
# --- Library targets ---
|
||||||
add_library(fastsync_shared STATIC ${SHARED_SRCS})
|
add_library(fastsync_shared STATIC ${SHARED_SRCS})
|
||||||
target_include_directories(fastsync_shared PUBLIC src/shared)
|
target_include_directories(fastsync_shared PUBLIC src/shared)
|
||||||
target_link_libraries(fastsync_shared PUBLIC Threads::Threads ${ZSTD_LIBRARY} OpenSSL::SSL
|
target_link_libraries(fastsync_shared PUBLIC Threads::Threads ${ZSTD_LIBRARY} ${ZLIB_LIBRARY}
|
||||||
OpenSSL::Crypto xxhash)
|
${LZ4_LIBRARY} OpenSSL::SSL OpenSSL::Crypto xxhash)
|
||||||
|
|
||||||
add_library(fastsync_client_core STATIC ${CLIENT_CORE_SRCS})
|
add_library(fastsync_client_core STATIC ${CLIENT_CORE_SRCS})
|
||||||
target_include_directories(fastsync_client_core PUBLIC src/client)
|
target_include_directories(fastsync_client_core PUBLIC src/client)
|
||||||
@@ -209,10 +230,13 @@ set(TEST_SRCS
|
|||||||
tests/test_daemon_limits.c
|
tests/test_daemon_limits.c
|
||||||
tests/test_data.c
|
tests/test_data.c
|
||||||
tests/test_delay_updates.c
|
tests/test_delay_updates.c
|
||||||
|
tests/test_delete_plan.c
|
||||||
tests/test_delta.c
|
tests/test_delta.c
|
||||||
tests/test_file.c
|
tests/test_file.c
|
||||||
tests/test_file_list.c
|
tests/test_file_list.c
|
||||||
tests/test_file_sendfile.c
|
tests/test_file_sendfile.c
|
||||||
|
tests/test_filter.c
|
||||||
|
tests/test_format.c
|
||||||
tests/test_fuzz_smoke.c
|
tests/test_fuzz_smoke.c
|
||||||
tests/test_glob.c
|
tests/test_glob.c
|
||||||
tests/test_hardlink.c
|
tests/test_hardlink.c
|
||||||
@@ -273,7 +297,7 @@ if(ENABLE_FUZZ)
|
|||||||
target_include_directories(${FUZZ_NAME} PRIVATE tests src/shared src/server)
|
target_include_directories(${FUZZ_NAME} PRIVATE tests src/shared src/server)
|
||||||
target_compile_options(${FUZZ_NAME} PRIVATE -fsanitize=fuzzer,address,undefined -fno-omit-frame-pointer)
|
target_compile_options(${FUZZ_NAME} PRIVATE -fsanitize=fuzzer,address,undefined -fno-omit-frame-pointer)
|
||||||
target_link_options(${FUZZ_NAME} PRIVATE -fsanitize=fuzzer,address,undefined)
|
target_link_options(${FUZZ_NAME} PRIVATE -fsanitize=fuzzer,address,undefined)
|
||||||
target_link_libraries(${FUZZ_NAME} PRIVATE Threads::Threads ${ZSTD_LIBRARY} OpenSSL::SSL
|
target_link_libraries(${FUZZ_NAME} PRIVATE Threads::Threads ${ZSTD_LIBRARY} ${ZLIB_LIBRARY}
|
||||||
OpenSSL::Crypto xxhash)
|
${LZ4_LIBRARY} OpenSSL::SSL OpenSSL::Crypto xxhash)
|
||||||
endforeach()
|
endforeach()
|
||||||
endif()
|
endif()
|
||||||
|
|||||||
+15
-1
@@ -2,8 +2,22 @@ FROM ubuntu:24.04
|
|||||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
gcc g++ make libc6-dev cmake libzstd-dev libssl-dev git ca-certificates curl cppcheck clang-format \
|
gcc g++ make libc6-dev cmake libzstd-dev libssl-dev git ca-certificates curl cppcheck clang-format \
|
||||||
python3 python3-pip python3-venv openssl openssh-client \
|
python3 python3-pip python3-venv openssl openssh-client \
|
||||||
lcov valgrind clang libclang-rt-18-dev && \
|
lcov valgrind clang libclang-rt-18-dev \
|
||||||
|
acl attr zlib1g-dev liblz4-dev libxxhash-dev && \
|
||||||
pip3 install --break-system-packages pytest pytest-xdist && \
|
pip3 install --break-system-packages pytest pytest-xdist && \
|
||||||
curl -fsSL https://deb.nodesource.com/setup_20.x | bash - && \
|
curl -fsSL https://deb.nodesource.com/setup_20.x | bash - && \
|
||||||
apt-get install -y --no-install-recommends nodejs && \
|
apt-get install -y --no-install-recommends nodejs && \
|
||||||
rm -rf /var/lib/apt/lists/*
|
rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
# rsync is used as the reference implementation for drop-in parity tests.
|
||||||
|
# Ubuntu 24.04 ships 3.2.7, so build the pinned 3.4.1 reference from source.
|
||||||
|
ARG RSYNC_VERSION=3.4.1
|
||||||
|
ARG RSYNC_SHA256=2924bcb3a1ed8b551fc101f740b9f0fe0a202b115027647cf69850d65fd88c52
|
||||||
|
RUN curl -fsSL "https://download.samba.org/pub/rsync/src/rsync-${RSYNC_VERSION}.tar.gz" -o /tmp/rsync.tar.gz && \
|
||||||
|
echo "${RSYNC_SHA256} /tmp/rsync.tar.gz" | sha256sum -c - && \
|
||||||
|
tar -xzf /tmp/rsync.tar.gz -C /tmp && \
|
||||||
|
cd "/tmp/rsync-${RSYNC_VERSION}" && \
|
||||||
|
./configure --enable-zstd --enable-xxhash --enable-lz4 && \
|
||||||
|
make -j"$(nproc)" && \
|
||||||
|
make install && \
|
||||||
|
rm -rf "/tmp/rsync-${RSYNC_VERSION}" /tmp/rsync.tar.gz
|
||||||
|
|||||||
+260
@@ -0,0 +1,260 @@
|
|||||||
|
# FastSync — Session Handoff (2026-09-21)
|
||||||
|
|
||||||
|
## Current status
|
||||||
|
- **Release `v2.28.0`** is tagged and merged to `main`: tag `v2.28.0` points at
|
||||||
|
`ee6523a`, and the PR #304 merge commit `b4d54504` is on `main`.
|
||||||
|
- **`dev` is at `0fbb9de`** — the merge of parity cycle 2.29 (PR #305). The old
|
||||||
|
`558782d` (incremental-check flake fix) is an ancestor.
|
||||||
|
- **`PROTOCOL_VERSION` = `"2.28.0"`** (`src/shared/config.h`); CMake
|
||||||
|
`project(FastFileTransfer VERSION 2.28.0)`.
|
||||||
|
- **Parity cycle 2.29 is merged to `dev`** (PR #305), no wire change. It closed
|
||||||
|
the scanner-order, delete-timing, relative-basis and fuzzy-eligibility
|
||||||
|
residuals and improved the `--info`/`--stats`/`--debug` partials. Parity
|
||||||
|
matrix: **120 ✅ / 10 ⚠️ / 27 ❌ = 157**. Remaining ⚠️ rows: `--info`,
|
||||||
|
`--debug`, `--msgs2stderr`, `--stats`, `--progress`, `--delete-before`, the
|
||||||
|
three basis-dir options, and `-y`/`--fuzzy`.
|
||||||
|
- **Audit cycle complete on branch `fix/audit-cycle`** (branched from `dev` @
|
||||||
|
`0fbb9de`), integration PR to `dev` pending. No wire change
|
||||||
|
(`PROTOCOL_VERSION` stays 2.28.0). It lands the receiver/client security and
|
||||||
|
correctness fixes — `--temp-dir` symlink-escape confinement, special-bit
|
||||||
|
masking under a super-off policy, daemon `umask(022)`, the `-z` decompression
|
||||||
|
ceiling raised to the 256 MiB whole-file bound, `--bwlimit` pacing the
|
||||||
|
plaintext `--sendfile` path, `--partial-dir` implying `--partial`, rejection
|
||||||
|
of unsupported filter modifiers (`x`/`e`/`n`/`w`), client-side
|
||||||
|
`MAX_FILTER_RULES` enforcement, unknown wire `Status` rejection, and the
|
||||||
|
accompanying refactors/docs. The parity matrix is unchanged at
|
||||||
|
**120 ✅ / 10 ⚠️ / 27 ❌ = 157**; this docs pass (worktree `fix/audit-docs2`)
|
||||||
|
corrects the `RSYNC_COMPAT.md` summary tally to match the rows.
|
||||||
|
|
||||||
|
|
||||||
|
## What landed this session
|
||||||
|
1. **Wave 8 (refactors):** Config X-macro wire table; single-owner `authorized_root`;
|
||||||
|
daemon per-module/per-host caps + cross-process auth lockout (`daemon_limits.[ch]`);
|
||||||
|
`Data` charge returns to its owning `ProtocolSession`.
|
||||||
|
2. **Wave 9 (protocol 2.21.0):** optional `STATUS_ERROR_DETAIL` rejection reasons;
|
||||||
|
server-contacting `--dry-run` (`STATUS_DRY_RUN_TRANSFER`, receiver mutates nothing).
|
||||||
|
3. **Security wave:** ran 5 parallel audits (wire parsing; daemon/transport/TLS/auth;
|
||||||
|
receiver confinement; client/CLI/SSH; crypto/memory/limits). Fixed all HIGH and the
|
||||||
|
confirmed MEDIUMs:
|
||||||
|
- SSH `-o ProxyCommand=…` argument injection (RCE) — reject leading `-`, insert `--`.
|
||||||
|
- Truncated zstd frame infinite CPU loop (remote DoS).
|
||||||
|
- FIFO receiver opens lacked `O_NONBLOCK` (indefinite hang).
|
||||||
|
- `--inplace` could write a FIFO/device (bypass of `--write-devices` gate).
|
||||||
|
- `--force` not gated by server `--allow-delete`.
|
||||||
|
- Privileged standalone server defaulted super activities on; added `--allow-super`
|
||||||
|
(never honored with `--stdio`).
|
||||||
|
- `--dry-run` content/hash oracle on `read only`/basis files removed.
|
||||||
|
- Empty `hosts allow`/`deny`/`auth users` now rejected.
|
||||||
|
- TLS: AEAD-only 1.2 + server preference, TOCTOU-safe key load, IP-SAN verify,
|
||||||
|
CN-truncation guard. Glob backtracking bounded; line reads bounded; ACL xattrs
|
||||||
|
gated on `--acls`; decompression/chunk memory charged; pre-auth `basis_count`
|
||||||
|
NULL-deref fixed.
|
||||||
|
4. **Tooling:** benchmark accuracy (data mix, verification, percentiles, `tc`,
|
||||||
|
`build-bench/`, `--warm` mode); `shell.nix` full toolchain and no build-on-entry;
|
||||||
|
docs state push-only / remote-source unsupported.
|
||||||
|
5. **Preserve-attribute split (protocol 2.22.0)** landed on `feat/preserve-attr-split`: per-attribute `-p/-t/-o/-g` + `--no-*` negations, `-a` = `-rlptgoD`, and the 2.21.0 → 2.22.0 wire bump.
|
||||||
|
6. **Rsync-parity wave (protocol 2.23.0)** on `feat/rsync-parity`: rsync short options/clustering/attached values (`-r`/`-b`/`-L`/`-B`, `-av`, `-aAX`, `-B1000`, `-essh`, `-MOPT`), `-c` checksum quick-check, `--checksum-choice`/`--compress-choice` validation and seed randomization, rsync timeout/max-alloc defaults, temp-dir confinement + `EXDEV` fallback, ownership/mapping parity (numeric-ids modifier, map ranges/`*`/empty-FROM, `--chown`+map conflicts, fake-super resolved-owner record), verbatim symlink storage with rsync `--safe-links`/`--munge-links`, socket recreation under `--specials`, `--chmod` 3.4.1 semantics, and delete scoping + `--max-delete` partial/exit-25. Wire: appended delete-manifest synchronized-directory section and `STATUS_DELETE_LIMIT`.
|
||||||
|
7. **Parity-completion wave (protocol 2.24.0 → 2.26.0)** on `feat/parity-completion`: per-directory delete plans (`STATUS_DELETE_PLAN`) for `--delete-during`/`--delete-delay`; receiver `STATUS_STATS` counters feeding `--stats`/`--progress` and `--out-format %b/%c/%C`, plus `-n --delete` lines; `lz4`/`zlib`/`zlibx` compression and `md4`/`sha1`/`none` checksums with `auto` negotiation (default `xxh128`/`zstd`); general `-R`/`--no-implied-dirs`/`-d`; the full filter grammar (`merge`/`dir-merge`/`hide`/`show`/`protect`/`risk`/`clear` + modifiers) and corrected `-F`/`-FF`; receiver-side `--chown`/map TO-name resolution; absolute basis dirs + `--link-dest` relink; receiver-side `--ignore-existing` short-circuit; `--preallocate` over `--sparse` via `fallocate(2)`; `--iconv=.`/`-`/`--no-iconv`; lone `-h` help; aliases `--ignore-non-existing`/`--protect-args`/`--msgs2stderr`; and the full `--info`/`--debug` vocabulary. `RSYNC_COMPAT.md` reclassifies the matrix to 106 ✅ / 27 ⚠️ / 23 ❌; the later rsync-parity-stats pass (`fix/parity-stats`) moves it to 107 ✅ / 25 ⚠️ / 24 ❌ (see item 8).
|
||||||
|
8. **rsync-parity-stats pass** on `fix/parity-stats` (no wire change, `PROTOCOL_VERSION` stays `2.26.0`): `--delete-delay` now reports only entries it actually removes, while the `--max-delete` budget is charged at plan/snapshot time (`planned`, via `defer_add`) to bound the deferred list (a refilled deferred directory that survives `ENOTEMPTY` is not reported but still consumes budget); `--stats` gained the `(reg/dir/link/special)` `Number of files` breakdown and now counts only regular files actually stored for `Number of regular files transferred`/transferred size/literal data (up-to-date re-runs report 0); `Total file size` includes symlink target lengths; `--progress` prints the leading `./` root line and counts it in `to-chk` so a single-file transfer matches rsync; and `%C` uses the selected transfer checksum with `checksum_digest_file` supporting md4/sha1/none, byte-identical to rsync for every algorithm. `--out-format` reclassified ❌ (`%b`/delta-`%c` are protocol-specific). Differential + regression tests added; full suite + ASan + clang-format + cppcheck clean.
|
||||||
|
9. **Option-parity wave (protocol 2.26.0 → 2.27.0, on `fix/parity-options`):**
|
||||||
|
`--bwlimit` now ports rsync 3.4.1's units/quantization and paces like its
|
||||||
|
leaky bucket; `--ignore-errors` reproduces rsync's default (an I/O error
|
||||||
|
skips deletion unless the flag is set; the readable tree still transfers and
|
||||||
|
the run exits 23) across every delete timing; the `--info` categories with a
|
||||||
|
FastSync event (`name`/`flist`/`del`/`remove`/`nonreg`/`progress`) emit
|
||||||
|
rsync's line format, with real-run `deleting`/`*deleting` lines carried over
|
||||||
|
the new trailing config bool `report_deletes` (golden wire updated by
|
||||||
|
`tests/test_config.c`). Two residuals were reclassified **divergent**: `-M`
|
||||||
|
over daemon/TCP (no argv channel in FastSync's binary config handshake;
|
||||||
|
rsync-daemon differential pins the rsync behavior) and receiver-side
|
||||||
|
`protect`/`risk` re-derivation for destination-only entries (would need a
|
||||||
|
receiver filter engine; differential pins the divergence — **reversed by
|
||||||
|
track 4a below**, which adds that engine). The options pass
|
||||||
|
stands at **110 ✅ / 21 ⚠️ / 26 ❌**. New `tests/integration/test_option_parity.py`
|
||||||
|
holds the rsync differentials (bwlimit parse+rate, info lines, real-setpriv
|
||||||
|
`--ignore-errors`, rsync-daemon `-M`, filter-protect pin).
|
||||||
|
|
||||||
|
10. **rsync-parity-fs pass** on `fix/parity-fs` (no wire change of its own; integrated
|
||||||
|
on top of the 2.27.0 options wave): recursive transfers now recreate empty source directories (and
|
||||||
|
`-m/--prune-empty-dirs` still suppresses them), a directory entry replaces a
|
||||||
|
blocking destination regular file, and `-R --no-implied-dirs --files-from`
|
||||||
|
places a listed file under a missing implied parent with default attributes
|
||||||
|
instead of refusing (real rsync 3.4.1 parity, differential-tested). `--iconv`
|
||||||
|
now reproduces rsync's push direction (destination charset = the spec's REMOTE
|
||||||
|
half; a server `--iconv` overrides), and `-T/--temp-dir` relative semantics are
|
||||||
|
confirmed identical while the absolute-path confinement is a deliberate
|
||||||
|
divergence. The basis-dir options, `--delay-updates` and `--dry-run` were
|
||||||
|
reclassified to ❌ after a differential test reproduced each exact residual
|
||||||
|
(basis content verification, fixed staging-name collision, and dry-run
|
||||||
|
would-delete over-report). `--fuzzy` was also reclassified to ❌ (deterministic
|
||||||
|
heuristic with a 10× size window, not rsync's matcher), but its residual is the
|
||||||
|
candidate-selection heuristic itself: the final tree is byte-exact by design, so
|
||||||
|
it is pinned by the `TestFuzzy` threshold suite rather than a byte-level rsync
|
||||||
|
differential. (Track 5b later found the name heuristic is rsync's own and moved
|
||||||
|
the row ❌ → ⚠️, leaving only the narrower delta size window; see entry 15.) The parity-review pass then moved `--delete-delay` to ⚠️ (the
|
||||||
|
plan-time `--max-delete` charge and non-recursive deferred removal differ from
|
||||||
|
rsync when a snapshotted entry fails removal). Differential-gate allowlist
|
||||||
|
entries `min_size`/`empty_dirs_recursive`/`dirs_plain` were removed. The
|
||||||
|
integrated stats+options+fs branch stands at **111 ✅ / 13 ⚠️ / 33 ❌ = 157**;
|
||||||
|
full suite + ASan + clang-format + cppcheck clean.
|
||||||
|
|
||||||
|
11. **No-wire parity track 1** on `feat/parity-2.28` (no protocol change):
|
||||||
|
`-n --delete` now sends the same filter-excluded + size-pruned protected
|
||||||
|
prefixes and synchronized-directory scope as a real run (dry-run would-delete
|
||||||
|
matches rsync for source-derived protections; the destination-only exclude
|
||||||
|
residual was later closed by track 4a, readdir ordering remains);
|
||||||
|
`--delete-delay` now charges
|
||||||
|
`--max-delete` on actual removals and re-scans a queued directory at commit
|
||||||
|
to remove content created after the plan, with an independent deferred-list
|
||||||
|
cap (only partial-delete ordering remains); and `--info=name2` emits `NAME is
|
||||||
|
uptodate` plus the leading `./` root name line for `--info=name` (only the
|
||||||
|
root-line trigger condition and receiver-side `skip` wording remain). Matrix
|
||||||
|
now **111 ✅ / 14 ⚠️ / 32 ❌ = 157**; differential + unit tests added in
|
||||||
|
`test_features.py`, `test_option_parity.py`, the unit test
|
||||||
|
`tests/test_delete_plan.c`,
|
||||||
|
`test_delete_delay_budget_parity.py`, `test_delete_timing_parity.py`.
|
||||||
|
12. **No-wire parity track 2b** on `feat/parity-2.28` (no protocol change):
|
||||||
|
`--progress`/`-P`/`--info=progress` (when not `--quiet`) now run an opt-in
|
||||||
|
paths-only metadata pre-count (no file reads/hashing) that supplies rsync's
|
||||||
|
full file-list total for the `to-chk` denominator and the directory names,
|
||||||
|
and emits per-directory/symlink/special name lines, in both the sequential
|
||||||
|
and `--threads` paths. `--delete-during`/`--delete-delay` reuse their
|
||||||
|
keep-set pre-scan instead of a second walk; non-progress runs are
|
||||||
|
unaffected. Differential tests (`progress`/`progress_threads` over a new
|
||||||
|
`multidir` corpus) match rsync's name set and `to-chk` denominator on a
|
||||||
|
fresh transfer, and the single-file byte-identical test still passes;
|
||||||
|
emission order (rsync's sorted depth-first vs FastSync's readdir/BFS stream)
|
||||||
|
plus re-run over-naming (unconditional `./`, ancestor dirs named with a
|
||||||
|
transferred child, and no quick-check for symlinks/empty dirs) remain the
|
||||||
|
caveats, so the row stays ⚠️ and the matrix is unchanged at
|
||||||
|
**111 ✅ / 14 ⚠️ / 32 ❌ = 157**.
|
||||||
|
|
||||||
|
13. **Wire parity track 4a** on `feat/parity-2.28` (`PROTOCOL_VERSION` stays
|
||||||
|
`2.28.0`): the receiver now has a delete-time filter engine. The sender
|
||||||
|
compiles its root-level selection rules exactly as the scanner does
|
||||||
|
(`filter_base_build`) and streams them as one bounded, self-describing
|
||||||
|
config-frame block (action, sides, anchored, dir-only, negate, owner,
|
||||||
|
pattern; bounded rule count and pattern bytes, unknown action/sides is a
|
||||||
|
protocol error). The receiver reconstructs `protect_rules` and applies them
|
||||||
|
first-match-wins to each extraneous destination path in every delete timing
|
||||||
|
(the whole-tree commit walker, the `--delete-during`/`--delete-delay`
|
||||||
|
per-directory plans, and the `-n` would-delete enumeration), so a
|
||||||
|
`P *.log` rule protects a destination-only `extra.log` like rsync (with
|
||||||
|
`risk` cancelling); the sender-derived protected-prefix behavior is
|
||||||
|
preserved when no rules are sent and `--delete-excluded` semantics are
|
||||||
|
unchanged. Per-directory merge (`:`/`.`) receiver re-derivation remains the
|
||||||
|
residual. `TestFilterProtect` (real + dry-run) plus differential cases
|
||||||
|
`filter_protect`, `filter_protect_during`, `filter_protect_delay` added and
|
||||||
|
the `--filter=RULE` row moves ❌ → ✅: matrix now
|
||||||
|
**115 ✅ / 11 ⚠️ / 31 ❌ = 157**; unit tests, the three named integration
|
||||||
|
files, clang-format and cppcheck clean.
|
||||||
|
|
||||||
|
14. **Wire parity track 5a** on `feat/parity-2.28` (`PROTOCOL_VERSION` stays
|
||||||
|
`2.28.0` by project decision): the three basis-dir options now default to
|
||||||
|
rsync's metadata quick-check (equal size + equal mtime, or size alone under
|
||||||
|
`--size-only`; `-I` disables matching) instead of FastSync's historical
|
||||||
|
xxHash64 content equality, so a same-size/different-content basis is trusted
|
||||||
|
exactly as rsync trusts it. A new FastSync-only, long-only `--verify-basis`
|
||||||
|
flag restores the strict whole-file content equality; its bool is appended to
|
||||||
|
the basis block of the config frame (golden wire frame 882 → 886 bytes).
|
||||||
|
`--verify-basis` streams the confined basis descriptor to hash it, and a
|
||||||
|
basis hit is no longer capped at the 256 MiB whole-file payload bound:
|
||||||
|
`--copy-dest` streams the basis through a bounded buffer and `--link-dest`'s
|
||||||
|
copy fallback streams from the basis, so an over-limit hit materializes (a
|
||||||
|
basis MISS still falls back to the normal transfer and keeps its own bound).
|
||||||
|
A `--copy-dest` hit re-applies the SOURCE attributes (the sender transmits
|
||||||
|
the source metadata with the basis check frame), matching rsync's
|
||||||
|
"copy then fix attributes"; a `--link-dest` success keeps the shared inode's
|
||||||
|
attributes (writing through it would mutate the basis). Differential cases
|
||||||
|
`copy_dest` and `verify_basis` added; `test_basis_dir_size_only_content_residual`
|
||||||
|
converted to a passing parity assertion; `TestBasisDestDirs` updated for the
|
||||||
|
new default + `--verify-basis`; unit tests cover the quick-check/verify
|
||||||
|
decision and the same-size/different-content handshake. The
|
||||||
|
`--compare-dest`/`--copy-dest`/`--link-dest` rows move ❌ → ⚠️ (relative-DIR
|
||||||
|
resolution base and over-limit MISS refusal): matrix now
|
||||||
|
**116 ✅ / 13 ⚠️ / 28 ❌ = 157**.
|
||||||
|
|
||||||
|
15. **No-wire parity track 5b** on `feat/parity-2.28` (`PROTOCOL_VERSION` stays
|
||||||
|
`2.28.0` by project decision): `-y`/`--fuzzy` reclassified ❌ → ⚠️. A probe
|
||||||
|
against real rsync 3.4.1 (pinned `-B8192`, repeated-content 64 KiB corpus)
|
||||||
|
showed the name heuristic is already rsync's (`util1.c fuzzy_distance` /
|
||||||
|
`find_filename_suffix` + the exact size+mtime pass) and the output is always
|
||||||
|
byte-exact; the only residual is candidate ELIGIBILITY, because FastSync's
|
||||||
|
`delta_should_attempt` gate caps the size ratio at 10× and requires both
|
||||||
|
files ≥ 16 KiB while rsync will reuse a basis from 0.25× to 10000× and below
|
||||||
|
16 KiB. The choice is observable only as `--stats` bandwidth counters. Added
|
||||||
|
differential case `fuzzy_basis` (same-suffix sibling, one name edit,
|
||||||
|
identical content, block size pinned) asserting tree **and** normalized
|
||||||
|
`--stats` parity where the choices coincide, plus `TestFuzzy` pinning the
|
||||||
|
window boundary on both sides (>10× and <16 KiB siblings declined by
|
||||||
|
FastSync while rsync uses them, both trees byte-identical). Matrix now
|
||||||
|
**116 ✅ / 14 ⚠️ / 27 ❌ = 157**.
|
||||||
|
|
||||||
|
16. **Lockstep delete-default track 6** on `feat/parity-2.28` (`PROTOCOL_VERSION`
|
||||||
|
stays `2.28.0`): plain `--delete` now defaults to rsync's delete-during
|
||||||
|
(`--del`) timing, normalized on the client onto the existing `delete_during`
|
||||||
|
wire bool. The old late whole-tree commit is opt-in via `--delete-after` or
|
||||||
|
the FastSync-only long `--delete-commit` (identical `delete_after` timing).
|
||||||
|
`-d/--dirs` still falls back to the end commit, `--delay-updates` still
|
||||||
|
deletes before publication, and `--files-from`/`-R` scope is unchanged. The
|
||||||
|
`STATUS_DELETE_PLAN` frame gained a one-int `apply` flag so the per-run
|
||||||
|
config block (including `--delete-missing-args` exact paths) is always
|
||||||
|
transmitted, on a config-only carrier when the scope allows no directory
|
||||||
|
plan — fixing a latent bug with a file-only `--files-from` list. Differential
|
||||||
|
cases `delete`/`delete_commit`/`filter_protect_after` plus the extended
|
||||||
|
`test_delete_timing_parity.py` (plain `--delete` mid-abort removes reached
|
||||||
|
extras, `--delete-commit` defers) pass; full `-m "not setpriv"` suite,
|
||||||
|
clang-format and cppcheck clean. Matrix unchanged at
|
||||||
|
**116 ✅ / 14 ⚠️ / 27 ❌ = 157** (the `--delete`/`--delete-during` rows stay
|
||||||
|
⚠️ for the abort boundary; `--delete-after` stays ✅).
|
||||||
|
|
||||||
|
17. **Audit cycle** on `fix/audit-cycle` (from `dev` @ `0fbb9de`;
|
||||||
|
`PROTOCOL_VERSION` stays `2.28.0`): a security/correctness pass over the
|
||||||
|
parity-2.29 baseline. It raises the decompression ceiling to the 256 MiB
|
||||||
|
protocol whole-file bound (`-z` on 100–256 MiB files now works), paces the
|
||||||
|
plaintext-TCP `--sendfile` path with `--bwlimit`, confines the `--temp-dir`
|
||||||
|
scratch dir by the fd's real path (symlink escape refused), masks
|
||||||
|
client-controlled setuid/setgid/sticky bits when super activities are not
|
||||||
|
permitted, sets the daemon umask to `022`, makes `--partial-dir` imply
|
||||||
|
`--partial`, rejects the unsupported filter modifiers (`x`/`e`/`n`/`w`),
|
||||||
|
enforces `MAX_FILTER_RULES` client-side, rejects unknown wire `Status`
|
||||||
|
values, and hardens credentials/signal handling (with the accompanying
|
||||||
|
refactors and docs). No row changes classification, so the matrix stays
|
||||||
|
**120 ✅ / 10 ⚠️ / 27 ❌ = 157**. This docs pass is on `fix/audit-docs2`.
|
||||||
|
|
||||||
|
## Next steps
|
||||||
|
1. **Open and merge the audit-cycle PR** (`fix/audit-cycle`, including this
|
||||||
|
`fix/audit-docs2` docs pass) into `dev` once reviewed. `dev` is the default
|
||||||
|
branch; all PRs target `dev`, never `main` directly.
|
||||||
|
2. **Remaining deferred items:**
|
||||||
|
- **Large structural refactors:** delete-engine consolidation
|
||||||
|
(`delete_extras_fd`/`manifest_delete_extras`/the delete-plan path),
|
||||||
|
god-function splits, and translation-unit splits.
|
||||||
|
- **`--progress`/`--info` receiver→sender event channel:** the root `./`
|
||||||
|
line, ancestor-directory suppression, receiver-side `skip`/`backup` echo,
|
||||||
|
and symlink/empty-dir quick-check feedback.
|
||||||
|
- **`--delete-before` phase-0 keep-set** (rsync fixes the file list before
|
||||||
|
the data pass; FastSync keeps its pre-scan snapshot race).
|
||||||
|
- **>256 MiB single-file streaming** (B4, the general whole-file limit).
|
||||||
|
- **Wire native-size framing:** lengths are native `size_t` and the protocol
|
||||||
|
assumes homogeneous word size/endianness — document or move to fixed-width
|
||||||
|
framing.
|
||||||
|
- **SCRAM-like daemon auth channel binding:** no TLS channel binding today
|
||||||
|
(and it is not RFC 5802).
|
||||||
|
- Still-open security nits: the pre-auth config/daemon-auth handshake has no
|
||||||
|
aggregate wall-clock deadline (per-message timeout only — slowloris holds
|
||||||
|
connection slots); the per-source registry fails open when the shared table
|
||||||
|
is full (per-module/global caps and host ACLs still apply).
|
||||||
|
3. **Out of scope / intentional:** pull (remote source) mode is **not** planned —
|
||||||
|
FastSync is push-only; see `RSYNC_COMPAT.md#direction`.
|
||||||
|
|
||||||
|
## Key facts / commands
|
||||||
|
- CI image: `gitea.tap-tap.win/taptap/fastsync-ci:v11` (alias `fastsync-ci:local`).
|
||||||
|
- Build/test: `cmake -B build -S . -DSTRICT_WARNINGS=ON && cmake --build build -j$(nproc) && ./build/tests`
|
||||||
|
then `python3 -m pytest tests/integration/ -n 4 --dist=load -m "not setpriv"`.
|
||||||
|
- Dev shell: `nix-shell` (provides clang-format, cppcheck, pytest-xdist, openssh,
|
||||||
|
rsync, iproute2, valgrind, lcov; does not build on entry).
|
||||||
|
- Gitea API token: supplied out-of-band via the `TOKEN` environment variable; it is
|
||||||
|
intentionally **not** recorded in this file.
|
||||||
|
- CI polling: `GET /api/v1/repos/TapTap/FastSync/actions/runs?limit=N`, match `head_sha`,
|
||||||
|
then `/actions/runs/<id>/jobs`.
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
#FastSync
|
# FastSync
|
||||||
|
|
||||||
FastSync is a high-performance file synchronization tool designed to become a
|
FastSync is a high-performance file synchronization tool designed to become a
|
||||||
drop-in replacement for common `rsync` workflows. It keeps the familiar
|
drop-in replacement for common `rsync` workflows. It keeps the familiar
|
||||||
@@ -7,7 +7,7 @@ multithreading, streaming zstd compression, chunking, zero-copy TCP transfers,
|
|||||||
and native TCP/TLS transports.
|
and native TCP/TLS transports.
|
||||||
|
|
||||||
The release version is FastSync's client/server protocol version (printed by
|
The release version is FastSync's client/server protocol version (printed by
|
||||||
`fastsync --version`); client and server must match. See
|
`./build/client --version`); client and server must match. See
|
||||||
[CHANGELOG.md](CHANGELOG.md) for the history.
|
[CHANGELOG.md](CHANGELOG.md) for the history.
|
||||||
|
|
||||||
The compatibility target is straightforward:
|
The compatibility target is straightforward:
|
||||||
@@ -28,7 +28,7 @@ FastSync uses a producer-consumer transfer pipeline and can combine several
|
|||||||
optimizations for large or high-latency transfers:
|
optimizations for large or high-latency transfers:
|
||||||
|
|
||||||
- Multithreaded scanning, loading, and sending.
|
- Multithreaded scanning, loading, and sending.
|
||||||
- Streaming zstd compression with levels 1 through 22.
|
- Streaming compression (zstd by default, plus lz4/zlib/zlibx) with levels 1 through 22.
|
||||||
- Configurable file chunking and compact chunk serialization.
|
- Configurable file chunking and compact chunk serialization.
|
||||||
- `sendfile()` zero-copy transfers over TCP.
|
- `sendfile()` zero-copy transfers over TCP.
|
||||||
- Batched incremental checks to reduce round trips.
|
- Batched incremental checks to reduce round trips.
|
||||||
@@ -51,28 +51,52 @@ replacement for every rsync feature or protocol mode.
|
|||||||
- Rsync-style source and destination arguments.
|
- Rsync-style source and destination arguments.
|
||||||
- SSH transport using `user@host:destination` paths below the remote authorized root.
|
- SSH transport using `user@host:destination` paths below the remote authorized root.
|
||||||
- TCP client/server transfers.
|
- TCP client/server transfers.
|
||||||
- Dry runs, excludes, includes, size filters, backups, statistics, and
|
- Dry runs (server-contacting since protocol 2.21.0 for server-routed targets),
|
||||||
bandwidth limiting.
|
excludes, includes, size filters, backups, statistics, and bandwidth
|
||||||
- Incremental size/mtime checks and optional xxHash64 content checks.
|
limiting.
|
||||||
|
- Incremental size/mtime checks and optional content checks (`xxh128` by
|
||||||
|
default, selectable with `--checksum-choice`).
|
||||||
- FastSync-native delta transfer for changed files.
|
- FastSync-native delta transfer for changed files.
|
||||||
- Optional mode and timestamp preservation.
|
- Optional mode and timestamp preservation.
|
||||||
- Delete manifests with server-side delete authorization.
|
- Delete manifests with server-side delete authorization.
|
||||||
- Temporary-file writes with atomic rename by default.
|
- Temporary-file writes with atomic rename by default.
|
||||||
- Path traversal checks and destination-root confinement.
|
- Path traversal checks and destination-root confinement.
|
||||||
|
|
||||||
### Not yet equivalent to rsync
|
### Boundaries and documented divergences
|
||||||
|
|
||||||
|
The items below summarize FastSync's rsync compatibility status — recently
|
||||||
|
closed gaps and the remaining known divergences. Each row of the detailed
|
||||||
|
matrix is classified as parity, caveat, or divergent in
|
||||||
|
[`RSYNC_COMPAT.md`](RSYNC_COMPAT.md).
|
||||||
|
|
||||||
- The FastSync wire protocol is not the rsync wire protocol.
|
- The FastSync wire protocol is not the rsync wire protocol.
|
||||||
- SSH mode requires `fastsync-server` on the remote host.
|
- SSH mode requires `fastsync-server` on the remote host.
|
||||||
- Archive mode does not yet provide all of rsync's `-rlptgoD` behavior.
|
- Archive mode covers rsync's `-rlptgoD` behavior — links, permissions, times,
|
||||||
- Symlink transfer is incomplete; link targets are not yet recreated in all
|
owner, group, devices, and special files — and does not imply compression or
|
||||||
modes.
|
multithreading (see [Client](#client)). Ownership application is still
|
||||||
- Owner/group, ACL, xattr, and hard-link handling is incomplete or
|
privilege-gated: a receiver that cannot `chown` logs a warning and skips it.
|
||||||
unavailable.
|
Under `-p` the source mode is copied exactly, including group/other-write
|
||||||
|
bits; setuid/setgid/sticky bits are copied only when super-user activities are
|
||||||
|
permitted, and are masked under `SUPER_MODE_OFF`/`--no-super` (see
|
||||||
|
[`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)).
|
||||||
|
- Symlink transfer stores targets **verbatim** (`-l`/`--links`), including
|
||||||
|
absolute and `..`-bearing targets, matching rsync. The receiver does not
|
||||||
|
enforce a containment predicate by default; `--safe-links` drops unsafe
|
||||||
|
targets on the sender, and `--munge-links` rewrites them with rsync's
|
||||||
|
`/rsyncd-munged/` marker. `--trust-sender` does not affect symlink targets.
|
||||||
|
A destination later consumed by a link-following tool can therefore follow a
|
||||||
|
link outside the receive root — use `--safe-links` for untrusted sources.
|
||||||
|
- Hard links (`-H`/`--hard-links`), extended attributes (`-X`/`--xattrs`), and
|
||||||
|
POSIX ACLs (`-A`/`--acls`) are preserved; owner/group is applied through
|
||||||
|
`-o`/`-g` (or an `-a`/`--archive` transfer), through the opt-in identity flags
|
||||||
|
(`--chown`/`--usermap`/`--groupmap`/`--numeric-ids`/`--copy-as`), and only when
|
||||||
|
the receiver has permission. See
|
||||||
|
[`RSYNC_COMPAT.md`](RSYNC_COMPAT.md) for the exact semantics and documented
|
||||||
|
divergences.
|
||||||
- Device and special-file preservation is implemented with documented
|
- Device and special-file preservation is implemented with documented
|
||||||
divergences: recreated device nodes require `CAP_MKNOD` on the receiver (a
|
divergences: recreated device nodes require `CAP_MKNOD` on the receiver (a
|
||||||
non-root receiver skips the entry), and sockets cannot be recreated (FIFOs
|
non-root receiver skips the entry), while FIFOs **and unix sockets** are
|
||||||
are).
|
recreated (`--specials`).
|
||||||
- Sparse-file hole preservation (`-S`, `--sparse`) is implemented receiver-side:
|
- Sparse-file hole preservation (`-S`, `--sparse`) is implemented receiver-side:
|
||||||
long all-zero runs are written as holes (no wire change; the full file image
|
long all-zero runs are written as holes (no wire change; the full file image
|
||||||
is already in memory).
|
is already in memory).
|
||||||
@@ -80,78 +104,164 @@ replacement for every rsync feature or protocol mode.
|
|||||||
the write atomic (temp + rename). With `--partial`, a failed/interrupted write
|
the write atomic (temp + rename). With `--partial`, a failed/interrupted write
|
||||||
now retains the already-written temp at the destination path (best-effort) so
|
now retains the already-written temp at the destination path (best-effort) so
|
||||||
a later `--append`/`--append-verify` run can resume it.
|
a later `--append`/`--append-verify` run can resume it.
|
||||||
- `--dirs` is not implemented. Its compatibility aliases `--old-dirs` and
|
- `-d`/`--dirs` and its aliases `--old-dirs`/`--old-d` transfer the named
|
||||||
`--old-d` are recognized but rejected explicitly rather than silently using
|
directory entries without recursing into their contents.
|
||||||
FastSync's recursive directory behavior.
|
|
||||||
- Short-option names are now rsync-parity (Phase 7 Wave A): FastSync's former
|
- Short-option names are now rsync-parity (Phase 7 Wave A): FastSync's former
|
||||||
collisions were renamed (`-j`/`--threads`, `--preserve`, `--sendfile`,
|
collisions were renamed (`-j`/`--threads`, `--preserve`, `--sendfile`,
|
||||||
`--chunk-serialization`, `--timeout`, `--ssh-port`), so `-m`, `-M`, `-f`,
|
`--chunk-serialization`, `--timeout`, `--ssh-port`), so `-m`, `-M`, `-f`,
|
||||||
`-s`, `-T`, `-p`, `-c`, `-a`, and `-z` follow rsync. See `RSYNC_COMPAT.md`.
|
`-s`, `-T`, `-p`, `-c`, `-a`, and `-z` follow rsync.
|
||||||
|
- Short-option clustering (`-av`, `-aAX`, `-rlpt`) and attached values
|
||||||
|
(`-B1000`, `-essh`, `-MOPT`, `--opt=value`) are accepted, matching rsync.
|
||||||
|
- `-r`, `-b`, `-L`, and `-B` are parsed with the rsync short names.
|
||||||
|
- `--stats` prints the counters FastSync can observe plus the receiver-only
|
||||||
|
counters reported over the wire (`Matched data`, deleted files, and the
|
||||||
|
created/literal counters); `Number of files` and `Number of created files`
|
||||||
|
carry rsync's per-type breakdown. `--progress` prints rsync-style per-file
|
||||||
|
blocks including the leading `./` line, and (when progress is requested) a
|
||||||
|
paths-only pre-count supplies rsync's `to-chk` denominator.
|
||||||
|
- Codecs match rsync 3.4.1: `zstd`/`lz4`/`zlib`/`zlibx` compression and
|
||||||
|
`xxh128`/`xxh3`/`xxh64`/`md5`/`md4`/`sha1`/`none` checksums. `auto` honors
|
||||||
|
`RSYNC_COMPRESS_LIST`/`RSYNC_CHECKSUM_LIST` and otherwise follows rsync's
|
||||||
|
compiled-in order. An omitted `--compress-level` uses the codec's rsync
|
||||||
|
default (zstd 3, zlib/zlibx 6, lz4 ignored); `zlib`/`zlibx` share the
|
||||||
|
literal-only zlib path (rsync's zlibx semantics), and the transfer checksum is
|
||||||
|
not separately selectable.
|
||||||
|
|
||||||
The detailed flag matrix is maintained in
|
The detailed flag matrix is maintained in
|
||||||
[`RSYNC_COMPAT.md`](RSYNC_COMPAT.md). It distinguishes implemented,
|
[`RSYNC_COMPAT.md`](RSYNC_COMPAT.md). It reports each row as **parity**,
|
||||||
partial, alternate, and planned behavior.
|
**caveat** (works with a documented divergence), or **divergent** (not
|
||||||
|
supported), rather than treating "parsed" as parity.
|
||||||
|
|
||||||
## Quick Start
|
## Quick Start
|
||||||
|
|
||||||
### Build
|
### Build
|
||||||
|
|
||||||
`compile_commands.json` is a symlink to `build/compile_commands.json` and is used by clangd/editor tooling; its target is generated by the build, so it dangles until the first build.
|
```bash
|
||||||
|
cmake -B build -S .
|
||||||
|
cmake --build build -j$(nproc)
|
||||||
|
```
|
||||||
|
|
||||||
|
This produces `./build/client` and `./build/server`. `compile_commands.json` is a symlink to `build/compile_commands.json` and is used by clangd/editor tooling; its target is generated by the build, so it dangles until the first build.
|
||||||
|
|
||||||
### Client
|
### Client
|
||||||
|
|
||||||
| Argument | Description |
|
| Argument | Description |
|
||||||
|----------|-------------|
|
|----------|-------------|
|
||||||
| Positional | `<source> <dest>` — automatic SSH detection if dest contains `:` |
|
| Positional | `<source> <dest>` — automatic SSH detection if dest contains `:` |
|
||||||
| `-c, --checksum` | Verify content by checksum instead of size+mtime |
|
| `-c, --checksum` | Verify content by checksum instead of size+mtime (implies the incremental checksum quick-check) |
|
||||||
| `-z, --compress [level]` | Enable streaming zstd compression (level 1–22, default 5) |
|
| `--checksum-choice <alg>` | Whole-file checksum algorithm: `xxh128` (default), `xxh3`, `xxh64`/`xxhash`, `md5`, `md4`, `sha1`, `none`, or `auto` (plus rsync's two-name `transfer,pre-transfer` form) |
|
||||||
| `-a, --archive` | rsync archive mode (`-rlptgoD`): links, metadata, devices and specials (not compression/multithreading) |
|
| `-z, --compress [level]` | Enable streaming compression (default `zstd`; level 1–22, default 5) |
|
||||||
|
| `--compress-choice <alg>` | Compression algorithm: `zstd` (default), `lz4`, `zlib`, `zlibx`, `none`, or `auto` |
|
||||||
|
| `--skip-compress <list>` | Skip compression for suffixes (`/`- or `,`-separated); defaults to rsync 3.4.1's built-in suffix list |
|
||||||
|
| `-a, --archive` | rsync archive mode (`-rlptgoD`): links, perms, times, owner, group, devices and specials; ownership application stays privilege-gated (not compression/multithreading) |
|
||||||
| `-j, --threads[=N]` | Multithreading mode; `N` (1–256) sets the parallel scanner worker count, bare `-j`/`--threads` uses the default |
|
| `-j, --threads[=N]` | Multithreading mode; `N` (1–256) sets the parallel scanner worker count, bare `-j`/`--threads` uses the default |
|
||||||
| `-m` | rsync `--prune-empty-dirs` (short form now rsync-parity) |
|
| `-m` | rsync `--prune-empty-dirs` (short form now rsync-parity) |
|
||||||
|
| `-r, --recursive` | Recurse into directories (FastSync is always recursive; accepted for rsync compatibility) |
|
||||||
|
| `-d, --dirs` | Transfer the named directory entries without recursing into their contents; aliases `--old-dirs`/`--old-d` |
|
||||||
|
| `-R, --relative` | Use rsync's relative path semantics (including the `/./` cut); with `--files-from`, preserve each listed entry's relative path below the destination root |
|
||||||
| `--chunk-serialization` | Chunk serialization (batch all files per chunk; long form only) |
|
| `--chunk-serialization` | Chunk serialization (batch all files per chunk; long form only) |
|
||||||
| `-s` | rsync `--secluded-args` compatibility no-op (remote SSH argv is already injection-safe) |
|
| `-s` | rsync `--secluded-args` compatibility no-op (remote SSH argv is already injection-safe) |
|
||||||
| `--sendfile` | Sendfile zero-copy. Incompatible with compression / chunk serialization. TCP only. Long form only. |
|
| `--sendfile` | Sendfile zero-copy. Incompatible with compression / chunk serialization. TCP only. Long form only. |
|
||||||
| `--preserve` | Preserve supported file metadata (mode and mtime; ownership and atime are unsupported) |
|
| `--preallocate` | Allocate destination file space up front (fail-fast on a full disk) |
|
||||||
| `-n, --dry-run` | Scan and print what would be transferred |
|
| `--append` | Resume a shorter destination by appending only its tail (prefix not verified; requires `--incremental`) |
|
||||||
| `-p, --perms` | Preserve permission bits (part of the metadata bundle) |
|
| `--append-verify` | Like `--append`, but verifies the retained prefix checksum first (falls back to a full transfer on mismatch) |
|
||||||
| `--ssh-port <port>` | SSH port (default: 22) |
|
| `-W, --whole-file` | Transfer changed files without delta processing; `--no-whole-file` clears it |
|
||||||
| `-v, --verbose` | Enable debug logging |
|
| `-B <n>, --block-size <n>` | Delta block size in bytes (alias `--delta-block`) |
|
||||||
| `-q, --quiet` | Suppress non-error output |
|
| `--checksum-seed <n>` | Seed for the whole-file xxHash digest; an unset/`0` seed is randomized per transfer, matching rsync |
|
||||||
| `--progress` | Show real-time transfer speed |
|
| `-I, --ignore-times` | Transfer files even when size and mtime match |
|
||||||
| `-P` | Enables partial-transfer mode + progress output; interrupted writes retain the already-written temp for resumption |
|
| `--size-only` | Skip incremental files matching in size, ignoring mtime |
|
||||||
| `--delete` | Delete files on receiver not present in source (default timing: delete-after, i.e. only after the whole transfer succeeded) |
|
| `--preserve` | Preserve mode and mtime (`-p` + `-t`; add `-o`/`-g` for owner/group or `-U`/`--atimes` for atime; `-N`/`--crtimes` captures birth time but cannot apply it) |
|
||||||
|
| `-U, --atimes` | Preserve access times. Captured with the metadata payload; does not enable ownership. |
|
||||||
|
| `-N, --crtimes` | Capture birth time; cannot be applied (documented divergence) |
|
||||||
|
| `-p, --perms` | Preserve permission bits. The source mode is copied exactly, including group/other-write bits; setuid/setgid/sticky are copied only when super-user activities are permitted (`SUPER_MODE_OFF`/`--no-super` masks them) |
|
||||||
|
| `-t, --times` | Preserve modification times |
|
||||||
|
| `-o, --owner` | Preserve the source owner (privilege-gated; mapped by name on the receiver with a numeric fallback) |
|
||||||
|
| `-g, --group` | Preserve the source group (privilege-gated; mapped by name on the receiver with a numeric fallback) |
|
||||||
|
| `--no-perms`, `--no-times`, `--no-owner`, `--no-group`, `--no-preserve` | Negate the per-attribute flags (short `--no-p`/`--no-t`/`--no-o`/`--no-g`; `--no-preserve` clears all four) |
|
||||||
|
| `-E, --executability` | Preserve executable permission bits |
|
||||||
|
| `-X, --xattrs` | Preserve user `user.*` extended attributes |
|
||||||
|
| `-A, --acls` | Preserve POSIX ACLs |
|
||||||
|
| `--chmod <changes>` | Modify transferred permissions (rsync syntax) |
|
||||||
|
| `--chown=USER:GROUP` | Override the ownership of transferred files |
|
||||||
|
| `--usermap=MAP` | Map usernames when applying ownership |
|
||||||
|
| `--groupmap=MAP` | Map group names when applying ownership |
|
||||||
|
| `--numeric-ids` | Apply source numeric uid/gid directly instead of mapping by name |
|
||||||
|
| `--copy-as=USER[:GROUP]` | Force every written entry to USER[:GROUP] (requires a privileged receiver) |
|
||||||
|
| `--fake-super` | Record the resolved owner plus full mode/rdev in rsync's reserved `user.rsync.%stat` xattr (rsync 3.4.1 grammar) and replay the permission bits; never performs a real chown |
|
||||||
|
| `--super` | Permit the receiver to attempt confined super-user activities (device nodes) |
|
||||||
|
| `-D` | Preserve device and special files (implies `--devices --specials`) |
|
||||||
|
| `--devices` | Recreate device nodes on the destination (privileged; skipped without `CAP_MKNOD`) |
|
||||||
|
| `--specials` | Recreate special files: FIFOs and unix sockets |
|
||||||
|
| `--remove-source-files` | Remove regular source files after a successful transfer |
|
||||||
|
| `--exclude <pattern>` | Exclude files matching glob pattern (repeatable) |
|
||||||
|
| `--exclude-from <file>` | Read exclude patterns from a file (one per line) |
|
||||||
|
| `--include <pattern>` | Only transfer files matching glob pattern (repeatable, whitelist) |
|
||||||
|
| `--include-from <file>` | Read include patterns from a file |
|
||||||
|
| `--files-from <file>` | Read the source file list from FILE (paths relative to the source root) |
|
||||||
|
| `--max-size <n>` | Skip files larger than n bytes |
|
||||||
|
| `--min-size <n>` | Skip files smaller than n bytes |
|
||||||
|
| `-x, --one-file-system` | Do not cross filesystem boundaries; the mount-point directory entry is emitted (empty at the destination) without descending |
|
||||||
|
| `--max-alloc <SIZE>` | Maximum single allocation (binary units: B, K, M, G, T, P, E; default 1G; `0` = no local limit, matching rsync) |
|
||||||
|
| `-u, --update` | Skip files newer than the source on the receiver |
|
||||||
|
| `--incremental` | Skip files unchanged since last transfer (size + mtime). Auto-enables `--preserve`. Incompatible with `--chunk-serialization`. |
|
||||||
|
| `--existing` | Skip files not already present at the destination; update existing files normally. |
|
||||||
|
| `--ignore-existing` | Skip files that already exist on the receiver; like rsync it does not apply to directories or symlinks. |
|
||||||
|
| `--compare-dest <dir>` | Extra comparison basis: unchanged files are not transferred (requires/implies `--incremental`; a basis MISS above the 256 MiB whole-file payload bound is refused — see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)) |
|
||||||
|
| `--copy-dest <dir>` | Like `--compare-dest`, but copies the unchanged file from DIR into the destination (same basis-size caveat; see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)) |
|
||||||
|
| `--link-dest <dir>` | Like `--copy-dest`, but hard-links the unchanged file from DIR (repeatable; earlier DIRs win; same basis-size caveat; see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)) |
|
||||||
|
| `--verify-basis` | FastSync-only: require a basis hit (`--compare-dest`/`--copy-dest`/`--link-dest`) to match the source by whole-file digest instead of trusting the size+mtime quick-check (default matches rsync) |
|
||||||
|
| `--delete` | Delete files on receiver not present in source (default timing: delete-during, matching rsync, so destination space is freed progressively). Scoped to the synchronized directories, so `--files-from` subsets are safe |
|
||||||
| `--delete-before` | Delete extras before the transfer starts (implies `--delete`) |
|
| `--delete-before` | Delete extras before the transfer starts (implies `--delete`) |
|
||||||
| `--delete-during`, `--del` | Delete extras once the keep-set is known, before data is applied (implies `--delete`) |
|
| `--delete-during`, `--del` | Delete extras once the keep-set is known, before data is applied (implies `--delete`) |
|
||||||
| `--delete-delay` | Delete extras only after a successful transfer (implies `--delete`) |
|
| `--delete-delay` | Delete extras only after a successful transfer (implies `--delete`) |
|
||||||
| `--delete-after` | Explicit delete-after timing (implies `--delete`) |
|
| `--delete-after` | Explicit delete-after timing (implies `--delete`) |
|
||||||
| `--exclude <pattern>` | Exclude files matching glob pattern (repeatable) |
|
| `--delete-commit` | FastSync-only: keep the pre-2.28 atomic timing — delete only after the whole transfer succeeded (identical timing to `--delete-after`) |
|
||||||
| `--exclude-from <file>` | Read exclude patterns from a file (one per line) |
|
| `--delete-excluded` | Also delete filter-excluded destination mirrors (size-pruned mirrors stay protected) |
|
||||||
| `--include <pattern>` | Only transfer files matching glob pattern (repeatable, whitelist) |
|
| `--max-delete <n>` | Delete at most n destination entries; the rest are skipped and the run exits 25 (partial), matching rsync |
|
||||||
| `--max-size <n>` | Skip files larger than n bytes |
|
| `--delay-updates` | Put updated files into place only at the end of the transfer (`--force` is honored at publication; the fixed `.fastsync-stage` staging name diverges from rsync — see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)) |
|
||||||
| `--min-size <n>` | Skip files smaller than n bytes |
|
| `-T, --temp-dir <dir>` | Scratch directory for temp files before the atomic install; confined to the receive root (a relative path resolves below it; an absolute path is accepted only when it canonicalizes inside it), with an `EXDEV` non-atomic copy fallback |
|
||||||
| `--max-alloc <SIZE>` | Maximum single allocation (binary units: B, K, M, G, T, P, E; default 1G) |
|
| `-n, --dry-run` | Report what would be transferred without mutating the destination. Since protocol 2.21.0 a server-routed target contacts the receiver and reports would-transfer based on receiver state; a plain local destination keeps the client-side scan. Never mutates or deletes. |
|
||||||
| `--incremental` | Skip files unchanged since last transfer (size + mtime). Auto-enables `--preserve`. Incompatible with `--chunk-serialization`. |
|
| `-v, --verbose` | Enable debug logging |
|
||||||
| `--existing` | Skip files not already present at the destination; update existing files normally. |
|
| `-q, --quiet` | Suppress non-error output |
|
||||||
| `--bwlimit <KB/s>` | Bandwidth limit in kilobytes per second |
|
| `--progress` | Show rsync-style per-file progress blocks from the receiver's wire counters; the root `./` line is printed whenever progress is active (rsync prints it only when the transfer root is created) |
|
||||||
| `--chunk-size <n>` | Chunk size in bytes (default: 10485760) |
|
| `-P` | Enables partial-transfer mode + progress output; interrupted writes retain the already-written temp for resumption |
|
||||||
| `--timeout <sec>` | Positive I/O timeout in seconds, applied to both the socket (`SO_RCVTIMEO`/`SO_SNDTIMEO`, built-in default 30 s) and the per-message protocol poll deadline (built-in default 60 s). Omit the option to keep both built-ins; `0` is rejected. The server side keeps the built-in 60 s protocol window (the value is not sent on the wire). |
|
| `--stats` | Print transfer statistics at end (bytes, files, timing), including the receiver-only counters reported over the wire; `Number of files` and `Number of created files` carry rsync's per-type breakdown (deleted files are reported as a single total) |
|
||||||
| `--contimeout <sec>` | Connection timeout in seconds (default: 10) |
|
| `-i, --itemize-changes` | Print an rsync-style per-file change line |
|
||||||
| `--backup` | Backup existing destination files before overwriting |
|
| `--out-format=FORMAT` | Output format for changed files (`%f %n %l %b %c %C %i %M %%`) |
|
||||||
| `--backup-dir <dir>` | Target directory for backups (requires `--backup`) |
|
| `--list-only` | List source files instead of transferring |
|
||||||
| `--stats` | Print transfer statistics at end (bytes, files, timing) |
|
| `--fsync` | Fsync every written file before publication |
|
||||||
| `-h, --human-readable` | Format transfer byte sizes with binary units |
|
| `-h, --human-readable` | Format transfer byte/rate counts with rsync's decimal (base-1000) units |
|
||||||
| `--max-depth <n>` | Maximum directory depth to recurse (0 = unlimited, default: 0) |
|
| `--max-depth <n>` | Maximum directory depth to recurse (0 = unlimited, default: 0) |
|
||||||
| `--log-file <path>` | Write log messages to file instead of stderr |
|
| `--log-file <path>` | Write log messages to file instead of stderr |
|
||||||
|
| `--write-batch=FILE` | Run the normal live transfer and also emit a self-contained batch file of the source tree (FastSync-native format, not rsync-interoperable) |
|
||||||
|
| `--only-write-batch=FILE` | Emit the batch file only (no destination, no server); FastSync-native format, not rsync-interoperable |
|
||||||
|
| `--read-batch=FILE` | Apply a batch file to the destination (no source, no server); FastSync-native format, not rsync-interoperable |
|
||||||
| `--source-dir <path>` | Source directory (overrides `FASTSYNC_SOURCE_DIR`) |
|
| `--source-dir <path>` | Source directory (overrides `FASTSYNC_SOURCE_DIR`) |
|
||||||
| `--dest-dir <path>` | Server destination directory (overrides `FASTSYNC_DEST_DIR`) |
|
| `--dest-dir <path>` | Server destination directory (overrides `FASTSYNC_DEST_DIR`) |
|
||||||
| `--save-to-disk` | Write received files to disk |
|
| `--save-to-disk` | Write received files to disk |
|
||||||
| `--server-host <ip>` | Server IP address (default: `127.0.0.1`) |
|
| `--server-host <ip>` | Server IP address (default: `127.0.0.1`) |
|
||||||
| `--server-port <n>` | Server port (default: `8080`) |
|
| `--server-port <n>` | Server port (default: `8080`) |
|
||||||
|
| `--ssh-port <port>` | SSH port (default: 22) |
|
||||||
|
| `-e, --rsh <command>` | Remote shell to launch for the SSH transport (default: `ssh`; may include arguments, e.g. `-e "ssh -p 2222"`) |
|
||||||
|
| `-M, --remote-option=OPT` | Append OPT to the remote server invocation over SSH (repeatable) |
|
||||||
|
| `--address <ip>` | Bind the outgoing client socket to this source address |
|
||||||
|
| `-4, --ipv4` | Force IPv4 for destination resolution |
|
||||||
|
| `-6, --ipv6` | Force IPv6 for destination resolution |
|
||||||
|
| `--sockopts=OPTS` | Comma-separated OPT=VAL socket options applied before connect (`TCP_NODELAY`, `SO_KEEPALIVE`, `SO_RCVBUF`, `SO_SNDBUF`, `SO_REUSEADDR`) |
|
||||||
|
| `--bwlimit <RATE>` | Bandwidth limit, using rsync's exact `parse_size_arg` grammar: a bare value is KiB/s; `K`/`M`/`G`/`T`/`P` are binary suffixes; `KB`/`MB` are decimal and `KiB`/`MiB` binary; decimals are accepted and quantized to whole KiB; `0` (or empty) means no limit. Also paces `--sendfile` transfers |
|
||||||
|
| `--chunk-size <n>` | Chunk size in bytes (default: 10485760) |
|
||||||
|
| `--timeout <sec>` | I/O timeout in seconds, applied to both the socket (`SO_RCVTIMEO`/`SO_SNDTIMEO`) and the per-message protocol poll deadline. Default `0` = disabled (matching rsync); `0` disables it. `--no-timeout` is the negation. The value is not sent on the wire; the server side keeps its own safe floor. |
|
||||||
|
| `--contimeout <sec>` | Connection timeout in seconds (default: 60, matching rsync); `0` disables it (`--no-contimeout` is the negation) |
|
||||||
|
| `--stop-after=MINS` | Stop the transfer after MINS minutes (a positive integer); whatever was already transferred is kept |
|
||||||
|
| `--stop-at=TIME` | Stop at an absolute time. Accepts rsync's `parse_time` forms (`Y-M-DTh:m`, `Y/M/DTh:m`, `Y-M-D`, `M-D`, `D`, `h:m`, `:m`, `T h:m`; omitted fields resolve to the next matching point in the local timezone), plus `now+N[smhd]` and FastSync's `HH:MM`/`HH:MM:SS` clock-time spelling. An early stop skips the late `--delete` keep-set |
|
||||||
|
| `-b, --backup` | Backup existing destination files before overwriting |
|
||||||
|
| `--backup-dir <dir>` | Target directory for backups (requires `--backup`) |
|
||||||
| `--tls` | Enable TLS encryption |
|
| `--tls` | Enable TLS encryption |
|
||||||
| `--cert <path>` | TLS certificate file (PEM) |
|
| `--cert <path>` | TLS certificate file (PEM) |
|
||||||
| `--key <path>` | TLS private key file (PEM) |
|
| `--key <path>` | TLS private key file (PEM) |
|
||||||
| `--ca <path>` | TLS CA certificate file for verification (PEM) |
|
| `--ca <path>` | TLS CA certificate file for verification (PEM) |
|
||||||
| `--client-cn <name>` | TLS client certificate common name; mandatory with `--tls` (a TLS connection always verifies the client CN) |
|
|
||||||
|
The exhaustive rsync flag matrix is in [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md).
|
||||||
|
|
||||||
**Per-message vs. connection timeouts.** `--timeout` bounds each individual protocol
|
**Per-message vs. connection timeouts.** `--timeout` bounds each individual protocol
|
||||||
send/receive (the `poll()` deadline), so a peer that stops mid-frame is dropped. It
|
send/receive (the `poll()` deadline), so a peer that stops mid-frame is dropped. It
|
||||||
@@ -190,60 +300,69 @@ transfer is never aborted.
|
|||||||
| `FASTSYNC_SOURCE_DIR` | — | Source directory fallback |
|
| `FASTSYNC_SOURCE_DIR` | — | Source directory fallback |
|
||||||
| `FASTSYNC_DEST_DIR` | — | Destination directory fallback |
|
| `FASTSYNC_DEST_DIR` | — | Destination directory fallback |
|
||||||
| `FASTSYNC_SAVE_TO_DISK` | `false` | Disk persistence fallback |
|
| `FASTSYNC_SAVE_TO_DISK` | `false` | Disk persistence fallback |
|
||||||
| `FASTSYNC_SSH_PORT` | `22` | Default SSH port |
|
|
||||||
| `FASTSYNC_SERVER_HOST` | `127.0.0.1` | Default server host |
|
|
||||||
| `FASTSYNC_SERVER_PORT` | `8080` | Default server port |
|
|
||||||
| `FASTSYNC_TLS_CERT` | — | Default TLS certificate path |
|
|
||||||
| `FASTSYNC_TLS_KEY` | — | Default TLS private key path |
|
|
||||||
| `FASTSYNC_TLS_CA` | — | Default TLS CA certificate path |
|
|
||||||
|
|
||||||
## Implementation Details
|
## Implementation Details
|
||||||
|
|
||||||
### Data Structures
|
### Data Structures
|
||||||
1. **Chunk** — collection of files (~10 MB total by default)
|
|
||||||
2. **File** — path, content (`Data`), optional `FileMetadata` pointer
|
1. **Chunk** — collection of files (~10 MB total by default).
|
||||||
3. **FileMetadata** — `mode`, `uid`, `gid`, `mtime_sec`, `mtime_nsec`;
|
2. **File** — path, content (`Data`), optional `FileMetadata` pointer.
|
||||||
uid / gid are advisory wire fields and are never applied by the receiver;
|
3. **FileMetadata** — `mode`, `uid`, `gid`, `mtime_sec`, `mtime_nsec` (plus
|
||||||
atime is unsupported
|
atime/crtime fields). `uid`/`gid` are applied only through the opt-in
|
||||||
4. **Config** — runtime parameters (transported over wire, TLS settings excluded). Includes `timeout`, `contimeout`, `quiet`, `backup`, `backup_dir`, `stats`, `max_depth`, `log_file`.
|
identity path; atime is preserved with `-U`/`--atimes`; crtime is captured
|
||||||
5. **Queue** — thread-safe bounded queue with condition variables
|
but cannot be set on the destination.
|
||||||
6. **DirectoryScanner** — recursive BFS traversal with exclude and include pattern support, max-depth enforcement
|
4. **Config** — runtime parameters. Most cross the wire (TLS settings
|
||||||
|
excluded); `backup` and `backup_dir` are in the serialized wire table, while
|
||||||
|
`timeout`, `contimeout`, `quiet`, `stats`, `max_depth`, and `log_file` are
|
||||||
|
client-only.
|
||||||
|
5. **Queue** — thread-safe bounded queue with condition variables.
|
||||||
|
6. **DirectoryScanner** — recursive traversal that buffers and sorts each
|
||||||
|
directory (non-directories ascending, then directories ascending) and walks
|
||||||
|
depth-first in rsync flist order, with exclude and include pattern support and
|
||||||
|
max-depth enforcement.
|
||||||
|
|
||||||
### Key Algorithms
|
### Key Algorithms
|
||||||
1. **File scanning** — BFS directory traversal;
|
|
||||||
entries matched against exclude and include patterns,
|
1. **File scanning** — sorted depth-first traversal in rsync flist order (each
|
||||||
max - depth enforced 2. * *Chunking ** — files accumulated until `chunk_size` threshold,
|
directory's non-directories ascending, then its directories ascending);
|
||||||
then flushed 3. *
|
entries matched against exclude and include patterns, with max-depth
|
||||||
*Compression ** — streaming zstd
|
enforced. The `--threads` parallel scanner remains unordered.
|
||||||
via `ZSTD_compressStream2` / `ZSTD_decompressStream` 4. *
|
2. **Chunking** — files accumulated until the `chunk_size` threshold (default
|
||||||
*Network protocol ** — status -
|
10 MiB) is reached, then flushed.
|
||||||
code - driven exchange with metadata packing,
|
3. **Compression** — streaming zstd via `ZSTD_compressStream2()` /
|
||||||
keep - alive,
|
`ZSTD_decompressStream()`, with lz4 and zlib/zlibx codecs also supported
|
||||||
and abort support 5. * *Incremental check ** — client sends `STATUS_CHECK` + path + size +
|
(selectable with `--compress-choice`).
|
||||||
mtime and,
|
4. **Network protocol** — status-code-driven exchange with metadata packing,
|
||||||
with `--checksum`, XXH64 content checksum; server compares against destination. Can be batched via `STATUS_CHECK_BATCH` for reduced round-trips.
|
keep-alive, and abort support.
|
||||||
6. **Bandwidth limiting** — token-bucket algorithm with `nanosleep` throttling on 64 KB write chunks
|
5. **Incremental check** — the client sends `STATUS_CHECK` + path + size +
|
||||||
7. **Metadata restoration** — `chmod()`, `chown()`, `utimensat()` on the receiving side
|
mtime and, with `--checksum`, a whole-file content checksum (`xxh128` by
|
||||||
8. **`--delete`** — sender tracks all sent paths;
|
default; selectable via `--checksum-choice`/`--cc`, seeded by
|
||||||
receiver walks destination tree and removes unlisted files / directories 9. *
|
`--checksum-seed`); the server compares against the destination. Can be
|
||||||
*SSH transport *
|
batched via `STATUS_CHECK_BATCH` for reduced round-trips.
|
||||||
* — `socketpair()` + `fork()` + `execvp("ssh",
|
6. **Bandwidth limiting** — token-bucket algorithm with sleep throttling on
|
||||||
...)` with `ControlMaster` and port support
|
64 KiB write chunks.
|
||||||
10. *
|
7. **Metadata restoration** — mode via `chmod()`/`fchmod()`, times via
|
||||||
*TLS transport ** — OpenSSL `SSL_CTX` with TLS
|
`utimensat()`/`futimens()`, and ownership only with an identity flag via
|
||||||
1.2 minimum,
|
fd-relative `fchown()`/`fchownat()`.
|
||||||
mutual CA verification,
|
8. **`--delete`** — the sender tracks all sent paths; the receiver walks the
|
||||||
transparent `SSL_read`/`SSL_write` via `io_set_ssl()` 11. *
|
destination tree and removes unlisted files and directories.
|
||||||
*Path traversal protection ** — `has_path_traversal()` rejects any file path
|
9. **SSH transport** — `socketpair()` + `fork()` + `execvp("ssh", ...)` with
|
||||||
containing `..` components,
|
`ControlMaster` and port support.
|
||||||
preventing directory escape attacks 12. *
|
10. **TLS transport** — OpenSSL `SSL_CTX` with TLS 1.2 minimum, mutual CA
|
||||||
*Connection limiting ** — server tracks active connections and rejects
|
verification, and transparent `SSL_read()`/`SSL_write()` via
|
||||||
new ones beyond `max_connections` (default 100)13. *
|
`io_set_ssl()`.
|
||||||
*Keep
|
11. **Path traversal protection** — `has_path_traversal()` rejects any file
|
||||||
- alive ** — idle connections receive periodic `STATUS_KEEPALIVE` to detect half
|
path containing `..` components, preventing directory escape attacks.
|
||||||
- open TCP connections 14. * *Abort handling ** — `SIGINT` sets an abort flag; the next protocol operation sends `STATUS_ABORT` for clean server cleanup
|
12. **Connection limiting** — the server tracks active connections and rejects
|
||||||
15. **Atomic writes** — files are written to a `.tmp` suffix then atomically renamed via `rename()`, preventing partial files
|
new ones beyond `max_connections` (default 100).
|
||||||
16. **Backup** — before overwriting, existing files are moved to `--backup-dir` (or same directory with `~` suffix) preserving the original
|
13. **Keep-alive** — idle connections receive periodic `STATUS_KEEPALIVE` to
|
||||||
|
detect half-open TCP connections.
|
||||||
|
14. **Abort handling** — `SIGINT` sets an abort flag; the next protocol
|
||||||
|
operation sends `STATUS_ABORT` for clean server cleanup.
|
||||||
|
15. **Atomic writes** — files are written to a `.tmp` suffix then atomically
|
||||||
|
renamed via `rename()`, preventing partial files.
|
||||||
|
16. **Backup** — before overwriting, existing files are moved to `--backup-dir`
|
||||||
|
(or the same directory with a `~` suffix), preserving the original.
|
||||||
|
|
||||||
## Security Features
|
## Security Features
|
||||||
|
|
||||||
@@ -267,6 +386,8 @@ Received files are written to a temporary path (suffixed with `.tmp`) and then a
|
|||||||
- C11 compiler
|
- C11 compiler
|
||||||
- CMake >= 3.22
|
- CMake >= 3.22
|
||||||
- zstd library
|
- zstd library
|
||||||
|
- zlib library
|
||||||
|
- lz4 library
|
||||||
- OpenSSL (development headers and libraries)
|
- OpenSSL (development headers and libraries)
|
||||||
- pthreads
|
- pthreads
|
||||||
- SSH client (for SSH transport mode only)
|
- SSH client (for SSH transport mode only)
|
||||||
@@ -275,12 +396,12 @@ Received files are written to a temporary path (suffixed with `.tmp`) and then a
|
|||||||
|
|
||||||
**Ubuntu/Debian:**
|
**Ubuntu/Debian:**
|
||||||
```bash
|
```bash
|
||||||
sudo apt install cmake build-essential libzstd-dev libssl-dev openssh-client
|
sudo apt install cmake build-essential libzstd-dev zlib1g-dev liblz4-dev libssl-dev openssh-client
|
||||||
```
|
```
|
||||||
|
|
||||||
**Nix:**
|
**Nix:**
|
||||||
```bash
|
```bash
|
||||||
nix-shell # provides zstd, openssl, cmake, gcc
|
nix-shell # provides zstd, zlib, lz4, openssl, cmake, gcc
|
||||||
```
|
```
|
||||||
|
|
||||||
## Building
|
## Building
|
||||||
@@ -300,7 +421,8 @@ cmake --build build -j$(nproc)
|
|||||||
|
|
||||||
### SSH transfer
|
### SSH transfer
|
||||||
|
|
||||||
The remote host must have `fastsync-server` available in `PATH`, or use
|
The remote host must have `fastsync-server` available in `PATH` (install or
|
||||||
|
copy the built `./build/server` there as `fastsync-server`), or use
|
||||||
`--fastsync-server-path`. SSH starts `fastsync-server --stdio` in its remote
|
`--fastsync-server-path`. SSH starts `fastsync-server --stdio` in its remote
|
||||||
working directory, so use a destination below that directory unless the
|
working directory, so use a destination below that directory unless the
|
||||||
remote server is otherwise configured with a matching authorized root.
|
remote server is otherwise configured with a matching authorized root.
|
||||||
@@ -341,8 +463,13 @@ Plain TCP requires the explicit `--allow-unauthenticated` server option. Use TLS
|
|||||||
authenticated network connections.
|
authenticated network connections.
|
||||||
|
|
||||||
### TLS transfer
|
### TLS transfer
|
||||||
|
|
||||||
|
Server TLS requires `--cert`, `--key`, `--ca`, and `--client-cn`; the client
|
||||||
|
requires `--cert`, `--key`, and `--ca`.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
./build/server --destination-root /path/to --tls --cert server.pem --key server-key.pem -p 8443
|
./build/server --destination-root /path/to --tls --cert server.pem --key server-key.pem \
|
||||||
|
--ca ca.pem --client-cn client -p 8443
|
||||||
./build/client --tls --cert client.pem --key client-key.pem --ca ca.pem \
|
./build/client --tls --cert client.pem --key client-key.pem --ca ca.pem \
|
||||||
--server-host example.com --server-port 8443 \
|
--server-host example.com --server-port 8443 \
|
||||||
--source-dir /path/to/source --dest-dir /path/to/destination \
|
--source-dir /path/to/source --dest-dir /path/to/destination \
|
||||||
@@ -355,32 +482,32 @@ These examples show the intended rsync-style workflow. Options marked as
|
|||||||
FastSync-native are optional performance or transport extensions.
|
FastSync-native are optional performance or transport extensions.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
#Basic synchronization
|
# Basic synchronization
|
||||||
./build/client /source/ /destination/
|
./build/client /source/ /destination/
|
||||||
|
|
||||||
#Archive - style synchronization(current FastSync archive behavior)
|
# Archive-style synchronization (current FastSync archive behavior)
|
||||||
./build/client -a /source/ user@host:destination/
|
./build/client -a /source/ user@host:destination/
|
||||||
|
|
||||||
#Preview a transfer without changing the destination
|
# Preview a transfer without changing the destination
|
||||||
./build/client -n /source/ /destination/
|
./build/client -n /source/ /destination/
|
||||||
|
|
||||||
#Exclude temporary and object files
|
# Exclude temporary and object files
|
||||||
./build/client --exclude '*.tmp' --exclude '*.o' \
|
./build/client --exclude '*.tmp' --exclude '*.o' \
|
||||||
/source/ user@host:destination/
|
/source/ user@host:destination/
|
||||||
|
|
||||||
#Remove destination entries not present in the source
|
# Remove destination entries not present in the source
|
||||||
./build/client --delete /source/ user@host:destination/
|
./build/client --delete /source/ user@host:destination/
|
||||||
|
|
||||||
#Skip unchanged files using size and modification time
|
# Skip unchanged files using size and modification time
|
||||||
./build/client --incremental /source/ user@host:destination/
|
./build/client --incremental /source/ user@host:destination/
|
||||||
|
|
||||||
#Verify content when size and time are not sufficient
|
# Verify content when size and time are not sufficient
|
||||||
./build/client --incremental --checksum /source/ user@host:destination/
|
./build/client --incremental --checksum /source/ user@host:destination/
|
||||||
|
|
||||||
#Preserve supported mode and timestamp metadata
|
# Preserve supported mode and timestamp metadata
|
||||||
./build/client --preserve /source/ user@host:destination/
|
./build/client --preserve /source/ user@host:destination/
|
||||||
|
|
||||||
#Keep backups of overwritten destination files
|
# Keep backups of overwritten destination files
|
||||||
./build/client --backup --backup-dir backups \
|
./build/client --backup --backup-dir backups \
|
||||||
/source/ user@host:destination/
|
/source/ user@host:destination/
|
||||||
```
|
```
|
||||||
@@ -393,11 +520,11 @@ features without changing the meaning of ordinary compatibility options.
|
|||||||
| Option | Purpose |
|
| Option | Purpose |
|
||||||
|---|---|
|
|---|---|
|
||||||
| `-j`, `--threads[=N]` | Enable the multithreaded scanner/loader/sender pipeline. `N` (1–256) sets the parallel scanner worker count; bare `-j`/`--threads` uses the default. |
|
| `-j`, `--threads[=N]` | Enable the multithreaded scanner/loader/sender pipeline. `N` (1–256) sets the parallel scanner worker count; bare `-j`/`--threads` uses the default. |
|
||||||
| `-z [level]`, `--compress [level]` | Enable streaming zstd compression, levels 1-22. |
|
| `-z [level]`, `--compress [level]` | Enable streaming compression (default `zstd`), levels 1-22. |
|
||||||
| `--compress-level <n>` | Set the zstd compression level. |
|
| `--compress-level <n>` | Set the compression level (1-22). Omitted, each codec uses its rsync default: zstd 3, zlib/zlibx 6, lz4 ignored. |
|
||||||
| `--zc <alg>` | Alias for `--compress-choice`. FastSync supports `zstd` and `none`. |
|
| `--zc <alg>` | Alias for `--compress-choice`. FastSync supports `zstd` (default), `lz4`, `zlib`, `zlibx`, `none`, and `auto`; `zlib`/`zlibx` share the same literal-only zlib path. |
|
||||||
| `--zl <n>` | Alias for `--compress-level`. |
|
| `--zl <n>` | Alias for `--compress-level`. |
|
||||||
| `--skip-compress <list>` | Skip compression for comma-separated suffixes; incompatible with `--chunk-serialization`. |
|
| `--skip-compress <list>` | Skip compression for `/`- or `,`-separated suffixes; defaults to rsync 3.4.1's built-in list. Incompatible with `--chunk-serialization`. |
|
||||||
| `--compress-threads <n>` | Use `n` zstd compression workers. Requires compression and a zstd build with threaded support; the setting affects sender CPU work only. |
|
| `--compress-threads <n>` | Use `n` zstd compression workers. Requires compression and a zstd build with threaded support; the setting affects sender CPU work only. |
|
||||||
| `--chunk-size <bytes>` | Set the transfer chunk size. |
|
| `--chunk-size <bytes>` | Set the transfer chunk size. |
|
||||||
| `--chunk-serialization` | Enable FastSync chunk serialization (long form only; `-s` is rsync's `--secluded-args`). |
|
| `--chunk-serialization` | Enable FastSync chunk serialization (long form only; `-s` is rsync's `--secluded-args`). |
|
||||||
@@ -408,11 +535,11 @@ features without changing the meaning of ordinary compatibility options.
|
|||||||
| `--server-host <host>` | Select the TCP server host. |
|
| `--server-host <host>` | Select the TCP server host. |
|
||||||
| `--server-port <port>` | Select the TCP server port (`--port <port>` and `--port=<port>` are rsync-friendly aliases). |
|
| `--server-port <port>` | Select the TCP server port (`--port <port>` and `--port=<port>` are rsync-friendly aliases). |
|
||||||
| `--tls` | Enable TLS for TCP transport. |
|
| `--tls` | Enable TLS for TCP transport. |
|
||||||
| `--bwlimit <KB/s>` | Apply token-bucket bandwidth limiting. |
|
| `--bwlimit <RATE>` | Apply token-bucket bandwidth limiting with rsync's exact `parse_size_arg` grammar (bare = KiB/s, `K`/`M`/`G`/`T`/`P` binary, `KB`/`MB` decimal, `KiB`/`MiB` binary, decimals quantized to whole KiB, `0`/empty = no limit; also paces `--sendfile` transfers). |
|
||||||
| `--progress` | Show transfer progress and throughput. |
|
| `--progress` | Show rsync-style per-file progress blocks from the receiver's wire counters; the root `./` line is printed whenever progress is active (rsync prints it only when the transfer root is created). |
|
||||||
| `--stats` | Print transfer statistics. |
|
| `--stats` | Print transfer statistics, including the receiver-only counters reported over the wire; `Number of files`/`Number of created files` carry rsync's per-type breakdown (deleted files are a single total). |
|
||||||
| `--timeout <seconds>` | Set the socket **and** per-message protocol I/O timeout (positive seconds). Omit to keep the built-in 30 s socket / 60 s protocol defaults. |
|
| `--timeout <seconds>` | Set the socket **and** per-message protocol I/O timeout. Default `0` = disabled (matching rsync); `0` disables it. |
|
||||||
| `--contimeout <seconds>` | Set connection timeout. |
|
| `--contimeout <seconds>` | Connection timeout (default 60, matching rsync); `0` disables it. |
|
||||||
|
|
||||||
Short-option conflicts with rsync have been resolved for the CLI namespace
|
Short-option conflicts with rsync have been resolved for the CLI namespace
|
||||||
(Phase 7): `-c` is now rsync's `--checksum`, `-m` is `--prune-empty-dirs`, `-M`
|
(Phase 7): `-c` is now rsync's `--checksum`, `-m` is `--prune-empty-dirs`, `-M`
|
||||||
@@ -421,7 +548,8 @@ is `--remote-option`, `-f` is `--filter`, `-s` is `--secluded-args`, `-p` is
|
|||||||
long-form-only or new shorts: multithreading is `-j`/`--threads`, metadata
|
long-form-only or new shorts: multithreading is `-j`/`--threads`, metadata
|
||||||
is `--preserve`, sendfile is `--sendfile`, chunk serialization is
|
is `--preserve`, sendfile is `--sendfile`, chunk serialization is
|
||||||
`--chunk-serialization`, timeout is `--timeout`, and SSH port is `--ssh-port`.
|
`--chunk-serialization`, timeout is `--timeout`, and SSH port is `--ssh-port`.
|
||||||
`-a`/`--archive` is now real rsync archive (`-rlptgoD`).
|
`-a`/`--archive` is now rsync archive `-rlptgoD` (owner/group implied, but the
|
||||||
|
receiver still needs privilege to apply them).
|
||||||
|
|
||||||
`--secluded-args` (and its short form `-s`) is accepted as a compatibility
|
`--secluded-args` (and its short form `-s`) is accepted as a compatibility
|
||||||
no-op. It does not change FastSync's transport or protocol behavior, because
|
no-op. It does not change FastSync's transport or protocol behavior, because
|
||||||
@@ -433,42 +561,103 @@ remote SSH argv is already built injection-safe.
|
|||||||
|
|
||||||
| Option | Description |
|
| Option | Description |
|
||||||
|---|---|
|
|---|---|
|
||||||
| `-a`, `--archive` | rsync archive mode (`-rlptgoD`): links, metadata, devices and specials. |
|
| `-a`, `--archive` | rsync archive mode (`-rlptgoD`): links, perms, times, owner, group, devices and specials; ownership application stays privilege-gated. |
|
||||||
| `-n`, `--dry-run` | Scan and report without writing files. |
|
| `-n`, `--dry-run` | Report what would be transferred without mutating the destination. Since protocol 2.21.0 a server-routed target contacts the receiver and reports would-transfer based on receiver state; a plain local destination keeps the client-side scan. Never mutates or deletes. |
|
||||||
| `--delete` | Request removal of destination entries absent from the source. The server must allow deletion. Default timing is delete-after: extras are removed only after the whole transfer succeeded. |
|
| `--remove-source-files` | Remove regular source files after a successful transfer. |
|
||||||
|
| `--incremental` | Skip files matching destination size and mtime. Auto-enables `--preserve`. Incompatible with `--chunk-serialization`. |
|
||||||
|
| `-c, --checksum` | Verify content by checksum (implies the incremental quick-check). Algorithm selectable with `--checksum-choice`. |
|
||||||
|
| `--checksum-choice <alg>` | Whole-file checksum algorithm: `xxh64`/`xxhash` (default), `xxh3`, `xxh128`, `md5`, or `auto`. |
|
||||||
|
| `--checksum-seed <n>` | Seed for the whole-file xxHash digest; an unset/`0` seed is randomized per transfer, matching rsync. |
|
||||||
|
| `--size-only` | Skip incremental files matching in size, ignoring mtime. |
|
||||||
|
| `-I, --ignore-times` | Transfer files even when size and mtime match. |
|
||||||
|
| `-u, --update` | Skip files newer than the source on the receiver. |
|
||||||
|
| `--ignore-existing` | Skip files that already exist on the receiver; like rsync it does not apply to directories or symlinks. |
|
||||||
|
| `-@, --modify-window <sec>` | Modification-time tolerance (seconds) for the incremental/basis quick-check; `0` requires an exact mtime match. |
|
||||||
|
| `-W, --whole-file` | Transfer changed files without delta processing (`--no-whole-file` clears it). |
|
||||||
|
| `-B <n>, --block-size <n>` | Delta block size in bytes (alias `--delta-block`). |
|
||||||
|
| `-d, --dirs` | Transfer the named directory entries without recursing into their contents (aliases `--old-dirs`/`--old-d`). |
|
||||||
|
| `-R, --relative` | Use rsync's relative path semantics (including the `/./` cut); with `--files-from`, preserve each listed entry's relative path below the destination root. |
|
||||||
|
| `--files-from <file>` | Read the source file list from FILE (paths relative to the source root). |
|
||||||
|
| `-0, --from0` | Treat entries in `--files-from` files as NUL-delimited instead of newline-delimited. |
|
||||||
|
| `--delay-updates` | Put updated files into place only at the end of the transfer (the fixed `.fastsync-stage` staging name diverges from rsync; see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)). |
|
||||||
|
| `--compare-dest <dir>` | Extra comparison basis: unchanged files are not transferred (requires/implies `--incremental`; a basis MISS above the 256 MiB whole-file payload bound is refused — see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)). |
|
||||||
|
| `--copy-dest <dir>` | Like `--compare-dest`, but copies the unchanged file from DIR into the destination (same basis-size caveat; see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)). |
|
||||||
|
| `--link-dest <dir>` | Like `--copy-dest`, but hard-links the unchanged file from DIR (repeatable; earlier DIRs win; same basis-size caveat; see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)). |
|
||||||
|
| `--verify-basis` | FastSync-only: require a basis hit to match the source by whole-file digest instead of trusting the size+mtime quick-check (default matches rsync). |
|
||||||
|
| `--preallocate` | Allocate destination file space up front (fail-fast on a full disk). |
|
||||||
|
| `--append` | Resume a shorter destination by appending only its tail (prefix not verified; requires `--incremental`). |
|
||||||
|
| `--append-verify` | Like `--append`, but verifies the retained prefix checksum first (falls back to a full transfer on mismatch). |
|
||||||
|
| `--delete` | Request removal of destination entries absent from the source. The server must allow deletion. Default timing is delete-during (matching rsync's `--del`): extras are removed per directory as the transfer proceeds, so destination space is freed progressively. Scoped to the synchronized directories, so `--files-from` subsets are safe. |
|
||||||
| `--delete-before` | Delete extras before the transfer starts (implies `--delete`). |
|
| `--delete-before` | Delete extras before the transfer starts (implies `--delete`). |
|
||||||
| `--delete-during`, `--del` | Delete extras once the keep-set manifest is known, before data is applied (implies `--delete`; early mode, same engine behaviour as `--delete-before`). |
|
| `--delete-during`, `--del` | Delete each directory's extras as that directory is processed (implies `--delete`). Since protocol 2.24.0 the sender streams a per-directory `STATUS_DELETE_PLAN` frame as it reaches each source directory; this is also the default timing of a plain `--delete`. |
|
||||||
| `--delete-delay` | Delete extras only after a successful transfer (implies `--delete`; commit mode, same behaviour as `--delete-after`). |
|
| `--delete-delay` | Record extras per directory during the scan but remove them only after a successful transfer (implies `--delete`). Uses the same per-directory `STATUS_DELETE_PLAN` frames as `--delete-during`, applied late. |
|
||||||
|
| `--delete-commit` | FastSync-only: atomic delete-after timing (only after the whole transfer succeeded). |
|
||||||
| `--delete-after` | Explicit delete-after timing: delete only after the transfer succeeded (implies `--delete`). |
|
| `--delete-after` | Explicit delete-after timing: delete only after the transfer succeeded (implies `--delete`). |
|
||||||
|
| `--delete-excluded` | Also delete filter-excluded destination mirrors (size-pruned mirrors stay protected). |
|
||||||
|
| `--max-delete <n>` | Delete at most n destination entries; the rest are skipped and the run exits 25 (partial), matching rsync. |
|
||||||
|
| `--force` | Allow an incoming file/symlink to replace a destination directory (also during `--delay-updates` publication). |
|
||||||
| `--exclude <pattern>` | Exclude matching paths. Repeatable. |
|
| `--exclude <pattern>` | Exclude matching paths. Repeatable. |
|
||||||
| `--include <pattern>` | Include matching paths. Repeatable. |
|
| `--include <pattern>` | Include matching paths. Repeatable. |
|
||||||
| `--exclude-from <file>` | Read exclude patterns from a file. |
|
| `--exclude-from <file>` | Read exclude patterns from a file. |
|
||||||
| `--include-from <file>` | Read include patterns from a file. |
|
| `--include-from <file>` | Read include patterns from a file. |
|
||||||
|
| `-f, --filter=RULE` | Add an rsync-style filter rule (`+`/`-`, `include`/`exclude`, `merge`/`.`, `dir-merge`/`:`, `hide`/`H`, `show`/`S`, `protect`/`P`, `risk`/`R`, `clear`/`!`, and modifiers; repeatable). |
|
||||||
| `--max-size <bytes>` | Skip files larger than the limit. |
|
| `--max-size <bytes>` | Skip files larger than the limit. |
|
||||||
| `--min-size <bytes>` | Skip files smaller than the limit. |
|
| `--min-size <bytes>` | Skip files smaller than the limit. |
|
||||||
| `--max-depth <n>` | Limit recursive scanning depth;
|
| `--max-alloc <SIZE>` | Maximum single allocation (binary units; default 1G; `0` = no local limit). |
|
||||||
zero means unlimited.| | `--incremental` | Skip files matching destination size and mtime.|
|
| `--max-depth <n>` | Limit recursive scanning depth; zero means unlimited. |
|
||||||
| `--checksum` | Include xxHash64 content checks in incremental comparisons.| | `--backup` |
|
| `-b, --backup` | Back up overwritten files. |
|
||||||
Back up overwritten files.| | `--backup - dir<dir>` | Store backups under a separate directory.|
|
| `-T, --temp-dir <dir>` | Scratch directory for temp files before the atomic install (confined to the receive root: relative resolves below it, absolute must canonicalize inside it; `EXDEV` falls back to a non-atomic copy). |
|
||||||
| `--suffix<suffix>` | Set the backup filename suffix.| | `--partial` |
|
| `--backup-dir <dir>` | Store backups under a separate directory (requires `--backup`). |
|
||||||
Select partial - transfer handling. On failed/interrupted writes the
|
| `--suffix <suffix>` | Set the backup filename suffix (default: `~`). |
|
||||||
already-written temp file is retained (best-effort) for resumption.|
|
| `--partial` | Select partial-transfer handling. On failed/interrupted writes the already-written temp file is retained (best-effort) for resumption. With `--partial --partial-dir <dir>`, completed files are written under the partial directory and installed atomically. |
|
||||||
With `--partial --partial-dir <dir>`, completed files are written under the
|
| `--partial-dir <dir>` | Set a relative partial-transfer directory below the server destination root. Implies `--partial`. Rejected together with `--inplace` (`--inplace cannot be used with --partial-dir`, matching rsync), because the inplace path bypasses partial/temp staging. |
|
||||||
partial directory and installed atomically. | | `--partial - dir<dir>` |
|
| `--inplace` | Write directly to the destination instead of using a temporary file. Cannot be combined with `--partial-dir`. |
|
||||||
Set a relative partial - transfer directory below the server destination root.
|
| `--fsync` | Fsync every written file before publication. |
|
||||||
Use with `--partial`. |
|
| `--write-batch=FILE` | Run the normal live transfer and also emit a self-contained batch file of the source tree (FastSync-native format, not rsync-interoperable). |
|
||||||
| `--inplace` | Write directly to the destination instead of using a temporary file. |
|
| `--only-write-batch=FILE` | Emit the batch file only (no destination, no server); FastSync-native format, not rsync-interoperable. |
|
||||||
|
| `--read-batch=FILE` | Apply a batch file to the destination (no source, no server); FastSync-native format, not rsync-interoperable. |
|
||||||
|
| `--stop-after=MINS` | Stop the transfer after MINS minutes; whatever was already transferred is kept. |
|
||||||
|
| `--stop-at=TIME` | Stop at an absolute time. Accepts rsync's `parse_time` forms (`Y-M-DTh:m`, `Y/M/DTh:m`, `Y-M-D`, `M-D`, `D`, `h:m`, `:m`, `T h:m`; omitted fields resolve to the next matching point in the local timezone), plus `now+N[smhd]` and FastSync's `HH:MM`/`HH:MM:SS` clock-time spelling. An early stop skips the late `--delete` keep-set. |
|
||||||
|
|
||||||
### Metadata and links
|
### Metadata and links
|
||||||
|
|
||||||
| Option | Description |
|
| Option | Description |
|
||||||
|---|---|
|
|---|---|
|
||||||
| `--preserve` | Preserve supported file metadata, currently mode and modification time (long form only). |
|
| `--preserve` | Preserve mode and mtime (long form only; equivalent to `-p` + `-t`). Add `-o`/`-g` for owner/group, `-U`/`--atimes` for atime, or an identity flag (`--chown`/`--usermap`/`--groupmap`/`--numeric-ids`/`--copy-as`) for mapped ownership. |
|
||||||
| `-l`, `--links` | Request symlink preservation;
|
| `-U`, `--atimes` | Preserve access times. Captured with the metadata payload; does not enable ownership. |
|
||||||
link-target transfer remains incomplete. |
|
| `-N`, `--crtimes` | Capture birth time and transmit it; it cannot be applied because no portable filesystem call can set a birth time (documented divergence). |
|
||||||
| `--copy-links` | Copy symlink referents. |
|
| `-p`, `--perms` | Preserve permission bits. One of the four per-attribute preserve flags (with `-t`/`-o`/`-g`); under `-p` the source mode is copied exactly (group/other-write included; setuid/setgid/sticky included only when super-user activities are permitted, masked under `SUPER_MODE_OFF`/`--no-super`), matching rsync otherwise. |
|
||||||
| `--safe-links` | Skip symlinks that point outside the transfer tree. |
|
| `-t`, `--times` | Preserve modification times. Independent of the other attributes; `-O`/`--omit-dir-times` suppresses directories only. |
|
||||||
|
| `-O`, `--omit-dir-times` | Do not apply modification times to directories. |
|
||||||
|
| `-J`, `--omit-link-times` | Do not apply times to symlinks. |
|
||||||
|
| `--open-noatime` | Open source files with `O_NOATIME` so reading for a transfer does not update their access time (client-only). |
|
||||||
|
| `-o`, `--owner` | Preserve the source owner (uid). Mapped by name on the receiver with a raw-numeric fallback (only numeric ids cross the wire); application is privilege-gated. |
|
||||||
|
| `-g`, `--group` | Preserve the source group (gid). Same name-mapping/numeric-fallback and privilege gating as `-o`. |
|
||||||
|
| `--no-perms`, `--no-times`, `--no-owner`, `--no-group` | Negate each per-attribute flag (also `--no-p`/`--no-t`/`--no-o`/`--no-g`); `--no-preserve` clears all four. |
|
||||||
|
| `-E`, `--executability` | Preserve executable permission bits. |
|
||||||
|
| `-X`, `--xattrs` | Preserve user `user.*` extended attributes. |
|
||||||
|
| `-A`, `--acls` | Preserve POSIX ACLs. |
|
||||||
|
| `--chmod <changes>` | Modify transferred permissions (rsync syntax, including `D`/`F`/`X` selectors and `s`/`t`); does not imply `-p`. |
|
||||||
|
| `--chown=USER:GROUP` | Override the ownership of transferred files (`USER:GROUP`, `USER`, or `:GROUP`); conflicts with `--usermap`/`--groupmap` on the same side. |
|
||||||
|
| `--usermap=MAP` | Map usernames when applying ownership (`FROM:TO` rules; names, ids, `LOW-HIGH` ranges, `*`, empty-`FROM`). |
|
||||||
|
| `--groupmap=MAP` | Map group names when applying ownership (same syntax as `--usermap`). |
|
||||||
|
| `--numeric-ids` | Mapping modifier: apply the source numeric uid/gid directly instead of mapping by name (combine with `-o`/`-g`, `-a`, or a map). |
|
||||||
|
| `--copy-as=USER[:GROUP]` | Force every written entry to USER[:GROUP]; requires a privileged receiver. |
|
||||||
|
| `--fake-super` | Record the resolved owner plus full mode/rdev in rsync's reserved `user.rsync.%stat` xattr (rsync 3.4.1 grammar) and replay the permission bits; never performs a real chown. |
|
||||||
|
| `--super` | Permit the receiver to attempt confined super-user activities (device nodes). |
|
||||||
|
| `--no-super` | Forbid those super-user activities even when the receiver is root. |
|
||||||
|
| `-l`, `--links` | Copy symlinks as symlinks; the target is stored verbatim (absolute and `..`-bearing targets included), matching rsync. |
|
||||||
|
| `-L`, `--copy-links` | Copy symlink referents (a broken referent makes the run exit 23, matching rsync). |
|
||||||
|
| `--safe-links` | Skip symlinks whose target points outside the transfer tree (applied on the sender). |
|
||||||
| `--copy-unsafe-links` | Copy unsafe symlink referents. |
|
| `--copy-unsafe-links` | Copy unsafe symlink referents. |
|
||||||
|
| `--munge-links` | Rewrite stored symlink targets with rsync's `/rsyncd-munged/` marker. |
|
||||||
|
| `-k`, `--copy-dirlinks` | Treat a symlink to a directory as a real directory on the sender. |
|
||||||
|
| `-K`, `--keep-dirlinks` | Follow an existing destination symlink-to-directory (confined to the receive root). |
|
||||||
|
| `-H`, `--hard-links` | Preserve hard-link relationships across the transfer. |
|
||||||
|
| `-D` | Preserve device and special files (implies `--devices --specials`). |
|
||||||
|
| `--devices` | Recreate device nodes on the destination (privileged; skipped without `CAP_MKNOD`). |
|
||||||
|
| `--specials` | Recreate special files: FIFOs and unix sockets. |
|
||||||
|
| `--copy-devices` | Copy a source device's content as an ordinary regular file on the destination (rsync's non-privileged safe mode) instead of recreating the device node. |
|
||||||
| `-S`, `--sparse` | Sparse-file handling: receiver preserves holes (zero runs are written as holes; no wire change). |
|
| `-S`, `--sparse` | Sparse-file handling: receiver preserves holes (zero runs are written as holes; no wire change). |
|
||||||
|
|
||||||
### Output and logging
|
### Output and logging
|
||||||
@@ -476,9 +665,18 @@ link-target transfer remains incomplete. |
|
|||||||
| Option | Description |
|
| Option | Description |
|
||||||
|---|---|
|
|---|---|
|
||||||
| `-v`, `--verbose` | Enable debug logging. |
|
| `-v`, `--verbose` | Enable debug logging. |
|
||||||
| `--progress` | Show live transfer progress. |
|
| `-q`, `--quiet` | Suppress non-error output. |
|
||||||
| `--stats` | Print transfer statistics. |
|
| `--progress` | Show rsync-style per-file progress blocks; the root `./` line is printed whenever progress is active (rsync prints it only when the transfer root is created). |
|
||||||
|
| `--stats` | Print transfer statistics, including the receiver-only counters reported over the wire; `Number of files`/`Number of created files` carry rsync's per-type breakdown (deleted files are a single total). |
|
||||||
|
| `-i, --itemize-changes` | Print an rsync-style per-file change line. |
|
||||||
|
| `--out-format=FORMAT` | Output format for changed files (`%f %n %l %b %c %C %i %M %%`). |
|
||||||
|
| `--list-only` | List source files instead of transferring. |
|
||||||
|
| `--outbuf=MODE` | stdout/stderr buffering: `N` (none/unbuffered), `L` (line-buffered), or `B` (block-buffered, default). |
|
||||||
| `--log-file <path>` | Write log output to a file. |
|
| `--log-file <path>` | Write log output to a file. |
|
||||||
|
| `--log-file-format=FORMAT` | Per-file log-line format (requires `--log-file`). |
|
||||||
|
| `--stderr=MODE` | Route logging to stderr: `errors` or `all`. |
|
||||||
|
| `--msgs2stderr` | Route all messages to stderr (deprecated spelling of `--stderr=all`). |
|
||||||
|
| `--no-msgs2stderr` | Select errors-only stderr (deprecated spelling; the default). |
|
||||||
| `-V`, `--version` | Print the FastSync protocol version. |
|
| `-V`, `--version` | Print the FastSync protocol version. |
|
||||||
| `--help` | Print command usage. |
|
| `--help` | Print command usage. |
|
||||||
|
|
||||||
@@ -487,31 +685,61 @@ link-target transfer remains incomplete. |
|
|||||||
| Option | Description |
|
| Option | Description |
|
||||||
|---|---|
|
|---|---|
|
||||||
| `--ssh-port <port>` | SSH port for the SSH transport (default: 22). Note the short `-p` is now rsync's `--perms`. |
|
| `--ssh-port <port>` | SSH port for the SSH transport (default: 22). Note the short `-p` is now rsync's `--perms`. |
|
||||||
| `--fastsync-server-path <path>` | Remote FastSync server path for SSH mode. |
|
| `-e`, `--rsh <command>` | Remote shell to launch for the SSH transport (default: `ssh`; may include arguments). |
|
||||||
|
| `--fastsync-server-path <path>` | Remote FastSync server path for SSH mode (client-only; never crosses the wire). |
|
||||||
|
| `--rsync-path <path>` | Alias for `--fastsync-server-path`. |
|
||||||
|
| `-M`, `--remote-option=OPT` | Append OPT to the remote server invocation over SSH (repeatable; rejected for daemon/TCP destinations). |
|
||||||
|
| `--trust-sender` | Receiver-local: trust the remote sender's file list and skip path re-validation (does not affect symlink targets). **On the client this flag alone is inert** — it is never sent on the wire; the server must be started with its own `--trust-sender`, or the client must forward it with `-M--trust-sender` (SSH only). |
|
||||||
|
| `--timeout <sec>` | Socket + per-message I/O timeout; default `0` = disabled. |
|
||||||
|
| `--contimeout <sec>` | Connection timeout; default 60; `0` disables. |
|
||||||
| `--source-dir <path>` | Set the source directory explicitly. |
|
| `--source-dir <path>` | Set the source directory explicitly. |
|
||||||
| `--dest-dir <path>` | Set the destination directory explicitly. |
|
| `--dest-dir <path>` | Set the destination directory explicitly. |
|
||||||
| `--save-to-disk` | Enable server-side disk persistence. |
|
| `--save-to-disk` | Enable server-side disk persistence. |
|
||||||
| `--server-host <host>` | TCP server address. |
|
| `--server-host <host>` | TCP server address. |
|
||||||
| `--server-port <port>` | TCP server port. `--port <port>` / `--port=<port>` is an alias. |
|
| `--server-port <port>` | TCP server port. `--port <port>` / `--port=<port>` is an alias. |
|
||||||
| `--tls` | Enable TLS. Requires `--cert` and `--key`. |
|
| `--address <ip>` | Bind the outgoing client socket to this source address. |
|
||||||
|
| `-4`, `--ipv4` | Force IPv4 for destination resolution. |
|
||||||
|
| `-6`, `--ipv6` | Force IPv6 for destination resolution. |
|
||||||
|
| `--sockopts=OPTS` | Comma-separated OPT=VAL socket options applied before connect. |
|
||||||
|
| `--blocking-io` | SSH transport only: leave the socket without read/write timeouts so it blocks naturally (no effect on TCP). |
|
||||||
|
| `--protocol=NUM` | Force the wire protocol version; must equal the current `PROTOCOL_VERSION` (FastSync cannot speak older/virtual wire formats). |
|
||||||
|
| `--old-args` | Accepted for rsync CLI compatibility; no effect (the remote server path is always safely quoted). |
|
||||||
|
| `--iconv=LOCAL[,REMOTE]` | Convert file-name charsets at the wire boundary (`LOCAL` is our names' charset, `REMOTE` the peer's, defaulting to `LOCAL`). |
|
||||||
|
| `--no-iconv` | Disable `--iconv` charset conversion (same as `--iconv=-`). |
|
||||||
|
| `--tls` | Enable TLS. Requires `--cert`, `--key`, and `--ca`. |
|
||||||
| `--cert <path>` | TLS certificate file. |
|
| `--cert <path>` | TLS certificate file. |
|
||||||
| `--key <path>` | TLS private key file. |
|
| `--key <path>` | TLS private key file. |
|
||||||
| `--ca <path>` | CA file for peer verification. |
|
| `--ca <path>` | CA file for peer verification (always required with `--tls`). |
|
||||||
|
|
||||||
## Server Options
|
## Server Options
|
||||||
|
|
||||||
| Option | Description |
|
| Option | Description |
|
||||||
|---|---|
|
|---|---|
|
||||||
| `--stdio` | Serve one SSH connection over standard input/output. |
|
| `--stdio` | Serve one SSH connection over standard input/output. |
|
||||||
| `-p <port>` | TCP listen port. |
|
| `--daemon` | Run as a persistent daemon listener using a module config file; the daemon default port is 873 (unlike `-p`, which defaults to 8080). |
|
||||||
|
| `--config=FILE` | Daemon config file (default: `~/.config/fastsync/fastsyncd.conf`, else `/etc/fastsyncd.conf`). Requires `--daemon`. |
|
||||||
|
| `--dparam=KEY=VALUE` | Override one global config key on the command line. Requires `--daemon`. |
|
||||||
|
| `--no-detach` | Stay in the foreground (default detaches to the background when running `--daemon`). |
|
||||||
|
| `-p, --port <port>` | TCP listen port (default: 8080, range: 1–65535). |
|
||||||
| `--tls` | Enable TLS. |
|
| `--tls` | Enable TLS. |
|
||||||
| `--cert <path>` | TLS certificate file. |
|
| `--cert <path>` | TLS certificate file (PEM). |
|
||||||
| `--key <path>` | TLS private key file. |
|
| `--key <path>` | TLS private key file (PEM). |
|
||||||
| `--ca <path>` | CA file for peer verification. |
|
| `--ca <path>` | CA file for peer verification (PEM). |
|
||||||
| `--destination-root <path>` | Confine received files to this server-side root;
|
| `--client-cn <name>` | TLS client certificate CN; mandatory with `--tls` (the server verifies the client CN). |
|
||||||
defaults to the current directory. |
|
| `--destination-root <path>` | Confine received files to this server-side root; defaults to the current directory. |
|
||||||
|
| `--address <addr>` | Bind the listening socket to this address. |
|
||||||
|
| `-4`, `--ipv4` | Bind an IPv4 socket (default). |
|
||||||
|
| `-6`, `--ipv6` | Bind an IPv6 socket. |
|
||||||
| `--allow-delete` | Permit client delete manifests. Deletion is refused by default. This also gates `--force` (which can recursively replace/remove a destination directory tree). |
|
| `--allow-delete` | Permit client delete manifests. Deletion is refused by default. This also gates `--force` (which can recursively replace/remove a destination directory tree). |
|
||||||
| `--allow-super` | Standalone TCP listener only: keep super-user activities enabled for a **root** receiver. Without it a root standalone server forces `SUPER_MODE_OFF`, so client `--devices`/`--write-devices`/`--super` and client-chosen ownership requests are skipped/refused. Rejected with `--stdio` (the SSH remote argv is client-composed; use a forced command if the default must hold). No effect when not root. Daemon modules opt in per module with `client owner = yes`. |
|
| `--allow-super` | Standalone TCP listener only: keep super-user activities enabled for a **root** receiver. Without it a root standalone server forces `SUPER_MODE_OFF`, so client `--devices`/`--write-devices`/`--super` and client-chosen ownership requests are skipped/refused. Rejected with `--stdio` (the SSH remote argv is client-composed; use a forced command if the default must hold). No effect when not root. Daemon modules opt in per module with `client owner = yes`. |
|
||||||
|
| `--trust-sender` | Trust the remote sender's file list: skip the receiver's up-front path-traversal re-validation (fewer checks, faster, potentially unsafe; off by default). It does not affect symlink targets, which are stored verbatim either way. A client `--trust-sender` is never sent over the wire — the server must set this flag itself, or the client must forward it via `-M--trust-sender`. |
|
||||||
|
| `--no-super` | Operator veto: never attempt super-user activities (ownership, device nodes) even as root, and refuse any client `--copy-as`/`--super` request. |
|
||||||
|
| `--allow-unauthenticated` | Permit plaintext/anonymous network clients; an auth-required module still accepts only opted-in loopback plaintext. |
|
||||||
|
| `--iconv=LOCAL[,REMOTE]` | Declare this server's LOCAL charset for file-name conversion. |
|
||||||
|
| `--password-file=FILE` | Credential store for modules that declare `auth users`. Requires `--daemon`. FastSync-native SCRAM/PBKDF2 format, not rsync-interoperable. |
|
||||||
|
| `--early-input=FILE` | Second credential store layered over `--password-file`. Requires `--daemon`. FastSync-native format, not rsync-interoperable. |
|
||||||
|
| `--hash-credentials <file>` | Read `<file>`'s `user:password` lines and print PBKDF2 credential-store lines to stdout, then exit. Cannot be combined with `--daemon` or `--stdio`. FastSync-native, not rsync-interoperable. |
|
||||||
|
| `--iterations N` | PBKDF2 iteration count for `--hash-credentials` (default 600000, range 100000–10000000). Requires `--hash-credentials`. FastSync-native, not rsync-interoperable. |
|
||||||
| `-v`, `--verbose` | Enable debug logging. |
|
| `-v`, `--verbose` | Enable debug logging. |
|
||||||
| `--help` | Print server usage. |
|
| `--help` | Print server usage. |
|
||||||
|
|
||||||
@@ -540,8 +768,17 @@ and `address`, the global section accepts:
|
|||||||
- `hosts allow` / `hosts deny` — comma- and/or whitespace-separated host access
|
- `hosts allow` / `hosts deny` — comma- and/or whitespace-separated host access
|
||||||
patterns.
|
patterns.
|
||||||
|
|
||||||
A `[module]` may also set `max connections` (0 = unlimited; enforced per module
|
A `[module]` requires `path`, and may also set `read only`, `write only`,
|
||||||
across all connection children) and its own `hosts allow`/`hosts deny`.
|
`client owner`, `auth users`, `max connections` (0 = unlimited; enforced per
|
||||||
|
module across all connection children), and its own `hosts allow`/`hosts deny`.
|
||||||
|
|
||||||
|
Like rsync, a module is **read-only by default**: a bare `[module]` with only a
|
||||||
|
`path` refuses a write transfer. Opt a module into writability explicitly with
|
||||||
|
`read only = no` or `write only = yes`; a global `read only` value in the
|
||||||
|
section before the first `[module]` sets the default for later modules, and a
|
||||||
|
module's own `read only`/`write only = yes` always wins over it. An
|
||||||
|
rsync-style `write only = yes` is mapped to writability because FastSync is
|
||||||
|
push-only (a module can never be read from the network).
|
||||||
|
|
||||||
The per-host cap and the shared auth lockout identify a source by its numeric
|
The per-host cap and the shared auth lockout identify a source by its numeric
|
||||||
peer IP. **Loopback peers (127.0.0.0/8, IPv6 `::1`) are exempt**: every local
|
peer IP. **Loopback peers (127.0.0.0/8, IPv6 `::1`) are exempt**: every local
|
||||||
@@ -595,7 +832,7 @@ before the module list, before authentication, and the connecting peer address
|
|||||||
|
|
||||||
## Protocol and Security
|
## Protocol and Security
|
||||||
|
|
||||||
FastSync protocol version `2.21.0` is shared by the client and server. The
|
FastSync protocol version `2.29.0` is shared by the client and server. The
|
||||||
current protocol is sender-driven and includes configuration negotiation,
|
current protocol is sender-driven and includes configuration negotiation,
|
||||||
including the maximum allocation limit, incremental checks, checksums,
|
including the maximum allocation limit, incremental checks, checksums,
|
||||||
manifests, keep-alives, abort handling, per-file remove-source results, and
|
manifests, keep-alives, abort handling, per-file remove-source results, and
|
||||||
@@ -647,10 +884,9 @@ mandates `--client-cn`, so a TLS connection to an auth-required module always
|
|||||||
has its client CN verified (`--client-cn` matches the certificate's CN only, not
|
has its client CN verified (`--client-cn` matches the certificate's CN only, not
|
||||||
a subjectAltName, which is acceptable for a private CA).
|
a subjectAltName, which is acceptable for a private CA).
|
||||||
|
|
||||||
TLS provides encrypted TCP transport. Supplying `--ca` enables certificate
|
TLS provides encrypted TCP transport. Both the client and the server require
|
||||||
verification; without it, traffic is encrypted but peer identity is not
|
`--ca` together with `--tls`, so peer certificates are always verified
|
||||||
verified. Use certificate verification for deployments where authentication
|
(`SSL_VERIFY_PEER`, depth 4). The default TCP transport is not encrypted.
|
||||||
matters. The default TCP transport is not encrypted.
|
|
||||||
|
|
||||||
The receiver protects its destination root with path validation, `openat()`
|
The receiver protects its destination root with path validation, `openat()`
|
||||||
directory traversal, `O_NOFOLLOW`, temporary files, and atomic renames. Delete
|
directory traversal, `O_NOFOLLOW`, temporary files, and atomic renames. Delete
|
||||||
@@ -661,18 +897,29 @@ operations require the server's explicit `--allow-delete` policy.
|
|||||||
The project will reach the drop-in replacement goal in stages:
|
The project will reach the drop-in replacement goal in stages:
|
||||||
|
|
||||||
1. Correct rsync option meanings, including short options, combined options,
|
1. Correct rsync option meanings, including short options, combined options,
|
||||||
and `--option=value` syntax.
|
and `--option=value` syntax — **done** in the rsync-parity wave: `-r`/`-b`/
|
||||||
|
`-L`/`-B`, short-option clustering (`-av`, `-aAX`, `-rlpt`), and attached
|
||||||
|
values (`-B1000`, `-essh`, `-MOPT`) all parse.
|
||||||
2. Add differential tests that compare FastSync and rsync contents, metadata,
|
2. Add differential tests that compare FastSync and rsync contents, metadata,
|
||||||
links, deletes, filters, dry runs, and exit codes.
|
links, deletes, filters, dry runs, and exit codes — **done** for the
|
||||||
3. Make `-a` implement the expected recursive, links, permissions, times,
|
completion wave's scope; the tests live in `tests/integration/` and skip
|
||||||
owner/group, and supported special-file behavior.
|
cleanly when rsync is unavailable.
|
||||||
4. Complete symlink, sparse-file, metadata, delete-policy, and resumable-write
|
3. `-a` implements full rsync `-rlptgoD`; under `-p` the source mode is copied
|
||||||
semantics.
|
exactly, including group/other-write bits, with setuid/setgid/sticky copied
|
||||||
|
only when super-user activities are permitted (masked under
|
||||||
|
`SUPER_MODE_OFF`/`--no-super`). Ownership application stays privilege-gated,
|
||||||
|
as in rsync.
|
||||||
|
4. Symlink (verbatim storage), sparse-file, metadata, delete-policy (including
|
||||||
|
`--max-delete` partial + exit 25, per-directory `--delete-during`/
|
||||||
|
`--delete-delay`), codecs, and resumable-write semantics are implemented;
|
||||||
|
remaining work is the documented edge cases, which the **Parity Completion
|
||||||
|
Wave** section of `RSYNC_COMPAT.md` enumerates honestly.
|
||||||
5. Add rsync remote-shell and daemon protocol interoperability.
|
5. Add rsync remote-shell and daemon protocol interoperability.
|
||||||
6. Keep FastSync performance options as negotiated, optional extensions.
|
6. Keep FastSync performance options as negotiated, optional extensions.
|
||||||
|
|
||||||
The exhaustive implementation matrix and compatibility notes are in
|
The exhaustive implementation matrix and compatibility notes are in
|
||||||
[`RSYNC_COMPAT.md`](RSYNC_COMPAT.md).
|
[`RSYNC_COMPAT.md`](RSYNC_COMPAT.md); each row is classified as parity, caveat,
|
||||||
|
or divergent.
|
||||||
|
|
||||||
## Testing
|
## Testing
|
||||||
|
|
||||||
@@ -709,10 +956,13 @@ rsync protocol or filesystem-semantic compatibility.
|
|||||||
|
|
||||||
## Performance Guidance
|
## Performance Guidance
|
||||||
|
|
||||||
- Use `-m` for workloads with many files or enough CPU parallelism.
|
- Use `-j`/`--threads` for workloads with many files or enough CPU parallelism
|
||||||
- Use `-c` or `-z` when network bandwidth is more constrained than CPU.
|
(`-m` is `--prune-empty-dirs`).
|
||||||
|
- Use `-z` when network bandwidth is more constrained than CPU (`-c` is
|
||||||
|
`--checksum`, not a bandwidth option).
|
||||||
- Tune `--chunk-size` for file sizes, memory limits, and network latency.
|
- Tune `--chunk-size` for file sizes, memory limits, and network latency.
|
||||||
- Use `-f` for large uncompressed TCP transfers where zero-copy I/O helps.
|
- Use `--sendfile` for large uncompressed TCP transfers where zero-copy I/O
|
||||||
|
helps (`-f` is `--filter`).
|
||||||
- Use `--incremental` to avoid retransmitting unchanged files.
|
- Use `--incremental` to avoid retransmitting unchanged files.
|
||||||
- Use `--delta` for changed files when both endpoints are FastSync peers.
|
- Use `--delta` for changed files when both endpoints are FastSync peers.
|
||||||
- Use `--bwlimit` when sharing a link with other traffic.
|
- Use `--bwlimit` when sharing a link with other traffic.
|
||||||
|
|||||||
+717
-281
File diff suppressed because one or more lines are too long
@@ -8,3 +8,6 @@ markers =
|
|||||||
daemon_detach: real double-fork backgrounding path (--daemon without
|
daemon_detach: real double-fork backgrounding path (--daemon without
|
||||||
--no-detach); slower/fragile, so it runs in the full suite but not the
|
--no-detach); slower/fragile, so it runs in the full suite but not the
|
||||||
fast PR gate
|
fast PR gate
|
||||||
|
parity: differential rsync-parity case (full set; runs on push to
|
||||||
|
dev/main)
|
||||||
|
parity_ci: fast differential rsync-parity subset (runs on the PR gate)
|
||||||
|
|||||||
@@ -38,6 +38,8 @@ pkgs.mkShell {
|
|||||||
|
|
||||||
buildInputs = with pkgs; [
|
buildInputs = with pkgs; [
|
||||||
zstd
|
zstd
|
||||||
|
zlib
|
||||||
|
lz4
|
||||||
openssl
|
openssl
|
||||||
];
|
];
|
||||||
|
|
||||||
@@ -54,6 +56,6 @@ pkgs.mkShell {
|
|||||||
echo "FastSync dev shell ready."
|
echo "FastSync dev shell ready."
|
||||||
echo " Build: cmake -B build -S . && cmake --build build -j\$(nproc)"
|
echo " Build: cmake -B build -S . && cmake --build build -j\$(nproc)"
|
||||||
echo " Unit: ./build/tests"
|
echo " Unit: ./build/tests"
|
||||||
echo " CI parity: docker run --rm --user \"\$(id -u):\$(id -g)\" -v \"\$PWD:/workspace\" -w /workspace gitea.tap-tap.win/taptap/fastsync-ci:v10 ..."
|
echo " CI parity: docker run --rm --user \"\$(id -u):\$(id -g)\" -v \"\$PWD:/workspace\" -w /workspace gitea.tap-tap.win/taptap/fastsync-ci:v11 ..."
|
||||||
'';
|
'';
|
||||||
}
|
}
|
||||||
|
|||||||
+564
-157
@@ -1,5 +1,8 @@
|
|||||||
#include "change_list.h"
|
#include "change_list.h"
|
||||||
|
#include "checksum.h"
|
||||||
|
#include "log.h"
|
||||||
#include "utils.h"
|
#include "utils.h"
|
||||||
|
#include <fcntl.h>
|
||||||
#include <limits.h>
|
#include <limits.h>
|
||||||
#include <stdint.h>
|
#include <stdint.h>
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
@@ -7,21 +10,7 @@
|
|||||||
#include <string.h>
|
#include <string.h>
|
||||||
#include <sys/stat.h>
|
#include <sys/stat.h>
|
||||||
#include <time.h>
|
#include <time.h>
|
||||||
|
#include <unistd.h>
|
||||||
/* Itemize code emitted for a transferred regular file.
|
|
||||||
*
|
|
||||||
* Layout (rsync-compatible 11-char item): `>f` marks a regular file that was
|
|
||||||
* transferred to the remote host; the trailing nine markers are, in order,
|
|
||||||
* c(hecksum) s(ize) t(ime) p(erms) o(wner) g(roup) u(ser/acl) a(ttrs) x(attrs).
|
|
||||||
* Every marker is `+` (FastSync does not compare each attribute on the
|
|
||||||
* receiving side, so a sent file is reported as fully updated). Files that
|
|
||||||
* are already up to date print no line at all, matching rsync's single -i
|
|
||||||
* which only itemizes changes.
|
|
||||||
*
|
|
||||||
* Because the scanner only yields regular-file transfer candidates, `>d`
|
|
||||||
* (directory) lines are never produced; directories are not transferred as
|
|
||||||
* items by FastSync. */
|
|
||||||
#define ITEMIZE_SENT_FILE ">f+++++++++"
|
|
||||||
|
|
||||||
typedef struct {
|
typedef struct {
|
||||||
char* data;
|
char* data;
|
||||||
@@ -80,103 +69,23 @@ static bool strbuf_append(StrBuf* buf, const char* text) {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
static bool strbuf_append_ull(StrBuf* buf, unsigned long long value) {
|
|
||||||
char digits[32];
|
|
||||||
int written = snprintf(digits, sizeof(digits), "%llu", value);
|
|
||||||
if (written < 0 || (size_t)written >= sizeof(digits))
|
|
||||||
return false;
|
|
||||||
return strbuf_append(buf, digits);
|
|
||||||
}
|
|
||||||
|
|
||||||
static bool strbuf_append_longlong(StrBuf* buf, long long value) {
|
|
||||||
char digits[32];
|
|
||||||
int written = snprintf(digits, sizeof(digits), "%lld", value);
|
|
||||||
if (written < 0 || (size_t)written >= sizeof(digits))
|
|
||||||
return false;
|
|
||||||
return strbuf_append(buf, digits);
|
|
||||||
}
|
|
||||||
|
|
||||||
bool change_list_enabled(const Config* config) {
|
bool change_list_enabled(const Config* config) {
|
||||||
return config != NULL && (config->itemize_changes || config->out_format != NULL ||
|
return config != NULL && (config->itemize_changes || config->out_format != NULL ||
|
||||||
(config->log_file != NULL && config->log_file_format != NULL));
|
(config->log_file != NULL && config->log_file_format != NULL) ||
|
||||||
|
(config->info_level & LOG_INFO_NAME) != 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
char* change_render_itemize(const ChangeEvent* event) {
|
/* Emitted once, lazily, ahead of the first --info=name entry: rsync prints the
|
||||||
if (event == NULL || event->decision != CHANGE_SENT)
|
* transfer-root `./` name line when the root directory is (re)created. */
|
||||||
return str_dup("");
|
static bool name_root_printed = false;
|
||||||
const char* code = event->is_directory ? ">d+++++++++" : ITEMIZE_SENT_FILE;
|
|
||||||
StrBuf line = {0};
|
void change_reset_name_root(void) {
|
||||||
bool ok = strbuf_append(&line, code) && strbuf_append(&line, " ") &&
|
name_root_printed = false;
|
||||||
strbuf_append(&line, event->path != NULL ? event->path : "");
|
|
||||||
if (!ok) {
|
|
||||||
strbuf_free(&line);
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
return line.data;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
static const char* leaf_name(const char* path) {
|
/* ---- Itemize code ---- */
|
||||||
if (path == NULL)
|
|
||||||
return "";
|
|
||||||
const char* slash = strrchr(path, '/');
|
|
||||||
return slash != NULL && slash[1] != '\0' ? slash + 1 : path;
|
|
||||||
}
|
|
||||||
|
|
||||||
char* change_render_format(const char* format, const ChangeEvent* event) {
|
/* Format the permission bits as an `ls -l` string, e.g. `-rw-r--r--`. */
|
||||||
if (format == NULL)
|
|
||||||
return NULL;
|
|
||||||
StrBuf line = {0};
|
|
||||||
bool ok = true;
|
|
||||||
for (const char* p = format; *p != '\0' && ok;) {
|
|
||||||
if (*p != '%') {
|
|
||||||
ok = strbuf_append_char(&line, *p);
|
|
||||||
p++;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
char token = p[1];
|
|
||||||
if (token == '\0') {
|
|
||||||
ok = strbuf_append_char(&line, '%');
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
switch (token) {
|
|
||||||
case '%':
|
|
||||||
ok = strbuf_append_char(&line, '%');
|
|
||||||
break;
|
|
||||||
case 'f':
|
|
||||||
ok = strbuf_append(&line, event->path != NULL ? event->path : "");
|
|
||||||
break;
|
|
||||||
case 'n':
|
|
||||||
ok = strbuf_append(&line, leaf_name(event->path));
|
|
||||||
break;
|
|
||||||
case 'l':
|
|
||||||
ok = strbuf_append_ull(&line, event->size);
|
|
||||||
break;
|
|
||||||
case 'b':
|
|
||||||
ok = strbuf_append_ull(&line, event->bytes_sent);
|
|
||||||
break;
|
|
||||||
case 'M':
|
|
||||||
ok = strbuf_append_longlong(&line, (long long)event->mtime_sec);
|
|
||||||
break;
|
|
||||||
default:
|
|
||||||
/* Unknown escape sequences are preserved verbatim. */
|
|
||||||
ok = strbuf_append_char(&line, '%') && strbuf_append_char(&line, token);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
p += 2;
|
|
||||||
}
|
|
||||||
if (!ok) {
|
|
||||||
strbuf_free(&line);
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
if (line.data == NULL) {
|
|
||||||
line.data = str_dup("");
|
|
||||||
if (!line.data)
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
return line.data;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Format a mode as an `ls -l` permission string, e.g. `-rw-r--r--`. */
|
|
||||||
static void mode_to_ls_string(mode_t mode, char out[11]) {
|
static void mode_to_ls_string(mode_t mode, char out[11]) {
|
||||||
out[0] = S_ISDIR(mode) ? 'd'
|
out[0] = S_ISDIR(mode) ? 'd'
|
||||||
: S_ISLNK(mode) ? 'l'
|
: S_ISLNK(mode) ? 'l'
|
||||||
@@ -198,29 +107,101 @@ static void mode_to_ls_string(mode_t mode, char out[11]) {
|
|||||||
out[10] = '\0';
|
out[10] = '\0';
|
||||||
}
|
}
|
||||||
|
|
||||||
char* change_render_list_line(mode_t mode, unsigned long long size, time_t mtime,
|
static char itemize_type_char(const ChangeEvent* event) {
|
||||||
const char* path) {
|
if (event->is_directory)
|
||||||
char permission[11];
|
return 'd';
|
||||||
mode_to_ls_string(mode, permission);
|
if (event->is_symlink)
|
||||||
char date[32];
|
return 'L';
|
||||||
struct tm broken_down;
|
if (event->is_special) {
|
||||||
if (localtime_r(&mtime, &broken_down) != NULL) {
|
if (S_ISCHR(event->mode) || S_ISBLK(event->mode))
|
||||||
if (strftime(date, sizeof(date), "%Y/%m/%d %H:%M:%S", &broken_down) == 0)
|
return 'D';
|
||||||
snprintf(date, sizeof(date), "?");
|
return 'S';
|
||||||
} else {
|
|
||||||
snprintf(date, sizeof(date), "?");
|
|
||||||
}
|
}
|
||||||
|
return 'f';
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool times_match(const Config* config, const ChangeEvent* event) {
|
||||||
|
if (!event->dest.known || !event->dest.existed)
|
||||||
|
return false;
|
||||||
|
if (event->mtime_sec == event->dest.mtime_sec)
|
||||||
|
return event->mtime_nsec == event->dest.mtime_nsec;
|
||||||
|
long long delta = (long long)event->mtime_sec - (long long)event->dest.mtime_sec;
|
||||||
|
if (delta < 0)
|
||||||
|
delta = -delta;
|
||||||
|
return delta <= (long long)config->modify_window;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Fill the 11-character itemize code (10 chars + NUL). `created` means the
|
||||||
|
* destination entry did not exist, so every attribute marker is `+`. */
|
||||||
|
static void itemize_code(const Config* config, const ChangeEvent* event, char code[12]) {
|
||||||
|
bool known = event->dest.known;
|
||||||
|
bool created = !known || !event->dest.existed;
|
||||||
|
char update;
|
||||||
|
if (event->is_hardlink)
|
||||||
|
update = 'h';
|
||||||
|
else if (created)
|
||||||
|
update = (event->is_directory || event->is_symlink || event->is_special) ? 'c' : '>';
|
||||||
|
else if (event->is_directory)
|
||||||
|
/* rsync: an existing directory that only has attribute changes carries no
|
||||||
|
transfer, so the update column is `.` rather than `>`. */
|
||||||
|
update = '.';
|
||||||
|
else
|
||||||
|
update = '>';
|
||||||
|
code[0] = update;
|
||||||
|
code[1] = itemize_type_char(event);
|
||||||
|
if (created) {
|
||||||
|
for (int i = 0; i < 9; i++)
|
||||||
|
code[2 + i] = '+';
|
||||||
|
code[11] = '\0';
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
bool size_diff = event->size != event->dest.size;
|
||||||
|
bool time_diff = !times_match(config, event);
|
||||||
|
bool perms_diff = (event->mode & 07777) != (event->dest.mode & 07777);
|
||||||
|
bool owner_diff = event->uid != (uid_t)event->dest.uid;
|
||||||
|
bool group_diff = event->gid != (gid_t)event->dest.gid;
|
||||||
|
code[2] = '.'; /* checksum: no destination digest available */
|
||||||
|
code[3] = size_diff ? 's' : '.';
|
||||||
|
code[4] = time_diff ? 't' : '.';
|
||||||
|
code[5] = (config->preserve_perms && perms_diff) ? 'p' : '.';
|
||||||
|
code[6] = (config->preserve_owner && owner_diff) ? 'o' : '.';
|
||||||
|
code[7] = (config->preserve_group && group_diff) ? 'g' : '.';
|
||||||
|
code[8] = '.'; /* reserved */
|
||||||
|
code[9] = '.'; /* acl: not compared */
|
||||||
|
code[10] = '.';
|
||||||
|
code[11] = '\0';
|
||||||
|
}
|
||||||
|
|
||||||
|
/* rsync %n: the transfer-relative name, with a trailing slash for directories.
|
||||||
|
* The transfer root is `.` (so `%n` renders `./`), matching rsync's root entry. */
|
||||||
|
static bool append_name(StrBuf* buf, const ChangeEvent* event) {
|
||||||
|
const char* name = event->name != NULL ? event->name : "";
|
||||||
|
if (event->is_directory && name[0] == '\0')
|
||||||
|
return strbuf_append(buf, "./");
|
||||||
|
if (!strbuf_append(buf, name))
|
||||||
|
return false;
|
||||||
|
if (event->is_directory && name[strlen(name) - 1] != '/')
|
||||||
|
return strbuf_append_char(buf, '/');
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* rsync %L: " -> target" for a symlink, " => target" for a hard link, else "". */
|
||||||
|
static bool append_link_suffix(StrBuf* buf, const ChangeEvent* event) {
|
||||||
|
if (event->is_symlink && event->symlink_target != NULL)
|
||||||
|
return strbuf_append(buf, " -> ") && strbuf_append(buf, event->symlink_target);
|
||||||
|
if (event->is_hardlink && event->hardlink_target != NULL)
|
||||||
|
return strbuf_append(buf, " => ") && strbuf_append(buf, event->hardlink_target);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
char* change_render_itemize(const Config* config, const ChangeEvent* event) {
|
||||||
|
if (event == NULL || event->decision != CHANGE_SENT)
|
||||||
|
return str_dup("");
|
||||||
|
char code[12];
|
||||||
|
itemize_code(config, event, code);
|
||||||
StrBuf line = {0};
|
StrBuf line = {0};
|
||||||
char size_field[32];
|
bool ok = strbuf_append(&line, code) && strbuf_append_char(&line, ' ') &&
|
||||||
int written = snprintf(size_field, sizeof(size_field), "%llu", size);
|
append_name(&line, event) && append_link_suffix(&line, event);
|
||||||
if (written < 0 || (size_t)written >= sizeof(size_field)) {
|
|
||||||
strbuf_free(&line);
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
bool ok = strbuf_append(&line, permission) && strbuf_append_char(&line, ' ') &&
|
|
||||||
strbuf_append(&line, size_field) && strbuf_append_char(&line, ' ') &&
|
|
||||||
strbuf_append(&line, date) && strbuf_append_char(&line, ' ') &&
|
|
||||||
strbuf_append(&line, path != NULL ? path : "");
|
|
||||||
if (!ok) {
|
if (!ok) {
|
||||||
strbuf_free(&line);
|
strbuf_free(&line);
|
||||||
return NULL;
|
return NULL;
|
||||||
@@ -228,6 +209,276 @@ char* change_render_list_line(mode_t mode, unsigned long long size, time_t mtime
|
|||||||
return line.data;
|
return line.data;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* rsync's `--info=name` line for an updated entry: the transfer-relative name
|
||||||
|
* (trailing slash for directories) plus the ` -> target` / ` => target` link
|
||||||
|
* suffix. `--info=name` does not alter an itemize/out-format run. */
|
||||||
|
static char* change_render_name(const ChangeEvent* event) {
|
||||||
|
StrBuf line = {0};
|
||||||
|
bool ok = append_name(&line, event) && append_link_suffix(&line, event);
|
||||||
|
if (!ok) {
|
||||||
|
strbuf_free(&line);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
if (line.data == NULL) {
|
||||||
|
line.data = str_dup("");
|
||||||
|
if (!line.data)
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
return line.data;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* rsync's `--info=name2` line for an unchanged entry: `NAME is uptodate`. */
|
||||||
|
static char* change_render_name_uptodate(const ChangeEvent* event) {
|
||||||
|
char* name = change_render_name(event);
|
||||||
|
if (name == NULL)
|
||||||
|
return NULL;
|
||||||
|
size_t length = strlen(name);
|
||||||
|
char* line = malloc(length + sizeof(" is uptodate"));
|
||||||
|
if (line == NULL) {
|
||||||
|
free(name);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
memcpy(line, name, length);
|
||||||
|
memcpy(line + length, " is uptodate", sizeof(" is uptodate"));
|
||||||
|
free(name);
|
||||||
|
return line;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ---- --out-format / --log-file-format ---- */
|
||||||
|
|
||||||
|
/* rsync 3.4.1's `%C` uses the negotiated TRANSFER checksum (the first name of a
|
||||||
|
* two-name "transfer,pre-transfer" --checksum-choice), not the pre-transfer
|
||||||
|
* whole-file digest FastSync compares against on the wire. The default "auto"
|
||||||
|
* resolves to xxh128, so an explicit selection and the default both render the
|
||||||
|
* selected algorithm's digest. */
|
||||||
|
static ChecksumAlgo out_format_checksum_algo(const Config* config) {
|
||||||
|
return (ChecksumAlgo)config->cli.checksum_transfer_algo;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Render a digest as rsync's sum_as_hex: xxh128 prints the HIGH 64-bit half
|
||||||
|
* before the low half, and xxh64/xxh3 print their 64-bit value big-endian; every
|
||||||
|
* other algorithm prints its bytes in order. */
|
||||||
|
static void digest_to_hex(ChecksumAlgo algo, const uint8_t* digest, size_t len, char* out) {
|
||||||
|
if (algo == CHECKSUM_ALGO_XXH128 && len == 16) {
|
||||||
|
uint64_t low = 0;
|
||||||
|
uint64_t high = 0;
|
||||||
|
memcpy(&low, digest, sizeof(low));
|
||||||
|
memcpy(&high, digest + 8, sizeof(high));
|
||||||
|
snprintf(out, len * 2 + 1, "%016llx%016llx", (unsigned long long)high, (unsigned long long)low);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if ((algo == CHECKSUM_ALGO_XXH64 || algo == CHECKSUM_ALGO_XXH3) && len == 8) {
|
||||||
|
uint64_t value = 0;
|
||||||
|
memcpy(&value, digest, sizeof(value));
|
||||||
|
snprintf(out, len * 2 + 1, "%016llx", (unsigned long long)value);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
static const char hex[] = "0123456789abcdef";
|
||||||
|
for (size_t i = 0; i < len; i++) {
|
||||||
|
out[i * 2] = hex[(digest[i] >> 4) & 0xf];
|
||||||
|
out[i * 2 + 1] = hex[digest[i] & 0xf];
|
||||||
|
}
|
||||||
|
out[len * 2] = '\0';
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool format_uses_checksum(const char* format) {
|
||||||
|
if (format == NULL)
|
||||||
|
return false;
|
||||||
|
for (const char* p = format; *p != '\0';) {
|
||||||
|
if (*p != '%') {
|
||||||
|
p++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
char token = p[1];
|
||||||
|
if (token == '\0')
|
||||||
|
break;
|
||||||
|
if (token == 'C')
|
||||||
|
return true;
|
||||||
|
p += 2;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Fill event->checksum/checksum_known for a transferred regular file. A
|
||||||
|
* non-regular entry (or a hard-link sibling) leaves checksum_known false, which
|
||||||
|
* renders as spaces like rsync. */
|
||||||
|
static void fill_event_checksum(const Config* config, const File* file, ChangeEvent* event) {
|
||||||
|
if (file == NULL || file->is_dir || file->is_symlink || file->is_special ||
|
||||||
|
(file->link_group != 0 && !file->link_first))
|
||||||
|
return;
|
||||||
|
if (!format_uses_checksum(config->out_format) && !format_uses_checksum(config->log_file_format))
|
||||||
|
return;
|
||||||
|
if (file->path == NULL)
|
||||||
|
return;
|
||||||
|
ChecksumAlgo algo = out_format_checksum_algo(config);
|
||||||
|
/* rsync renders `--checksum-choice=none` as a blank 2-character column. */
|
||||||
|
if (algo == CHECKSUM_ALGO_NONE)
|
||||||
|
return;
|
||||||
|
uint8_t digest[CHECKSUM_MAX_DIGEST_LEN];
|
||||||
|
size_t len = 0;
|
||||||
|
/* rsync's %C is the transfer checksum, which is always seeded with 0 (it is
|
||||||
|
* independent of --checksum-seed, as rsync 3.4.1 demonstrates). */
|
||||||
|
if (!checksum_digest_file(algo, 0, file->path, digest, sizeof(digest), &len))
|
||||||
|
return;
|
||||||
|
digest_to_hex(algo, digest, len, event->checksum);
|
||||||
|
event->checksum_known = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
char* change_render_format(const char* format, const Config* config, const ChangeEvent* event) {
|
||||||
|
if (format == NULL || event == NULL)
|
||||||
|
return NULL;
|
||||||
|
StrBuf line = {0};
|
||||||
|
bool ok = true;
|
||||||
|
for (const char* p = format; *p != '\0' && ok;) {
|
||||||
|
if (*p != '%') {
|
||||||
|
ok = strbuf_append_char(&line, *p);
|
||||||
|
p++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
char token = p[1];
|
||||||
|
if (token == '\0') {
|
||||||
|
ok = strbuf_append_char(&line, '%');
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
switch (token) {
|
||||||
|
case '%':
|
||||||
|
ok = strbuf_append_char(&line, '%');
|
||||||
|
break;
|
||||||
|
case 'i': {
|
||||||
|
if (event->deleted) {
|
||||||
|
/* rsync's ITEM_DELETED itemize code: `*deleting ` (11 chars). */
|
||||||
|
ok = strbuf_append(&line, "*deleting ");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
char code[12];
|
||||||
|
itemize_code(config, event, code);
|
||||||
|
ok = strbuf_append(&line, code);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case 'f':
|
||||||
|
ok = strbuf_append(&line, event->path != NULL ? event->path : "");
|
||||||
|
break;
|
||||||
|
case 'n':
|
||||||
|
ok = append_name(&line, event);
|
||||||
|
break;
|
||||||
|
case 'L':
|
||||||
|
ok = append_link_suffix(&line, event);
|
||||||
|
break;
|
||||||
|
case 'l': {
|
||||||
|
char digits[32];
|
||||||
|
int written = snprintf(digits, sizeof(digits), "%llu", event->size);
|
||||||
|
ok = written >= 0 && (size_t)written < sizeof(digits) && strbuf_append(&line, digits);
|
||||||
|
} break;
|
||||||
|
case 'b': {
|
||||||
|
char digits[32];
|
||||||
|
int written = snprintf(digits, sizeof(digits), "%llu", event->bytes_sent);
|
||||||
|
ok = written >= 0 && (size_t)written < sizeof(digits) && strbuf_append(&line, digits);
|
||||||
|
} break;
|
||||||
|
case 'c': {
|
||||||
|
char digits[32];
|
||||||
|
int written = snprintf(digits, sizeof(digits), "%llu", event->bytes_read);
|
||||||
|
ok = written >= 0 && (size_t)written < sizeof(digits) && strbuf_append(&line, digits);
|
||||||
|
} break;
|
||||||
|
case 'C': {
|
||||||
|
if (event->checksum_known) {
|
||||||
|
ok = strbuf_append(&line, event->checksum);
|
||||||
|
} else {
|
||||||
|
/* rsync pads a non-regular / untransferred / `none` entry with spaces;
|
||||||
|
`none` renders as a blank 2-character column. */
|
||||||
|
ChecksumAlgo algo = out_format_checksum_algo(config);
|
||||||
|
int width = algo == CHECKSUM_ALGO_NONE ? 2 : checksum_digest_len(algo) * 2;
|
||||||
|
for (int i = 0; i < width && ok; i++)
|
||||||
|
ok = strbuf_append_char(&line, ' ');
|
||||||
|
}
|
||||||
|
} break;
|
||||||
|
case 'M': {
|
||||||
|
char when[32];
|
||||||
|
if (format_rsync_datetime(event->mtime_sec, true, when, sizeof(when)))
|
||||||
|
ok = strbuf_append(&line, when);
|
||||||
|
} break;
|
||||||
|
case 't': {
|
||||||
|
char when[32];
|
||||||
|
if (format_rsync_datetime(time(NULL), false, when, sizeof(when)))
|
||||||
|
ok = strbuf_append(&line, when);
|
||||||
|
} break;
|
||||||
|
case 'o':
|
||||||
|
ok = strbuf_append(&line, "send");
|
||||||
|
break;
|
||||||
|
case 'p': {
|
||||||
|
char digits[32];
|
||||||
|
int written = snprintf(digits, sizeof(digits), "%ld", (long)getpid());
|
||||||
|
ok = written >= 0 && (size_t)written < sizeof(digits) && strbuf_append(&line, digits);
|
||||||
|
} break;
|
||||||
|
case 'B': {
|
||||||
|
char permission[11];
|
||||||
|
mode_to_ls_string(event->mode, permission);
|
||||||
|
ok = strbuf_append(&line, permission + 1);
|
||||||
|
} break;
|
||||||
|
case 'U': {
|
||||||
|
char digits[32];
|
||||||
|
int written = snprintf(digits, sizeof(digits), "%u", (unsigned)event->uid);
|
||||||
|
ok = written >= 0 && (size_t)written < sizeof(digits) && strbuf_append(&line, digits);
|
||||||
|
} break;
|
||||||
|
case 'G': {
|
||||||
|
char digits[32];
|
||||||
|
int written = snprintf(digits, sizeof(digits), "%u", (unsigned)event->gid);
|
||||||
|
ok = written >= 0 && (size_t)written < sizeof(digits) && strbuf_append(&line, digits);
|
||||||
|
} break;
|
||||||
|
default:
|
||||||
|
/* Unknown escape sequences are preserved verbatim. */
|
||||||
|
ok = strbuf_append_char(&line, '%') && strbuf_append_char(&line, token);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
p += 2;
|
||||||
|
}
|
||||||
|
if (!ok) {
|
||||||
|
strbuf_free(&line);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
if (line.data == NULL) {
|
||||||
|
line.data = str_dup("");
|
||||||
|
if (!line.data)
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
return line.data;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ---- --list-only ---- */
|
||||||
|
|
||||||
|
char* change_render_list_line(const Config* config, const ChangeEvent* event) {
|
||||||
|
(void)config;
|
||||||
|
if (event == NULL)
|
||||||
|
return NULL;
|
||||||
|
char permission[11];
|
||||||
|
mode_to_ls_string(event->mode, permission);
|
||||||
|
char date[32];
|
||||||
|
if (!format_rsync_datetime(event->mtime_sec, false, date, sizeof(date)))
|
||||||
|
snprintf(date, sizeof(date), "?");
|
||||||
|
StrBuf line = {0};
|
||||||
|
char size_field[40];
|
||||||
|
char grouped[32];
|
||||||
|
if (!format_big_num(event->size, false, grouped, sizeof(grouped))) {
|
||||||
|
strbuf_free(&line);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
int written = snprintf(size_field, sizeof(size_field), "%15s", grouped);
|
||||||
|
if (written < 0 || (size_t)written >= sizeof(size_field)) {
|
||||||
|
strbuf_free(&line);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
const char* name = event->name != NULL && event->name[0] != '\0' ? event->name : ".";
|
||||||
|
bool ok = strbuf_append(&line, permission) && strbuf_append(&line, size_field) &&
|
||||||
|
strbuf_append_char(&line, ' ') && strbuf_append(&line, date) &&
|
||||||
|
strbuf_append_char(&line, ' ') && strbuf_append(&line, name);
|
||||||
|
if (!ok) {
|
||||||
|
strbuf_free(&line);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
return line.data;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ---- Event emission ---- */
|
||||||
|
|
||||||
static void print_escaped_line(FILE* stream, const char* line, bool eight_bit_output) {
|
static void print_escaped_line(FILE* stream, const char* line, bool eight_bit_output) {
|
||||||
char* escaped = output_escape(line, eight_bit_output);
|
char* escaped = output_escape(line, eight_bit_output);
|
||||||
if (escaped != NULL) {
|
if (escaped != NULL) {
|
||||||
@@ -242,20 +493,49 @@ static void print_escaped_line(FILE* stream, const char* line, bool eight_bit_ou
|
|||||||
void change_emit(const Config* config, const ChangeEvent* event) {
|
void change_emit(const Config* config, const ChangeEvent* event) {
|
||||||
if (event == NULL || !change_list_enabled(config))
|
if (event == NULL || !change_list_enabled(config))
|
||||||
return;
|
return;
|
||||||
if (event->decision == CHANGE_UP_TO_DATE)
|
|
||||||
return;
|
|
||||||
bool to_stdout = config->itemize_changes || config->out_format != NULL;
|
bool to_stdout = config->itemize_changes || config->out_format != NULL;
|
||||||
bool to_log = config->log_file != NULL && config->log_file_format != NULL;
|
bool to_log = config->log_file != NULL && config->log_file_format != NULL;
|
||||||
|
bool progress_active = config->show_progress || (config->info_level & LOG_INFO_PROGRESS);
|
||||||
|
if (event->decision == CHANGE_UP_TO_DATE) {
|
||||||
|
/* --info=name2 prints `NAME is uptodate` for entries the receiver already
|
||||||
|
had. An itemize/out-format run reports them through its own format (or
|
||||||
|
not at all), the progress stream has no frame for them, and neither the
|
||||||
|
itemize nor the log-file stream previously reported an up-to-date entry,
|
||||||
|
so nothing else here changes. */
|
||||||
|
if (!to_stdout && (config->info_level & LOG_INFO_NAME_UPTODATE) != 0 && !progress_active) {
|
||||||
|
char* line = change_render_name_uptodate(event);
|
||||||
|
if (line != NULL) {
|
||||||
|
print_escaped_line(stdout, line, config->eight_bit_output);
|
||||||
|
free(line);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
if (to_stdout) {
|
if (to_stdout) {
|
||||||
char* line = config->out_format != NULL ? change_render_format(config->out_format, event)
|
char* line = config->out_format != NULL
|
||||||
: change_render_itemize(event);
|
? change_render_format(config->out_format, config, event)
|
||||||
|
: change_render_itemize(config, event);
|
||||||
|
if (line != NULL) {
|
||||||
|
print_escaped_line(stdout, line, config->eight_bit_output);
|
||||||
|
free(line);
|
||||||
|
}
|
||||||
|
} else if ((config->info_level & LOG_INFO_NAME) != 0 && !progress_active) {
|
||||||
|
/* --info=name without -i/--out-format: print the updated entry's name. The
|
||||||
|
--progress path owns the name line when progress output is active (it
|
||||||
|
emits the same names before the progress frames), so do not duplicate.
|
||||||
|
The transfer-root `./` line precedes the first such name. */
|
||||||
|
if (!name_root_printed) {
|
||||||
|
name_root_printed = true;
|
||||||
|
fputs("./\n", stdout);
|
||||||
|
}
|
||||||
|
char* line = change_render_name(event);
|
||||||
if (line != NULL) {
|
if (line != NULL) {
|
||||||
print_escaped_line(stdout, line, config->eight_bit_output);
|
print_escaped_line(stdout, line, config->eight_bit_output);
|
||||||
free(line);
|
free(line);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (to_log) {
|
if (to_log) {
|
||||||
char* line = change_render_format(config->log_file_format, event);
|
char* line = change_render_format(config->log_file_format, config, event);
|
||||||
if (line != NULL) {
|
if (line != NULL) {
|
||||||
print_escaped_line(config->log_file, line, config->eight_bit_output);
|
print_escaped_line(config->log_file, line, config->eight_bit_output);
|
||||||
free(line);
|
free(line);
|
||||||
@@ -266,9 +546,6 @@ void change_emit(const Config* config, const ChangeEvent* event) {
|
|||||||
static bool format_uses_mtime(const char* format) {
|
static bool format_uses_mtime(const char* format) {
|
||||||
if (format == NULL)
|
if (format == NULL)
|
||||||
return false;
|
return false;
|
||||||
/* Mirror change_render_format's tokenizer: "%%" is a literal percent (so
|
|
||||||
* "%%M" does NOT expand %M) and unknown "%X" escapes consume both chars.
|
|
||||||
* This keeps the optional stat() fallback below in step with the renderer. */
|
|
||||||
for (const char* p = format; *p != '\0';) {
|
for (const char* p = format; *p != '\0';) {
|
||||||
if (*p != '%') {
|
if (*p != '%') {
|
||||||
p++;
|
p++;
|
||||||
@@ -284,46 +561,176 @@ static bool format_uses_mtime(const char* format) {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
void change_emit_file_sent(const Config* config, const File* file) {
|
/* Relative path of an entry below the transfer root (no leading slash). Uses
|
||||||
|
* the sender-side send_path override when present (bare-relative -R layout). */
|
||||||
|
static char* relative_name(const Config* config, const File* file) {
|
||||||
|
const char* full = file_wire_path(file);
|
||||||
|
if (file->send_path != NULL)
|
||||||
|
return str_dup(full != NULL ? full : "");
|
||||||
|
const char* root = config->send_directory;
|
||||||
|
if (root == NULL || full == NULL)
|
||||||
|
return str_dup(full != NULL ? full : "");
|
||||||
|
size_t root_len = strlen(root);
|
||||||
|
while (root_len > 1 && root[root_len - 1] == '/')
|
||||||
|
root_len--;
|
||||||
|
if (strncmp(root, full, root_len) == 0) {
|
||||||
|
if (full[root_len] == '\0')
|
||||||
|
return str_dup("");
|
||||||
|
if (full[root_len] == '/')
|
||||||
|
return str_dup(full + root_len + 1);
|
||||||
|
}
|
||||||
|
return str_dup(full);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* rsync %f long form: the source argument as typed (leading '/' removed,
|
||||||
|
* trailing '/' removed, leading "./" removed) joined to the relative name. */
|
||||||
|
static char* display_name(const Config* config, const char* name) {
|
||||||
|
const char* root = config->send_directory;
|
||||||
|
if (root == NULL)
|
||||||
|
return str_dup(name != NULL ? name : "");
|
||||||
|
const char* p = root;
|
||||||
|
while (*p == '/')
|
||||||
|
p++;
|
||||||
|
if (p[0] == '.' && p[1] == '/')
|
||||||
|
p += 2;
|
||||||
|
size_t root_len = strlen(p);
|
||||||
|
while (root_len > 0 && p[root_len - 1] == '/')
|
||||||
|
root_len--;
|
||||||
|
size_t name_len = name != NULL ? strlen(name) : 0;
|
||||||
|
if (root_len == 0 && name_len == 0)
|
||||||
|
return str_dup("");
|
||||||
|
char* out = malloc(root_len + (root_len > 0 && name_len > 0 ? 1 : 0) + name_len + 1);
|
||||||
|
if (!out)
|
||||||
|
return NULL;
|
||||||
|
size_t offset = 0;
|
||||||
|
if (root_len > 0) {
|
||||||
|
memcpy(out, p, root_len);
|
||||||
|
offset = root_len;
|
||||||
|
}
|
||||||
|
if (root_len > 0 && name_len > 0)
|
||||||
|
out[offset++] = '/';
|
||||||
|
if (name_len > 0)
|
||||||
|
memcpy(out + offset, name, name_len);
|
||||||
|
out[offset + name_len] = '\0';
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void fill_event_from_file(const Config* config, const File* file, ChangeEvent* event,
|
||||||
|
char** name_out, char** path_out) {
|
||||||
|
char* name = relative_name(config, file);
|
||||||
|
char* path = display_name(config, name);
|
||||||
|
event->name = name;
|
||||||
|
event->path = path;
|
||||||
|
*name_out = name;
|
||||||
|
*path_out = path;
|
||||||
|
if (file->metadata != NULL) {
|
||||||
|
event->mtime_sec = file->metadata->mtime_sec;
|
||||||
|
event->mtime_nsec = file->metadata->mtime_nsec;
|
||||||
|
event->mode = file->metadata->mode;
|
||||||
|
event->uid = file->metadata->uid;
|
||||||
|
event->gid = file->metadata->gid;
|
||||||
|
} else if (format_uses_mtime(config->out_format) || format_uses_mtime(config->log_file_format)) {
|
||||||
|
struct stat st;
|
||||||
|
if (file->path != NULL && stat(file->path, &st) == 0) {
|
||||||
|
event->mtime_sec = st.st_mtime;
|
||||||
|
event->mtime_nsec = st.st_mtim.tv_nsec;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void change_emit_file_sent_bytes(const Config* config, const File* file,
|
||||||
|
unsigned long long bytes_sent, unsigned long long bytes_read) {
|
||||||
if (file == NULL || !change_list_enabled(config))
|
if (file == NULL || !change_list_enabled(config))
|
||||||
return;
|
return;
|
||||||
ChangeEvent event;
|
ChangeEvent event;
|
||||||
memset(&event, 0, sizeof(event));
|
memset(&event, 0, sizeof(event));
|
||||||
/* The displayed path is the one transmitted (with -R + --files-from this is
|
|
||||||
the bare relative destination path); the metadata fallback below still
|
|
||||||
stats the local absolute path. */
|
|
||||||
event.path = file_wire_path(file);
|
|
||||||
event.decision = CHANGE_SENT;
|
event.decision = CHANGE_SENT;
|
||||||
event.is_directory = false;
|
event.is_directory = false;
|
||||||
|
event.is_symlink = false;
|
||||||
|
event.is_special = false;
|
||||||
|
event.is_hardlink = false;
|
||||||
event.size = file->data != NULL ? file->data->size : 0;
|
event.size = file->data != NULL ? file->data->size : 0;
|
||||||
/* FastSync has no wire-byte counter yet, so %b reports the source length
|
event.dest = file->dest_state;
|
||||||
* that had to be delivered (always equal to %l); the actual bytes written
|
if (file->is_symlink) {
|
||||||
* to the socket (compressed/delta) are not measured. */
|
event.is_symlink = true;
|
||||||
event.bytes_sent = event.size;
|
event.symlink_target = file->symlink_target;
|
||||||
if (file->metadata != NULL) {
|
event.size = file->symlink_target != NULL ? strlen(file->symlink_target) : 0;
|
||||||
event.mtime_sec = file->metadata->mtime_sec;
|
event.bytes_sent = 0;
|
||||||
} else if (format_uses_mtime(config->out_format) || format_uses_mtime(config->log_file_format)) {
|
} else if (file->is_special) {
|
||||||
/* Best-effort fallback for %M when no metadata was captured (no -M): the
|
event.is_special = true;
|
||||||
* path is stat()ed just to fill the field, and any failure leaves 0. */
|
event.bytes_sent = 0;
|
||||||
struct stat st;
|
} else if (file->link_group != 0 && !file->link_first) {
|
||||||
if (file->path != NULL && stat(file->path, &st) == 0)
|
event.is_hardlink = true;
|
||||||
event.mtime_sec = st.st_mtime;
|
event.hardlink_target = file->hardlink_target;
|
||||||
|
event.bytes_sent = 0;
|
||||||
|
} else {
|
||||||
|
event.bytes_sent = bytes_sent;
|
||||||
|
/* rsync's %c is the block-checksum bytes received for the file. Even a
|
||||||
|
* whole-file transfer (no basis; --append/--inplace included) receives
|
||||||
|
* rsync's 16-byte sum header, so rsync reports 16; a dry run transfers
|
||||||
|
* nothing and reports 0. FastSync's whole-file path has no sum header, so
|
||||||
|
* report rsync's value for parity. With delta enabled the real received
|
||||||
|
* bytes are kept, but FastSync's signature framing differs from rsync's so
|
||||||
|
* those stay numerically divergent. */
|
||||||
|
bool delta_active = config->use_delta && !config->whole_file;
|
||||||
|
event.bytes_read = (!config->dry_run && !delta_active) ? 16 : bytes_read;
|
||||||
}
|
}
|
||||||
change_emit(config, &event);
|
char* name = NULL;
|
||||||
|
char* path = NULL;
|
||||||
|
fill_event_from_file(config, file, &event, &name, &path);
|
||||||
|
if (name != NULL && path != NULL) {
|
||||||
|
fill_event_checksum(config, file, &event);
|
||||||
|
change_emit(config, &event);
|
||||||
|
}
|
||||||
|
free(name);
|
||||||
|
free(path);
|
||||||
|
}
|
||||||
|
|
||||||
|
void change_emit_file_sent(const Config* config, const File* file) {
|
||||||
|
if (file == NULL)
|
||||||
|
return;
|
||||||
|
unsigned long long payload = file->data != NULL ? file->data->size : 0;
|
||||||
|
change_emit_file_sent_bytes(config, file, payload, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
void change_emit_file_uptodate(const Config* config, const File* file) {
|
||||||
|
if (file == NULL || !change_list_enabled(config))
|
||||||
|
return;
|
||||||
|
ChangeEvent event;
|
||||||
|
memset(&event, 0, sizeof(event));
|
||||||
|
event.decision = CHANGE_UP_TO_DATE;
|
||||||
|
event.is_directory = false;
|
||||||
|
event.is_symlink = file->is_symlink;
|
||||||
|
event.is_special = file->is_special;
|
||||||
|
event.is_hardlink = file->link_group != 0 && !file->link_first;
|
||||||
|
event.symlink_target = file->symlink_target;
|
||||||
|
event.hardlink_target = file->hardlink_target;
|
||||||
|
event.size = file->data != NULL ? file->data->size : 0;
|
||||||
|
event.dest = file->dest_state;
|
||||||
|
char* name = NULL;
|
||||||
|
char* path = NULL;
|
||||||
|
fill_event_from_file(config, file, &event, &name, &path);
|
||||||
|
if (name != NULL && path != NULL)
|
||||||
|
change_emit(config, &event);
|
||||||
|
free(name);
|
||||||
|
free(path);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Build and emit a CHANGE_SENT event for an explicit directory entry (-d). */
|
|
||||||
void change_emit_dir_sent(const Config* config, const File* file) {
|
void change_emit_dir_sent(const Config* config, const File* file) {
|
||||||
if (file == NULL || !change_list_enabled(config))
|
if (file == NULL || !change_list_enabled(config))
|
||||||
return;
|
return;
|
||||||
ChangeEvent event;
|
ChangeEvent event;
|
||||||
memset(&event, 0, sizeof(event));
|
memset(&event, 0, sizeof(event));
|
||||||
event.path = file_wire_path(file);
|
|
||||||
event.decision = CHANGE_SENT;
|
event.decision = CHANGE_SENT;
|
||||||
event.is_directory = true;
|
event.is_directory = true;
|
||||||
event.size = 0;
|
event.size = 0;
|
||||||
event.bytes_sent = 0;
|
event.bytes_sent = 0;
|
||||||
if (file->metadata != NULL)
|
event.dest = file->dest_state;
|
||||||
event.mtime_sec = file->metadata->mtime_sec;
|
char* name = NULL;
|
||||||
change_emit(config, &event);
|
char* path = NULL;
|
||||||
|
fill_event_from_file(config, file, &event, &name, &path);
|
||||||
|
if (name != NULL && path != NULL)
|
||||||
|
change_emit(config, &event);
|
||||||
|
free(name);
|
||||||
|
free(path);
|
||||||
}
|
}
|
||||||
|
|||||||
+61
-25
@@ -2,7 +2,9 @@
|
|||||||
#define CHANGE_LIST_H
|
#define CHANGE_LIST_H
|
||||||
|
|
||||||
#include "config.h"
|
#include "config.h"
|
||||||
|
#include "checksum.h"
|
||||||
#include "file_types.h"
|
#include "file_types.h"
|
||||||
|
#include "format.h"
|
||||||
#include <stdbool.h>
|
#include <stdbool.h>
|
||||||
#include <sys/stat.h>
|
#include <sys/stat.h>
|
||||||
#include <time.h>
|
#include <time.h>
|
||||||
@@ -26,42 +28,59 @@ typedef enum {
|
|||||||
} ChangeDecision;
|
} ChangeDecision;
|
||||||
|
|
||||||
typedef struct {
|
typedef struct {
|
||||||
const char* path; /* full source path */
|
const char* path; /* long-form display path (rsync %f) */
|
||||||
|
const char* name; /* transfer-relative path (rsync %n), no trailing slash */
|
||||||
ChangeDecision decision;
|
ChangeDecision decision;
|
||||||
bool is_directory;
|
bool is_directory;
|
||||||
unsigned long long size; /* source file length in bytes */
|
bool is_symlink;
|
||||||
/* The number of bytes reported for a sent file. FastSync has no wire-byte
|
bool is_special;
|
||||||
* counter, so this is always the source length (== size / %l); actual
|
bool is_hardlink; /* a hard-link sibling (linked, no data sent) */
|
||||||
* post-compression/delta bytes on the wire are not counted. */
|
bool deleted; /* a would-delete report (-n --delete); no source file */
|
||||||
unsigned long long bytes_sent;
|
const char* symlink_target;
|
||||||
time_t mtime_sec; /* 0 when unknown */
|
const char* hardlink_target;
|
||||||
|
unsigned long long size; /* source file length in bytes */
|
||||||
|
unsigned long long bytes_sent; /* wire bytes actually transferred (rsync %b) */
|
||||||
|
unsigned long long bytes_read; /* wire bytes read back for this file (rsync %c) */
|
||||||
|
/* rsync %C: whole-file checksum hex for a transferred regular file. Only
|
||||||
|
* filled when the active format uses %C (checksum_known == false otherwise,
|
||||||
|
* which renders as spaces like rsync for non-regular entries). */
|
||||||
|
bool checksum_known;
|
||||||
|
char checksum[CHECKSUM_MAX_DIGEST_LEN * 2 + 1];
|
||||||
|
time_t mtime_sec;
|
||||||
|
long mtime_nsec;
|
||||||
|
mode_t mode;
|
||||||
|
uid_t uid;
|
||||||
|
gid_t gid;
|
||||||
|
/* Receiver-reported pre-transfer destination state (OutputDestState.known is
|
||||||
|
* false when no report was requested/received). */
|
||||||
|
OutputDestState dest;
|
||||||
} ChangeEvent;
|
} ChangeEvent;
|
||||||
|
|
||||||
/* True when any output mode is active and per-file events matter. */
|
/* True when any output mode is active and per-file events matter. */
|
||||||
bool change_list_enabled(const Config* config);
|
bool change_list_enabled(const Config* config);
|
||||||
|
|
||||||
/* Render the rsync-style itemize line for a transferred file:
|
/* Render the rsync-style itemize line for a transferred item
|
||||||
* `>f+++++++++ <path>`
|
* (`%i %n%L`): `>f+++++++++ sub/b.txt`. Caller frees the result. */
|
||||||
* The 11-char code is `>f` (regular file transferred to the remote host)
|
char* change_render_itemize(const Config* config, const ChangeEvent* event);
|
||||||
* followed by c/s/t/p/o/g/u/a/x markers that are all `+` (value will be set
|
|
||||||
* / differs) because FastSync does not separately compare checksums, size,
|
|
||||||
* mtime, perms, owner, group, uid, acl, or xattr on the receiving side, so a
|
|
||||||
* sent file is reported as fully updated. Up-to-date files print no line
|
|
||||||
* (rsync single `-i` only shows changes). Caller frees the result. */
|
|
||||||
char* change_render_itemize(const ChangeEvent* event);
|
|
||||||
|
|
||||||
/* Expand an --out-format/--log-file-format template. Tokens:
|
/* Expand an --out-format/--log-file-format template. Supported tokens:
|
||||||
* %f full source path %b "bytes sent" == the source length (%l);
|
* %i itemize code %n transfer-relative name (dir: trailing /)
|
||||||
* %n leaf (base) name actual post-compression/delta wire bytes
|
* %f long display path %l file length in bytes
|
||||||
* %l file length in bytes are not counted
|
* %b wire bytes transferred %c block-checksum bytes received (rsync: 16
|
||||||
* %M mtime in whole seconds %% a literal percent sign
|
* for a whole-file transfer, 0 for a dry run)
|
||||||
|
* %C whole-file checksum hex (xxh128 by default; spaces for non-regular)
|
||||||
|
* %M mtime (YYYY/MM/DD-HH:MM:SS)
|
||||||
|
* %t current time %o operation ("send"/"del.")
|
||||||
|
* %p pid %B permission bits without the type char
|
||||||
|
* %U uid %G gid
|
||||||
|
* %L " -> target" / " => target" %% a literal percent sign
|
||||||
* Unknown %X sequences are preserved verbatim. Caller frees the result. */
|
* Unknown %X sequences are preserved verbatim. Caller frees the result. */
|
||||||
char* change_render_format(const char* format, const ChangeEvent* event);
|
char* change_render_format(const char* format, const Config* config, const ChangeEvent* event);
|
||||||
|
|
||||||
/* Render one --list-only long-listing entry:
|
/* Render one --list-only long-listing entry:
|
||||||
* `-rw-r--r-- 12 2026/09/06 10:00:00 <path>`
|
* `-rw-r--r-- 12 2026/09/06 10:00:00 sub/b.txt`
|
||||||
* (ls -l style columns; mtime in the local time zone). Caller frees it. */
|
* (ls -l style columns; mtime in the local time zone). Caller frees it. */
|
||||||
char* change_render_list_line(mode_t mode, unsigned long long size, time_t mtime, const char* path);
|
char* change_render_list_line(const Config* config, const ChangeEvent* event);
|
||||||
|
|
||||||
/* Emit an event to every active destination:
|
/* Emit an event to every active destination:
|
||||||
* stdout: --itemize-changes line, or the --out-format expansion when set;
|
* stdout: --itemize-changes line, or the --out-format expansion when set;
|
||||||
@@ -69,10 +88,27 @@ char* change_render_list_line(mode_t mode, unsigned long long size, time_t mtime
|
|||||||
* CHANGE_UP_TO_DATE events produce no output. */
|
* CHANGE_UP_TO_DATE events produce no output. */
|
||||||
void change_emit(const Config* config, const ChangeEvent* event);
|
void change_emit(const Config* config, const ChangeEvent* event);
|
||||||
|
|
||||||
/* Build and emit a CHANGE_SENT event for a file the client just sent. */
|
/* Build and emit a CHANGE_SENT event for a file the client just sent. `bytes_sent`
|
||||||
|
* is the process-wide wire-byte delta for this file (rsync's %b) and `bytes_read`
|
||||||
|
* the received bytes used for the delta handshake; pass 0 when unknown. For a
|
||||||
|
* whole-file transfer %c is pinned to rsync's 16-byte sum header regardless. */
|
||||||
|
void change_emit_file_sent_bytes(const Config* config, const File* file,
|
||||||
|
unsigned long long bytes_sent, unsigned long long bytes_read);
|
||||||
|
|
||||||
|
/* Build and emit a CHANGE_SENT event for a file the client just sent, deriving
|
||||||
|
* the wire byte counts from the source payload length. */
|
||||||
void change_emit_file_sent(const Config* config, const File* file);
|
void change_emit_file_sent(const Config* config, const File* file);
|
||||||
|
|
||||||
/* Build and emit a CHANGE_SENT event for an explicit directory entry (-d). */
|
/* Build and emit a CHANGE_SENT event for an explicit directory entry (-d). */
|
||||||
void change_emit_dir_sent(const Config* config, const File* file);
|
void change_emit_dir_sent(const Config* config, const File* file);
|
||||||
|
|
||||||
|
/* Build and emit a CHANGE_UP_TO_DATE event for a file the receiver already had.
|
||||||
|
* With --info=name2 it renders rsync's "NAME is uptodate" line (no output
|
||||||
|
* otherwise). */
|
||||||
|
void change_emit_file_uptodate(const Config* config, const File* file);
|
||||||
|
|
||||||
|
/* Reset the lazy transfer-root `./` line emitted ahead of the first
|
||||||
|
* --info=name entry. Call once at the start of a transfer. */
|
||||||
|
void change_reset_name_root(void);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
+1250
-228
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,682 @@
|
|||||||
|
#include "client_send_internal.h"
|
||||||
|
#include "array_list.h"
|
||||||
|
#include "change_list.h"
|
||||||
|
#include "charset.h"
|
||||||
|
#include "config.h"
|
||||||
|
#include "data.h"
|
||||||
|
#include "delta.h"
|
||||||
|
#include "file.h"
|
||||||
|
#include "format.h"
|
||||||
|
#include "log.h"
|
||||||
|
#include "protocol.h"
|
||||||
|
#include "scanner.h"
|
||||||
|
#include "transport_tls.h"
|
||||||
|
#include "utils.h"
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
|
#include <sys/stat.h>
|
||||||
|
#include <time.h>
|
||||||
|
|
||||||
|
/* True when --dry-run should contact a receiver rather than running the
|
||||||
|
* client-side local manifest. Any target a real run would reach over the wire
|
||||||
|
* selects the server-contacting path: a remote (SSH host:path), a daemon
|
||||||
|
* (host::module/path), an explicit --server-host, --server-port/--port, TLS, or
|
||||||
|
* a source-bind --address. A plain local destination (none of these) keeps the
|
||||||
|
* original client-side behavior, which never dials the default 127.0.0.1:8080. */
|
||||||
|
bool dry_run_targets_server(const Config* config) {
|
||||||
|
if (!config)
|
||||||
|
return false;
|
||||||
|
if (config->transport == TRANSPORT_SSH)
|
||||||
|
return true;
|
||||||
|
if (config->module && config->module[0] != '\0')
|
||||||
|
return true;
|
||||||
|
if (config->cli.server_host_set || config->cli.server_port_set)
|
||||||
|
return true;
|
||||||
|
if (config->use_tls)
|
||||||
|
return true;
|
||||||
|
if (config->address != NULL)
|
||||||
|
return true;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool add_chunk_to_manifest(ArrayList* manifest, const Chunk* chunk) {
|
||||||
|
if (!manifest)
|
||||||
|
return true;
|
||||||
|
for (int i = 0; i < chunk->element_count; i++) {
|
||||||
|
const char* path = file_wire_path(chunk->items[i]);
|
||||||
|
if (*path == '/')
|
||||||
|
path++;
|
||||||
|
char* entry = str_dup(path);
|
||||||
|
if (!entry) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Failed to allocate manifest entry");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (!array_list_add(manifest, entry)) {
|
||||||
|
free(entry);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Print dry-run manifest showing files that would be transferred. Returns 0 on success. */
|
||||||
|
int send_dry_run_manifest(const Config* config) {
|
||||||
|
int skipped = 0;
|
||||||
|
ArrayList* missing_dest = NULL;
|
||||||
|
if (config->delete_missing_args) {
|
||||||
|
missing_dest = array_list_create(free);
|
||||||
|
if (!missing_dest)
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
if (!files_from_list_check(config, missing_dest, &skipped)) {
|
||||||
|
if (missing_dest)
|
||||||
|
array_list_delete(missing_dest);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
PreparedScanner prepared;
|
||||||
|
if (!prepare_scanner(config, 0, &prepared)) {
|
||||||
|
if (missing_dest)
|
||||||
|
array_list_delete(missing_dest);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
DirectoryScanner* scanner =
|
||||||
|
directory_scanner_create_with_options(config->send_directory, &prepared.options);
|
||||||
|
if (!scanner) {
|
||||||
|
prepared_scanner_destroy(&prepared);
|
||||||
|
if (missing_dest)
|
||||||
|
array_list_delete(missing_dest);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
Chunk* chunk;
|
||||||
|
int file_count = 0;
|
||||||
|
unsigned long long total_bytes = 0;
|
||||||
|
char size_buffer[32];
|
||||||
|
if (!config->quiet)
|
||||||
|
printf("Dry run: files to be transferred\n");
|
||||||
|
while ((chunk = directory_scanner_next(scanner)) != NULL) {
|
||||||
|
for (int i = 0; i < chunk->element_count; i++) {
|
||||||
|
if (!config->quiet) {
|
||||||
|
char* escaped_path =
|
||||||
|
output_escape(file_wire_path(chunk->items[i]), config->eight_bit_output);
|
||||||
|
if (!escaped_path) {
|
||||||
|
chunk_destroy(chunk);
|
||||||
|
directory_scanner_destroy(scanner);
|
||||||
|
prepared_scanner_destroy(&prepared);
|
||||||
|
if (missing_dest)
|
||||||
|
array_list_delete(missing_dest);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
if (config->human_readable)
|
||||||
|
printf(
|
||||||
|
" %s (%s)\n", escaped_path,
|
||||||
|
display_bytes(chunk->items[i]->data->size, true, size_buffer, sizeof(size_buffer)));
|
||||||
|
else
|
||||||
|
printf(" %s (%zu bytes)\n", escaped_path, chunk->items[i]->data->size);
|
||||||
|
free(escaped_path);
|
||||||
|
}
|
||||||
|
total_bytes += chunk->items[i]->data->size;
|
||||||
|
file_count++;
|
||||||
|
}
|
||||||
|
chunk_destroy(chunk);
|
||||||
|
}
|
||||||
|
directory_scanner_destroy(scanner);
|
||||||
|
prepared_scanner_destroy(&prepared);
|
||||||
|
/* --delete-missing-args: the missing entries' destination mirrors render as
|
||||||
|
would-be deletions (rsync's dry-run also lists its *deleting lines). */
|
||||||
|
if (missing_dest && !config->quiet) {
|
||||||
|
for (int i = 0; i < missing_dest->size; i++) {
|
||||||
|
char* escaped = output_escape((char*)missing_dest->items[i], config->eight_bit_output);
|
||||||
|
printf(" %s (missing; would be deleted)\n", escaped ? escaped : "<allocation failed>");
|
||||||
|
free(escaped);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (missing_dest)
|
||||||
|
array_list_delete(missing_dest);
|
||||||
|
if (!config->quiet) {
|
||||||
|
if (config->human_readable)
|
||||||
|
printf("Total: %d files, %s\n", file_count,
|
||||||
|
display_bytes(total_bytes, true, size_buffer, sizeof(size_buffer)));
|
||||||
|
else
|
||||||
|
printf("Total: %d files, %.1f MB\n", file_count, (double)total_bytes / (double)BYTES_PER_MIB);
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
char* name; /* transfer-relative name ("" == the source root) */
|
||||||
|
mode_t mode;
|
||||||
|
unsigned long long size;
|
||||||
|
time_t mtime;
|
||||||
|
long mtime_nsec;
|
||||||
|
bool is_dir;
|
||||||
|
bool is_symlink;
|
||||||
|
char* link_target;
|
||||||
|
} ListEntry;
|
||||||
|
|
||||||
|
static void list_entries_destroy(ListEntry* entries, size_t count) {
|
||||||
|
if (entries == NULL)
|
||||||
|
return;
|
||||||
|
for (size_t i = 0; i < count; i++) {
|
||||||
|
free(entries[i].name);
|
||||||
|
free(entries[i].link_target);
|
||||||
|
}
|
||||||
|
free(entries);
|
||||||
|
}
|
||||||
|
|
||||||
|
static int compare_list_entries(const void* left, const void* right) {
|
||||||
|
const ListEntry* a = (const ListEntry*)left;
|
||||||
|
const ListEntry* b = (const ListEntry*)right;
|
||||||
|
return strcmp(a->name, b->name);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Relative path of an entry below `root` ("" for the root itself). Mirrors
|
||||||
|
* change_list's relative_name for list-only rendering. */
|
||||||
|
static char* list_relative_name(const char* root, const char* full) {
|
||||||
|
if (root == NULL || full == NULL)
|
||||||
|
return str_dup(full != NULL ? full : "");
|
||||||
|
size_t root_len = strlen(root);
|
||||||
|
while (root_len > 1 && root[root_len - 1] == '/')
|
||||||
|
root_len--;
|
||||||
|
if (strncmp(root, full, root_len) == 0) {
|
||||||
|
if (full[root_len] == '\0')
|
||||||
|
return str_dup("");
|
||||||
|
if (full[root_len] == '/')
|
||||||
|
return str_dup(full + root_len + 1);
|
||||||
|
}
|
||||||
|
return str_dup(full);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* --list-only: print an ls-style listing of the entries that WOULD be
|
||||||
|
* transferred and exit without contacting the server or writing anything.
|
||||||
|
* Names are transfer-relative (rsync prints `a.txt`, `sub/b.txt`, `.`) and
|
||||||
|
* directory entries are included. Returns 0 on success, 1 on error. */
|
||||||
|
int send_list_only(const Config* config) {
|
||||||
|
int skipped = 0;
|
||||||
|
if (!files_from_list_check(config, NULL, &skipped))
|
||||||
|
return 1;
|
||||||
|
PreparedScanner prepared;
|
||||||
|
if (!prepare_scanner(config, 0, &prepared))
|
||||||
|
return 1;
|
||||||
|
prepared.options.use_metadata = true; /* capture mode + mtime for the listing */
|
||||||
|
prepared.options.list_dirs = true;
|
||||||
|
DirectoryScanner* scanner =
|
||||||
|
directory_scanner_create_with_options(config->send_directory, &prepared.options);
|
||||||
|
if (!scanner) {
|
||||||
|
prepared_scanner_destroy(&prepared);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
ListEntry* entries = NULL;
|
||||||
|
size_t count = 0;
|
||||||
|
size_t capacity = 0;
|
||||||
|
bool oom = false;
|
||||||
|
|
||||||
|
/* rsync lists the source root itself (as "."). Only when the source is a
|
||||||
|
* directory and no --files-from subset is in effect. */
|
||||||
|
if (config->files_from_set == NULL && config->send_directory != NULL) {
|
||||||
|
struct stat st;
|
||||||
|
if (stat(config->send_directory, &st) == 0 && S_ISDIR(st.st_mode)) {
|
||||||
|
capacity = 64;
|
||||||
|
entries = calloc(capacity, sizeof(ListEntry));
|
||||||
|
if (entries == NULL) {
|
||||||
|
oom = true;
|
||||||
|
} else if ((entries[0].name = str_dup("")) == NULL) {
|
||||||
|
/* A NULL name would be dereferenced by qsort/render: fail the listing. */
|
||||||
|
oom = true;
|
||||||
|
} else {
|
||||||
|
entries[0].mode = st.st_mode;
|
||||||
|
entries[0].mtime = st.st_mtime;
|
||||||
|
entries[0].mtime_nsec = st.st_mtim.tv_nsec;
|
||||||
|
entries[0].size = (unsigned long long)st.st_size;
|
||||||
|
entries[0].is_dir = true;
|
||||||
|
count = 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Chunk* chunk;
|
||||||
|
while (!oom && (chunk = directory_scanner_next(scanner)) != NULL) {
|
||||||
|
for (int i = 0; i < chunk->element_count; i++) {
|
||||||
|
File* f = chunk->items[i];
|
||||||
|
if (f == NULL)
|
||||||
|
continue;
|
||||||
|
if (count == capacity) {
|
||||||
|
size_t new_capacity = capacity > 0 ? capacity * 2 : 64;
|
||||||
|
if (new_capacity <= capacity) {
|
||||||
|
oom = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
ListEntry* grown = realloc(entries, new_capacity * sizeof(ListEntry));
|
||||||
|
if (!grown) {
|
||||||
|
oom = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
entries = grown;
|
||||||
|
memset(entries + capacity, 0, (new_capacity - capacity) * sizeof(ListEntry));
|
||||||
|
capacity = new_capacity;
|
||||||
|
}
|
||||||
|
char* name = list_relative_name(config->send_directory, file_wire_path(f));
|
||||||
|
if (!name) {
|
||||||
|
oom = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
mode_t mode = 0;
|
||||||
|
time_t mtime = 0;
|
||||||
|
long mtime_nsec = 0;
|
||||||
|
if (f->metadata != NULL) {
|
||||||
|
mode = f->metadata->mode;
|
||||||
|
mtime = f->metadata->mtime_sec;
|
||||||
|
mtime_nsec = f->metadata->mtime_nsec;
|
||||||
|
} else {
|
||||||
|
struct stat st;
|
||||||
|
if (lstat(f->path, &st) == 0) {
|
||||||
|
mode = st.st_mode;
|
||||||
|
mtime = st.st_mtime;
|
||||||
|
mtime_nsec = st.st_mtim.tv_nsec;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
entries[count].name = name;
|
||||||
|
entries[count].mode = mode;
|
||||||
|
entries[count].mtime = mtime;
|
||||||
|
entries[count].mtime_nsec = mtime_nsec;
|
||||||
|
if (f->is_symlink)
|
||||||
|
entries[count].size = f->symlink_target != NULL ? strlen(f->symlink_target) : 0;
|
||||||
|
else if (f->is_dir) {
|
||||||
|
struct stat dir_st;
|
||||||
|
entries[count].size = stat(f->path, &dir_st) == 0 ? (unsigned long long)dir_st.st_size : 0;
|
||||||
|
} else
|
||||||
|
entries[count].size = f->data != NULL ? f->data->size : 0;
|
||||||
|
entries[count].is_dir = f->is_dir;
|
||||||
|
entries[count].is_symlink = f->is_symlink;
|
||||||
|
entries[count].link_target =
|
||||||
|
f->is_symlink && f->symlink_target ? str_dup(f->symlink_target) : NULL;
|
||||||
|
count++;
|
||||||
|
}
|
||||||
|
chunk_destroy(chunk);
|
||||||
|
}
|
||||||
|
bool failed = oom || directory_scanner_failed(scanner) || directory_scanner_had_io_error(scanner);
|
||||||
|
directory_scanner_destroy(scanner);
|
||||||
|
prepared_scanner_destroy(&prepared);
|
||||||
|
if (failed) {
|
||||||
|
list_entries_destroy(entries, count);
|
||||||
|
if (oom)
|
||||||
|
log_message(LOG_LEVEL_ERROR, "memory allocation failed while listing");
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
if (count > 1)
|
||||||
|
qsort(entries, count, sizeof(ListEntry), compare_list_entries);
|
||||||
|
for (size_t i = 0; i < count; i++) {
|
||||||
|
ChangeEvent event;
|
||||||
|
memset(&event, 0, sizeof(event));
|
||||||
|
event.name = entries[i].name;
|
||||||
|
event.path = entries[i].name;
|
||||||
|
event.mode = entries[i].mode;
|
||||||
|
event.size = entries[i].size;
|
||||||
|
event.mtime_sec = entries[i].mtime;
|
||||||
|
event.mtime_nsec = entries[i].mtime_nsec;
|
||||||
|
event.is_directory = entries[i].is_dir;
|
||||||
|
event.is_symlink = entries[i].is_symlink;
|
||||||
|
event.symlink_target = entries[i].link_target;
|
||||||
|
char* line = change_render_list_line(config, &event);
|
||||||
|
if (line != NULL) {
|
||||||
|
char* escaped = output_escape(line, config->eight_bit_output);
|
||||||
|
printf("%s\n", escaped != NULL ? escaped : line);
|
||||||
|
free(escaped);
|
||||||
|
free(line);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
list_entries_destroy(entries, count);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Send the delete manifest to the server. Returns 0 on success, -1 on
|
||||||
|
failure. It carries FOUR sections: the keep-set paths, the protected
|
||||||
|
excluded prefixes, the --delete-missing-args exact-delete paths, and the
|
||||||
|
destination-relative directories the sender synchronized this run.
|
||||||
|
When --delete-excluded is given `protected` is empty: excluded destination
|
||||||
|
mirrors are then ordinary extras and are removed. When
|
||||||
|
--delete-missing-args is active `missing_args` holds the destination mirrors
|
||||||
|
of missing --files-from entries: each is an explicit receiver-side deletion
|
||||||
|
request, independent of the extras walk. `synced_dirs` confines the extras
|
||||||
|
walk to entries directly inside a synchronized directory. A NULL
|
||||||
|
keep-set / protected / missing / dirs list transmits an empty section. All
|
||||||
|
four sections are unbounded on the sender; the receiver enforces
|
||||||
|
MAX_MANIFEST_ENTRIES per section and a single MAX_MANIFEST_BYTES budget
|
||||||
|
shared across the sections, rejecting (with STATUS_ERROR) an over-budget
|
||||||
|
frame. A heavily filtered source whose exclusion list is large therefore
|
||||||
|
fails the run cleanly on the receiver rather than being truncated. */
|
||||||
|
int send_delete_manifest(int fd, ArrayList* manifest, ArrayList* protected_prefixes,
|
||||||
|
ArrayList* size_skipped, ArrayList* missing_args, ArrayList* synced_dirs) {
|
||||||
|
if (!send_status(fd, STATUS_MANIFEST))
|
||||||
|
return -1;
|
||||||
|
int keep_count = manifest ? manifest->size : 0;
|
||||||
|
if (!send_int(fd, keep_count))
|
||||||
|
return -1;
|
||||||
|
for (int i = 0; i < keep_count; i++) {
|
||||||
|
if (!send_wire_str(fd, (char*)manifest->items[i]))
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
/* The receiver has ONE protected-prefix section; filter-excluded prefixes
|
||||||
|
(dropped under --delete-excluded) and size-pruned prefixes (always
|
||||||
|
protected) are concatenated into it. */
|
||||||
|
int protected_count =
|
||||||
|
(protected_prefixes ? protected_prefixes->size : 0) + (size_skipped ? size_skipped->size : 0);
|
||||||
|
if (!send_int(fd, protected_count))
|
||||||
|
return -1;
|
||||||
|
if (protected_prefixes) {
|
||||||
|
for (int i = 0; i < protected_prefixes->size; i++) {
|
||||||
|
if (!send_wire_str(fd, (char*)protected_prefixes->items[i]))
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (size_skipped) {
|
||||||
|
for (int i = 0; i < size_skipped->size; i++) {
|
||||||
|
if (!send_wire_str(fd, (char*)size_skipped->items[i]))
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
int missing_count = missing_args ? missing_args->size : 0;
|
||||||
|
if (!send_int(fd, missing_count))
|
||||||
|
return -1;
|
||||||
|
for (int i = 0; i < missing_count; i++) {
|
||||||
|
if (!send_wire_str(fd, (char*)missing_args->items[i]))
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
int dirs_count = synced_dirs ? synced_dirs->size : 0;
|
||||||
|
if (!send_int(fd, dirs_count))
|
||||||
|
return -1;
|
||||||
|
for (int i = 0; i < dirs_count; i++) {
|
||||||
|
if (!send_wire_str(fd, (char*)synced_dirs->items[i]))
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Transmit the keep-set manifest and wait for the receiver's verdict. Used by
|
||||||
|
--delete-before/--delete-during, where the extras are removed on the receiver
|
||||||
|
BEFORE the first byte of file data is sent: the receiver acknowledges with
|
||||||
|
STATUS_OK once the bounded delete committed, or STATUS_ERROR if it could not
|
||||||
|
(in which case the sender aborts without streaming any data). The ACK may
|
||||||
|
take much longer than an ordinary per-message round trip because the receiver
|
||||||
|
performs the whole bounded deletion walk (up to MAX_SERVER_DELETE_COUNT
|
||||||
|
unlinks) before replying, so the wait uses a generous explicit deadline
|
||||||
|
instead of the default 60 s receive window. */
|
||||||
|
#define DELETE_ACK_TIMEOUT_SEC 3600
|
||||||
|
/* While waiting for the (potentially slow) receiver-side deletion, send a
|
||||||
|
* STATUS_KEEPALIVE at most this often so the connection is demonstrably alive
|
||||||
|
* and neither side's per-message timeout trips. */
|
||||||
|
#define DELETE_ACK_KEEPALIVE_SEC 10
|
||||||
|
|
||||||
|
bool send_delete_manifest_early(Client* client, ArrayList* manifest, ArrayList* protected_prefixes,
|
||||||
|
ArrayList* size_skipped, ArrayList* missing_args,
|
||||||
|
ArrayList* synced_dirs) {
|
||||||
|
if (!client || !manifest)
|
||||||
|
return false;
|
||||||
|
if (send_delete_manifest(client->file_descriptor, manifest, protected_prefixes, size_skipped,
|
||||||
|
missing_args, synced_dirs) != 0)
|
||||||
|
return false;
|
||||||
|
Status ack;
|
||||||
|
/* The wait is long (up to an hour) and runs inline on this thread: a helper
|
||||||
|
* thread would race the non-thread-safe protocol send path, so keepalives are
|
||||||
|
* emitted from this wait loop itself. A Ctrl-C/SIGTERM abort flag also ends
|
||||||
|
* the wait; the caller then best-effort sends STATUS_ABORT. */
|
||||||
|
if (!receive_status_keepalive(client->file_descriptor, &ack, DELETE_ACK_TIMEOUT_SEC,
|
||||||
|
DELETE_ACK_KEEPALIVE_SEC, client_abort_pending)) {
|
||||||
|
/* A Ctrl-C/SIGTERM abort ends the wait above; tell the receiver before the
|
||||||
|
caller tears the connection down (best-effort). */
|
||||||
|
if (client_abort_pending()) {
|
||||||
|
log_info_message(LOG_INFO_MISC,
|
||||||
|
"Abort requested while awaiting delete ack; sending STATUS_ABORT");
|
||||||
|
send_status(client->file_descriptor, STATUS_ABORT);
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (ack != STATUS_OK) {
|
||||||
|
log_server_rejection("Server failed to delete files before the transfer");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Server-contacting --dry-run. Connects to the configured remote/daemon and
|
||||||
|
* runs the normal per-file incremental decision WITHOUT transmitting any file
|
||||||
|
* data: the receiver (which also sees dry_run=true on the wire) answers
|
||||||
|
* STATUS_OK for an up-to-date file and STATUS_DRY_RUN_TRANSFER for a file it
|
||||||
|
* would otherwise write, mutating nothing on either side. The would-transfer
|
||||||
|
* set and the same trailer as the local dry-run are printed. A
|
||||||
|
* --compare-dest exact basis hit with no destination copy is reported as a
|
||||||
|
* skip by the receiver.
|
||||||
|
*
|
||||||
|
* Only regular files take the receiver-consulted check; directory / symlink /
|
||||||
|
* special / hard-link-sibling entries have no per-file content check, so they
|
||||||
|
* are reported conservatively as would-transfer and their frames are never
|
||||||
|
* sent (which is what keeps the receiver mutation-free). --delete* is
|
||||||
|
* deliberately NOT transmitted in dry-run, so no deletion can occur; the
|
||||||
|
* would-delete manifest report is a documented follow-up.
|
||||||
|
*
|
||||||
|
* Returns 0 on success, 1 on error. */
|
||||||
|
int send_dry_run_remote(Config* config) {
|
||||||
|
int from_skipped = 0;
|
||||||
|
ArrayList* missing_args = NULL;
|
||||||
|
if (config->delete_missing_args) {
|
||||||
|
missing_args = array_list_create(free);
|
||||||
|
if (!missing_args)
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
if (!files_from_list_check(config, missing_args, &from_skipped)) {
|
||||||
|
if (missing_args)
|
||||||
|
array_list_delete(missing_args);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
if (missing_args)
|
||||||
|
array_list_delete(missing_args);
|
||||||
|
/* A live session may follow, so arm graceful abort handling. */
|
||||||
|
client_set_abort_armed(true);
|
||||||
|
Client* client = connect_transfer_client(config);
|
||||||
|
if (!client) {
|
||||||
|
if (config->transport == TRANSPORT_TCP)
|
||||||
|
log_message(LOG_LEVEL_ERROR, "could not connect to server%s",
|
||||||
|
config->use_tls ? " via TLS" : "");
|
||||||
|
client_set_abort_armed(false);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
ProtocolSession session;
|
||||||
|
protocol_session_init(&session, client->file_descriptor, client->file_descriptor);
|
||||||
|
protocol_session_set_io_timeout(&session, config->timeout);
|
||||||
|
protocol_session_set_ssl(&session, (SSL*)client->ssl);
|
||||||
|
protocol_session_bind(&session);
|
||||||
|
|
||||||
|
int ret = 1;
|
||||||
|
time_t dry_start = time(NULL);
|
||||||
|
ReceiverStats dry_stats;
|
||||||
|
memset(&dry_stats, 0, sizeof(dry_stats));
|
||||||
|
PreparedScanner prepared;
|
||||||
|
memset(&prepared, 0, sizeof(prepared));
|
||||||
|
DirectoryScanner* scanner = NULL;
|
||||||
|
ArrayList* dry_manifest = NULL;
|
||||||
|
ArrayList* dry_dirs = NULL;
|
||||||
|
ArrayList* dry_excluded = NULL;
|
||||||
|
ArrayList* dry_size_skipped = NULL;
|
||||||
|
if (!config_send(client->file_descriptor, config))
|
||||||
|
goto dry_fail;
|
||||||
|
receive_daemon_motd(client, config);
|
||||||
|
if (!prepare_scanner(config, 0, &prepared))
|
||||||
|
goto dry_fail;
|
||||||
|
/* -n --delete: build the same keep-set manifest, protected prefixes, and
|
||||||
|
synchronized-directory scope a real run would send, so the receiver's
|
||||||
|
read-only extras walk enumerates exactly the deletions a real run makes. */
|
||||||
|
if (config->use_delete) {
|
||||||
|
dry_manifest = array_list_create(free);
|
||||||
|
dry_dirs = array_list_create(free);
|
||||||
|
dry_size_skipped = array_list_create(free);
|
||||||
|
if (!dry_manifest || !dry_dirs || !dry_size_skipped)
|
||||||
|
goto dry_fail;
|
||||||
|
if (!config->delete_excluded) {
|
||||||
|
dry_excluded = array_list_create(free);
|
||||||
|
if (!dry_excluded)
|
||||||
|
goto dry_fail;
|
||||||
|
prepared.options.excluded_paths = dry_excluded;
|
||||||
|
}
|
||||||
|
prepared.options.size_skipped_paths = dry_size_skipped;
|
||||||
|
/* A --files-from subset confines the extras walk to the directories the
|
||||||
|
scan synchronized; a full recursive transfer marks the root itself. */
|
||||||
|
if (config->files_from_set == NULL) {
|
||||||
|
char* root_marker = delete_scope_root_marker(config);
|
||||||
|
if (!root_marker || !array_list_add(dry_dirs, root_marker)) {
|
||||||
|
free(root_marker);
|
||||||
|
goto dry_fail;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
prepared.options.synced_dirs = dry_dirs;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
scanner = directory_scanner_create_with_options(config->send_directory, &prepared.options);
|
||||||
|
if (!scanner)
|
||||||
|
goto dry_fail;
|
||||||
|
|
||||||
|
int file_count = 0;
|
||||||
|
unsigned long long total_bytes = 0;
|
||||||
|
char size_buffer[32];
|
||||||
|
if (!config->quiet)
|
||||||
|
printf("Dry run: files to be transferred\n");
|
||||||
|
Chunk* chunk;
|
||||||
|
while ((chunk = directory_scanner_next(scanner)) != NULL) {
|
||||||
|
if (dry_manifest && !add_chunk_to_manifest(dry_manifest, chunk)) {
|
||||||
|
chunk_destroy(chunk);
|
||||||
|
goto dry_fail;
|
||||||
|
}
|
||||||
|
for (int i = 0; i < chunk->element_count; i++) {
|
||||||
|
File* f = chunk->items[i];
|
||||||
|
if (!f)
|
||||||
|
continue;
|
||||||
|
unsigned long long fsize = f->data ? f->data->size : 0;
|
||||||
|
bool would;
|
||||||
|
if (f->is_dir || f->is_symlink || f->is_special ||
|
||||||
|
(f->link_group != 0 && !f->link_first && f->hardlink_target != NULL)) {
|
||||||
|
/* No receiver-side content check exists for these frame types; a real
|
||||||
|
run would (re)create them, so report would-transfer and send no
|
||||||
|
frame (the receiver must stay mutation-free). */
|
||||||
|
would = true;
|
||||||
|
} else if (fsize > MAX_RECEIVE_WHOLE_FILE_SIZE && !config->use_incremental &&
|
||||||
|
!config_has_basis(config)) {
|
||||||
|
/* A non-incremental run streams a >whole-file-limit source without the
|
||||||
|
STATUS_CHECK handshake, so no read-only receiver decision is possible
|
||||||
|
(and none is needed: a real run would transfer it). */
|
||||||
|
would = true;
|
||||||
|
} else {
|
||||||
|
DeltaSignature* sig = NULL;
|
||||||
|
unsigned long long resume_offset = 0;
|
||||||
|
int rc = incremental_check(client, f, config, &sig, &resume_offset);
|
||||||
|
delta_signature_destroy(sig);
|
||||||
|
if (rc < 0) {
|
||||||
|
chunk_destroy(chunk);
|
||||||
|
goto dry_fail;
|
||||||
|
}
|
||||||
|
if (rc == 1)
|
||||||
|
continue; /* up to date; nothing to report */
|
||||||
|
if (rc != 4) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Unexpected receiver reply during dry-run");
|
||||||
|
chunk_destroy(chunk);
|
||||||
|
goto dry_fail;
|
||||||
|
}
|
||||||
|
would = true;
|
||||||
|
}
|
||||||
|
if (would) {
|
||||||
|
if (!config->quiet) {
|
||||||
|
char* escaped_path = output_escape(file_wire_path(f), config->eight_bit_output);
|
||||||
|
if (!escaped_path) {
|
||||||
|
chunk_destroy(chunk);
|
||||||
|
goto dry_fail;
|
||||||
|
}
|
||||||
|
if (config->human_readable)
|
||||||
|
printf(" %s (%s)\n", escaped_path,
|
||||||
|
display_bytes(fsize, true, size_buffer, sizeof(size_buffer)));
|
||||||
|
else
|
||||||
|
printf(" %s (%llu bytes)\n", escaped_path, fsize);
|
||||||
|
free(escaped_path);
|
||||||
|
}
|
||||||
|
total_bytes += fsize;
|
||||||
|
file_count++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
chunk_destroy(chunk);
|
||||||
|
}
|
||||||
|
bool io_error = directory_scanner_had_io_error(scanner);
|
||||||
|
if (directory_scanner_failed(scanner))
|
||||||
|
goto dry_fail;
|
||||||
|
if (io_error)
|
||||||
|
log_message(LOG_LEVEL_WARNING, "source scan hit an unreadable directory");
|
||||||
|
/* Send the keep-set manifest (no data frames) so the receiver can enumerate
|
||||||
|
the destination extras; an early-timing delete ACKs before it will accept
|
||||||
|
the terminal FINISHED. */
|
||||||
|
bool early_delete = config->use_delete && config_delete_timing_early(config);
|
||||||
|
if (dry_manifest) {
|
||||||
|
if (send_delete_manifest(client->file_descriptor, dry_manifest, dry_excluded, dry_size_skipped,
|
||||||
|
NULL, dry_dirs) != 0)
|
||||||
|
goto dry_fail;
|
||||||
|
if (early_delete) {
|
||||||
|
Status ack;
|
||||||
|
if (!receive_status_keepalive(client->file_descriptor, &ack, DELETE_ACK_TIMEOUT_SEC,
|
||||||
|
DELETE_ACK_KEEPALIVE_SEC, client_abort_pending) ||
|
||||||
|
ack != STATUS_OK)
|
||||||
|
goto dry_fail;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
/* Terminate the stream so the receiver emits its success frame; no data frame
|
||||||
|
is ever sent in dry-run. */
|
||||||
|
if (!send_status(client->file_descriptor, STATUS_FINISHED))
|
||||||
|
goto dry_fail;
|
||||||
|
Status status;
|
||||||
|
if (!receive_status(client->file_descriptor, &status))
|
||||||
|
goto dry_fail;
|
||||||
|
if (status == STATUS_STATS) {
|
||||||
|
ArrayList* would_delete = array_list_create(free);
|
||||||
|
if (!would_delete)
|
||||||
|
goto dry_fail;
|
||||||
|
if (!receive_stats_record(client->file_descriptor, &dry_stats, would_delete)) {
|
||||||
|
array_list_delete(would_delete);
|
||||||
|
goto dry_fail;
|
||||||
|
}
|
||||||
|
print_delete_reports(config, would_delete);
|
||||||
|
array_list_delete(would_delete);
|
||||||
|
if (!receive_status(client->file_descriptor, &status))
|
||||||
|
goto dry_fail;
|
||||||
|
}
|
||||||
|
if (status != STATUS_OK)
|
||||||
|
goto dry_fail;
|
||||||
|
if (!config->quiet) {
|
||||||
|
if (config->human_readable)
|
||||||
|
printf("Total: %d files, %s\n", file_count,
|
||||||
|
display_bytes(total_bytes, true, size_buffer, sizeof(size_buffer)));
|
||||||
|
else
|
||||||
|
printf("Total: %d files, %.1f MB\n", file_count, (double)total_bytes / (double)BYTES_PER_MIB);
|
||||||
|
}
|
||||||
|
{
|
||||||
|
TransferStats dry_transfer;
|
||||||
|
memset(&dry_transfer, 0, sizeof(dry_transfer));
|
||||||
|
dry_transfer.flist_reg = (unsigned long long)file_count;
|
||||||
|
dry_transfer.total_file_size = total_bytes;
|
||||||
|
dry_transfer.transferred_regular = (unsigned long long)file_count;
|
||||||
|
dry_transfer.transferred_file_size = total_bytes;
|
||||||
|
dry_transfer.literal_data = total_bytes;
|
||||||
|
report_transfer_stats(config, &dry_transfer, dry_start, &dry_stats);
|
||||||
|
}
|
||||||
|
ret = io_error ? 1 : 0;
|
||||||
|
|
||||||
|
dry_fail:
|
||||||
|
if (dry_manifest)
|
||||||
|
array_list_delete(dry_manifest);
|
||||||
|
if (dry_dirs)
|
||||||
|
array_list_delete(dry_dirs);
|
||||||
|
if (dry_excluded)
|
||||||
|
array_list_delete(dry_excluded);
|
||||||
|
if (dry_size_skipped)
|
||||||
|
array_list_delete(dry_size_skipped);
|
||||||
|
if (scanner)
|
||||||
|
directory_scanner_destroy(scanner);
|
||||||
|
prepared_scanner_destroy(&prepared);
|
||||||
|
disconnect_transfer_client(client);
|
||||||
|
protocol_session_unbind();
|
||||||
|
client_set_abort_armed(false);
|
||||||
|
return ret;
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,470 @@
|
|||||||
|
#include "client_send_internal.h"
|
||||||
|
#include "array_list.h"
|
||||||
|
#include "charset.h"
|
||||||
|
#include "config.h"
|
||||||
|
#include "delete_plan.h"
|
||||||
|
#include "file.h"
|
||||||
|
#include "file_list.h"
|
||||||
|
#include "filter.h"
|
||||||
|
#include "hardlink.h"
|
||||||
|
#include "log.h"
|
||||||
|
#include "scanner.h"
|
||||||
|
#include "utils.h"
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
|
#include <sys/stat.h>
|
||||||
|
|
||||||
|
/* Build the scanner options for one scan. Returns false and logs on failure. */
|
||||||
|
bool prepare_scanner(const Config* config, int num_threads, PreparedScanner* out) {
|
||||||
|
if (!out)
|
||||||
|
return false;
|
||||||
|
out->base_filters = NULL;
|
||||||
|
out->hardlinks = NULL;
|
||||||
|
out->relative_prefix = NULL;
|
||||||
|
memset(&out->options, 0, sizeof(out->options));
|
||||||
|
|
||||||
|
int rule_count = config->filters ? config->filters->size : 0;
|
||||||
|
const char** texts = NULL;
|
||||||
|
if (rule_count > 0) {
|
||||||
|
texts = malloc((size_t)rule_count * sizeof(char*));
|
||||||
|
if (!texts) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "memory allocation failed for filter rules");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
for (int i = 0; i < rule_count; i++)
|
||||||
|
texts[i] = (const char*)config->filters->items[i];
|
||||||
|
}
|
||||||
|
if (rule_count > 0 || config->cvs_exclude) {
|
||||||
|
char err[160];
|
||||||
|
out->base_filters = filter_base_build(texts, rule_count, config->cvs_exclude,
|
||||||
|
config->delete_excluded, err, sizeof(err));
|
||||||
|
free(texts);
|
||||||
|
if (!out->base_filters) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "invalid filter rule: %s", err);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
free(texts);
|
||||||
|
}
|
||||||
|
|
||||||
|
ScannerOptions* options = &out->options;
|
||||||
|
options->use_metadata = config->use_metadata;
|
||||||
|
options->preserve_atimes = config->preserve_atimes;
|
||||||
|
options->preserve_crtimes = config->preserve_crtimes;
|
||||||
|
options->preserve_xattrs = config->preserve_xattrs;
|
||||||
|
options->preserve_acls = config->preserve_acls;
|
||||||
|
options->chunk_size = config->chunk_size;
|
||||||
|
/* --exclude/--include are compiled, in command-line order, into the SAME
|
||||||
|
* ordered filter rule list as --filter/-f (see config_add_selection_rule), so
|
||||||
|
* the legacy per-kind arrays are deliberately NOT passed to the scanner:
|
||||||
|
* doing so would re-apply them with the old "excludes first, then includes as
|
||||||
|
* a mandatory whitelist" precedence and defeat rsync's first-match-wins
|
||||||
|
* ordering. The arrays remain populated purely for the Config API surface. */
|
||||||
|
options->exclude_patterns = NULL;
|
||||||
|
options->exclude_count = 0;
|
||||||
|
options->include_patterns = NULL;
|
||||||
|
options->include_count = 0;
|
||||||
|
options->max_size = config->max_size;
|
||||||
|
options->min_size = config->min_size;
|
||||||
|
options->max_depth = config->max_depth;
|
||||||
|
options->num_threads = num_threads;
|
||||||
|
options->follow_symlinks = config->follow_symlinks;
|
||||||
|
options->copy_links = config->copy_links;
|
||||||
|
options->safe_links = config->safe_links;
|
||||||
|
options->copy_unsafe_links = config->copy_unsafe_links;
|
||||||
|
options->copy_dirlinks = config->copy_dirlinks;
|
||||||
|
options->munge_links = config->munge_links;
|
||||||
|
options->checksum = config->checksum;
|
||||||
|
options->one_file_system = config->one_file_system;
|
||||||
|
options->preserve_devices = config->preserve_devices;
|
||||||
|
options->preserve_specials = config->preserve_specials;
|
||||||
|
options->copy_devices = config->copy_devices;
|
||||||
|
options->file_list = (const FileListSet*)config->files_from_set;
|
||||||
|
options->base_filters = out->base_filters;
|
||||||
|
options->per_dir_filters = config->per_dir_filter;
|
||||||
|
options->delete_excluded = config->delete_excluded;
|
||||||
|
options->exclude_per_dir_filter_files = config->per_dir_filter_count >= 2;
|
||||||
|
options->dirs = config->dirs;
|
||||||
|
options->relative = config->relative;
|
||||||
|
/* A real recursive transfer recreates empty source directories (rsync
|
||||||
|
parity); low-level scanner users leave this off. */
|
||||||
|
options->emit_empty_dirs = true;
|
||||||
|
/* --no-implied-dirs only has meaning with -R (rsync): without it the option
|
||||||
|
is a documented no-op, so the scanner must not suppress directory
|
||||||
|
metadata. */
|
||||||
|
options->no_implied_dirs = config->no_implied_dirs && config->relative;
|
||||||
|
/* -R/--relative outside --files-from reconstructs every destination path from
|
||||||
|
* the source spec (rsync's '/./' cut point). With --files-from the listed
|
||||||
|
* entry already supplies the bare relative path, so no prefix is built. */
|
||||||
|
if (config->relative && config->files_from_set == NULL && config->send_directory) {
|
||||||
|
out->relative_prefix = scanner_relative_prefix(config->send_directory);
|
||||||
|
if (!out->relative_prefix) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "memory allocation failed building --relative path prefix");
|
||||||
|
filter_rule_list_free(out->base_filters);
|
||||||
|
out->base_filters = NULL;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
options->relative_prefix = out->relative_prefix;
|
||||||
|
}
|
||||||
|
options->prune_empty_dirs = config->prune_empty_dirs;
|
||||||
|
options->ignore_io_errors = config->ignore_errors;
|
||||||
|
options->ignore_missing_args = config->ignore_missing_args || config->delete_missing_args;
|
||||||
|
options->note_nonreg = (config->info_level & LOG_INFO_NONREG) != 0 && !config->quiet;
|
||||||
|
options->note_mount = (config->info_level & LOG_INFO_MOUNT) != 0 && !config->quiet;
|
||||||
|
options->send_directory = config->send_directory;
|
||||||
|
options->eight_bit_output = config->eight_bit_output;
|
||||||
|
options->excluded_paths = NULL;
|
||||||
|
options->excluded_mutex = NULL;
|
||||||
|
options->size_skipped_paths = NULL;
|
||||||
|
options->synced_dirs = NULL;
|
||||||
|
options->hardlinks = NULL;
|
||||||
|
/* Set by the real send paths; NULL for the metadata-only scans (progress
|
||||||
|
pre-count, batch) that must not perturb the sender's --stats counter. */
|
||||||
|
options->dir_count = NULL;
|
||||||
|
/* P7 Wave D: capture source directory metadata when a directory attribute is
|
||||||
|
requested (-p for modes, -t for times unless -O omits them). Whether they
|
||||||
|
are APPLIED is decided receiver-side. */
|
||||||
|
options->capture_dir_times = dir_metadata_should_capture(config);
|
||||||
|
options->dir_entries = NULL;
|
||||||
|
options->dir_entries_mutex = NULL;
|
||||||
|
if (config->preserve_hard_links) {
|
||||||
|
out->hardlinks = hardlink_table_create();
|
||||||
|
if (!out->hardlinks) {
|
||||||
|
filter_rule_list_free(out->base_filters);
|
||||||
|
out->base_filters = NULL;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
options->hardlinks = out->hardlinks;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
void prepared_scanner_destroy(PreparedScanner* prepared) {
|
||||||
|
if (!prepared)
|
||||||
|
return;
|
||||||
|
filter_rule_list_free(prepared->base_filters);
|
||||||
|
prepared->base_filters = NULL;
|
||||||
|
hardlink_table_destroy(prepared->hardlinks);
|
||||||
|
prepared->hardlinks = NULL;
|
||||||
|
free(prepared->relative_prefix);
|
||||||
|
prepared->relative_prefix = NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* -R/--relative implied directories: rsync transmits the metadata of the
|
||||||
|
* parent directories implied by the source path (every prefix component above
|
||||||
|
* the source root) so the receiver applies their attributes to the created
|
||||||
|
* parents. FastSync's scan only covers the source root and below, so append
|
||||||
|
* one metadata-only directory entry per implied ancestor. --no-implied-dirs
|
||||||
|
* suppresses this exactly like rsync. A missing ancestor is never fatal. */
|
||||||
|
bool append_implied_dir_times(const Config* config, ArrayList* dir_entries) {
|
||||||
|
if (!dir_entries || !config->relative || config->files_from_set != NULL ||
|
||||||
|
config->no_implied_dirs || !config->send_directory)
|
||||||
|
return true;
|
||||||
|
char* prefix = scanner_relative_prefix(config->send_directory);
|
||||||
|
if (!prefix)
|
||||||
|
return true;
|
||||||
|
int ncomp = 0;
|
||||||
|
for (const char* s = prefix; *s;) {
|
||||||
|
while (*s == '/')
|
||||||
|
s++;
|
||||||
|
if (!*s)
|
||||||
|
break;
|
||||||
|
while (*s && *s != '/')
|
||||||
|
s++;
|
||||||
|
ncomp++;
|
||||||
|
}
|
||||||
|
if (ncomp <= 1) {
|
||||||
|
free(prefix);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
char* fs = str_dup(config->send_directory);
|
||||||
|
if (!fs) {
|
||||||
|
free(prefix);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
size_t flen = strlen(fs);
|
||||||
|
while (flen > 1 && fs[flen - 1] == '/')
|
||||||
|
fs[--flen] = '\0';
|
||||||
|
bool ok = true;
|
||||||
|
/* Walk the source path upwards one component at a time (fs is truncated in
|
||||||
|
place, so each step targets the next implied ancestor). */
|
||||||
|
for (int depth = ncomp - 2; depth >= 0 && ok; depth--) {
|
||||||
|
char* slash = strrchr(fs, '/');
|
||||||
|
if (!slash || slash == fs)
|
||||||
|
break;
|
||||||
|
*slash = '\0';
|
||||||
|
char* p = prefix;
|
||||||
|
int c = 0;
|
||||||
|
while (c <= depth) {
|
||||||
|
while (*p == '/')
|
||||||
|
p++;
|
||||||
|
while (*p && *p != '/')
|
||||||
|
p++;
|
||||||
|
c++;
|
||||||
|
}
|
||||||
|
char saved = *p;
|
||||||
|
*p = '\0';
|
||||||
|
struct stat st;
|
||||||
|
if (stat(fs, &st) == 0 && S_ISDIR(st.st_mode)) {
|
||||||
|
File* file = file_create(fs);
|
||||||
|
if (!file) {
|
||||||
|
ok = false;
|
||||||
|
} else {
|
||||||
|
file->is_dir = true;
|
||||||
|
file->metadata =
|
||||||
|
file_metadata_create(fs, &st, config->preserve_atimes, config->preserve_crtimes);
|
||||||
|
file->send_path = str_dup(prefix);
|
||||||
|
if (!file->metadata || !file->send_path || !array_list_add(dir_entries, file)) {
|
||||||
|
file_destroy(file);
|
||||||
|
ok = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
*p = saved;
|
||||||
|
}
|
||||||
|
free(fs);
|
||||||
|
free(prefix);
|
||||||
|
return ok;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* The delete-walk root scope for a full (non---files-from) transfer: rsync
|
||||||
|
* confines --delete to the directories it actually transferred. A plain
|
||||||
|
* recursive run mirrors the source under the receive root, so "." (the whole
|
||||||
|
* tree) is correct; an -R run transfers only the reconstructed prefix subtree,
|
||||||
|
* so the walk is scoped to that prefix instead. Returns a malloc'd wire path
|
||||||
|
* (or "."), or NULL on allocation failure. */
|
||||||
|
char* delete_scope_root_marker(const Config* config) {
|
||||||
|
if (config->relative && config->files_from_set == NULL && config->send_directory) {
|
||||||
|
char* prefix = scanner_relative_prefix(config->send_directory);
|
||||||
|
if (!prefix)
|
||||||
|
return NULL;
|
||||||
|
if (prefix[0] != '\0')
|
||||||
|
return prefix;
|
||||||
|
free(prefix);
|
||||||
|
}
|
||||||
|
return str_dup(".");
|
||||||
|
}
|
||||||
|
|
||||||
|
/* The -R destination prefix that confines a per-directory delete walk, or NULL
|
||||||
|
* when the whole receive root is in scope. The marker was installed into
|
||||||
|
* `synced_dirs` by delete_scope_root_marker(); for a plain recursive transfer
|
||||||
|
* it is "." (whole root) and for --files-from the list is not a single prefix. */
|
||||||
|
const char* delete_plan_walk_root(const Config* config, const ArrayList* synced_dirs) {
|
||||||
|
if (!config || config->files_from_set != NULL || !config->relative || !config->send_directory)
|
||||||
|
return NULL;
|
||||||
|
if (!synced_dirs || synced_dirs->size != 1)
|
||||||
|
return NULL;
|
||||||
|
const char* marker = (const char*)synced_dirs->items[0];
|
||||||
|
if (marker[0] == '\0' || strcmp(marker, ".") == 0)
|
||||||
|
return NULL;
|
||||||
|
return marker;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* The destination-relative mirror path for a missing --files-from entry: where
|
||||||
|
a PRESENT entry with the same name would have been written. With -R that is
|
||||||
|
the entry's bare relative path (the bare wire path the receiver uses);
|
||||||
|
otherwise it is the full source mirror below the destination root
|
||||||
|
(`send_directory` joined to the entry, leading '/' stripped), exactly the
|
||||||
|
path the manifest records for a present sibling. Returns an owned string, or
|
||||||
|
NULL on allocation failure. */
|
||||||
|
static char* files_from_missing_dest_path(const Config* config, const char* entry) {
|
||||||
|
if (config->relative)
|
||||||
|
return str_dup(entry);
|
||||||
|
char* joined = path_cat(config->send_directory, entry);
|
||||||
|
if (!joined)
|
||||||
|
return NULL;
|
||||||
|
const char* rel = *joined == '/' ? joined + 1 : joined;
|
||||||
|
char* dup = str_dup(rel);
|
||||||
|
free(joined);
|
||||||
|
return dup;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* --files-from semantics: every listed entry must resolve under the source
|
||||||
|
* root, otherwise rsync reports a hard error instead of silently transferring
|
||||||
|
* nothing. An entry of "." (the whole tree) and listed-but-empty directories
|
||||||
|
* are valid. An empty list is valid too: rsync transfers nothing and exits 0.
|
||||||
|
* With --ignore-missing-args
|
||||||
|
* (implied by --delete-missing-args) a listed-but-missing entry is instead
|
||||||
|
* skipped: nothing is transferred for it, it never enters the keep-set and the
|
||||||
|
* run succeeds for the rest (an all-missing non-empty list succeeds
|
||||||
|
* transferring nothing, matching rsync). With --delete-missing-args
|
||||||
|
* `missing_dest` (when non-NULL) collects the entry's destination-relative
|
||||||
|
* mirror for the receiver's exact-deletion request. Runs before any
|
||||||
|
* transfer so the failure/skip is surfaced uniformly in the single-threaded,
|
||||||
|
* -m, dry-run and --list-only paths. */
|
||||||
|
bool files_from_list_check(const Config* config, ArrayList* missing_dest, int* skipped_out) {
|
||||||
|
*skipped_out = 0;
|
||||||
|
const FileListSet* set = (const FileListSet*)config->files_from_set;
|
||||||
|
if (!set)
|
||||||
|
return true;
|
||||||
|
if (!config->send_directory) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "--files-from requires a source directory");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (set->count == 0) {
|
||||||
|
/* rsync treats an empty --files-from list as "nothing to transfer" and
|
||||||
|
exits 0 (the source directory is still a valid source arg), so this is
|
||||||
|
not an error. Nothing passes the (empty) allow-set, so no file is sent
|
||||||
|
and no keep-set entry is produced. */
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
bool ignore = config->ignore_missing_args || config->delete_missing_args;
|
||||||
|
for (int i = 0; i < set->count; i++) {
|
||||||
|
const char* entry = set->entries[i];
|
||||||
|
if (entry[0] == '\0')
|
||||||
|
continue; /* "." == list the whole tree */
|
||||||
|
char* full = path_cat(config->send_directory, entry);
|
||||||
|
if (!full) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "memory allocation failed while validating --files-from");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
struct stat st;
|
||||||
|
if (lstat(full, &st) != 0) {
|
||||||
|
free(full);
|
||||||
|
if (ignore) {
|
||||||
|
(*skipped_out)++;
|
||||||
|
char* escaped_entry = output_escape(entry, log_get_8_bit_output());
|
||||||
|
log_info_message(LOG_INFO_MISC, "skipping missing --files-from entry '%s'",
|
||||||
|
escaped_entry ? escaped_entry : "<allocation failed>");
|
||||||
|
free(escaped_entry);
|
||||||
|
if (config->delete_missing_args && missing_dest) {
|
||||||
|
char* mirror = files_from_missing_dest_path(config, entry);
|
||||||
|
if (!mirror || !array_list_add(missing_dest, mirror)) {
|
||||||
|
free(mirror);
|
||||||
|
log_message(LOG_LEVEL_ERROR, "memory allocation failed while validating --files-from");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
char* escaped_entry = output_escape(entry, log_get_8_bit_output());
|
||||||
|
char* escaped_src = output_escape(config->send_directory, log_get_8_bit_output());
|
||||||
|
log_message(LOG_LEVEL_ERROR, "--files-from entry '%s' not found in source '%s'",
|
||||||
|
escaped_entry ? escaped_entry : "<allocation failed>",
|
||||||
|
escaped_src ? escaped_src : "<allocation failed>");
|
||||||
|
free(escaped_entry);
|
||||||
|
free(escaped_src);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
free(full);
|
||||||
|
}
|
||||||
|
if (*skipped_out > 0) {
|
||||||
|
if (config->delete_missing_args) {
|
||||||
|
/* --list-only never deletes and a --dry-run only shows intent, so the
|
||||||
|
summary must not claim a real deletion happened in those modes. */
|
||||||
|
if (config->list_only)
|
||||||
|
log_message(LOG_LEVEL_WARNING,
|
||||||
|
"--delete-missing-args: %d missing --files-from entr%s skipped (--list-only "
|
||||||
|
"never deletes)",
|
||||||
|
*skipped_out, *skipped_out == 1 ? "y" : "ies");
|
||||||
|
else if (config->dry_run)
|
||||||
|
log_message(LOG_LEVEL_WARNING,
|
||||||
|
"--delete-missing-args: %d missing --files-from entr%s would be deleted from "
|
||||||
|
"the destination (dry run)",
|
||||||
|
*skipped_out, *skipped_out == 1 ? "y" : "ies");
|
||||||
|
else
|
||||||
|
log_message(
|
||||||
|
LOG_LEVEL_WARNING,
|
||||||
|
"--delete-missing-args: %d missing --files-from entr%s will be deleted from the "
|
||||||
|
"destination",
|
||||||
|
*skipped_out, *skipped_out == 1 ? "y" : "ies");
|
||||||
|
} else if (config->ignore_missing_args)
|
||||||
|
log_message(LOG_LEVEL_WARNING,
|
||||||
|
"--ignore-missing-args: ignored %d missing --files-from entr%s", *skipped_out,
|
||||||
|
*skipped_out == 1 ? "y" : "ies");
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Walk the whole source tree once collecting only destination-relative wire
|
||||||
|
paths, loading and sending nothing. --delete-before/--delete-during need the
|
||||||
|
complete keep-set manifest before the first data byte, so it is built by a
|
||||||
|
dedicated pre-scan pass and transmitted early; the data pass then re-scans
|
||||||
|
with a fresh scanner. --delete-before additionally replays this very scan as
|
||||||
|
its data pass (rsync's single file list), so `chunks_out` (optional) retains
|
||||||
|
the scanned Chunk objects for the caller to send instead of destroying them;
|
||||||
|
the caller owns the list and must give it a chunk_destroy destructor. A
|
||||||
|
source I/O error is fatal unless the options carry --ignore-errors, in which
|
||||||
|
case the scan continues past the unreadable directory and *io_error_out
|
||||||
|
reports it (the caller still performs the deletion but reports the run as
|
||||||
|
errored). */
|
||||||
|
bool scan_paths_only(const Config* config, const ScannerOptions* options, ArrayList* manifest,
|
||||||
|
DeletePlanSender* plans, bool* io_error_out,
|
||||||
|
unsigned long long* non_dir_count_out, ArrayList* chunks_out,
|
||||||
|
bool emit_nonreg) {
|
||||||
|
if (io_error_out)
|
||||||
|
*io_error_out = false;
|
||||||
|
if (non_dir_count_out)
|
||||||
|
*non_dir_count_out = 0;
|
||||||
|
ScannerOptions local = *options;
|
||||||
|
/* The pre-scan is normally a paths-only pass with no client output: it must
|
||||||
|
not emit --info=nonreg lines because the data pass re-scans and emits them
|
||||||
|
once. When the caller replays this scan as the data pass (--delete-before)
|
||||||
|
there is no later scan, so it opts in and the lines are emitted here. */
|
||||||
|
local.note_nonreg = emit_nonreg && options->note_nonreg;
|
||||||
|
DirectoryScanner* scanner = directory_scanner_create_with_options(config->send_directory, &local);
|
||||||
|
if (!scanner)
|
||||||
|
return false;
|
||||||
|
bool ok = true;
|
||||||
|
Chunk* chunk;
|
||||||
|
while ((chunk = directory_scanner_next(scanner)) != NULL) {
|
||||||
|
if (non_dir_count_out) {
|
||||||
|
for (int i = 0; i < chunk->element_count; i++) {
|
||||||
|
const File* f = chunk->items[i];
|
||||||
|
if (f && !f->is_dir)
|
||||||
|
(*non_dir_count_out)++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (manifest && !add_chunk_to_manifest(manifest, chunk)) {
|
||||||
|
ok = false;
|
||||||
|
chunk_destroy(chunk);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if (plans) {
|
||||||
|
for (int i = 0; i < chunk->element_count; i++) {
|
||||||
|
File* f = chunk->items[i];
|
||||||
|
if (!f)
|
||||||
|
continue;
|
||||||
|
const char* path = file_wire_path(f);
|
||||||
|
if (!delete_plan_sender_add(plans, path, f->is_dir)) {
|
||||||
|
ok = false;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!ok) {
|
||||||
|
chunk_destroy(chunk);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (chunks_out) {
|
||||||
|
/* Retain the chunk for the caller's data pass; ownership moves with it. */
|
||||||
|
if (!array_list_add(chunks_out, chunk)) {
|
||||||
|
ok = false;
|
||||||
|
chunk_destroy(chunk);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
chunk_destroy(chunk);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (ok) {
|
||||||
|
/* Keep every traversed source directory, including empty ones, so a plan
|
||||||
|
no longer removes the destination directory itself. Their own plans are
|
||||||
|
emitted after the data stream (no file frame triggers them). */
|
||||||
|
if (plans && options->plan_dirs) {
|
||||||
|
for (int i = 0; i < options->plan_dirs->size; i++) {
|
||||||
|
if (!delete_plan_sender_add(plans, (const char*)options->plan_dirs->items[i], true)) {
|
||||||
|
ok = false;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (ok && directory_scanner_failed(scanner))
|
||||||
|
ok = false;
|
||||||
|
if (io_error_out)
|
||||||
|
*io_error_out = directory_scanner_had_io_error(scanner);
|
||||||
|
directory_scanner_destroy(scanner);
|
||||||
|
return ok;
|
||||||
|
}
|
||||||
+775
-1247
File diff suppressed because it is too large
Load Diff
@@ -22,7 +22,13 @@ void client_set_abort_armed(bool armed);
|
|||||||
* never free it, and the caller retains ownership (freeing it with
|
* never free it, and the caller retains ownership (freeing it with
|
||||||
* config_delete() once the call returns). */
|
* config_delete() once the call returns). */
|
||||||
int send_files(Config* config);
|
int send_files(Config* config);
|
||||||
int send_files_multithreaded(Config** config);
|
int send_files_multithreaded(Config* config);
|
||||||
|
/* rsync's --ignore-errors deletion gate: with no I/O error during the scan the
|
||||||
|
* deletion phase always proceeds; with one it is suppressed unless
|
||||||
|
* `--ignore-errors` was given. Exposed so the decision can be unit-tested
|
||||||
|
* without a privileged (mode-000) source directory. See client_send.c. */
|
||||||
|
bool ignore_errors_allows_delete(const Config* config, bool had_io_error);
|
||||||
|
|
||||||
/* Phase 6 residual-batch (client-only). See client_send.c. */
|
/* Phase 6 residual-batch (client-only). See client_send.c. */
|
||||||
int write_batch_from_source(const Config* config, const char* batch_path);
|
int write_batch_from_source(const Config* config, const char* batch_path);
|
||||||
int apply_batch_to_dest(const Config* config, const char* batch_path, const char* dest_root);
|
int apply_batch_to_dest(const Config* config, const char* batch_path, const char* dest_root);
|
||||||
|
|||||||
@@ -0,0 +1,95 @@
|
|||||||
|
#ifndef CLIENT_SEND_INTERNAL_H
|
||||||
|
#define CLIENT_SEND_INTERNAL_H
|
||||||
|
|
||||||
|
/* Declarations shared between the client_send.c transfer orchestration and the
|
||||||
|
* reporting (client_report.c), scanner-preparation (client_scan.c) and
|
||||||
|
* manifest/list/dry-run (client_manifest.c) translation units that were split
|
||||||
|
* out of it. Nothing here is part of the public client_send.h facade. */
|
||||||
|
|
||||||
|
#include "array_list.h"
|
||||||
|
#include "client_send.h"
|
||||||
|
#include "config.h"
|
||||||
|
#include "delete_plan.h"
|
||||||
|
#include "delta.h"
|
||||||
|
#include "format.h"
|
||||||
|
#include "log.h"
|
||||||
|
#include "scanner.h"
|
||||||
|
#include <stdatomic.h>
|
||||||
|
#include <stdbool.h>
|
||||||
|
#include <stddef.h>
|
||||||
|
#include <time.h>
|
||||||
|
|
||||||
|
/* One mebibyte in bytes; the unit used by the --stats/--progress lines.
|
||||||
|
Always cast to double when dividing so the output stays fractional. */
|
||||||
|
#define BYTES_PER_MIB (1024ULL * 1024ULL)
|
||||||
|
|
||||||
|
/* Compiled scanner inputs that are shared read-only across scanner instances
|
||||||
|
* and, in -m mode, across worker threads. `base_filters` owns the compiled
|
||||||
|
* command-line + -C rules; the FileListSet allow-set lives in the Config.
|
||||||
|
* `hardlinks` owns the --hard-links/-H link-group detection table (NULL when
|
||||||
|
* off) and is shared (mutex-guarded) across every scanner/worker of one scan. */
|
||||||
|
typedef struct {
|
||||||
|
ScannerOptions options;
|
||||||
|
FilterRuleList* base_filters; /* owned; may be NULL */
|
||||||
|
HardLinkTable* hardlinks; /* owned; may be NULL */
|
||||||
|
char* relative_prefix; /* owned -R prefix; may be NULL */
|
||||||
|
} PreparedScanner;
|
||||||
|
|
||||||
|
/* client_scan.c */
|
||||||
|
bool prepare_scanner(const Config* config, int num_threads, PreparedScanner* out);
|
||||||
|
void prepared_scanner_destroy(PreparedScanner* prepared);
|
||||||
|
bool append_implied_dir_times(const Config* config, ArrayList* dir_entries);
|
||||||
|
char* delete_scope_root_marker(const Config* config);
|
||||||
|
const char* delete_plan_walk_root(const Config* config, const ArrayList* synced_dirs);
|
||||||
|
bool files_from_list_check(const Config* config, ArrayList* missing_dest, int* skipped_out);
|
||||||
|
bool scan_paths_only(const Config* config, const ScannerOptions* options, ArrayList* manifest,
|
||||||
|
DeletePlanSender* plans, bool* io_error_out,
|
||||||
|
unsigned long long* non_dir_count_out, ArrayList* chunks_out,
|
||||||
|
bool emit_nonreg);
|
||||||
|
|
||||||
|
/* client_report.c */
|
||||||
|
void log_server_rejection(const char* context);
|
||||||
|
const char* display_bytes(unsigned long long bytes, bool human_readable, char* buffer,
|
||||||
|
size_t buffer_size);
|
||||||
|
unsigned long long dir_count_for_stats(const Config* config, const ArrayList* dir_entries,
|
||||||
|
atomic_ullong* counter);
|
||||||
|
void report_transfer_stats(const Config* config, const TransferStats* stats, time_t start,
|
||||||
|
const ReceiverStats* recv);
|
||||||
|
void transfer_stats_note_entry(TransferStats* stats, const File* file);
|
||||||
|
void transfer_stats_note_transferred(TransferStats* stats, const File* file);
|
||||||
|
bool info_flag_enabled(const Config* config, LogInfoFlag flag);
|
||||||
|
void print_delete_reports(const Config* config, const ArrayList* paths);
|
||||||
|
const char* delete_display_path(const Config* config, const char* path);
|
||||||
|
bool progress_requested(const Config* config);
|
||||||
|
void client_progress_cleanup(void);
|
||||||
|
void client_progress_begin(const Config* config);
|
||||||
|
void client_progress_file(const Config* config, const File* file);
|
||||||
|
void client_progress_name(const Config* config, const File* file);
|
||||||
|
/* Emit a transferred entry's ancestor directories (as -i/--out-format change
|
||||||
|
* lines or --progress name lines) before the entry's own line. */
|
||||||
|
void client_change_emit_ancestors(const Config* config, const File* file);
|
||||||
|
void client_progress_uptodate(const Config* config, const File* file);
|
||||||
|
void client_progress_prepare(const Config* config, const ArrayList* plan_dirs,
|
||||||
|
unsigned long long plan_non_dir_count);
|
||||||
|
bool receive_stats_record(int fd, ReceiverStats* stats, ArrayList* would_delete);
|
||||||
|
|
||||||
|
/* client_send.c */
|
||||||
|
void receive_daemon_motd(Client* client, const Config* config);
|
||||||
|
Client* connect_transfer_client(const Config* config);
|
||||||
|
void disconnect_transfer_client(Client* client);
|
||||||
|
int incremental_check(Client* client, File* file, const Config* config, DeltaSignature** out_sig,
|
||||||
|
unsigned long long* resume_offset);
|
||||||
|
|
||||||
|
/* client_manifest.c */
|
||||||
|
bool dry_run_targets_server(const Config* config);
|
||||||
|
bool add_chunk_to_manifest(ArrayList* manifest, const Chunk* chunk);
|
||||||
|
int send_dry_run_manifest(const Config* config);
|
||||||
|
int send_list_only(const Config* config);
|
||||||
|
int send_dry_run_remote(Config* config);
|
||||||
|
int send_delete_manifest(int fd, ArrayList* manifest, ArrayList* protected_prefixes,
|
||||||
|
ArrayList* size_skipped, ArrayList* missing_args, ArrayList* synced_dirs);
|
||||||
|
bool send_delete_manifest_early(Client* client, ArrayList* manifest, ArrayList* protected_prefixes,
|
||||||
|
ArrayList* size_skipped, ArrayList* missing_args,
|
||||||
|
ArrayList* synced_dirs);
|
||||||
|
|
||||||
|
#endif
|
||||||
@@ -53,18 +53,35 @@ bool validate_config(const Config* config) {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
if (config->compression_threads > 0 && !config->use_compression) {
|
if (config->compression_threads > 0 && !config->use_compression) {
|
||||||
log_message(LOG_LEVEL_ERROR, "--compress-threads requires compression (-c or -z)");
|
log_message(LOG_LEVEL_ERROR, "--compress-threads requires compression (-z/--compress)");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
if (config->transport == TRANSPORT_SSH && config->use_sendfile) {
|
if (config->transport == TRANSPORT_SSH && config->use_sendfile) {
|
||||||
log_message(LOG_LEVEL_ERROR, "-f/--sendfile is not supported with SSH transport");
|
log_message(LOG_LEVEL_ERROR, "-f/--sendfile is not supported with SSH transport");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
/* -M/--remote-option appends an option to the REMOTE server's argv, which
|
||||||
|
* only exists on the SSH (user@host:path) transport. A daemon
|
||||||
|
* (host::module/path) or local TCP destination has no remote command line,
|
||||||
|
* so the option would be silently ignored; reject it by name instead. */
|
||||||
|
if (config->remote_option_count > 0 && config->transport != TRANSPORT_SSH) {
|
||||||
|
log_message(LOG_LEVEL_ERROR,
|
||||||
|
"-M/--remote-option is only valid with the SSH transport (user@host:path); it "
|
||||||
|
"cannot be used with a daemon (host::module/path) or local TCP destination");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
/* -4 and -6 are mutually exclusive: a socket address family cannot be both. */
|
/* -4 and -6 are mutually exclusive: a socket address family cannot be both. */
|
||||||
if (config->ipv4 && config->ipv6) {
|
if (config->ipv4 && config->ipv6) {
|
||||||
log_message(LOG_LEVEL_ERROR, "-4/--ipv4 and -6/--ipv6 are mutually exclusive");
|
log_message(LOG_LEVEL_ERROR, "-4/--ipv4 and -6/--ipv6 are mutually exclusive");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
/* rsync 3.4.1 rejects --inplace together with --partial-dir (exit 1): the
|
||||||
|
inplace write path bypasses partial staging, so a partial-dir name would be
|
||||||
|
silently ignored. Match rsync's message and refuse before any I/O. */
|
||||||
|
if (config->inplace && config->partial_dir) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "--inplace cannot be used with --partial-dir");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
if (config->log_file_format && !config->log_file) {
|
if (config->log_file_format && !config->log_file) {
|
||||||
log_message(LOG_LEVEL_ERROR, "--log-file-format requires --log-file");
|
log_message(LOG_LEVEL_ERROR, "--log-file-format requires --log-file");
|
||||||
return false;
|
return false;
|
||||||
@@ -92,6 +109,16 @@ bool validate_config(const Config* config) {
|
|||||||
log_message(LOG_LEVEL_ERROR, "%s", invariants_error);
|
log_message(LOG_LEVEL_ERROR, "%s", invariants_error);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
/* The receiver rejects a protect-rule block with more than MAX_FILTER_RULES
|
||||||
|
entries as an opaque protocol error; reject an over-limit --filter set here,
|
||||||
|
before any network I/O, with an actionable message. send_protect_entries()
|
||||||
|
re-checks the final built count because cvs-exclude / merge rules can
|
||||||
|
expand it beyond config->filters->size. */
|
||||||
|
if (config->filters && config->filters->size > MAX_FILTER_RULES) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "too many filter rules: %d (maximum %d)", config->filters->size,
|
||||||
|
MAX_FILTER_RULES);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
/* --protocol: FastSync has exactly one wire format, so the forced version
|
/* --protocol: FastSync has exactly one wire format, so the forced version
|
||||||
must equal the current PROTOCOL_VERSION exactly. Rejected here, before any
|
must equal the current PROTOCOL_VERSION exactly. Rejected here, before any
|
||||||
network I/O, rather than letting the server hit its own mismatch check. */
|
network I/O, rather than letting the server hit its own mismatch check. */
|
||||||
|
|||||||
+669
-1125
File diff suppressed because it is too large
Load Diff
+113
-16
@@ -10,6 +10,7 @@
|
|||||||
#include "stop_condition.h"
|
#include "stop_condition.h"
|
||||||
#include <dirent.h>
|
#include <dirent.h>
|
||||||
#include <stdbool.h>
|
#include <stdbool.h>
|
||||||
|
#include <stddef.h>
|
||||||
#include <stdatomic.h>
|
#include <stdatomic.h>
|
||||||
#include <sys/types.h>
|
#include <sys/types.h>
|
||||||
#include <threads.h>
|
#include <threads.h>
|
||||||
@@ -50,7 +51,7 @@ typedef struct {
|
|||||||
bool copy_dirlinks;
|
bool copy_dirlinks;
|
||||||
bool munge_links;
|
bool munge_links;
|
||||||
bool checksum;
|
bool checksum;
|
||||||
bool one_file_system;
|
int one_file_system;
|
||||||
/* Phase 4 special/devices: whether device nodes (--devices) and special files
|
/* Phase 4 special/devices: whether device nodes (--devices) and special files
|
||||||
* (--specials) are preserved via recreation, and whether --copy-devices
|
* (--specials) are preserved via recreation, and whether --copy-devices
|
||||||
* copies a device's content as an ordinary regular file. */
|
* copies a device's content as an ordinary regular file. */
|
||||||
@@ -63,8 +64,23 @@ typedef struct {
|
|||||||
const FileListSet* file_list; /* --files-from allow-set, or NULL */
|
const FileListSet* file_list; /* --files-from allow-set, or NULL */
|
||||||
const FilterRuleList* base_filters; /* command-line + -C rules, or NULL */
|
const FilterRuleList* base_filters; /* command-line + -C rules, or NULL */
|
||||||
bool per_dir_filters; /* -F: read .rsync-filter per directory */
|
bool per_dir_filters; /* -F: read .rsync-filter per directory */
|
||||||
bool dirs; /* -d/--dirs: transfer dir entries, no recursion */
|
/* --delete-excluded: per-directory plain rules become sender-only, so they no
|
||||||
bool relative; /* -R/--relative (dest rel paths, with --files-from) */
|
longer protect the receiver from deletion. */
|
||||||
|
bool delete_excluded;
|
||||||
|
/* -FF: also exclude the per-directory filter files themselves from the
|
||||||
|
transfer (single -F transfers them). */
|
||||||
|
bool exclude_per_dir_filter_files;
|
||||||
|
bool dirs; /* -d/--dirs: transfer dir entries, no recursion */
|
||||||
|
bool relative; /* -R/--relative (dest rel paths, with --files-from) */
|
||||||
|
/* -R/--relative outside --files-from: the destination-relative path prefix
|
||||||
|
* reconstructed from the source spec (rsync's '/./' cut point), or NULL when
|
||||||
|
* -R is off or --files-from is in use (the bare-relative path then comes from
|
||||||
|
* the listed entry). Borrowed read-only; owned by client_send. */
|
||||||
|
const char* relative_prefix;
|
||||||
|
/* --list-only: emit an is_dir File for every traversed directory (the listing
|
||||||
|
* includes directory entries, matching rsync). Client-only; never set on a
|
||||||
|
* real transfer, which relies on implicit parent creation. */
|
||||||
|
bool list_dirs;
|
||||||
/* --prune-empty-dirs (long only): in --dirs mode an empty source directory's
|
/* --prune-empty-dirs (long only): in --dirs mode an empty source directory's
|
||||||
explicit entry is omitted from the transfer file list (so nothing is
|
explicit entry is omitted from the transfer file list (so nothing is
|
||||||
created at the destination and it can be pruned by --delete); explicitly
|
created at the destination and it can be pruned by --delete); explicitly
|
||||||
@@ -73,20 +89,62 @@ typedef struct {
|
|||||||
bool prune_empty_dirs;
|
bool prune_empty_dirs;
|
||||||
/* Delete-excluded protection sink (optional): when non-NULL the scanner
|
/* Delete-excluded protection sink (optional): when non-NULL the scanner
|
||||||
* appends the destination-relative path of every entry it prunes because a
|
* appends the destination-relative path of every entry it prunes because a
|
||||||
* USER SELECTION rule excluded it (--filter/-C/per-dir rules, the legacy
|
* USER SELECTION rule excluded it (--filter/-C/per-dir rules and the legacy
|
||||||
* --exclude/--include layer, and --max-size/--min-size). The sender turns
|
* --exclude/--include layer). The sender turns this list into the manifest's
|
||||||
* this list into the manifest's protected prefixes so `--delete` leaves the
|
* protected prefixes so `--delete` leaves the destination mirror of excluded
|
||||||
* destination mirror of excluded source paths alone (rsync's default), and
|
* source paths alone (rsync's default), and drops it when --delete-excluded
|
||||||
* empties it when --delete-excluded opts back into deleting them. NOT
|
* opts back into deleting them. NOT recorded for --files-from subset pruning
|
||||||
* recorded for --files-from subset pruning (whose delete semantics stay
|
* (whose delete semantics derive from the synchronized-directory set) or for
|
||||||
* keep-set-only) or for -R/--files-from relative wire paths. When
|
* -R/--files-from relative wire paths. When `excluded_mutex` is non-NULL it
|
||||||
* `excluded_mutex` is non-NULL it is taken around every append (the parallel
|
* is taken around every append (the parallel scanner shares one list across
|
||||||
* scanner shares one list across its worker threads). */
|
* its worker threads). */
|
||||||
ArrayList* excluded_paths;
|
ArrayList* excluded_paths;
|
||||||
mtx_t* excluded_mutex;
|
mtx_t* excluded_mutex;
|
||||||
/* --ignore-errors: an unreadable directory during the scan is recorded as an
|
/* Size-prune protection sink (optional): when non-NULL the scanner appends
|
||||||
* I/O error and skipped instead of aborting the scan. Client-only. */
|
* the destination-relative path of every entry it skipped because of
|
||||||
|
* --max-size/--min-size. rsync never deletes a size-skipped source mirror,
|
||||||
|
* even under --delete-excluded, so the sender always transmits this list as
|
||||||
|
* protected prefixes (unlike excluded_paths, which --delete-excluded drops).
|
||||||
|
* Guarded by `excluded_mutex` like excluded_paths. */
|
||||||
|
ArrayList* size_skipped_paths;
|
||||||
|
/* Synchronized-directory sink (optional): when non-NULL the scanner appends
|
||||||
|
* the destination-relative path of every directory it is about to traverse
|
||||||
|
* that lies inside a --files-from listed directory (or of every traversed
|
||||||
|
* directory when there is no list). The sender sends this set with the delete
|
||||||
|
* manifest so the receiver confines its extras walk to synchronized
|
||||||
|
* directories, exactly like rsync; the receive root is the "." sentinel.
|
||||||
|
* Guarded by `excluded_mutex`. */
|
||||||
|
ArrayList* synced_dirs;
|
||||||
|
/* Delete-plan directory sink (optional): when non-NULL the scanner appends
|
||||||
|
* the destination-relative path of every directory it traverses (except the
|
||||||
|
* receive root). The per-directory --delete-during/--delete-delay plan
|
||||||
|
* builder uses this to keep an empty in-scope source directory (rsync keeps
|
||||||
|
* it) and to emit its plan after the data stream, when no file frame would
|
||||||
|
* otherwise trigger it. Guarded by `excluded_mutex`. */
|
||||||
|
ArrayList* plan_dirs;
|
||||||
|
/* --ignore-errors: an unreadable subdirectory no longer aborts the scan (it
|
||||||
|
* is always skipped so the rest of the tree transfers); this flag is kept so
|
||||||
|
* the client can distinguish the option state when deciding deletion policy.
|
||||||
|
* Client-only. */
|
||||||
bool ignore_io_errors;
|
bool ignore_io_errors;
|
||||||
|
/* --info=nonreg: print rsync's `skipping non-regular file "NAME"` line for a
|
||||||
|
* non-regular entry that is not being preserved. Client-only. */
|
||||||
|
bool note_nonreg;
|
||||||
|
/* --info=mount: print rsync's `[sender] skipping mount-point dir NAME` when
|
||||||
|
* -xx drops a mount-point directory. Client-only. */
|
||||||
|
bool note_mount;
|
||||||
|
/* --stats directory accounting for a `-r` run (no -t/-p): a shared counter of
|
||||||
|
* traversed directories that are NOT otherwise represented by an inline
|
||||||
|
* directory entry (rsync still counts every directory in `Number of files`).
|
||||||
|
* Incremented when a directory is opened and decremented when an empty
|
||||||
|
* directory is emitted inline (so it is counted exactly once). Atomic
|
||||||
|
* because the parallel scanner's workers share it; NULL disables the
|
||||||
|
* accounting. Client-only. */
|
||||||
|
atomic_ullong* dir_count;
|
||||||
|
/* Source root and 8-bit-output policy used to render a `--info=nonreg` name
|
||||||
|
* relative to the transfer root. Borrowed read-only. */
|
||||||
|
const char* send_directory;
|
||||||
|
bool eight_bit_output;
|
||||||
/* --ignore-missing-args (implied by --delete-missing-args): an explicitly
|
/* --ignore-missing-args (implied by --delete-missing-args): an explicitly
|
||||||
* --files-from-listed entry that does not exist under the source is skipped
|
* --files-from-listed entry that does not exist under the source is skipped
|
||||||
* instead of failing (the --dirs generator is the only scanner path that
|
* instead of failing (the --dirs generator is the only scanner path that
|
||||||
@@ -114,6 +172,17 @@ typedef struct {
|
|||||||
bool capture_dir_times;
|
bool capture_dir_times;
|
||||||
ArrayList* dir_entries;
|
ArrayList* dir_entries;
|
||||||
mtx_t* dir_entries_mutex;
|
mtx_t* dir_entries_mutex;
|
||||||
|
/* Recreate empty source directories on a recursive transfer: emit a
|
||||||
|
* payload-less directory entry for every traversed directory that produced
|
||||||
|
* no transferred/descended child. Off by default so low-level scanner users
|
||||||
|
* (unit helpers, --list-only) see only the historical file list; the real
|
||||||
|
* sender sets it in prepare_scanner. */
|
||||||
|
bool emit_empty_dirs;
|
||||||
|
/* --no-implied-dirs with -R + --files-from: a directory that is only an
|
||||||
|
* implied parent of a listed entry (not itself listed, nor below a listed
|
||||||
|
* directory) must not carry source metadata; it is created with default
|
||||||
|
* attributes at the destination, matching rsync. */
|
||||||
|
bool no_implied_dirs;
|
||||||
} ScannerOptions;
|
} ScannerOptions;
|
||||||
|
|
||||||
/* Internal per-scanner filter state. FilterNode chains represent the ordered
|
/* Internal per-scanner filter state. FilterNode chains represent the ordered
|
||||||
@@ -131,6 +200,22 @@ typedef struct {
|
|||||||
int current_depth;
|
int current_depth;
|
||||||
dev_t root_dev;
|
dev_t root_dev;
|
||||||
bool failed;
|
bool failed;
|
||||||
|
/* rsync-order traversal: each opened directory's entries are inspected once
|
||||||
|
and buffered (an internal SortedEntry[] owned here) sorted as rsync's flist
|
||||||
|
orders them -- non-directories ascending, then directories ascending. The
|
||||||
|
entries are walked in order and child directories are collected in
|
||||||
|
`pending_dirs` (an ArrayList of DirEntry*, owned here) and pushed onto the
|
||||||
|
LIFO `directories` stack in reverse at directory exhaustion, so the emitted
|
||||||
|
stream is depth-first like rsync. `sorted_*` are reset per directory. */
|
||||||
|
void* sorted_entries;
|
||||||
|
size_t sorted_count;
|
||||||
|
size_t sorted_index;
|
||||||
|
void* pending_dirs;
|
||||||
|
/* Recursive scan: whether the open directory yielded any transferred or
|
||||||
|
descended entry. When it did not, closing it emits a directory entry so
|
||||||
|
the empty source directory is recreated at the destination (rsync
|
||||||
|
parity). */
|
||||||
|
bool current_dir_produced;
|
||||||
/* Phase 2 (files-from / filter layer). */
|
/* Phase 2 (files-from / filter layer). */
|
||||||
char* root_path; /* transfer root (fs path) for rel computation */
|
char* root_path; /* transfer root (fs path) for rel computation */
|
||||||
char* current_rel; /* rel path of the open directory ("" == root) */
|
char* current_rel; /* rel path of the open directory ("" == root) */
|
||||||
@@ -149,6 +234,10 @@ typedef struct {
|
|||||||
--ignore-errors the scan continues past it and the caller decides what to
|
--ignore-errors the scan continues past it and the caller decides what to
|
||||||
do; `failed` is reserved for fatal errors that always abort the scan. */
|
do; `failed` is reserved for fatal errors that always abort the scan. */
|
||||||
bool io_error;
|
bool io_error;
|
||||||
|
/* The transfer ROOT could not be opened. It is always fatal, even under
|
||||||
|
--ignore-errors, but the client still maps it to rsync's partial-transfer
|
||||||
|
exit (23) rather than a generic failure. */
|
||||||
|
bool root_io_error;
|
||||||
} DirectoryScanner;
|
} DirectoryScanner;
|
||||||
|
|
||||||
typedef struct {
|
typedef struct {
|
||||||
@@ -168,7 +257,8 @@ typedef struct {
|
|||||||
int completed;
|
int completed;
|
||||||
Chunk* initial_chunk;
|
Chunk* initial_chunk;
|
||||||
ProtocolSession* allocation_session;
|
ProtocolSession* allocation_session;
|
||||||
FilterNode* root_filter_node; /* root .rsync-filter context (owned by ps) */
|
FilterNode* root_filter_node; /* root .rsync-filter context (owned by ps) */
|
||||||
|
const ScannerOptions* options; /* borrowed scan options (--info=nonreg output) */
|
||||||
} ParallelScanner;
|
} ParallelScanner;
|
||||||
|
|
||||||
DirectoryScanner* directory_scanner_create(const char* root_directory, bool use_metadata,
|
DirectoryScanner* directory_scanner_create(const char* root_directory, bool use_metadata,
|
||||||
@@ -187,13 +277,20 @@ void directory_scanner_destroy(DirectoryScanner* scanner);
|
|||||||
/* --one-file-system (-x) decision: a directory entry may be descended into
|
/* --one-file-system (-x) decision: a directory entry may be descended into
|
||||||
* only when the option is disabled or the entry lives on the same device as
|
* only when the option is disabled or the entry lives on the same device as
|
||||||
* the transfer root. Exposed so tests can exercise the rule directly. */
|
* the transfer root. Exposed so tests can exercise the rule directly. */
|
||||||
bool scanner_same_filesystem(bool one_file_system, dev_t root_device, dev_t entry_device);
|
bool scanner_same_filesystem(int one_file_system, dev_t root_device, dev_t entry_device);
|
||||||
|
|
||||||
/* Relative path of an on-disk path below `root` ("" == the root itself, NULL
|
/* Relative path of an on-disk path below `root` ("" == the root itself, NULL
|
||||||
* when `fs_path` is not under `root`). Handles trailing slashes and a root of
|
* when `fs_path` is not under `root`). Handles trailing slashes and a root of
|
||||||
* "/". Exposed so tests can exercise the mapping directly. */
|
* "/". Exposed so tests can exercise the mapping directly. */
|
||||||
char* scanner_path_relative(const char* root, const char* fs_path);
|
char* scanner_path_relative(const char* root, const char* fs_path);
|
||||||
|
|
||||||
|
/* -R/--relative destination-relative prefix reconstructed from a source spec:
|
||||||
|
* the path after rsync's first '.' path component (the '/./' cut point), with
|
||||||
|
* leading/trailing slashes removed, or the whole spec (normalized) when there
|
||||||
|
* is no cut. Returns "" for the receive root, or NULL when `spec` is NULL or
|
||||||
|
* allocation fails. Exposed so tests can exercise the mapping directly. */
|
||||||
|
char* scanner_relative_prefix(const char* spec);
|
||||||
|
|
||||||
ParallelScanner* parallel_scanner_create_with_options(const char* root_directory,
|
ParallelScanner* parallel_scanner_create_with_options(const char* root_directory,
|
||||||
const ScannerOptions* options,
|
const ScannerOptions* options,
|
||||||
ProtocolSession* allocation_session);
|
ProtocolSession* allocation_session);
|
||||||
|
|||||||
@@ -0,0 +1,675 @@
|
|||||||
|
#include "log.h"
|
||||||
|
#include "scanner.h"
|
||||||
|
#include "scanner_internal.h"
|
||||||
|
#include "array_list.h"
|
||||||
|
#include "chunk.h"
|
||||||
|
#include "file.h"
|
||||||
|
#include "queue.h"
|
||||||
|
#include "utils.h"
|
||||||
|
#include <dirent.h>
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
|
#include <sys/stat.h>
|
||||||
|
#include <sys/sysmacros.h>
|
||||||
|
#include <threads.h>
|
||||||
|
#include <unistd.h>
|
||||||
|
#include <limits.h>
|
||||||
|
|
||||||
|
#include "xattr.h"
|
||||||
|
|
||||||
|
/* A chain node: `own` holds the .rsync-filter rules of one directory, `parent`
|
||||||
|
* the context that directory inherited (nearest ancestor with a filter file).
|
||||||
|
* The chain for a directory's contents runs from that directory's own node up
|
||||||
|
* to the root; the command-line base rules are evaluated after the whole
|
||||||
|
* chain. */
|
||||||
|
struct FilterNode {
|
||||||
|
FilterNode* parent;
|
||||||
|
FilterRuleList* own;
|
||||||
|
};
|
||||||
|
|
||||||
|
void filter_node_destroy(void* item) {
|
||||||
|
if (item) {
|
||||||
|
FilterNode* node = (FilterNode*)item;
|
||||||
|
if (node->own)
|
||||||
|
filter_rule_list_free(node->own);
|
||||||
|
free(node);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
FilterNode* filter_node_alloc(FilterNode* parent, FilterRuleList* own) {
|
||||||
|
FilterNode* node = malloc(sizeof(FilterNode));
|
||||||
|
if (!node)
|
||||||
|
return NULL;
|
||||||
|
node->parent = parent;
|
||||||
|
node->own = own;
|
||||||
|
return node;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Evaluate a rule chain for one entry. rsync precedence, highest first: the
|
||||||
|
* innermost (current) directory's .rsync-filter rules, then each ancestor's,
|
||||||
|
* then the root's, and finally the command-line base rules (--filter/-C). The
|
||||||
|
* sender-side verdict decides whether the entry is hidden from the transfer;
|
||||||
|
* the receiver-side verdict decides whether its destination mirror is protected
|
||||||
|
* from --delete. Each side takes the FIRST matching rule independently. */
|
||||||
|
typedef struct {
|
||||||
|
bool hide; /* sender-side exclude matched */
|
||||||
|
bool protect; /* receiver-side exclude matched */
|
||||||
|
} FilterOutcome;
|
||||||
|
|
||||||
|
static void chain_rules_outcome(const FilterRuleList* base, const FilterNode* node, const char* rel,
|
||||||
|
const char* leaf, bool is_dir, FilterOutcome* out) {
|
||||||
|
memset(out, 0, sizeof(*out));
|
||||||
|
bool sender_decided = false;
|
||||||
|
bool receiver_decided = false;
|
||||||
|
const FilterNode* n = node;
|
||||||
|
while (!sender_decided || !receiver_decided) {
|
||||||
|
const FilterRuleList* list = n ? n->own : base;
|
||||||
|
if (list) {
|
||||||
|
if (!sender_decided) {
|
||||||
|
FilterAction action = filter_rules_apply_side(list, rel, leaf, is_dir, FILTER_SIDE_SENDER);
|
||||||
|
if (action != FILTER_ACTION_NONE) {
|
||||||
|
out->hide = action == FILTER_ACTION_EXCLUDE;
|
||||||
|
sender_decided = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!receiver_decided) {
|
||||||
|
FilterAction action =
|
||||||
|
filter_rules_apply_side(list, rel, leaf, is_dir, FILTER_SIDE_RECEIVER);
|
||||||
|
if (action != FILTER_ACTION_NONE) {
|
||||||
|
out->protect = action == FILTER_ACTION_PROTECT;
|
||||||
|
receiver_decided = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!n)
|
||||||
|
break;
|
||||||
|
n = n->parent;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool entry_allowed(const FilterRuleList* base, const FilterNode* node, const char* rel,
|
||||||
|
const char* leaf, bool is_dir, bool exclude_filter_files,
|
||||||
|
bool* protect_out) {
|
||||||
|
/* -FF: per-directory .rsync-filter files are never transferred (single -F
|
||||||
|
transfers them, matching rsync). */
|
||||||
|
if (exclude_filter_files && !is_dir && strcmp(leaf, ".rsync-filter") == 0) {
|
||||||
|
if (protect_out)
|
||||||
|
*protect_out = false;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
FilterOutcome outcome;
|
||||||
|
chain_rules_outcome(base, node, rel, leaf, is_dir, &outcome);
|
||||||
|
if (protect_out)
|
||||||
|
*protect_out = outcome.protect;
|
||||||
|
return !outcome.hide;
|
||||||
|
}
|
||||||
|
|
||||||
|
void dir_entry_destroy(void* item) {
|
||||||
|
if (item) {
|
||||||
|
DirEntry* de = (DirEntry*)item;
|
||||||
|
free(de->path);
|
||||||
|
free(de);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
DirEntry* dir_entry_create(const char* path, int depth, FilterNode* context) {
|
||||||
|
DirEntry* de = malloc(sizeof(DirEntry));
|
||||||
|
if (!de)
|
||||||
|
return NULL;
|
||||||
|
de->path = str_dup(path);
|
||||||
|
if (!de->path) {
|
||||||
|
free(de);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
de->depth = depth;
|
||||||
|
de->context = context;
|
||||||
|
return de;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Apply rsync's symlink-resolution precedence to one S_ISLNK entry:
|
||||||
|
* --copy-links dereferences every symlink;
|
||||||
|
* --copy-unsafe-links dereferences only targets unsafe_symlink() flags;
|
||||||
|
* -k/--copy-dirlinks dereferences only a symlink whose referent is a dir;
|
||||||
|
* --safe-links (receiver-side in rsync; modelled here) ignores an unsafe
|
||||||
|
* target that would otherwise be carried; with --munge-links
|
||||||
|
* every stored target becomes absolute, so --safe-links then
|
||||||
|
* ignores every symlink, exactly as rsync documents;
|
||||||
|
* -l/--links carries the link.
|
||||||
|
* `link_rel` is the symlink's transfer-relative path (incl. name) and is used
|
||||||
|
* only for the lexical unsafe test. `target` receives the raw link value. */
|
||||||
|
LinkAction scanner_link_action(const ScannerOptions* options, const char* path,
|
||||||
|
const char* link_rel, char* target, size_t target_size) {
|
||||||
|
if (!options->follow_symlinks && !options->copy_links && !options->safe_links &&
|
||||||
|
!options->copy_unsafe_links && !options->copy_dirlinks)
|
||||||
|
return LINK_ACTION_SKIP;
|
||||||
|
ssize_t length = readlink(path, target, target_size - 1);
|
||||||
|
if (length < 0)
|
||||||
|
return LINK_ACTION_SKIP;
|
||||||
|
target[length] = '\0';
|
||||||
|
|
||||||
|
bool unsafe = file_symlink_unsafe(target, link_rel);
|
||||||
|
if (options->copy_links || (options->copy_unsafe_links && unsafe))
|
||||||
|
return LINK_ACTION_DEREF;
|
||||||
|
if (options->copy_dirlinks) {
|
||||||
|
struct stat ref;
|
||||||
|
if (stat(path, &ref) == 0 && S_ISDIR(ref.st_mode))
|
||||||
|
return LINK_ACTION_DEREF;
|
||||||
|
}
|
||||||
|
if (options->safe_links && (unsafe || options->munge_links))
|
||||||
|
return LINK_ACTION_SKIP_PROTECTED;
|
||||||
|
if (!options->follow_symlinks || target[0] == '\0')
|
||||||
|
return LINK_ACTION_SKIP;
|
||||||
|
return LINK_ACTION_CARRY;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* --one-file-system (-x) decision. Only directories can carry a different
|
||||||
|
* device than their parent (mount points), so this is checked when a child
|
||||||
|
* directory is about to be descended into. */
|
||||||
|
bool scanner_same_filesystem(int one_file_system, dev_t root_device, dev_t entry_device) {
|
||||||
|
return one_file_system <= 0 || entry_device == root_device;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Build a payload-less directory File carrying the captured metadata (when
|
||||||
|
* requested). Used by -x mount-point emission and --list-only directory
|
||||||
|
* entries. Returns NULL on allocation failure. */
|
||||||
|
File* scanner_build_dir_file(const char* path, const struct stat* stats,
|
||||||
|
const ScannerOptions* options) {
|
||||||
|
File* dir = file_create(path);
|
||||||
|
if (dir == NULL)
|
||||||
|
return NULL;
|
||||||
|
dir->is_dir = true;
|
||||||
|
if (options->use_metadata) {
|
||||||
|
dir->metadata =
|
||||||
|
file_metadata_create(dir->path, stats, options->preserve_atimes, options->preserve_crtimes);
|
||||||
|
if (!dir->metadata) {
|
||||||
|
file_destroy(dir);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return dir;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Relative path of an on-disk path below `root`. The transfer root may be
|
||||||
|
* given with a trailing slash; the returned rel path never has one and is ""
|
||||||
|
* for the root itself. A root of "/" is handled (its children start at "/").
|
||||||
|
* Exposed so tests can exercise the mapping directly. */
|
||||||
|
char* scanner_path_relative(const char* root, const char* fs_path) {
|
||||||
|
size_t root_len = strlen(root);
|
||||||
|
while (root_len > 1 && root[root_len - 1] == '/')
|
||||||
|
root_len--;
|
||||||
|
if (strncmp(root, fs_path, root_len) != 0)
|
||||||
|
return NULL;
|
||||||
|
if (root_len == 1 && root[0] == '/') {
|
||||||
|
if (fs_path[1] == '\0')
|
||||||
|
return str_dup("");
|
||||||
|
return str_dup(fs_path + 1);
|
||||||
|
}
|
||||||
|
if (fs_path[root_len] == '\0')
|
||||||
|
return str_dup("");
|
||||||
|
if (fs_path[root_len] != '/')
|
||||||
|
return NULL;
|
||||||
|
return str_dup(fs_path + root_len + 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* -R/--relative destination-relative prefix reconstructed from a source spec:
|
||||||
|
* everything after the first '.' path component (rsync's '/./' cut point),
|
||||||
|
* with leading/trailing slashes removed; or the whole spec (normalized) when
|
||||||
|
* there is no cut. Returns "" for the receive root. Exposed for tests. */
|
||||||
|
char* scanner_relative_prefix(const char* spec) {
|
||||||
|
if (!spec || spec[0] == '\0')
|
||||||
|
return NULL;
|
||||||
|
const char* after = spec;
|
||||||
|
if (spec[0] == '.' && spec[1] == '/') {
|
||||||
|
after = spec + 2;
|
||||||
|
} else {
|
||||||
|
const char* cut = strstr(spec, "/./");
|
||||||
|
if (cut)
|
||||||
|
after = cut + 3;
|
||||||
|
}
|
||||||
|
size_t cap = strlen(spec) + 1;
|
||||||
|
char* out = malloc(cap);
|
||||||
|
if (!out)
|
||||||
|
return NULL;
|
||||||
|
size_t len = 0;
|
||||||
|
for (const char* s = after; *s;) {
|
||||||
|
while (*s == '/')
|
||||||
|
s++;
|
||||||
|
const char* comp = s;
|
||||||
|
while (*s && *s != '/')
|
||||||
|
s++;
|
||||||
|
size_t clen = (size_t)(s - comp);
|
||||||
|
if (clen == 0 || (clen == 1 && comp[0] == '.'))
|
||||||
|
continue;
|
||||||
|
if (len)
|
||||||
|
out[len++] = '/';
|
||||||
|
memcpy(out + len, comp, clen);
|
||||||
|
len += clen;
|
||||||
|
}
|
||||||
|
out[len] = '\0';
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Relative path of a child entry below the current directory. */
|
||||||
|
char* child_rel_path(const char* parent_rel, const char* name) {
|
||||||
|
if (!parent_rel || parent_rel[0] == '\0')
|
||||||
|
return str_dup(name);
|
||||||
|
return path_cat(parent_rel, name);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Destination-relative wire path for an entry under an -R prefix. */
|
||||||
|
char* scanner_prefix_send_path(const char* prefix, const char* rel) {
|
||||||
|
if (prefix[0] == '\0')
|
||||||
|
return str_dup(rel);
|
||||||
|
if (rel[0] == '\0')
|
||||||
|
return str_dup(prefix);
|
||||||
|
return path_cat(prefix, rel);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Apply the --files-from allow-set and the filter layer to one entry. On
|
||||||
|
* return `*protect_out` is true when a receiver-side rule protects the entry's
|
||||||
|
* destination mirror from deletion. */
|
||||||
|
bool entry_passes_selection(const FileListSet* file_list, const FilterRuleList* base,
|
||||||
|
const FilterNode* node, const char* rel, const char* leaf, bool is_dir,
|
||||||
|
bool per_dir_filters, bool exclude_filter_files, bool* protect_out) {
|
||||||
|
if (protect_out)
|
||||||
|
*protect_out = false;
|
||||||
|
if (file_list && !file_list_affects(file_list, rel))
|
||||||
|
return false;
|
||||||
|
if (base || per_dir_filters)
|
||||||
|
return entry_allowed(base, node, rel, leaf, is_dir, exclude_filter_files, protect_out);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Best-effort capture of the file's whitelisted xattrs (-X/-A). A failure to
|
||||||
|
* read xattrs is non-fatal: the file is transferred without them. A symlink
|
||||||
|
* entry reads the LINK's own xattrs (never the referent's) with the no-follow
|
||||||
|
* variant; on Linux the VFS refuses xattrs on symlinks, so that yields NULL. */
|
||||||
|
void scanner_capture_xattrs(const DirectoryScanner* scanner, File* file) {
|
||||||
|
if (!scanner || !file || !(scanner->options.preserve_xattrs || scanner->options.preserve_acls))
|
||||||
|
return;
|
||||||
|
file->xattrs = file->is_symlink
|
||||||
|
? xattr_capture_path_nofollow(file->path, scanner->options.preserve_acls)
|
||||||
|
: xattr_capture_path(file->path, scanner->options.preserve_acls);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Apply --hard-links (-H) detection to one regular File. On a sibling (a
|
||||||
|
* later member of an already-seen source inode) the File keeps the group id
|
||||||
|
* and the first member's wire path but carries NO data payload (size 0); the
|
||||||
|
* first member is left untouched (data present, link_first). Allocation
|
||||||
|
* failure is fatal: the scanner is marked failed. */
|
||||||
|
void scanner_assign_hardlink(DirectoryScanner* scanner, HardLinkTable* table, File* file,
|
||||||
|
const struct stat* stats) {
|
||||||
|
if (!table || !file || !stats)
|
||||||
|
return;
|
||||||
|
int gid;
|
||||||
|
bool is_first;
|
||||||
|
char* first_path = NULL;
|
||||||
|
if (!hardlink_table_assign(table, file_wire_path(file), stats->st_dev, stats->st_ino, &gid,
|
||||||
|
&is_first, &first_path)) {
|
||||||
|
if (scanner)
|
||||||
|
scanner->failed = true;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
file->link_group = gid;
|
||||||
|
file->link_first = is_first;
|
||||||
|
if (!is_first) {
|
||||||
|
file->hardlink_target = first_path;
|
||||||
|
file->data->size = 0;
|
||||||
|
} else {
|
||||||
|
free(first_path);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Phase 4 special/devices decision for one non-regular entry, matching rsync:
|
||||||
|
- a char/block device is RECREATED as a node under -D/--devices, unless
|
||||||
|
--copy-devices asks for its content to be copied into a regular file;
|
||||||
|
- a FIFO/socket is RECREATED under --specials;
|
||||||
|
- when the matching flag is absent the entry is SKIPPED ("skipping
|
||||||
|
non-regular file"), exactly like rsync's default, instead of being
|
||||||
|
silently copied as a zero-length regular file;
|
||||||
|
- anything else (regular/directory) is left to the normal data path. */
|
||||||
|
ScannerSpecial scanner_prepare_special(bool preserve_devices, bool preserve_specials,
|
||||||
|
bool copy_devices, File* file, const struct stat* stats) {
|
||||||
|
if (!file || !stats)
|
||||||
|
return SCANNER_SPECIAL_REGULAR;
|
||||||
|
bool is_device = S_ISCHR(stats->st_mode) || S_ISBLK(stats->st_mode);
|
||||||
|
bool is_fifo = S_ISFIFO(stats->st_mode);
|
||||||
|
bool is_socket = S_ISSOCK(stats->st_mode);
|
||||||
|
if (!is_device && !is_fifo && !is_socket)
|
||||||
|
return SCANNER_SPECIAL_REGULAR;
|
||||||
|
if (is_device && copy_devices)
|
||||||
|
return SCANNER_SPECIAL_REGULAR; /* copy device content as a regular file */
|
||||||
|
bool preserve = is_device ? preserve_devices : preserve_specials;
|
||||||
|
if (!preserve)
|
||||||
|
return SCANNER_SPECIAL_SKIP;
|
||||||
|
file->is_special = true;
|
||||||
|
file->data->size = 0;
|
||||||
|
file->data->data = NULL;
|
||||||
|
if (is_device) {
|
||||||
|
file->rdev_major = (int32_t)major(stats->st_rdev);
|
||||||
|
file->rdev_minor = (int32_t)minor(stats->st_rdev);
|
||||||
|
}
|
||||||
|
return SCANNER_SPECIAL_RECREATE;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Append `rel` to the caller's exclusion sink, taking `mtx` when shared across
|
||||||
|
parallel worker threads. Returns false on allocation failure (list left
|
||||||
|
unchanged). */
|
||||||
|
bool excluded_sink_append(ArrayList* list, mtx_t* mtx, const char* rel) {
|
||||||
|
if (!list)
|
||||||
|
return true;
|
||||||
|
char* dup = str_dup(rel);
|
||||||
|
if (!dup)
|
||||||
|
return false;
|
||||||
|
if (mtx)
|
||||||
|
mtx_lock(mtx);
|
||||||
|
bool ok = array_list_add(list, dup);
|
||||||
|
if (mtx)
|
||||||
|
mtx_unlock(mtx);
|
||||||
|
if (!ok)
|
||||||
|
free(dup);
|
||||||
|
return ok;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Record one pruned filesystem path in a delete-protection sink. The stored
|
||||||
|
form is the entry's wire/destination-relative path (a single leading '/'
|
||||||
|
removed, exactly how manifest keep entries are stored), so the receiver's
|
||||||
|
walker prefixes match the destination layout. An allocation failure is a
|
||||||
|
fatal scan error. */
|
||||||
|
static void scanner_record_protected(DirectoryScanner* scanner, const char* fs_path,
|
||||||
|
ArrayList* sink) {
|
||||||
|
if (!sink || !fs_path)
|
||||||
|
return;
|
||||||
|
const char* rel = *fs_path == '/' ? fs_path + 1 : fs_path;
|
||||||
|
if (!excluded_sink_append(sink, scanner->options.excluded_mutex, rel))
|
||||||
|
scanner->failed = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* rsync's `--info=nonreg` line for a non-regular entry that is not being
|
||||||
|
* preserved: `skipping non-regular file "NAME"`. The name is the path relative
|
||||||
|
* to the transfer root, so it matches rsync's displayed name. */
|
||||||
|
void scanner_note_nonreg(const ScannerOptions* options, const char* fs_path) {
|
||||||
|
if (!options || !options->note_nonreg || !fs_path)
|
||||||
|
return;
|
||||||
|
const char* rel = utils_strip_transfer_root(fs_path, options->send_directory);
|
||||||
|
char* escaped = output_escape(rel, options->eight_bit_output);
|
||||||
|
printf("skipping non-regular file \"%s\"\n", escaped ? escaped : rel);
|
||||||
|
free(escaped);
|
||||||
|
fflush(stdout);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* rsync 3.4.1's `--info=mount` line, emitted when `-xx` drops a mount-point
|
||||||
|
* directory: `[sender] skipping mount-point dir NAME` (the client is the
|
||||||
|
* sender). Plain `-x` keeps the empty directory and prints nothing, matching
|
||||||
|
* rsync. */
|
||||||
|
void scanner_note_mount(const ScannerOptions* options, const char* fs_path) {
|
||||||
|
if (!options || !options->note_mount || !fs_path)
|
||||||
|
return;
|
||||||
|
const char* rel = utils_strip_transfer_root(fs_path, options->send_directory);
|
||||||
|
char* escaped = output_escape(rel, options->eight_bit_output);
|
||||||
|
printf("[sender] skipping mount-point dir %s\n", escaped ? escaped : rel);
|
||||||
|
free(escaped);
|
||||||
|
fflush(stdout);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* --debug=filter: a selection/filter decision dropped an entry. */
|
||||||
|
void scanner_note_filter(const ScannerOptions* options, const char* name) {
|
||||||
|
if (!options || !log_debug_enabled(LOG_DEBUG_FILTER) || !name)
|
||||||
|
return;
|
||||||
|
log_debug_message(LOG_DEBUG_FILTER, "filter: excluded %s", name);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Account for a directory that will not be represented by an inline directory
|
||||||
|
* entry. Paired with scanner_dir_count_uncount for empty directories that are
|
||||||
|
* emitted inline, so every traversed directory is counted exactly once. */
|
||||||
|
void scanner_dir_count_count(const ScannerOptions* options) {
|
||||||
|
if (options && options->dir_count)
|
||||||
|
atomic_fetch_add(options->dir_count, 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
void scanner_dir_count_uncount(const ScannerOptions* options) {
|
||||||
|
if (options && options->dir_count)
|
||||||
|
atomic_fetch_sub(options->dir_count, 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* A user-selection exclusion (--filter/-C/per-dir or --exclude/--include). */
|
||||||
|
void scanner_record_excluded(DirectoryScanner* scanner, const char* fs_path) {
|
||||||
|
scanner_record_protected(scanner, fs_path, scanner->options.excluded_paths);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* A --max-size/--min-size prune (always protected, even under --delete-excluded). */
|
||||||
|
void scanner_record_size_skipped(DirectoryScanner* scanner, const char* fs_path) {
|
||||||
|
scanner_record_protected(scanner, fs_path, scanner->options.size_skipped_paths);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Record a directory the scan synchronized. `fs_path` is its absolute path and
|
||||||
|
`rel` its path relative to the transfer root ("" for the root); the stored
|
||||||
|
form matches the wire layout (the bare relative path in -R+--files-from, else
|
||||||
|
the source path with a leading '/' removed, with "." for the receive root).
|
||||||
|
Returns false on allocation failure. */
|
||||||
|
bool scanner_record_synced_dir(const ScannerOptions* options, const char* fs_path, const char* rel,
|
||||||
|
bool relative_mode) {
|
||||||
|
if (!options->synced_dirs && !options->plan_dirs)
|
||||||
|
return true;
|
||||||
|
if (!file_list_dir_in_scope(options->file_list, rel))
|
||||||
|
return true;
|
||||||
|
char* prefixed = NULL;
|
||||||
|
const char* dest;
|
||||||
|
if (relative_mode) {
|
||||||
|
dest = rel;
|
||||||
|
} else if (options->relative_prefix) {
|
||||||
|
prefixed = scanner_prefix_send_path(options->relative_prefix, rel);
|
||||||
|
if (!prefixed)
|
||||||
|
return false;
|
||||||
|
dest = prefixed;
|
||||||
|
} else {
|
||||||
|
dest = fs_path;
|
||||||
|
}
|
||||||
|
if (dest[0] == '/')
|
||||||
|
dest++;
|
||||||
|
if (dest[0] == '\0')
|
||||||
|
dest = ".";
|
||||||
|
bool ok = true;
|
||||||
|
if (options->synced_dirs)
|
||||||
|
ok = excluded_sink_append(options->synced_dirs, options->excluded_mutex, dest);
|
||||||
|
/* The delete-plan keep set needs an entry for every traversed source
|
||||||
|
directory, including empty ones, so its destination mirror is kept rather
|
||||||
|
than deleted as an extra; the receive root (".") is implicit. */
|
||||||
|
if (ok && options->plan_dirs && strcmp(dest, ".") != 0)
|
||||||
|
ok = excluded_sink_append(options->plan_dirs, options->excluded_mutex, dest);
|
||||||
|
free(prefixed);
|
||||||
|
return ok;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Read every per-directory filter file that applies to `dir_path` (its
|
||||||
|
* .rsync-filter when -F is active, plus each registered "dir-merge NAME") into a
|
||||||
|
* fresh list. Returns NULL on allocation/parse failure (message in `err`);
|
||||||
|
* returns an empty list (and *any_exists=false) when no file exists. */
|
||||||
|
FilterRuleList* read_dir_filters(const ScannerOptions* options, const char* dir_path,
|
||||||
|
const char* rel, bool* any_exists, char* err, size_t err_size) {
|
||||||
|
if (err && err_size > 0)
|
||||||
|
err[0] = '\0';
|
||||||
|
const FilterRuleList* base = options->base_filters;
|
||||||
|
bool have_names = options->per_dir_filters || (base && base->dir_merge_count > 0);
|
||||||
|
if (any_exists)
|
||||||
|
*any_exists = false;
|
||||||
|
if (!have_names)
|
||||||
|
return NULL;
|
||||||
|
FilterRuleList* own = filter_rule_list_create();
|
||||||
|
if (!own) {
|
||||||
|
snprintf(err, err_size, "memory allocation failed");
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
FilterParseOptions opts = {.delete_excluded = options->delete_excluded, .cvs_exclude = false};
|
||||||
|
bool exists = false;
|
||||||
|
if (options->per_dir_filters) {
|
||||||
|
if (!filter_file_append(own, dir_path, ".rsync-filter", rel, &opts, &exists, err, err_size))
|
||||||
|
goto fail;
|
||||||
|
if (exists && any_exists)
|
||||||
|
*any_exists = true;
|
||||||
|
}
|
||||||
|
if (base) {
|
||||||
|
for (int i = 0; i < base->dir_merge_count; i++) {
|
||||||
|
if (!filter_file_append(own, dir_path, base->dir_merge_names[i], rel, &opts, &exists, err,
|
||||||
|
err_size))
|
||||||
|
goto fail;
|
||||||
|
if (exists && any_exists)
|
||||||
|
*any_exists = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return own;
|
||||||
|
fail:
|
||||||
|
filter_rule_list_free(own);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Merge the open directory's own per-directory filter files (the default
|
||||||
|
* .rsync-filter when -F is active, plus every "dir-merge NAME" registered on the
|
||||||
|
* base rule list) into the inherited context, returning the context used for
|
||||||
|
* this directory's entries. On a parse error the scanner is marked failed.
|
||||||
|
* Returns 0 on success, -1 on failure. */
|
||||||
|
int open_directory_filter_context(DirectoryScanner* scanner, const FilterNode* inherited) {
|
||||||
|
char err[256];
|
||||||
|
bool any_exists = false;
|
||||||
|
FilterRuleList* own = read_dir_filters(&scanner->options, scanner->current_path,
|
||||||
|
scanner->current_rel ? scanner->current_rel : "",
|
||||||
|
&any_exists, err, sizeof(err));
|
||||||
|
if (!own) {
|
||||||
|
/* read_dir_filters() leaves `err` set on a parse/allocation failure even
|
||||||
|
when an earlier merge file in the same directory existed (any_exists true);
|
||||||
|
key off the error text rather than any_exists so an invalid per-directory
|
||||||
|
filter file can never be silently ignored. */
|
||||||
|
if (err[0] == '\0') {
|
||||||
|
scanner->current_node = (FilterNode*)inherited;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
char* escaped_path = output_escape(scanner->current_path, log_get_8_bit_output());
|
||||||
|
log_message(LOG_LEVEL_ERROR, "invalid per-directory filter in %s: %s",
|
||||||
|
escaped_path ? escaped_path : "<allocation failed>", err);
|
||||||
|
free(escaped_path);
|
||||||
|
scanner->failed = true;
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
if (any_exists && (own->count > 0 || own->dir_merge_count > 0)) {
|
||||||
|
FilterNode* node = filter_node_alloc((FilterNode*)inherited, own);
|
||||||
|
if (!node || !array_list_add(scanner->filter_nodes, node)) {
|
||||||
|
filter_node_destroy(node);
|
||||||
|
scanner->failed = true;
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
scanner->current_node = node;
|
||||||
|
} else {
|
||||||
|
filter_rule_list_free(own);
|
||||||
|
scanner->current_node = (FilterNode*)inherited;
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Inspect symlinks, resolve the entry type, and apply file filters once for both scanners.
|
||||||
|
* `link_rel` is the entry's path relative to the transfer root (including its
|
||||||
|
* name), used for the lexical rsync unsafe-symlink test. */
|
||||||
|
int scanner_inspect_entry(const ScannerOptions* options, const char* containing_dir,
|
||||||
|
const char* link_rel, const char* name, ScannerEntry* entry) {
|
||||||
|
entry->excluded = false;
|
||||||
|
entry->size_excluded = false;
|
||||||
|
entry->referent_error = false;
|
||||||
|
entry->is_symlink = false;
|
||||||
|
entry->link_target = NULL;
|
||||||
|
entry->path = path_cat(containing_dir, name);
|
||||||
|
if (!entry->path)
|
||||||
|
return -1;
|
||||||
|
|
||||||
|
struct stat link_stats;
|
||||||
|
if (lstat(entry->path, &link_stats) != 0) {
|
||||||
|
free(entry->path);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
if (!S_ISLNK(link_stats.st_mode))
|
||||||
|
goto regular;
|
||||||
|
|
||||||
|
char link_target[4096];
|
||||||
|
switch (scanner_link_action(options, entry->path, link_rel, link_target, sizeof(link_target))) {
|
||||||
|
case LINK_ACTION_SKIP:
|
||||||
|
goto skip;
|
||||||
|
case LINK_ACTION_SKIP_PROTECTED:
|
||||||
|
/* --safe-links ignored the link, but rsync still counts it as present in
|
||||||
|
the transfer, so its destination mirror survives --delete. Record it as
|
||||||
|
an excluded path (the same delete-protection channel as a filter prune). */
|
||||||
|
entry->excluded = true;
|
||||||
|
goto skip;
|
||||||
|
case LINK_ACTION_DEREF:
|
||||||
|
if (stat(entry->path, &entry->stats) != 0) {
|
||||||
|
/* rsync reports "symlink has no referent" and continues with a partial
|
||||||
|
transfer (exit 23); record the error so the run exits 23 too. */
|
||||||
|
char* escaped = output_escape(entry->path, log_get_8_bit_output());
|
||||||
|
log_message(LOG_LEVEL_WARNING, "symlink has no referent: %s",
|
||||||
|
escaped ? escaped : "<allocation failed>");
|
||||||
|
free(escaped);
|
||||||
|
entry->referent_error = true;
|
||||||
|
goto skip;
|
||||||
|
}
|
||||||
|
entry->is_directory = S_ISDIR(entry->stats.st_mode);
|
||||||
|
if (entry->is_directory)
|
||||||
|
return 1;
|
||||||
|
goto apply_filters;
|
||||||
|
case LINK_ACTION_CARRY:
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Carry the link as a symlink. --munge-links is applied by the RECEIVER (it
|
||||||
|
prefixes every stored target with /rsyncd-munged/); when the SOURCE already
|
||||||
|
holds a munged value the sender strips it so the receiver re-munges a clean
|
||||||
|
target, round-tripping a munged tree exactly like rsync. */
|
||||||
|
entry->is_symlink = true;
|
||||||
|
entry->stats = link_stats;
|
||||||
|
entry->is_directory = false;
|
||||||
|
entry->link_target = str_dup(link_target);
|
||||||
|
if (!entry->link_target)
|
||||||
|
goto skip;
|
||||||
|
if (options->munge_links)
|
||||||
|
file_symlink_unmunge(entry->link_target);
|
||||||
|
goto apply_filters;
|
||||||
|
|
||||||
|
regular:
|
||||||
|
/* Not a symlink: the lstat() above already described this entry, and lstat
|
||||||
|
and stat are identical for every non-symlink, so reuse that result instead
|
||||||
|
of issuing a redundant stat() on the scanner hot path. stat() is still
|
||||||
|
used on the dereference paths above/below for actual symlinks (copy-links,
|
||||||
|
safe/copy-unsafe links, and -k symlinks-to-directories). */
|
||||||
|
entry->stats = link_stats;
|
||||||
|
entry->is_directory = S_ISDIR(link_stats.st_mode);
|
||||||
|
if (entry->is_directory)
|
||||||
|
return 1;
|
||||||
|
|
||||||
|
apply_filters:
|
||||||
|
for (int i = 0; i < options->exclude_count; i++)
|
||||||
|
if (glob_match(options->exclude_patterns[i], name)) {
|
||||||
|
entry->excluded = true;
|
||||||
|
goto skip;
|
||||||
|
}
|
||||||
|
if (options->include_count > 0) {
|
||||||
|
bool included = false;
|
||||||
|
for (int i = 0; i < options->include_count; i++)
|
||||||
|
if (glob_match(options->include_patterns[i], name))
|
||||||
|
included = true;
|
||||||
|
if (!included) {
|
||||||
|
entry->excluded = true;
|
||||||
|
goto skip;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ((options->max_size > 0 && (unsigned long long)entry->stats.st_size > options->max_size) ||
|
||||||
|
(options->min_size > 0 && (unsigned long long)entry->stats.st_size < options->min_size)) {
|
||||||
|
entry->excluded = true;
|
||||||
|
entry->size_excluded = true;
|
||||||
|
goto skip;
|
||||||
|
}
|
||||||
|
return 1;
|
||||||
|
|
||||||
|
skip:
|
||||||
|
free(entry->path);
|
||||||
|
entry->path = NULL;
|
||||||
|
free(entry->link_target);
|
||||||
|
entry->link_target = NULL;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
@@ -0,0 +1,107 @@
|
|||||||
|
#ifndef SCANNER_INTERNAL_H
|
||||||
|
#define SCANNER_INTERNAL_H
|
||||||
|
|
||||||
|
/* Internal declarations shared between the scanner translation units
|
||||||
|
* (scanner_filter.c, scanner.c, scanner_parallel.c). Nothing here is part of
|
||||||
|
* the public scanner façade (scanner.h); every symbol stays internal to the
|
||||||
|
* client module. */
|
||||||
|
|
||||||
|
#include "array_list.h"
|
||||||
|
#include "file.h"
|
||||||
|
#include "scanner.h"
|
||||||
|
#include <stdbool.h>
|
||||||
|
#include <stddef.h>
|
||||||
|
#include <sys/stat.h>
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
char* path;
|
||||||
|
int depth;
|
||||||
|
FilterNode* context; /* inherited per-directory filter context */
|
||||||
|
} DirEntry;
|
||||||
|
|
||||||
|
/* How rsync's readlink_stat()/generator resolves one source symlink. */
|
||||||
|
typedef enum {
|
||||||
|
LINK_ACTION_SKIP, /* not transferred (no link option) */
|
||||||
|
LINK_ACTION_SKIP_PROTECTED, /* ignored as unsafe by --safe-links; rsync keeps
|
||||||
|
it in the transfer, so its destination mirror
|
||||||
|
must be protected from --delete */
|
||||||
|
LINK_ACTION_DEREF, /* follow the referent (--copy-links, an unsafe
|
||||||
|
target under --copy-unsafe-links, or -k dir) */
|
||||||
|
LINK_ACTION_CARRY, /* transmit the link itself (-l) */
|
||||||
|
} LinkAction;
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
char* path;
|
||||||
|
struct stat stats;
|
||||||
|
bool is_directory;
|
||||||
|
/* True when the entry should be carried through as a SYMLINK (is_symlink)
|
||||||
|
rather than a dereferenced file/directory. When true, `link_target` holds
|
||||||
|
the owned target string to transmit (sender-munged under --munge-links);
|
||||||
|
ownership transfers to the File built from this entry. */
|
||||||
|
bool is_symlink;
|
||||||
|
char* link_target;
|
||||||
|
/* True when the entry was pruned by a user selection rule (--filter/-C/per-dir
|
||||||
|
rules or the --exclude/--include layer) rather than skipped for another
|
||||||
|
reason (unreadable, symlink policy, not applicable). */
|
||||||
|
bool excluded;
|
||||||
|
/* True when the entry was skipped specifically by --max-size/--min-size.
|
||||||
|
Size pruning protects the destination mirror even under --delete-excluded,
|
||||||
|
so it is recorded into a separate sink from `excluded`. */
|
||||||
|
bool size_excluded;
|
||||||
|
/* True when a symlink selected for dereferencing (-L/--copy-links or an
|
||||||
|
unsafe target under --copy-unsafe-links) had no usable referent (a broken
|
||||||
|
link or a stat() failure). rsync still reports this as a partial transfer
|
||||||
|
(exit 23) even though the entry is skipped, so the scanner records it as a
|
||||||
|
non-fatal I/O error. */
|
||||||
|
bool referent_error;
|
||||||
|
} ScannerEntry;
|
||||||
|
|
||||||
|
typedef enum {
|
||||||
|
SCANNER_SPECIAL_REGULAR, /* ordinary file: transfer content */
|
||||||
|
SCANNER_SPECIAL_RECREATE, /* is_special node to recreate on the receiver */
|
||||||
|
SCANNER_SPECIAL_SKIP, /* non-regular entry not requested: skip */
|
||||||
|
} ScannerSpecial;
|
||||||
|
|
||||||
|
/* scanner_filter.c */
|
||||||
|
void filter_node_destroy(void* item);
|
||||||
|
FilterNode* filter_node_alloc(FilterNode* parent, FilterRuleList* own);
|
||||||
|
void dir_entry_destroy(void* item);
|
||||||
|
DirEntry* dir_entry_create(const char* path, int depth, FilterNode* context);
|
||||||
|
LinkAction scanner_link_action(const ScannerOptions* options, const char* path,
|
||||||
|
const char* link_rel, char* target, size_t target_size);
|
||||||
|
File* scanner_build_dir_file(const char* path, const struct stat* stats,
|
||||||
|
const ScannerOptions* options);
|
||||||
|
char* child_rel_path(const char* parent_rel, const char* name);
|
||||||
|
char* scanner_prefix_send_path(const char* prefix, const char* rel);
|
||||||
|
bool entry_passes_selection(const FileListSet* file_list, const FilterRuleList* base,
|
||||||
|
const FilterNode* node, const char* rel, const char* leaf, bool is_dir,
|
||||||
|
bool per_dir_filters, bool exclude_filter_files, bool* protect_out);
|
||||||
|
void scanner_capture_xattrs(const DirectoryScanner* scanner, File* file);
|
||||||
|
void scanner_assign_hardlink(DirectoryScanner* scanner, HardLinkTable* table, File* file,
|
||||||
|
const struct stat* stats);
|
||||||
|
ScannerSpecial scanner_prepare_special(bool preserve_devices, bool preserve_specials,
|
||||||
|
bool copy_devices, File* file, const struct stat* stats);
|
||||||
|
bool excluded_sink_append(ArrayList* list, mtx_t* mtx, const char* rel);
|
||||||
|
void scanner_note_nonreg(const ScannerOptions* options, const char* fs_path);
|
||||||
|
void scanner_note_mount(const ScannerOptions* options, const char* fs_path);
|
||||||
|
void scanner_note_filter(const ScannerOptions* options, const char* name);
|
||||||
|
void scanner_dir_count_count(const ScannerOptions* options);
|
||||||
|
void scanner_dir_count_uncount(const ScannerOptions* options);
|
||||||
|
void scanner_record_excluded(DirectoryScanner* scanner, const char* fs_path);
|
||||||
|
void scanner_record_size_skipped(DirectoryScanner* scanner, const char* fs_path);
|
||||||
|
bool scanner_record_synced_dir(const ScannerOptions* options, const char* fs_path, const char* rel,
|
||||||
|
bool relative_mode);
|
||||||
|
FilterRuleList* read_dir_filters(const ScannerOptions* options, const char* dir_path,
|
||||||
|
const char* rel, bool* any_exists, char* err, size_t err_size);
|
||||||
|
int open_directory_filter_context(DirectoryScanner* scanner, const FilterNode* inherited);
|
||||||
|
int scanner_inspect_entry(const ScannerOptions* options, const char* containing_dir,
|
||||||
|
const char* link_rel, const char* name, ScannerEntry* entry);
|
||||||
|
|
||||||
|
/* scanner.c */
|
||||||
|
bool scanner_capture_dir_time(ArrayList* dir_entries, mtx_t* mutex, const char* root_path,
|
||||||
|
const char* fs_path, bool relative_mode, const char* relative_prefix,
|
||||||
|
bool preserve_atimes, bool preserve_crtimes, bool preserve_xattrs,
|
||||||
|
bool preserve_acls, bool no_implied_dirs,
|
||||||
|
const FileListSet* file_list);
|
||||||
|
|
||||||
|
#endif
|
||||||
@@ -0,0 +1,705 @@
|
|||||||
|
#include "log.h"
|
||||||
|
#include "scanner.h"
|
||||||
|
#include "scanner_internal.h"
|
||||||
|
#include "array_list.h"
|
||||||
|
#include "chunk.h"
|
||||||
|
#include "file.h"
|
||||||
|
#include "queue.h"
|
||||||
|
#include "utils.h"
|
||||||
|
#include <dirent.h>
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
|
#include <sys/stat.h>
|
||||||
|
#include <sys/sysmacros.h>
|
||||||
|
#include <threads.h>
|
||||||
|
#include <unistd.h>
|
||||||
|
#include <limits.h>
|
||||||
|
|
||||||
|
#include "xattr.h"
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
ParallelScanner* ps;
|
||||||
|
char** dirs;
|
||||||
|
int dir_count;
|
||||||
|
char* root_dir; /* the transfer root, for relative-path computation */
|
||||||
|
ScannerOptions options;
|
||||||
|
ProtocolSession* allocation_session;
|
||||||
|
} ParallelWorkerArg;
|
||||||
|
|
||||||
|
static int parallel_worker_thread(void* arg) {
|
||||||
|
ParallelWorkerArg* wa = (ParallelWorkerArg*)arg;
|
||||||
|
ProtocolSession* allocation_session = wa->allocation_session;
|
||||||
|
if (allocation_session)
|
||||||
|
protocol_session_bind(allocation_session);
|
||||||
|
for (int i = 0; i < wa->dir_count; i++) {
|
||||||
|
DirectoryScanner* ds = directory_scanner_create_with_options(wa->dirs[i], &wa->options);
|
||||||
|
if (!ds) {
|
||||||
|
mtx_lock(&wa->ps->result_mutex);
|
||||||
|
wa->ps->failed = true;
|
||||||
|
atomic_store(&wa->ps->cancelled, true);
|
||||||
|
cnd_broadcast(&wa->ps->result_not_empty);
|
||||||
|
cnd_broadcast(&wa->ps->result_not_full);
|
||||||
|
mtx_unlock(&wa->ps->result_mutex);
|
||||||
|
for (int j = i; j < wa->dir_count; j++)
|
||||||
|
free(wa->dirs[j]);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
/* Root .rsync-filter rules (parsed by the parallel scanner) apply to the
|
||||||
|
* contents of every assigned subdirectory. Relative paths (used by the
|
||||||
|
* allow-set and per-directory rules) are computed against the transfer
|
||||||
|
* root, not the subdirectory the worker is seeded with. Exclusion
|
||||||
|
* recording shares one caller-owned list across the workers. */
|
||||||
|
free(ds->root_path);
|
||||||
|
ds->root_path = str_dup(wa->root_dir);
|
||||||
|
ds->seed_node = wa->ps->root_filter_node;
|
||||||
|
ds->options.excluded_mutex = &wa->ps->result_mutex;
|
||||||
|
Chunk* chunk;
|
||||||
|
while ((chunk = directory_scanner_next(ds)) != NULL) {
|
||||||
|
if (!queue_enqueue_multithreaded_cancel(wa->ps->result_queue, chunk, &wa->ps->result_mutex,
|
||||||
|
&wa->ps->result_not_empty, &wa->ps->result_not_full,
|
||||||
|
&wa->ps->cancelled)) {
|
||||||
|
chunk_destroy(chunk);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (directory_scanner_failed(ds)) {
|
||||||
|
mtx_lock(&wa->ps->result_mutex);
|
||||||
|
wa->ps->failed = true;
|
||||||
|
atomic_store(&wa->ps->cancelled, true);
|
||||||
|
cnd_broadcast(&wa->ps->result_not_empty);
|
||||||
|
cnd_broadcast(&wa->ps->result_not_full);
|
||||||
|
mtx_unlock(&wa->ps->result_mutex);
|
||||||
|
} else if (directory_scanner_had_io_error(ds)) {
|
||||||
|
/* --ignore-errors path: an unreadable directory was skipped, not fatal. */
|
||||||
|
mtx_lock(&wa->ps->result_mutex);
|
||||||
|
wa->ps->io_error = true;
|
||||||
|
mtx_unlock(&wa->ps->result_mutex);
|
||||||
|
}
|
||||||
|
directory_scanner_destroy(ds);
|
||||||
|
free(wa->dirs[i]);
|
||||||
|
}
|
||||||
|
ParallelScanner* ps = wa->ps;
|
||||||
|
free(wa->root_dir);
|
||||||
|
free(wa->dirs);
|
||||||
|
free(wa);
|
||||||
|
mtx_lock(&ps->result_mutex);
|
||||||
|
ps->completed++;
|
||||||
|
if (ps->completed >= ps->expected_threads) {
|
||||||
|
ps->done = true;
|
||||||
|
cnd_signal(&ps->result_not_empty);
|
||||||
|
}
|
||||||
|
mtx_unlock(&ps->result_mutex);
|
||||||
|
if (allocation_session)
|
||||||
|
protocol_session_unbind();
|
||||||
|
return thrd_success;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void parallel_scanner_creation_failed(ParallelScanner* ps) {
|
||||||
|
mtx_lock(&ps->result_mutex);
|
||||||
|
ps->failed = true;
|
||||||
|
atomic_store(&ps->cancelled, true);
|
||||||
|
ps->expected_threads = ps->created_threads;
|
||||||
|
if (ps->completed >= ps->expected_threads)
|
||||||
|
ps->done = true;
|
||||||
|
cnd_broadcast(&ps->result_not_empty);
|
||||||
|
cnd_broadcast(&ps->result_not_full);
|
||||||
|
mtx_unlock(&ps->result_mutex);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Initialize result queue and synchronization primitives. Returns true on success. */
|
||||||
|
static bool parallel_scanner_init(ParallelScanner* ps) {
|
||||||
|
ps->result_queue = queue_create(100, chunk_destroy);
|
||||||
|
if (!ps->result_queue)
|
||||||
|
return false;
|
||||||
|
atomic_init(&ps->cancelled, false);
|
||||||
|
int init = 0;
|
||||||
|
bool ok = true;
|
||||||
|
if (mtx_init(&ps->result_mutex, mtx_plain) != thrd_success)
|
||||||
|
ok = false;
|
||||||
|
if (ok) {
|
||||||
|
init++;
|
||||||
|
if (cnd_init(&ps->result_not_empty) != thrd_success)
|
||||||
|
ok = false;
|
||||||
|
}
|
||||||
|
if (ok) {
|
||||||
|
// cppcheck-suppress unreadVariable
|
||||||
|
init++;
|
||||||
|
if (cnd_init(&ps->result_not_full) != thrd_success)
|
||||||
|
ok = false;
|
||||||
|
}
|
||||||
|
if (!ok) {
|
||||||
|
if (init >= 3)
|
||||||
|
cnd_destroy(&ps->result_not_full);
|
||||||
|
if (init >= 2)
|
||||||
|
cnd_destroy(&ps->result_not_empty);
|
||||||
|
if (init >= 1)
|
||||||
|
mtx_destroy(&ps->result_mutex);
|
||||||
|
queue_destroy(ps->result_queue);
|
||||||
|
ps->result_queue = NULL;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Split files into chunks of roughly chunk_size bytes. Returns the first chunk (also stored
|
||||||
|
* chunks beyond the first are enqueued on `queue`). Nulls out consumed entries in `files`.
|
||||||
|
* Sets *failed on allocation/enqueue errors. */
|
||||||
|
static Chunk* batch_files(ArrayList* files, unsigned long long chunk_size, Queue* queue,
|
||||||
|
bool* failed) {
|
||||||
|
Chunk* first = NULL;
|
||||||
|
if (files->size <= 0)
|
||||||
|
return NULL;
|
||||||
|
ArrayList* batch = array_list_create(NULL);
|
||||||
|
if (!batch) {
|
||||||
|
*failed = true;
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
unsigned long long batch_size = 0;
|
||||||
|
for (int i = 0; i < files->size; i++) {
|
||||||
|
File* f = (File*)files->items[i];
|
||||||
|
if (!array_list_add(batch, f)) {
|
||||||
|
*failed = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
batch_size += f->data->size;
|
||||||
|
if (batch_size >= chunk_size || i == files->size - 1) {
|
||||||
|
void** items = array_list_to_array(batch);
|
||||||
|
if (!items) {
|
||||||
|
*failed = true;
|
||||||
|
array_list_delete(batch);
|
||||||
|
batch = NULL;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
Chunk* c = chunk_create((File**)items, batch->size);
|
||||||
|
free(items);
|
||||||
|
if (!c) {
|
||||||
|
*failed = true;
|
||||||
|
array_list_delete(batch);
|
||||||
|
batch = NULL;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
int batch_start = i - batch->size + 1;
|
||||||
|
for (int j = batch_start; j <= i; j++)
|
||||||
|
files->items[j] = NULL;
|
||||||
|
batch->item_destroyer = NULL;
|
||||||
|
array_list_delete(batch);
|
||||||
|
batch = NULL;
|
||||||
|
if (!first) {
|
||||||
|
first = c;
|
||||||
|
} else {
|
||||||
|
if (!queue_enqueue(queue, c)) {
|
||||||
|
chunk_destroy(c);
|
||||||
|
*failed = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (i < files->size - 1) {
|
||||||
|
batch = array_list_create(NULL);
|
||||||
|
if (!batch) {
|
||||||
|
*failed = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
batch_size = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (batch) {
|
||||||
|
batch->item_destroyer = NULL;
|
||||||
|
array_list_delete(batch);
|
||||||
|
}
|
||||||
|
return first;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Scan one root-directory entry into either the subdirs or files list. */
|
||||||
|
static void scan_root_entry(const ScannerOptions* options, const FilterNode* root_node,
|
||||||
|
const char* root_directory, const struct dirent* entry,
|
||||||
|
ArrayList* root_files, ArrayList* subdirs, dev_t root_dev,
|
||||||
|
ParallelScanner* ps) {
|
||||||
|
ScannerEntry inspected;
|
||||||
|
int inspection =
|
||||||
|
scanner_inspect_entry(options, root_directory, entry->d_name, entry->d_name, &inspected);
|
||||||
|
if (inspection < 0) {
|
||||||
|
ps->failed = true;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (inspection == 0) {
|
||||||
|
if (inspected.referent_error)
|
||||||
|
ps->io_error = true;
|
||||||
|
ArrayList* sink = NULL;
|
||||||
|
if (inspected.excluded)
|
||||||
|
sink = inspected.size_excluded ? options->size_skipped_paths : options->excluded_paths;
|
||||||
|
if (sink) {
|
||||||
|
/* A root-level prune protects the destination mirror of the entry's wire
|
||||||
|
path: under -R + --files-from that is the bare relative name, otherwise
|
||||||
|
it is the full source path with a leading '/' removed (matching the
|
||||||
|
send_path/file_wire_path the scanner hands the sender). */
|
||||||
|
if (options->relative && options->file_list != NULL) {
|
||||||
|
if (!excluded_sink_append(sink, options->excluded_mutex, entry->d_name))
|
||||||
|
ps->failed = true;
|
||||||
|
} else if (options->relative_prefix) {
|
||||||
|
char* wrel = scanner_prefix_send_path(options->relative_prefix, entry->d_name);
|
||||||
|
if (!wrel) {
|
||||||
|
ps->failed = true;
|
||||||
|
} else {
|
||||||
|
if (!excluded_sink_append(sink, options->excluded_mutex, wrel))
|
||||||
|
ps->failed = true;
|
||||||
|
free(wrel);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
char* abs_path = path_cat(root_directory, entry->d_name);
|
||||||
|
if (!abs_path) {
|
||||||
|
ps->failed = true;
|
||||||
|
} else {
|
||||||
|
const char* rel = *abs_path == '/' ? abs_path + 1 : abs_path;
|
||||||
|
if (!excluded_sink_append(sink, options->excluded_mutex, rel))
|
||||||
|
ps->failed = true;
|
||||||
|
free(abs_path);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
char* cur_path = inspected.path;
|
||||||
|
struct stat st = inspected.stats;
|
||||||
|
bool is_dir = inspected.is_directory;
|
||||||
|
char* rel = str_dup(entry->d_name);
|
||||||
|
if (!rel) {
|
||||||
|
free(cur_path);
|
||||||
|
ps->failed = true;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
bool protect = false;
|
||||||
|
bool passes = entry_passes_selection(options->file_list, options->base_filters, root_node, rel,
|
||||||
|
entry->d_name, is_dir, options->per_dir_filters,
|
||||||
|
options->exclude_per_dir_filter_files, &protect);
|
||||||
|
/* -R + --files-from: root-level files keep their bare relative send path. */
|
||||||
|
bool use_rel = options->relative && options->file_list != NULL;
|
||||||
|
if (!passes || protect) {
|
||||||
|
/* --files-from subset pruning is not a filter exclusion; -R bare-wire-path
|
||||||
|
exclusions are never recorded (see ScannerOptions.excluded_paths). */
|
||||||
|
bool files_from_prune = options->file_list && !file_list_affects(options->file_list, rel);
|
||||||
|
if ((!files_from_prune && !use_rel) || protect) {
|
||||||
|
const char* rel_path;
|
||||||
|
char* prefixed = NULL;
|
||||||
|
if (use_rel) {
|
||||||
|
/* -R + --files-from: the destination/wire path is the bare relative
|
||||||
|
name, not the source path. */
|
||||||
|
rel_path = rel;
|
||||||
|
} else if (options->relative_prefix) {
|
||||||
|
prefixed = scanner_prefix_send_path(options->relative_prefix, entry->d_name);
|
||||||
|
if (!prefixed) {
|
||||||
|
free(rel);
|
||||||
|
free(cur_path);
|
||||||
|
ps->failed = true;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
rel_path = prefixed;
|
||||||
|
} else {
|
||||||
|
rel_path = *cur_path == '/' ? cur_path + 1 : cur_path;
|
||||||
|
}
|
||||||
|
if (options->excluded_paths &&
|
||||||
|
!excluded_sink_append(options->excluded_paths, options->excluded_mutex, rel_path))
|
||||||
|
ps->failed = true;
|
||||||
|
free(prefixed);
|
||||||
|
}
|
||||||
|
if (!passes) {
|
||||||
|
scanner_note_filter(options, entry->d_name);
|
||||||
|
free(rel);
|
||||||
|
free(cur_path);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (is_dir) {
|
||||||
|
if (!scanner_same_filesystem(options->one_file_system, root_dev, st.st_dev)) {
|
||||||
|
if (options->one_file_system > 1) {
|
||||||
|
/* -xx: drop the mount-point directory entirely (rsync) and print the
|
||||||
|
--info=mount line when enabled. */
|
||||||
|
scanner_note_mount(options, cur_path);
|
||||||
|
free(rel);
|
||||||
|
free(cur_path);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
/* -x/--one-file-system: emit the mount-point directory entry (empty) but
|
||||||
|
do not descend into it (see the sequential scanner for the same rule). */
|
||||||
|
File* mount = file_create(cur_path);
|
||||||
|
free(cur_path);
|
||||||
|
if (mount == NULL) {
|
||||||
|
free(rel);
|
||||||
|
ps->failed = true;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
mount->is_dir = true;
|
||||||
|
if (options->use_metadata) {
|
||||||
|
mount->metadata = file_metadata_create(mount->path, &st, options->preserve_atimes,
|
||||||
|
options->preserve_crtimes);
|
||||||
|
if (!mount->metadata) {
|
||||||
|
free(rel);
|
||||||
|
file_destroy(mount);
|
||||||
|
ps->failed = true;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (options->relative_prefix) {
|
||||||
|
mount->send_path = scanner_prefix_send_path(options->relative_prefix, rel);
|
||||||
|
if (!mount->send_path) {
|
||||||
|
free(rel);
|
||||||
|
file_destroy(mount);
|
||||||
|
ps->failed = true;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
free(rel);
|
||||||
|
if (!array_list_add(root_files, mount)) {
|
||||||
|
file_destroy(mount);
|
||||||
|
ps->failed = true;
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
free(rel);
|
||||||
|
if (!array_list_add(subdirs, cur_path)) {
|
||||||
|
free(cur_path);
|
||||||
|
ps->failed = true;
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
File* file = file_create(cur_path);
|
||||||
|
free(cur_path);
|
||||||
|
if (!file) {
|
||||||
|
free(rel);
|
||||||
|
free(inspected.link_target);
|
||||||
|
inspected.link_target = NULL;
|
||||||
|
ps->failed = true;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (inspected.is_symlink) {
|
||||||
|
file->is_symlink = true;
|
||||||
|
file->symlink_target = inspected.link_target;
|
||||||
|
inspected.link_target = NULL;
|
||||||
|
} else {
|
||||||
|
file->data->size = st.st_size;
|
||||||
|
}
|
||||||
|
if (use_rel) {
|
||||||
|
file->send_path = rel;
|
||||||
|
rel = NULL;
|
||||||
|
} else if (options->relative_prefix) {
|
||||||
|
file->send_path = scanner_prefix_send_path(options->relative_prefix, rel);
|
||||||
|
free(rel);
|
||||||
|
rel = NULL;
|
||||||
|
if (!file->send_path) {
|
||||||
|
file_destroy(file);
|
||||||
|
ps->failed = true;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ScannerSpecial special = scanner_prepare_special(
|
||||||
|
options->preserve_devices, options->preserve_specials, options->copy_devices, file, &st);
|
||||||
|
if (special == SCANNER_SPECIAL_SKIP) {
|
||||||
|
scanner_note_nonreg(ps->options, file->path);
|
||||||
|
free(rel);
|
||||||
|
file_destroy(file);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (options->hardlinks && S_ISREG(st.st_mode)) {
|
||||||
|
int gid;
|
||||||
|
bool is_first;
|
||||||
|
char* first_path = NULL;
|
||||||
|
if (!hardlink_table_assign((HardLinkTable*)options->hardlinks, file_wire_path(file), st.st_dev,
|
||||||
|
st.st_ino, &gid, &is_first, &first_path)) {
|
||||||
|
ps->failed = true;
|
||||||
|
} else {
|
||||||
|
file->link_group = gid;
|
||||||
|
file->link_first = is_first;
|
||||||
|
if (!is_first) {
|
||||||
|
file->hardlink_target = first_path;
|
||||||
|
file->data->size = 0;
|
||||||
|
} else {
|
||||||
|
free(first_path);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (options->use_metadata)
|
||||||
|
file->metadata =
|
||||||
|
file_metadata_create(file->path, &st, options->preserve_atimes, options->preserve_crtimes);
|
||||||
|
if (options->use_metadata && !file->metadata) {
|
||||||
|
free(rel);
|
||||||
|
file_destroy(file);
|
||||||
|
ps->failed = true;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if ((options->preserve_xattrs || options->preserve_acls) &&
|
||||||
|
!(file->link_group != 0 && !file->link_first))
|
||||||
|
file->xattrs = file->is_symlink
|
||||||
|
? xattr_capture_path_nofollow(file->path, options->preserve_acls)
|
||||||
|
: xattr_capture_path(file->path, options->preserve_acls);
|
||||||
|
if (!array_list_add(root_files, file)) {
|
||||||
|
free(rel);
|
||||||
|
file_destroy(file);
|
||||||
|
ps->failed = true;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
free(rel);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Scan the root directory itself, collecting root files and subdirectories.
|
||||||
|
* Returns false if the root directory could not be opened. */
|
||||||
|
static bool scan_root_directory(ParallelScanner* ps, const char* root_directory,
|
||||||
|
const ScannerOptions* options, const FilterNode* root_node,
|
||||||
|
dev_t root_dev, ArrayList* root_files, ArrayList* subdirs) {
|
||||||
|
DIR* dir = opendir(root_directory);
|
||||||
|
if (!dir) {
|
||||||
|
log_perror("Could not open root directory for parallel scan");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
/* The parallel scanner opens the transfer root directly (not through
|
||||||
|
open_next_directory), so record it as synchronized here. */
|
||||||
|
if (!scanner_record_synced_dir(options, root_directory, "",
|
||||||
|
options->relative && options->file_list != NULL)) {
|
||||||
|
closedir(dir);
|
||||||
|
ps->failed = true;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
log_debug_message(LOG_DEBUG_FLIST, "flist: scanning %s", root_directory);
|
||||||
|
const struct dirent* entry;
|
||||||
|
while ((entry = readdir(dir)) != NULL) {
|
||||||
|
if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
|
||||||
|
continue;
|
||||||
|
scan_root_entry(options, root_node, root_directory, entry, root_files, subdirs, root_dev, ps);
|
||||||
|
}
|
||||||
|
closedir(dir);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Spawn worker threads, one per group of subdirectories. */
|
||||||
|
static void spawn_parallel_workers(ParallelScanner* ps, ArrayList* subdirs,
|
||||||
|
const ScannerOptions* options, const char* root_directory,
|
||||||
|
unsigned long long cs) {
|
||||||
|
if (subdirs->size <= 0)
|
||||||
|
return;
|
||||||
|
int n = options->num_threads > 0 ? options->num_threads : 4;
|
||||||
|
if (n > subdirs->size)
|
||||||
|
n = subdirs->size;
|
||||||
|
|
||||||
|
ps->num_threads = n;
|
||||||
|
ps->expected_threads = n;
|
||||||
|
ps->threads = calloc(n, sizeof(thrd_t));
|
||||||
|
if (!ps->threads) {
|
||||||
|
ps->num_threads = 0;
|
||||||
|
ps->expected_threads = 0;
|
||||||
|
ps->failed = true;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
int dirs_per_thread = subdirs->size / n;
|
||||||
|
int remainder = subdirs->size % n;
|
||||||
|
int start = 0;
|
||||||
|
ps->num_threads = 0;
|
||||||
|
for (int t = 0; t < n; t++) {
|
||||||
|
int count = dirs_per_thread + (t < remainder ? 1 : 0);
|
||||||
|
if (count == 0)
|
||||||
|
break;
|
||||||
|
ParallelWorkerArg* wa = calloc(1, sizeof(ParallelWorkerArg));
|
||||||
|
if (!wa) {
|
||||||
|
parallel_scanner_creation_failed(ps);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
wa->ps = ps;
|
||||||
|
wa->dirs = calloc(count, sizeof(char*));
|
||||||
|
wa->root_dir = str_dup(root_directory);
|
||||||
|
if (!wa->dirs || !wa->root_dir) {
|
||||||
|
free(wa->root_dir);
|
||||||
|
free(wa->dirs);
|
||||||
|
free(wa);
|
||||||
|
parallel_scanner_creation_failed(ps);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
bool dup_ok = true;
|
||||||
|
for (int j = 0; j < count; j++) {
|
||||||
|
wa->dirs[j] = str_dup((char*)subdirs->items[start + j]);
|
||||||
|
if (!wa->dirs[j])
|
||||||
|
dup_ok = false;
|
||||||
|
}
|
||||||
|
if (!dup_ok) {
|
||||||
|
for (int j = 0; j < count; j++)
|
||||||
|
free(wa->dirs[j]);
|
||||||
|
free(wa->root_dir);
|
||||||
|
free(wa->dirs);
|
||||||
|
free(wa);
|
||||||
|
parallel_scanner_creation_failed(ps);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
wa->dir_count = count;
|
||||||
|
wa->options = *options;
|
||||||
|
wa->options.chunk_size = cs;
|
||||||
|
wa->allocation_session = ps->allocation_session;
|
||||||
|
start += count;
|
||||||
|
if (thrd_create(&ps->threads[t], parallel_worker_thread, wa) != thrd_success) {
|
||||||
|
for (int j = 0; j < count; j++)
|
||||||
|
free(wa->dirs[j]);
|
||||||
|
free(wa->root_dir);
|
||||||
|
free(wa->dirs);
|
||||||
|
free(wa);
|
||||||
|
parallel_scanner_creation_failed(ps);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
ps->num_threads++;
|
||||||
|
ps->created_threads++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
ParallelScanner* parallel_scanner_create_with_options(const char* root_directory,
|
||||||
|
const ScannerOptions* options,
|
||||||
|
ProtocolSession* allocation_session) {
|
||||||
|
if (!root_directory || !options)
|
||||||
|
return NULL;
|
||||||
|
ParallelScanner* ps = calloc(1, sizeof(ParallelScanner));
|
||||||
|
if (!ps)
|
||||||
|
return NULL;
|
||||||
|
if (!parallel_scanner_init(ps)) {
|
||||||
|
free(ps);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
ps->allocation_session = allocation_session;
|
||||||
|
ps->options = options;
|
||||||
|
|
||||||
|
ArrayList* root_files = array_list_create(file_destroy);
|
||||||
|
ArrayList* subdirs = array_list_create(free);
|
||||||
|
if (!root_files || !subdirs) {
|
||||||
|
array_list_delete(root_files);
|
||||||
|
array_list_delete(subdirs);
|
||||||
|
parallel_scanner_destroy(ps);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
dev_t root_dev = 0;
|
||||||
|
if (options->one_file_system) {
|
||||||
|
struct stat root_stats;
|
||||||
|
if (stat(root_directory, &root_stats) != 0) {
|
||||||
|
log_perror("Could not stat source directory");
|
||||||
|
array_list_delete(root_files);
|
||||||
|
array_list_delete(subdirs);
|
||||||
|
parallel_scanner_destroy(ps);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
root_dev = root_stats.st_dev;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Build the root directory's per-directory filter context once; workers seed
|
||||||
|
* their scanners with it so per-dir rules behave identically to the sequential
|
||||||
|
* scanner. */
|
||||||
|
FilterNode* root_node = NULL;
|
||||||
|
{
|
||||||
|
char err[256];
|
||||||
|
bool any_exists = false;
|
||||||
|
FilterRuleList* own =
|
||||||
|
read_dir_filters(options, root_directory, "", &any_exists, err, sizeof(err));
|
||||||
|
if (!own) {
|
||||||
|
/* A parse/allocation failure must fail the scan even when an earlier
|
||||||
|
merge file in the same directory existed (see the sequential scanner). */
|
||||||
|
if (err[0] != '\0') {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "invalid per-directory filter in %s: %s", root_directory, err);
|
||||||
|
array_list_delete(root_files);
|
||||||
|
array_list_delete(subdirs);
|
||||||
|
parallel_scanner_destroy(ps);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
/* no files exist: leave root_node NULL */
|
||||||
|
} else if (any_exists && (own->count > 0 || own->dir_merge_count > 0)) {
|
||||||
|
root_node = filter_node_alloc(NULL, own);
|
||||||
|
if (!root_node) {
|
||||||
|
filter_rule_list_free(own);
|
||||||
|
array_list_delete(root_files);
|
||||||
|
array_list_delete(subdirs);
|
||||||
|
parallel_scanner_destroy(ps);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
filter_rule_list_free(own);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ps->root_filter_node = root_node;
|
||||||
|
|
||||||
|
if (!scan_root_directory(ps, root_directory, options, root_node, root_dev, root_files, subdirs)) {
|
||||||
|
array_list_delete(root_files);
|
||||||
|
array_list_delete(subdirs);
|
||||||
|
parallel_scanner_destroy(ps);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
/* The root itself is a traversed directory (rsync counts it in
|
||||||
|
`Number of files`); the worker DirectoryScanners account for every
|
||||||
|
subdirectory below it. */
|
||||||
|
scanner_dir_count_count(options);
|
||||||
|
/* P7 Wave D: the parallel scanner never runs a DirectoryScanner over the
|
||||||
|
transfer root itself (it hands the root's immediate subdirectories to
|
||||||
|
workers), so capture the root's directory time here. */
|
||||||
|
if (options->capture_dir_times &&
|
||||||
|
!scanner_capture_dir_time(
|
||||||
|
options->dir_entries, options->dir_entries_mutex, root_directory, root_directory,
|
||||||
|
options->relative && options->file_list != NULL, options->relative_prefix,
|
||||||
|
options->preserve_atimes, options->preserve_crtimes, options->preserve_xattrs,
|
||||||
|
options->preserve_acls, options->no_implied_dirs, options->file_list)) {
|
||||||
|
array_list_delete(root_files);
|
||||||
|
array_list_delete(subdirs);
|
||||||
|
parallel_scanner_destroy(ps);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
unsigned long long cs = options->chunk_size > 0 ? options->chunk_size : DESIRED_CHUNK_SIZE;
|
||||||
|
ps->initial_chunk = batch_files(root_files, cs, ps->result_queue, &ps->failed);
|
||||||
|
array_list_delete(root_files);
|
||||||
|
|
||||||
|
spawn_parallel_workers(ps, subdirs, options, root_directory, cs);
|
||||||
|
array_list_delete(subdirs);
|
||||||
|
return ps;
|
||||||
|
}
|
||||||
|
|
||||||
|
Chunk* parallel_scanner_next(ParallelScanner* ps) {
|
||||||
|
if (ps->initial_chunk) {
|
||||||
|
Chunk* c = ps->initial_chunk;
|
||||||
|
ps->initial_chunk = NULL;
|
||||||
|
return c;
|
||||||
|
}
|
||||||
|
if (ps->num_threads == 0) {
|
||||||
|
mtx_lock(&ps->result_mutex);
|
||||||
|
if (!queue_is_empty(ps->result_queue)) {
|
||||||
|
Chunk* chunk = queue_dequeue(ps->result_queue);
|
||||||
|
mtx_unlock(&ps->result_mutex);
|
||||||
|
return chunk;
|
||||||
|
}
|
||||||
|
ps->done = true;
|
||||||
|
mtx_unlock(&ps->result_mutex);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
Chunk* chunk = queue_dequeue_multithreaded(
|
||||||
|
ps->result_queue, &ps->result_mutex, &ps->result_not_empty, &ps->result_not_full, &ps->done);
|
||||||
|
return chunk;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool parallel_scanner_failed(const ParallelScanner* ps) {
|
||||||
|
return ps == NULL || ps->failed;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool parallel_scanner_had_io_error(const ParallelScanner* ps) {
|
||||||
|
return ps != NULL && ps->io_error;
|
||||||
|
}
|
||||||
|
|
||||||
|
void parallel_scanner_destroy(ParallelScanner* ps) {
|
||||||
|
if (!ps)
|
||||||
|
return;
|
||||||
|
mtx_lock(&ps->result_mutex);
|
||||||
|
ps->done = true;
|
||||||
|
atomic_store(&ps->cancelled, true);
|
||||||
|
cnd_broadcast(&ps->result_not_empty);
|
||||||
|
cnd_broadcast(&ps->result_not_full);
|
||||||
|
mtx_unlock(&ps->result_mutex);
|
||||||
|
for (int i = 0; i < ps->num_threads; i++)
|
||||||
|
thrd_join(ps->threads[i], NULL);
|
||||||
|
free(ps->threads);
|
||||||
|
if (ps->root_filter_node)
|
||||||
|
filter_node_destroy(ps->root_filter_node);
|
||||||
|
if (ps->initial_chunk)
|
||||||
|
chunk_destroy(ps->initial_chunk);
|
||||||
|
queue_destroy(ps->result_queue);
|
||||||
|
mtx_destroy(&ps->result_mutex);
|
||||||
|
cnd_destroy(&ps->result_not_empty);
|
||||||
|
cnd_destroy(&ps->result_not_full);
|
||||||
|
free(ps);
|
||||||
|
}
|
||||||
+172
-91
@@ -19,20 +19,30 @@ void print_usage(void) {
|
|||||||
printf("\n");
|
printf("\n");
|
||||||
printf("Options:\n");
|
printf("Options:\n");
|
||||||
printf(" -c, --checksum Verify content by checksum instead of size+mtime\n");
|
printf(" -c, --checksum Verify content by checksum instead of size+mtime\n");
|
||||||
printf(" -z, --compress [level] Enable compression (level 1-22, default 5)\n");
|
printf(" -z, --compress [level] Enable compression. The default level is\n");
|
||||||
printf(" -a, --archive rsync archive mode (-rlptgoD): links, metadata,\n");
|
printf(" per-codec: zstd 3 (range 1-22), zlib/zlibx 6, lz4\n");
|
||||||
printf(" devices and specials (not compression/multithreading)\n");
|
printf(" ignores the level\n");
|
||||||
|
printf(" -a, --archive rsync archive mode (-rlptgoD): links, perms, times,\n");
|
||||||
|
printf(" owner, group, devices and specials; not\n");
|
||||||
|
printf(" compression/multithreading\n");
|
||||||
|
printf(" -r, --recursive Recurse into directories (FastSync is always recursive)\n");
|
||||||
|
printf(" --inc-recursive Accepted for rsync CLI compatibility; no effect (FastSync\n");
|
||||||
|
printf(" always performs a full scan, so the destination is identical)\n");
|
||||||
|
printf(" --no-inc-recursive Accepted for rsync CLI compatibility; no effect\n");
|
||||||
printf(" -n, --dry-run Show what would be transferred\n");
|
printf(" -n, --dry-run Show what would be transferred\n");
|
||||||
printf(" --remove-source-files Remove regular source files after successful transfer\n");
|
printf(" --remove-source-files Remove regular source files after successful transfer\n");
|
||||||
printf(" -p, --perms Preserve permission bits (part of the metadata bundle)\n");
|
printf(" -p, --perms Preserve permission bits\n");
|
||||||
|
printf(" -t, --times Preserve modification times\n");
|
||||||
|
printf(" -o, --owner Preserve owner (uid)\n");
|
||||||
|
printf(" -g, --group Preserve group (gid)\n");
|
||||||
printf(" --ssh-port <port> SSH port (default: 22)\n");
|
printf(" --ssh-port <port> SSH port (default: 22)\n");
|
||||||
printf(" -e, --rsh <command> Remote shell to launch on the client for the SSH\n");
|
printf(" -e, --rsh <command> Remote shell to launch on the client for the SSH\n");
|
||||||
printf(" transport (default: ssh). The command may include\n");
|
printf(" transport (default: ssh). The command may include\n");
|
||||||
printf(" arguments, e.g. -e \"ssh -p 2222\"\n");
|
printf(" arguments, e.g. -e \"ssh -p 2222\"\n");
|
||||||
printf(" --rsync-path <path> Alias for --fastsync-server-path (path to the\n");
|
printf(" --rsync-path <path> Alias for --fastsync-server-path (path to the\n");
|
||||||
printf(" fastsync server binary on the remote side)\n");
|
printf(" fastsync server binary on the remote side)\n");
|
||||||
printf(" --blocking-io Leave the SSH transport socket without read/write\n");
|
printf(" --blocking-io SSH transport only: leave the socket without read/write\n");
|
||||||
printf(" timeouts so it blocks naturally\n");
|
printf(" timeouts so it blocks naturally (no effect on TCP)\n");
|
||||||
printf(" --outbuf=MODE stdout/stderr buffering: N (none/unbuffered),\n");
|
printf(" --outbuf=MODE stdout/stderr buffering: N (none/unbuffered),\n");
|
||||||
printf(" L (line-buffered), or B (block-buffered, default)\n");
|
printf(" L (line-buffered), or B (block-buffered, default)\n");
|
||||||
printf(" --progress Show transfer progress\n");
|
printf(" --progress Show transfer progress\n");
|
||||||
@@ -44,6 +54,7 @@ void print_usage(void) {
|
|||||||
printf(" converted before transmission and back on receipt; a\n");
|
printf(" converted before transmission and back on receipt; a\n");
|
||||||
printf(" name that cannot be represented in the target charset\n");
|
printf(" name that cannot be represented in the target charset\n");
|
||||||
printf(" fails that transfer cleanly (rsync-compatible)\n");
|
printf(" fails that transfer cleanly (rsync-compatible)\n");
|
||||||
|
printf(" --no-iconv Disable --iconv charset conversion (same as --iconv=-)\n");
|
||||||
printf(" --protocol=NUM Force the wire protocol version (must equal the current\n");
|
printf(" --protocol=NUM Force the wire protocol version (must equal the current\n");
|
||||||
printf(" PROTOCOL_VERSION; FastSync cannot speak older/virtual\n");
|
printf(" PROTOCOL_VERSION; FastSync cannot speak older/virtual\n");
|
||||||
printf(" wire formats)\n");
|
printf(" wire formats)\n");
|
||||||
@@ -54,24 +65,27 @@ void print_usage(void) {
|
|||||||
printf(" Emit the batch file only (no destination, no server)\n");
|
printf(" Emit the batch file only (no destination, no server)\n");
|
||||||
printf(" --read-batch=FILE Apply the batch file to the destination (no source, no\n");
|
printf(" --read-batch=FILE Apply the batch file to the destination (no source, no\n");
|
||||||
printf(" server); takes only the destination as an argument\n");
|
printf(" server); takes only the destination as an argument\n");
|
||||||
|
printf(" NOTE: the FastSync batch format is NOT interoperable with rsync's batch\n");
|
||||||
|
printf(" files (different container format); do not mix the two tools.\n");
|
||||||
printf(" --delete Delete files on receiver not in source\n");
|
printf(" --delete Delete files on receiver not in source\n");
|
||||||
printf(" (default timing: delete only after the whole\n");
|
printf(" (default timing: delete-during, like rsync --del)\n");
|
||||||
printf(" transfer has succeeded)\n");
|
|
||||||
printf(" --delete-before Delete extras before the transfer starts\n");
|
printf(" --delete-before Delete extras before the transfer starts\n");
|
||||||
printf(" (implies --delete)\n");
|
printf(" (implies --delete)\n");
|
||||||
printf(" --delete-during Delete extras once the keep-set manifest is known,\n");
|
printf(" --delete-during Delete a directory's extras as that directory is\n");
|
||||||
printf(" before the data is applied (implies --delete)\n");
|
printf(" processed (implies --delete)\n");
|
||||||
printf(" --del Alias for --delete-during\n");
|
printf(" --del Alias for --delete-during\n");
|
||||||
printf(" --delete-delay Delete extras only after a successful transfer\n");
|
printf(" --delete-delay Record the extras during the scan but remove them\n");
|
||||||
printf(" (implies --delete)\n");
|
printf(" only after a successful transfer (implies --delete)\n");
|
||||||
printf(" --delete-after Delete only after the whole transfer succeeded\n");
|
printf(" --delete-after Delete only after the whole transfer succeeded\n");
|
||||||
printf(" (the default --delete timing; implies --delete)\n");
|
printf(" (implies --delete)\n");
|
||||||
|
printf(" --delete-commit FastSync-only: restore the late whole-tree commit\n");
|
||||||
|
printf(" (identical to --delete-after; implies --delete)\n");
|
||||||
printf(" --delete-excluded Also delete destination files that were excluded on\n");
|
printf(" --delete-excluded Also delete destination files that were excluded on\n");
|
||||||
printf(" the source (default protects them, matching rsync)\n");
|
printf(" the source (default protects them, matching rsync)\n");
|
||||||
printf(" --max-delete=NUM Never delete more than NUM destination entries per run;\n");
|
printf(" --max-delete=NUM Delete at most NUM destination entries per run; if the\n");
|
||||||
printf(" if the extras would exceed NUM, nothing is deleted and\n");
|
printf(" extras exceed NUM, the rest are skipped and the run is\n");
|
||||||
printf(" the run fails with a clear error (implies --delete only\n");
|
printf(" reported as partial (exit 25, matching rsync). Only\n");
|
||||||
printf(" when used with it)\n");
|
printf(" applies together with --delete\n");
|
||||||
printf(" --ignore-errors Continue (and still delete) when a source directory is\n");
|
printf(" --ignore-errors Continue (and still delete) when a source directory is\n");
|
||||||
printf(" unreadable during the scan, instead of aborting with no\n");
|
printf(" unreadable during the scan, instead of aborting with no\n");
|
||||||
printf(" deletion\n");
|
printf(" deletion\n");
|
||||||
@@ -83,10 +97,11 @@ void print_usage(void) {
|
|||||||
printf(" entry's destination mirror receiver-side. Independent of\n");
|
printf(" entry's destination mirror receiver-side. Independent of\n");
|
||||||
printf(" --delete (it does not imply --delete; a non-empty directory\n");
|
printf(" --delete (it does not imply --delete; a non-empty directory\n");
|
||||||
printf(" mirror is removed only with --force or --delete)\n");
|
printf(" mirror is removed only with --force or --delete)\n");
|
||||||
printf(" -m, --prune-empty-dirs Do not transfer empty directory entries (--dirs mode);\n");
|
printf(" -m, --prune-empty-dirs Do not create empty directories (a recursive transfer\n");
|
||||||
printf(" recursive transfers never send empty dirs\n");
|
printf(" otherwise recreates them, like rsync)\n");
|
||||||
printf(" Note: each timing flag implies --delete. Combining a timing flag with\n");
|
printf(" Note: each timing flag implies --delete. Combining a timing flag with\n");
|
||||||
printf(" --no-delete (in either order) is rejected as a config error.\n");
|
printf(" --no-delete (in either order) is rejected as a config error, as is more\n");
|
||||||
|
printf(" than one timing flag.\n");
|
||||||
printf(" --ignore-existing Skip files that already exist on receiver\n");
|
printf(" --ignore-existing Skip files that already exist on receiver\n");
|
||||||
printf(" --delay-updates Put updated files into place only at the end of transfer\n");
|
printf(" --delay-updates Put updated files into place only at the end of transfer\n");
|
||||||
printf(" --dirs, -d, --old-dirs, --old-d Transfer the named directory entries without\n");
|
printf(" --dirs, -d, --old-dirs, --old-d Transfer the named directory entries without\n");
|
||||||
@@ -96,24 +111,28 @@ void print_usage(void) {
|
|||||||
printf(" -R, --relative With --files-from, preserve each listed entry's relative path\n");
|
printf(" -R, --relative With --files-from, preserve each listed entry's relative path\n");
|
||||||
printf(" below the destination root instead of mirroring the full\n");
|
printf(" below the destination root instead of mirroring the full\n");
|
||||||
printf(" source path (no effect without --files-from)\n");
|
printf(" source path (no effect without --files-from)\n");
|
||||||
printf(" --no-implied-dirs With -R --files-from, refuse to place a listed file whose\n");
|
printf(" --no-implied-dirs With -R, do not apply the source metadata of a listed file's\n");
|
||||||
printf(" parent directory is not itself listed\n");
|
printf(" implied parent directories (they are still created with\n");
|
||||||
|
printf(" default attributes)\n");
|
||||||
printf(" --mkpath Create the destination root directory on the server when it\n");
|
printf(" --mkpath Create the destination root directory on the server when it\n");
|
||||||
printf(" does not exist yet\n");
|
printf(" does not exist yet\n");
|
||||||
printf(" --exclude <pattern> Exclude files matching pattern\n");
|
printf(" --exclude <pattern>, --exclude=<pattern> Exclude files matching pattern\n");
|
||||||
printf(" --include <pattern> Only include files matching pattern\n");
|
printf(" --include <pattern>, --include=<pattern> Only include files matching pattern\n");
|
||||||
printf(" --exclude-from <file> Read exclude patterns from file\n");
|
printf(" --exclude-from <file>, --exclude-from=<file> Read exclude patterns from file\n");
|
||||||
printf(" --include-from <file> Read include patterns from file\n");
|
printf(" --include-from <file>, --include-from=<file> Read include patterns from file\n");
|
||||||
printf(" --files-from <file> Read the source file list from FILE (paths relative to the "
|
printf(" --files-from <file> Read the source file list from FILE (paths relative to the "
|
||||||
"source root)\n");
|
"source root)\n");
|
||||||
printf(" -0, --from0 Entries in --files-from are NUL-delimited\n");
|
printf(" -0, --from0 Entries in --files-from are NUL-delimited\n");
|
||||||
printf(" -f, --filter=RULE rsync-style filter rule (+/- include/exclude; repeatable;\n");
|
printf(" -f, --filter=RULE rsync-style filter rule: exclude/- include/+ hide/H show/S\n");
|
||||||
printf(" both --filter=RULE and the -f RULE / -f=RULE short forms work)\n");
|
printf(" protect/P risk/R merge/. dir-merge/: clear/! with modifiers\n");
|
||||||
|
printf(" (repeatable; --filter=RULE and -f RULE / -f=RULE both work)\n");
|
||||||
printf(" -C, --cvs-exclude Auto-ignore common CVS/SCM files (.git/, .svn/, *.o, *~, ...)\n");
|
printf(" -C, --cvs-exclude Auto-ignore common CVS/SCM files (.git/, .svn/, *.o, *~, ...)\n");
|
||||||
printf(" -F Apply per-directory .rsync-filter files during the scan\n");
|
printf(" -F Apply per-directory .rsync-filter files; repeated -FF also\n");
|
||||||
|
printf(" excludes the .rsync-filter files themselves\n");
|
||||||
printf(" --max-size <n> Skip files larger than n bytes\n");
|
printf(" --max-size <n> Skip files larger than n bytes\n");
|
||||||
printf(" --min-size <n> Skip files smaller than n bytes\n");
|
printf(" --min-size <n> Skip files smaller than n bytes\n");
|
||||||
printf(" --max-alloc <SIZE> Maximum single allocation (default: 1G)\n");
|
printf(" --max-alloc <SIZE> Maximum single allocation (default: 1G; 0 = no limit,\n");
|
||||||
|
printf(" matching rsync)\n");
|
||||||
printf(" --incremental Skip files unchanged since last transfer\n");
|
printf(" --incremental Skip files unchanged since last transfer\n");
|
||||||
printf(" --size-only Skip incremental files matching in size, ignoring mtime\n");
|
printf(" --size-only Skip incremental files matching in size, ignoring mtime\n");
|
||||||
printf(" -I, --ignore-times Transfer files even when size and mtime match\n");
|
printf(" -I, --ignore-times Transfer files even when size and mtime match\n");
|
||||||
@@ -127,22 +146,29 @@ void print_usage(void) {
|
|||||||
printf(" into the destination instead of transferring its data\n");
|
printf(" into the destination instead of transferring its data\n");
|
||||||
printf(" --link-dest <dir> Like --copy-dest, but hard-links the unchanged file from DIR\n");
|
printf(" --link-dest <dir> Like --copy-dest, but hard-links the unchanged file from DIR\n");
|
||||||
printf(" into the destination (repeatable; earlier DIRs win)\n");
|
printf(" into the destination (repeatable; earlier DIRs win)\n");
|
||||||
|
printf(" --verify-basis FastSync-only: require a basis hit's content to match the\n");
|
||||||
|
printf(" source by whole-file digest instead of trusting rsync's\n");
|
||||||
|
printf(" size+mtime (or --size-only) quick-check\n");
|
||||||
printf(" --checksum-choice, --cc <alg> Whole-file checksum algorithm for --incremental/\n");
|
printf(" --checksum-choice, --cc <alg> Whole-file checksum algorithm for --incremental/\n");
|
||||||
printf(" --checksum compares (xxh64/xxhash or md5; default xxh64 with\n");
|
printf(" --checksum compares. Accepted: xxh128 (default), xxh3, xxh64\n");
|
||||||
printf(" seed 0). The seed comes from --checksum-seed\n");
|
printf(" (aka xxhash), md5, md4, sha1, or none. A two-name\n");
|
||||||
printf(" --checksum-seed <num> Seed for the whole-file xxHash64 digest (and the delta\n");
|
printf(" 'transfer,pre-transfer' form is accepted like rsync; 'none' as\n");
|
||||||
printf(" block strong hash, low 32 bits); md5 ignores the seed. The\n");
|
printf(" the pre-transfer algorithm is rejected with --checksum\n");
|
||||||
printf(" digest algorithm and seed must match on sender and receiver\n");
|
printf(" --checksum-seed <num> Seed for the whole-file xxHash digest (and the delta\n");
|
||||||
|
printf(" block strong hash, low 32 bits); md5 ignores the seed. A seed\n");
|
||||||
|
printf(" of 0 (the default) is randomized per transfer, exactly like\n");
|
||||||
|
printf(" rsync, and the chosen seed is sent to the receiver\n");
|
||||||
printf(" --delta Delta transfer for changed files (requires --incremental)\n");
|
printf(" --delta Delta transfer for changed files (requires --incremental)\n");
|
||||||
printf(" -W, --whole-file Transfer changed files without delta processing\n");
|
printf(" -W, --whole-file Transfer changed files without delta processing\n");
|
||||||
|
printf(" --no-whole-file rsync spelling that clears -W/--whole-file\n");
|
||||||
printf(" -y, --fuzzy Use a similar-named file already in the destination\n");
|
printf(" -y, --fuzzy Use a similar-named file already in the destination\n");
|
||||||
printf(" directory as the delta basis when the destination has no\n");
|
printf(" directory as the delta basis when the destination has no\n");
|
||||||
printf(" usable file at the exact path (saves bandwidth; implies\n");
|
printf(" usable file at the exact path (saves bandwidth; implies\n");
|
||||||
printf(" --incremental and --delta; inert with --whole-file,\n");
|
printf(" --incremental and --delta; inert with --whole-file,\n");
|
||||||
printf(" --no-delta, or --no-incremental)\n");
|
printf(" --no-delta, or --no-incremental)\n");
|
||||||
printf(" --no-fuzzy Disable --fuzzy\n");
|
printf(" --no-fuzzy Disable --fuzzy\n");
|
||||||
printf(" --delta-block <n>, --block-size <n>\n");
|
printf(" -B <n>, --block-size <n>, --delta-block <n>\n");
|
||||||
printf(" Delta block size in bytes (default: %d)\n", DELTA_BLOCK_SIZE_DEFAULT);
|
printf(" Delta block size in bytes (default: %u)\n", DELTA_BLOCK_SIZE_DEFAULT);
|
||||||
printf(" --delta-max <n> Max file size for delta transfer (default: %llu)\n",
|
printf(" --delta-max <n> Max file size for delta transfer (default: %llu)\n",
|
||||||
DELTA_MAX_FILE_SIZE);
|
DELTA_MAX_FILE_SIZE);
|
||||||
printf(" -j, --threads[=N] Enable the multithreaded scanner/loader/sender\n");
|
printf(" -j, --threads[=N] Enable the multithreaded scanner/loader/sender\n");
|
||||||
@@ -152,52 +178,71 @@ void print_usage(void) {
|
|||||||
printf(" --chunk-serialization Enable chunk serialization (long form only)\n");
|
printf(" --chunk-serialization Enable chunk serialization (long form only)\n");
|
||||||
printf(" -s, --secluded-args Protect-args compatibility option (no effect; remote\n");
|
printf(" -s, --secluded-args Protect-args compatibility option (no effect; remote\n");
|
||||||
printf(" SSH argv is already built injection-safe)\n");
|
printf(" SSH argv is already built injection-safe)\n");
|
||||||
printf(" --sendfile Enable sendfile zero-copy (TCP only; long form only)\n");
|
printf(" --sendfile Enable sendfile zero-copy (TCP only; long form only;\n");
|
||||||
printf(" --compress-choice <alg> Compression algorithm (default: zstd)\n");
|
printf(" -f is bound to --filter, not --sendfile)\n");
|
||||||
|
printf(" --compress-choice <alg> Compression algorithm: zstd (default), lz4, zlib,\n");
|
||||||
|
printf(" zlibx, none, or auto\n");
|
||||||
printf(" --zc <alg> Alias for --compress-choice\n");
|
printf(" --zc <alg> Alias for --compress-choice\n");
|
||||||
printf(" -v, --verbose Enable debug logging\n");
|
printf(" -v, --verbose Enable debug logging\n");
|
||||||
printf(" -q, --quiet Suppress non-error output\n");
|
printf(" -q, --quiet Suppress non-error output\n");
|
||||||
printf(" --debug=FLAGS Fine-grained debug logging (use --debug=help for flags)\n");
|
printf(" --debug=FLAGS Fine-grained debug logging (use --debug=help for flags)\n");
|
||||||
printf(" --info=FLAGS Fine-grained info: copy,misc,skip,stats,all,none\n");
|
printf(" --info=FLAGS Fine-grained info: copy,name,misc,skip,stats,all,none\n");
|
||||||
printf(" none suppresses info even with --verbose\n");
|
printf(" (use --info=help for flags; none suppresses --verbose)\n");
|
||||||
printf(" --preserve Preserve file metadata (long form only)\n");
|
printf(" --preserve Preserve permissions and times (= -pt; long form only)\n");
|
||||||
|
printf(" --no-perms Negate -p/--perms\n");
|
||||||
|
printf(" --no-times Negate -t/--times\n");
|
||||||
|
printf(" --no-owner Negate -o/--owner\n");
|
||||||
|
printf(" --no-group Negate -g/--group\n");
|
||||||
|
printf(" --no-preserve Disable metadata preservation (negates --preserve)\n");
|
||||||
printf(" -E, --executability Preserve executable permission bits\n");
|
printf(" -E, --executability Preserve executable permission bits\n");
|
||||||
|
printf(" -U, --atimes Preserve access times\n");
|
||||||
|
printf(" -N, --crtimes Capture birth time; cannot be applied (documented\n");
|
||||||
|
printf(" divergence)\n");
|
||||||
|
printf(" -O, --omit-dir-times Do not apply modification times to directories\n");
|
||||||
|
printf(" -J, --omit-link-times Do not apply times to symlinks\n");
|
||||||
|
printf(" --open-noatime Open source files with O_NOATIME so reading for a\n");
|
||||||
|
printf(" transfer does not update their access time\n");
|
||||||
printf(" -X, --xattrs Preserve user extended attributes (user.* only;\n");
|
printf(" -X, --xattrs Preserve user extended attributes (user.* only;\n");
|
||||||
printf(" privileged security.*/trusted.* namespaces are\n");
|
printf(" privileged security.*/trusted.* namespaces are\n");
|
||||||
printf(" never captured or applied)\n");
|
printf(" never captured or applied)\n");
|
||||||
printf(" -A, --acls Preserve POSIX ACLs (the system.posix_acl_* xattrs;\n");
|
printf(" -A, --acls Preserve POSIX ACLs (the system.posix_acl_* xattrs;\n");
|
||||||
printf(" setting an ACL the receiver is not permitted to\n");
|
printf(" setting an ACL the receiver is not permitted to\n");
|
||||||
printf(" set is warned and skipped, never fatal)\n");
|
printf(" set is warned and skipped, never fatal)\n");
|
||||||
printf(" --fake-super Store the source uid/gid/mode/mtime in a reserved\n");
|
printf(" --fake-super Store the source mode/rdev/uid/gid in rsync's\n");
|
||||||
printf(" user.fastsync.stat xattr on each written file and\n");
|
printf(" reserved user.rsync.%%stat xattr on each written\n");
|
||||||
printf(" re-apply it (fd-relative) on a privileged run; the\n");
|
printf(" file (interoperable with rsync); it never performs a\n");
|
||||||
printf(" recording format diverges from rsync's user.rsync.%%stat%%\n");
|
printf(" real chown, so an unprivileged receiver records the\n");
|
||||||
|
printf(" privileged stat for a later restore\n");
|
||||||
printf(" --super Permit the receiver to attempt super-user activities\n");
|
printf(" --super Permit the receiver to attempt super-user activities\n");
|
||||||
printf(" (char/block device-node creation, --write-devices)\n");
|
printf(" (char/block device-node creation, --write-devices)\n");
|
||||||
printf(" within the confined receive root. Never elevates\n");
|
printf(" within the confined receive root. Never elevates\n");
|
||||||
printf(" privileges and never bypasses confinement; ownership\n");
|
printf(" privileges and never bypasses confinement; ownership\n");
|
||||||
printf(" is still applied only with an explicit identity flag\n");
|
printf(" is still applied only with -o/--owner, -g/--group, or an\n");
|
||||||
printf(" (--numeric-ids/--chown/--usermap/--groupmap/--copy-as)\n");
|
printf(" explicit identity flag (--chown/--usermap/--groupmap/\n");
|
||||||
|
printf(" --copy-as); --numeric-ids only changes how ids map\n");
|
||||||
printf(" --no-super Forbid those super-user activities even when the\n");
|
printf(" --no-super Forbid those super-user activities even when the\n");
|
||||||
printf(" receiver is running as root\n");
|
printf(" receiver is running as root\n");
|
||||||
printf(" --chmod <changes> Modify transferred permissions (rsync syntax)\n");
|
printf(
|
||||||
printf(" --numeric-ids Do not map uid/gid by name: use the source numeric\n");
|
" --chmod <changes> Modify new/transferred permissions (rsync syntax; implies no -p)\n");
|
||||||
printf(" ids directly when applying ownership\n");
|
printf(" --numeric-ids Map uid/gid by id instead of by name (a modifier, not\n");
|
||||||
|
printf(" an ownership request: combine with -o/-g or a map)\n");
|
||||||
printf(" --usermap=MAP Map usernames when applying ownership: comma-separated\n");
|
printf(" --usermap=MAP Map usernames when applying ownership: comma-separated\n");
|
||||||
printf(" FROM:TO rules, first match wins. FROM/TO are names\n");
|
printf(" FROM:TO rules, first match wins. FROM is a name (from\n");
|
||||||
printf(" (resolved on the source machine), * (match any /\n");
|
printf(" the source), an id, an inclusive LOW-HIGH range, *\n");
|
||||||
printf(" current user), or @N numeric ids. e.g. *:nobody\n");
|
printf(" (any id), or empty (ids with no name). TO is an id, *\n");
|
||||||
|
printf(" (current user), or a name resolved on the receiver.\n");
|
||||||
|
printf(" e.g. 0-99:nobody,*:normal (cannot mix with --chown)\n");
|
||||||
printf(" --groupmap=MAP Map group names when applying ownership (same syntax)\n");
|
printf(" --groupmap=MAP Map group names when applying ownership (same syntax)\n");
|
||||||
printf(" --chown=USER:GROUP Override the ownership of transferred files. Forms:\n");
|
printf(" --chown=USER:GROUP Override the ownership of transferred files. Forms:\n");
|
||||||
printf(" USER:GROUP, USER (owner only), :GROUP (group only); a\n");
|
printf(" USER:GROUP, USER (owner only), :GROUP (group only); a\n");
|
||||||
printf(" value of * means the current/root user as appropriate.\n");
|
printf(" value of * means the current/root user as appropriate.\n");
|
||||||
printf(" Names resolve on the source machine; @N for numerics.\n");
|
printf(" Names resolve on the source machine; @N for numerics.\n");
|
||||||
printf(" (Metadata is enabled with --preserve; -M now means\n");
|
printf(" (Implies owner/group metadata; -M now means rsync's\n");
|
||||||
printf(" rsync's --remote-option.)\n");
|
printf(" --remote-option.)\n");
|
||||||
printf(" --copy-as=USER[:GROUP] Force every written entry (files, dirs, symlinks\n");
|
printf(" --copy-as=USER[:GROUP] Force every written entry (files, dirs, symlinks\n");
|
||||||
printf(" and special nodes) to USER[:GROUP], resolved on the\n");
|
printf(" and special nodes) to USER[:GROUP], resolved on the\n");
|
||||||
printf(" source machine like --chown. Requires a privileged\n");
|
printf(" source machine like --chown. Requires a privileged\n");
|
||||||
printf(" (root) receiver and implies --preserve; an\n");
|
printf(" (root) receiver and implies owner/group metadata; an\n");
|
||||||
printf(" unprivileged receiver refuses the transfer. Never\n");
|
printf(" unprivileged receiver refuses the transfer. Never\n");
|
||||||
printf(" switches process credentials (safe-subset; see\n");
|
printf(" switches process credentials (safe-subset; see\n");
|
||||||
printf(" RSYNC_COMPAT.md). A daemon refuses it.\n");
|
printf(" RSYNC_COMPAT.md). A daemon refuses it.\n");
|
||||||
@@ -209,65 +254,86 @@ void print_usage(void) {
|
|||||||
printf(" --server-port <n> Server port (default: 8080)\n");
|
printf(" --server-port <n> Server port (default: 8080)\n");
|
||||||
printf(" --port <n> Alias for --server-port\n");
|
printf(" --port <n> Alias for --server-port\n");
|
||||||
printf(" --password-file <f> Authenticate a host::module/path daemon destination.\n");
|
printf(" --password-file <f> Authenticate a host::module/path daemon destination.\n");
|
||||||
printf(" The file's first user:password line supplies the\n");
|
printf(" FastSync-native SCRAM/PBKDF2 credential scheme (NOT\n");
|
||||||
printf(" username and password (only a SHA-256 digest of the\n");
|
printf(" rsync's --password-file): the file's first user:password\n");
|
||||||
printf(" password is sent; keep the file mode 0600)\n");
|
printf(" line supplies the username and password; no password or\n");
|
||||||
|
printf(" reusable digest is sent (keep the file mode 0600)\n");
|
||||||
printf(" --no-motd Suppress display of the daemon's MOTD (the server\n");
|
printf(" --no-motd Suppress display of the daemon's MOTD (the server\n");
|
||||||
printf(" still sends it; the client just does not show it)\n");
|
printf(" still sends it; the client just does not show it)\n");
|
||||||
printf(" --bwlimit <KB/s> Bandwidth limit in kilobytes per second\n");
|
printf(" --bwlimit=RATE Limit socket I/O bandwidth (default unit KiB/s,\n");
|
||||||
|
printf(" rsync-style: 0 = no limit; K/M/G/T/P suffixes are\n");
|
||||||
|
printf(" binary, KB/MB decimal, KiB/MiB binary; decimals allowed)\n");
|
||||||
printf(" --tls Enable TLS encryption\n");
|
printf(" --tls Enable TLS encryption\n");
|
||||||
printf(" --cert <path> TLS certificate file (PEM)\n");
|
printf(" --cert <path> TLS certificate file (PEM)\n");
|
||||||
printf(" --key <path> TLS private key file (PEM)\n");
|
printf(" --key <path> TLS private key file (PEM)\n");
|
||||||
printf(" --ca <path> TLS CA certificate file (PEM)\n");
|
printf(" --ca <path> TLS CA certificate file (PEM)\n");
|
||||||
printf(" --timeout <sec> I/O timeout in seconds (default: 30; long form only)\n");
|
printf(" --timeout <sec> I/O timeout in seconds (default: 0 = disabled, matching\n");
|
||||||
printf(" --contimeout <sec> Connection timeout in seconds (default: 10)\n");
|
printf(" rsync). 0 disables it; --no-timeout is the same\n");
|
||||||
|
printf(" --contimeout <sec> Connection timeout in seconds (default: 60, matching\n");
|
||||||
|
printf(" rsync); 0 disables it (--no-contimeout)\n");
|
||||||
printf(" --stop-after=MINS Stop the transfer after MINS minutes (a positive\n");
|
printf(" --stop-after=MINS Stop the transfer after MINS minutes (a positive\n");
|
||||||
printf(" integer); whatever was already transferred is kept\n");
|
printf(" integer); whatever was already transferred is kept\n");
|
||||||
printf(" --stop-at=TIME Stop at an absolute time: HH:MM, HH:MM:SS, or\n");
|
printf(" --stop-at=TIME Stop at an absolute time. Accepts rsync's date form\n");
|
||||||
printf(" now+N[smhd] (a time already in the past stops the\n");
|
printf(" (Y-M-DTh:m, Y/M/DTh:m, abbreviable fields such as 12-31,\n");
|
||||||
printf(" transfer immediately; client-only). An early stop\n");
|
printf(" 14:00, :59, 1) plus FastSync's HH:MM[:SS] and now+N[smhd]\n");
|
||||||
printf(" skips the late --delete keep-set so it cannot delete\n");
|
printf(" (a time already in the past stops the transfer\n");
|
||||||
printf(" source mirrors that were not yet scanned\n");
|
printf(" immediately; client-only). An early stop skips the late\n");
|
||||||
|
printf(" --delete keep-set so it cannot delete source mirrors that\n");
|
||||||
|
printf(" were not yet scanned\n");
|
||||||
printf(" --address <ip> Bind the outgoing client socket to this source address\n");
|
printf(" --address <ip> Bind the outgoing client socket to this source address\n");
|
||||||
printf(" -4, --ipv4 Force IPv4 for destination resolution\n");
|
printf(" -4, --ipv4 Force IPv4 for destination resolution\n");
|
||||||
printf(" -6, --ipv6 Force IPv6 for destination resolution\n");
|
printf(" -6, --ipv6 Force IPv6 for destination resolution\n");
|
||||||
printf(" --sockopts=OPTS Comma-separated OPT=VAL socket options applied before connect:\n");
|
printf(" --sockopts=OPTS Comma-separated OPT=VAL socket options applied before connect:\n");
|
||||||
printf(" TCP_NODELAY, SO_KEEPALIVE, SO_RCVBUF, SO_SNDBUF, SO_REUSEADDR\n");
|
printf(" TCP_NODELAY, SO_KEEPALIVE, SO_RCVBUF, SO_SNDBUF, SO_REUSEADDR\n");
|
||||||
printf(" --backup Backup existing files before overwriting\n");
|
printf(" -b, --backup Backup existing files before overwriting\n");
|
||||||
printf(" --backup-dir <dir> Directory for backups (requires --backup)\n");
|
printf(" --backup-dir <dir> Directory for backups (requires --backup)\n");
|
||||||
printf(" --suffix <str> Backup suffix (default: ~)\n");
|
printf(" --suffix <str> Backup suffix (default: ~)\n");
|
||||||
printf(" --stats Print transfer statistics at end\n");
|
printf(" --stats Print transfer statistics at end\n");
|
||||||
printf(" -i, --itemize-changes Print an rsync-style per-file change line\n");
|
printf(" -i, --itemize-changes Print an rsync-style per-file change line\n");
|
||||||
printf(" --out-format=FORMAT Output format for changed files (%%f %%n %%l %%b %%M %%%%)\n");
|
printf(" --out-format=FORMAT Output format (%%f %%n %%l %%b %%c %%C %%i %%M %%%%)\n");
|
||||||
printf(" --list-only List source files instead of transferring\n");
|
printf(" --list-only List source files instead of transferring\n");
|
||||||
printf(" --log-file-format=FORMAT Per-file log line format (needs --log-file)\n");
|
printf(" --log-file-format=FORMAT Per-file log line format (needs --log-file)\n");
|
||||||
printf(" -h, --human-readable Print byte sizes in human-readable form\n");
|
printf(" -h, --human-readable Print byte sizes in human-readable form\n");
|
||||||
printf(" --max-depth <n> Maximum directory depth (0=unlimited)\n");
|
printf(" --max-depth <n> Maximum directory depth (0=unlimited)\n");
|
||||||
printf(" -x, --one-file-system Do not cross filesystem boundaries\n");
|
printf(" -x, --one-file-system Do not cross filesystem boundaries\n");
|
||||||
printf(" --log-file <path> Write log messages to file\n");
|
printf(" --log-file <path>, --log-file=<path> Write log messages to file\n");
|
||||||
printf(" --stderr=MODE Route logging to stderr: errors or all\n");
|
printf(" --stderr=MODE Route logging to stderr: errors or all\n");
|
||||||
|
printf(" --msgs2stderr Route all messages to stderr (deprecated spelling of\n");
|
||||||
|
printf(" --stderr=all)\n");
|
||||||
|
printf(" --no-msgs2stderr Select errors-only stderr (deprecated spelling; the\n");
|
||||||
|
printf(" default)\n");
|
||||||
printf(" --partial Keep partial files on interrupted transfer\n");
|
printf(" --partial Keep partial files on interrupted transfer\n");
|
||||||
printf(" --partial-dir <dir> Directory for partial files\n");
|
printf(" --partial-dir <dir> Directory for partial files (implies --partial)\n");
|
||||||
printf(" -T, --temp-dir <dir> Scratch dir for temp files before atomic install\n");
|
printf(" -T, --temp-dir <dir> Scratch dir for temp files before atomic install.\n");
|
||||||
|
printf(" Confined to the receive root: a relative dir resolves below\n");
|
||||||
|
printf(" it and an absolute/traversal dir is rejected. The dir must\n");
|
||||||
|
printf(" already exist; a different filesystem falls back to a\n");
|
||||||
|
printf(" non-atomic copy instead of aborting\n");
|
||||||
printf(" --fastsync-server-path <path>\n");
|
printf(" --fastsync-server-path <path>\n");
|
||||||
printf(" Path to fastsync-server on remote (default: fastsync-server)\n");
|
printf(" Path to fastsync-server on remote (default: fastsync-server)\n");
|
||||||
printf(" --old-args Accepted for rsync CLI compatibility; no effect (the\n");
|
printf(" --old-args Accepted for rsync CLI compatibility; no effect (the\n");
|
||||||
printf(" remote server path is always safely quoted now)\n");
|
printf(" remote server path is always safely quoted now)\n");
|
||||||
printf(" -M, --remote-option=OPT Append OPT to the REMOTE server invocation over SSH\n");
|
printf(" -M, --remote-option=OPT Append OPT to the REMOTE server invocation. SSH\n");
|
||||||
printf(" (repeatable; each value is single-quote-escaped on the remote\n");
|
printf(" transport ONLY (user@host:path): a daemon (host::module) or\n");
|
||||||
printf(" command line; empty values and values with control characters\n");
|
printf(" local TCP destination rejects it (no remote command line to\n");
|
||||||
printf(" are rejected; -M OPT, -M=OPT and --remote-option=OPT work)\n");
|
printf(" append to). Repeatable; each value is single-quote-escaped on\n");
|
||||||
printf(" --trust-sender Trust the remote sender's file list: the receiver skips its\n");
|
printf(" the remote command line; empty values and values with control\n");
|
||||||
printf(" own up-front path-traversal/containment re-validation of the\n");
|
printf(" characters are rejected; -M OPT, -M=OPT and\n");
|
||||||
printf(" incoming file list (fewer checks, faster, potentially unsafe).\n");
|
printf(" --remote-option=OPT work\n");
|
||||||
printf(" Local receiver policy: never sent to the peer, off by default\n");
|
printf(" --trust-sender RECEIVER-LOCAL policy: trust the remote sender's file list\n");
|
||||||
|
printf(" and skip the receiver's own up-front path-traversal/\n");
|
||||||
|
printf(" containment re-validation of the incoming list (fewer checks,\n");
|
||||||
|
printf(" faster, potentially unsafe). It is never sent to the peer, so\n");
|
||||||
|
printf(" for a push it must be enabled on the receiving SERVER\n");
|
||||||
|
printf(" (fastsync-server --trust-sender) or forwarded with\n");
|
||||||
|
printf(" -M--trust-sender; the client flag alone has no effect\n");
|
||||||
printf(" -l, --links Copy symlinks as symlinks\n");
|
printf(" -l, --links Copy symlinks as symlinks\n");
|
||||||
printf(" --copy-links Transform symlinks into referent files\n");
|
printf(" -L, --copy-links Transform symlinks into referent files\n");
|
||||||
printf(" --safe-links Skip symlinks that point outside transfer tree\n");
|
printf(" --safe-links Skip symlinks whose target points outside the tree\n");
|
||||||
printf(" --copy-unsafe-links Only transform unsafe symlinks into referent files\n");
|
printf(" --copy-unsafe-links Copy unsafe symlinks (outside tree) as referent files\n");
|
||||||
printf(" -k, --copy-dirlinks Transform symlinks to directories into real dirs\n");
|
printf(" -k, --copy-dirlinks Transform symlinks to directories into real dirs\n");
|
||||||
printf(" -K, --keep-dirlinks Keep an existing symlink-to-dir as that dir\n");
|
printf(" -K, --keep-dirlinks Keep an existing symlink-to-dir as that dir\n");
|
||||||
printf(" --munge-links Munge symlink targets on the wire (sender)\n");
|
printf(" --munge-links Munge stored symlink targets (/rsyncd-munged/) on the receiver\n");
|
||||||
printf(" -H, --hard-links Preserve hard-link relationships across the transfer\n");
|
printf(" -H, --hard-links Preserve hard-link relationships across the transfer\n");
|
||||||
printf(" -S, --sparse Handle sparse files efficiently\n");
|
printf(" -S, --sparse Handle sparse files efficiently\n");
|
||||||
printf(
|
printf(
|
||||||
@@ -275,8 +341,7 @@ void print_usage(void) {
|
|||||||
printf(
|
printf(
|
||||||
" --devices Recreate device nodes on the destination (privileged; skipped when\n");
|
" --devices Recreate device nodes on the destination (privileged; skipped when\n");
|
||||||
printf(" the receiver lacks CAP_MKNOD)\n");
|
printf(" the receiver lacks CAP_MKNOD)\n");
|
||||||
printf(" --specials Recreate special files (FIFOs) on the destination (sockets "
|
printf(" --specials Recreate special files (FIFOs, sockets) on the destination\n");
|
||||||
"skipped)\n");
|
|
||||||
printf(" --copy-devices Copy a source device's content as a regular file instead\n");
|
printf(" --copy-devices Copy a source device's content as a regular file instead\n");
|
||||||
printf(" --write-devices Write received data into an existing destination device node\n");
|
printf(" --write-devices Write received data into an existing destination device node\n");
|
||||||
printf(" --inplace Update files in-place (no temp+rename)\n");
|
printf(" --inplace Update files in-place (no temp+rename)\n");
|
||||||
@@ -287,9 +352,12 @@ void print_usage(void) {
|
|||||||
printf(" --append-verify Like --append, but verifies the retained prefix checksum\n");
|
printf(" --append-verify Like --append, but verifies the retained prefix checksum\n");
|
||||||
printf(" before appending (falls back to a full transfer on mismatch)\n");
|
printf(" before appending (falls back to a full transfer on mismatch)\n");
|
||||||
printf(" --fsync Fsync every written file before publication\n");
|
printf(" --fsync Fsync every written file before publication\n");
|
||||||
printf(" --compress-level <n> Compression level (default: 5)\n");
|
printf(" --compress-level <n> Compression level (per-codec default: zstd 3,\n");
|
||||||
|
printf(" zlib/zlibx 6, lz4 ignores it)\n");
|
||||||
printf(" --zl <n> Alias for --compress-level\n");
|
printf(" --zl <n> Alias for --compress-level\n");
|
||||||
printf(" --skip-compress=LIST Skip compression for comma-separated suffixes\n");
|
printf(" --skip-compress=LIST Skip compression for suffixes in LIST (separated by\n");
|
||||||
|
printf(" '/' as in rsync, or ','); a leading dot is optional. The\n");
|
||||||
|
printf(" default is rsync 3.4.1's built-in skip-compress list\n");
|
||||||
printf(" --compress-threads <n> Compression worker threads (requires zstd threaded support)\n");
|
printf(" --compress-threads <n> Compression worker threads (requires zstd threaded support)\n");
|
||||||
printf(" --no-OPTION Disable a supported boolean option\n");
|
printf(" --no-OPTION Disable a supported boolean option\n");
|
||||||
printf(" --help Show this help\n");
|
printf(" --help Show this help\n");
|
||||||
@@ -297,7 +365,20 @@ void print_usage(void) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
void print_debug_usage(void) {
|
void print_debug_usage(void) {
|
||||||
printf("Supported debug flags: IO,PROTO,PACK,UTIL,ALL,NONE\n");
|
printf("Emitting debug flags: IO,PROTO,PACK,UTIL,FLIST,DEL,HASH,DELTASUM,\n");
|
||||||
|
printf("RECV,FILTER,SEND,ALL,NONE\n");
|
||||||
|
printf("Also accepted for rsync CLI parity (silent): ACL,BACKUP,BIND,CHDIR,\n");
|
||||||
|
printf("CONNECT,CMD,DUP,EXIT,FUZZY,GENR,HLINK,ICONV,NSTR,OWN,TIME.\n");
|
||||||
printf("Flags may be comma-separated, for example: --debug=io,proto\n");
|
printf("Flags may be comma-separated, for example: --debug=io,proto\n");
|
||||||
printf("Other rsync debug flags are unsupported and rejected.\n");
|
printf("An optional level suffix is accepted (e.g. --debug=io2); level 0\n");
|
||||||
|
printf("silences that item. Unknown names are rejected.\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
void print_info_usage(void) {
|
||||||
|
printf("Emitting info flags: COPY,MISC,SKIP,STATS,DEL,REMOVE,NAME,FLIST,\n");
|
||||||
|
printf("NONREG,PROGRESS,MOUNT,ALL,NONE\n");
|
||||||
|
printf("Also accepted for rsync CLI parity (silent): BACKUP,SYMS,SYMSAFE.\n");
|
||||||
|
printf("Flags may be comma-separated, for example: --info=name,stats\n");
|
||||||
|
printf("An optional level suffix is accepted (e.g. --info=stats2); level 0\n");
|
||||||
|
printf("silences that item. Unknown names are rejected.\n");
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,5 +3,6 @@
|
|||||||
|
|
||||||
void print_usage(void);
|
void print_usage(void);
|
||||||
void print_debug_usage(void);
|
void print_debug_usage(void);
|
||||||
|
void print_info_usage(void);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
+492
-153
@@ -3,6 +3,7 @@
|
|||||||
#include "charset.h"
|
#include "charset.h"
|
||||||
#include "chunk.h"
|
#include "chunk.h"
|
||||||
#include "config.h"
|
#include "config.h"
|
||||||
|
#include "delete_plan.h"
|
||||||
#include "delay_updates.h"
|
#include "delay_updates.h"
|
||||||
#include "file.h"
|
#include "file.h"
|
||||||
#include "file_receive.h"
|
#include "file_receive.h"
|
||||||
@@ -11,6 +12,7 @@
|
|||||||
#include "protocol.h"
|
#include "protocol.h"
|
||||||
#include "utils.h"
|
#include "utils.h"
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
#include <sys/stat.h>
|
#include <sys/stat.h>
|
||||||
#include <time.h>
|
#include <time.h>
|
||||||
|
|
||||||
@@ -43,17 +45,79 @@ void receiver_outcomes_destroy(ReceiverOutcomes* outcomes) {
|
|||||||
/* End-of-transfer success frame. When --remove-source-files was negotiated
|
/* End-of-transfer success frame. When --remove-source-files was negotiated
|
||||||
each processed data file is acknowledged first (STATUS_NEXT = written,
|
each processed data file is acknowledged first (STATUS_NEXT = written,
|
||||||
STATUS_OK = skipped) so the sender never removes a source the receiver did
|
STATUS_OK = skipped) so the sender never removes a source the receiver did
|
||||||
not actually store. The frame always ends with a plain STATUS_OK. */
|
not actually store. The frame ends with `final_status` (STATUS_OK, or
|
||||||
bool receiver_send_final_success(int fd, const Config* config, const ReceiverOutcomes* outcomes) {
|
STATUS_DELETE_LIMIT when a --max-delete commit was capped). */
|
||||||
|
bool receiver_send_final_success(int fd, const Config* config, const ReceiverOutcomes* outcomes,
|
||||||
|
Status final_status) {
|
||||||
if (!config->remove_source_files)
|
if (!config->remove_source_files)
|
||||||
return send_status(fd, STATUS_OK);
|
return send_status(fd, final_status);
|
||||||
size_t count = outcomes ? outcomes->count : 0;
|
size_t count = outcomes ? outcomes->count : 0;
|
||||||
for (size_t i = 0; i < count; i++) {
|
for (size_t i = 0; i < count; i++) {
|
||||||
Status per_file = outcomes->entries[i] == FILE_SAVE_WRITTEN ? STATUS_NEXT : STATUS_OK;
|
Status per_file;
|
||||||
|
if (outcomes->entries[i] == FILE_SAVE_WRITTEN)
|
||||||
|
per_file = STATUS_NEXT;
|
||||||
|
else if (outcomes->entries[i] == FILE_SAVE_FAILED)
|
||||||
|
per_file = STATUS_ERROR;
|
||||||
|
else
|
||||||
|
per_file = STATUS_OK;
|
||||||
if (!send_status(fd, per_file))
|
if (!send_status(fd, per_file))
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
return send_status(fd, STATUS_OK);
|
return send_status(fd, final_status);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool receiver_send_stats_frame(int fd, const Config* config, const ReceiverStats* stats,
|
||||||
|
const struct ArrayList* would_delete,
|
||||||
|
const struct ArrayList* deleted_paths) {
|
||||||
|
if (!config->report_stats)
|
||||||
|
return true;
|
||||||
|
ReceiverStats local;
|
||||||
|
memset(&local, 0, sizeof(local));
|
||||||
|
const ReceiverStats* out = stats ? stats : &local;
|
||||||
|
/* The path list carries the dry-run would-delete set for a -n run and the
|
||||||
|
actually-removed set for a real --info=del run. */
|
||||||
|
const struct ArrayList* paths =
|
||||||
|
config->dry_run ? would_delete : (config->report_deletes ? deleted_paths : NULL);
|
||||||
|
size_t count = paths ? (size_t)paths->size : 0;
|
||||||
|
if (count > (size_t)MAX_MANIFEST_ENTRIES)
|
||||||
|
count = MAX_MANIFEST_ENTRIES;
|
||||||
|
ReceiverStats record = *out;
|
||||||
|
record.would_delete_count = count;
|
||||||
|
if (!send_status(fd, STATUS_STATS) || !format_stats_send(fd, &record) ||
|
||||||
|
!send_int(fd, (int)count))
|
||||||
|
return false;
|
||||||
|
for (size_t i = 0; i < count; i++) {
|
||||||
|
const char* path = (const char*)paths->items[i];
|
||||||
|
if (!send_wire_str(fd, path ? path : ""))
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Add a delete commit's tally to the sink's end-of-transfer wire counters (when
|
||||||
|
the sink reports them). Runs on the receiving thread, so no locking. */
|
||||||
|
static void receiver_tally_deleted(const ReceiverSink* sink, size_t deleted) {
|
||||||
|
if (sink && sink->stats && deleted > 0)
|
||||||
|
sink->stats->deleted_files += deleted;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Observer for --info=del: record each truly-removed destination-relative path
|
||||||
|
in the ArrayList passed as the observer context, so the terminal STATUS_STATS
|
||||||
|
frame can list it. A failed append is best-effort (the deletion already
|
||||||
|
happened; output is cosmetic). Shared by the single-threaded receiver and
|
||||||
|
the -m pipeline's deferred commit. */
|
||||||
|
void receiver_record_deleted_path(void* context, const char* rel_path) {
|
||||||
|
ArrayList* paths = context;
|
||||||
|
if (!paths || !rel_path)
|
||||||
|
return;
|
||||||
|
/* Bound the retained list like the keep-set manifest: only MAX_MANIFEST_ENTRIES
|
||||||
|
paths are ever transmitted in the terminal STATUS_STATS frame, so recording
|
||||||
|
more only grows memory. A hostile/huge deletion set is therefore capped. */
|
||||||
|
if ((size_t)paths->size >= (size_t)MAX_MANIFEST_ENTRIES)
|
||||||
|
return;
|
||||||
|
char* copy = str_dup(rel_path);
|
||||||
|
if (copy && !array_list_add(paths, copy))
|
||||||
|
free(copy);
|
||||||
}
|
}
|
||||||
|
|
||||||
static bool receiver_process_chunk(Chunk* chunk, const ReceiverSink* sink) {
|
static bool receiver_process_chunk(Chunk* chunk, const ReceiverSink* sink) {
|
||||||
@@ -242,21 +306,281 @@ static bool receiver_note_status(const struct timespec* session_start,
|
|||||||
}
|
}
|
||||||
|
|
||||||
int receiver_process(Config* config, int file_descriptor, const ReceiverSink* sink) {
|
int receiver_process(Config* config, int file_descriptor, const ReceiverSink* sink) {
|
||||||
return receiver_process_pending(config, file_descriptor, sink, NULL);
|
return receiver_process_pending(config, file_descriptor, sink, NULL, NULL);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Per-connection state threaded through the status handlers below. The parked
|
||||||
|
keep-set / per-directory session live here so one teardown helper can release
|
||||||
|
them on every exit path. */
|
||||||
|
typedef struct {
|
||||||
|
Config* config;
|
||||||
|
int fd;
|
||||||
|
const ReceiverSink* sink;
|
||||||
|
DeleteManifest** pending_manifest;
|
||||||
|
DeletePlanSession** pending_plans;
|
||||||
|
/* Parked keep-set for the late/commit timing. Every exit path frees it
|
||||||
|
exactly once; the only exception is the successful FINISHED handoff, which
|
||||||
|
transfers ownership to *pending_manifest (used by the -m receiver). */
|
||||||
|
DeleteManifest* deferred_manifest;
|
||||||
|
/* Per-directory delete session for --delete-during/--delete-delay. During the
|
||||||
|
loop it applies plans inline (during) or snapshots their extras (delay); on
|
||||||
|
a successful FINISHED it is either committed here or handed to
|
||||||
|
*pending_plans so the -m caller commits after its disk writer drained. */
|
||||||
|
DeletePlanSession* plan_session;
|
||||||
|
bool early_delete;
|
||||||
|
bool per_dir_delete;
|
||||||
|
bool delete_limit_noted;
|
||||||
|
} ReceiverPendingState;
|
||||||
|
|
||||||
|
/* Outcome of one frame handler. NEXT reads the following status frame; FAIL
|
||||||
|
tears the connection down without a peer STATUS_ERROR; ERROR tears it down
|
||||||
|
and (when the sink owns error reporting) emits STATUS_ERROR. */
|
||||||
|
typedef enum {
|
||||||
|
RECEIVER_STEP_NEXT,
|
||||||
|
RECEIVER_STEP_FAIL,
|
||||||
|
RECEIVER_STEP_ERROR,
|
||||||
|
} ReceiverStep;
|
||||||
|
|
||||||
|
static ReceiverStep receiver_handle_keepalive(ReceiverPendingState* state) {
|
||||||
|
if (!send_status(state->fd, STATUS_KEEPALIVE))
|
||||||
|
return RECEIVER_STEP_FAIL;
|
||||||
|
return RECEIVER_STEP_NEXT;
|
||||||
|
}
|
||||||
|
|
||||||
|
static ReceiverStep receiver_handle_abort(ReceiverPendingState* state) {
|
||||||
|
(void)state;
|
||||||
|
log_message(LOG_LEVEL_INFO, "Received abort from client, cleaning up");
|
||||||
|
return RECEIVER_STEP_FAIL;
|
||||||
|
}
|
||||||
|
|
||||||
|
static ReceiverStep receiver_handle_check(ReceiverPendingState* state) {
|
||||||
|
bool skipped = false;
|
||||||
|
bool would_transfer = false;
|
||||||
|
File* file = receive_incremental_check_ex(state->fd, state->config, &skipped, &would_transfer);
|
||||||
|
if (state->config->dry_run) {
|
||||||
|
/* Server-contacting --dry-run: the reply has already been sent
|
||||||
|
(STATUS_OK = up to date, STATUS_DRY_RUN_TRANSFER = would transfer) and
|
||||||
|
nothing may be stored. Both flags false means a genuine protocol
|
||||||
|
error (STATUS_ERROR already sent or sent by receive_error below). */
|
||||||
|
if (!skipped && !would_transfer)
|
||||||
|
return RECEIVER_STEP_ERROR;
|
||||||
|
} else if (!skipped && (!file || !state->sink->store_file(file, state->sink->context))) {
|
||||||
|
return RECEIVER_STEP_ERROR;
|
||||||
|
}
|
||||||
|
return RECEIVER_STEP_NEXT;
|
||||||
|
}
|
||||||
|
|
||||||
|
static ReceiverStep receiver_handle_chunk(ReceiverPendingState* state) {
|
||||||
|
Chunk* chunk = receive_chunk_data(state->fd, state->config);
|
||||||
|
if (!chunk || !receiver_process_chunk(chunk, state->sink))
|
||||||
|
return RECEIVER_STEP_ERROR;
|
||||||
|
return RECEIVER_STEP_NEXT;
|
||||||
|
}
|
||||||
|
|
||||||
|
static ReceiverStep receiver_handle_check_batch(ReceiverPendingState* state) {
|
||||||
|
if (!receiver_process_batch(state->config, state->fd))
|
||||||
|
return RECEIVER_STEP_FAIL;
|
||||||
|
return RECEIVER_STEP_NEXT;
|
||||||
|
}
|
||||||
|
|
||||||
|
static ReceiverStep receiver_handle_mkdir(ReceiverPendingState* state) {
|
||||||
|
File* dir = file_receive_directory(state->fd, state->config);
|
||||||
|
if (!dir || !state->sink->store_file(dir, state->sink->context))
|
||||||
|
return RECEIVER_STEP_ERROR;
|
||||||
|
return RECEIVER_STEP_NEXT;
|
||||||
|
}
|
||||||
|
|
||||||
|
static ReceiverStep receiver_handle_dir_times(const ReceiverPendingState* state) {
|
||||||
|
if (!receiver_process_dir_times(state->fd, state->config, state->sink))
|
||||||
|
return RECEIVER_STEP_ERROR;
|
||||||
|
return RECEIVER_STEP_NEXT;
|
||||||
|
}
|
||||||
|
|
||||||
|
static ReceiverStep receiver_handle_hardlink(ReceiverPendingState* state) {
|
||||||
|
File* file = file_receive_hardlink(state->fd);
|
||||||
|
if (!file || !state->sink->store_file(file, state->sink->context))
|
||||||
|
return RECEIVER_STEP_ERROR;
|
||||||
|
return RECEIVER_STEP_NEXT;
|
||||||
|
}
|
||||||
|
|
||||||
|
static ReceiverStep receiver_handle_symlink(ReceiverPendingState* state) {
|
||||||
|
File* sym = file_receive_symlink(state->fd, state->config);
|
||||||
|
if (!sym || !state->sink->store_file(sym, state->sink->context))
|
||||||
|
return RECEIVER_STEP_ERROR;
|
||||||
|
return RECEIVER_STEP_NEXT;
|
||||||
|
}
|
||||||
|
|
||||||
|
static ReceiverStep receiver_handle_special(ReceiverPendingState* state) {
|
||||||
|
File* file = file_receive_special(state->fd);
|
||||||
|
if (!file || !state->sink->store_file(file, state->sink->context))
|
||||||
|
return RECEIVER_STEP_ERROR;
|
||||||
|
return RECEIVER_STEP_NEXT;
|
||||||
|
}
|
||||||
|
|
||||||
|
static ReceiverStep receiver_handle_manifest(ReceiverPendingState* state) {
|
||||||
|
Config* config = state->config;
|
||||||
|
int fd = state->fd;
|
||||||
|
const ReceiverSink* sink = state->sink;
|
||||||
|
DeleteManifest* manifest = receive_manifest_entries(fd);
|
||||||
|
if (!manifest)
|
||||||
|
return RECEIVER_STEP_FAIL; /* receive_manifest_entries already sent STATUS_ERROR */
|
||||||
|
if (config->dry_run) {
|
||||||
|
/* Server-contacting --dry-run mutates nothing, so a keep-set manifest
|
||||||
|
is consumed and discarded. The early-delete mode still needs its ACK
|
||||||
|
so a sender blocked on the delete handshake is not left hanging.
|
||||||
|
When would-delete reporting is armed, enumerate (read-only) the
|
||||||
|
destination extras so the terminal STATUS_STATS frame can list them. */
|
||||||
|
if (config->use_delete && sink->would_delete) {
|
||||||
|
size_t count = 0;
|
||||||
|
if (!manifest_would_delete_list(config, manifest, sink->would_delete, &count))
|
||||||
|
log_message(LOG_LEVEL_WARNING, "dry-run: could not enumerate would-delete paths");
|
||||||
|
}
|
||||||
|
delete_manifest_free(manifest);
|
||||||
|
if (state->early_delete && !send_status(fd, STATUS_OK))
|
||||||
|
return RECEIVER_STEP_FAIL;
|
||||||
|
return RECEIVER_STEP_NEXT;
|
||||||
|
}
|
||||||
|
if (state->early_delete) {
|
||||||
|
/* --delete-before: the whole-tree manifest is authoritative the moment
|
||||||
|
it arrives, before any file data. Delete now and acknowledge so the
|
||||||
|
sender only starts streaming once the deletion committed (or failed).
|
||||||
|
A later transfer failure does not restore these deletions. A
|
||||||
|
--max-delete-capped commit still succeeds and the transfer proceeds;
|
||||||
|
the terminal success frame reports the cap. */
|
||||||
|
size_t deleted = 0;
|
||||||
|
DeletePathObserver observer =
|
||||||
|
(config->report_deletes && sink->deleted_paths) ? receiver_record_deleted_path : NULL;
|
||||||
|
DeleteCommitResult deletion =
|
||||||
|
(config->use_delete || config->delete_missing_args)
|
||||||
|
? manifest_delete_all_observed(config, manifest, &deleted, observer,
|
||||||
|
(void*)sink->deleted_paths)
|
||||||
|
: DELETE_COMMIT_OK;
|
||||||
|
receiver_tally_deleted(sink, deleted);
|
||||||
|
delete_manifest_free(manifest);
|
||||||
|
if (deletion == DELETE_COMMIT_ERROR) {
|
||||||
|
send_status(fd, STATUS_ERROR);
|
||||||
|
return RECEIVER_STEP_FAIL;
|
||||||
|
}
|
||||||
|
if (deletion == DELETE_COMMIT_LIMIT_REACHED && sink->note_delete_limit)
|
||||||
|
sink->note_delete_limit(sink->context);
|
||||||
|
if (!send_status(fd, STATUS_OK))
|
||||||
|
return RECEIVER_STEP_FAIL;
|
||||||
|
} else if (config->use_delete || config->delete_missing_args) {
|
||||||
|
/* Plain --delete / --delete-after and the --delete-missing-args
|
||||||
|
exact-path deletions: hold the manifest and commit it only after
|
||||||
|
STATUS_FINISHED. The per-directory modes never send this frame. */
|
||||||
|
if (state->deferred_manifest) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Received a second delete manifest");
|
||||||
|
delete_manifest_free(state->deferred_manifest);
|
||||||
|
state->deferred_manifest = NULL;
|
||||||
|
delete_manifest_free(manifest);
|
||||||
|
send_status(fd, STATUS_ERROR);
|
||||||
|
return RECEIVER_STEP_FAIL;
|
||||||
|
}
|
||||||
|
state->deferred_manifest = manifest;
|
||||||
|
} else {
|
||||||
|
delete_manifest_free(manifest);
|
||||||
|
}
|
||||||
|
return RECEIVER_STEP_NEXT;
|
||||||
|
}
|
||||||
|
|
||||||
|
static ReceiverStep receiver_handle_delete_plan(ReceiverPendingState* state) {
|
||||||
|
Config* config = state->config;
|
||||||
|
int fd = state->fd;
|
||||||
|
const ReceiverSink* sink = state->sink;
|
||||||
|
if (!state->per_dir_delete) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Received a per-directory delete plan without a per-dir "
|
||||||
|
"delete timing");
|
||||||
|
send_status(fd, STATUS_ERROR);
|
||||||
|
return RECEIVER_STEP_FAIL;
|
||||||
|
}
|
||||||
|
if (!state->plan_session) {
|
||||||
|
state->plan_session = delete_plan_session_create(config);
|
||||||
|
if (state->plan_session && config->report_deletes && sink->deleted_paths)
|
||||||
|
delete_plan_session_set_delete_observer(state->plan_session, receiver_record_deleted_path,
|
||||||
|
(void*)sink->deleted_paths);
|
||||||
|
}
|
||||||
|
if (!state->plan_session || delete_plan_session_receive(state->plan_session, config, fd) != 0)
|
||||||
|
return RECEIVER_STEP_FAIL;
|
||||||
|
if (delete_plan_session_limit_reached(state->plan_session) && !state->delete_limit_noted &&
|
||||||
|
sink->note_delete_limit) {
|
||||||
|
sink->note_delete_limit(sink->context);
|
||||||
|
state->delete_limit_noted = true;
|
||||||
|
}
|
||||||
|
return RECEIVER_STEP_NEXT;
|
||||||
|
}
|
||||||
|
|
||||||
|
static ReceiverStep receiver_handle_file(ReceiverPendingState* state) {
|
||||||
|
File* file = file_receive(state->config, state->fd);
|
||||||
|
if (!file) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Failed to receive file");
|
||||||
|
return RECEIVER_STEP_ERROR;
|
||||||
|
}
|
||||||
|
if (!state->sink->store_file(file, state->sink->context))
|
||||||
|
return RECEIVER_STEP_ERROR;
|
||||||
|
return RECEIVER_STEP_NEXT;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* One dispatch per admitted frame type; STATUS_NEXT (and any other
|
||||||
|
data-bearing status) falls through to the regular file receiver. */
|
||||||
|
static ReceiverStep receiver_dispatch_status(ReceiverPendingState* state, Status status) {
|
||||||
|
switch (status) {
|
||||||
|
case STATUS_KEEPALIVE:
|
||||||
|
return receiver_handle_keepalive(state);
|
||||||
|
case STATUS_ABORT:
|
||||||
|
return receiver_handle_abort(state);
|
||||||
|
case STATUS_CHECK:
|
||||||
|
return receiver_handle_check(state);
|
||||||
|
case STATUS_CHUNK:
|
||||||
|
return receiver_handle_chunk(state);
|
||||||
|
case STATUS_CHECK_BATCH:
|
||||||
|
return receiver_handle_check_batch(state);
|
||||||
|
case STATUS_MKDIR:
|
||||||
|
return receiver_handle_mkdir(state);
|
||||||
|
case STATUS_DIR_TIMES:
|
||||||
|
return receiver_handle_dir_times(state);
|
||||||
|
case STATUS_HARDLINK:
|
||||||
|
return receiver_handle_hardlink(state);
|
||||||
|
case STATUS_SYMLINK:
|
||||||
|
return receiver_handle_symlink(state);
|
||||||
|
case STATUS_SPECIAL:
|
||||||
|
return receiver_handle_special(state);
|
||||||
|
case STATUS_MANIFEST:
|
||||||
|
return receiver_handle_manifest(state);
|
||||||
|
case STATUS_DELETE_PLAN:
|
||||||
|
return receiver_handle_delete_plan(state);
|
||||||
|
default:
|
||||||
|
return receiver_handle_file(state);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Release the parked keep-set / per-directory session exactly once on every
|
||||||
|
failure exit. Never commit a deletion for a failed stream. */
|
||||||
|
static void receiver_drop_pending(ReceiverPendingState* state) {
|
||||||
|
if (state->deferred_manifest) {
|
||||||
|
delete_manifest_free(state->deferred_manifest);
|
||||||
|
state->deferred_manifest = NULL;
|
||||||
|
}
|
||||||
|
if (state->plan_session) {
|
||||||
|
delete_plan_session_destroy(state->plan_session);
|
||||||
|
state->plan_session = NULL;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Runs the whole receive loop. The delete manifest may legitimately arrive
|
/* Runs the whole receive loop. The delete manifest may legitimately arrive
|
||||||
either FIRST (--delete-before / --delete-during: the sender transmits the
|
either FIRST (--delete-before / --delete-during: the sender transmits the
|
||||||
validated keep-set before any file data) or LAST (plain --delete /
|
validated keep-set before any file data) or LAST (--delete-after /
|
||||||
--delete-after / --delete-delay: the manifest closes the data stream). In
|
--delete-commit / --delete-delay: the manifest closes the data stream). In
|
||||||
the early modes the receiver deletes as soon as the manifest has been read
|
the early modes the receiver deletes as soon as the manifest has been read
|
||||||
and acknowledges with STATUS_OK so the sender only starts streaming once the
|
and acknowledges with STATUS_OK so the sender only starts streaming once the
|
||||||
deletion has committed (or failed); in the late modes the manifest is held
|
deletion has committed (or failed); in the late modes the manifest is held
|
||||||
and the deletion is committed only after the terminal STATUS_FINISHED proves
|
and the deletion is committed only after the terminal STATUS_FINISHED proves
|
||||||
the whole transfer succeeded. See receiver_process_pending() for how the -m
|
the whole transfer succeeded. A plain --delete defaults to the per-directory
|
||||||
receiver defers that commit until its disk writer has drained. */
|
delete-during plan mode (no manifest at all). See the per-frame handlers
|
||||||
|
above for how the -m receiver defers that commit until its disk writer has
|
||||||
|
drained. */
|
||||||
int receiver_process_pending(Config* config, int file_descriptor, const ReceiverSink* sink,
|
int receiver_process_pending(Config* config, int file_descriptor, const ReceiverSink* sink,
|
||||||
DeleteManifest** pending_manifest) {
|
DeleteManifest** pending_manifest, DeletePlanSession** pending_plans) {
|
||||||
Status status;
|
Status status;
|
||||||
if (!receive_status(file_descriptor, &status))
|
if (!receive_status(file_descriptor, &status))
|
||||||
return -1;
|
return -1;
|
||||||
@@ -269,121 +593,29 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver
|
|||||||
last_progress = session_start;
|
last_progress = session_start;
|
||||||
if (!receiver_note_status(&session_start, &last_progress, status, file_descriptor, sink))
|
if (!receiver_note_status(&session_start, &last_progress, status, file_descriptor, sink))
|
||||||
return -1;
|
return -1;
|
||||||
bool early_delete = config_delete_timing_early(config);
|
ReceiverPendingState state = {
|
||||||
/* Parked keep-set for the late/commit timing. Every exit path below frees it
|
.config = config,
|
||||||
exactly once; the only exception is the successful FINISHED handoff, which
|
.fd = file_descriptor,
|
||||||
transfers ownership to *pending_manifest (used by the -m receiver). */
|
.sink = sink,
|
||||||
DeleteManifest* deferred_manifest = NULL;
|
.pending_manifest = pending_manifest,
|
||||||
|
.pending_plans = pending_plans,
|
||||||
|
.deferred_manifest = NULL,
|
||||||
|
.plan_session = NULL,
|
||||||
|
.early_delete = config_delete_timing_early(config),
|
||||||
|
.per_dir_delete = config_delete_timing_per_dir(config),
|
||||||
|
.delete_limit_noted = false,
|
||||||
|
};
|
||||||
|
bool notify_peer = false;
|
||||||
while (status == STATUS_NEXT || status == STATUS_CHUNK || status == STATUS_CHECK ||
|
while (status == STATUS_NEXT || status == STATUS_CHUNK || status == STATUS_CHECK ||
|
||||||
status == STATUS_KEEPALIVE || status == STATUS_ABORT || status == STATUS_CHECK_BATCH ||
|
status == STATUS_KEEPALIVE || status == STATUS_ABORT || status == STATUS_CHECK_BATCH ||
|
||||||
status == STATUS_MKDIR || status == STATUS_MANIFEST || status == STATUS_HARDLINK ||
|
status == STATUS_MKDIR || status == STATUS_MANIFEST || status == STATUS_HARDLINK ||
|
||||||
status == STATUS_SYMLINK || status == STATUS_SPECIAL || status == STATUS_DIR_TIMES) {
|
status == STATUS_SYMLINK || status == STATUS_SPECIAL || status == STATUS_DIR_TIMES ||
|
||||||
if (status == STATUS_KEEPALIVE) {
|
status == STATUS_DELETE_PLAN) {
|
||||||
if (!send_status(file_descriptor, STATUS_KEEPALIVE))
|
ReceiverStep step = receiver_dispatch_status(&state, status);
|
||||||
goto fail;
|
if (step == RECEIVER_STEP_FAIL)
|
||||||
goto next_status;
|
|
||||||
}
|
|
||||||
if (status == STATUS_ABORT) {
|
|
||||||
log_message(LOG_LEVEL_INFO, "Received abort from client, cleaning up");
|
|
||||||
goto fail;
|
goto fail;
|
||||||
}
|
if (step == RECEIVER_STEP_ERROR)
|
||||||
if (status == STATUS_CHECK) {
|
goto receive_error;
|
||||||
bool skipped = false;
|
|
||||||
bool would_transfer = false;
|
|
||||||
File* file = receive_incremental_check_ex(file_descriptor, config, &skipped, &would_transfer);
|
|
||||||
if (config->dry_run) {
|
|
||||||
/* Server-contacting --dry-run: the reply has already been sent
|
|
||||||
(STATUS_OK = up to date, STATUS_DRY_RUN_TRANSFER = would transfer) and
|
|
||||||
nothing may be stored. Both flags false means a genuine protocol
|
|
||||||
error (STATUS_ERROR already sent or sent by receive_error below). */
|
|
||||||
if (!skipped && !would_transfer)
|
|
||||||
goto receive_error;
|
|
||||||
} else if (!skipped && (!file || !sink->store_file(file, sink->context))) {
|
|
||||||
goto receive_error;
|
|
||||||
}
|
|
||||||
} else if (status == STATUS_CHUNK) {
|
|
||||||
Chunk* chunk = receive_chunk_data(file_descriptor, config);
|
|
||||||
if (!chunk || !receiver_process_chunk(chunk, sink))
|
|
||||||
goto receive_error;
|
|
||||||
} else if (status == STATUS_CHECK_BATCH) {
|
|
||||||
if (!receiver_process_batch(config, file_descriptor))
|
|
||||||
goto fail;
|
|
||||||
goto next_status;
|
|
||||||
} else if (status == STATUS_MKDIR) {
|
|
||||||
File* dir = file_receive_directory(file_descriptor, config);
|
|
||||||
if (!dir || !sink->store_file(dir, sink->context))
|
|
||||||
goto receive_error;
|
|
||||||
} else if (status == STATUS_DIR_TIMES) {
|
|
||||||
if (!receiver_process_dir_times(file_descriptor, config, sink))
|
|
||||||
goto receive_error;
|
|
||||||
} else if (status == STATUS_HARDLINK) {
|
|
||||||
File* file = file_receive_hardlink(file_descriptor);
|
|
||||||
if (!file || !sink->store_file(file, sink->context))
|
|
||||||
goto receive_error;
|
|
||||||
} else if (status == STATUS_SYMLINK) {
|
|
||||||
File* sym = file_receive_symlink(file_descriptor, config);
|
|
||||||
if (!sym || !sink->store_file(sym, sink->context))
|
|
||||||
goto receive_error;
|
|
||||||
} else if (status == STATUS_SPECIAL) {
|
|
||||||
File* file = file_receive_special(file_descriptor);
|
|
||||||
if (!file || !sink->store_file(file, sink->context))
|
|
||||||
goto receive_error;
|
|
||||||
} else if (status == STATUS_MANIFEST) {
|
|
||||||
DeleteManifest* manifest = receive_manifest_entries(file_descriptor);
|
|
||||||
if (!manifest)
|
|
||||||
goto fail; /* receive_manifest_entries already sent STATUS_ERROR */
|
|
||||||
if (config->dry_run) {
|
|
||||||
/* Server-contacting --dry-run mutates nothing, so a keep-set manifest
|
|
||||||
is consumed and discarded. The early-delete mode still needs its ACK
|
|
||||||
so a sender blocked on the delete handshake is not left hanging. */
|
|
||||||
delete_manifest_free(manifest);
|
|
||||||
if (early_delete && !send_status(file_descriptor, STATUS_OK))
|
|
||||||
goto fail;
|
|
||||||
goto next_status;
|
|
||||||
}
|
|
||||||
if (early_delete) {
|
|
||||||
/* --delete-before / --delete-during: the manifest is authoritative the
|
|
||||||
moment it arrives, before any file data. Delete now and acknowledge
|
|
||||||
so the sender only starts streaming once the deletion committed (or
|
|
||||||
failed). This is the rsync delete-before/delete-during window: a
|
|
||||||
later transfer failure does not restore these deletions. */
|
|
||||||
bool deletion_ok = (config->use_delete || config->delete_missing_args)
|
|
||||||
? manifest_delete_all(config, manifest)
|
|
||||||
: true;
|
|
||||||
delete_manifest_free(manifest);
|
|
||||||
if (!deletion_ok) {
|
|
||||||
send_status(file_descriptor, STATUS_ERROR);
|
|
||||||
goto fail;
|
|
||||||
}
|
|
||||||
if (!send_status(file_descriptor, STATUS_OK))
|
|
||||||
goto fail;
|
|
||||||
} else if (config->use_delete || config->delete_missing_args) {
|
|
||||||
/* Plain --delete / --delete-after / --delete-delay and the
|
|
||||||
--delete-missing-args exact-path deletions: hold the manifest and
|
|
||||||
commit it only after STATUS_FINISHED. */
|
|
||||||
if (deferred_manifest) {
|
|
||||||
log_message(LOG_LEVEL_ERROR, "Received a second delete manifest");
|
|
||||||
delete_manifest_free(deferred_manifest);
|
|
||||||
deferred_manifest = NULL;
|
|
||||||
delete_manifest_free(manifest);
|
|
||||||
send_status(file_descriptor, STATUS_ERROR);
|
|
||||||
goto fail;
|
|
||||||
}
|
|
||||||
deferred_manifest = manifest;
|
|
||||||
} else {
|
|
||||||
delete_manifest_free(manifest);
|
|
||||||
}
|
|
||||||
goto next_status;
|
|
||||||
} else {
|
|
||||||
File* file = file_receive(config, file_descriptor);
|
|
||||||
if (!file) {
|
|
||||||
log_message(LOG_LEVEL_ERROR, "Failed to receive file");
|
|
||||||
goto receive_error;
|
|
||||||
}
|
|
||||||
if (!sink->store_file(file, sink->context))
|
|
||||||
goto receive_error;
|
|
||||||
}
|
|
||||||
next_status:
|
|
||||||
if (!receive_status(file_descriptor, &status))
|
if (!receive_status(file_descriptor, &status))
|
||||||
goto receive_error;
|
goto receive_error;
|
||||||
if (!receiver_note_status(&session_start, &last_progress, status, file_descriptor, sink))
|
if (!receiver_note_status(&session_start, &last_progress, status, file_descriptor, sink))
|
||||||
@@ -402,18 +634,55 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver
|
|||||||
disk writer may still be draining; the caller commits after the writer has
|
disk writer may still be draining; the caller commits after the writer has
|
||||||
joined so no extra file is removed unless the transfer is known to have
|
joined so no extra file is removed unless the transfer is known to have
|
||||||
succeeded. */
|
succeeded. */
|
||||||
if (deferred_manifest) {
|
if (state.deferred_manifest) {
|
||||||
if (pending_manifest) {
|
if (state.pending_manifest) {
|
||||||
*pending_manifest = deferred_manifest;
|
*state.pending_manifest = state.deferred_manifest;
|
||||||
deferred_manifest = NULL;
|
state.deferred_manifest = NULL;
|
||||||
} else {
|
} else {
|
||||||
bool deletion_ok = manifest_delete_all(config, deferred_manifest);
|
size_t deleted = 0;
|
||||||
delete_manifest_free(deferred_manifest);
|
DeletePathObserver observer =
|
||||||
deferred_manifest = NULL;
|
(config->report_deletes && sink->deleted_paths) ? receiver_record_deleted_path : NULL;
|
||||||
if (!deletion_ok) {
|
DeleteCommitResult deletion = manifest_delete_all_observed(
|
||||||
|
config, state.deferred_manifest, &deleted, observer, (void*)sink->deleted_paths);
|
||||||
|
receiver_tally_deleted(sink, deleted);
|
||||||
|
delete_manifest_free(state.deferred_manifest);
|
||||||
|
state.deferred_manifest = NULL;
|
||||||
|
if (deletion == DELETE_COMMIT_ERROR) {
|
||||||
send_status(file_descriptor, STATUS_ERROR);
|
send_status(file_descriptor, STATUS_ERROR);
|
||||||
goto fail;
|
goto fail;
|
||||||
}
|
}
|
||||||
|
if (deletion == DELETE_COMMIT_LIMIT_REACHED && sink->note_delete_limit)
|
||||||
|
sink->note_delete_limit(sink->context);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
/* Per-directory deletion: --delete-during already applied each plan inline, so
|
||||||
|
this only finishes the missing-args deletions; --delete-delay committed
|
||||||
|
nothing yet and applies its decompressed snapshot here. The -m receiver
|
||||||
|
hands the session to its caller instead, which commits after the disk
|
||||||
|
writer drained. */
|
||||||
|
if (state.plan_session) {
|
||||||
|
if (config->report_deletes && sink->deleted_paths)
|
||||||
|
delete_plan_session_set_delete_observer(state.plan_session, receiver_record_deleted_path,
|
||||||
|
(void*)sink->deleted_paths);
|
||||||
|
if (state.pending_plans) {
|
||||||
|
*state.pending_plans = state.plan_session;
|
||||||
|
state.plan_session = NULL;
|
||||||
|
} else if (config->dry_run) {
|
||||||
|
/* Central dry-run no-op: never commit a deletion for a -n run. */
|
||||||
|
delete_plan_session_destroy(state.plan_session);
|
||||||
|
state.plan_session = NULL;
|
||||||
|
} else {
|
||||||
|
DeleteCommitResult deletion = delete_plan_session_commit(state.plan_session, config);
|
||||||
|
bool limit = delete_plan_session_limit_reached(state.plan_session);
|
||||||
|
receiver_tally_deleted(sink, delete_plan_session_deleted(state.plan_session));
|
||||||
|
delete_plan_session_destroy(state.plan_session);
|
||||||
|
state.plan_session = NULL;
|
||||||
|
if (deletion == DELETE_COMMIT_ERROR) {
|
||||||
|
send_status(file_descriptor, STATUS_ERROR);
|
||||||
|
goto fail;
|
||||||
|
}
|
||||||
|
if (limit && !state.delete_limit_noted && sink->note_delete_limit)
|
||||||
|
sink->note_delete_limit(sink->context);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (sink->send_success) {
|
if (sink->send_success) {
|
||||||
@@ -426,21 +695,14 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver
|
|||||||
}
|
}
|
||||||
return 0;
|
return 0;
|
||||||
|
|
||||||
|
receive_error:
|
||||||
|
notify_peer = true;
|
||||||
fail:
|
fail:
|
||||||
/* Failure exits that must not (or already did) report a STATUS_ERROR. The
|
/* Failure exits that must not (or already did) report a STATUS_ERROR. The
|
||||||
parked keep-set is dropped: never commit a deletion for a failed stream. */
|
parked keep-set/session is dropped: never commit a deletion for a failed
|
||||||
if (deferred_manifest) {
|
stream. */
|
||||||
delete_manifest_free(deferred_manifest);
|
receiver_drop_pending(&state);
|
||||||
deferred_manifest = NULL;
|
if (notify_peer && sink->send_error)
|
||||||
}
|
|
||||||
return -1;
|
|
||||||
|
|
||||||
receive_error:
|
|
||||||
if (deferred_manifest) {
|
|
||||||
delete_manifest_free(deferred_manifest);
|
|
||||||
deferred_manifest = NULL;
|
|
||||||
}
|
|
||||||
if (sink->send_error)
|
|
||||||
send_status(file_descriptor, STATUS_ERROR);
|
send_status(file_descriptor, STATUS_ERROR);
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
@@ -454,11 +716,27 @@ typedef struct {
|
|||||||
after the whole transfer (and its delete/publication phases) has run so a
|
after the whole transfer (and its delete/publication phases) has run so a
|
||||||
child write never clobbers a directory mtime. */
|
child write never clobbers a directory mtime. */
|
||||||
DirTimeList dir_times;
|
DirTimeList dir_times;
|
||||||
|
/* Set when a --max-delete commit was capped; the terminal frame then carries
|
||||||
|
STATUS_DELETE_LIMIT so the sender exits 25 like rsync. */
|
||||||
|
bool delete_limit_reached;
|
||||||
|
/* End-of-transfer wire counters (protocol 2.25.0) and the -n/--dry-run
|
||||||
|
--delete would-delete path list collected while processing the manifest. */
|
||||||
|
ReceiverStats stats;
|
||||||
|
ArrayList* would_delete;
|
||||||
|
/* --info=del: actually-removed paths collected during the delete commit. */
|
||||||
|
ArrayList* deleted_paths;
|
||||||
|
/* Per-run count of entries that failed to materialize without aborting the
|
||||||
|
stream (currently ONLY a --devices mknod EPERM/EACCES). A nonzero count
|
||||||
|
makes the terminal frame carry a non-OK status so the client exits
|
||||||
|
non-zero, matching rsync's continue-and-exit-partial behavior. */
|
||||||
|
size_t failed_entries;
|
||||||
} ReceiverSaveContext;
|
} ReceiverSaveContext;
|
||||||
|
|
||||||
static bool receiver_save_file(File* file, void* context_pointer) {
|
static bool receiver_save_file(File* file, void* context_pointer) {
|
||||||
ReceiverSaveContext* context = context_pointer;
|
ReceiverSaveContext* context = context_pointer;
|
||||||
FileSaveResult result = FILE_SAVE_ERROR;
|
FileSaveResult result = FILE_SAVE_ERROR;
|
||||||
|
bool created = false;
|
||||||
|
unsigned created_dirs = 0;
|
||||||
if (context->config->dry_run) {
|
if (context->config->dry_run) {
|
||||||
/* Defense in depth: a dry-run receiver mutates nothing even if a data
|
/* Defense in depth: a dry-run receiver mutates nothing even if a data
|
||||||
frame reaches the sink (the sender is not supposed to send one). */
|
frame reaches the sink (the sender is not supposed to send one). */
|
||||||
@@ -468,14 +746,29 @@ static bool receiver_save_file(File* file, void* context_pointer) {
|
|||||||
--remove-source-files sender keeps its source. */
|
--remove-source-files sender keeps its source. */
|
||||||
result = FILE_SAVE_SKIPPED;
|
result = FILE_SAVE_SKIPPED;
|
||||||
} else {
|
} else {
|
||||||
result = file_save_to_disk_full(context->config->receive_root_directory, file, context->config);
|
result = file_save_to_disk_full_ex(context->config->receive_root_directory, file,
|
||||||
|
context->config, &created, &created_dirs);
|
||||||
}
|
}
|
||||||
/* A directory's times are deferred, never applied inline: collect the
|
/* Wire-stats tally: bytes reconstructed from the basis file (delta matches)
|
||||||
metadata now and apply it at the end. -O/--omit-dir-times is honored by
|
count as matched data in the end-of-transfer report. */
|
||||||
dir_time_list_apply's caller (see receiver_send_success_frame). */
|
if (result != FILE_SAVE_ERROR && file->matched_bytes > 0)
|
||||||
|
context->stats.matched_data += file->matched_bytes;
|
||||||
|
/* --devices parity: a device node the receiver could not mknod (EPERM/EACCES)
|
||||||
|
is counted per-run but does not abort the transfer. The terminal frame
|
||||||
|
turns a nonzero count into a non-OK status so the client exits non-zero. */
|
||||||
|
if (result == FILE_SAVE_FAILED)
|
||||||
|
context->failed_entries++;
|
||||||
|
/* Protocol 2.28.0: receiver-observed literal bytes and the created-entry
|
||||||
|
breakdown (regular/dir/link/special) for the `--stats` report. */
|
||||||
|
if (result == FILE_SAVE_WRITTEN)
|
||||||
|
receiver_stats_note_saved(&context->stats, file, created, created_dirs);
|
||||||
|
/* A directory's metadata is deferred, never applied inline: collect it now
|
||||||
|
and apply it at the end. -O/--omit-dir-times and --preserve_perms/-times
|
||||||
|
are honored by dir_metadata_list_apply's caller (see
|
||||||
|
receiver_send_success_frame). */
|
||||||
if (result != FILE_SAVE_ERROR && file->is_dir && file->metadata &&
|
if (result != FILE_SAVE_ERROR && file->is_dir && file->metadata &&
|
||||||
dir_times_should_capture(context->config) &&
|
dir_metadata_should_capture(context->config) &&
|
||||||
!dir_time_list_add(&context->dir_times, file->path, file->metadata)) {
|
!dir_time_list_add(&context->dir_times, file->path, file->metadata, file->xattrs)) {
|
||||||
file_destroy(file);
|
file_destroy(file);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
@@ -493,12 +786,37 @@ static bool receiver_save_file(File* file, void* context_pointer) {
|
|||||||
return result != FILE_SAVE_ERROR;
|
return result != FILE_SAVE_ERROR;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void receiver_note_delete_limit(void* context_pointer) {
|
||||||
|
ReceiverSaveContext* context = context_pointer;
|
||||||
|
context->delete_limit_reached = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Terminal status for a run. A capped --delete limit wins (rsync exit 25);
|
||||||
|
otherwise any per-entry failure (for example an unprivileged --devices
|
||||||
|
mknod) makes the terminal frame non-OK so the client exits non-zero. rsync
|
||||||
|
reports 23 here; mapping the client's exact exit code to 23 is a separate,
|
||||||
|
pre-existing concern. A clean run keeps STATUS_OK. */
|
||||||
|
static Status receiver_final_status(bool delete_limit_reached, size_t failed_entries) {
|
||||||
|
if (delete_limit_reached)
|
||||||
|
return STATUS_DELETE_LIMIT;
|
||||||
|
return failed_entries > 0 ? STATUS_ERROR : STATUS_OK;
|
||||||
|
}
|
||||||
|
|
||||||
static bool receiver_send_success_frame(int fd, void* context_pointer) {
|
static bool receiver_send_success_frame(int fd, void* context_pointer) {
|
||||||
ReceiverSaveContext* context = context_pointer;
|
ReceiverSaveContext* context = context_pointer;
|
||||||
|
if (context->failed_entries > 0)
|
||||||
|
log_message(LOG_LEVEL_WARNING,
|
||||||
|
"%zu entr%s failed to materialize; continuing (partial transfer)",
|
||||||
|
context->failed_entries, context->failed_entries == 1 ? "y" : "ies");
|
||||||
|
Status final_status =
|
||||||
|
receiver_final_status(context->delete_limit_reached, context->failed_entries);
|
||||||
|
if (!receiver_send_stats_frame(fd, context->config, &context->stats, context->would_delete,
|
||||||
|
context->deleted_paths))
|
||||||
|
return false;
|
||||||
/* Server-contacting --dry-run: nothing was staged or written, so there is
|
/* Server-contacting --dry-run: nothing was staged or written, so there is
|
||||||
nothing to publish and no directory times to stamp. */
|
nothing to publish and no directory times to stamp. */
|
||||||
if (context->config->dry_run)
|
if (context->config->dry_run)
|
||||||
return receiver_send_final_success(fd, context->config, &context->outcomes);
|
return receiver_send_final_success(fd, context->config, &context->outcomes, final_status);
|
||||||
/* --delay-updates: the whole protocol stream (including manifest/delete
|
/* --delay-updates: the whole protocol stream (including manifest/delete
|
||||||
handling, which ran inside receiver_process) has succeeded and every
|
handling, which ran inside receiver_process) has succeeded and every
|
||||||
staged file was fully written. Publish them atomically now, before the
|
staged file was fully written. Publish them atomically now, before the
|
||||||
@@ -514,18 +832,39 @@ static bool receiver_send_success_frame(int fd, void* context_pointer) {
|
|||||||
phases have committed, so it is finally safe to stamp directory times.
|
phases have committed, so it is finally safe to stamp directory times.
|
||||||
This runs after the deferred deletion because receiver_process commits it
|
This runs after the deferred deletion because receiver_process commits it
|
||||||
before calling this success frame. */
|
before calling this success frame. */
|
||||||
dir_time_list_apply(&context->dir_times, context->config->receive_root_directory);
|
dir_metadata_list_apply(&context->dir_times, context->config->receive_root_directory,
|
||||||
return receiver_send_final_success(fd, context->config, &context->outcomes);
|
context->config);
|
||||||
|
return receiver_send_final_success(fd, context->config, &context->outcomes, final_status);
|
||||||
}
|
}
|
||||||
|
|
||||||
int receiver_receive_files(Config* config, int file_descriptor) {
|
int receiver_receive_files(Config* config, int file_descriptor) {
|
||||||
ReceiverSaveContext context = {.config = config, .outcomes = {0}};
|
ReceiverSaveContext context = {.config = config, .outcomes = {0}};
|
||||||
dir_time_list_init(&context.dir_times);
|
dir_time_list_init(&context.dir_times);
|
||||||
ReceiverSink sink = {receiver_save_file, &context, true, true, receiver_send_success_frame};
|
context.would_delete = array_list_create(free);
|
||||||
|
/* report_deletes (--info=del / -i / --out-format under --delete) is the only
|
||||||
|
reason to retain the actually-removed paths; a plain --delete must not
|
||||||
|
str_dup every removal. NULL is handled by every consumer. */
|
||||||
|
context.deleted_paths = config->report_deletes ? array_list_create(free) : NULL;
|
||||||
|
if (!context.would_delete || (config->report_deletes && !context.deleted_paths)) {
|
||||||
|
array_list_delete(context.would_delete);
|
||||||
|
array_list_delete(context.deleted_paths);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
ReceiverSink sink = {receiver_save_file,
|
||||||
|
&context,
|
||||||
|
true,
|
||||||
|
true,
|
||||||
|
receiver_send_success_frame,
|
||||||
|
receiver_note_delete_limit,
|
||||||
|
&context.stats,
|
||||||
|
context.would_delete,
|
||||||
|
context.deleted_paths};
|
||||||
int ret = receiver_process(config, file_descriptor, &sink);
|
int ret = receiver_process(config, file_descriptor, &sink);
|
||||||
if (ret != 0 && config->delay_updates && config->delay_context)
|
if (ret != 0 && config->delay_updates && config->delay_context)
|
||||||
delay_updates_cleanup(config->delay_context);
|
delay_updates_cleanup(config->delay_context);
|
||||||
receiver_outcomes_destroy(&context.outcomes);
|
receiver_outcomes_destroy(&context.outcomes);
|
||||||
dir_time_list_free(&context.dir_times);
|
dir_time_list_free(&context.dir_times);
|
||||||
|
array_list_delete(context.would_delete);
|
||||||
|
array_list_delete(context.deleted_paths);
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
|
|||||||
+43
-5
@@ -2,8 +2,10 @@
|
|||||||
#define RECEIVER_H
|
#define RECEIVER_H
|
||||||
|
|
||||||
#include "config.h"
|
#include "config.h"
|
||||||
|
#include "delete_plan.h"
|
||||||
#include "file.h"
|
#include "file.h"
|
||||||
#include "file_receive.h"
|
#include "file_receive.h"
|
||||||
|
#include "protocol.h"
|
||||||
#include <stdbool.h>
|
#include <stdbool.h>
|
||||||
#include <time.h>
|
#include <time.h>
|
||||||
|
|
||||||
@@ -21,6 +23,12 @@ typedef struct {
|
|||||||
|
|
||||||
typedef bool (*ReceiverSuccessFrame)(int fd, void* context);
|
typedef bool (*ReceiverSuccessFrame)(int fd, void* context);
|
||||||
|
|
||||||
|
/* Records that a --max-delete commit stopped with extras left over, so the
|
||||||
|
caller's terminal success frame can carry STATUS_DELETE_LIMIT instead of
|
||||||
|
STATUS_OK. The commit runs on the receiver thread, so the flag is stored in
|
||||||
|
the sink's own context rather than in a shared global. */
|
||||||
|
typedef void (*ReceiverNoteDeleteLimit)(void* context);
|
||||||
|
|
||||||
typedef struct {
|
typedef struct {
|
||||||
ReceiverFileSink store_file;
|
ReceiverFileSink store_file;
|
||||||
void* context;
|
void* context;
|
||||||
@@ -28,23 +36,53 @@ typedef struct {
|
|||||||
bool send_success;
|
bool send_success;
|
||||||
/* Emits the end-of-transfer success frame. When the sender requested
|
/* Emits the end-of-transfer success frame. When the sender requested
|
||||||
--remove-source-files this includes one per-file status per processed
|
--remove-source-files this includes one per-file status per processed
|
||||||
data file followed by the final STATUS_OK; otherwise just STATUS_OK. */
|
data file followed by the final status; otherwise just the final status. */
|
||||||
ReceiverSuccessFrame send_success_frame;
|
ReceiverSuccessFrame send_success_frame;
|
||||||
|
/* Optional; may be NULL when the sink has no --max-delete handling. */
|
||||||
|
ReceiverNoteDeleteLimit note_delete_limit;
|
||||||
|
/* Optional end-of-transfer wire counters (protocol 2.25.0). When non-NULL
|
||||||
|
and the wire config carries report_stats, the success frame is preceded by
|
||||||
|
a STATUS_STATS record; `would_delete` (optional, receiver-owned strings)
|
||||||
|
carries the -n/--dry-run --delete path list. */
|
||||||
|
ReceiverStats* stats;
|
||||||
|
struct ArrayList* would_delete;
|
||||||
|
/* When --info=del requested it, receiver-owned strings for every path the
|
||||||
|
deletion commit ACTUALLY removed, sent in the terminal STATUS_STATS frame's
|
||||||
|
path list so the sender can print rsync's `deleting PATH` lines. */
|
||||||
|
struct ArrayList* deleted_paths;
|
||||||
} ReceiverSink;
|
} ReceiverSink;
|
||||||
|
|
||||||
bool receiver_outcomes_append(ReceiverOutcomes* outcomes, unsigned char code);
|
bool receiver_outcomes_append(ReceiverOutcomes* outcomes, unsigned char code);
|
||||||
void receiver_outcomes_destroy(ReceiverOutcomes* outcomes);
|
void receiver_outcomes_destroy(ReceiverOutcomes* outcomes);
|
||||||
bool receiver_send_final_success(int fd, const Config* config, const ReceiverOutcomes* outcomes);
|
|
||||||
|
/* DeletePathObserver implementation for --info=del: `context` is an ArrayList*
|
||||||
|
that receives owned copies of every truly-removed destination-relative path.
|
||||||
|
Shared by the single-threaded receiver and the -m pipeline's deferred commit. */
|
||||||
|
void receiver_record_deleted_path(void* context, const char* rel_path);
|
||||||
|
|
||||||
|
/* Send the terminal success frame. `final_status` is usually STATUS_OK, or
|
||||||
|
STATUS_DELETE_LIMIT when a --max-delete commit was capped. */
|
||||||
|
bool receiver_send_final_success(int fd, const Config* config, const ReceiverOutcomes* outcomes,
|
||||||
|
Status final_status);
|
||||||
|
|
||||||
|
/* Emit STATUS_STATS (a fixed ReceiverStats record plus, when `would_delete` is
|
||||||
|
non-NULL, a count and that many wire strings) when the wire config requested
|
||||||
|
report_stats. A no-op otherwise. */
|
||||||
|
bool receiver_send_stats_frame(int fd, const Config* config, const ReceiverStats* stats,
|
||||||
|
const struct ArrayList* would_delete,
|
||||||
|
const struct ArrayList* deleted_paths);
|
||||||
|
|
||||||
int receiver_process(Config* config, int file_descriptor, const ReceiverSink* sink);
|
int receiver_process(Config* config, int file_descriptor, const ReceiverSink* sink);
|
||||||
/* receiver_process with an escape hatch for the commit-style (late) deletion:
|
/* receiver_process with an escape hatch for the commit-style (late) deletion:
|
||||||
when `pending_manifest` is non-NULL the receiver does NOT delete at
|
when `pending_manifest` is non-NULL the receiver does NOT delete at
|
||||||
STATUS_FINISHED itself; instead it stores the owned keep-set manifest there
|
STATUS_FINISHED itself; instead it stores the owned keep-set manifest there
|
||||||
(leaving *pending_manifest untouched on early modes/errors) so the caller can
|
(leaving *pending_manifest untouched on early modes/errors) so the caller can
|
||||||
commit the deletion only after its disk writer has fully drained. Pass NULL
|
commit the deletion only after its disk writer has fully drained. Likewise,
|
||||||
to keep the default behaviour (delete before the success frame). */
|
when `pending_plans` is non-NULL the --delete-delay per-directory session is
|
||||||
|
handed to the caller instead of being committed at STATUS_FINISHED. Pass NULL
|
||||||
|
for either to keep the default behaviour (delete before the success frame). */
|
||||||
int receiver_process_pending(Config* config, int file_descriptor, const ReceiverSink* sink,
|
int receiver_process_pending(Config* config, int file_descriptor, const ReceiverSink* sink,
|
||||||
DeleteManifest** pending_manifest);
|
DeleteManifest** pending_manifest, DeletePlanSession** pending_plans);
|
||||||
int receiver_receive_files(Config* config, int file_descriptor);
|
int receiver_receive_files(Config* config, int file_descriptor);
|
||||||
|
|
||||||
/* ---- Connection time bounds (anti-slowloris) ----
|
/* ---- Connection time bounds (anti-slowloris) ----
|
||||||
|
|||||||
@@ -27,6 +27,12 @@ PipelineContextReceiver* pipeline_context_receiver_create(Config* config, Queue*
|
|||||||
context->queued_bytes = 0;
|
context->queued_bytes = 0;
|
||||||
context->max_queue_bytes = 0;
|
context->max_queue_bytes = 0;
|
||||||
context->deferred_manifest = NULL;
|
context->deferred_manifest = NULL;
|
||||||
|
context->deferred_plans = NULL;
|
||||||
|
context->delete_limit_reached = false;
|
||||||
|
context->failed_entries = 0;
|
||||||
|
memset(&context->stats, 0, sizeof(context->stats));
|
||||||
|
context->would_delete = NULL;
|
||||||
|
context->deleted_paths = NULL;
|
||||||
atomic_init(&context->cancelled, false);
|
atomic_init(&context->cancelled, false);
|
||||||
int init = 0;
|
int init = 0;
|
||||||
if (mtx_init(&context->mutex, mtx_plain) != thrd_success)
|
if (mtx_init(&context->mutex, mtx_plain) != thrd_success)
|
||||||
@@ -39,6 +45,18 @@ PipelineContextReceiver* pipeline_context_receiver_create(Config* config, Queue*
|
|||||||
goto fail;
|
goto fail;
|
||||||
// cppcheck-suppress unreadVariable
|
// cppcheck-suppress unreadVariable
|
||||||
init++;
|
init++;
|
||||||
|
context->would_delete = array_list_create(free);
|
||||||
|
if (!context->would_delete)
|
||||||
|
goto fail;
|
||||||
|
/* The actually-removed path list is only needed to render rsync's
|
||||||
|
`deleting PATH` lines, which the client requests via report_deletes
|
||||||
|
(--info=del / -i / --out-format under --delete). A plain --delete run must
|
||||||
|
not allocate it or observe every removal. */
|
||||||
|
if (config->report_deletes) {
|
||||||
|
context->deleted_paths = array_list_create(free);
|
||||||
|
if (!context->deleted_paths)
|
||||||
|
goto fail;
|
||||||
|
}
|
||||||
return context;
|
return context;
|
||||||
|
|
||||||
fail:
|
fail:
|
||||||
@@ -49,6 +67,12 @@ fail:
|
|||||||
cnd_destroy(&context->condition_not_full);
|
cnd_destroy(&context->condition_not_full);
|
||||||
if (init >= 1)
|
if (init >= 1)
|
||||||
mtx_destroy(&context->mutex);
|
mtx_destroy(&context->mutex);
|
||||||
|
/* Free every list that was already created before the failing allocation:
|
||||||
|
`context` itself is freed below, so they would otherwise leak. */
|
||||||
|
if (context->would_delete)
|
||||||
|
array_list_delete(context->would_delete);
|
||||||
|
if (context->deleted_paths)
|
||||||
|
array_list_delete(context->deleted_paths);
|
||||||
free(context);
|
free(context);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
@@ -57,9 +81,15 @@ void pipeline_context_receiver_destroy(PipelineContextReceiver* context) {
|
|||||||
config_delete(context->config);
|
config_delete(context->config);
|
||||||
if (context->deferred_manifest)
|
if (context->deferred_manifest)
|
||||||
delete_manifest_free(context->deferred_manifest);
|
delete_manifest_free(context->deferred_manifest);
|
||||||
|
if (context->deferred_plans)
|
||||||
|
delete_plan_session_destroy(context->deferred_plans);
|
||||||
queue_destroy(context->queue);
|
queue_destroy(context->queue);
|
||||||
receiver_outcomes_destroy(&context->outcomes);
|
receiver_outcomes_destroy(&context->outcomes);
|
||||||
dir_time_list_free(&context->dir_times);
|
dir_time_list_free(&context->dir_times);
|
||||||
|
if (context->would_delete)
|
||||||
|
array_list_delete(context->would_delete);
|
||||||
|
if (context->deleted_paths)
|
||||||
|
array_list_delete(context->deleted_paths);
|
||||||
mtx_destroy(&context->mutex);
|
mtx_destroy(&context->mutex);
|
||||||
cnd_destroy(&context->condition_not_full);
|
cnd_destroy(&context->condition_not_full);
|
||||||
cnd_destroy(&context->condition_not_empty);
|
cnd_destroy(&context->condition_not_empty);
|
||||||
@@ -132,9 +162,23 @@ bool pipeline_context_receiver_enqueue_file(PipelineContextReceiver* context, Fi
|
|||||||
|
|
||||||
static bool receiver_enqueue_file(File* file, void* context_pointer) {
|
static bool receiver_enqueue_file(File* file, void* context_pointer) {
|
||||||
PipelineContextReceiver* context = (PipelineContextReceiver*)context_pointer;
|
PipelineContextReceiver* context = (PipelineContextReceiver*)context_pointer;
|
||||||
|
if (file && file->matched_bytes > 0) {
|
||||||
|
mtx_lock(&context->mutex);
|
||||||
|
context->stats.matched_data += file->matched_bytes;
|
||||||
|
mtx_unlock(&context->mutex);
|
||||||
|
}
|
||||||
return pipeline_context_receiver_enqueue_file(context, file);
|
return pipeline_context_receiver_enqueue_file(context, file);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Early delete modes (--delete-before/--delete-during) commit the manifest
|
||||||
|
inside receiver_process_pending on this thread; record a capped commit so
|
||||||
|
server.c's terminal frame can report STATUS_DELETE_LIMIT. The plain bool is
|
||||||
|
safe: receive_thread writes it before the main thread joins the thread. */
|
||||||
|
static void receiver_pipeline_note_delete_limit(void* context_pointer) {
|
||||||
|
PipelineContextReceiver* context = (PipelineContextReceiver*)context_pointer;
|
||||||
|
context->delete_limit_reached = true;
|
||||||
|
}
|
||||||
|
|
||||||
static void receiver_thread_fail(PipelineContextReceiver* context) {
|
static void receiver_thread_fail(PipelineContextReceiver* context) {
|
||||||
mtx_lock(&context->mutex);
|
mtx_lock(&context->mutex);
|
||||||
atomic_store(&context->cancelled, true);
|
atomic_store(&context->cancelled, true);
|
||||||
@@ -152,9 +196,17 @@ int receive_thread(void* pipeline_context) {
|
|||||||
const Config* config = context->config;
|
const Config* config = context->config;
|
||||||
mtx_unlock(&context->mutex);
|
mtx_unlock(&context->mutex);
|
||||||
|
|
||||||
ReceiverSink sink = {receiver_enqueue_file, context, false, false, NULL};
|
ReceiverSink sink = {receiver_enqueue_file,
|
||||||
if (receiver_process_pending((Config*)config, file_descriptor, &sink,
|
context,
|
||||||
&context->deferred_manifest) != 0) {
|
false,
|
||||||
|
false,
|
||||||
|
NULL,
|
||||||
|
receiver_pipeline_note_delete_limit,
|
||||||
|
&context->stats,
|
||||||
|
context->would_delete,
|
||||||
|
context->deleted_paths};
|
||||||
|
if (receiver_process_pending((Config*)config, file_descriptor, &sink, &context->deferred_manifest,
|
||||||
|
&context->deferred_plans) != 0) {
|
||||||
receiver_thread_fail(context);
|
receiver_thread_fail(context);
|
||||||
protocol_session_unbind();
|
protocol_session_unbind();
|
||||||
return thrd_error;
|
return thrd_error;
|
||||||
@@ -196,12 +248,30 @@ int write_thread(void* pipeline_context) {
|
|||||||
}
|
}
|
||||||
size_t file_bytes = file->data ? file->data->size : 0;
|
size_t file_bytes = file->data ? file->data->size : 0;
|
||||||
FileSaveResult result = FILE_SAVE_SKIPPED;
|
FileSaveResult result = FILE_SAVE_SKIPPED;
|
||||||
|
bool created = false;
|
||||||
|
unsigned created_dirs = 0;
|
||||||
/* Server-contacting --dry-run: never write. The receiver thread does not
|
/* Server-contacting --dry-run: never write. The receiver thread does not
|
||||||
enqueue anything on the dry-run path, but this keeps the writer thread
|
enqueue anything on the dry-run path, but this keeps the writer thread
|
||||||
provably mutation-free if a data frame ever reached it. */
|
provably mutation-free if a data frame ever reached it. */
|
||||||
bool dry_run = context->config->dry_run;
|
bool dry_run = context->config->dry_run;
|
||||||
if (save_to_disk && !dry_run) {
|
if (save_to_disk && !dry_run) {
|
||||||
result = file_save_to_disk_full(root_directory, file, context->config);
|
result =
|
||||||
|
file_save_to_disk_full_ex(root_directory, file, context->config, &created, &created_dirs);
|
||||||
|
if (result == FILE_SAVE_WRITTEN) {
|
||||||
|
/* Protocol 2.28.0: fold the receiver-observed literal bytes and the
|
||||||
|
created-entry type into the shared stats block under its mutex (the
|
||||||
|
receive thread also writes stats.matched_data). */
|
||||||
|
mtx_lock(&context->mutex);
|
||||||
|
receiver_stats_note_saved(&context->stats, file, created, created_dirs);
|
||||||
|
mtx_unlock(&context->mutex);
|
||||||
|
}
|
||||||
|
/* --devices parity: a device node that could not be mknod'ed is counted
|
||||||
|
per-run but does NOT abort the transfer. */
|
||||||
|
if (result == FILE_SAVE_FAILED) {
|
||||||
|
mtx_lock(&context->mutex);
|
||||||
|
context->failed_entries++;
|
||||||
|
mtx_unlock(&context->mutex);
|
||||||
|
}
|
||||||
if (result == FILE_SAVE_ERROR) {
|
if (result == FILE_SAVE_ERROR) {
|
||||||
file_destroy(file);
|
file_destroy(file);
|
||||||
pipeline_context_receiver_note_bytes_released(context, file_bytes);
|
pipeline_context_receiver_note_bytes_released(context, file_bytes);
|
||||||
@@ -220,8 +290,8 @@ int write_thread(void* pipeline_context) {
|
|||||||
write would clobber them); accumulate the metadata here and let the
|
write would clobber them); accumulate the metadata here and let the
|
||||||
caller apply it once every writer has drained. */
|
caller apply it once every writer has drained. */
|
||||||
if (!dry_run && result != FILE_SAVE_ERROR && file->is_dir && file->metadata &&
|
if (!dry_run && result != FILE_SAVE_ERROR && file->is_dir && file->metadata &&
|
||||||
dir_times_should_capture(context->config) &&
|
dir_metadata_should_capture(context->config) &&
|
||||||
!dir_time_list_add(&context->dir_times, file->path, file->metadata)) {
|
!dir_time_list_add(&context->dir_times, file->path, file->metadata, file->xattrs)) {
|
||||||
file_destroy(file);
|
file_destroy(file);
|
||||||
pipeline_context_receiver_note_bytes_released(context, file_bytes);
|
pipeline_context_receiver_note_bytes_released(context, file_bytes);
|
||||||
mtx_lock(&context->mutex);
|
mtx_lock(&context->mutex);
|
||||||
|
|||||||
@@ -41,10 +41,34 @@ typedef struct PipelineContextReceiver {
|
|||||||
transfer truly succeeded. NULL in the early delete modes (which delete at
|
transfer truly succeeded. NULL in the early delete modes (which delete at
|
||||||
the manifest). */
|
the manifest). */
|
||||||
DeleteManifest* deferred_manifest;
|
DeleteManifest* deferred_manifest;
|
||||||
|
/* Per-directory delete session for --delete-delay: receive_thread snapshots
|
||||||
|
each plan's extras as it arrives and hands the session here instead of
|
||||||
|
committing while the disk writer may still be draining; server.c commits it
|
||||||
|
after both threads joined. NULL for every other timing. */
|
||||||
|
DeletePlanSession* deferred_plans;
|
||||||
|
/* Set by server.c when the deferred delete commit hit the --max-delete
|
||||||
|
budget; the terminal success frame then carries STATUS_DELETE_LIMIT
|
||||||
|
(rsync exit 25) while the transfer itself still succeeds. */
|
||||||
|
bool delete_limit_reached;
|
||||||
/* P7 Wave D: directory metadata collected by write_thread from received
|
/* P7 Wave D: directory metadata collected by write_thread from received
|
||||||
directory entries. Only write_thread mutates it (before it joins); the
|
directory entries. Only write_thread mutates it (before it joins); the
|
||||||
caller (server.c) applies it after the delete/delay-updates phase. */
|
caller (server.c) applies it after the delete/delay-updates phase. */
|
||||||
DirTimeList dir_times;
|
DirTimeList dir_times;
|
||||||
|
/* End-of-transfer wire counters (protocol 2.25.0). receive_thread accumulates
|
||||||
|
matched_data under `mutex`; server.c adds the delete-commit tallies after
|
||||||
|
both threads join and emits the STATUS_STATS frame. */
|
||||||
|
ReceiverStats stats;
|
||||||
|
/* -n/--dry-run --delete would-delete path list, collected by receive_thread
|
||||||
|
and reported in the STATUS_STATS frame. */
|
||||||
|
struct ArrayList* would_delete;
|
||||||
|
/* --info=del actually-removed path list, collected by the deferred delete
|
||||||
|
commit in server.c and reported in the STATUS_STATS frame. */
|
||||||
|
struct ArrayList* deleted_paths;
|
||||||
|
/* Per-run count of entries that failed to materialize without aborting the
|
||||||
|
stream (currently ONLY a --devices mknod EPERM/EACCES). write_thread
|
||||||
|
increments it under `mutex`; server.c turns a nonzero count into a non-OK
|
||||||
|
terminal status so the client exits non-zero. */
|
||||||
|
size_t failed_entries;
|
||||||
} PipelineContextReceiver;
|
} PipelineContextReceiver;
|
||||||
|
|
||||||
PipelineContextReceiver* pipeline_context_receiver_create(Config* config, Queue* queue_receiver,
|
PipelineContextReceiver* pipeline_context_receiver_create(Config* config, Queue* queue_receiver,
|
||||||
|
|||||||
+357
-174
@@ -226,11 +226,6 @@ static void release_authorization(void) {
|
|||||||
close(root_fd);
|
close(root_fd);
|
||||||
}
|
}
|
||||||
|
|
||||||
static bool path_is_within(const char* root, const char* path) {
|
|
||||||
size_t n = strlen(root);
|
|
||||||
return strncmp(root, path, n) == 0 && (path[n] == '\0' || path[n] == '/');
|
|
||||||
}
|
|
||||||
|
|
||||||
/* --mkpath contract: when the client's destination root directory does not
|
/* --mkpath contract: when the client's destination root directory does not
|
||||||
exist yet on the server side, --mkpath tells the server to create it (and
|
exist yet on the server side, --mkpath tells the server to create it (and
|
||||||
any missing leading components) below the authorized root at connection
|
any missing leading components) below the authorized root at connection
|
||||||
@@ -389,8 +384,12 @@ static const char* module_gate_check_ownership(const Config* config, const Daemo
|
|||||||
ModuleGateContext* gate_ctx) {
|
ModuleGateContext* gate_ctx) {
|
||||||
if (module->client_owner)
|
if (module->client_owner)
|
||||||
return NULL;
|
return NULL;
|
||||||
/* Ownership: refuse the whole transfer up front (a clear failure). */
|
/* Ownership: refuse the whole transfer up front (a clear failure) for a
|
||||||
if (identity_ownership_requested(config)) {
|
* client-CHOSEN owner/group request. A plain -o/-g/-a preserve-source
|
||||||
|
* request is deliberately not in this narrow set: it falls through to the
|
||||||
|
* super-mode override below, which forces all ownership activity off for this
|
||||||
|
* connection so no chown happens (the transfer itself still succeeds). */
|
||||||
|
if (identity_explicit_ownership_requested(config)) {
|
||||||
log_message(LOG_LEVEL_ERROR,
|
log_message(LOG_LEVEL_ERROR,
|
||||||
"daemon module '%s' refuses client-chosen ownership/super-user activities "
|
"daemon module '%s' refuses client-chosen ownership/super-user activities "
|
||||||
"(no `client owner = yes` opt-in); refusing",
|
"(no `client owner = yes` opt-in); refusing",
|
||||||
@@ -706,56 +705,85 @@ static const char* server_module_gate(const Config* config, void* context) {
|
|||||||
return module_gate_install_root(config, module);
|
return module_gate_install_root(config, module);
|
||||||
}
|
}
|
||||||
|
|
||||||
void handler(int file_descriptor) {
|
/* Per-connection state threaded through the handler phase helpers below. The
|
||||||
SSL* ssl = io_get_ssl();
|
* fields are a faithful split of the former handler() locals: the protocol
|
||||||
|
* session, the config-frame gate context, the accepted config, the optional
|
||||||
|
* multithreaded pipeline context and the teardown bookkeeping all live here so
|
||||||
|
* the single `done` epilogue in handler() can release them exactly as before. */
|
||||||
|
typedef struct ServerSession {
|
||||||
|
int fd;
|
||||||
|
SSL* ssl;
|
||||||
ProtocolSession session;
|
ProtocolSession session;
|
||||||
protocol_session_init(&session, file_descriptor, file_descriptor);
|
|
||||||
protocol_session_set_ssl(&session, ssl);
|
|
||||||
protocol_session_bind(&session);
|
|
||||||
ModuleGateContext gate_ctx;
|
ModuleGateContext gate_ctx;
|
||||||
gate_ctx.ssl = ssl;
|
Config* config;
|
||||||
gate_ctx.fd = file_descriptor;
|
PipelineContextReceiver* context;
|
||||||
gate_ctx.super_mode_override = -1;
|
char* joined_destination;
|
||||||
gate_ctx.has_peer_ip = false;
|
bool charset_ready;
|
||||||
gate_ctx.peer_ip[0] = '\0';
|
} ServerSession;
|
||||||
gate_ctx.is_local = false;
|
|
||||||
/* All teardown state starts empty so the single `done` epilogue is safe to
|
/* Phase 1 -- config receipt + validation. Receives the client config frame
|
||||||
* reach from any error path (including before the config frame arrives). */
|
* through the module gate, applies the super-mode override the gate recorded
|
||||||
Config* config = NULL;
|
* exactly once, and installs the per-connection protocol/compression state.
|
||||||
PipelineContextReceiver* context = NULL;
|
* Returns false when the config frame was refused (the gate has already
|
||||||
char* joined_destination = NULL;
|
* answered the client); the caller jumps to the shared `done` epilogue. */
|
||||||
bool charset_ready = false;
|
static bool server_accept_config(ServerSession* state) {
|
||||||
config = config_receive_with_validate(file_descriptor, server_module_gate, &gate_ctx);
|
state->config = config_receive_with_validate(state->fd, server_module_gate, &state->gate_ctx);
|
||||||
if (config == NULL) {
|
if (state->config == NULL) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Failed to receive config");
|
log_message(LOG_LEVEL_ERROR, "Failed to receive config");
|
||||||
goto done;
|
return false;
|
||||||
}
|
}
|
||||||
/* Apply the super-mode veto the gate decided on (operator --no-super, or a
|
/* Apply the super-mode veto the gate decided on (operator --no-super, or a
|
||||||
* daemon module without the `client owner = yes` opt-in) exactly once, so
|
* daemon module without the `client owner = yes` opt-in) exactly once, so
|
||||||
* every downstream gate (identity_apply_ownership via privilege_super_permitted,
|
* every downstream gate (identity_apply_ownership via privilege_super_permitted,
|
||||||
* device-node creation) sees SUPER_MODE_OFF. The gate never mutated the
|
* device-node creation) sees SUPER_MODE_OFF. The gate never mutated the
|
||||||
* received config. */
|
* received config. */
|
||||||
if (gate_ctx.super_mode_override != -1)
|
if (state->gate_ctx.super_mode_override != -1)
|
||||||
config->super_mode = (SuperMode)gate_ctx.super_mode_override;
|
state->config->super_mode = (SuperMode)state->gate_ctx.super_mode_override;
|
||||||
protocol_set_8_bit_output(config->eight_bit_output);
|
/* Install the codec this connection negotiated before the receiver/writer
|
||||||
|
* threads start (the server forks per connection, so the process-global
|
||||||
|
* codec is private to this session). */
|
||||||
|
compression_set_algo((CompressionAlgo)state->config->compression_algo);
|
||||||
|
/* If the client requested ownership but the effective super mode forbids it
|
||||||
|
* (operator --no-super, a privileged standalone receiver's secure default, or
|
||||||
|
* a daemon module without `client owner = yes`), say so ONCE per connection so
|
||||||
|
* a successful -a/-o/-g transfer is not mistaken for preserved ownership. */
|
||||||
|
if (state->config->super_mode == SUPER_MODE_OFF && identity_ownership_requested(state->config))
|
||||||
|
log_message(LOG_LEVEL_WARNING,
|
||||||
|
"requested ownership will NOT be applied: super-user activities are disabled "
|
||||||
|
"for this connection (operator veto, or module without `client owner = yes`)");
|
||||||
|
protocol_set_8_bit_output(state->config->eight_bit_output);
|
||||||
/* Server-side per-message protocol deadline for every frame from here on.
|
/* Server-side per-message protocol deadline for every frame from here on.
|
||||||
* `timeout` is not serialized, so this is the server's own config (the server
|
* `timeout` is not serialized, so this is the server's own config (the server
|
||||||
* has no --timeout CLI and defaults it to 0): the built-in 60 s window stays
|
* has no --timeout CLI and defaults it to 0). A client's --timeout tightens
|
||||||
* in effect. A client's --timeout tightens only that client's own protocol
|
* only that client's own protocol I/O; the server floors its own deadline at
|
||||||
* I/O and the server's socket read/write timeout is the transport default. */
|
* SERVER_IO_TIMEOUT_SEC so a silent peer can never hold a session slot
|
||||||
protocol_session_set_io_timeout(&session, config->timeout);
|
* forever (the socket layer gets the same floor at startup). */
|
||||||
|
protocol_session_set_io_timeout(&state->session,
|
||||||
|
protocol_server_io_timeout_sec(state->config->timeout));
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Phase 2 -- security gates. The ORDER here is load-bearing and must not be
|
||||||
|
* merged or reordered: transport/authentication (plaintext refusal, TLS
|
||||||
|
* client-CN verification), then daemon-root confinement (absolute-destination
|
||||||
|
* rejection, traversal + within-authorized-root), then delete/force
|
||||||
|
* authorization -- exactly the sequence the former handler() used. Returns
|
||||||
|
* false after logging the matching rejection; the caller jumps to the shared
|
||||||
|
* `done` epilogue. */
|
||||||
|
static bool server_apply_security_gates(ServerSession* state) {
|
||||||
|
Config* config = state->config;
|
||||||
const char* authorized_root = utils_get_authorized_root_path();
|
const char* authorized_root = utils_get_authorized_root_path();
|
||||||
if (!authorized_root) {
|
if (!authorized_root) {
|
||||||
log_message(LOG_LEVEL_ERROR, "No server-side destination root configured");
|
log_message(LOG_LEVEL_ERROR, "No server-side destination root configured");
|
||||||
goto done;
|
return false;
|
||||||
}
|
}
|
||||||
if (!allow_unauthenticated && ssl == NULL) {
|
if (!allow_unauthenticated && state->ssl == NULL) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Rejected unauthenticated plaintext connection");
|
log_message(LOG_LEVEL_ERROR, "Rejected unauthenticated plaintext connection");
|
||||||
goto done;
|
return false;
|
||||||
}
|
}
|
||||||
if (ssl && required_client_cn && !tls_client_identity_allowed(ssl)) {
|
if (state->ssl && required_client_cn && !tls_client_identity_allowed(state->ssl)) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Rejected TLS client with unauthorized identity");
|
log_message(LOG_LEVEL_ERROR, "Rejected TLS client with unauthorized identity");
|
||||||
goto done;
|
return false;
|
||||||
}
|
}
|
||||||
/* Daemon mode: the module's root is the authorized root (installed by
|
/* Daemon mode: the module's root is the authorized root (installed by
|
||||||
server_module_gate), and the client's destination is a MODULE-RELATIVE
|
server_module_gate), and the client's destination is a MODULE-RELATIVE
|
||||||
@@ -765,27 +793,27 @@ void handler(int file_descriptor) {
|
|||||||
if (g_daemon_conf && config->receive_root_directory && config->receive_root_directory[0] == '/') {
|
if (g_daemon_conf && config->receive_root_directory && config->receive_root_directory[0] == '/') {
|
||||||
log_message(LOG_LEVEL_ERROR, "Rejected absolute daemon destination (must be relative to the "
|
log_message(LOG_LEVEL_ERROR, "Rejected absolute daemon destination (must be relative to the "
|
||||||
"selected module root)");
|
"selected module root)");
|
||||||
goto done;
|
return false;
|
||||||
}
|
}
|
||||||
char* destination = config->receive_root_directory;
|
char* destination = config->receive_root_directory;
|
||||||
if (destination && destination[0] != '/')
|
if (destination && destination[0] != '/')
|
||||||
joined_destination = path_cat(authorized_root, destination);
|
state->joined_destination = path_cat(authorized_root, destination);
|
||||||
if (joined_destination)
|
if (state->joined_destination)
|
||||||
destination = joined_destination;
|
destination = state->joined_destination;
|
||||||
if (!destination || has_path_traversal(destination) ||
|
if (!destination || has_path_traversal(destination) ||
|
||||||
!path_is_within(authorized_root, destination)) {
|
!path_is_within_root(authorized_root, destination)) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Rejected destination outside authorized root");
|
log_message(LOG_LEVEL_ERROR, "Rejected destination outside authorized root");
|
||||||
free(joined_destination);
|
free(state->joined_destination);
|
||||||
joined_destination = NULL;
|
state->joined_destination = NULL;
|
||||||
goto done;
|
return false;
|
||||||
}
|
}
|
||||||
if (joined_destination) {
|
if (state->joined_destination) {
|
||||||
free(config->receive_root_directory);
|
free(config->receive_root_directory);
|
||||||
config->receive_root_directory = joined_destination;
|
config->receive_root_directory = state->joined_destination;
|
||||||
joined_destination = NULL;
|
state->joined_destination = NULL;
|
||||||
}
|
}
|
||||||
if (!config->receive_root_directory) {
|
if (!config->receive_root_directory) {
|
||||||
goto done;
|
return false;
|
||||||
}
|
}
|
||||||
config->use_delete = config->use_delete && allow_delete;
|
config->use_delete = config->use_delete && allow_delete;
|
||||||
/* --force (receiver-side) is deletion authority too: it lets an incoming
|
/* --force (receiver-side) is deletion authority too: it lets an incoming
|
||||||
@@ -795,6 +823,18 @@ void handler(int file_descriptor) {
|
|||||||
* --delete-missing-args, so a client cannot use --force to bypass the delete
|
* --delete-missing-args, so a client cannot use --force to bypass the delete
|
||||||
* policy. */
|
* policy. */
|
||||||
config->force_delete = config->force_delete && allow_delete;
|
config->force_delete = config->force_delete && allow_delete;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Phase 3 -- session preparation. Installs the negotiated conversion, applies
|
||||||
|
* the remaining deletion policy, materializes the destination root (--mkpath),
|
||||||
|
* creates the --delay-updates staging tree, snapshots the identity policy, and
|
||||||
|
* publishes the --keep-dirlinks/--trust-sender globals and the daemon MOTD.
|
||||||
|
* All of it must happen before any receiver/writer thread is spawned. Returns
|
||||||
|
* false after logging the matching failure; the caller jumps to the shared
|
||||||
|
* `done` epilogue. */
|
||||||
|
static bool server_prepare_session(ServerSession* state) {
|
||||||
|
Config* config = state->config;
|
||||||
/* --iconv (protocol 2.16.0): install the receiver-side wire->local conversion
|
/* --iconv (protocol 2.16.0): install the receiver-side wire->local conversion
|
||||||
now that the client's full CONVERT_SPEC has been received and validated,
|
now that the client's full CONVERT_SPEC has been received and validated,
|
||||||
before any received file name is decoded. The server's own --iconv (if
|
before any received file name is decoded. The server's own --iconv (if
|
||||||
@@ -805,14 +845,14 @@ void handler(int file_descriptor) {
|
|||||||
if (!charset_wire_init_receiver(config->iconv_spec, server_iconv_spec)) {
|
if (!charset_wire_init_receiver(config->iconv_spec, server_iconv_spec)) {
|
||||||
log_message(LOG_LEVEL_ERROR,
|
log_message(LOG_LEVEL_ERROR,
|
||||||
"--iconv: unsupported charset conversion requested (LOCAL[,REMOTE])");
|
"--iconv: unsupported charset conversion requested (LOCAL[,REMOTE])");
|
||||||
goto done;
|
return false;
|
||||||
}
|
}
|
||||||
charset_ready = true;
|
state->charset_ready = true;
|
||||||
}
|
}
|
||||||
/* --delete-missing-args deletes destination mirrors receiver-side, so it is
|
/* --delete-missing-args deletes destination mirrors receiver-side, so it is
|
||||||
deletion and stays gated by the same --allow-delete server policy. When
|
* deletion and stays gated by the same --allow-delete server policy. When
|
||||||
the server policy is off the flag is inert (the missing entries are still
|
* the server policy is off the flag is inert (the missing entries are still
|
||||||
skipped via its implied --ignore-missing-args, but nothing is deleted). */
|
* skipped via its implied --ignore-missing-args, but nothing is deleted). */
|
||||||
config->delete_missing_args = config->delete_missing_args && allow_delete;
|
config->delete_missing_args = config->delete_missing_args && allow_delete;
|
||||||
/* --mkpath: create the destination root (and its missing leading components)
|
/* --mkpath: create the destination root (and its missing leading components)
|
||||||
* before anything else; without it the root must pre-exist. The precondition
|
* before anything else; without it the root must pre-exist. The precondition
|
||||||
@@ -827,7 +867,7 @@ void handler(int file_descriptor) {
|
|||||||
log_message(LOG_LEVEL_ERROR, "destination root is not available: %s",
|
log_message(LOG_LEVEL_ERROR, "destination root is not available: %s",
|
||||||
escaped_root ? escaped_root : "<allocation failed>");
|
escaped_root ? escaped_root : "<allocation failed>");
|
||||||
free(escaped_root);
|
free(escaped_root);
|
||||||
goto done;
|
return false;
|
||||||
}
|
}
|
||||||
/* A --delay-updates transfer stages under a private 0700 directory inside
|
/* A --delay-updates transfer stages under a private 0700 directory inside
|
||||||
the receive root. Create it up front (wiping leftovers of any previously
|
the receive root. Create it up front (wiping leftovers of any previously
|
||||||
@@ -837,7 +877,7 @@ void handler(int file_descriptor) {
|
|||||||
config->delay_context = delay_updates_context_create(config->receive_root_directory);
|
config->delay_context = delay_updates_context_create(config->receive_root_directory);
|
||||||
if (!config->delay_context || !delay_updates_prepare(config->delay_context)) {
|
if (!config->delay_context || !delay_updates_prepare(config->delay_context)) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Failed to initialize --delay-updates staging area");
|
log_message(LOG_LEVEL_ERROR, "Failed to initialize --delay-updates staging area");
|
||||||
goto done;
|
return false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
/* Preserve the negotiated identity policy for the fd-relative ownership
|
/* Preserve the negotiated identity policy for the fd-relative ownership
|
||||||
@@ -847,7 +887,7 @@ void handler(int file_descriptor) {
|
|||||||
rather than silently applying the wrong ownership policy. */
|
rather than silently applying the wrong ownership policy. */
|
||||||
if (!identity_set_active(config)) {
|
if (!identity_set_active(config)) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Failed to activate identity policy");
|
log_message(LOG_LEVEL_ERROR, "Failed to activate identity policy");
|
||||||
goto done;
|
return false;
|
||||||
}
|
}
|
||||||
/* Persist the negotiated --keep-dirlinks policy once, here at config-accept,
|
/* Persist the negotiated --keep-dirlinks policy once, here at config-accept,
|
||||||
before any multithreaded receiver/writer threads are spawned, so the
|
before any multithreaded receiver/writer threads are spawned, so the
|
||||||
@@ -875,104 +915,201 @@ void handler(int file_descriptor) {
|
|||||||
Wave C note in config.h). */
|
Wave C note in config.h). */
|
||||||
if (g_daemon_conf) {
|
if (g_daemon_conf) {
|
||||||
char* motd = motd_read_file(g_daemon_conf->global.motd_file);
|
char* motd = motd_read_file(g_daemon_conf->global.motd_file);
|
||||||
if (!motd_send(file_descriptor, motd ? motd : "")) {
|
if (!motd_send(state->fd, motd ? motd : "")) {
|
||||||
free(motd);
|
free(motd);
|
||||||
log_message(LOG_LEVEL_ERROR, "Failed to send daemon MOTD");
|
log_message(LOG_LEVEL_ERROR, "Failed to send daemon MOTD");
|
||||||
goto done;
|
return false;
|
||||||
}
|
}
|
||||||
free(motd);
|
free(motd);
|
||||||
}
|
}
|
||||||
if (config->use_multithreading) {
|
return true;
|
||||||
Queue* q = queue_create(100, file_destroy);
|
}
|
||||||
if (q == NULL)
|
|
||||||
goto done;
|
/* Phase 4a -- transfer via the multithreaded receiver. Spawns the receive/write
|
||||||
context = pipeline_context_receiver_create(config, q, file_descriptor, ssl);
|
* thread pair, joins them, then commits the late deletion, --delay-updates
|
||||||
if (context == NULL) {
|
* publication and directory times before emitting the terminal stats/success
|
||||||
queue_destroy(q);
|
* frame. On any failure the helper just returns; the caller's `done` epilogue
|
||||||
goto done;
|
* releases the pipeline context (which owns the config and queue) exactly as the
|
||||||
}
|
* former inline code did. */
|
||||||
protocol_session_set_max_alloc(&context->session, config->max_alloc);
|
static void server_run_mt_receiver(ServerSession* state) {
|
||||||
protocol_session_set_io_timeout(&context->session, config->timeout);
|
Config* config = state->config;
|
||||||
atomic_store(&context->session.total_allocated_bytes,
|
Queue* q = queue_create(100, file_destroy);
|
||||||
atomic_load(&session.total_allocated_bytes));
|
if (q == NULL)
|
||||||
pipeline_context_receiver_set_queue_byte_limit(context, RECEIVER_QUEUE_MAX_BYTES);
|
return;
|
||||||
thrd_t receiver = {0};
|
state->context = pipeline_context_receiver_create(config, q, state->fd, state->ssl);
|
||||||
thrd_t writer = {0};
|
if (state->context == NULL) {
|
||||||
bool receiver_created = thrd_create(&receiver, receive_thread, context) == thrd_success;
|
queue_destroy(q);
|
||||||
bool writer_created = false;
|
return;
|
||||||
if (receiver_created)
|
|
||||||
writer_created = thrd_create(&writer, write_thread, context) == thrd_success;
|
|
||||||
if (!receiver_created || !writer_created) {
|
|
||||||
log_perror("Error creating Threads");
|
|
||||||
if (receiver_created) {
|
|
||||||
mtx_lock(&context->mutex);
|
|
||||||
atomic_store(&context->cancelled, true);
|
|
||||||
cnd_broadcast(&context->condition_not_full);
|
|
||||||
cnd_broadcast(&context->condition_not_empty);
|
|
||||||
mtx_unlock(&context->mutex);
|
|
||||||
/* Unblock a worker parked in socket I/O without closing the fd (the
|
|
||||||
* child owns the single close). shutdown() only affects sockets; for
|
|
||||||
* the --stdio pipe the receiver's per-message poll timeout still
|
|
||||||
* bounds the join, so do nothing there rather than close a descriptor
|
|
||||||
* another thread may still be using. */
|
|
||||||
struct stat fd_stat;
|
|
||||||
if (fstat(file_descriptor, &fd_stat) == 0 && S_ISSOCK(fd_stat.st_mode))
|
|
||||||
shutdown(file_descriptor, SHUT_RDWR);
|
|
||||||
thrd_join(receiver, NULL);
|
|
||||||
}
|
|
||||||
if (writer_created)
|
|
||||||
thrd_join(writer, NULL);
|
|
||||||
goto done;
|
|
||||||
}
|
|
||||||
int receiver_result;
|
|
||||||
int writer_result;
|
|
||||||
thrd_join(receiver, &receiver_result);
|
|
||||||
thrd_join(writer, &writer_result);
|
|
||||||
bool transfer_ok = receiver_result == thrd_success && writer_result == thrd_success;
|
|
||||||
if (transfer_ok && !config->dry_run) {
|
|
||||||
/* Commit-style (late) deletion: receive_thread handed the keep-set
|
|
||||||
manifest here instead of deleting while write_thread might still be
|
|
||||||
draining, so by now every file is on disk and the whole transfer is
|
|
||||||
known to have succeeded. Remove the extras before publishing a
|
|
||||||
--delay-updates run; the walker skips the staging directory. A
|
|
||||||
server-contacting --dry-run deletes nothing (no manifest is sent). */
|
|
||||||
if (context->deferred_manifest) {
|
|
||||||
if (!manifest_delete_all(config, context->deferred_manifest)) {
|
|
||||||
transfer_ok = false;
|
|
||||||
}
|
|
||||||
delete_manifest_free(context->deferred_manifest);
|
|
||||||
context->deferred_manifest = NULL;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (transfer_ok && !config->dry_run) {
|
|
||||||
/* --delay-updates: receive_thread has finished the whole protocol stream
|
|
||||||
(including manifest/delete handling) and write_thread has drained its
|
|
||||||
queue, so every staged file is complete. Publish atomically before the
|
|
||||||
success/outcome frame so a --remove-source-files sender only learns of
|
|
||||||
files that were actually installed. */
|
|
||||||
if (config->delay_updates && config->delay_context &&
|
|
||||||
!delay_updates_publish(config->delay_context, config)) {
|
|
||||||
transfer_ok = false;
|
|
||||||
}
|
|
||||||
/* P7 Wave D: all writers have joined and the late deletion (and
|
|
||||||
--delay-updates publication) has committed above, so it is finally safe
|
|
||||||
to stamp directory times; a directory's mtime must not be clobbered by
|
|
||||||
its children or by an extra removal. */
|
|
||||||
if (transfer_ok)
|
|
||||||
dir_time_list_apply(&context->dir_times, config->receive_root_directory);
|
|
||||||
}
|
|
||||||
if (transfer_ok) {
|
|
||||||
if (!receiver_send_final_success(file_descriptor, config, &context->outcomes))
|
|
||||||
transfer_ok = false;
|
|
||||||
} else {
|
|
||||||
send_error_detail(file_descriptor, "transfer failed on receiver");
|
|
||||||
}
|
|
||||||
if (!transfer_ok)
|
|
||||||
log_message(LOG_LEVEL_ERROR, "Transfer failed");
|
|
||||||
} else {
|
|
||||||
if (receiver_receive_files(config, file_descriptor) != 0)
|
|
||||||
log_message(LOG_LEVEL_ERROR, "Transfer failed");
|
|
||||||
}
|
}
|
||||||
|
protocol_session_set_max_alloc(&state->context->session, config->max_alloc);
|
||||||
|
protocol_session_set_io_timeout(&state->context->session,
|
||||||
|
protocol_server_io_timeout_sec(config->timeout));
|
||||||
|
atomic_store(&state->context->session.total_allocated_bytes,
|
||||||
|
atomic_load(&state->session.total_allocated_bytes));
|
||||||
|
pipeline_context_receiver_set_queue_byte_limit(state->context, RECEIVER_QUEUE_MAX_BYTES);
|
||||||
|
thrd_t receiver = {0};
|
||||||
|
thrd_t writer = {0};
|
||||||
|
bool receiver_created = thrd_create(&receiver, receive_thread, state->context) == thrd_success;
|
||||||
|
bool writer_created = false;
|
||||||
|
if (receiver_created)
|
||||||
|
writer_created = thrd_create(&writer, write_thread, state->context) == thrd_success;
|
||||||
|
if (!receiver_created || !writer_created) {
|
||||||
|
log_perror("Error creating Threads");
|
||||||
|
if (receiver_created) {
|
||||||
|
mtx_lock(&state->context->mutex);
|
||||||
|
atomic_store(&state->context->cancelled, true);
|
||||||
|
cnd_broadcast(&state->context->condition_not_full);
|
||||||
|
cnd_broadcast(&state->context->condition_not_empty);
|
||||||
|
mtx_unlock(&state->context->mutex);
|
||||||
|
/* Unblock a worker parked in socket I/O without closing the fd (the
|
||||||
|
* child owns the single close). shutdown() only affects sockets; for
|
||||||
|
* the --stdio pipe the receiver's per-message poll timeout still
|
||||||
|
* bounds the join, so do nothing there rather than close a descriptor
|
||||||
|
* another thread may still be using. */
|
||||||
|
struct stat fd_stat;
|
||||||
|
if (fstat(state->fd, &fd_stat) == 0 && S_ISSOCK(fd_stat.st_mode))
|
||||||
|
shutdown(state->fd, SHUT_RDWR);
|
||||||
|
thrd_join(receiver, NULL);
|
||||||
|
}
|
||||||
|
if (writer_created)
|
||||||
|
thrd_join(writer, NULL);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
int receiver_result;
|
||||||
|
int writer_result;
|
||||||
|
thrd_join(receiver, &receiver_result);
|
||||||
|
thrd_join(writer, &writer_result);
|
||||||
|
bool transfer_ok = receiver_result == thrd_success && writer_result == thrd_success;
|
||||||
|
PipelineContextReceiver* context = state->context;
|
||||||
|
if (transfer_ok && !config->dry_run) {
|
||||||
|
/* Commit-style (late) deletion: receive_thread handed the keep-set
|
||||||
|
manifest here instead of deleting while write_thread might still be
|
||||||
|
draining, so by now every file is on disk and the whole transfer is
|
||||||
|
known to have succeeded. Remove the extras before publishing a
|
||||||
|
--delay-updates run; the walker skips the staging directory. A
|
||||||
|
server-contacting --dry-run deletes nothing (no manifest is sent). */
|
||||||
|
if (context->deferred_manifest) {
|
||||||
|
size_t deleted = 0;
|
||||||
|
DeletePathObserver observer = config->report_deletes ? receiver_record_deleted_path : NULL;
|
||||||
|
DeleteCommitResult deletion = manifest_delete_all_observed(
|
||||||
|
config, context->deferred_manifest, &deleted, observer, (void*)context->deleted_paths);
|
||||||
|
context->stats.deleted_files += deleted;
|
||||||
|
if (deletion == DELETE_COMMIT_ERROR) {
|
||||||
|
transfer_ok = false;
|
||||||
|
} else if (deletion == DELETE_COMMIT_LIMIT_REACHED) {
|
||||||
|
/* The transfer still succeeds; the terminal frame reports the capped
|
||||||
|
deletion so the sender exits 25 like rsync. */
|
||||||
|
context->delete_limit_reached = true;
|
||||||
|
}
|
||||||
|
delete_manifest_free(context->deferred_manifest);
|
||||||
|
context->deferred_manifest = NULL;
|
||||||
|
}
|
||||||
|
/* --delete-delay: receive_thread snapshotted each plan's extras as it
|
||||||
|
arrived; with the disk writer drained, commit the deferred removals.
|
||||||
|
--delete-during already applied its plans on the receive thread. */
|
||||||
|
if (context->deferred_plans) {
|
||||||
|
/* Defence in depth (the enclosing block already excludes dry-run): a
|
||||||
|
-n run never commits a deletion. */
|
||||||
|
if (config->report_deletes)
|
||||||
|
delete_plan_session_set_delete_observer(
|
||||||
|
context->deferred_plans, receiver_record_deleted_path, (void*)context->deleted_paths);
|
||||||
|
DeleteCommitResult deletion =
|
||||||
|
config->dry_run ? DELETE_COMMIT_OK
|
||||||
|
: delete_plan_session_commit(context->deferred_plans, config);
|
||||||
|
context->stats.deleted_files += delete_plan_session_deleted(context->deferred_plans);
|
||||||
|
if (deletion == DELETE_COMMIT_ERROR) {
|
||||||
|
transfer_ok = false;
|
||||||
|
} else if (deletion == DELETE_COMMIT_LIMIT_REACHED) {
|
||||||
|
context->delete_limit_reached = true;
|
||||||
|
}
|
||||||
|
delete_plan_session_destroy(context->deferred_plans);
|
||||||
|
context->deferred_plans = NULL;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (transfer_ok && !config->dry_run) {
|
||||||
|
/* --delay-updates: receive_thread has finished the whole protocol stream
|
||||||
|
(including manifest/delete handling) and write_thread has drained its
|
||||||
|
queue, so every staged file is complete. Publish atomically before the
|
||||||
|
success/outcome frame so a --remove-source-files sender only learns of
|
||||||
|
files that were actually installed. */
|
||||||
|
if (config->delay_updates && config->delay_context &&
|
||||||
|
!delay_updates_publish(config->delay_context, config)) {
|
||||||
|
transfer_ok = false;
|
||||||
|
}
|
||||||
|
/* P7 Wave D: all writers have joined and the late deletion (and
|
||||||
|
--delay-updates publication) has committed above, so it is finally safe
|
||||||
|
to stamp directory times; a directory's mtime must not be clobbered by
|
||||||
|
its children or by an extra removal. */
|
||||||
|
if (transfer_ok)
|
||||||
|
dir_metadata_list_apply(&context->dir_times, config->receive_root_directory, config);
|
||||||
|
}
|
||||||
|
if (transfer_ok) {
|
||||||
|
if (context->failed_entries > 0)
|
||||||
|
log_message(LOG_LEVEL_WARNING,
|
||||||
|
"%zu entr%s failed to materialize; continuing (partial transfer)",
|
||||||
|
context->failed_entries, context->failed_entries == 1 ? "y" : "ies");
|
||||||
|
Status final_status = context->delete_limit_reached
|
||||||
|
? STATUS_DELETE_LIMIT
|
||||||
|
: (context->failed_entries > 0 ? STATUS_ERROR : STATUS_OK);
|
||||||
|
/* Emit the optional wire-stats record first (protocol 2.25.0), then the
|
||||||
|
success/outcome frame, exactly like the single-threaded receiver. */
|
||||||
|
if (!receiver_send_stats_frame(state->fd, config, &context->stats, context->would_delete,
|
||||||
|
context->deleted_paths) ||
|
||||||
|
!receiver_send_final_success(state->fd, config, &context->outcomes, final_status))
|
||||||
|
transfer_ok = false;
|
||||||
|
} else {
|
||||||
|
send_error_detail(state->fd, "transfer failed on receiver");
|
||||||
|
}
|
||||||
|
if (!transfer_ok)
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Transfer failed");
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Phase 4b -- transfer via the single-threaded receiver. Failure is logged
|
||||||
|
* exactly as before; the caller's `done` epilogue then releases the config. */
|
||||||
|
static void server_run_st_receiver(ServerSession* state) {
|
||||||
|
if (receiver_receive_files(state->config, state->fd) != 0)
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Transfer failed");
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Phase 4 dispatch -- choose the receiver implementation the config asks for.
|
||||||
|
* Both helpers own their success/failure logging; the caller falls through to
|
||||||
|
* the shared `done` epilogue either way. */
|
||||||
|
static void server_run_transfer(ServerSession* state) {
|
||||||
|
if (state->config->use_multithreading)
|
||||||
|
server_run_mt_receiver(state);
|
||||||
|
else
|
||||||
|
server_run_st_receiver(state);
|
||||||
|
}
|
||||||
|
|
||||||
|
void handler(int file_descriptor) {
|
||||||
|
/* Single per-connection state; every phase helper below advances it and
|
||||||
|
* returns false on a logged failure. All teardown state starts empty so the
|
||||||
|
* single `done` epilogue is safe to reach from any error path (including
|
||||||
|
* before the config frame arrives). */
|
||||||
|
ServerSession state;
|
||||||
|
state.fd = file_descriptor;
|
||||||
|
state.ssl = io_get_ssl();
|
||||||
|
protocol_session_init(&state.session, file_descriptor, file_descriptor);
|
||||||
|
protocol_session_set_ssl(&state.session, state.ssl);
|
||||||
|
protocol_session_bind(&state.session);
|
||||||
|
state.gate_ctx.ssl = state.ssl;
|
||||||
|
state.gate_ctx.fd = file_descriptor;
|
||||||
|
state.gate_ctx.super_mode_override = -1;
|
||||||
|
state.gate_ctx.has_peer_ip = false;
|
||||||
|
state.gate_ctx.peer_ip[0] = '\0';
|
||||||
|
state.gate_ctx.is_local = false;
|
||||||
|
state.config = NULL;
|
||||||
|
state.context = NULL;
|
||||||
|
state.joined_destination = NULL;
|
||||||
|
state.charset_ready = false;
|
||||||
|
|
||||||
|
if (!server_accept_config(&state))
|
||||||
|
goto done;
|
||||||
|
if (!server_apply_security_gates(&state))
|
||||||
|
goto done;
|
||||||
|
if (!server_prepare_session(&state))
|
||||||
|
goto done;
|
||||||
|
server_run_transfer(&state);
|
||||||
|
|
||||||
done:
|
done:
|
||||||
/* Single cleanup epilogue: every error path jumps here, so the iconv
|
/* Single cleanup epilogue: every error path jumps here, so the iconv
|
||||||
@@ -981,38 +1118,63 @@ done:
|
|||||||
* connection fd is deliberately NOT closed here -- the child functions own
|
* connection fd is deliberately NOT closed here -- the child functions own
|
||||||
* its single close (plain_child_fn / tls_child_fn), and the --stdio call
|
* its single close (plain_child_fn / tls_child_fn), and the --stdio call
|
||||||
* site must leave stdin/stdout open. */
|
* site must leave stdin/stdout open. */
|
||||||
if (charset_ready)
|
if (state.charset_ready)
|
||||||
charset_wire_free();
|
charset_wire_free();
|
||||||
/* The delay-updates staging tree is released by config_delete (which the
|
/* The delay-updates staging tree is released by config_delete (which the
|
||||||
branch below always reaches), so it is cleaned exactly once. */
|
branch below always reaches), so it is cleaned exactly once. */
|
||||||
identity_clear_active();
|
identity_clear_active();
|
||||||
protocol_session_unbind();
|
protocol_session_unbind();
|
||||||
if (context != NULL) {
|
if (state.context != NULL) {
|
||||||
/* context owns both the config and the queue it was created with. */
|
/* context owns both the config and the queue it was created with. */
|
||||||
pipeline_context_receiver_destroy(context);
|
pipeline_context_receiver_destroy(state.context);
|
||||||
context = NULL;
|
state.context = NULL;
|
||||||
config = NULL;
|
state.config = NULL;
|
||||||
} else {
|
} else {
|
||||||
config_delete(config);
|
config_delete(state.config);
|
||||||
config = NULL;
|
state.config = NULL;
|
||||||
}
|
}
|
||||||
free(joined_destination);
|
free(state.joined_destination);
|
||||||
}
|
}
|
||||||
|
|
||||||
#ifndef FASTSYNC_SERVER_AS_LIB
|
#ifndef FASTSYNC_SERVER_AS_LIB
|
||||||
static Server* g_server = NULL;
|
static Server* g_server = NULL;
|
||||||
|
|
||||||
|
/* Signal handler for the foreground daemon/standalone listener.
|
||||||
|
*
|
||||||
|
* Async-signal-safety: _exit(2) is on the POSIX async-signal-safe list and is
|
||||||
|
* the ONLY thing done here. The previous body called server_delete()
|
||||||
|
* (close/free/SSL_CTX_free), daemon_conf_free() and credentials_free(); none of
|
||||||
|
* those (free/malloc, and much of OpenSSL teardown) are async-signal-safe, so a
|
||||||
|
* signal delivered while the main thread was inside malloc/free could deadlock
|
||||||
|
* or corrupt the heap.
|
||||||
|
*
|
||||||
|
* Residual (documented, not hidden): the in-memory teardown is skipped on the
|
||||||
|
* signal path. That is safe because the parent daemon owns no persistent
|
||||||
|
* resource that survives process exit -- the listening socket is closed by the
|
||||||
|
* kernel, the connection registry is an anonymous MAP_SHARED mapping with no
|
||||||
|
* named backing object, and the daemon config/credential stores are plain heap
|
||||||
|
* allocations. Connection children are separate processes and handle their own
|
||||||
|
* temp files/locks. The normal (non-signal) shutdown path in main() still runs
|
||||||
|
* the full teardown, so no cleanup is dropped on the common path. Wiring the
|
||||||
|
* accept loop (transport_tcp.c, outside this change's scope) to a flag-based
|
||||||
|
* self-pipe shutdown would let the frees run context-safely; it is deliberately
|
||||||
|
* deferred rather than risk restructuring the daemon loop. */
|
||||||
static void cleanup(int sig) {
|
static void cleanup(int sig) {
|
||||||
(void)sig;
|
(void)sig;
|
||||||
if (g_server)
|
|
||||||
server_delete(&g_server);
|
|
||||||
daemon_conf_free(g_daemon_conf);
|
|
||||||
g_daemon_conf = NULL;
|
|
||||||
credentials_free(g_credentials);
|
|
||||||
g_credentials = NULL;
|
|
||||||
_exit(0);
|
_exit(0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Install a signal handler with sigaction(2) (the required async-signal-safe
|
||||||
|
* install primitive; signal(3) is not specified to be async-signal-safe). */
|
||||||
|
static void install_cleanup_handler(int signo) {
|
||||||
|
struct sigaction action;
|
||||||
|
memset(&action, 0, sizeof(action));
|
||||||
|
action.sa_handler = cleanup;
|
||||||
|
sigemptyset(&action.sa_mask);
|
||||||
|
action.sa_flags = 0;
|
||||||
|
sigaction(signo, &action, NULL);
|
||||||
|
}
|
||||||
|
|
||||||
static void print_server_usage(void) {
|
static void print_server_usage(void) {
|
||||||
printf("FastSync Server\n");
|
printf("FastSync Server\n");
|
||||||
printf("Usage: fastsync-server [options]\n\n");
|
printf("Usage: fastsync-server [options]\n\n");
|
||||||
@@ -1029,8 +1191,9 @@ static void print_server_usage(void) {
|
|||||||
printf(" hosts allow, hosts deny)\n");
|
printf(" hosts allow, hosts deny)\n");
|
||||||
printf(" --no-detach Stay in the foreground (default detaches to\n");
|
printf(" --no-detach Stay in the foreground (default detaches to\n");
|
||||||
printf(" background when running --daemon)\n");
|
printf(" background when running --daemon)\n");
|
||||||
printf(" --password-file=FILE Credential store for modules that declare\n");
|
printf(" --password-file=FILE FastSync-native SCRAM/PBKDF2 credential store (NOT\n");
|
||||||
printf(" 'auth users' (line format:\n");
|
printf(" rsync's auth scheme) for modules that declare 'auth\n");
|
||||||
|
printf(" users' (line format:\n");
|
||||||
printf(" user:$fastsync$1$pbkdf2-sha256$iters$salt$stored$server,\n");
|
printf(" user:$fastsync$1$pbkdf2-sha256$iters$salt$stored$server,\n");
|
||||||
printf(" generated by --hash-credentials). Legacy\n");
|
printf(" generated by --hash-credentials). Legacy\n");
|
||||||
printf(" user:SHA256HEX lines are rejected. Requires\n");
|
printf(" user:SHA256HEX lines are rejected. Requires\n");
|
||||||
@@ -1039,7 +1202,7 @@ static void print_server_usage(void) {
|
|||||||
printf(" --early-input=FILE Second credential store layered over\n");
|
printf(" --early-input=FILE Second credential store layered over\n");
|
||||||
printf(" --password-file (same format); usually a secrets-\n");
|
printf(" --password-file (same format); usually a secrets-\n");
|
||||||
printf(" manager/process-substitution file. Requires --daemon\n");
|
printf(" manager/process-substitution file. Requires --daemon\n");
|
||||||
printf(" -p <port> TCP port (default: 8080, range: 1-65535)\n");
|
printf(" -p, --port <port> TCP port (default: 8080, range: 1-65535)\n");
|
||||||
printf(" --tls Enable TLS encryption\n");
|
printf(" --tls Enable TLS encryption\n");
|
||||||
printf(" --cert <path> TLS certificate file (PEM)\n");
|
printf(" --cert <path> TLS certificate file (PEM)\n");
|
||||||
printf(" --key <path> TLS private key file (PEM)\n");
|
printf(" --key <path> TLS private key file (PEM)\n");
|
||||||
@@ -1050,7 +1213,9 @@ static void print_server_usage(void) {
|
|||||||
printf(" -4, --ipv4 Bind an IPv4 socket (default)\n");
|
printf(" -4, --ipv4 Bind an IPv4 socket (default)\n");
|
||||||
printf(" -6, --ipv6 Bind an IPv6 socket\n");
|
printf(" -6, --ipv6 Bind an IPv6 socket\n");
|
||||||
printf(" --allow-delete Permit manifest deletion\n");
|
printf(" --allow-delete Permit manifest deletion\n");
|
||||||
printf(" --trust-sender Trust the remote sender's file list\n");
|
printf(" --trust-sender Trust the remote sender's file list (receiver-local;\n");
|
||||||
|
printf(" this server-side flag is the only one that matters -- a\n");
|
||||||
|
printf(" client --trust-sender is never sent to the server)\n");
|
||||||
printf(" --no-super Operator veto: never attempt super-user activities\n");
|
printf(" --no-super Operator veto: never attempt super-user activities\n");
|
||||||
printf(" (ownership, device nodes) even as root, and refuse\n");
|
printf(" (ownership, device nodes) even as root, and refuse\n");
|
||||||
printf(" any client --copy-as/--super request\n");
|
printf(" any client --copy-as/--super request\n");
|
||||||
@@ -1122,15 +1287,29 @@ static bool daemonize(void) {
|
|||||||
close(devnull);
|
close(devnull);
|
||||||
}
|
}
|
||||||
/* Do not pin the launch CWD (module-relative 'path' entries would resolve
|
/* Do not pin the launch CWD (module-relative 'path' entries would resolve
|
||||||
* against an unstable working directory) and drop the restrictive host umask
|
* against an unstable working directory). Set a conservative daemon umask
|
||||||
* so modules can create files/dirs with the modes the config requests. */
|
* of 022 (the conventional service default): rsync never forces umask 0 --
|
||||||
|
* it reads and restores the inherited umask and creates new entries as
|
||||||
|
* 0777 & ~umask / source & ~umask without -p. Forcing 0 here made every
|
||||||
|
* implied parent directory world-writable (0777) whenever -p metadata was not
|
||||||
|
* applied. 022 gives 0755 directories and source&~022 files, matching rsync
|
||||||
|
* under a normal daemon umask; -p/-a still restore the exact source mode via
|
||||||
|
* fchmod, which is unaffected by the umask. */
|
||||||
if (chdir("/") != 0)
|
if (chdir("/") != 0)
|
||||||
log_message(LOG_LEVEL_WARNING, "daemon: chdir to / failed: %s", strerror(errno));
|
log_message(LOG_LEVEL_WARNING, "daemon: chdir to / failed: %s", strerror(errno));
|
||||||
umask(0);
|
umask(022);
|
||||||
|
/* Refresh the cached umask: main() captured the launch umask before this
|
||||||
|
* (single-threaded) umask(022), and file_mode_base() must see the daemon's
|
||||||
|
* actual umask. */
|
||||||
|
file_umask_capture();
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
int main(int argc, char* argv[]) {
|
int main(int argc, char* argv[]) {
|
||||||
|
/* Capture the process umask now, while still single-threaded: the cached
|
||||||
|
* value is what file_mode_base() uses, and reading it later would race with
|
||||||
|
* receiver threads creating files. */
|
||||||
|
file_umask_capture();
|
||||||
ServerCliOptions opts;
|
ServerCliOptions opts;
|
||||||
char cli_err[512];
|
char cli_err[512];
|
||||||
int parse_result = server_cli_parse(argc, argv, &opts, cli_err, sizeof(cli_err));
|
int parse_result = server_cli_parse(argc, argv, &opts, cli_err, sizeof(cli_err));
|
||||||
@@ -1189,8 +1368,12 @@ int main(int argc, char* argv[]) {
|
|||||||
* this process-global policy cannot be re-enabled by a future caller. */
|
* this process-global policy cannot be re-enabled by a future caller. */
|
||||||
server_allow_super = opts.allow_super && !opts.stdio_mode;
|
server_allow_super = opts.allow_super && !opts.stdio_mode;
|
||||||
server_iconv_spec = opts.iconv_spec;
|
server_iconv_spec = opts.iconv_spec;
|
||||||
signal(SIGINT, cleanup);
|
install_cleanup_handler(SIGINT);
|
||||||
signal(SIGTERM, cleanup);
|
install_cleanup_handler(SIGTERM);
|
||||||
|
/* Server-owned socket deadline floor: the client default --timeout=0 would
|
||||||
|
* otherwise leave accepted sockets without SO_RCVTIMEO/SO_SNDTIMEO and let a
|
||||||
|
* silent peer hold a connection (and its process slot) forever. */
|
||||||
|
tcp_set_timeouts(SERVER_IO_TIMEOUT_SEC, SERVER_IO_TIMEOUT_SEC);
|
||||||
|
|
||||||
if (opts.stdio_mode) {
|
if (opts.stdio_mode) {
|
||||||
/* SSH authenticates the stdio transport outside of FastSync. */
|
/* SSH authenticates the stdio transport outside of FastSync. */
|
||||||
|
|||||||
+14
-16
@@ -3,20 +3,12 @@
|
|||||||
#include "credentials.h"
|
#include "credentials.h"
|
||||||
#include "utils.h"
|
#include "utils.h"
|
||||||
#include <limits.h>
|
#include <limits.h>
|
||||||
#include <stdarg.h>
|
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
#include <sys/socket.h>
|
#include <sys/socket.h>
|
||||||
|
|
||||||
static void set_error(char* err, size_t err_size, const char* fmt, ...) {
|
#define set_error utils_set_error
|
||||||
if (!err || err_size == 0)
|
|
||||||
return;
|
|
||||||
va_list args;
|
|
||||||
va_start(args, fmt);
|
|
||||||
vsnprintf(err, err_size, fmt, args);
|
|
||||||
va_end(args);
|
|
||||||
}
|
|
||||||
|
|
||||||
void server_cli_options_default(ServerCliOptions* opts) {
|
void server_cli_options_default(ServerCliOptions* opts) {
|
||||||
if (!opts)
|
if (!opts)
|
||||||
@@ -192,14 +184,20 @@ int server_cli_parse(int argc, char* argv[], ServerCliOptions* opts, char* err,
|
|||||||
inline_value = argv[++i];
|
inline_value = argv[++i];
|
||||||
}
|
}
|
||||||
opts->iconv_spec = inline_value;
|
opts->iconv_spec = inline_value;
|
||||||
} else if (arg_is(argv[i], "-p")) {
|
} else if (arg_is(argv[i], "-p") || arg_has_value(argv[i], "--port", &inline_value)) {
|
||||||
if (i + 1 >= argc) {
|
if (inline_value) {
|
||||||
set_error(err, err_size, "missing argument for -p");
|
opts->port_set = true;
|
||||||
return -1;
|
if (parse_port_arg(inline_value, &opts->port, err, err_size) != 0)
|
||||||
|
return -1;
|
||||||
|
} else {
|
||||||
|
if (i + 1 >= argc) {
|
||||||
|
set_error(err, err_size, "missing argument for %s", argv[i]);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
opts->port_set = true;
|
||||||
|
if (parse_port_arg(argv[++i], &opts->port, err, err_size) != 0)
|
||||||
|
return -1;
|
||||||
}
|
}
|
||||||
opts->port_set = true;
|
|
||||||
if (parse_port_arg(argv[++i], &opts->port, err, err_size) != 0)
|
|
||||||
return -1;
|
|
||||||
} else {
|
} else {
|
||||||
if (arg_has_value(argv[i], "--config", &inline_value)) {
|
if (arg_has_value(argv[i], "--config", &inline_value)) {
|
||||||
if (!inline_value) {
|
if (!inline_value) {
|
||||||
|
|||||||
@@ -9,7 +9,7 @@
|
|||||||
ArrayList* array_list_create(void (*item_destroyer)(void* item)) {
|
ArrayList* array_list_create(void (*item_destroyer)(void* item)) {
|
||||||
ArrayList* list = (ArrayList*)protocol_alloc(sizeof(ArrayList));
|
ArrayList* list = (ArrayList*)protocol_alloc(sizeof(ArrayList));
|
||||||
if (list == NULL) {
|
if (list == NULL) {
|
||||||
log_perror("ERROR: Could not allocate memory for array list struct");
|
log_message(LOG_LEVEL_ERROR, "%s", "ERROR: Could not allocate memory for array list struct");
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -47,7 +47,7 @@ static bool array_list_extend(ArrayList* array_list) {
|
|||||||
new_capacity = INITIAL_ARRAY_SIZE;
|
new_capacity = INITIAL_ARRAY_SIZE;
|
||||||
void* new_items = protocol_realloc(array_list->items, new_capacity * sizeof(void*));
|
void* new_items = protocol_realloc(array_list->items, new_capacity * sizeof(void*));
|
||||||
if (new_items == NULL) {
|
if (new_items == NULL) {
|
||||||
log_perror("ERROR: Could not reallocate memory for array list items");
|
log_message(LOG_LEVEL_ERROR, "%s", "ERROR: Could not reallocate memory for array list items");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
array_list->items = new_items;
|
array_list->items = new_items;
|
||||||
@@ -73,7 +73,7 @@ void** array_list_to_array(const ArrayList* array_list) {
|
|||||||
}
|
}
|
||||||
void** array = protocol_alloc(array_list->size * sizeof(void*));
|
void** array = protocol_alloc(array_list->size * sizeof(void*));
|
||||||
if (array == NULL) {
|
if (array == NULL) {
|
||||||
log_perror("Could not malloc space for array from array list!");
|
log_message(LOG_LEVEL_ERROR, "%s", "Could not malloc space for array from array list!");
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
memcpy(array, array_list->items, array_list->size * sizeof(void*));
|
memcpy(array, array_list->items, array_list->size * sizeof(void*));
|
||||||
|
|||||||
+55
-20
@@ -2,6 +2,7 @@
|
|||||||
#include "data.h"
|
#include "data.h"
|
||||||
#include "file.h"
|
#include "file.h"
|
||||||
#include "file_receive.h"
|
#include "file_receive.h"
|
||||||
|
#include "identity.h"
|
||||||
#include "log.h"
|
#include "log.h"
|
||||||
#include <errno.h>
|
#include <errno.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
@@ -10,11 +11,15 @@
|
|||||||
|
|
||||||
/* Serialization metadata mode for the batch stream, captured from the config at
|
/* Serialization metadata mode for the batch stream, captured from the config at
|
||||||
* batch_write_header time. The header persists it into the file so a batch is
|
* batch_write_header time. The header persists it into the file so a batch is
|
||||||
* self-describing: batch_read_apply re-reads it from the file (not from the
|
* self-describing about whether per-entry metadata was CAPTURED in the stream:
|
||||||
* reading config), so a batch written with -M is applied identically by an
|
* batch_read_apply re-reads it from the file (not from the reading config) to
|
||||||
* invoking process regardless of its own -M setting. The batch driver is a
|
* decode the chunk records correctly. Which attributes are actually APPLIED,
|
||||||
* single sequential scan pass within one thread, so this module-level flag is
|
* however, comes from the INVOKING process's per-attribute config (the
|
||||||
* safe. */
|
* FileAttrPolicy and the dir-metadata gate), so a batch written with -M is NOT
|
||||||
|
* automatically applied identically by an invoking process with a different
|
||||||
|
* -p/-t/-o/-g: --read-batch must be invoked with the same -p/-t/-o/-g as the
|
||||||
|
* write side (rsync requires the same options). The batch driver is a single
|
||||||
|
* sequential scan pass within one thread, so this module-level flag is safe. */
|
||||||
static bool batch_metadata_mode = false;
|
static bool batch_metadata_mode = false;
|
||||||
|
|
||||||
static bool write_all_bytes(int fd, const void* data, size_t size) {
|
static bool write_all_bytes(int fd, const void* data, size_t size) {
|
||||||
@@ -91,22 +96,37 @@ int batch_read_apply(int fd, const Config* config, const char* dest_root) {
|
|||||||
if (fd < 0 || dest_root == NULL || dest_root[0] == '\0')
|
if (fd < 0 || dest_root == NULL || dest_root[0] == '\0')
|
||||||
return -1;
|
return -1;
|
||||||
|
|
||||||
|
/* Directory metadata is deferred to the end of the apply (a child write would
|
||||||
|
* otherwise clobber its parent's mtime/mode). The batch header's single
|
||||||
|
* metadata bit only says whether metadata is present in the stream; which
|
||||||
|
* attributes are APPLIED comes from the invoking process's config, so
|
||||||
|
* --read-batch must be invoked with the same -p/-t/-o/-g as the write side
|
||||||
|
* (rsync requires the same options). The identity snapshot is activated so
|
||||||
|
* -o/-g and the explicit ownership flags can apply. */
|
||||||
|
DirTimeList dir_times;
|
||||||
|
dir_time_list_init(&dir_times);
|
||||||
|
int result = -1;
|
||||||
|
if (!identity_set_active(config)) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "batch: could not activate the identity policy");
|
||||||
|
goto done;
|
||||||
|
}
|
||||||
|
|
||||||
char magic[BATCH_MAGIC_LEN];
|
char magic[BATCH_MAGIC_LEN];
|
||||||
bool eof = false;
|
bool eof = false;
|
||||||
if (!read_exact(fd, magic, BATCH_MAGIC_LEN, &eof) || eof ||
|
if (!read_exact(fd, magic, BATCH_MAGIC_LEN, &eof) || eof ||
|
||||||
memcmp(magic, BATCH_MAGIC, BATCH_MAGIC_LEN) != 0) {
|
memcmp(magic, BATCH_MAGIC, BATCH_MAGIC_LEN) != 0) {
|
||||||
log_message(LOG_LEVEL_ERROR, "batch: malformed header (bad magic)");
|
log_message(LOG_LEVEL_ERROR, "batch: malformed header (bad magic)");
|
||||||
return -1;
|
goto done;
|
||||||
}
|
}
|
||||||
unsigned char version;
|
unsigned char version;
|
||||||
if (!read_exact(fd, &version, 1, &eof) || eof || version != BATCH_FORMAT_VERSION) {
|
if (!read_exact(fd, &version, 1, &eof) || eof || version != BATCH_FORMAT_VERSION) {
|
||||||
log_message(LOG_LEVEL_ERROR, "batch: malformed header (bad or missing format version)");
|
log_message(LOG_LEVEL_ERROR, "batch: malformed header (bad or missing format version)");
|
||||||
return -1;
|
goto done;
|
||||||
}
|
}
|
||||||
unsigned char mode;
|
unsigned char mode;
|
||||||
if (!read_exact(fd, &mode, 1, &eof) || eof || (mode != 0 && mode != 1)) {
|
if (!read_exact(fd, &mode, 1, &eof) || eof || (mode != 0 && mode != 1)) {
|
||||||
log_message(LOG_LEVEL_ERROR, "batch: malformed header (bad metadata flag)");
|
log_message(LOG_LEVEL_ERROR, "batch: malformed header (bad metadata flag)");
|
||||||
return -1;
|
goto done;
|
||||||
}
|
}
|
||||||
bool use_metadata = mode == 1;
|
bool use_metadata = mode == 1;
|
||||||
|
|
||||||
@@ -114,47 +134,62 @@ int batch_read_apply(int fd, const Config* config, const char* dest_root) {
|
|||||||
unsigned long long length;
|
unsigned long long length;
|
||||||
if (!read_exact(fd, &length, sizeof(length), &eof)) {
|
if (!read_exact(fd, &length, sizeof(length), &eof)) {
|
||||||
log_message(LOG_LEVEL_ERROR, "batch: truncated length prefix");
|
log_message(LOG_LEVEL_ERROR, "batch: truncated length prefix");
|
||||||
return -1;
|
goto done;
|
||||||
}
|
}
|
||||||
if (eof)
|
if (eof)
|
||||||
break; /* clean end of stream */
|
break; /* clean end of stream */
|
||||||
if (length == 0 || length > BATCH_MAX_RECORD) {
|
if (length == 0 || length > BATCH_MAX_RECORD) {
|
||||||
log_message(LOG_LEVEL_ERROR, "batch: rejected record length %llu (valid range 1..%llu)",
|
log_message(LOG_LEVEL_ERROR, "batch: rejected record length %llu (valid range 1..%llu)",
|
||||||
length, (unsigned long long)BATCH_MAX_RECORD);
|
length, (unsigned long long)BATCH_MAX_RECORD);
|
||||||
return -1;
|
goto done;
|
||||||
}
|
}
|
||||||
char* record = (char*)malloc((size_t)length);
|
char* record = (char*)malloc((size_t)length);
|
||||||
if (record == NULL) {
|
if (record == NULL) {
|
||||||
log_message(LOG_LEVEL_ERROR, "batch: could not allocate a %llu-byte record", length);
|
log_message(LOG_LEVEL_ERROR, "batch: could not allocate a %llu-byte record", length);
|
||||||
return -1;
|
goto done;
|
||||||
}
|
}
|
||||||
if (!read_exact(fd, record, (size_t)length, &eof) || eof) {
|
if (!read_exact(fd, record, (size_t)length, &eof) || eof) {
|
||||||
log_message(LOG_LEVEL_ERROR, "batch: truncated chunk record");
|
log_message(LOG_LEVEL_ERROR, "batch: truncated chunk record");
|
||||||
free(record);
|
free(record);
|
||||||
return -1;
|
goto done;
|
||||||
}
|
}
|
||||||
Data* data = data_create(record, (size_t)length);
|
Data* data = data_create(record, (size_t)length);
|
||||||
if (data == NULL)
|
if (data == NULL)
|
||||||
return -1; /* data_create frees `record` on failure */
|
goto done; /* data_create frees `record` on failure */
|
||||||
Chunk* chunk = chunk_deserialize(data, use_metadata);
|
Chunk* chunk = chunk_deserialize(data, use_metadata);
|
||||||
data_destroy(data);
|
data_destroy(data);
|
||||||
if (chunk == NULL) {
|
if (chunk == NULL) {
|
||||||
log_message(LOG_LEVEL_ERROR, "batch: rejected malformed chunk record");
|
log_message(LOG_LEVEL_ERROR, "batch: rejected malformed chunk record");
|
||||||
return -1;
|
goto done;
|
||||||
}
|
}
|
||||||
for (int i = 0; i < chunk->element_count; i++) {
|
for (int i = 0; i < chunk->element_count; i++) {
|
||||||
File* file = chunk->items[i];
|
File* file = chunk->items[i];
|
||||||
chunk->items[i] = NULL;
|
chunk->items[i] = NULL;
|
||||||
if (file == NULL)
|
if (file == NULL)
|
||||||
continue;
|
continue;
|
||||||
FileSaveResult result = file_save_to_disk_full(dest_root, file, config);
|
FileSaveResult save = file_save_to_disk_full(dest_root, file, config);
|
||||||
file_destroy(file);
|
/* Accumulate directory metadata (when it applies) before the File is
|
||||||
if (result == FILE_SAVE_ERROR) {
|
* destroyed; applied once the whole stream has been consumed. */
|
||||||
|
if (save != FILE_SAVE_ERROR && file->is_dir && file->metadata &&
|
||||||
|
dir_metadata_should_capture(config) &&
|
||||||
|
!dir_time_list_add(&dir_times, file->path, file->metadata, file->xattrs)) {
|
||||||
|
file_destroy(file);
|
||||||
chunk_destroy(chunk);
|
chunk_destroy(chunk);
|
||||||
return -1;
|
goto done;
|
||||||
|
}
|
||||||
|
file_destroy(file);
|
||||||
|
if (save == FILE_SAVE_ERROR) {
|
||||||
|
chunk_destroy(chunk);
|
||||||
|
goto done;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
chunk_destroy(chunk);
|
chunk_destroy(chunk);
|
||||||
}
|
}
|
||||||
return 0;
|
dir_metadata_list_apply(&dir_times, dest_root, config);
|
||||||
}
|
result = 0;
|
||||||
|
|
||||||
|
done:
|
||||||
|
identity_clear_active();
|
||||||
|
dir_time_list_free(&dir_times);
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|||||||
+15
-10
@@ -168,9 +168,14 @@ bool charset_spec_valid_direction(const char* from_charset, const char* to_chars
|
|||||||
return direction_probe_valid(from_charset, to_charset);
|
return direction_probe_valid(from_charset, to_charset);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* The receiver's real conversion is wire(client REMOTE) -> server-local (the
|
/* The receiver's conversion is wire charset -> destination charset. rsync's
|
||||||
* server's own --iconv LOCAL half, or the client's LOCAL half when the server
|
* CONVERT_SPEC is LOCAL,REMOTE and "stays the same whether you're pushing or
|
||||||
* has no --iconv). A dedicated pre-ack check so an impossible direction is
|
* pulling", so for a PUSH (FastSync's only direction) the destination end's
|
||||||
|
* charset is the spec's REMOTE half: the client converts LOCAL -> REMOTE on the
|
||||||
|
* sender and the receiver writes the wire bytes verbatim. Only a server that
|
||||||
|
* declares its OWN --iconv (the daemon "charset" analog) has a different local
|
||||||
|
* charset, and then it is that spec's LOCAL half and the receiver converts
|
||||||
|
* wire -> server-local. A dedicated pre-ack check so an impossible direction is
|
||||||
* rejected before the connection instead of refusing mid-transfer. */
|
* rejected before the connection instead of refusing mid-transfer. */
|
||||||
bool charset_wire_receiver_spec_valid(const char* spec, const char* server_spec) {
|
bool charset_wire_receiver_spec_valid(const char* spec, const char* server_spec) {
|
||||||
if (!spec)
|
if (!spec)
|
||||||
@@ -180,7 +185,7 @@ bool charset_wire_receiver_spec_valid(const char* spec, const char* server_spec)
|
|||||||
if (charset_spec_parse(spec, &local, &remote) != 0)
|
if (charset_spec_parse(spec, &local, &remote) != 0)
|
||||||
return false;
|
return false;
|
||||||
const char* wire = remote;
|
const char* wire = remote;
|
||||||
const char* target_local = local;
|
const char* target_local = remote;
|
||||||
char* server_local = NULL;
|
char* server_local = NULL;
|
||||||
char* server_remote = NULL;
|
char* server_remote = NULL;
|
||||||
if (server_spec) {
|
if (server_spec) {
|
||||||
@@ -302,13 +307,13 @@ bool charset_wire_init_receiver(const char* spec, const char* server_spec) {
|
|||||||
char* remote;
|
char* remote;
|
||||||
if (charset_spec_parse(spec, &local, &remote) != 0)
|
if (charset_spec_parse(spec, &local, &remote) != 0)
|
||||||
return false;
|
return false;
|
||||||
/* The wire charset is the client spec's REMOTE half; the local charset is
|
/* The wire charset is the client spec's REMOTE half (rsync's LOCAL,REMOTE
|
||||||
* the client spec's LOCAL half unless the server was itself started with
|
* spec stays the same push or pull, so on a push the destination end's
|
||||||
* --iconv naming a different local charset (the server halves above never
|
* charset is REMOTE and the receiver writes the wire bytes verbatim). Only a
|
||||||
* travel, so the server's own flag is the only way its local charset can
|
* server started with its own --iconv declares a different local charset (the
|
||||||
* differ from what the client assumed). */
|
* server halves above never travel), and then it is that spec's LOCAL half. */
|
||||||
const char* wire = remote;
|
const char* wire = remote;
|
||||||
const char* target_local = local;
|
const char* target_local = remote;
|
||||||
char* server_local = NULL;
|
char* server_local = NULL;
|
||||||
char* server_remote = NULL;
|
char* server_remote = NULL;
|
||||||
if (server_spec) {
|
if (server_spec) {
|
||||||
|
|||||||
@@ -57,8 +57,9 @@ void charset_conversion_close(void* conversion);
|
|||||||
/* Process-wide wire conversion. charset_wire_init_sender (client side) opens
|
/* Process-wide wire conversion. charset_wire_init_sender (client side) opens
|
||||||
* LOCAL->REMOTE; charset_wire_init_receiver (server side) opens
|
* LOCAL->REMOTE; charset_wire_init_receiver (server side) opens
|
||||||
* wire(REMOTE)->server-local. server_spec is the server's own --iconv, whose
|
* wire(REMOTE)->server-local. server_spec is the server's own --iconv, whose
|
||||||
* LOCAL half may override the local charset the client assumed; NULL reuses
|
* LOCAL half overrides the destination charset; NULL means the destination
|
||||||
* the client spec's LOCAL half. Both return false on an unsupported spec.
|
* charset is the client spec's REMOTE half (rsync's push semantics: the wire
|
||||||
|
* bytes are written verbatim). Both return false on an unsupported spec.
|
||||||
* The state is freed with charset_wire_free. */
|
* The state is freed with charset_wire_free. */
|
||||||
bool charset_wire_init_sender(const char* spec);
|
bool charset_wire_init_sender(const char* spec);
|
||||||
bool charset_wire_init_receiver(const char* spec, const char* server_spec);
|
bool charset_wire_init_receiver(const char* spec, const char* server_spec);
|
||||||
@@ -66,9 +67,9 @@ void charset_wire_free(void);
|
|||||||
bool charset_wire_active(void);
|
bool charset_wire_active(void);
|
||||||
|
|
||||||
/* Pre-ack receiver-direction sanity (see charset_wire_init_receiver): true
|
/* Pre-ack receiver-direction sanity (see charset_wire_init_receiver): true
|
||||||
* when the exact wire->server-local conversion the receiver will use (client
|
* when the exact wire->destination conversion the receiver will use (client
|
||||||
* spec's REMOTE half into the server's own LOCAL half, or the client's LOCAL
|
* spec's REMOTE half into the server's own LOCAL half, or REMOTE->REMOTE when
|
||||||
* half when the server has no --iconv) opens and produces NUL-free output. */
|
* the server has no --iconv) opens and produces NUL-free output. */
|
||||||
bool charset_wire_receiver_spec_valid(const char* spec, const char* server_spec);
|
bool charset_wire_receiver_spec_valid(const char* spec, const char* server_spec);
|
||||||
|
|
||||||
/* Convert a path across the wire in the process direction. Returns a malloc'd
|
/* Convert a path across the wire in the process direction. Returns a malloc'd
|
||||||
|
|||||||
+370
-18
@@ -1,12 +1,171 @@
|
|||||||
#include "checksum.h"
|
#include "checksum.h"
|
||||||
|
#include "utils.h"
|
||||||
|
#include <fcntl.h>
|
||||||
#include <openssl/evp.h>
|
#include <openssl/evp.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
#include <strings.h>
|
#include <strings.h>
|
||||||
|
#include <unistd.h>
|
||||||
|
|
||||||
/* delta.c owns the single XXH_IMPLEMENTATION that provides the xxHash symbols
|
/* delta.c owns the single XXH_IMPLEMENTATION that provides the xxHash symbols
|
||||||
* for the whole binary; this TU only needs the declarations. */
|
* for the whole binary; this TU only needs the declarations. The streaming
|
||||||
|
* state structs and XXH3_update are exposed only with XXH_STATIC_LINKING_ONLY. */
|
||||||
|
#define XXH_STATIC_LINKING_ONLY
|
||||||
#include <xxhash.h>
|
#include <xxhash.h>
|
||||||
|
|
||||||
|
/* ---------------------------------------------------------------------------
|
||||||
|
* Self-contained MD4 (RFC 1320). OpenSSL's MD4 lives in the legacy provider
|
||||||
|
* and is not guaranteed present, so FastSync carries its own implementation to
|
||||||
|
* keep --checksum-choice=md4 working on every build.
|
||||||
|
* ------------------------------------------------------------------------- */
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
uint32_t state[4];
|
||||||
|
uint64_t bit_count;
|
||||||
|
uint8_t buffer[64];
|
||||||
|
size_t buffer_len;
|
||||||
|
} Md4Ctx;
|
||||||
|
|
||||||
|
static uint32_t md4_rotl(uint32_t x, int n) {
|
||||||
|
return (x << n) | (x >> (32 - n));
|
||||||
|
}
|
||||||
|
|
||||||
|
static void md4_transform(uint32_t state[4], const uint8_t block[64]) {
|
||||||
|
uint32_t x[16];
|
||||||
|
for (int i = 0; i < 16; i++)
|
||||||
|
x[i] = (uint32_t)block[i * 4] | ((uint32_t)block[i * 4 + 1] << 8) |
|
||||||
|
((uint32_t)block[i * 4 + 2] << 16) | ((uint32_t)block[i * 4 + 3] << 24);
|
||||||
|
|
||||||
|
uint32_t a = state[0], b = state[1], c = state[2], d = state[3];
|
||||||
|
|
||||||
|
#define F(x, y, z) (((x) & (y)) | (~(x) & (z)))
|
||||||
|
#define G(x, y, z) (((x) & (y)) | ((x) & (z)) | ((y) & (z)))
|
||||||
|
#define H(x, y, z) ((x) ^ (y) ^ (z))
|
||||||
|
#define ROUND1(a, b, c, d, k, s) a = md4_rotl(a + F(b, c, d) + x[k], s)
|
||||||
|
#define ROUND2(a, b, c, d, k, s) a = md4_rotl(a + G(b, c, d) + x[k] + 0x5a827999u, s)
|
||||||
|
#define ROUND3(a, b, c, d, k, s) a = md4_rotl(a + H(b, c, d) + x[k] + 0x6ed9eba1u, s)
|
||||||
|
|
||||||
|
ROUND1(a, b, c, d, 0, 3);
|
||||||
|
ROUND1(d, a, b, c, 1, 7);
|
||||||
|
ROUND1(c, d, a, b, 2, 11);
|
||||||
|
ROUND1(b, c, d, a, 3, 19);
|
||||||
|
ROUND1(a, b, c, d, 4, 3);
|
||||||
|
ROUND1(d, a, b, c, 5, 7);
|
||||||
|
ROUND1(c, d, a, b, 6, 11);
|
||||||
|
ROUND1(b, c, d, a, 7, 19);
|
||||||
|
ROUND1(a, b, c, d, 8, 3);
|
||||||
|
ROUND1(d, a, b, c, 9, 7);
|
||||||
|
ROUND1(c, d, a, b, 10, 11);
|
||||||
|
ROUND1(b, c, d, a, 11, 19);
|
||||||
|
ROUND1(a, b, c, d, 12, 3);
|
||||||
|
ROUND1(d, a, b, c, 13, 7);
|
||||||
|
ROUND1(c, d, a, b, 14, 11);
|
||||||
|
ROUND1(b, c, d, a, 15, 19);
|
||||||
|
|
||||||
|
ROUND2(a, b, c, d, 0, 3);
|
||||||
|
ROUND2(d, a, b, c, 4, 5);
|
||||||
|
ROUND2(c, d, a, b, 8, 9);
|
||||||
|
ROUND2(b, c, d, a, 12, 13);
|
||||||
|
ROUND2(a, b, c, d, 1, 3);
|
||||||
|
ROUND2(d, a, b, c, 5, 5);
|
||||||
|
ROUND2(c, d, a, b, 9, 9);
|
||||||
|
ROUND2(b, c, d, a, 13, 13);
|
||||||
|
ROUND2(a, b, c, d, 2, 3);
|
||||||
|
ROUND2(d, a, b, c, 6, 5);
|
||||||
|
ROUND2(c, d, a, b, 10, 9);
|
||||||
|
ROUND2(b, c, d, a, 14, 13);
|
||||||
|
ROUND2(a, b, c, d, 3, 3);
|
||||||
|
ROUND2(d, a, b, c, 7, 5);
|
||||||
|
ROUND2(c, d, a, b, 11, 9);
|
||||||
|
ROUND2(b, c, d, a, 15, 13);
|
||||||
|
|
||||||
|
ROUND3(a, b, c, d, 0, 3);
|
||||||
|
ROUND3(d, a, b, c, 8, 9);
|
||||||
|
ROUND3(c, d, a, b, 4, 11);
|
||||||
|
ROUND3(b, c, d, a, 12, 15);
|
||||||
|
ROUND3(a, b, c, d, 2, 3);
|
||||||
|
ROUND3(d, a, b, c, 10, 9);
|
||||||
|
ROUND3(c, d, a, b, 6, 11);
|
||||||
|
ROUND3(b, c, d, a, 14, 15);
|
||||||
|
ROUND3(a, b, c, d, 1, 3);
|
||||||
|
ROUND3(d, a, b, c, 9, 9);
|
||||||
|
ROUND3(c, d, a, b, 5, 11);
|
||||||
|
ROUND3(b, c, d, a, 13, 15);
|
||||||
|
ROUND3(a, b, c, d, 3, 3);
|
||||||
|
ROUND3(d, a, b, c, 11, 9);
|
||||||
|
ROUND3(c, d, a, b, 7, 11);
|
||||||
|
ROUND3(b, c, d, a, 15, 15);
|
||||||
|
|
||||||
|
#undef F
|
||||||
|
#undef G
|
||||||
|
#undef H
|
||||||
|
#undef ROUND1
|
||||||
|
#undef ROUND2
|
||||||
|
#undef ROUND3
|
||||||
|
|
||||||
|
state[0] += a;
|
||||||
|
state[1] += b;
|
||||||
|
state[2] += c;
|
||||||
|
state[3] += d;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void md4_init(Md4Ctx* ctx) {
|
||||||
|
ctx->state[0] = 0x67452301u;
|
||||||
|
ctx->state[1] = 0xefcdab89u;
|
||||||
|
ctx->state[2] = 0x98badcfeu;
|
||||||
|
ctx->state[3] = 0x10325476u;
|
||||||
|
ctx->bit_count = 0;
|
||||||
|
ctx->buffer_len = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void md4_update(Md4Ctx* ctx, const uint8_t* data, size_t len) {
|
||||||
|
ctx->bit_count += (uint64_t)len * 8;
|
||||||
|
while (len > 0) {
|
||||||
|
size_t space = sizeof(ctx->buffer) - ctx->buffer_len;
|
||||||
|
size_t take = len < space ? len : space;
|
||||||
|
memcpy(ctx->buffer + ctx->buffer_len, data, take);
|
||||||
|
ctx->buffer_len += take;
|
||||||
|
data += take;
|
||||||
|
len -= take;
|
||||||
|
if (ctx->buffer_len == sizeof(ctx->buffer)) {
|
||||||
|
md4_transform(ctx->state, ctx->buffer);
|
||||||
|
ctx->buffer_len = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static void md4_final(Md4Ctx* ctx, uint8_t out[16]) {
|
||||||
|
uint64_t bit_count = ctx->bit_count;
|
||||||
|
uint8_t pad = 0x80;
|
||||||
|
md4_update(ctx, &pad, 1);
|
||||||
|
uint8_t zero = 0;
|
||||||
|
while (ctx->buffer_len != 56)
|
||||||
|
md4_update(ctx, &zero, 1);
|
||||||
|
uint8_t length_le[8];
|
||||||
|
for (int i = 0; i < 8; i++)
|
||||||
|
length_le[i] = (uint8_t)((bit_count >> (8 * i)) & 0xff);
|
||||||
|
md4_update(ctx, length_le, sizeof(length_le));
|
||||||
|
for (int i = 0; i < 4; i++) {
|
||||||
|
out[i * 4] = (uint8_t)(ctx->state[i] & 0xff);
|
||||||
|
out[i * 4 + 1] = (uint8_t)((ctx->state[i] >> 8) & 0xff);
|
||||||
|
out[i * 4 + 2] = (uint8_t)((ctx->state[i] >> 16) & 0xff);
|
||||||
|
out[i * 4 + 3] = (uint8_t)((ctx->state[i] >> 24) & 0xff);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* One-shot EVP digest (md5/sha1). Returns false when OpenSSL refuses. */
|
||||||
|
static bool evp_digest(const EVP_MD* md, const void* data, size_t size, uint8_t* out,
|
||||||
|
size_t out_capacity, size_t* out_len) {
|
||||||
|
static const uint8_t empty = 0;
|
||||||
|
const void* input = data ? data : ∅
|
||||||
|
unsigned int digest_len = 0;
|
||||||
|
if (EVP_Digest(input, size, out, &digest_len, md, NULL) != 1)
|
||||||
|
return false;
|
||||||
|
if (digest_len > out_capacity)
|
||||||
|
return false;
|
||||||
|
*out_len = digest_len;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
bool checksum_digest(ChecksumAlgo algo, uint64_t seed, const void* data, size_t size, uint8_t* out,
|
bool checksum_digest(ChecksumAlgo algo, uint64_t seed, const void* data, size_t size, uint8_t* out,
|
||||||
size_t out_capacity, size_t* out_len) {
|
size_t out_capacity, size_t* out_len) {
|
||||||
if (!out || !out_len || out_capacity < CHECKSUM_MAX_DIGEST_LEN)
|
if (!out || !out_len || out_capacity < CHECKSUM_MAX_DIGEST_LEN)
|
||||||
@@ -14,30 +173,166 @@ bool checksum_digest(ChecksumAlgo algo, uint64_t seed, const void* data, size_t
|
|||||||
if (data == NULL && size != 0)
|
if (data == NULL && size != 0)
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
if (algo == CHECKSUM_ALGO_XXH64) {
|
switch (algo) {
|
||||||
|
case CHECKSUM_ALGO_XXH64: {
|
||||||
uint64_t digest = XXH64(data, size, seed);
|
uint64_t digest = XXH64(data, size, seed);
|
||||||
memcpy(out, &digest, sizeof(digest));
|
memcpy(out, &digest, sizeof(digest));
|
||||||
*out_len = sizeof(digest);
|
*out_len = sizeof(digest);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
case CHECKSUM_ALGO_XXH3: {
|
||||||
if (algo == CHECKSUM_ALGO_MD5) {
|
uint64_t digest = XXH3_64bits_withSeed(data, size, seed);
|
||||||
|
memcpy(out, &digest, sizeof(digest));
|
||||||
|
*out_len = sizeof(digest);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
case CHECKSUM_ALGO_XXH128: {
|
||||||
|
XXH128_hash_t digest = XXH3_128bits_withSeed(data, size, seed);
|
||||||
|
memcpy(out, &digest, sizeof(digest));
|
||||||
|
*out_len = sizeof(digest);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
case CHECKSUM_ALGO_MD5:
|
||||||
/* md5 takes no seed; the caller's seed is deliberately ignored (documented
|
/* md5 takes no seed; the caller's seed is deliberately ignored (documented
|
||||||
* in RSYNC_COMPAT.md). OpenSSL's one-shot EVP_Digest needs a non-NULL
|
* in RSYNC_COMPAT.md). */
|
||||||
* buffer even for an empty input, so map a NULL data + size==0 to an empty
|
return evp_digest(EVP_md5(), data, size, out, out_capacity, out_len);
|
||||||
* buffer. */
|
case CHECKSUM_ALGO_MD4: {
|
||||||
static const uint8_t empty = 0;
|
Md4Ctx ctx;
|
||||||
const void* input = data ? data : ∅
|
md4_init(&ctx);
|
||||||
unsigned int digest_len = 0;
|
md4_update(&ctx, (const uint8_t*)data, size);
|
||||||
if (EVP_Digest(input, size, out, &digest_len, EVP_md5(), NULL) != 1)
|
md4_final(&ctx, out);
|
||||||
return false;
|
*out_len = 16;
|
||||||
if (digest_len > out_capacity)
|
return true;
|
||||||
return false;
|
}
|
||||||
*out_len = digest_len;
|
case CHECKSUM_ALGO_SHA1:
|
||||||
|
/* sha1 takes no seed; the caller's seed is deliberately ignored. */
|
||||||
|
return evp_digest(EVP_sha1(), data, size, out, out_capacity, out_len);
|
||||||
|
case CHECKSUM_ALGO_NONE:
|
||||||
|
/* No checksum requested: an empty digest is the successful result. */
|
||||||
|
*out_len = 0;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool checksum_digest_file(ChecksumAlgo algo, uint64_t seed, const char* path, uint8_t* out,
|
||||||
|
size_t out_capacity, size_t* out_len) {
|
||||||
|
if (!path || !out || !out_len || out_capacity < CHECKSUM_MAX_DIGEST_LEN)
|
||||||
|
return false;
|
||||||
|
|
||||||
|
int fd = open(path, O_RDONLY | O_CLOEXEC);
|
||||||
|
if (fd < 0)
|
||||||
|
return false;
|
||||||
|
|
||||||
|
bool ok = checksum_digest_fd(algo, seed, fd, out, out_capacity, out_len);
|
||||||
|
close(fd);
|
||||||
|
return ok;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool checksum_digest_fd(ChecksumAlgo algo, uint64_t seed, int fd, uint8_t* out, size_t out_capacity,
|
||||||
|
size_t* out_len) {
|
||||||
|
if (fd < 0 || !out || !out_len || out_capacity < CHECKSUM_MAX_DIGEST_LEN)
|
||||||
|
return false;
|
||||||
|
|
||||||
|
if (algo == CHECKSUM_ALGO_NONE) {
|
||||||
|
/* No checksum requested: nothing to read; an empty digest succeeds. */
|
||||||
|
*out_len = 0;
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
return false;
|
uint8_t buffer[64 * 1024];
|
||||||
|
bool ok = false;
|
||||||
|
lseek(fd, 0, SEEK_SET);
|
||||||
|
|
||||||
|
if (algo == CHECKSUM_ALGO_MD5 || algo == CHECKSUM_ALGO_SHA1) {
|
||||||
|
const EVP_MD* md = algo == CHECKSUM_ALGO_MD5 ? EVP_md5() : EVP_sha1();
|
||||||
|
EVP_MD_CTX* ctx = EVP_MD_CTX_new();
|
||||||
|
if (!ctx)
|
||||||
|
return false;
|
||||||
|
unsigned int digest_len = 0;
|
||||||
|
if (EVP_DigestInit_ex(ctx, md, NULL) == 1) {
|
||||||
|
ok = true;
|
||||||
|
ssize_t got;
|
||||||
|
while ((got = read(fd, buffer, sizeof(buffer))) > 0) {
|
||||||
|
if (EVP_DigestUpdate(ctx, buffer, (size_t)got) != 1) {
|
||||||
|
ok = false;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (got < 0)
|
||||||
|
ok = false;
|
||||||
|
if (ok && EVP_DigestFinal_ex(ctx, out, &digest_len) == 1 && digest_len <= out_capacity)
|
||||||
|
*out_len = digest_len;
|
||||||
|
else
|
||||||
|
ok = false;
|
||||||
|
}
|
||||||
|
EVP_MD_CTX_free(ctx);
|
||||||
|
return ok;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (algo == CHECKSUM_ALGO_MD4) {
|
||||||
|
Md4Ctx ctx;
|
||||||
|
md4_init(&ctx);
|
||||||
|
ok = true;
|
||||||
|
ssize_t got;
|
||||||
|
while ((got = read(fd, buffer, sizeof(buffer))) > 0)
|
||||||
|
md4_update(&ctx, buffer, (size_t)got);
|
||||||
|
if (got < 0)
|
||||||
|
ok = false;
|
||||||
|
if (ok) {
|
||||||
|
md4_final(&ctx, out);
|
||||||
|
*out_len = 16;
|
||||||
|
}
|
||||||
|
return ok;
|
||||||
|
}
|
||||||
|
|
||||||
|
XXH64_state_t xxh64;
|
||||||
|
XXH3_state_t* xxh3 = NULL;
|
||||||
|
if (algo == CHECKSUM_ALGO_XXH64) {
|
||||||
|
XXH64_reset(&xxh64, seed);
|
||||||
|
} else if (algo == CHECKSUM_ALGO_XXH3 || algo == CHECKSUM_ALGO_XXH128) {
|
||||||
|
xxh3 = XXH3_createState();
|
||||||
|
if (!xxh3)
|
||||||
|
return false;
|
||||||
|
if (algo == CHECKSUM_ALGO_XXH3)
|
||||||
|
XXH3_64bits_reset_withSeed(xxh3, seed);
|
||||||
|
else
|
||||||
|
XXH3_128bits_reset_withSeed(xxh3, seed);
|
||||||
|
} else {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
ok = true;
|
||||||
|
ssize_t got;
|
||||||
|
while ((got = read(fd, buffer, sizeof(buffer))) > 0) {
|
||||||
|
if (algo == CHECKSUM_ALGO_XXH64)
|
||||||
|
XXH64_update(&xxh64, buffer, (size_t)got);
|
||||||
|
else if (XXH3_64bits_update(xxh3, buffer, (size_t)got) == XXH_ERROR) {
|
||||||
|
ok = false;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (got < 0)
|
||||||
|
ok = false;
|
||||||
|
|
||||||
|
if (ok) {
|
||||||
|
if (algo == CHECKSUM_ALGO_XXH64) {
|
||||||
|
uint64_t digest = XXH64_digest(&xxh64);
|
||||||
|
memcpy(out, &digest, sizeof(digest));
|
||||||
|
*out_len = sizeof(digest);
|
||||||
|
} else if (algo == CHECKSUM_ALGO_XXH3) {
|
||||||
|
uint64_t digest = XXH3_64bits_digest(xxh3);
|
||||||
|
memcpy(out, &digest, sizeof(digest));
|
||||||
|
*out_len = sizeof(digest);
|
||||||
|
} else {
|
||||||
|
XXH128_hash_t digest = XXH3_128bits_digest(xxh3);
|
||||||
|
memcpy(out, &digest, sizeof(digest));
|
||||||
|
*out_len = sizeof(digest);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (xxh3)
|
||||||
|
XXH3_freeState(xxh3);
|
||||||
|
return ok;
|
||||||
}
|
}
|
||||||
|
|
||||||
int checksum_algo_from_name(const char* name) {
|
int checksum_algo_from_name(const char* name) {
|
||||||
@@ -45,8 +340,18 @@ int checksum_algo_from_name(const char* name) {
|
|||||||
return -1;
|
return -1;
|
||||||
if (strcasecmp(name, "xxh64") == 0 || strcasecmp(name, "xxhash") == 0)
|
if (strcasecmp(name, "xxh64") == 0 || strcasecmp(name, "xxhash") == 0)
|
||||||
return (int)CHECKSUM_ALGO_XXH64;
|
return (int)CHECKSUM_ALGO_XXH64;
|
||||||
|
if (strcasecmp(name, "xxh3") == 0)
|
||||||
|
return (int)CHECKSUM_ALGO_XXH3;
|
||||||
|
if (strcasecmp(name, "xxh128") == 0)
|
||||||
|
return (int)CHECKSUM_ALGO_XXH128;
|
||||||
if (strcasecmp(name, "md5") == 0)
|
if (strcasecmp(name, "md5") == 0)
|
||||||
return (int)CHECKSUM_ALGO_MD5;
|
return (int)CHECKSUM_ALGO_MD5;
|
||||||
|
if (strcasecmp(name, "md4") == 0)
|
||||||
|
return (int)CHECKSUM_ALGO_MD4;
|
||||||
|
if (strcasecmp(name, "sha1") == 0)
|
||||||
|
return (int)CHECKSUM_ALGO_SHA1;
|
||||||
|
if (strcasecmp(name, "none") == 0)
|
||||||
|
return (int)CHECKSUM_ALGO_NONE;
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -54,22 +359,69 @@ const char* checksum_algo_name(ChecksumAlgo algo) {
|
|||||||
switch (algo) {
|
switch (algo) {
|
||||||
case CHECKSUM_ALGO_XXH64:
|
case CHECKSUM_ALGO_XXH64:
|
||||||
return "xxh64";
|
return "xxh64";
|
||||||
|
case CHECKSUM_ALGO_XXH3:
|
||||||
|
return "xxh3";
|
||||||
|
case CHECKSUM_ALGO_XXH128:
|
||||||
|
return "xxh128";
|
||||||
case CHECKSUM_ALGO_MD5:
|
case CHECKSUM_ALGO_MD5:
|
||||||
return "md5";
|
return "md5";
|
||||||
|
case CHECKSUM_ALGO_MD4:
|
||||||
|
return "md4";
|
||||||
|
case CHECKSUM_ALGO_SHA1:
|
||||||
|
return "sha1";
|
||||||
|
case CHECKSUM_ALGO_NONE:
|
||||||
|
return "none";
|
||||||
}
|
}
|
||||||
return "<unknown>";
|
return "<unknown>";
|
||||||
}
|
}
|
||||||
|
|
||||||
bool checksum_algo_valid(int algo) {
|
bool checksum_algo_valid(int algo) {
|
||||||
return algo == (int)CHECKSUM_ALGO_XXH64 || algo == (int)CHECKSUM_ALGO_MD5;
|
return algo == (int)CHECKSUM_ALGO_XXH64 || algo == (int)CHECKSUM_ALGO_MD5 ||
|
||||||
|
algo == (int)CHECKSUM_ALGO_XXH3 || algo == (int)CHECKSUM_ALGO_XXH128 ||
|
||||||
|
algo == (int)CHECKSUM_ALGO_MD4 || algo == (int)CHECKSUM_ALGO_SHA1 ||
|
||||||
|
algo == (int)CHECKSUM_ALGO_NONE;
|
||||||
}
|
}
|
||||||
|
|
||||||
uint8_t checksum_digest_len(ChecksumAlgo algo) {
|
uint8_t checksum_digest_len(ChecksumAlgo algo) {
|
||||||
switch (algo) {
|
switch (algo) {
|
||||||
case CHECKSUM_ALGO_XXH64:
|
case CHECKSUM_ALGO_XXH64:
|
||||||
|
case CHECKSUM_ALGO_XXH3:
|
||||||
return 8;
|
return 8;
|
||||||
|
case CHECKSUM_ALGO_XXH128:
|
||||||
case CHECKSUM_ALGO_MD5:
|
case CHECKSUM_ALGO_MD5:
|
||||||
|
case CHECKSUM_ALGO_MD4:
|
||||||
return 16;
|
return 16;
|
||||||
|
case CHECKSUM_ALGO_SHA1:
|
||||||
|
return 20;
|
||||||
|
case CHECKSUM_ALGO_NONE:
|
||||||
|
return 0;
|
||||||
}
|
}
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static ChecksumAlgo compiled_checksum_preference_first(void) {
|
||||||
|
/* rsync 3.4.1 default preference order; every entry is compiled in, so this
|
||||||
|
* resolves to xxh128. */
|
||||||
|
static const ChecksumAlgo preference[] = {
|
||||||
|
CHECKSUM_ALGO_XXH128, CHECKSUM_ALGO_XXH3, CHECKSUM_ALGO_XXH64, CHECKSUM_ALGO_MD5,
|
||||||
|
CHECKSUM_ALGO_MD4, CHECKSUM_ALGO_SHA1, CHECKSUM_ALGO_NONE,
|
||||||
|
};
|
||||||
|
for (size_t i = 0; i < sizeof(preference) / sizeof(preference[0]); i++) {
|
||||||
|
if (checksum_algo_valid((int)preference[i]))
|
||||||
|
return preference[i];
|
||||||
|
}
|
||||||
|
return CHECKSUM_ALGO_XXH64;
|
||||||
|
}
|
||||||
|
|
||||||
|
int checksum_choice_resolve(void) {
|
||||||
|
bool specified = false;
|
||||||
|
int env = env_choice_first("RSYNC_CHECKSUM_LIST", checksum_algo_from_name, &specified);
|
||||||
|
if (specified)
|
||||||
|
return env; /* -1 = the list named no supported checksum */
|
||||||
|
return (int)compiled_checksum_preference_first();
|
||||||
|
}
|
||||||
|
|
||||||
|
ChecksumAlgo checksum_negotiate_default(void) {
|
||||||
|
int resolved = checksum_choice_resolve();
|
||||||
|
return resolved >= 0 ? (ChecksumAlgo)resolved : compiled_checksum_preference_first();
|
||||||
|
}
|
||||||
|
|||||||
+55
-11
@@ -8,18 +8,35 @@
|
|||||||
/* Whole-file content-digest algorithms selectable with --checksum-choice and
|
/* Whole-file content-digest algorithms selectable with --checksum-choice and
|
||||||
* seeded with --checksum-seed. The ids are the values actually placed on the
|
* seeded with --checksum-seed. The ids are the values actually placed on the
|
||||||
* wire (config frame), so they must be kept stable and validated on receive.
|
* wire (config frame), so they must be kept stable and validated on receive.
|
||||||
* CHECKSUM_ALGO_XXH64 == 0 is the default and is byte-for-byte what FastSync
|
* CHECKSUM_ALGO_XXH64 == 0 is the historical FastSync default and its numeric
|
||||||
* computed before these options existed (xxHash64 with seed 0). */
|
* value is preserved. The full set mirrors the algorithms rsync 3.4.1 can be
|
||||||
typedef enum { CHECKSUM_ALGO_XXH64 = 0, CHECKSUM_ALGO_MD5 = 1 } ChecksumAlgo;
|
* built with; every one of them is implemented here. */
|
||||||
|
typedef enum {
|
||||||
|
CHECKSUM_ALGO_XXH64 = 0,
|
||||||
|
CHECKSUM_ALGO_MD5 = 1,
|
||||||
|
CHECKSUM_ALGO_XXH3 = 2,
|
||||||
|
CHECKSUM_ALGO_XXH128 = 3,
|
||||||
|
CHECKSUM_ALGO_MD4 = 4,
|
||||||
|
CHECKSUM_ALGO_SHA1 = 5,
|
||||||
|
CHECKSUM_ALGO_NONE = 6
|
||||||
|
} ChecksumAlgo;
|
||||||
|
|
||||||
/* md5 digest is 16 bytes, the longest supported. */
|
/* FastSync's negotiated default (rsync 3.4.1 auto-negotiates xxh128 first).
|
||||||
#define CHECKSUM_MAX_DIGEST_LEN 16
|
* The wire default for Config->checksum_algo is this value. */
|
||||||
|
#define CHECKSUM_ALGO_DEFAULT CHECKSUM_ALGO_XXH128
|
||||||
|
|
||||||
|
/* sha1 digest is 20 bytes, the longest supported. */
|
||||||
|
#define CHECKSUM_MAX_DIGEST_LEN 20
|
||||||
|
|
||||||
/* Compute the whole-file digest of the first `size` bytes of `data`.
|
/* Compute the whole-file digest of the first `size` bytes of `data`.
|
||||||
*
|
*
|
||||||
* - CHECKSUM_ALGO_XXH64: xxHash64(data, size, seed) (full 64-bit seed).
|
* - CHECKSUM_ALGO_XXH64: xxHash64(data, size, seed) (full 64-bit seed).
|
||||||
* - CHECKSUM_ALGO_MD5: md5(data, size) via OpenSSL EVP.
|
* - CHECKSUM_ALGO_XXH3: XXH3_64bits_withSeed(data, size, seed).
|
||||||
* md5 has no seed, so `seed` is ignored (documented).
|
* - CHECKSUM_ALGO_XXH128: XXH3_128bits_withSeed(data, size, seed).
|
||||||
|
* - CHECKSUM_ALGO_MD5: md5(data, size) via OpenSSL EVP (seed ignored).
|
||||||
|
* - CHECKSUM_ALGO_MD4: md4(data, size), self-contained RFC 1320 (seed ignored).
|
||||||
|
* - CHECKSUM_ALGO_SHA1: sha1(data, size) via OpenSSL EVP (seed ignored).
|
||||||
|
* - CHECKSUM_ALGO_NONE: no digest; *out_len is 0 and nothing is written.
|
||||||
* - `size == 0` hashes the empty input (plus its seed), not a NULL input.
|
* - `size == 0` hashes the empty input (plus its seed), not a NULL input.
|
||||||
*
|
*
|
||||||
* Writes up to `out_capacity` bytes into `out`, storing the digest length in
|
* Writes up to `out_capacity` bytes into `out`, storing the digest length in
|
||||||
@@ -28,9 +45,23 @@ typedef enum { CHECKSUM_ALGO_XXH64 = 0, CHECKSUM_ALGO_MD5 = 1 } ChecksumAlgo;
|
|||||||
bool checksum_digest(ChecksumAlgo algo, uint64_t seed, const void* data, size_t size, uint8_t* out,
|
bool checksum_digest(ChecksumAlgo algo, uint64_t seed, const void* data, size_t size, uint8_t* out,
|
||||||
size_t out_capacity, size_t* out_len);
|
size_t out_capacity, size_t* out_len);
|
||||||
|
|
||||||
|
/* Streaming whole-file digest: hash the contents of `path` without holding the
|
||||||
|
* whole file in memory. Same digest/capacity contract as checksum_digest.
|
||||||
|
* Returns false on open/read failure or an undersized buffer. */
|
||||||
|
bool checksum_digest_file(ChecksumAlgo algo, uint64_t seed, const char* path, uint8_t* out,
|
||||||
|
size_t out_capacity, size_t* out_len);
|
||||||
|
|
||||||
|
/* Descriptor form of the streaming digest: rewinds `fd` to the start and hashes
|
||||||
|
* to EOF without closing it. Used by the --verify-basis path to hash an
|
||||||
|
* already-open, root-confined basis descriptor. Same contract as
|
||||||
|
* checksum_digest_file. */
|
||||||
|
bool checksum_digest_fd(ChecksumAlgo algo, uint64_t seed, int fd, uint8_t* out, size_t out_capacity,
|
||||||
|
size_t* out_len);
|
||||||
|
|
||||||
/* Resolve a --checksum-choice string (case-insensitive) to an algorithm id.
|
/* Resolve a --checksum-choice string (case-insensitive) to an algorithm id.
|
||||||
* Accepts "xxh64" and "xxhash" (both map to CHECKSUM_ALGO_XXH64, rsync's
|
* Accepts "xxh64"/"xxhash", "xxh3", "xxh128", "md5", "md4", "sha1", "none".
|
||||||
* xxhash spelling) and "md5". Returns -1 for any unsupported name. */
|
* "auto" is not an algorithm here; the caller resolves it to the negotiated
|
||||||
|
* default. Returns -1 for any unrecognized name. */
|
||||||
int checksum_algo_from_name(const char* name);
|
int checksum_algo_from_name(const char* name);
|
||||||
|
|
||||||
/* Canonical name of an algorithm (used in CLI error messages). */
|
/* Canonical name of an algorithm (used in CLI error messages). */
|
||||||
@@ -39,7 +70,20 @@ const char* checksum_algo_name(ChecksumAlgo algo);
|
|||||||
/* True when `algo` is a supported id (used by config receive validation). */
|
/* True when `algo` is a supported id (used by config receive validation). */
|
||||||
bool checksum_algo_valid(int algo);
|
bool checksum_algo_valid(int algo);
|
||||||
|
|
||||||
/* Digest length in bytes for an algorithm (xxx64 = 8, md5 = 16). */
|
/* Digest length in bytes for an algorithm (xxh64/xxh3 = 8,
|
||||||
|
* md5/md4/xxh128 = 16, sha1 = 20, none = 0). */
|
||||||
uint8_t checksum_digest_len(ChecksumAlgo algo);
|
uint8_t checksum_digest_len(ChecksumAlgo algo);
|
||||||
|
|
||||||
#endif /* CHECKSUM_H */
|
/* Pick the first algorithm from FastSync's compiled-in preference list that is
|
||||||
|
* supported on this build (rsync 3.4.1's `--version` order:
|
||||||
|
* xxh128 xxh3 xxh64 md5 md4 sha1 none). Used to resolve "auto". */
|
||||||
|
ChecksumAlgo checksum_negotiate_default(void);
|
||||||
|
|
||||||
|
/* Resolve "auto" the way rsync does: the first supported name in
|
||||||
|
* RSYNC_CHECKSUM_LIST (whitespace-separated, client half ends at '&'), then the
|
||||||
|
* compiled-in preference order when the variable is unset/blank. Returns -1
|
||||||
|
* when the variable is set but names no supported checksum (rsync's failed
|
||||||
|
* negotiation), otherwise a valid ChecksumAlgo id. */
|
||||||
|
int checksum_choice_resolve(void);
|
||||||
|
|
||||||
|
#endif /* CHECKSUM_H */
|
||||||
|
|||||||
+170
-77
@@ -1,90 +1,183 @@
|
|||||||
#include "chmod.h"
|
#include "chmod.h"
|
||||||
|
#include "file.h"
|
||||||
#include <stddef.h>
|
#include <stddef.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
|
|
||||||
static bool parse_clause(mode_t* mode, const char* begin, const char* end) {
|
/* rsync's --chmod parser (parse_chmod + tweak_mode). A single clause is
|
||||||
const char* p = begin;
|
* applied as it is completed, so repeated clauses and repeated --chmod options
|
||||||
unsigned who = 0;
|
* (joined with commas by the CLI) accumulate exactly like rsync. The D/F
|
||||||
while (p < end && strchr("ugoa", *p)) {
|
* selectors restrict a clause to directories/files; X adds execute only to
|
||||||
if (*p == 'a')
|
* directories or files that were already executable. */
|
||||||
who = 7;
|
|
||||||
else
|
#define CHMOD_BITS 07777
|
||||||
who |= *p == 'u' ? 1U : (*p == 'g' ? 2U : 4U);
|
#define CHMOD_FLAG_X_KEEP (1U << 0)
|
||||||
p++;
|
#define CHMOD_FLAG_DIRS_ONLY (1U << 1)
|
||||||
}
|
#define CHMOD_FLAG_FILES_ONLY (1U << 2)
|
||||||
if (who == 0)
|
|
||||||
who = 7;
|
enum chmod_op { CHMOD_OP_ADD = 1, CHMOD_OP_SUB, CHMOD_OP_EQ, CHMOD_OP_SET };
|
||||||
if (p == end || (*p != '+' && *p != '-' && *p != '='))
|
enum chmod_state {
|
||||||
return false;
|
CHMOD_STATE_ERROR,
|
||||||
char operation = *p++;
|
CHMOD_STATE_1ST_HALF,
|
||||||
mode_t bits = 0;
|
CHMOD_STATE_2ND_HALF,
|
||||||
while (p < end) {
|
CHMOD_STATE_OCTAL
|
||||||
mode_t bit;
|
};
|
||||||
switch (*p++) {
|
|
||||||
case 'r':
|
|
||||||
bit = 4;
|
|
||||||
break;
|
|
||||||
case 'w':
|
|
||||||
bit = 2;
|
|
||||||
break;
|
|
||||||
case 'x':
|
|
||||||
bit = 1;
|
|
||||||
break;
|
|
||||||
default:
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
bits |= bit;
|
|
||||||
}
|
|
||||||
for (unsigned class_index = 0; class_index < 3; class_index++) {
|
|
||||||
unsigned class_bit = 1U << class_index;
|
|
||||||
if (!(who & class_bit))
|
|
||||||
continue;
|
|
||||||
mode_t shift = (mode_t)((2U - class_index) * 3U);
|
|
||||||
mode_t mask = (mode_t)(7U << shift);
|
|
||||||
mode_t class_bits = (mode_t)(bits << shift);
|
|
||||||
if (operation == '+')
|
|
||||||
*mode |= class_bits;
|
|
||||||
else if (operation == '-')
|
|
||||||
*mode &= ~class_bits;
|
|
||||||
else
|
|
||||||
*mode = (*mode & ~mask) | class_bits;
|
|
||||||
}
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
bool chmod_apply(mode_t mode, const char* spec, mode_t* result) {
|
bool chmod_apply(mode_t mode, const char* spec, mode_t* result) {
|
||||||
if (!spec || !*spec || !result)
|
if (!spec || !*spec || !result)
|
||||||
return false;
|
return false;
|
||||||
bool numeric = true;
|
const mode_t nonperm = mode & ~(mode_t)CHMOD_BITS;
|
||||||
size_t length = strlen(spec);
|
const bool initially_executable = (mode & 0111) != 0;
|
||||||
if (length > 4)
|
|
||||||
numeric = false;
|
|
||||||
for (size_t i = 0; i < length && numeric; i++)
|
|
||||||
numeric = spec[i] >= '0' && spec[i] <= '7';
|
|
||||||
if (numeric) {
|
|
||||||
if (length == 0 || length > 4)
|
|
||||||
return false;
|
|
||||||
mode_t parsed = 0;
|
|
||||||
for (size_t i = 0; i < length; i++)
|
|
||||||
parsed = (mode_t)((parsed << 3) | (spec[i] - '0'));
|
|
||||||
*result = parsed;
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
mode_t changed = mode;
|
mode_t changed = mode;
|
||||||
const char* begin = spec;
|
int state = CHMOD_STATE_1ST_HALF;
|
||||||
while (*begin) {
|
unsigned where = 0;
|
||||||
const char* end = strchr(begin, ',');
|
int what = 0, op = 0, topbits = 0, topoct = 0, flags = 0;
|
||||||
if (!end)
|
const char* p = spec;
|
||||||
end = begin + strlen(begin);
|
while (state != CHMOD_STATE_ERROR) {
|
||||||
if (!parse_clause(&changed, begin, end))
|
if (*p == '\0' || *p == ',') {
|
||||||
return false;
|
int bits;
|
||||||
if (*end == '\0')
|
if (!op) {
|
||||||
|
state = CHMOD_STATE_ERROR;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if (where)
|
||||||
|
bits = (int)(where * (unsigned)what);
|
||||||
|
else {
|
||||||
|
where = 0111;
|
||||||
|
bits = (int)((where * (unsigned)what) & ~(unsigned)file_process_umask());
|
||||||
|
}
|
||||||
|
int mode_and, mode_or;
|
||||||
|
switch (op) {
|
||||||
|
case CHMOD_OP_ADD:
|
||||||
|
mode_and = CHMOD_BITS;
|
||||||
|
mode_or = bits + topoct;
|
||||||
|
break;
|
||||||
|
case CHMOD_OP_SUB:
|
||||||
|
mode_and = CHMOD_BITS - bits - topoct;
|
||||||
|
mode_or = 0;
|
||||||
|
break;
|
||||||
|
case CHMOD_OP_EQ:
|
||||||
|
mode_and = CHMOD_BITS - (int)(where * 7U) - (topoct ? topbits : 0);
|
||||||
|
mode_or = bits + topoct;
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
mode_and = 0;
|
||||||
|
mode_or = bits;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
bool is_dir = S_ISDIR(nonperm);
|
||||||
|
if (!((flags & CHMOD_FLAG_DIRS_ONLY) && !is_dir) &&
|
||||||
|
!((flags & CHMOD_FLAG_FILES_ONLY) && is_dir)) {
|
||||||
|
changed &= (mode_t)mode_and;
|
||||||
|
if ((flags & CHMOD_FLAG_X_KEEP) && !initially_executable && !is_dir)
|
||||||
|
changed |= (mode_t)(mode_or & ~0111);
|
||||||
|
else
|
||||||
|
changed |= (mode_t)mode_or;
|
||||||
|
}
|
||||||
|
if (*p == '\0')
|
||||||
|
break;
|
||||||
|
p++;
|
||||||
|
state = CHMOD_STATE_1ST_HALF;
|
||||||
|
where = 0;
|
||||||
|
what = op = topoct = topbits = flags = 0;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
switch (state) {
|
||||||
|
case CHMOD_STATE_1ST_HALF:
|
||||||
|
switch (*p) {
|
||||||
|
case 'D':
|
||||||
|
if (flags & CHMOD_FLAG_FILES_ONLY) {
|
||||||
|
state = CHMOD_STATE_ERROR;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
flags |= CHMOD_FLAG_DIRS_ONLY;
|
||||||
|
break;
|
||||||
|
case 'F':
|
||||||
|
if (flags & CHMOD_FLAG_DIRS_ONLY) {
|
||||||
|
state = CHMOD_STATE_ERROR;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
flags |= CHMOD_FLAG_FILES_ONLY;
|
||||||
|
break;
|
||||||
|
case 'u':
|
||||||
|
where |= 0100;
|
||||||
|
topbits |= 04000;
|
||||||
|
break;
|
||||||
|
case 'g':
|
||||||
|
where |= 0010;
|
||||||
|
topbits |= 02000;
|
||||||
|
break;
|
||||||
|
case 'o':
|
||||||
|
where |= 0001;
|
||||||
|
break;
|
||||||
|
case 'a':
|
||||||
|
where |= 0111;
|
||||||
|
break;
|
||||||
|
case '+':
|
||||||
|
op = CHMOD_OP_ADD;
|
||||||
|
state = CHMOD_STATE_2ND_HALF;
|
||||||
|
break;
|
||||||
|
case '-':
|
||||||
|
op = CHMOD_OP_SUB;
|
||||||
|
state = CHMOD_STATE_2ND_HALF;
|
||||||
|
break;
|
||||||
|
case '=':
|
||||||
|
op = CHMOD_OP_EQ;
|
||||||
|
state = CHMOD_STATE_2ND_HALF;
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
if (*p >= '0' && *p <= '7' && !where) {
|
||||||
|
op = CHMOD_OP_SET;
|
||||||
|
state = CHMOD_STATE_OCTAL;
|
||||||
|
where = 1;
|
||||||
|
what = *p - '0';
|
||||||
|
} else {
|
||||||
|
state = CHMOD_STATE_ERROR;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
break;
|
break;
|
||||||
begin = end + 1;
|
case CHMOD_STATE_2ND_HALF:
|
||||||
if (!*begin)
|
switch (*p) {
|
||||||
return false;
|
case 'r':
|
||||||
|
what |= 4;
|
||||||
|
break;
|
||||||
|
case 'w':
|
||||||
|
what |= 2;
|
||||||
|
break;
|
||||||
|
case 'X':
|
||||||
|
flags |= CHMOD_FLAG_X_KEEP;
|
||||||
|
/* fall through */
|
||||||
|
case 'x':
|
||||||
|
what |= 1;
|
||||||
|
break;
|
||||||
|
case 's':
|
||||||
|
if (topbits)
|
||||||
|
topoct |= topbits;
|
||||||
|
else
|
||||||
|
topoct = 04000;
|
||||||
|
break;
|
||||||
|
case 't':
|
||||||
|
topoct |= 01000;
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
state = CHMOD_STATE_ERROR;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
if (*p >= '0' && *p <= '7') {
|
||||||
|
what = what * 8 + (*p - '0');
|
||||||
|
if (what > CHMOD_BITS)
|
||||||
|
state = CHMOD_STATE_ERROR;
|
||||||
|
} else {
|
||||||
|
state = CHMOD_STATE_ERROR;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
p++;
|
||||||
}
|
}
|
||||||
*result = changed;
|
if (state == CHMOD_STATE_ERROR)
|
||||||
|
return false;
|
||||||
|
*result = (changed & (mode_t)CHMOD_BITS) | nonperm;
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|||||||
+4
-1
@@ -4,7 +4,10 @@
|
|||||||
#include <stdbool.h>
|
#include <stdbool.h>
|
||||||
#include <sys/stat.h>
|
#include <sys/stat.h>
|
||||||
|
|
||||||
/* Apply the supported rsync --chmod syntax to a permission mode. */
|
/* Apply rsync's --chmod syntax to a permission mode, including the D/F/X
|
||||||
|
* selectors and the s/t special bits. `mode` should carry the file type bits
|
||||||
|
* (S_IFDIR/S_IFREG) so D/F/X can be evaluated; the type bits are preserved in
|
||||||
|
* `result`. A spec may contain comma-separated clauses, which accumulate. */
|
||||||
bool chmod_apply(mode_t mode, const char* spec, mode_t* result);
|
bool chmod_apply(mode_t mode, const char* spec, mode_t* result);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
+5
-4
@@ -17,8 +17,9 @@
|
|||||||
#include "protocol.h"
|
#include "protocol.h"
|
||||||
#include "utils.h"
|
#include "utils.h"
|
||||||
|
|
||||||
/* Maximum individual file data size within a chunk (64 MB) */
|
/* Maximum individual file data size within a chunk (64 MB). Distinct from the
|
||||||
#define MAX_FILE_DATA_SIZE (64ULL * 1024 * 1024)
|
* receiver's whole-file MAX_FILE_DATA_SIZE (256 MB) in file_receive.c. */
|
||||||
|
#define MAX_CHUNK_FILE_DATA_SIZE (64ULL * 1024 * 1024)
|
||||||
#define MAX_FILES_PER_CHUNK 65536U
|
#define MAX_FILES_PER_CHUNK 65536U
|
||||||
|
|
||||||
/* Reserve `charge` against `session`'s connection budget. This mirrors the
|
/* Reserve `charge` against `session`'s connection budget. This mirrors the
|
||||||
@@ -382,9 +383,9 @@ Chunk* chunk_deserialize(Data* data, bool use_metadata) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Reject individual file data larger than the maximum allowed size.
|
// Reject individual file data larger than the maximum allowed size.
|
||||||
if (file_data_size > MAX_FILE_DATA_SIZE) {
|
if (file_data_size > MAX_CHUNK_FILE_DATA_SIZE) {
|
||||||
log_message(LOG_LEVEL_ERROR, "File data size %zu exceeds maximum %llu", file_data_size,
|
log_message(LOG_LEVEL_ERROR, "File data size %zu exceeds maximum %llu", file_data_size,
|
||||||
(unsigned long long)MAX_FILE_DATA_SIZE);
|
(unsigned long long)MAX_CHUNK_FILE_DATA_SIZE);
|
||||||
goto error;
|
goto error;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+396
-37
@@ -2,40 +2,205 @@
|
|||||||
#include "data.h"
|
#include "data.h"
|
||||||
#include "log.h"
|
#include "log.h"
|
||||||
#include "protocol.h"
|
#include "protocol.h"
|
||||||
|
#include "utils.h"
|
||||||
#include <limits.h>
|
#include <limits.h>
|
||||||
|
#include <lz4.h>
|
||||||
|
#include <stdatomic.h>
|
||||||
#include <stdint.h>
|
#include <stdint.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
#include <strings.h>
|
#include <strings.h>
|
||||||
#include <threads.h>
|
#include <threads.h>
|
||||||
#include <unistd.h>
|
#include <unistd.h>
|
||||||
|
#include <zlib.h>
|
||||||
#include <zstd.h>
|
#include <zstd.h>
|
||||||
|
|
||||||
#define INITIAL_DECOMPRESS_BUF_SIZE (1024 * 1024)
|
#define INITIAL_DECOMPRESS_BUF_SIZE (1024 * 1024)
|
||||||
#define MAX_DECOMPRESSED_SIZE (100ULL * 1024 * 1024) /* 100 MB hard ceiling */
|
|
||||||
|
|
||||||
static char* SKIP_COMPRESSION_EXTENSIONS[] = {".jpg", ".jpeg", ".png", ".gif", ".mp4", ".mkv",
|
/* Hard ceiling for a single decompression. The sender compresses whole files
|
||||||
".zip", ".gz", ".xz", ".zst", NULL};
|
* up to the protocol's whole-file receive bound, so the decompressor must
|
||||||
|
* accept payloads that large; referencing the protocol constant keeps the two
|
||||||
|
* bounds from drifting apart (they previously did: a 100 MB ceiling rejected
|
||||||
|
* 100-256 MB files). This remains a real bomb guard -- every allocation in the
|
||||||
|
* paths below is clamped to it -- so it must not exceed the protocol bound. */
|
||||||
|
#define MAX_DECOMPRESSED_SIZE MAX_RECEIVE_WHOLE_FILE_SIZE
|
||||||
|
|
||||||
|
/* rsync 3.4.1's built-in skip-compress suffix list (the `--skip-compress`
|
||||||
|
* defaults, in the man page's order). rsync stores it as space-separated
|
||||||
|
* "*.suffix" globs; FastSync matches the plain suffix after the final dot, so
|
||||||
|
* the leading "*." is omitted here. A user --skip-compress list replaces this
|
||||||
|
* default entirely (matching rsync). */
|
||||||
|
#define DEFAULT_SKIP_COMPRESS_SUFFIXES \
|
||||||
|
"3g2 3gp 7z aac ace apk avi bz2 deb dmg ear f4v flac flv gpg gz iso jar jpeg jpg lrz lz lz4 " \
|
||||||
|
"lzma " \
|
||||||
|
"lzo m1a m1v m2a m2ts m2v m4a m4b m4p m4r m4v mka mkv mov mp1 mp2 mp3 mp4 mpa mpeg mpg mpv mts " \
|
||||||
|
"odb odf odg odi odm odp ods odt oga ogg ogm ogv ogx opus otg oth otp ots ott oxt png qt rar " \
|
||||||
|
"rpm " \
|
||||||
|
"rz rzip spx squashfs sxc sxd sxg sxm sxw sz tbz tbz2 tgz tlz ts txz tzo vob war webm webp xz " \
|
||||||
|
"z " \
|
||||||
|
"zip zst"
|
||||||
|
|
||||||
|
/* Self-describing compressed frames: the first byte is the CompressionAlgo id.
|
||||||
|
* zlib/lz4 store the uncompressed size as a little-endian uint32 after the
|
||||||
|
* codec byte so decompression can be exactly pre-sized and bounded. */
|
||||||
|
#define LZ4_SIZE_PREFIX_LEN 4
|
||||||
|
|
||||||
|
static _Atomic int g_compression_algo = COMPRESSION_ALGO_ZSTD;
|
||||||
|
|
||||||
|
/* Case-insensitive match of a bare suffix (no leading dot) against a
|
||||||
|
* space-separated suffix list. */
|
||||||
|
static bool suffix_in_list(const char* name, const char* list) {
|
||||||
|
size_t name_len = strlen(name);
|
||||||
|
while (*list) {
|
||||||
|
while (*list == ' ')
|
||||||
|
list++;
|
||||||
|
const char* start = list;
|
||||||
|
while (*list && *list != ' ')
|
||||||
|
list++;
|
||||||
|
size_t len = (size_t)(list - start);
|
||||||
|
if (len == name_len && strncasecmp(name, start, len) == 0)
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
bool compression_should_skip_with_suffixes(const char* path, char* const* suffixes, int count) {
|
bool compression_should_skip_with_suffixes(const char* path, char* const* suffixes, int count) {
|
||||||
if (!path)
|
if (!path)
|
||||||
return false;
|
return false;
|
||||||
const char* dot = strrchr(path, '.');
|
const char* dot = strrchr(path, '.');
|
||||||
if (!dot)
|
if (!dot || dot[1] == '\0')
|
||||||
return false;
|
return false;
|
||||||
if (count < 0) {
|
const char* name = dot + 1;
|
||||||
suffixes = SKIP_COMPRESSION_EXTENSIONS;
|
/* count < 0 (the user gave no --skip-compress) selects rsync's built-in
|
||||||
count = 0;
|
* default list; a non-negative count is the user's explicit list. */
|
||||||
while (SKIP_COMPRESSION_EXTENSIONS[count])
|
if (count < 0)
|
||||||
count++;
|
return suffix_in_list(name, DEFAULT_SKIP_COMPRESS_SUFFIXES);
|
||||||
}
|
|
||||||
for (int i = 0; i < count; i++) {
|
for (int i = 0; i < count; i++) {
|
||||||
if (strcasecmp(dot, suffixes[i]) == 0)
|
const char* suffix = suffixes[i];
|
||||||
|
if (suffix[0] == '.')
|
||||||
|
suffix++;
|
||||||
|
if (strcasecmp(name, suffix) == 0)
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
int compression_algo_from_name(const char* name) {
|
||||||
|
if (!name)
|
||||||
|
return -1;
|
||||||
|
if (strcasecmp(name, "zstd") == 0)
|
||||||
|
return (int)COMPRESSION_ALGO_ZSTD;
|
||||||
|
if (strcasecmp(name, "lz4") == 0)
|
||||||
|
return (int)COMPRESSION_ALGO_LZ4;
|
||||||
|
if (strcasecmp(name, "zlib") == 0)
|
||||||
|
return (int)COMPRESSION_ALGO_ZLIB;
|
||||||
|
if (strcasecmp(name, "zlibx") == 0)
|
||||||
|
return (int)COMPRESSION_ALGO_ZLIBX;
|
||||||
|
if (strcasecmp(name, "none") == 0)
|
||||||
|
return (int)COMPRESSION_ALGO_NONE;
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
const char* compression_algo_name(CompressionAlgo algo) {
|
||||||
|
switch (algo) {
|
||||||
|
case COMPRESSION_ALGO_NONE:
|
||||||
|
return "none";
|
||||||
|
case COMPRESSION_ALGO_ZSTD:
|
||||||
|
return "zstd";
|
||||||
|
case COMPRESSION_ALGO_LZ4:
|
||||||
|
return "lz4";
|
||||||
|
case COMPRESSION_ALGO_ZLIB:
|
||||||
|
return "zlib";
|
||||||
|
case COMPRESSION_ALGO_ZLIBX:
|
||||||
|
return "zlibx";
|
||||||
|
}
|
||||||
|
return "<unknown>";
|
||||||
|
}
|
||||||
|
|
||||||
|
bool compression_algo_valid(int algo) {
|
||||||
|
return algo == (int)COMPRESSION_ALGO_NONE || algo == (int)COMPRESSION_ALGO_ZSTD ||
|
||||||
|
algo == (int)COMPRESSION_ALGO_LZ4 || algo == (int)COMPRESSION_ALGO_ZLIB ||
|
||||||
|
algo == (int)COMPRESSION_ALGO_ZLIBX;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool compression_algo_enabled(CompressionAlgo algo) {
|
||||||
|
return algo != COMPRESSION_ALGO_NONE;
|
||||||
|
}
|
||||||
|
|
||||||
|
static CompressionAlgo compiled_preference_first(void) {
|
||||||
|
/* rsync 3.4.1 default preference order; every entry is compiled in, so this
|
||||||
|
* resolves to zstd. */
|
||||||
|
static const CompressionAlgo preference[] = {
|
||||||
|
COMPRESSION_ALGO_ZSTD, COMPRESSION_ALGO_LZ4, COMPRESSION_ALGO_ZLIBX,
|
||||||
|
COMPRESSION_ALGO_ZLIB, COMPRESSION_ALGO_NONE,
|
||||||
|
};
|
||||||
|
for (size_t i = 0; i < sizeof(preference) / sizeof(preference[0]); i++) {
|
||||||
|
if (compression_algo_valid((int)preference[i]))
|
||||||
|
return preference[i];
|
||||||
|
}
|
||||||
|
return COMPRESSION_ALGO_ZSTD;
|
||||||
|
}
|
||||||
|
|
||||||
|
int compression_choice_resolve(void) {
|
||||||
|
bool specified = false;
|
||||||
|
int env = env_choice_first("RSYNC_COMPRESS_LIST", compression_algo_from_name, &specified);
|
||||||
|
if (specified)
|
||||||
|
return env; /* -1 = the list named no supported codec */
|
||||||
|
return (int)compiled_preference_first();
|
||||||
|
}
|
||||||
|
|
||||||
|
CompressionAlgo compression_negotiate_default(void) {
|
||||||
|
int resolved = compression_choice_resolve();
|
||||||
|
return resolved >= 0 ? (CompressionAlgo)resolved : compiled_preference_first();
|
||||||
|
}
|
||||||
|
|
||||||
|
int compression_default_level(CompressionAlgo algo) {
|
||||||
|
switch (algo) {
|
||||||
|
case COMPRESSION_ALGO_ZSTD:
|
||||||
|
return ZSTD_CLEVEL_DEFAULT;
|
||||||
|
case COMPRESSION_ALGO_ZLIB:
|
||||||
|
case COMPRESSION_ALGO_ZLIBX:
|
||||||
|
return 6; /* rsync resolves zlib's Z_DEFAULT_COMPRESSION (-1) to 6 */
|
||||||
|
case COMPRESSION_ALGO_LZ4:
|
||||||
|
return 1; /* rsync lz4 level is 0/ignored; positive keeps the gate on */
|
||||||
|
case COMPRESSION_ALGO_NONE:
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
int compression_clamp_level(CompressionAlgo algo, int level) {
|
||||||
|
switch (algo) {
|
||||||
|
case COMPRESSION_ALGO_ZSTD:
|
||||||
|
if (level < 1)
|
||||||
|
return 1;
|
||||||
|
if (level > 22)
|
||||||
|
return 22;
|
||||||
|
return level;
|
||||||
|
case COMPRESSION_ALGO_ZLIB:
|
||||||
|
case COMPRESSION_ALGO_ZLIBX:
|
||||||
|
if (level < 1)
|
||||||
|
return 1;
|
||||||
|
if (level > 9)
|
||||||
|
return 9;
|
||||||
|
return level;
|
||||||
|
case COMPRESSION_ALGO_LZ4:
|
||||||
|
return 1; /* ignored by lz4_compress; keeps the "compress" gate on */
|
||||||
|
case COMPRESSION_ALGO_NONE:
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
return level;
|
||||||
|
}
|
||||||
|
|
||||||
|
void compression_set_algo(CompressionAlgo algo) {
|
||||||
|
if (compression_algo_valid((int)algo))
|
||||||
|
atomic_store(&g_compression_algo, (int)algo);
|
||||||
|
}
|
||||||
|
|
||||||
|
CompressionAlgo compression_get_algo(void) {
|
||||||
|
return (CompressionAlgo)atomic_load(&g_compression_algo);
|
||||||
|
}
|
||||||
|
|
||||||
/* Per-thread cache of zstd contexts plus the grow-only compression scratch
|
/* Per-thread cache of zstd contexts plus the grow-only compression scratch
|
||||||
* buffer. zstd contexts are stateful and not safe to share between threads,
|
* buffer. zstd contexts are stateful and not safe to share between threads,
|
||||||
* so each thread keeps its own (see compression_get_thread_ctx). The cache is
|
* so each thread keeps its own (see compression_get_thread_ctx). The cache is
|
||||||
@@ -126,17 +291,25 @@ static void compression_ctx_put(CompressionThreadCtx* ctx) {
|
|||||||
compression_ctx_free(ctx);
|
compression_ctx_free(ctx);
|
||||||
}
|
}
|
||||||
|
|
||||||
Data* data_compress(Data* data_to_compress, int compression_level) {
|
/* Build a frame consisting of a copy of `src` prefixed by `codec`. */
|
||||||
return data_compress_with_threads(data_to_compress, compression_level, 0);
|
static Data* frame_with_codec(const void* src, size_t size, CompressionAlgo codec) {
|
||||||
|
if (size > SIZE_MAX - 1)
|
||||||
|
return NULL;
|
||||||
|
Data* out = data_create_empty(size + 1);
|
||||||
|
if (!out)
|
||||||
|
return NULL;
|
||||||
|
((uint8_t*)out->data)[0] = (uint8_t)codec;
|
||||||
|
if (size > 0)
|
||||||
|
memcpy((uint8_t*)out->data + 1, src, size);
|
||||||
|
out->size = size + 1;
|
||||||
|
return out;
|
||||||
}
|
}
|
||||||
|
|
||||||
Data* data_compress_with_threads(Data* data_to_compress, int compression_level,
|
static Data* zstd_compress(Data* in, int compression_level, int compression_threads) {
|
||||||
int compression_threads) {
|
size_t dst_size = ZSTD_compressBound(in->size);
|
||||||
if (!data_to_compress || (!data_to_compress->data && data_to_compress->size != 0) ||
|
if (dst_size > SIZE_MAX - 1)
|
||||||
compression_threads < 0 || compression_threads > COMPRESSION_MAX_THREADS)
|
|
||||||
return NULL;
|
return NULL;
|
||||||
log_message(LOG_LEVEL_DEBUG, "Starting to compress data");
|
dst_size += 1; /* codec prefix */
|
||||||
size_t dst_size = ZSTD_compressBound(data_to_compress->size);
|
|
||||||
|
|
||||||
CompressionThreadCtx* ctx = compression_get_thread_ctx();
|
CompressionThreadCtx* ctx = compression_get_thread_ctx();
|
||||||
if (ctx == NULL) {
|
if (ctx == NULL) {
|
||||||
@@ -187,7 +360,7 @@ Data* data_compress_with_threads(Data* data_to_compress, int compression_level,
|
|||||||
|
|
||||||
if (available_threads > 0) {
|
if (available_threads > 0) {
|
||||||
/* Streaming compression needs the source size before threaded mode can end a frame. */
|
/* Streaming compression needs the source size before threaded mode can end a frame. */
|
||||||
size_t zret = ZSTD_CCtx_setPledgedSrcSize(ctx->cctx, data_to_compress->size);
|
size_t zret = ZSTD_CCtx_setPledgedSrcSize(ctx->cctx, in->size);
|
||||||
if (ZSTD_isError(zret)) {
|
if (ZSTD_isError(zret)) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Failed to set compression source size: %s",
|
log_message(LOG_LEVEL_ERROR, "Failed to set compression source size: %s",
|
||||||
ZSTD_getErrorName(zret));
|
ZSTD_getErrorName(zret));
|
||||||
@@ -205,8 +378,8 @@ Data* data_compress_with_threads(Data* data_to_compress, int compression_level,
|
|||||||
ctx->out_cap = dst_size;
|
ctx->out_cap = dst_size;
|
||||||
}
|
}
|
||||||
|
|
||||||
ZSTD_inBuffer input = {data_to_compress->data, data_to_compress->size, 0};
|
ZSTD_inBuffer input = {in->data, in->size, 0};
|
||||||
ZSTD_outBuffer output = {ctx->out_buf, dst_size, 0};
|
ZSTD_outBuffer output = {(uint8_t*)ctx->out_buf + 1, dst_size - 1, 0};
|
||||||
|
|
||||||
size_t ret;
|
size_t ret;
|
||||||
do {
|
do {
|
||||||
@@ -219,30 +392,192 @@ Data* data_compress_with_threads(Data* data_to_compress, int compression_level,
|
|||||||
|
|
||||||
/* Hand off an exactly-sized copy; the scratch buffer stays cached so the next
|
/* Hand off an exactly-sized copy; the scratch buffer stays cached so the next
|
||||||
* call does not reallocate a ZSTD_compressBound-sized block. */
|
* call does not reallocate a ZSTD_compressBound-sized block. */
|
||||||
compressed_data = data_create_empty(output.pos);
|
compressed_data = data_create_empty(output.pos + 1);
|
||||||
if (compressed_data == NULL) {
|
if (compressed_data == NULL) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Failed to allocate compressed data");
|
log_message(LOG_LEVEL_ERROR, "Failed to allocate compressed data");
|
||||||
goto cleanup;
|
goto cleanup;
|
||||||
}
|
}
|
||||||
|
((uint8_t*)compressed_data->data)[0] = (uint8_t)COMPRESSION_ALGO_ZSTD;
|
||||||
if (output.pos > 0)
|
if (output.pos > 0)
|
||||||
memcpy(compressed_data->data, ctx->out_buf, output.pos);
|
memcpy((uint8_t*)compressed_data->data + 1, (uint8_t*)ctx->out_buf + 1, output.pos);
|
||||||
compressed_data->size = output.pos;
|
compressed_data->size = output.pos + 1;
|
||||||
|
|
||||||
log_debug_message(LOG_DEBUG_UTIL, "Data succesfully compressed from %zu to %zu",
|
log_debug_message(LOG_DEBUG_UTIL, "Data succesfully compressed from %zu to %zu", in->size,
|
||||||
data_to_compress->size, compressed_data->size);
|
compressed_data->size);
|
||||||
|
|
||||||
cleanup:
|
cleanup:
|
||||||
compression_ctx_put(ctx);
|
compression_ctx_put(ctx);
|
||||||
return compressed_data;
|
return compressed_data;
|
||||||
}
|
}
|
||||||
|
|
||||||
Data* data_decompress_limited(Data* compressed_data, size_t maximum_size) {
|
static Data* lz4_compress(Data* in) {
|
||||||
if (!compressed_data || (!compressed_data->data && compressed_data->size != 0) ||
|
int bound = LZ4_compressBound((int)in->size);
|
||||||
maximum_size == 0)
|
if (bound < 0 || in->size > (size_t)INT_MAX)
|
||||||
return NULL;
|
return NULL;
|
||||||
|
Data* out = data_create_empty((size_t)bound + 1 + LZ4_SIZE_PREFIX_LEN);
|
||||||
|
if (!out)
|
||||||
|
return NULL;
|
||||||
|
uint32_t raw_size = (uint32_t)in->size;
|
||||||
|
uint8_t* p = (uint8_t*)out->data;
|
||||||
|
p[0] = (uint8_t)COMPRESSION_ALGO_LZ4;
|
||||||
|
for (int i = 0; i < LZ4_SIZE_PREFIX_LEN; i++)
|
||||||
|
p[1 + i] = (uint8_t)((raw_size >> (8 * i)) & 0xff);
|
||||||
|
int written = 0;
|
||||||
|
if (in->size > 0) {
|
||||||
|
written = LZ4_compress_default((const char*)in->data, (char*)p + 1 + LZ4_SIZE_PREFIX_LEN,
|
||||||
|
(int)in->size, bound);
|
||||||
|
if (written <= 0) {
|
||||||
|
data_destroy(out);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out->size = (size_t)written + 1 + LZ4_SIZE_PREFIX_LEN;
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
static Data* zlib_compress(Data* in, CompressionAlgo algo, int compression_level) {
|
||||||
|
int level = compression_level;
|
||||||
|
if (level < 1)
|
||||||
|
level = Z_DEFAULT_COMPRESSION;
|
||||||
|
if (level > 9)
|
||||||
|
level = 9;
|
||||||
|
uLong bound = compressBound((uLong)in->size);
|
||||||
|
if (in->size > (size_t)ULONG_MAX)
|
||||||
|
return NULL;
|
||||||
|
Data* out = data_create_empty((size_t)bound + 1 + LZ4_SIZE_PREFIX_LEN);
|
||||||
|
if (!out)
|
||||||
|
return NULL;
|
||||||
|
uint32_t raw_size = (uint32_t)in->size;
|
||||||
|
uint8_t* p = (uint8_t*)out->data;
|
||||||
|
p[0] = (uint8_t)algo;
|
||||||
|
for (int i = 0; i < LZ4_SIZE_PREFIX_LEN; i++)
|
||||||
|
p[1 + i] = (uint8_t)((raw_size >> (8 * i)) & 0xff);
|
||||||
|
uLongf dest_len = bound;
|
||||||
|
int rc = compress2(p + 1 + LZ4_SIZE_PREFIX_LEN, &dest_len, (const Bytef*)in->data,
|
||||||
|
(uLong)in->size, level);
|
||||||
|
if (rc != Z_OK) {
|
||||||
|
data_destroy(out);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
out->size = (size_t)dest_len + 1 + LZ4_SIZE_PREFIX_LEN;
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
Data* data_compress_codec(Data* data_to_compress, CompressionAlgo algo, int compression_level,
|
||||||
|
int compression_threads) {
|
||||||
|
if (!data_to_compress || (!data_to_compress->data && data_to_compress->size != 0) ||
|
||||||
|
compression_threads < 0 || compression_threads > COMPRESSION_MAX_THREADS)
|
||||||
|
return NULL;
|
||||||
|
if (!compression_algo_valid((int)algo))
|
||||||
|
return NULL;
|
||||||
|
log_message(LOG_LEVEL_DEBUG, "Starting to compress data");
|
||||||
|
switch (algo) {
|
||||||
|
case COMPRESSION_ALGO_NONE:
|
||||||
|
return frame_with_codec(data_to_compress->data, data_to_compress->size, COMPRESSION_ALGO_NONE);
|
||||||
|
case COMPRESSION_ALGO_ZSTD:
|
||||||
|
return zstd_compress(data_to_compress, compression_level, compression_threads);
|
||||||
|
case COMPRESSION_ALGO_LZ4:
|
||||||
|
return lz4_compress(data_to_compress);
|
||||||
|
case COMPRESSION_ALGO_ZLIB:
|
||||||
|
case COMPRESSION_ALGO_ZLIBX:
|
||||||
|
return zlib_compress(data_to_compress, algo, compression_level);
|
||||||
|
}
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
Data* data_compress_with_threads(Data* data_to_compress, int compression_level,
|
||||||
|
int compression_threads) {
|
||||||
|
return data_compress_codec(data_to_compress, compression_get_algo(), compression_level,
|
||||||
|
compression_threads);
|
||||||
|
}
|
||||||
|
|
||||||
|
Data* data_compress(Data* data_to_compress, int compression_level) {
|
||||||
|
return data_compress_codec(data_to_compress, compression_get_algo(), compression_level, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
static Data* decompress_none(const Data* compressed_data, size_t maximum_size) {
|
||||||
|
size_t size = compressed_data->size - 1;
|
||||||
|
if (size > maximum_size)
|
||||||
|
return NULL;
|
||||||
|
Data* out = data_create_empty(size);
|
||||||
|
if (!out)
|
||||||
|
return NULL;
|
||||||
|
if (size > 0)
|
||||||
|
memcpy(out->data, (const uint8_t*)compressed_data->data + 1, size);
|
||||||
|
out->size = size;
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Read the 4-byte little-endian raw size stored after the codec byte. */
|
||||||
|
static bool read_raw_size(const Data* in, uint32_t* raw_size) {
|
||||||
|
if (in->size < 1 + LZ4_SIZE_PREFIX_LEN)
|
||||||
|
return false;
|
||||||
|
const uint8_t* p = (const uint8_t*)in->data;
|
||||||
|
uint32_t v = 0;
|
||||||
|
for (int i = 0; i < LZ4_SIZE_PREFIX_LEN; i++)
|
||||||
|
v |= (uint32_t)p[1 + i] << (8 * i);
|
||||||
|
*raw_size = v;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
static Data* lz4_decompress(Data* compressed_data, size_t maximum_size, size_t hard_limit) {
|
||||||
|
uint32_t raw_size = 0;
|
||||||
|
if (!read_raw_size(compressed_data, &raw_size))
|
||||||
|
return NULL;
|
||||||
|
if (raw_size > hard_limit || raw_size > maximum_size)
|
||||||
|
return NULL;
|
||||||
|
size_t comp_size = compressed_data->size - 1 - LZ4_SIZE_PREFIX_LEN;
|
||||||
|
Data* out = data_create_empty(raw_size);
|
||||||
|
if (!out)
|
||||||
|
return NULL;
|
||||||
|
if (raw_size == 0) {
|
||||||
|
out->size = 0;
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
int rc = LZ4_decompress_safe((const char*)compressed_data->data + 1 + LZ4_SIZE_PREFIX_LEN,
|
||||||
|
(char*)out->data, (int)comp_size, (int)raw_size);
|
||||||
|
if (rc < 0 || (uint32_t)rc != raw_size) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "LZ4 decompression failed");
|
||||||
|
data_destroy(out);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
out->size = raw_size;
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
static Data* zlib_decompress(Data* compressed_data, size_t maximum_size, size_t hard_limit) {
|
||||||
|
uint32_t raw_size = 0;
|
||||||
|
if (!read_raw_size(compressed_data, &raw_size))
|
||||||
|
return NULL;
|
||||||
|
if (raw_size > hard_limit || raw_size > maximum_size)
|
||||||
|
return NULL;
|
||||||
|
size_t comp_size = compressed_data->size - 1 - LZ4_SIZE_PREFIX_LEN;
|
||||||
|
Data* out = data_create_empty(raw_size);
|
||||||
|
if (!out)
|
||||||
|
return NULL;
|
||||||
|
if (raw_size == 0) {
|
||||||
|
out->size = 0;
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
uLongf dest_len = raw_size;
|
||||||
|
int rc =
|
||||||
|
uncompress((Bytef*)out->data, &dest_len,
|
||||||
|
(const Bytef*)compressed_data->data + 1 + LZ4_SIZE_PREFIX_LEN, (uLong)comp_size);
|
||||||
|
if (rc != Z_OK || dest_len != raw_size) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "zlib decompression failed");
|
||||||
|
data_destroy(out);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
out->size = raw_size;
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
static Data* zstd_decompress(Data* compressed_data, size_t maximum_size) {
|
||||||
|
/* The zstd frame starts after the codec byte. */
|
||||||
|
const void* frame = (const uint8_t*)compressed_data->data + 1;
|
||||||
|
size_t frame_size = compressed_data->size - 1;
|
||||||
log_debug_message(LOG_DEBUG_UTIL, "Start to decompress data");
|
log_debug_message(LOG_DEBUG_UTIL, "Start to decompress data");
|
||||||
unsigned long long dst_size =
|
unsigned long long dst_size = ZSTD_getFrameContentSize(frame, frame_size);
|
||||||
ZSTD_getFrameContentSize(compressed_data->data, compressed_data->size);
|
|
||||||
/* ZSTD_isError() is also true for ZSTD_CONTENTSIZE_ERROR and
|
/* ZSTD_isError() is also true for ZSTD_CONTENTSIZE_ERROR and
|
||||||
* ZSTD_CONTENTSIZE_UNKNOWN (both are encoded near (size_t)-1), so test the
|
* ZSTD_CONTENTSIZE_UNKNOWN (both are encoded near (size_t)-1), so test the
|
||||||
* sentinels explicitly instead of blanket-rejecting every error-ish value:
|
* sentinels explicitly instead of blanket-rejecting every error-ish value:
|
||||||
@@ -256,9 +591,9 @@ Data* data_decompress_limited(Data* compressed_data, size_t maximum_size) {
|
|||||||
// ZSTD_CONTENTSIZE_UNKNOWN (~2^64) can cause massive allocation;
|
// ZSTD_CONTENTSIZE_UNKNOWN (~2^64) can cause massive allocation;
|
||||||
// fall back to a conservative estimate (3x compressed size) when unknown.
|
// fall back to a conservative estimate (3x compressed size) when unknown.
|
||||||
if (dst_size == ZSTD_CONTENTSIZE_UNKNOWN) {
|
if (dst_size == ZSTD_CONTENTSIZE_UNKNOWN) {
|
||||||
if (compressed_data->size > ULLONG_MAX / 3)
|
if (frame_size > ULLONG_MAX / 3)
|
||||||
return NULL;
|
return NULL;
|
||||||
dst_size = compressed_data->size * 3;
|
dst_size = frame_size * 3;
|
||||||
if (dst_size < INITIAL_DECOMPRESS_BUF_SIZE)
|
if (dst_size < INITIAL_DECOMPRESS_BUF_SIZE)
|
||||||
dst_size = INITIAL_DECOMPRESS_BUF_SIZE;
|
dst_size = INITIAL_DECOMPRESS_BUF_SIZE;
|
||||||
}
|
}
|
||||||
@@ -295,7 +630,7 @@ Data* data_decompress_limited(Data* compressed_data, size_t maximum_size) {
|
|||||||
goto cleanup;
|
goto cleanup;
|
||||||
}
|
}
|
||||||
|
|
||||||
ZSTD_inBuffer input = {compressed_data->data, compressed_data->size, 0};
|
ZSTD_inBuffer input = {frame, frame_size, 0};
|
||||||
ZSTD_outBuffer output = {uncompressed_data->data, buf_size, 0};
|
ZSTD_outBuffer output = {uncompressed_data->data, buf_size, 0};
|
||||||
|
|
||||||
size_t ret;
|
size_t ret;
|
||||||
@@ -309,8 +644,7 @@ Data* data_decompress_limited(Data* compressed_data, size_t maximum_size) {
|
|||||||
}
|
}
|
||||||
if (ret > 0 && output.pos == output.size) {
|
if (ret > 0 && output.pos == output.size) {
|
||||||
if (buf_size >= hard_limit || buf_size > SIZE_MAX / 2) {
|
if (buf_size >= hard_limit || buf_size > SIZE_MAX / 2) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Decompressed data exceeds %llu bytes",
|
log_message(LOG_LEVEL_ERROR, "Decompressed data exceeds %llu bytes", hard_limit);
|
||||||
(unsigned long long)MAX_DECOMPRESSED_SIZE);
|
|
||||||
data_destroy(uncompressed_data);
|
data_destroy(uncompressed_data);
|
||||||
uncompressed_data = NULL;
|
uncompressed_data = NULL;
|
||||||
goto cleanup;
|
goto cleanup;
|
||||||
@@ -354,6 +688,31 @@ cleanup:
|
|||||||
return uncompressed_data;
|
return uncompressed_data;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Data* data_decompress_limited(Data* compressed_data, size_t maximum_size) {
|
||||||
|
if (!compressed_data || (!compressed_data->data && compressed_data->size != 0) ||
|
||||||
|
maximum_size == 0)
|
||||||
|
return NULL;
|
||||||
|
if (compressed_data->size < 1)
|
||||||
|
return NULL;
|
||||||
|
unsigned long long hard_limit =
|
||||||
|
maximum_size < MAX_DECOMPRESSED_SIZE ? maximum_size : MAX_DECOMPRESSED_SIZE;
|
||||||
|
uint8_t codec = ((const uint8_t*)compressed_data->data)[0];
|
||||||
|
if (!compression_algo_valid(codec))
|
||||||
|
return NULL;
|
||||||
|
switch ((CompressionAlgo)codec) {
|
||||||
|
case COMPRESSION_ALGO_NONE:
|
||||||
|
return decompress_none(compressed_data, (size_t)hard_limit);
|
||||||
|
case COMPRESSION_ALGO_ZSTD:
|
||||||
|
return zstd_decompress(compressed_data, (size_t)hard_limit);
|
||||||
|
case COMPRESSION_ALGO_LZ4:
|
||||||
|
return lz4_decompress(compressed_data, maximum_size, (size_t)hard_limit);
|
||||||
|
case COMPRESSION_ALGO_ZLIB:
|
||||||
|
case COMPRESSION_ALGO_ZLIBX:
|
||||||
|
return zlib_decompress(compressed_data, maximum_size, (size_t)hard_limit);
|
||||||
|
}
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
Data* data_decompress(Data* compressed_data) {
|
Data* data_decompress(Data* compressed_data) {
|
||||||
return data_decompress_limited(compressed_data, MAX_DECOMPRESSED_SIZE);
|
return data_decompress_limited(compressed_data, MAX_DECOMPRESSED_SIZE);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,11 +6,79 @@
|
|||||||
|
|
||||||
#define COMPRESSION_MAX_THREADS 64
|
#define COMPRESSION_MAX_THREADS 64
|
||||||
|
|
||||||
|
/* Compression algorithms selectable with --compress-choice / -z. The ids are
|
||||||
|
* the values placed on the wire (Config->compression_algo), so they must be
|
||||||
|
* kept stable. NONE is "no compression"; ZSTD is the historical FastSync
|
||||||
|
* default and the negotiated "auto" choice. ZLIBX is rsync's zlib-without-
|
||||||
|
* matched-data variant: FastSync compresses only the delta/token bytes (it does
|
||||||
|
* not put matched file data in the compression stream), so its zlib codec is
|
||||||
|
* already the "x" form and zlib/zlibx share the same implementation, recorded
|
||||||
|
* under distinct ids. */
|
||||||
|
typedef enum {
|
||||||
|
COMPRESSION_ALGO_NONE = 0,
|
||||||
|
COMPRESSION_ALGO_ZSTD = 1,
|
||||||
|
COMPRESSION_ALGO_LZ4 = 2,
|
||||||
|
COMPRESSION_ALGO_ZLIB = 3,
|
||||||
|
COMPRESSION_ALGO_ZLIBX = 4
|
||||||
|
} CompressionAlgo;
|
||||||
|
|
||||||
|
/* Resolve a --compress-choice string (case-insensitive) to an algorithm id.
|
||||||
|
* Accepts "zstd", "lz4", "zlib", "zlibx", "none". "auto" is not an algorithm
|
||||||
|
* here; the caller resolves it to the negotiated default. Returns -1 for any
|
||||||
|
* unrecognized name. */
|
||||||
|
int compression_algo_from_name(const char* name);
|
||||||
|
const char* compression_algo_name(CompressionAlgo algo);
|
||||||
|
bool compression_algo_valid(int algo);
|
||||||
|
|
||||||
|
/* Pick the first algorithm from FastSync's compiled-in preference list
|
||||||
|
* (rsync 3.4.1's `--version` order: zstd lz4 zlibx zlib none). Resolves
|
||||||
|
* "auto". */
|
||||||
|
CompressionAlgo compression_negotiate_default(void);
|
||||||
|
|
||||||
|
/* Resolve "auto" the way rsync does: the first supported name in
|
||||||
|
* RSYNC_COMPRESS_LIST (whitespace-separated, client half ends at '&'), then the
|
||||||
|
* compiled-in preference order when the variable is unset/blank. Returns -1
|
||||||
|
* when the variable is set but names no supported codec (rsync's failed
|
||||||
|
* negotiation), otherwise a valid CompressionAlgo id. */
|
||||||
|
int compression_choice_resolve(void);
|
||||||
|
|
||||||
|
/* rsync 3.4.1's per-codec default level, applied when the user did not pass
|
||||||
|
* --compress-level/--zl. zstd uses ZSTD_CLEVEL_DEFAULT (3) and zlib/zlibx the
|
||||||
|
* resolved Z_DEFAULT_COMPRESSION (6). lz4 has no tunable level in rsync
|
||||||
|
* (always the default acceleration); FastSync returns a positive placeholder so
|
||||||
|
* its "level > 0" compression gate stays engaged, and lz4_compress ignores the
|
||||||
|
* value, so the output is identical to rsync's. none is 0. */
|
||||||
|
int compression_default_level(CompressionAlgo algo);
|
||||||
|
|
||||||
|
/* Clamp an explicit --compress-level to the codec's accepted range the way
|
||||||
|
* rsync's init_compression_level() does: zstd 1..22, zlib/zlibx 1..9, lz4
|
||||||
|
* ignored (fixed positive placeholder), none 0. */
|
||||||
|
int compression_clamp_level(CompressionAlgo algo, int level);
|
||||||
|
|
||||||
|
/* True when the algorithm actually compresses (i.e. is not NONE). */
|
||||||
|
bool compression_algo_enabled(CompressionAlgo algo);
|
||||||
|
|
||||||
|
/* Select the process-wide codec used by the legacy wrappers below. Each
|
||||||
|
* process serves exactly one transfer config (the server forks per connection,
|
||||||
|
* the client configures itself before spawning transfer threads), so a
|
||||||
|
* process-global default is sufficient and constant for the lifetime of a
|
||||||
|
* transfer. Defaults to ZSTD when never set. Thread-safe. */
|
||||||
|
void compression_set_algo(CompressionAlgo algo);
|
||||||
|
CompressionAlgo compression_get_algo(void);
|
||||||
|
|
||||||
|
/* Codec-aware primitives. The compressed buffer is self-describing: its first
|
||||||
|
* byte is the CompressionAlgo id, so decompression never needs the codec passed
|
||||||
|
* separately (this keeps every existing Decompress call site source-compatible).
|
||||||
|
* `data_compress_codec` returns NULL on invalid input or an unsupported codec. */
|
||||||
|
Data* data_compress_codec(Data* data_to_compress, CompressionAlgo algo, int compression_level,
|
||||||
|
int compression_threads);
|
||||||
|
Data* data_decompress_limited(Data* compressed_data, size_t maximum_size);
|
||||||
|
|
||||||
|
/* Legacy zstd-default wrappers retained for existing callers/tests. */
|
||||||
Data* data_compress(Data* data_to_compress, int compression_level);
|
Data* data_compress(Data* data_to_compress, int compression_level);
|
||||||
Data* data_compress_with_threads(Data* data_to_compress, int compression_level,
|
Data* data_compress_with_threads(Data* data_to_compress, int compression_level,
|
||||||
int compression_threads);
|
int compression_threads);
|
||||||
Data* data_decompress(Data* compressed_data);
|
Data* data_decompress(Data* compressed_data);
|
||||||
Data* data_decompress_limited(Data* compressed_data, size_t maximum_size);
|
|
||||||
bool compression_should_skip_with_suffixes(const char* path, char* const* suffixes, int count);
|
bool compression_should_skip_with_suffixes(const char* path, char* const* suffixes, int count);
|
||||||
|
|
||||||
/* Release the calling thread's cached zstd contexts (compressor, decompressor
|
/* Release the calling thread's cached zstd contexts (compressor, decompressor
|
||||||
|
|||||||
+335
-54
@@ -14,12 +14,15 @@
|
|||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
|
#include <strings.h>
|
||||||
#include <limits.h>
|
#include <limits.h>
|
||||||
#include <errno.h>
|
#include <errno.h>
|
||||||
|
|
||||||
static void config_set_defaults(Config* config) {
|
static void config_set_defaults(Config* config) {
|
||||||
config->scanner_threads = 0;
|
config->scanner_threads = 0;
|
||||||
config->metadata_explicitly_disabled = false;
|
config->cli.preserve_perms_explicit_off = false;
|
||||||
|
config->cli.preserve_times_explicit_off = false;
|
||||||
|
config->cli.metadata_explicitly_disabled = false;
|
||||||
config->show_progress = false;
|
config->show_progress = false;
|
||||||
config->compression_threads = 0;
|
config->compression_threads = 0;
|
||||||
config->ssh_port = 22;
|
config->ssh_port = 22;
|
||||||
@@ -41,14 +44,16 @@ static void config_set_defaults(Config* config) {
|
|||||||
config->tls_ca = NULL;
|
config->tls_ca = NULL;
|
||||||
config->server_host = str_dup("127.0.0.1");
|
config->server_host = str_dup("127.0.0.1");
|
||||||
config->server_port = 8080;
|
config->server_port = 8080;
|
||||||
config->server_port_set = false;
|
config->cli.server_port_set = false;
|
||||||
config->server_host_set = false;
|
config->cli.server_host_set = false;
|
||||||
/* 0 means "--timeout not given": the transport keeps its own built-in 30 s
|
/* rsync defaults: --timeout=0 (I/O timeouts disabled) and --contimeout=60.
|
||||||
* socket timeout (tcp_set_timeouts ignores non-positive values) and the
|
* A value of 0 disables the client's own deadline on both the socket layer
|
||||||
* protocol layer keeps its built-in 60 s per-message deadline. A positive
|
* (tcp_set_timeouts) and the protocol layer
|
||||||
* value overrides BOTH (see protocol_session_set_io_timeout). */
|
* (protocol_session_set_io_timeout); a positive value sets it. A server
|
||||||
|
* session floors the deadline at SERVER_IO_TIMEOUT_SEC so 0 can never hold a
|
||||||
|
* connection open forever. */
|
||||||
config->timeout = 0;
|
config->timeout = 0;
|
||||||
config->contimeout = 10;
|
config->contimeout = 60;
|
||||||
config->quiet = false;
|
config->quiet = false;
|
||||||
config->stats = false;
|
config->stats = false;
|
||||||
config->max_depth = 0;
|
config->max_depth = 0;
|
||||||
@@ -63,19 +68,23 @@ static void config_set_defaults(Config* config) {
|
|||||||
config->human_readable = false;
|
config->human_readable = false;
|
||||||
config->ignore_errors = false;
|
config->ignore_errors = false;
|
||||||
config->ignore_missing_args = false;
|
config->ignore_missing_args = false;
|
||||||
|
config->cli.checksum_transfer_algo = CHECKSUM_ALGO_DEFAULT;
|
||||||
|
config->cli.cli_exit_code = 0;
|
||||||
|
config->cli.compression_level_set = false;
|
||||||
|
config->cli.checksum_choice_set = false;
|
||||||
config->filters = NULL;
|
config->filters = NULL;
|
||||||
config->files_from = NULL;
|
config->files_from = NULL;
|
||||||
config->files_from_set = NULL;
|
config->files_from_set = NULL;
|
||||||
config->from0 = false;
|
config->from0 = false;
|
||||||
config->cvs_exclude = false;
|
config->cvs_exclude = false;
|
||||||
config->per_dir_filter = false;
|
config->per_dir_filter = false;
|
||||||
config->one_file_system = false;
|
config->per_dir_filter_count = 0;
|
||||||
|
config->one_file_system = 0;
|
||||||
config->no_implied_dirs = false;
|
config->no_implied_dirs = false;
|
||||||
config->dirs = false;
|
config->dirs = false;
|
||||||
config->rsh_command = NULL;
|
config->rsh_command = NULL;
|
||||||
config->blocking_io = false;
|
config->blocking_io = false;
|
||||||
config->outbuf = OUTBUF_BLOCK;
|
config->outbuf = OUTBUF_BLOCK;
|
||||||
config->old_args = false;
|
|
||||||
config->remote_options = NULL;
|
config->remote_options = NULL;
|
||||||
config->remote_option_count = 0;
|
config->remote_option_count = 0;
|
||||||
config->address = NULL;
|
config->address = NULL;
|
||||||
@@ -91,7 +100,7 @@ static void config_set_defaults(Config* config) {
|
|||||||
config->trust_sender = false;
|
config->trust_sender = false;
|
||||||
config->stop_after_mins = 0;
|
config->stop_after_mins = 0;
|
||||||
config->stop_at = 0;
|
config->stop_at = 0;
|
||||||
config->stop_at_set = false;
|
config->cli.stop_at_set = false;
|
||||||
config->write_batch = NULL;
|
config->write_batch = NULL;
|
||||||
config->only_write_batch = NULL;
|
config->only_write_batch = NULL;
|
||||||
config->read_batch = NULL;
|
config->read_batch = NULL;
|
||||||
@@ -192,10 +201,14 @@ static bool validate_received_config(const Config* config) {
|
|||||||
valid_wire_bool(config->partial) && valid_wire_bool(config->delete_before) &&
|
valid_wire_bool(config->partial) && valid_wire_bool(config->delete_before) &&
|
||||||
valid_wire_bool(config->checksum) && valid_wire_bool(config->eight_bit_output) &&
|
valid_wire_bool(config->checksum) && valid_wire_bool(config->eight_bit_output) &&
|
||||||
valid_wire_bool(config->dry_run) && checksum_algo_valid(config->checksum_algo) &&
|
valid_wire_bool(config->dry_run) && checksum_algo_valid(config->checksum_algo) &&
|
||||||
identity_wire_valid(config) && valid_wire_bool(config->preserve_atimes) &&
|
compression_algo_valid(config->compression_algo) && identity_wire_valid(config) &&
|
||||||
valid_wire_bool(config->preserve_crtimes) && valid_wire_bool(config->omit_dir_times) &&
|
valid_wire_bool(config->preserve_atimes) && valid_wire_bool(config->preserve_crtimes) &&
|
||||||
valid_wire_bool(config->omit_link_times) && valid_wire_bool(config->munge_links) &&
|
valid_wire_bool(config->omit_dir_times) && valid_wire_bool(config->omit_link_times) &&
|
||||||
valid_wire_bool(config->keep_dirlinks) && valid_wire_bool(config->fake_super) &&
|
valid_wire_bool(config->preserve_perms) && valid_wire_bool(config->preserve_times) &&
|
||||||
|
valid_wire_bool(config->preserve_owner) && valid_wire_bool(config->preserve_group) &&
|
||||||
|
valid_wire_bool(config->munge_links) && valid_wire_bool(config->keep_dirlinks) &&
|
||||||
|
valid_wire_bool(config->fake_super) && valid_wire_bool(config->report_dest_info) &&
|
||||||
|
valid_wire_bool(config->report_stats) && valid_wire_bool(config->report_deletes) &&
|
||||||
(!config->copy_as_set || (config->copy_as_uid >= 0 && config->copy_as_gid >= 0)) &&
|
(!config->copy_as_set || (config->copy_as_uid >= 0 && config->copy_as_gid >= 0)) &&
|
||||||
(!config->use_compression ||
|
(!config->use_compression ||
|
||||||
(config->compression_level >= 1 && config->compression_level <= 22)) &&
|
(config->compression_level >= 1 && config->compression_level <= 22)) &&
|
||||||
@@ -203,8 +216,9 @@ static bool validate_received_config(const Config* config) {
|
|||||||
config->delta_block_size >= DELTA_BLOCK_SIZE_MIN &&
|
config->delta_block_size >= DELTA_BLOCK_SIZE_MIN &&
|
||||||
config->delta_block_size <= DELTA_BLOCK_SIZE_MAX &&
|
config->delta_block_size <= DELTA_BLOCK_SIZE_MAX &&
|
||||||
config->delta_max_file_size <= DELTA_MAX_FILE_SIZE && config->modify_window >= 0 &&
|
config->delta_max_file_size <= DELTA_MAX_FILE_SIZE && config->modify_window >= 0 &&
|
||||||
config->max_delete >= -1 && config->skip_compress_count >= 0 &&
|
config->max_delete >= -1 && config->max_alloc <= MAX_SERVER_ALLOC &&
|
||||||
config->skip_compress_count <= MAX_SKIP_COMPRESS_SUFFIXES && config->max_alloc > 0 &&
|
config->skip_compress_count >= 0 &&
|
||||||
|
config->skip_compress_count <= MAX_SKIP_COMPRESS_SUFFIXES &&
|
||||||
(!config->chmod_spec || !*config->chmod_spec ||
|
(!config->chmod_spec || !*config->chmod_spec ||
|
||||||
chmod_apply(0, config->chmod_spec, &(mode_t){0})) &&
|
chmod_apply(0, config->chmod_spec, &(mode_t){0})) &&
|
||||||
config->super_mode >= SUPER_MODE_AUTO && config->super_mode <= SUPER_MODE_OFF;
|
config->super_mode >= SUPER_MODE_AUTO && config->super_mode <= SUPER_MODE_OFF;
|
||||||
@@ -215,13 +229,26 @@ Config* config_create(void) {
|
|||||||
if (!config)
|
if (!config)
|
||||||
return NULL;
|
return NULL;
|
||||||
config_set_defaults(config);
|
config_set_defaults(config);
|
||||||
|
/* config_set_defaults() dups the default server host; a failure there leaves
|
||||||
|
* server_host NULL and would crash later consumers, so fail the whole create
|
||||||
|
* (every caller already handles a NULL return). */
|
||||||
|
if (!config->server_host) {
|
||||||
|
config_delete(config);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
return config;
|
return config;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool config_delete_timing_early(const Config* config) {
|
bool config_delete_timing_early(const Config* config) {
|
||||||
if (!config)
|
if (!config)
|
||||||
return false;
|
return false;
|
||||||
return config->delete_before || config->delete_during;
|
return config->delete_before;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool config_delete_timing_per_dir(const Config* config) {
|
||||||
|
if (!config)
|
||||||
|
return false;
|
||||||
|
return config->delete_during || config->delete_delay;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* A delete-timing flag is only meaningful together with --delete. At most one
|
/* A delete-timing flag is only meaningful together with --delete. At most one
|
||||||
@@ -292,40 +319,69 @@ const char* config_invariants_error(const Config* config) {
|
|||||||
"timing; at most one may be given and each implies --delete";
|
"timing; at most one may be given and each implies --delete";
|
||||||
if (config->iconv_spec && !charset_spec_valid(config->iconv_spec))
|
if (config->iconv_spec && !charset_spec_valid(config->iconv_spec))
|
||||||
return "--iconv requires LOCAL[,REMOTE] charset names supported by iconv";
|
return "--iconv requires LOCAL[,REMOTE] charset names supported by iconv";
|
||||||
|
if ((config->preserve_perms || config->preserve_times || config->preserve_owner ||
|
||||||
|
config->preserve_group || config->preserve_atimes || config->preserve_crtimes ||
|
||||||
|
config->use_executability) &&
|
||||||
|
!config->use_metadata)
|
||||||
|
return "a preservation attribute requires metadata transmission";
|
||||||
if (config->copy_as_set && !config->use_metadata)
|
if (config->copy_as_set && !config->use_metadata)
|
||||||
return "--copy-as requires metadata preservation and cannot be combined with --no-preserve";
|
return "--copy-as requires metadata preservation and cannot be combined with --no-preserve";
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool config_derived_use_metadata(const Config* config) {
|
||||||
|
if (!config)
|
||||||
|
return false;
|
||||||
|
if (config->preserve_perms || config->preserve_times || config->preserve_owner ||
|
||||||
|
config->preserve_group || config->preserve_atimes || config->preserve_crtimes ||
|
||||||
|
config->use_executability || config->preserve_xattrs || config->preserve_acls ||
|
||||||
|
config->fake_super || config->preserve_devices || config->preserve_specials ||
|
||||||
|
config->copy_devices || config->write_devices ||
|
||||||
|
(config->chmod_spec && config->chmod_spec[0]) || config->copy_as_set ||
|
||||||
|
config->chown_uid_set || config->chown_gid_set || config->usermap_count > 0 ||
|
||||||
|
config->groupmap_count > 0 || config->update)
|
||||||
|
return true;
|
||||||
|
return (config->use_incremental || config->use_delta) &&
|
||||||
|
!config->cli.metadata_explicitly_disabled;
|
||||||
|
}
|
||||||
|
|
||||||
bool config_has_basis(const Config* config) {
|
bool config_has_basis(const Config* config) {
|
||||||
return config && config->basis_count > 0;
|
return config && config->basis_count > 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* A basis-dir path travels from the client to the receiver and is resolved
|
/* A basis-dir path travels from the client to the receiver and is resolved
|
||||||
* below the destination root, so it must be a non-empty relative path with no
|
* below the destination root when relative, or used verbatim when absolute
|
||||||
* "." or ".." component and no traversal: an absolute or escaping path would
|
* (matching rsync). Either form must be non-empty, traversal-free (no "..")
|
||||||
* make the receiver read or link files outside its authorized root.
|
* and free of "." components: an escaping path would make the receiver read or
|
||||||
|
* link files outside its authorized root. An absolute path is still subject to
|
||||||
|
* the receiver's root confinement at open time (file_open_secure_parent), so a
|
||||||
|
* basis outside the authorized root is simply not found rather than an escape.
|
||||||
*
|
*
|
||||||
* Returns a malloc'd CANONICAL copy of an accepted path, or NULL when the path
|
* Returns a malloc'd CANONICAL copy of an accepted path, or NULL when the path
|
||||||
* is rejected. Canonicalization collapses interior empty components ("a//b" ->
|
* is rejected. Canonicalization collapses interior empty components ("a//b" ->
|
||||||
* "a/b"), drops "." components and trailing "/"s, so validation, the delete
|
* "a/b"), drops "." components and trailing "/"s, and preserves a leading '/'
|
||||||
* walker prefix match and the receiver's basis lookup all agree on one form.
|
* for absolute paths, so validation, the delete walker prefix match and the
|
||||||
* The normalizer is the single source of truth for both config_basis_path_valid
|
* receiver's basis lookup all agree on one form. The normalizer is the single
|
||||||
* and config_basis_append. */
|
* source of truth for both config_basis_path_valid and config_basis_append. */
|
||||||
static char* basis_path_normalize(const char* path) {
|
static char* basis_path_normalize(const char* path) {
|
||||||
if (!path || path[0] == '\0' || path[0] == '/' || has_path_traversal(path))
|
if (!path || path[0] == '\0' || has_path_traversal(path))
|
||||||
return NULL;
|
return NULL;
|
||||||
if (strcmp(path, ".") == 0)
|
bool absolute = path[0] == '/';
|
||||||
|
if (!absolute && strcmp(path, ".") == 0)
|
||||||
|
return NULL;
|
||||||
|
if (absolute && strcmp(path, "/") == 0)
|
||||||
return NULL;
|
return NULL;
|
||||||
char* dup = str_dup(path);
|
char* dup = str_dup(path);
|
||||||
if (!dup)
|
if (!dup)
|
||||||
return NULL;
|
return NULL;
|
||||||
size_t out_len = 0;
|
size_t out_len = 0;
|
||||||
char* out = malloc(strlen(path) + 1);
|
char* out = malloc(strlen(path) + 2);
|
||||||
if (!out) {
|
if (!out) {
|
||||||
free(dup);
|
free(dup);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
if (absolute)
|
||||||
|
out[out_len++] = '/';
|
||||||
char* saveptr = NULL;
|
char* saveptr = NULL;
|
||||||
bool ok = true;
|
bool ok = true;
|
||||||
for (char* part = strtok_r(dup, "/", &saveptr); part; part = strtok_r(NULL, "/", &saveptr)) {
|
for (char* part = strtok_r(dup, "/", &saveptr); part; part = strtok_r(NULL, "/", &saveptr)) {
|
||||||
@@ -335,14 +391,14 @@ static char* basis_path_normalize(const char* path) {
|
|||||||
}
|
}
|
||||||
if (strcmp(part, ".") == 0)
|
if (strcmp(part, ".") == 0)
|
||||||
continue;
|
continue;
|
||||||
if (out_len > 0)
|
if (out_len > 0 && out[out_len - 1] != '/')
|
||||||
out[out_len++] = '/';
|
out[out_len++] = '/';
|
||||||
size_t len = strlen(part);
|
size_t len = strlen(part);
|
||||||
memcpy(out + out_len, part, len);
|
memcpy(out + out_len, part, len);
|
||||||
out_len += len;
|
out_len += len;
|
||||||
}
|
}
|
||||||
free(dup);
|
free(dup);
|
||||||
if (!ok || out_len == 0) {
|
if (!ok || out_len == 0 || (absolute && out_len == 1)) {
|
||||||
free(out);
|
free(out);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
@@ -637,9 +693,15 @@ int config_parse_ssh_dest(Config* config) {
|
|||||||
return daemon_dest_parse_error("invalid remote destination user@host (must not be empty or "
|
return daemon_dest_parse_error("invalid remote destination user@host (must not be empty or "
|
||||||
"start with '-')",
|
"start with '-')",
|
||||||
dest);
|
dest);
|
||||||
config->transport = TRANSPORT_SSH;
|
char* ssh_destination = str_dup(dest);
|
||||||
config->ssh_destination = str_dup(dest);
|
|
||||||
char* path = str_dup(colon + 1);
|
char* path = str_dup(colon + 1);
|
||||||
|
if (!ssh_destination || !path) {
|
||||||
|
free(ssh_destination);
|
||||||
|
free(path);
|
||||||
|
return daemon_dest_parse_error("out of memory parsing remote destination", dest);
|
||||||
|
}
|
||||||
|
config->transport = TRANSPORT_SSH;
|
||||||
|
config->ssh_destination = ssh_destination;
|
||||||
free(config->receive_root_directory);
|
free(config->receive_root_directory);
|
||||||
config->receive_root_directory = path;
|
config->receive_root_directory = path;
|
||||||
return 0;
|
return 0;
|
||||||
@@ -725,15 +787,25 @@ void config_delete(Config* config) {
|
|||||||
free(config->skip_compress_suffixes[i]);
|
free(config->skip_compress_suffixes[i]);
|
||||||
free(config->skip_compress_suffixes);
|
free(config->skip_compress_suffixes);
|
||||||
}
|
}
|
||||||
free(config->usermap);
|
if (config->usermap) {
|
||||||
|
for (int i = 0; i < config->usermap_count; i++)
|
||||||
|
free(config->usermap[i].to_name);
|
||||||
|
free(config->usermap);
|
||||||
|
}
|
||||||
config->usermap = NULL;
|
config->usermap = NULL;
|
||||||
config->usermap_count = 0;
|
config->usermap_count = 0;
|
||||||
free(config->groupmap);
|
if (config->groupmap) {
|
||||||
|
for (int i = 0; i < config->groupmap_count; i++)
|
||||||
|
free(config->groupmap[i].to_name);
|
||||||
|
free(config->groupmap);
|
||||||
|
}
|
||||||
config->groupmap = NULL;
|
config->groupmap = NULL;
|
||||||
config->groupmap_count = 0;
|
config->groupmap_count = 0;
|
||||||
if (config->filters) {
|
if (config->filters) {
|
||||||
array_list_delete(config->filters);
|
array_list_delete(config->filters);
|
||||||
}
|
}
|
||||||
|
filter_rule_list_free(config->protect_rules);
|
||||||
|
config->protect_rules = NULL;
|
||||||
/* A --delay-updates staging tree is transient receiver state: remove any
|
/* A --delay-updates staging tree is transient receiver state: remove any
|
||||||
leftovers on every exit path (success already emptied it). */
|
leftovers on every exit path (success already emptied it). */
|
||||||
if (config->delay_context)
|
if (config->delay_context)
|
||||||
@@ -756,11 +828,14 @@ void config_delete(Config* config) {
|
|||||||
* ------------------------------------------------------------------------- */
|
* ------------------------------------------------------------------------- */
|
||||||
|
|
||||||
/* --max-alloc: raw 64-bit value, clamped server-side and installed as the
|
/* --max-alloc: raw 64-bit value, clamped server-side and installed as the
|
||||||
* session allocation ceiling. A zero value is rejected. */
|
* session allocation ceiling. A received 0 is rsync's "no alloc limit"; on the
|
||||||
|
* receive path it is mapped to the server ceiling so a client can never disable
|
||||||
|
* it (client-side 0 remains unlimited). Any value above the ceiling is clamped
|
||||||
|
* to it. */
|
||||||
static bool config_receive_max_alloc(int fd, unsigned long long* value) {
|
static bool config_receive_max_alloc(int fd, unsigned long long* value) {
|
||||||
if (!receive_n_data(fd, value, sizeof(*value)) || *value == 0)
|
if (!receive_n_data(fd, value, sizeof(*value)))
|
||||||
return false;
|
return false;
|
||||||
if (*value > MAX_SERVER_ALLOC)
|
if (*value == 0 || *value > MAX_SERVER_ALLOC)
|
||||||
*value = MAX_SERVER_ALLOC;
|
*value = MAX_SERVER_ALLOC;
|
||||||
protocol_session_set_max_alloc(NULL, *value);
|
protocol_session_set_max_alloc(NULL, *value);
|
||||||
return true;
|
return true;
|
||||||
@@ -842,6 +917,14 @@ static bool config_receive_checksum_algo(int fd, int* value) {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static bool config_receive_compression_algo(int fd, int* value) {
|
||||||
|
int algo;
|
||||||
|
if (!receive_int(fd, &algo) || !compression_algo_valid(algo))
|
||||||
|
return false;
|
||||||
|
*value = algo;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
static bool config_receive_super_mode(int fd, SuperMode* value) {
|
static bool config_receive_super_mode(int fd, SuperMode* value) {
|
||||||
int mode;
|
int mode;
|
||||||
if (!receive_int(fd, &mode) || mode < SUPER_MODE_AUTO || mode > SUPER_MODE_OFF)
|
if (!receive_int(fd, &mode) || mode < SUPER_MODE_AUTO || mode > SUPER_MODE_OFF)
|
||||||
@@ -958,9 +1041,138 @@ static bool receive_basis_entries(int fd, Config* c, ConfigStringBudget* budget)
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Receiver-side delete-protection rules (protocol 2.28.0). The sender compiles
|
||||||
|
* its command-line selection rules exactly as the scanner does and streams the
|
||||||
|
* result as one bounded, self-describing block (count + per-rule records); the
|
||||||
|
* receiver reconstructs a FilterRuleList for the --delete extras walk. owner
|
||||||
|
* and pattern are charged through the shared ConfigStringBudget and the block
|
||||||
|
* additionally enforces MAX_FILTER_RULES / MAX_FILTER_BYTES. */
|
||||||
|
static bool send_protect_entries(int fd, const Config* c) {
|
||||||
|
int count = c->filters ? c->filters->size : 0;
|
||||||
|
const char** texts = NULL;
|
||||||
|
if (count > 0) {
|
||||||
|
texts = malloc((size_t)count * sizeof(char*));
|
||||||
|
if (!texts)
|
||||||
|
return false;
|
||||||
|
for (int i = 0; i < count; i++)
|
||||||
|
texts[i] = (const char*)c->filters->items[i];
|
||||||
|
}
|
||||||
|
char err[160];
|
||||||
|
FilterRuleList* rules =
|
||||||
|
filter_base_build(texts, count, c->cvs_exclude, c->delete_excluded, err, sizeof(err));
|
||||||
|
free(texts);
|
||||||
|
if (!rules) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "invalid filter rule: %s", err);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
/* The receiver rejects any block with more than MAX_FILTER_RULES entries as a
|
||||||
|
* protocol error; refuse to emit such a frame at all. filter_base_build()
|
||||||
|
* can expand the client rule set (cvs-exclude, merge files), so this is the
|
||||||
|
* authoritative bound, not config->filters->size. */
|
||||||
|
if (rules->count < 0 || rules->count > MAX_FILTER_RULES) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "too many filter rules: %d (maximum %d)", rules->count,
|
||||||
|
MAX_FILTER_RULES);
|
||||||
|
filter_rule_list_free(rules);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
bool ok = send_int(fd, rules->count);
|
||||||
|
for (int i = 0; ok && i < rules->count; i++) {
|
||||||
|
const FilterRule* r = rules->items[i];
|
||||||
|
/* Mirror the receiver's limit so the peer never receives a rule it will
|
||||||
|
reject as a protocol error. */
|
||||||
|
if (r->pattern && strlen(r->pattern) > MAX_PROTECT_PATTERN_LEN) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "filter pattern exceeds %d bytes", MAX_PROTECT_PATTERN_LEN);
|
||||||
|
filter_rule_list_free(rules);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
ok = send_int(fd, (int)r->action) && send_int(fd, (int)r->sides) &&
|
||||||
|
send_int(fd, r->anchored ? 1 : 0) && send_int(fd, r->dir_only ? 1 : 0) &&
|
||||||
|
send_int(fd, r->negate ? 1 : 0) && send_str(fd, r->owner ? r->owner : "") &&
|
||||||
|
send_str(fd, r->pattern ? r->pattern : "");
|
||||||
|
}
|
||||||
|
filter_rule_list_free(rules);
|
||||||
|
return ok;
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool receive_protect_entries(int fd, Config* c, ConfigStringBudget* budget) {
|
||||||
|
int count;
|
||||||
|
if (!receive_int(fd, &count))
|
||||||
|
return false;
|
||||||
|
if (count < 0 || count > MAX_FILTER_RULES)
|
||||||
|
return false;
|
||||||
|
if (count == 0)
|
||||||
|
return true;
|
||||||
|
FilterRuleList* list = filter_rule_list_create();
|
||||||
|
if (!list)
|
||||||
|
return false;
|
||||||
|
size_t pattern_bytes = 0;
|
||||||
|
for (int i = 0; i < count; i++) {
|
||||||
|
int action;
|
||||||
|
int sides;
|
||||||
|
bool anchored;
|
||||||
|
bool dir_only;
|
||||||
|
bool negate;
|
||||||
|
if (!receive_int(fd, &action) ||
|
||||||
|
(action != FILTER_ACTION_EXCLUDE && action != FILTER_ACTION_INCLUDE) ||
|
||||||
|
!receive_int(fd, &sides) || sides < (int)FILTER_SIDE_SENDER ||
|
||||||
|
sides > (int)(FILTER_SIDE_SENDER | FILTER_SIDE_RECEIVER) ||
|
||||||
|
!receive_wire_bool(fd, &anchored) || !receive_wire_bool(fd, &dir_only) ||
|
||||||
|
!receive_wire_bool(fd, &negate))
|
||||||
|
goto fail;
|
||||||
|
char* owner = config_receive_str(fd, budget);
|
||||||
|
if (!owner)
|
||||||
|
goto fail;
|
||||||
|
char* pattern = config_receive_str(fd, budget);
|
||||||
|
if (!pattern || pattern[0] == '\0') {
|
||||||
|
free(owner);
|
||||||
|
free(pattern);
|
||||||
|
goto fail;
|
||||||
|
}
|
||||||
|
/* A pattern too long to be evaluated by glob_match against a PATH_MAX path
|
||||||
|
would silently fail to match and leave a protect rule inert (fail-open:
|
||||||
|
the entry is then deleted). Reject it up front as a protocol error
|
||||||
|
rather than accept a rule that can never shield anything. */
|
||||||
|
if (strlen(pattern) > MAX_PROTECT_PATTERN_LEN) {
|
||||||
|
free(owner);
|
||||||
|
free(pattern);
|
||||||
|
goto fail;
|
||||||
|
}
|
||||||
|
size_t bytes = strlen(owner) + strlen(pattern);
|
||||||
|
if (bytes > MAX_FILTER_BYTES - pattern_bytes) {
|
||||||
|
free(owner);
|
||||||
|
free(pattern);
|
||||||
|
goto fail;
|
||||||
|
}
|
||||||
|
pattern_bytes += bytes;
|
||||||
|
FilterRule* rule = calloc(1, sizeof(FilterRule));
|
||||||
|
if (!rule) {
|
||||||
|
free(owner);
|
||||||
|
free(pattern);
|
||||||
|
goto fail;
|
||||||
|
}
|
||||||
|
rule->action = (FilterAction)action;
|
||||||
|
rule->sides = (unsigned)sides;
|
||||||
|
rule->anchored = anchored;
|
||||||
|
rule->dir_only = dir_only;
|
||||||
|
rule->negate = negate;
|
||||||
|
rule->owner = owner;
|
||||||
|
rule->pattern = pattern;
|
||||||
|
if (!filter_rule_list_add(list, rule)) {
|
||||||
|
filter_rule_free(rule);
|
||||||
|
goto fail;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
c->protect_rules = list;
|
||||||
|
return true;
|
||||||
|
fail:
|
||||||
|
filter_rule_list_free(list);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
static bool send_identity_entries(int fd, const IdentityMap* map, int count) {
|
static bool send_identity_entries(int fd, const IdentityMap* map, int count) {
|
||||||
for (int i = 0; i < count; i++) {
|
for (int i = 0; i < count; i++) {
|
||||||
if (!send_int(fd, map[i].from) || !send_int(fd, map[i].to))
|
if (!send_int(fd, map[i].from) || !send_int(fd, map[i].from_hi) || !send_int(fd, map[i].to) ||
|
||||||
|
!send_str(fd, map[i].to_name ? map[i].to_name : ""))
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
return true;
|
return true;
|
||||||
@@ -968,20 +1180,32 @@ static bool send_identity_entries(int fd, const IdentityMap* map, int count) {
|
|||||||
|
|
||||||
static bool receive_identity_entries(int fd, ConfigStringBudget* budget, int count,
|
static bool receive_identity_entries(int fd, ConfigStringBudget* budget, int count,
|
||||||
IdentityMap** out) {
|
IdentityMap** out) {
|
||||||
(void)budget;
|
|
||||||
if (count <= 0)
|
if (count <= 0)
|
||||||
return true;
|
return true;
|
||||||
IdentityMap* map = calloc((size_t)count, sizeof(IdentityMap));
|
IdentityMap* map = calloc((size_t)count, sizeof(IdentityMap));
|
||||||
if (!map)
|
if (!map)
|
||||||
return false;
|
return false;
|
||||||
for (int i = 0; i < count; i++) {
|
for (int i = 0; i < count; i++) {
|
||||||
if (!receive_int(fd, &map[i].from) || !receive_int(fd, &map[i].to)) {
|
if (!receive_int(fd, &map[i].from) || !receive_int(fd, &map[i].from_hi) ||
|
||||||
free(map);
|
!receive_int(fd, &map[i].to))
|
||||||
return false;
|
goto fail;
|
||||||
|
char* name = config_receive_str(fd, budget);
|
||||||
|
if (!name)
|
||||||
|
goto fail;
|
||||||
|
if (name[0] == '\0') {
|
||||||
|
free(name);
|
||||||
|
map[i].to_name = NULL;
|
||||||
|
} else {
|
||||||
|
map[i].to_name = name;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
*out = map;
|
*out = map;
|
||||||
return true;
|
return true;
|
||||||
|
fail:
|
||||||
|
for (int i = 0; i < count; i++)
|
||||||
|
free(map[i].to_name);
|
||||||
|
free(map);
|
||||||
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* ---------------------------------------------------------------------------
|
/* ---------------------------------------------------------------------------
|
||||||
@@ -1030,6 +1254,9 @@ static bool receive_identity_entries(int fd, ConfigStringBudget* budget, int cou
|
|||||||
#define CONFIG_SEND_INT_CHECKSUM_ALGO(name) send_int(fd, c->name)
|
#define CONFIG_SEND_INT_CHECKSUM_ALGO(name) send_int(fd, c->name)
|
||||||
#define CONFIG_RECV_INT_CHECKSUM_ALGO(name) config_receive_checksum_algo(fd, &c->name)
|
#define CONFIG_RECV_INT_CHECKSUM_ALGO(name) config_receive_checksum_algo(fd, &c->name)
|
||||||
|
|
||||||
|
#define CONFIG_SEND_INT_COMPRESSION_ALGO(name) send_int(fd, c->name)
|
||||||
|
#define CONFIG_RECV_INT_COMPRESSION_ALGO(name) config_receive_compression_algo(fd, &c->name)
|
||||||
|
|
||||||
#define CONFIG_SEND_SUPERMODE(name) send_int(fd, (int)c->name)
|
#define CONFIG_SEND_SUPERMODE(name) send_int(fd, (int)c->name)
|
||||||
#define CONFIG_RECV_SUPERMODE(name) config_receive_super_mode(fd, &c->name)
|
#define CONFIG_RECV_SUPERMODE(name) config_receive_super_mode(fd, &c->name)
|
||||||
|
|
||||||
@@ -1069,6 +1296,9 @@ static bool receive_identity_entries(int fd, ConfigStringBudget* budget, int cou
|
|||||||
#define CONFIG_RECV_BLOCK_IDMAP(name) \
|
#define CONFIG_RECV_BLOCK_IDMAP(name) \
|
||||||
receive_identity_entries(fd, budget, c->name##_count, &c->name)
|
receive_identity_entries(fd, budget, c->name##_count, &c->name)
|
||||||
|
|
||||||
|
#define CONFIG_SEND_BLOCK_PROTECT_RULES(name) send_protect_entries(fd, c)
|
||||||
|
#define CONFIG_RECV_BLOCK_PROTECT_RULES(name) receive_protect_entries(fd, c, budget)
|
||||||
|
|
||||||
/* One table entry, applied in sequence. XSEND/XRECV are statement macros so
|
/* One table entry, applied in sequence. XSEND/XRECV are statement macros so
|
||||||
* consecutive entries read as a plain sequence of assignments. */
|
* consecutive entries read as a plain sequence of assignments. */
|
||||||
#define XSEND(name, ctype, def, kind) ok = ok && (CONFIG_SEND_##kind(name));
|
#define XSEND(name, ctype, def, kind) ok = ok && (CONFIG_SEND_##kind(name));
|
||||||
@@ -1104,6 +1334,9 @@ CONFIG_DEFINE_SEND(send_daemon_auth, CONFIG_WIRE_DAEMON_AUTH_FIELDS)
|
|||||||
CONFIG_DEFINE_SEND(send_iconv_spec, CONFIG_WIRE_ICONV_FIELDS)
|
CONFIG_DEFINE_SEND(send_iconv_spec, CONFIG_WIRE_ICONV_FIELDS)
|
||||||
CONFIG_DEFINE_SEND(send_privilege_options, CONFIG_WIRE_PRIVILEGE_FIELDS)
|
CONFIG_DEFINE_SEND(send_privilege_options, CONFIG_WIRE_PRIVILEGE_FIELDS)
|
||||||
CONFIG_DEFINE_SEND(send_copy_as_options, CONFIG_WIRE_COPY_AS_FIELDS)
|
CONFIG_DEFINE_SEND(send_copy_as_options, CONFIG_WIRE_COPY_AS_FIELDS)
|
||||||
|
CONFIG_DEFINE_SEND(send_output_options, CONFIG_WIRE_OUTPUT_FIELDS)
|
||||||
|
CONFIG_DEFINE_SEND(send_codec_options, CONFIG_WIRE_CODEC_FIELDS)
|
||||||
|
CONFIG_DEFINE_SEND(send_protect_options, CONFIG_WIRE_PROTECT_FIELDS)
|
||||||
|
|
||||||
CONFIG_DEFINE_RECV(receive_core_fields, CONFIG_WIRE_CORE_FIELDS)
|
CONFIG_DEFINE_RECV(receive_core_fields, CONFIG_WIRE_CORE_FIELDS)
|
||||||
CONFIG_DEFINE_RECV(receive_delta_fields, CONFIG_WIRE_DELTA_FIELDS)
|
CONFIG_DEFINE_RECV(receive_delta_fields, CONFIG_WIRE_DELTA_FIELDS)
|
||||||
@@ -1122,6 +1355,9 @@ CONFIG_DEFINE_RECV(receive_daemon_auth, CONFIG_WIRE_DAEMON_AUTH_FIELDS)
|
|||||||
CONFIG_DEFINE_RECV(receive_iconv_spec, CONFIG_WIRE_ICONV_FIELDS)
|
CONFIG_DEFINE_RECV(receive_iconv_spec, CONFIG_WIRE_ICONV_FIELDS)
|
||||||
CONFIG_DEFINE_RECV(receive_privilege_options, CONFIG_WIRE_PRIVILEGE_FIELDS)
|
CONFIG_DEFINE_RECV(receive_privilege_options, CONFIG_WIRE_PRIVILEGE_FIELDS)
|
||||||
CONFIG_DEFINE_RECV(receive_copy_as_options, CONFIG_WIRE_COPY_AS_FIELDS)
|
CONFIG_DEFINE_RECV(receive_copy_as_options, CONFIG_WIRE_COPY_AS_FIELDS)
|
||||||
|
CONFIG_DEFINE_RECV(receive_output_options, CONFIG_WIRE_OUTPUT_FIELDS)
|
||||||
|
CONFIG_DEFINE_RECV(receive_codec_options, CONFIG_WIRE_CODEC_FIELDS)
|
||||||
|
CONFIG_DEFINE_RECV(receive_protect_options, CONFIG_WIRE_PROTECT_FIELDS)
|
||||||
|
|
||||||
#undef XSEND
|
#undef XSEND
|
||||||
#undef XRECV
|
#undef XRECV
|
||||||
@@ -1238,7 +1474,10 @@ bool config_send_wire_block(int file_descriptor, const Config* config) {
|
|||||||
send_daemon_module(file_descriptor, config) && send_daemon_auth(file_descriptor, config) &&
|
send_daemon_module(file_descriptor, config) && send_daemon_auth(file_descriptor, config) &&
|
||||||
send_iconv_spec(file_descriptor, config) &&
|
send_iconv_spec(file_descriptor, config) &&
|
||||||
send_privilege_options(file_descriptor, config) &&
|
send_privilege_options(file_descriptor, config) &&
|
||||||
send_copy_as_options(file_descriptor, config);
|
send_copy_as_options(file_descriptor, config) &&
|
||||||
|
send_output_options(file_descriptor, config) &&
|
||||||
|
send_codec_options(file_descriptor, config) &&
|
||||||
|
send_protect_options(file_descriptor, config);
|
||||||
}
|
}
|
||||||
|
|
||||||
bool config_send(int file_descriptor, const Config* config) {
|
bool config_send(int file_descriptor, const Config* config) {
|
||||||
@@ -1308,18 +1547,60 @@ Config* config_receive_with_validate(int file_descriptor, ConfigValidateFunc val
|
|||||||
!receive_daemon_auth(file_descriptor, config, &budget) ||
|
!receive_daemon_auth(file_descriptor, config, &budget) ||
|
||||||
!receive_iconv_spec(file_descriptor, config, &budget) ||
|
!receive_iconv_spec(file_descriptor, config, &budget) ||
|
||||||
!receive_privilege_options(file_descriptor, config, &budget) ||
|
!receive_privilege_options(file_descriptor, config, &budget) ||
|
||||||
!receive_copy_as_options(file_descriptor, config, &budget))
|
!receive_copy_as_options(file_descriptor, config, &budget) ||
|
||||||
|
!receive_output_options(file_descriptor, config, &budget) ||
|
||||||
|
!receive_codec_options(file_descriptor, config, &budget) ||
|
||||||
|
!receive_protect_options(file_descriptor, config, &budget))
|
||||||
goto error;
|
goto error;
|
||||||
if (config->compress_choice[0] != '\0' && strcmp(config->compress_choice, "zstd") != 0 &&
|
/* Validate/normalize the negotiated codec. compress_choice is the human
|
||||||
strcmp(config->compress_choice, "none") != 0) {
|
* spelling (NULL or "" when -z was not given); compression_algo is the
|
||||||
char* escaped_choice = output_escape(config->compress_choice, config->eight_bit_output);
|
* concrete codec id the sender used. They must agree, and "auto" is
|
||||||
log_message(LOG_LEVEL_ERROR, "Unsupported compression choice: %s",
|
* canonicalized to FastSync's negotiated default so the stored spelling is
|
||||||
escaped_choice ? escaped_choice : "<allocation failed>");
|
* always concrete (a hostile/older client may still send "auto"). */
|
||||||
char detail[128];
|
if (config->compress_choice && config->compress_choice[0] != '\0') {
|
||||||
snprintf(detail, sizeof(detail), "unsupported compression choice: %s",
|
int choice_algo = compression_algo_from_name(config->compress_choice);
|
||||||
escaped_choice ? escaped_choice : "<allocation failed>");
|
if (choice_algo < 0 && strcasecmp(config->compress_choice, "auto") != 0) {
|
||||||
send_error_detail(file_descriptor, detail);
|
char* escaped_choice = output_escape(config->compress_choice, config->eight_bit_output);
|
||||||
free(escaped_choice);
|
log_message(LOG_LEVEL_ERROR, "Unsupported compression choice: %s",
|
||||||
|
escaped_choice ? escaped_choice : "<allocation failed>");
|
||||||
|
char detail[160];
|
||||||
|
snprintf(detail, sizeof(detail), "unsupported compression choice: %s",
|
||||||
|
escaped_choice ? escaped_choice : "<allocation failed>");
|
||||||
|
send_error_detail(file_descriptor, detail);
|
||||||
|
free(escaped_choice);
|
||||||
|
goto error;
|
||||||
|
}
|
||||||
|
if (choice_algo < 0)
|
||||||
|
choice_algo = (int)compression_negotiate_default();
|
||||||
|
if (strcasecmp(config->compress_choice, "auto") == 0 ||
|
||||||
|
choice_algo == (int)COMPRESSION_ALGO_NONE) {
|
||||||
|
const char* canonical = compression_algo_name((CompressionAlgo)choice_algo);
|
||||||
|
char* dup = str_dup(canonical);
|
||||||
|
if (!dup)
|
||||||
|
goto error;
|
||||||
|
free(config->compress_choice);
|
||||||
|
config->compress_choice = dup;
|
||||||
|
}
|
||||||
|
if (config->compression_algo != choice_algo) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Compression choice '%s' does not match codec id %d",
|
||||||
|
config->compress_choice, config->compression_algo);
|
||||||
|
send_error_detail(file_descriptor, "compression choice/codec mismatch");
|
||||||
|
goto error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
/* The concrete codec must exist only when compression is on. A client that
|
||||||
|
* left -z off has no codec in effect, but the field keeps whatever id it
|
||||||
|
* carried (the receiver never dispatches on it without use_compression), so
|
||||||
|
* the wire value round-trips untouched. */
|
||||||
|
if (config->use_compression && config->compression_algo == (int)COMPRESSION_ALGO_NONE) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Compression requested with the 'none' codec");
|
||||||
|
send_error_detail(file_descriptor, "compression requested with the none codec");
|
||||||
|
goto error;
|
||||||
|
}
|
||||||
|
/* rsync: "none" as the pre-transfer checksum is invalid with --checksum. */
|
||||||
|
if (config->checksum && config->checksum_algo == (int)CHECKSUM_ALGO_NONE) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Invalid checksum-choice for --checksum: none");
|
||||||
|
send_error_detail(file_descriptor, "checksum-choice 'none' cannot be used with --checksum");
|
||||||
goto error;
|
goto error;
|
||||||
}
|
}
|
||||||
if (!validate_received_config(config)) {
|
if (!validate_received_config(config)) {
|
||||||
|
|||||||
+368
-56
@@ -3,6 +3,8 @@
|
|||||||
|
|
||||||
#include "array_list.h"
|
#include "array_list.h"
|
||||||
#include "checksum.h"
|
#include "checksum.h"
|
||||||
|
#include "compression.h"
|
||||||
|
#include "filter.h"
|
||||||
#include <stdbool.h>
|
#include <stdbool.h>
|
||||||
#include <stdint.h>
|
#include <stdint.h>
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
@@ -39,15 +41,20 @@ typedef struct BasisDest {
|
|||||||
char* path; /* relative to the destination root (receiver-confined) */
|
char* path; /* relative to the destination root (receiver-confined) */
|
||||||
} BasisDest;
|
} BasisDest;
|
||||||
|
|
||||||
/* One resolved FROM:TO identity-mapping rule (--usermap / --groupmap). Both
|
/* One FROM:TO identity-mapping rule (--usermap / --groupmap). `from`/`from_hi`
|
||||||
* fields are numeric ids. IDENTITY_MATCH_ANY (-1) in `from` is rsync's '*'
|
* describe the sender-side FROM matcher (a single id when from_hi == from, an
|
||||||
* wildcard (matches any transmitted id); IDENTITY_CURRENT (-1) in `to` makes
|
* inclusive LOW-HIGH range, IDENTITY_MATCH_ANY for rsync's '*', or
|
||||||
* the receiver resolve the receiving process's own current euid/egid at apply
|
* IDENTITY_MATCH_UNNAMED for rsync's empty FROM). `to` is the receiver-side TO
|
||||||
* time. Names are resolved to numbers at parse time on the client (see
|
* numeric id (IDENTITY_CURRENT = the receiving process's own euid/egid) UNLESS
|
||||||
* identity.h for the exact subset). */
|
* `to_name` is non-NULL, in which case the receiver resolves the name against
|
||||||
|
* its own account database at apply time (rsync resolves TO names on the
|
||||||
|
* receiver) and `to` is ignored. FROM names/ranges/globs are resolved on the
|
||||||
|
* client (the sender) exactly as rsync matches them against sender names. */
|
||||||
typedef struct {
|
typedef struct {
|
||||||
int32_t from;
|
int32_t from;
|
||||||
|
int32_t from_hi;
|
||||||
int32_t to;
|
int32_t to;
|
||||||
|
char* to_name;
|
||||||
} IdentityMap;
|
} IdentityMap;
|
||||||
|
|
||||||
/* --sockopts=OPTIONS allowlist. Only these option names are accepted; anything
|
/* --sockopts=OPTIONS allowlist. Only these option names are accepted; anything
|
||||||
@@ -76,7 +83,7 @@ typedef struct {
|
|||||||
typedef enum SuperMode { SUPER_MODE_AUTO = 0, SUPER_MODE_ON = 1, SUPER_MODE_OFF = 2 } SuperMode;
|
typedef enum SuperMode { SUPER_MODE_AUTO = 0, SUPER_MODE_ON = 1, SUPER_MODE_OFF = 2 } SuperMode;
|
||||||
|
|
||||||
/* ===========================================================================
|
/* ===========================================================================
|
||||||
* Config wire-field table (single source of truth for protocol 2.21.0).
|
* Config wire-field table (single source of truth for protocol 2.29.0).
|
||||||
*
|
*
|
||||||
* Every field below crosses the wire. The table is the ONLY place a
|
* Every field below crosses the wire. The table is the ONLY place a
|
||||||
* serialized field is named: config.h expands CONFIG_WIRE_FIELDS() to declare
|
* serialized field is named: config.h expands CONFIG_WIRE_FIELDS() to declare
|
||||||
@@ -191,14 +198,23 @@ typedef enum SuperMode { SUPER_MODE_AUTO = 0, SUPER_MODE_ON = 1, SUPER_MODE_OFF
|
|||||||
X(skip_compress_count, int, 0, INT_SKIPCOUNT) \
|
X(skip_compress_count, int, 0, INT_SKIPCOUNT) \
|
||||||
X(skip_compress_suffixes, char**, NULL, BLOCK_SKIP_SUFFIXES)
|
X(skip_compress_suffixes, char**, NULL, BLOCK_SKIP_SUFFIXES)
|
||||||
|
|
||||||
|
/* FastSync-only --verify-basis (protocol 2.28.0, no version bump by project
|
||||||
|
* decision): restores the stricter content equality on a basis hit. By
|
||||||
|
* default a basis hit is accepted on rsync's metadata quick-check alone (equal
|
||||||
|
* size plus equal mtime, or size alone under --size-only); with this flag the
|
||||||
|
* receiver ALSO requires the basis bytes' whole-file digest (the negotiated
|
||||||
|
* --checksum-choice algorithm) to equal the sender's, exactly FastSync's
|
||||||
|
* historical behavior. It is a receiver policy and crosses the wire so the
|
||||||
|
* receiver knows whether to read and hash the basis content. */
|
||||||
#define CONFIG_WIRE_BASIS_FIELDS(X) \
|
#define CONFIG_WIRE_BASIS_FIELDS(X) \
|
||||||
X(basis_count, int, 0, INT_BASISCOUNT) \
|
X(basis_count, int, 0, INT_BASISCOUNT) \
|
||||||
X(basis_dirs, BasisDest*, NULL, BLOCK_BASIS)
|
X(basis_dirs, BasisDest*, NULL, BLOCK_BASIS) \
|
||||||
|
X(verify_basis, bool, false, BOOL)
|
||||||
|
|
||||||
#define CONFIG_WIRE_FUZZY_FIELDS(X) X(fuzzy, bool, false, BOOL)
|
#define CONFIG_WIRE_FUZZY_FIELDS(X) X(fuzzy, bool, false, BOOL)
|
||||||
|
|
||||||
#define CONFIG_WIRE_CHECKSUM_FIELDS(X) \
|
#define CONFIG_WIRE_CHECKSUM_FIELDS(X) \
|
||||||
X(checksum_algo, int, CHECKSUM_ALGO_XXH64, INT_CHECKSUM_ALGO) \
|
X(checksum_algo, int, CHECKSUM_ALGO_DEFAULT, INT_CHECKSUM_ALGO) \
|
||||||
X(checksum_seed, uint64_t, 0, RAW)
|
X(checksum_seed, uint64_t, 0, RAW)
|
||||||
|
|
||||||
#define CONFIG_WIRE_IDENTITY_FIELDS(X) \
|
#define CONFIG_WIRE_IDENTITY_FIELDS(X) \
|
||||||
@@ -216,7 +232,11 @@ typedef enum SuperMode { SUPER_MODE_AUTO = 0, SUPER_MODE_ON = 1, SUPER_MODE_OFF
|
|||||||
X(preserve_atimes, bool, false, BOOL) \
|
X(preserve_atimes, bool, false, BOOL) \
|
||||||
X(preserve_crtimes, bool, false, BOOL) \
|
X(preserve_crtimes, bool, false, BOOL) \
|
||||||
X(omit_dir_times, bool, false, BOOL) \
|
X(omit_dir_times, bool, false, BOOL) \
|
||||||
X(omit_link_times, bool, false, BOOL)
|
X(omit_link_times, bool, false, BOOL) \
|
||||||
|
X(preserve_perms, bool, false, BOOL) \
|
||||||
|
X(preserve_times, bool, false, BOOL) \
|
||||||
|
X(preserve_owner, bool, false, BOOL) \
|
||||||
|
X(preserve_group, bool, false, BOOL)
|
||||||
|
|
||||||
#define CONFIG_WIRE_SYMLINK_TRUST_FIELDS(X) \
|
#define CONFIG_WIRE_SYMLINK_TRUST_FIELDS(X) \
|
||||||
X(munge_links, bool, false, BOOL) \
|
X(munge_links, bool, false, BOOL) \
|
||||||
@@ -237,6 +257,66 @@ typedef enum SuperMode { SUPER_MODE_AUTO = 0, SUPER_MODE_ON = 1, SUPER_MODE_OFF
|
|||||||
X(copy_as_uid, int32_t, 0, COPY_AS_ID) \
|
X(copy_as_uid, int32_t, 0, COPY_AS_ID) \
|
||||||
X(copy_as_gid, int32_t, 0, COPY_AS_ID)
|
X(copy_as_gid, int32_t, 0, COPY_AS_ID)
|
||||||
|
|
||||||
|
/* Output-parity wave (protocol 2.23.0). report_dest_info tells the receiver to
|
||||||
|
* answer every per-file STATUS_CHECK with a STATUS_DEST_INFO snapshot of the
|
||||||
|
* pre-transfer destination entry (see protocol.h). It is set by the client
|
||||||
|
* only when -i/--itemize-changes or --out-format asks for per-file change
|
||||||
|
* output; the transfer decision itself is unchanged.
|
||||||
|
*
|
||||||
|
* Wire-stats wave (protocol 2.25.0). report_stats tells the receiver to send a
|
||||||
|
* STATUS_STATS frame immediately before its terminal success status carrying
|
||||||
|
* the receiver-only counters (matched data, deleted-file count) and,
|
||||||
|
* for -n/--dry-run --delete, the destination-relative paths it WOULD have
|
||||||
|
* deleted. It is set by the client only when --stats, --progress/-P, an
|
||||||
|
* --out-format token needs a wire counter (%b/%c), or a dry-run carries
|
||||||
|
* --delete; the transfer decision itself is unchanged.
|
||||||
|
*
|
||||||
|
* --info wave (protocol 2.27.0). report_deletes tells the receiver to include
|
||||||
|
* the destination-relative paths it ACTUALLY removed in its terminal
|
||||||
|
* STATUS_STATS record (the same path-list field the dry-run would-delete report
|
||||||
|
* uses), so the sender can print rsync's `deleting PATH`/`*deleting` lines for a
|
||||||
|
* real (non-dry-run) deletion. It is set when --delete is active and any of
|
||||||
|
* --info=del, -i/--itemize-changes or --out-format requests per-file change
|
||||||
|
* output; the transfer decision itself is unchanged. */
|
||||||
|
#define CONFIG_WIRE_OUTPUT_FIELDS(X) \
|
||||||
|
X(report_dest_info, bool, false, BOOL) \
|
||||||
|
X(report_stats, bool, false, BOOL) X(report_deletes, bool, false, BOOL)
|
||||||
|
|
||||||
|
/* Codec-negotiation wave (protocol 2.26.0). compression_algo is the concrete
|
||||||
|
* codec the client selected for this transfer (a CompressionAlgo id) and is the
|
||||||
|
* value the receiver validates and installs. It is the resolved result of
|
||||||
|
* --compress-choice / the "auto" negotiation so both peers agree exactly.
|
||||||
|
*
|
||||||
|
* Negotiation model: FastSync enforces a strict same-version handshake, so both
|
||||||
|
* peers carry the identical compiled-in codec set. The client resolves the
|
||||||
|
* effective algorithm deterministically and serializes it here; "auto" picks
|
||||||
|
* the first entry of the rsync 3.4.1 preference order
|
||||||
|
* (compression: zstd lz4 zlibx zlib none; checksum: xxh128 xxh3 xxh64 md5 md4
|
||||||
|
* sha1 none), and an explicit request wins. The receiver rejects (before
|
||||||
|
* STATUS_OK) any algorithm outside its own supported set, which is rsync's
|
||||||
|
* "no common choice is an error" behavior. The same resolver runs on both
|
||||||
|
* sides (compression_negotiate_default / checksum_negotiate_default), so the
|
||||||
|
* fallback is consistent.
|
||||||
|
*
|
||||||
|
* The field is appended after the output block so every pre-2.26 field keeps
|
||||||
|
* its wire position. */
|
||||||
|
#define CONFIG_WIRE_CODEC_FIELDS(X) \
|
||||||
|
X(compression_algo, int, COMPRESSION_ALGO_ZSTD, INT_COMPRESSION_ALGO)
|
||||||
|
|
||||||
|
/* Receiver-side delete-protection filter rules (protocol 2.28.0). The sender
|
||||||
|
* compiles its root-level selection rules exactly as the scanner does
|
||||||
|
* (filter_base_build over --filter/-f/--exclude/--include/-C) and streams them
|
||||||
|
* as one self-describing, bounded block (count followed by per-rule records).
|
||||||
|
* The receiver reconstructs `protect_rules` and evaluates them against
|
||||||
|
* DESTINATION-ONLY entries during the --delete extras walk, so a
|
||||||
|
* `protect`/`P` rule protects an extra that never appeared on the sender
|
||||||
|
* (rsync re-derives deletion protection from the filter list; FastSync
|
||||||
|
* historically derived it only from the source scan). `protect_rules` is NULL
|
||||||
|
* on the sender and is owned/freed by the receiver Config. Bounded by
|
||||||
|
* MAX_FILTER_RULES and MAX_FILTER_BYTES; an unknown action/sides is a protocol
|
||||||
|
* error. */
|
||||||
|
#define CONFIG_WIRE_PROTECT_FIELDS(X) X(protect_rules, FilterRuleList*, NULL, BLOCK_PROTECT_RULES)
|
||||||
|
|
||||||
/* All serialized fields, in exact wire order. Concatenating the per-segment
|
/* All serialized fields, in exact wire order. Concatenating the per-segment
|
||||||
* lists here is what keeps the declaration order = the wire order. */
|
* lists here is what keeps the declaration order = the wire order. */
|
||||||
#define CONFIG_WIRE_FIELDS(X) \
|
#define CONFIG_WIRE_FIELDS(X) \
|
||||||
@@ -257,7 +337,56 @@ typedef enum SuperMode { SUPER_MODE_AUTO = 0, SUPER_MODE_ON = 1, SUPER_MODE_OFF
|
|||||||
CONFIG_WIRE_DAEMON_AUTH_FIELDS(X) \
|
CONFIG_WIRE_DAEMON_AUTH_FIELDS(X) \
|
||||||
CONFIG_WIRE_ICONV_FIELDS(X) \
|
CONFIG_WIRE_ICONV_FIELDS(X) \
|
||||||
CONFIG_WIRE_PRIVILEGE_FIELDS(X) \
|
CONFIG_WIRE_PRIVILEGE_FIELDS(X) \
|
||||||
CONFIG_WIRE_COPY_AS_FIELDS(X)
|
CONFIG_WIRE_COPY_AS_FIELDS(X) \
|
||||||
|
CONFIG_WIRE_OUTPUT_FIELDS(X) \
|
||||||
|
CONFIG_WIRE_CODEC_FIELDS(X) \
|
||||||
|
CONFIG_WIRE_PROTECT_FIELDS(X)
|
||||||
|
|
||||||
|
/* Client-only, CLI-parse bookkeeping (never serialized). These members exist
|
||||||
|
* only so the client command-line parser can record HOW an option was
|
||||||
|
* specified (explicitly set, explicitly negated, or a parser-requested exit
|
||||||
|
* code); no other module and no wire peer ever needs them. Grouping them in
|
||||||
|
* one nested member keeps the public Config free of client-CLI-only state. */
|
||||||
|
typedef struct {
|
||||||
|
/* Set when the user explicitly turned an attribute off with --no-perms /
|
||||||
|
* --no-times (long or short form). --incremental/--delta historically
|
||||||
|
* auto-enabled mode and mtime preservation; these flags let
|
||||||
|
* cli_finalize_config restore that behavior while still honoring the
|
||||||
|
* explicit per-attribute negation. A later -p/-t re-enables the attribute
|
||||||
|
* directly, so the flag only prevents the incremental/delta implication,
|
||||||
|
* never a POSITIVE request. */
|
||||||
|
bool preserve_perms_explicit_off;
|
||||||
|
bool preserve_times_explicit_off;
|
||||||
|
/* Set by --no-preserve, the explicit opt-out of the whole preservation
|
||||||
|
* bundle, so the --incremental/--delta auto-preserve implication stays off. */
|
||||||
|
bool metadata_explicitly_disabled;
|
||||||
|
/* True when --server-port/--port was explicitly given. --dry-run uses it to
|
||||||
|
* decide whether a real server handshake was requested, so a plain local
|
||||||
|
* destination (no explicit port) keeps the existing client-side dry-run
|
||||||
|
* behavior instead of dialing the default 127.0.0.1:8080. */
|
||||||
|
bool server_port_set;
|
||||||
|
/* True when --server-host was explicitly given, and distinct from the
|
||||||
|
* "127.0.0.1" default: --dry-run uses it to route an explicit remote target
|
||||||
|
* to the server so it reports receiver state exactly like a real run,
|
||||||
|
* instead of silently running the client-side manifest. */
|
||||||
|
bool server_host_set;
|
||||||
|
/* Codec-negotiation CLI state. The effective pre-transfer checksum is
|
||||||
|
* Config->checksum_algo (serialized); checksum_transfer_algo is the rsync
|
||||||
|
* "transfer" half of a two-name --checksum-choice form (validated and used
|
||||||
|
* only to mirror rsync's whole-file forcing, since FastSync's per-block
|
||||||
|
* strong hash is fixed). cli_exit_code carries a parser-requested process
|
||||||
|
* exit status (rsync uses 4 for an unsupported checksum/compress algorithm)
|
||||||
|
* so main() can mirror it. */
|
||||||
|
int checksum_transfer_algo;
|
||||||
|
int cli_exit_code;
|
||||||
|
/* "The user explicitly chose" bits. They let the per-codec default level /
|
||||||
|
* checksum list be applied only when the corresponding rsync option was
|
||||||
|
* omitted (an explicit --compress-level / --checksum-choice always wins). */
|
||||||
|
bool compression_level_set;
|
||||||
|
bool checksum_choice_set;
|
||||||
|
/* True when --stop-at was given. */
|
||||||
|
bool stop_at_set;
|
||||||
|
} ConfigCliParse;
|
||||||
|
|
||||||
typedef struct Config {
|
typedef struct Config {
|
||||||
/* -j/--threads=N: number of parallel scanner worker threads for the -m
|
/* -j/--threads=N: number of parallel scanner worker threads for the -m
|
||||||
@@ -265,7 +394,8 @@ typedef struct Config {
|
|||||||
* scanner's built-in default" (4). CLIENT-ONLY: it is a local scheduling
|
* scanner's built-in default" (4). CLIENT-ONLY: it is a local scheduling
|
||||||
* concern and is NEVER serialized into the wire config frame. */
|
* concern and is NEVER serialized into the wire config frame. */
|
||||||
int scanner_threads;
|
int scanner_threads;
|
||||||
bool metadata_explicitly_disabled;
|
/* Client-only CLI-parse bookkeeping (never serialized). See ConfigCliParse. */
|
||||||
|
ConfigCliParse cli;
|
||||||
bool show_progress;
|
bool show_progress;
|
||||||
int compression_threads;
|
int compression_threads;
|
||||||
int ssh_port;
|
int ssh_port;
|
||||||
@@ -286,27 +416,17 @@ typedef struct Config {
|
|||||||
bool use_tls;
|
bool use_tls;
|
||||||
char* server_host;
|
char* server_host;
|
||||||
int server_port;
|
int server_port;
|
||||||
/* True when --server-port/--port was explicitly given. CLIENT-ONLY (never
|
|
||||||
* serialized): --dry-run uses it to decide whether a real server handshake
|
|
||||||
* was requested, so a plain local destination (no explicit port) keeps the
|
|
||||||
* existing client-side dry-run behavior instead of dialing the default
|
|
||||||
* 127.0.0.1:8080. */
|
|
||||||
bool server_port_set;
|
|
||||||
/* True when --server-host was explicitly given. CLIENT-ONLY (never
|
|
||||||
* serialized), and distinct from the "127.0.0.1" default: --dry-run uses it
|
|
||||||
* to route an explicit remote target to the server so it reports receiver
|
|
||||||
* state exactly like a real run, instead of silently running the client-side
|
|
||||||
* manifest. */
|
|
||||||
bool server_host_set;
|
|
||||||
char* tls_cert;
|
char* tls_cert;
|
||||||
char* tls_key;
|
char* tls_key;
|
||||||
char* tls_ca;
|
char* tls_ca;
|
||||||
/* --timeout: per-message I/O deadline in seconds. 0 (the default/unset
|
/* --timeout: per-message I/O deadline in seconds. 0 (rsync's default)
|
||||||
* sentinel) leaves the transport's built-in 30 s socket timeout and the
|
* disables the deadline entirely on the client's own socket and protocol
|
||||||
* protocol's built-in 60 s per-message deadline in place; a positive value
|
* layers; a positive value sets it. A server session never inherits the
|
||||||
* overrides both. See protocol_session_set_io_timeout. */
|
* disabled value: it applies the SERVER_IO_TIMEOUT_SEC floor (see
|
||||||
|
* protocol_server_io_timeout_sec and tcp_set_timeouts). */
|
||||||
int timeout;
|
int timeout;
|
||||||
/* --contimeout: connect()/accept timeout, transport layer only. */
|
/* --contimeout: connect()/accept timeout in seconds (rsync's default 60);
|
||||||
|
* 0 disables it. Transport layer only. */
|
||||||
int contimeout;
|
int contimeout;
|
||||||
bool quiet;
|
bool quiet;
|
||||||
bool stats;
|
bool stats;
|
||||||
@@ -349,9 +469,17 @@ typedef struct Config {
|
|||||||
bool from0; /* -0/--from0: NUL-delimited *-from files */
|
bool from0; /* -0/--from0: NUL-delimited *-from files */
|
||||||
bool cvs_exclude; /* -C/--cvs-exclude: standard CVS ignore set */
|
bool cvs_exclude; /* -C/--cvs-exclude: standard CVS ignore set */
|
||||||
bool per_dir_filter; /* -F: apply per-directory .rsync-filter files */
|
bool per_dir_filter; /* -F: apply per-directory .rsync-filter files */
|
||||||
bool one_file_system; /* -x/--one-file-system: do not cross filesystem boundaries */
|
/* -F click count. rsync's single -F means --filter='dir-merge
|
||||||
/* --no-implied-dirs: client-only. With -R + --files-from, refuse to place a
|
* /.rsync-filter' (the .rsync-filter files themselves are transferred); a
|
||||||
* listed file whose ancestor directory is not itself explicitly listed. */
|
* repeated -F adds --filter='- .rsync-filter' so they are excluded too. */
|
||||||
|
int per_dir_filter_count;
|
||||||
|
int one_file_system; /* -x/--one-file-system: do not cross filesystem boundaries.
|
||||||
|
Repeated -x (rsync's -xx) drops the mount-point
|
||||||
|
directory entirely instead of recreating it empty. */
|
||||||
|
/* --no-implied-dirs: client-only. With -R, do not transfer the source
|
||||||
|
* metadata of the parent directories implied by a listed path; an unlisted
|
||||||
|
* implied parent is still created (with default attributes) so the listed
|
||||||
|
* file can be placed, matching rsync. */
|
||||||
bool no_implied_dirs;
|
bool no_implied_dirs;
|
||||||
/* -d/--dirs: client-only. Transfer the directory entries named by the
|
/* -d/--dirs: client-only. Transfer the directory entries named by the
|
||||||
* source argument / --files-from list without recursing into contents. */
|
* source argument / --files-from list without recursing into contents. */
|
||||||
@@ -368,7 +496,6 @@ typedef struct Config {
|
|||||||
/* --outbuf mode (OutbufMode): stdout/stderr buffering. Client-only launch
|
/* --outbuf mode (OutbufMode): stdout/stderr buffering. Client-only launch
|
||||||
* concern: NEVER crosses the wire. */
|
* concern: NEVER crosses the wire. */
|
||||||
int outbuf;
|
int outbuf;
|
||||||
bool old_args;
|
|
||||||
/* --remote-option=OPT (Phase 5, long form only): one or more extra command-line
|
/* --remote-option=OPT (Phase 5, long form only): one or more extra command-line
|
||||||
* options to append to the REMOTE server invocation over SSH. CLIENT-ONLY:
|
* options to append to the REMOTE server invocation over SSH. CLIENT-ONLY:
|
||||||
* they are composed into the remote command line by ssh_build_remote_command()
|
* they are composed into the remote command line by ssh_build_remote_command()
|
||||||
@@ -438,7 +565,6 @@ typedef struct Config {
|
|||||||
* process and are NEVER serialized into the config frame. */
|
* process and are NEVER serialized into the config frame. */
|
||||||
int stop_after_mins; /* --stop-after=MINS minutes; 0 when unset */
|
int stop_after_mins; /* --stop-after=MINS minutes; 0 when unset */
|
||||||
time_t stop_at; /* --stop-at=... absolute wall-clock deadline */
|
time_t stop_at; /* --stop-at=... absolute wall-clock deadline */
|
||||||
bool stop_at_set; /* true when --stop-at was given */
|
|
||||||
|
|
||||||
/* Client-only residual-batch paths. A residual batch is a self-contained
|
/* Client-only residual-batch paths. A residual batch is a self-contained
|
||||||
* single-file record of the whole source tree (full file images using the
|
* single-file record of the whole source tree (full file images using the
|
||||||
@@ -519,13 +645,21 @@ typedef struct Config {
|
|||||||
/* rsync deletion-timing family (real from Phase 3). At most one of
|
/* rsync deletion-timing family (real from Phase 3). At most one of
|
||||||
delete_before / delete_during / delete_delay / delete_after may be set, and
|
delete_before / delete_during / delete_delay / delete_after may be set, and
|
||||||
only together with use_delete (the CLI implies --delete for each of them).
|
only together with use_delete (the CLI implies --delete for each of them).
|
||||||
delete_before and delete_during select the EARLY engine mode: the keep-set
|
delete_before selects the EARLY engine mode: the whole-tree keep-set
|
||||||
manifest is transmitted before any file data and extras are removed then,
|
manifest is transmitted before any file data and extras are removed then,
|
||||||
acknowledged, before the first data byte. delete_delay and delete_after
|
acknowledged, before the first data byte. delete_during and delete_delay
|
||||||
select the LATE commit mode: extras are removed only after the whole
|
select the per-directory delete-plan mode (protocol 2.24.0): one plan per
|
||||||
transfer has succeeded (plain --delete keeps this mode). The exact
|
source directory is streamed in directory order, and the receiver removes
|
||||||
semantics and the divergences from rsync are documented in RSYNC_COMPAT.md
|
each directory's extras when its plan arrives (during) or snapshots them
|
||||||
and in config_delete_timing_early() below. */
|
and removes them only after a successful transfer (delay). delete_after
|
||||||
|
keeps the whole-tree commit mode: extras are removed from a fresh
|
||||||
|
end-of-transfer destination scan only after the whole transfer succeeded.
|
||||||
|
A plain --delete with no explicit timing flag defaults to delete_during on
|
||||||
|
the client (cli_finalize_config), matching rsync's --del default; the old
|
||||||
|
late-commit behavior is selected explicitly by --delete-after or the
|
||||||
|
FastSync-only long spelling --delete-commit (an exact alias for
|
||||||
|
--delete-after, mapped onto the same wire field). See
|
||||||
|
config_delete_timing_early()/config_delete_timing_per_dir() below. */
|
||||||
/* partial_dir */
|
/* partial_dir */
|
||||||
// PR #174: Partial transfer resumption
|
// PR #174: Partial transfer resumption
|
||||||
/* suffix */
|
/* suffix */
|
||||||
@@ -563,13 +697,17 @@ typedef struct Config {
|
|||||||
* targets and, with -K, follows an in-root destination symlink-to-directory);
|
* targets and, with -K, follows an in-root destination symlink-to-directory);
|
||||||
* -k/--copy-dirlinks is sender-only and is never serialized. */
|
* -k/--copy-dirlinks is sender-only and is never serialized. */
|
||||||
/* numeric_ids */
|
/* numeric_ids */
|
||||||
/* --numeric-ids: no name lookup, use the transmitted numeric ids raw. */
|
/* --numeric-ids: a mapping MODIFIER only -- no name lookup, use the
|
||||||
|
* transmitted numeric ids raw. It does NOT by itself request ownership. */
|
||||||
/* chown_uid_set */
|
/* chown_uid_set */
|
||||||
/* --chown USER (owner) override; IDENTITY_CURRENT = the receiver's euid. */
|
/* --chown USER (owner) override; IDENTITY_CURRENT = the receiver's euid. */
|
||||||
/* chown_gid_set */
|
/* chown_gid_set */
|
||||||
/* --chown :GROUP (group) override; IDENTITY_CURRENT = the receiver's egid. */
|
/* --chown :GROUP (group) override; IDENTITY_CURRENT = the receiver's egid. */
|
||||||
/* usermap */
|
/* usermap */
|
||||||
/* --usermap / --groupmap entries, in order (first match wins). */
|
/* --usermap / --groupmap entries, in order (first match wins). Each entry's
|
||||||
|
* from/from_hi are a single id, an inclusive range, IDENTITY_MATCH_ANY ('*'),
|
||||||
|
* or IDENTITY_MATCH_UNNAMED (empty FROM); to_name carries a receiver-resolved
|
||||||
|
* TO name (rsync resolves TO names on the receiving side). */
|
||||||
/* preserve_atimes */
|
/* preserve_atimes */
|
||||||
/* -U/--atimes: preserve source access times on the destination. */
|
/* -U/--atimes: preserve source access times on the destination. */
|
||||||
/* preserve_crtimes */
|
/* preserve_crtimes */
|
||||||
@@ -579,10 +717,31 @@ typedef struct Config {
|
|||||||
/* -O/--omit-dir-times: do not apply mtimes to directories. */
|
/* -O/--omit-dir-times: do not apply mtimes to directories. */
|
||||||
/* omit_link_times */
|
/* omit_link_times */
|
||||||
/* -J/--omit-link-times: do not apply times to symlinks. */
|
/* -J/--omit-link-times: do not apply times to symlinks. */
|
||||||
|
/* preserve_perms */
|
||||||
|
/* -p/--perms: preserve the source permission bits (mode). One of the four
|
||||||
|
* per-attribute preservation flags split out of the former single
|
||||||
|
* use_metadata bundle; --chmod and -A/--acls also imply it. */
|
||||||
|
/* preserve_times */
|
||||||
|
/* -t/--times: preserve source modification times. Split out of the former
|
||||||
|
* use_metadata bundle; --preserve and -a/--archive imply it. */
|
||||||
|
/* preserve_owner */
|
||||||
|
/* -o/--owner: preserve the source owner (uid). Split out of the former
|
||||||
|
* use_metadata bundle; --usermap/--chown (and, when a uid is requested,
|
||||||
|
* --copy-as) imply it. Owner application still requires receiver privilege
|
||||||
|
* and is gated separately by the identity flags. */
|
||||||
|
/* preserve_group */
|
||||||
|
/* -g/--group: preserve the source group (gid). Split out of the former
|
||||||
|
* use_metadata bundle; --groupmap/--chown (and, when a gid is requested,
|
||||||
|
* --copy-as) imply it. */
|
||||||
/* fake_super */
|
/* fake_super */
|
||||||
/* --fake-super: receiver-only. When set, each written file additionally gets
|
/* --fake-super: receiver-only. When set, each written file additionally gets
|
||||||
* a reserved user.fastsync.stat xattr recording the source uid/gid/mode/mtime
|
* rsync's reserved user.rsync.%stat xattr recording the RESOLVED uid/gid (the
|
||||||
* so a later privileged restore could re-apply them. Crosses the wire. */
|
* source's own when no ownership request is active, else the --chown/--usermap
|
||||||
|
* result) plus the full mode and rdev, in rsync 3.4.1's grammar, so the tree is
|
||||||
|
* interoperable and a later privileged restore could re-apply them. mtime is
|
||||||
|
* carried by the file's own timestamp, exactly as rsync does it. It NEVER
|
||||||
|
* real-chowns: the point is to record the source ownership on an unprivileged
|
||||||
|
* receiver. Crosses the wire. */
|
||||||
/* module */
|
/* module */
|
||||||
/* Daemon module selection (Wave A, protocol 2.15.0). Client-composed from a
|
/* Daemon module selection (Wave A, protocol 2.15.0). Client-composed from a
|
||||||
* host::module/path destination; NULL or "" means "no module" (the ordinary
|
* host::module/path destination; NULL or "" means "no module" (the ordinary
|
||||||
@@ -623,7 +782,7 @@ typedef struct Config {
|
|||||||
* fd-relative confinement (file_open_secure_parent, O_NOFOLLOW, root checks);
|
* fd-relative confinement (file_open_secure_parent, O_NOFOLLOW, root checks);
|
||||||
* --super only permits an attempt that is already confined. Crosses the wire
|
* --super only permits an attempt that is already confined. Crosses the wire
|
||||||
* as a trailing int so the receiver can enforce the policy. See
|
* as a trailing int so the receiver can enforce the policy. See
|
||||||
* privilege_super_permitted() and identity_ownership_requested() in
|
* privilege_super_permitted() and identity_explicit_ownership_requested() in
|
||||||
* identity.h. */
|
* identity.h. */
|
||||||
/* copy_as_set */
|
/* copy_as_set */
|
||||||
/* --copy-as=USER[:GROUP] (P7 Wave E, protocol 2.18.0). Safe-subset
|
/* --copy-as=USER[:GROUP] (P7 Wave E, protocol 2.18.0). Safe-subset
|
||||||
@@ -803,12 +962,148 @@ typedef struct Config {
|
|||||||
* unknown status, or the unconsumed detail body, and the strict same-version
|
* unknown status, or the unconsumed detail body, and the strict same-version
|
||||||
* handshake (config_receive rejects a mismatched version before parsing
|
* handshake (config_receive rejects a mismatched version before parsing
|
||||||
* anything else) is what keeps a 2.21 client and a 2.20 server from ever
|
* anything else) is what keeps a 2.21 client and a 2.20 server from ever
|
||||||
* reaching that state. */
|
* reaching that state.
|
||||||
#define PROTOCOL_VERSION "2.21.0"
|
*
|
||||||
|
* Preserve-Attribute Split Wave: 2.21.0 -> 2.22.0.
|
||||||
|
*
|
||||||
|
* WHY the bump, grounded in the wire: this wave splits the former single
|
||||||
|
* use_metadata bundle into four independent rsync-compatible preservation
|
||||||
|
* attributes (preserve_perms / preserve_times / preserve_owner /
|
||||||
|
* preserve_group) so -p/-t/-o/-g (and their --no-* negations) become real
|
||||||
|
* drop-in flags. The binary config frame gains four serialized bools appended
|
||||||
|
* to CONFIG_WIRE_METADATA_TIMES_FIELDS after omit_link_times, in this fixed
|
||||||
|
* order: preserve_perms, preserve_times, preserve_owner, preserve_group. Any
|
||||||
|
* config-frame layout change must bump the protocol version: a peer that does
|
||||||
|
* not parse the new trailing bytes would desynchronize on the frame boundary,
|
||||||
|
* and the strict same-version handshake (config_receive rejects a mismatched
|
||||||
|
* version before parsing anything else) is what keeps a 2.22 client and a 2.21
|
||||||
|
* server from ever reaching that state. The fixed-width FileMetadata layout is
|
||||||
|
* UNCHANGED: the receiver still gates attribute application on use_metadata,
|
||||||
|
* which is now DERIVED from these attributes by config_derived_use_metadata().
|
||||||
|
*
|
||||||
|
* Rsync-Parity Wave: 2.22.0 -> 2.23.0.
|
||||||
|
*
|
||||||
|
* WHY the bump, grounded in the wire. Several independent changes land in this
|
||||||
|
* protocol version:
|
||||||
|
*
|
||||||
|
* (1) Ownership parity (#286/#294): each --usermap/--groupmap wire entry grows
|
||||||
|
* from two int32s to [from][from_hi][to][to_name]; `from_hi` carries an
|
||||||
|
* inclusive LOW-HIGH range (== from for a single/any/unnamed matcher) and the
|
||||||
|
* trailing string carries a TO NAME for the receiver to resolve (rsync resolves
|
||||||
|
* TO names on the receiving side). The STATUS_MKDIR and STATUS_DIR_TIMES frames
|
||||||
|
* also gain a bounded per-entry xattr block when -X/-A is negotiated, so
|
||||||
|
* directory xattrs/ACLs (including default ACLs) are preserved like regular-file
|
||||||
|
* xattrs.
|
||||||
|
*
|
||||||
|
* (2) Delete semantics (#290): the delete-manifest frame gains a fourth trailing
|
||||||
|
* section -- a synchronized-directory count followed by that many
|
||||||
|
* destination-relative directory paths (the receive root is "."). The receiver
|
||||||
|
* confines its extras walk to these directories, so `--files-from` with
|
||||||
|
* `--delete` only removes inside listed directory subtrees (rsync parity)
|
||||||
|
* instead of deleting every untransmitted path under the receive root. The
|
||||||
|
* frame stream also gains STATUS_DELETE_LIMIT, the terminal success status sent
|
||||||
|
* instead of STATUS_OK when a --max-delete commit removes up to the bound and
|
||||||
|
* skips the rest (the sender then exits 25 like rsync).
|
||||||
|
*
|
||||||
|
* Any config-frame layout or frame-sequence change must bump the protocol
|
||||||
|
* version: a 2.22 peer would desynchronize on the new entry bytes, the extra
|
||||||
|
* trailing section or the unknown status, and the strict same-version handshake
|
||||||
|
* (config_receive rejects a mismatched version before parsing anything else) is
|
||||||
|
* what keeps a 2.23 client and a 2.22 server from ever reaching that state.
|
||||||
|
*
|
||||||
|
* (3) Output parity (#291/#292): -i/--itemize-changes and --out-format must
|
||||||
|
* compare the source against the PRE-TRANSFER destination entry (new vs
|
||||||
|
* modified, and which of size/time/perms/owner/group differ), but FastSync's
|
||||||
|
* push sender never sees the destination. The receiver therefore answers a
|
||||||
|
* per-file STATUS_CHECK with a new STATUS_DEST_INFO frame (a fixed-width
|
||||||
|
* snapshot of the old entry) before its ordinary verdict when the config frame
|
||||||
|
* carries the new report_dest_info bool appended after the --copy-as block.
|
||||||
|
* This is both a config-frame layout change (one trailing bool) and a frame
|
||||||
|
* sequence change (the new status).
|
||||||
|
*
|
||||||
|
* (4) Delete timing (protocol 2.24.0): the sender streams one delete plan per
|
||||||
|
* source directory so --delete-during/--delete-delay reproduce rsync's deletion
|
||||||
|
* timing (the plan fields and STATUS_DELETE_PLAN are documented at the keep-set
|
||||||
|
* / delete-plan definitions below).
|
||||||
|
*
|
||||||
|
* (5) Wire-stats parity (protocol 2.25.0): --stats, --progress/-P and the
|
||||||
|
* --out-format %b/%c tokens need receiver-only and wire counters that the push
|
||||||
|
* sender cannot observe, and -n/--dry-run --delete must report the extras it
|
||||||
|
* would have removed without deleting anything. The config frame gains one
|
||||||
|
* trailing report_stats bool and the receiver emits a new STATUS_STATS frame
|
||||||
|
* (carrying matched data, the deleted-file count and the would-delete path
|
||||||
|
* list) immediately before its terminal success status.
|
||||||
|
*
|
||||||
|
* (6) Codec breadth + negotiation (protocol 2.26.0): the config frame gains one
|
||||||
|
* trailing int, compression_algo (a CompressionAlgo id), appended after the
|
||||||
|
* output block. It is the negotiated/effective compression codec and is what
|
||||||
|
* the receiver's self-describing decompressor validates against its own
|
||||||
|
* supported set. The checksum_algo wire value now also accepts md4/sha1/none,
|
||||||
|
* and its default changes to the rsync 3.4.1 auto-negotiated xxh128.
|
||||||
|
*
|
||||||
|
* Any config-frame layout change must bump the protocol version: a peer that
|
||||||
|
* does not parse the new trailing bytes would desynchronize on the frame
|
||||||
|
* boundary, and the strict same-version handshake (config_receive rejects a
|
||||||
|
* mismatched version before parsing anything else) keeps mixed deployments from
|
||||||
|
* ever reaching that state. */
|
||||||
|
/* (7) --info=del report (protocol 2.27.0): the config frame gains one trailing
|
||||||
|
* bool, report_deletes, appended after report_stats. When set, the receiver
|
||||||
|
* lists the paths it actually removed in the terminal STATUS_STATS path list
|
||||||
|
* (the same count-delimited list the -n/--dry-run would-delete report uses), so
|
||||||
|
* the sender can print rsync's `deleting PATH` lines for a real deletion. No
|
||||||
|
* change to the fixed STATUS_STATS record itself; only a new trailing config
|
||||||
|
* bool, which still requires the version bump for the strict lockstep. */
|
||||||
|
/* (8) --stats receiver-observed counters (protocol 2.28.0): the fixed
|
||||||
|
* STATUS_STATS record grows from three counters to eight. The receiver now
|
||||||
|
* reports the bytes it literally stored (`literal_data`) and the count of
|
||||||
|
* destination entries it newly CREATED, split by type
|
||||||
|
* (reg/dir/link/special), so the sender can print rsync's exact
|
||||||
|
* `Number of created files: N (reg: X, dir: Y, link: Z, special: W)` line and
|
||||||
|
* an exact `Literal data` total even for delta transfers. The config-frame
|
||||||
|
* LAYOUT is unchanged (no new config field), but the STATUS_STATS body grows,
|
||||||
|
* so a 2.27 peer that does not consume the five new fixed-width counters would
|
||||||
|
* desynchronize on the trailing would-delete path list; the strict
|
||||||
|
* same-version handshake (config_receive rejects a mismatched version before
|
||||||
|
* parsing anything else) keeps mixed deployments from ever reaching that
|
||||||
|
* state. */
|
||||||
|
/* (9) Receiver-side delete protection (still protocol 2.28.0): the config frame
|
||||||
|
* gains one trailing self-describing block carrying the sender's compiled base
|
||||||
|
* filter rules so the receiver can protect DESTINATION-ONLY entries from
|
||||||
|
* --delete with `protect`/`risk` rules (rsync parity). The block appends after
|
||||||
|
* compression_algo; see CONFIG_WIRE_PROTECT_FIELDS. */
|
||||||
|
/* (10) Symlink xattrs/ACLs (protocol 2.29.0): the config-frame LAYOUT is
|
||||||
|
* unchanged (the derived use_xattrs bit already crosses the wire), but the
|
||||||
|
* STATUS_SYMLINK frame BODY grows a trailing bounded xattr block when -X/-A is
|
||||||
|
* negotiated -- exactly the block STATUS_MKDIR, STATUS_DIR_TIMES and regular
|
||||||
|
* files already carry. The sender captures the symlink's OWN xattrs with
|
||||||
|
* llistxattr/lgetxattr (so it can never attach the REFERENT's attributes to the
|
||||||
|
* link) and the receiver re-applies them to the link itself with lsetxattr on a
|
||||||
|
* confined /proc/self/fd/<parent>/<leaf> path (there is no *at xattr syscall and
|
||||||
|
* fsetxattr cannot target a symlink). A 2.28 peer that does not consume the new
|
||||||
|
* trailing block would desynchronize after every symlink, so the protocol
|
||||||
|
* version must bump; the strict same-version handshake (config_receive rejects a
|
||||||
|
* mismatched version before parsing anything else) keeps a 2.29 client and a
|
||||||
|
* 2.28 server from ever reaching that state. */
|
||||||
|
#define PROTOCOL_VERSION "2.29.0"
|
||||||
#define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024)
|
#define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024)
|
||||||
/* Upper bound on total basis-dir entries (rsync caps --link-dest at 20). */
|
/* Upper bound on total basis-dir entries (rsync caps --link-dest at 20). */
|
||||||
#define MAX_BASIS_DIRS 64
|
#define MAX_BASIS_DIRS 64
|
||||||
|
|
||||||
|
/* Bounds on the received receiver-side delete-protection rule block. The rule
|
||||||
|
* count and the aggregate pattern+owner bytes are each capped so a hostile
|
||||||
|
* peer cannot pin unbounded pre-auth memory; both are validated strictly on
|
||||||
|
* receive (alongside the per-string ConfigStringBudget). */
|
||||||
|
/* A peer may supply protect rules; cap the list so a crafted config cannot make
|
||||||
|
* the receiver's delete walk evaluate an unbounded number of glob patterns per
|
||||||
|
* destination entry (glob_match is O(pattern x path)). 1024 is far above any
|
||||||
|
* legitimate selection. */
|
||||||
|
#define MAX_FILTER_RULES 1024
|
||||||
|
#define MAX_FILTER_BYTES (256 * 1024)
|
||||||
|
/* glob_match's DP is capped at 64 Mi work units; a pattern longer than this
|
||||||
|
* could exceed the cap against a PATH_MAX path and silently stop matching,
|
||||||
|
* leaving a protect rule inert. Reject such a rule at receive time. */
|
||||||
|
#define MAX_PROTECT_PATTERN_LEN 8192
|
||||||
|
|
||||||
/* Upper bound on the number of --skip-compress suffixes accepted from the wire.
|
/* Upper bound on the number of --skip-compress suffixes accepted from the wire.
|
||||||
* Each suffix is an independent wire string (up to MAX_STRING_SIZE = 64 KiB), so
|
* Each suffix is an independent wire string (up to MAX_STRING_SIZE = 64 KiB), so
|
||||||
* without this a hostile pre-auth client could otherwise retain
|
* without this a hostile pre-auth client could otherwise retain
|
||||||
@@ -829,9 +1124,11 @@ typedef struct Config {
|
|||||||
|
|
||||||
/* Identity-mapping sentinels and bounds (see identity.h for semantics).
|
/* Identity-mapping sentinels and bounds (see identity.h for semantics).
|
||||||
* IDENTITY_MATCH_ANY is a usermap/groupmap FROM '*' (matches any id);
|
* IDENTITY_MATCH_ANY is a usermap/groupmap FROM '*' (matches any id);
|
||||||
* IDENTITY_CURRENT is a chown / map TO '*' (resolve to the receiver's current
|
* IDENTITY_MATCH_UNNAMED is a FROM with an empty token (rsync's "ids with no
|
||||||
* euid/egid at apply time). */
|
* name on the sender"); IDENTITY_CURRENT is a chown / map TO '*' (resolve to
|
||||||
|
* the receiver's current euid/egid at apply time). */
|
||||||
#define IDENTITY_MATCH_ANY (-1)
|
#define IDENTITY_MATCH_ANY (-1)
|
||||||
|
#define IDENTITY_MATCH_UNNAMED (-2)
|
||||||
#define IDENTITY_CURRENT (-1)
|
#define IDENTITY_CURRENT (-1)
|
||||||
#define MAX_IDENTITY_MAP 128
|
#define MAX_IDENTITY_MAP 128
|
||||||
|
|
||||||
@@ -896,16 +1193,23 @@ int config_parse_daemon_dest(Config* config);
|
|||||||
* 0. */
|
* 0. */
|
||||||
int config_parse_transport_dest(Config* config);
|
int config_parse_transport_dest(Config* config);
|
||||||
|
|
||||||
/* True when the negotiated delete timing performs the extra-file deletion
|
/* True for the whole-tree delete-before timing: a complete keep-set manifest is
|
||||||
* BEFORE the transfer data (--delete-before / --delete-during). The flag is
|
* transmitted before any data and committed (with an ack) before the first data
|
||||||
* a pure function of the config and is used identically on the sender (to pick
|
* byte. Pure function of the config, used identically on the sender (to pick
|
||||||
* the manifest-first frame order) and the receiver (to delete when the early
|
* the manifest-first frame order) and the receiver (to delete when the early
|
||||||
* manifest arrives). When false the deletion is committed only after the whole
|
* manifest arrives). */
|
||||||
* transfer succeeded (--delete / --delete-after / --delete-delay). */
|
|
||||||
bool config_delete_timing_early(const Config* config);
|
bool config_delete_timing_early(const Config* config);
|
||||||
|
/* True for the per-directory timings (--delete-during / --delete-delay). The
|
||||||
|
* sender streams a delete plan per source directory in directory order; the
|
||||||
|
* receiver applies each plan on arrival (during) or snapshots its extras and
|
||||||
|
* commits them only after a fully-successful transfer (delay). */
|
||||||
|
bool config_delete_timing_per_dir(const Config* config);
|
||||||
/* Delete-timing sanity: with deletion enabled at most one timing flag may be
|
/* Delete-timing sanity: with deletion enabled at most one timing flag may be
|
||||||
* set (none = the default delete-after commit timing); without deletion no
|
* set; without deletion no timing flag may be set (each timing flag implies
|
||||||
* timing flag may be set (each timing flag implies --delete). */
|
* --delete). A plain --delete is normalized to delete_during by
|
||||||
|
* cli_finalize_config on the client, so a transmitted use_delete config always
|
||||||
|
* carries exactly one timing; the zero-timing case remains valid only for a
|
||||||
|
* config that has not been through the CLI. */
|
||||||
bool config_has_valid_delete_timing(const Config* config);
|
bool config_has_valid_delete_timing(const Config* config);
|
||||||
|
|
||||||
/* Single source of truth for the cross-field ("combination") invariants a
|
/* Single source of truth for the cross-field ("combination") invariants a
|
||||||
@@ -918,6 +1222,14 @@ bool config_has_valid_delete_timing(const Config* config);
|
|||||||
* validate_received_config() so the receiver enforces exactly the same
|
* validate_received_config() so the receiver enforces exactly the same
|
||||||
* invariants it relies on (the server is the trust boundary). */
|
* invariants it relies on (the server is the trust boundary). */
|
||||||
const char* config_invariants_error(const Config* config);
|
const char* config_invariants_error(const Config* config);
|
||||||
|
/* Single source of truth for the DERIVED transport bit (use_metadata): true
|
||||||
|
* when any configured preservation/ownership option requires the metadata
|
||||||
|
* frame to travel. Returns false when no such option is set (a bare run).
|
||||||
|
* This is a pure predicate over the config; the client lowers it into
|
||||||
|
* Config->use_metadata at the end of parsing so every implication (devices,
|
||||||
|
* executability, identity maps, incremental/delta, ...) is centralized here
|
||||||
|
* rather than scattered as direct writes. */
|
||||||
|
bool config_derived_use_metadata(const Config* config);
|
||||||
/* True when at least one --compare-dest/--copy-dest/--link-dest was set. */
|
/* True when at least one --compare-dest/--copy-dest/--link-dest was set. */
|
||||||
bool config_has_basis(const Config* config);
|
bool config_has_basis(const Config* config);
|
||||||
/* Append one basis-dir entry. Returns 0 on success, -1 on allocation failure. */
|
/* Append one basis-dir entry. Returns 0 on success, -1 on allocation failure. */
|
||||||
|
|||||||
+204
-18
@@ -8,12 +8,13 @@
|
|||||||
#include <openssl/evp.h>
|
#include <openssl/evp.h>
|
||||||
#include <openssl/params.h>
|
#include <openssl/params.h>
|
||||||
#include <openssl/rand.h>
|
#include <openssl/rand.h>
|
||||||
#include <stdarg.h>
|
#include <poll.h>
|
||||||
#include <stdint.h>
|
#include <stdint.h>
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
#include <sys/stat.h>
|
#include <sys/stat.h>
|
||||||
|
#include <time.h>
|
||||||
#include <unistd.h>
|
#include <unistd.h>
|
||||||
|
|
||||||
/* One store entry: a username and its salted PBKDF2 verifier. The plaintext
|
/* One store entry: a username and its salted PBKDF2 verifier. The plaintext
|
||||||
@@ -58,19 +59,37 @@ struct CredentialStore {
|
|||||||
static const uint8_t k_dummy_stored_key[CREDENTIAL_KEY_LEN] = {0};
|
static const uint8_t k_dummy_stored_key[CREDENTIAL_KEY_LEN] = {0};
|
||||||
static const uint8_t k_dummy_server_key[CREDENTIAL_KEY_LEN] = {0};
|
static const uint8_t k_dummy_server_key[CREDENTIAL_KEY_LEN] = {0};
|
||||||
|
|
||||||
static void set_error(char* err, size_t err_size, const char* fmt, ...) {
|
#define set_error utils_set_error
|
||||||
if (!err || err_size == 0)
|
|
||||||
return;
|
|
||||||
va_list args;
|
|
||||||
va_start(args, fmt);
|
|
||||||
vsnprintf(err, err_size, fmt, args);
|
|
||||||
va_end(args);
|
|
||||||
}
|
|
||||||
|
|
||||||
static bool is_comment_char(char c) {
|
static bool is_comment_char(char c) {
|
||||||
return c == '#' || c == ';';
|
return c == '#' || c == ';';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* True for a literal fd-backed store path: exactly "/dev/fd/<digits>" or
|
||||||
|
* "/proc/self/fd/<digits>", with no trailing component and no "..". These name
|
||||||
|
* the calling process's own open descriptors (e.g. a bash process substitution
|
||||||
|
* `<(...)`, which passes /dev/fd/N), and both prefixes are symlinks by
|
||||||
|
* construction. */
|
||||||
|
static bool is_fd_backed_path(const char* path) {
|
||||||
|
static const char* const prefixes[] = {"/dev/fd/", "/proc/self/fd/"};
|
||||||
|
if (!path)
|
||||||
|
return false;
|
||||||
|
for (size_t i = 0; i < sizeof(prefixes) / sizeof(prefixes[0]); i++) {
|
||||||
|
const char* prefix = prefixes[i];
|
||||||
|
size_t prefix_len = strlen(prefix);
|
||||||
|
if (strncmp(path, prefix, prefix_len) != 0)
|
||||||
|
continue;
|
||||||
|
const char* digits = path + prefix_len;
|
||||||
|
if (*digits < '0' || *digits > '9')
|
||||||
|
return false;
|
||||||
|
const char* p = digits;
|
||||||
|
while (*p >= '0' && *p <= '9')
|
||||||
|
p++;
|
||||||
|
return *p == '\0';
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
/* Open a --password-file / --early-input after verifying the EXACT inode we
|
/* Open a --password-file / --early-input after verifying the EXACT inode we
|
||||||
* will read: it must be owned by the effective user and grant no group/other
|
* will read: it must be owned by the effective user and grant no group/other
|
||||||
* permission bit (so 0600 and stricter modes such as 0400 are accepted),
|
* permission bit (so 0600 and stricter modes such as 0400 are accepted),
|
||||||
@@ -80,12 +99,31 @@ static bool is_comment_char(char c) {
|
|||||||
* path and then fstat the resulting fd (rather than stat()ing the path first
|
* path and then fstat the resulting fd (rather than stat()ing the path first
|
||||||
* and reopening it), so the permission decision is made on the same inode that
|
* and reopening it), so the permission decision is made on the same inode that
|
||||||
* is read and cannot be raced by swapping the path between check and open.
|
* is read and cannot be raced by swapping the path between check and open.
|
||||||
* The path may be a process-substitution pipe (`<(...)` -> /dev/fd/N), so
|
* O_NOFOLLOW refuses a symlinked path outright (ELOOP fails closed) instead of
|
||||||
* regular files and FIFOs are accepted when the ownership/mode checks pass.
|
* following it before the owner/mode gate can run. The one exception is a
|
||||||
|
* literal fd-backed path (/dev/fd/N or /proc/self/fd/N, see
|
||||||
|
* is_fd_backed_path): those entries are symlinks to the CALLING process's own
|
||||||
|
* descriptors, so following them is not the untrusted-symlink hazard
|
||||||
|
* O_NOFOLLOW guards against, and requiring O_NOFOLLOW would break the
|
||||||
|
* documented process-substitution/FIFO usage. For them only, O_NOFOLLOW is
|
||||||
|
* omitted; the same fstat owner/mode gate still applies to the resolved inode.
|
||||||
|
* O_NONBLOCK keeps the OPEN itself from
|
||||||
|
* blocking forever on a writer-less FIFO (a blocking O_RDONLY open would wait
|
||||||
|
* for a writer). The fd is left nonblocking for FIFOs so a read never blocks
|
||||||
|
* either; the read loop (secret_read_line) absorbs the resulting EAGAIN by
|
||||||
|
* waiting, under a bounded deadline, for the writer -- this is what makes a
|
||||||
|
* slow process substitution (`--password-file <(sleep 1; ...)`) work while a
|
||||||
|
* writer-less FIFO still fails after the deadline instead of hanging. Only
|
||||||
|
* regular files and FIFOs pass the ownership/mode checks; O_NONBLOCK is
|
||||||
|
* cleared for regular files, where it is a no-op anyway and no EAGAIN can
|
||||||
|
* occur, so their stdio read path is byte-for-byte unchanged.
|
||||||
*
|
*
|
||||||
* Returns a FILE* the caller must fclose, or NULL with `err` filled. */
|
* Returns a FILE* the caller must fclose, or NULL with `err` filled. */
|
||||||
static FILE* secret_file_open(const char* path, char* err, size_t err_size) {
|
static FILE* secret_file_open(const char* path, char* err, size_t err_size) {
|
||||||
int fd = open(path, O_RDONLY | O_CLOEXEC);
|
int flags = O_RDONLY | O_NONBLOCK | O_CLOEXEC;
|
||||||
|
if (!is_fd_backed_path(path))
|
||||||
|
flags |= O_NOFOLLOW;
|
||||||
|
int fd = open(path, flags);
|
||||||
if (fd < 0) {
|
if (fd < 0) {
|
||||||
set_error(err, err_size, "cannot open secret file '%s': %s", path, strerror(errno));
|
set_error(err, err_size, "cannot open secret file '%s': %s", path, strerror(errno));
|
||||||
return NULL;
|
return NULL;
|
||||||
@@ -105,6 +143,15 @@ static FILE* secret_file_open(const char* path, char* err, size_t err_size) {
|
|||||||
close(fd);
|
close(fd);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
/* O_NONBLOCK is only meaningful for the FIFO allowance. Restore blocking
|
||||||
|
* mode on a regular file so its read path is exactly as before; a no-op on
|
||||||
|
* most systems, but explicit. Failures here are ignored: O_NONBLOCK on a
|
||||||
|
* regular file does not affect reads either way. */
|
||||||
|
if (S_ISREG(st.st_mode)) {
|
||||||
|
int status_flags = fcntl(fd, F_GETFL);
|
||||||
|
if (status_flags >= 0)
|
||||||
|
(void)fcntl(fd, F_SETFL, status_flags & ~O_NONBLOCK);
|
||||||
|
}
|
||||||
FILE* fp = fdopen(fd, "r");
|
FILE* fp = fdopen(fd, "r");
|
||||||
if (!fp) {
|
if (!fp) {
|
||||||
set_error(err, err_size, "cannot read secret file '%s': %s", path, strerror(errno));
|
set_error(err, err_size, "cannot read secret file '%s': %s", path, strerror(errno));
|
||||||
@@ -114,6 +161,123 @@ static FILE* secret_file_open(const char* path, char* err, size_t err_size) {
|
|||||||
return fp;
|
return fp;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Overall bound on how long the reader waits for a process-substitution/FIFO
|
||||||
|
* writer to produce data before giving up. It must comfortably exceed a
|
||||||
|
* producer's startup delay (e.g. `--password-file <(sleep 1; ...)`) while still
|
||||||
|
* bounding a writer-less FIFO, so a stray or hostile FIFO cannot stall the
|
||||||
|
* daemon or client indefinitely. */
|
||||||
|
#define CREDENTIAL_FIFO_READ_TIMEOUT_MS 3000
|
||||||
|
|
||||||
|
/* Monotonic milliseconds, used only for the read deadline (wall-clock changes
|
||||||
|
* must not extend or shorten the wait). */
|
||||||
|
static int64_t credential_monotonic_ms(void) {
|
||||||
|
struct timespec ts;
|
||||||
|
if (clock_gettime(CLOCK_MONOTONIC, &ts) != 0)
|
||||||
|
return 0;
|
||||||
|
return (int64_t)ts.tv_sec * 1000 + (int64_t)(ts.tv_nsec / 1000000);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Wait until `fd` is readable or the deadline passes. Returns true when it is
|
||||||
|
* readable, false on timeout or a poll error (err filled). EINTR is retried
|
||||||
|
* against the same deadline, so signals cannot extend the wait. */
|
||||||
|
static bool credential_wait_readable(int fd, int64_t deadline, const char* label, const char* path,
|
||||||
|
char* err, size_t err_size) {
|
||||||
|
for (;;) {
|
||||||
|
int64_t remaining = deadline - credential_monotonic_ms();
|
||||||
|
if (remaining <= 0)
|
||||||
|
break;
|
||||||
|
if (remaining > INT_MAX)
|
||||||
|
remaining = INT_MAX;
|
||||||
|
struct pollfd pfd = {.fd = fd, .events = POLLIN, .revents = 0};
|
||||||
|
int rc = poll(&pfd, 1, (int)remaining);
|
||||||
|
if (rc > 0)
|
||||||
|
return true;
|
||||||
|
if (rc == 0)
|
||||||
|
break;
|
||||||
|
if (errno != EINTR) {
|
||||||
|
set_error(err, err_size, "error waiting for %s '%s': %s", label, path, strerror(errno));
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
set_error(err, err_size, "timed out after %d ms waiting for %s '%s'",
|
||||||
|
CREDENTIAL_FIFO_READ_TIMEOUT_MS, label, path);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
typedef enum {
|
||||||
|
SECRET_READ_LINE,
|
||||||
|
SECRET_READ_EOF,
|
||||||
|
SECRET_READ_ERROR,
|
||||||
|
} SecretReadResult;
|
||||||
|
|
||||||
|
/* Read one complete line from `fp` into `line` (capacity `cap`), including the
|
||||||
|
* trailing newline when present and always NUL-terminating. `*out_len`
|
||||||
|
* receives strlen(line).
|
||||||
|
*
|
||||||
|
* A regular file is read exactly as before: secret_file_open leaves it
|
||||||
|
* blocking, so fgets never sees EAGAIN. A FIFO stays nonblocking, so fgets
|
||||||
|
* returns NULL (or a partial line) with EAGAIN while the writer is still
|
||||||
|
* starting up; instead of treating that as a fatal error the loop clearerr()s
|
||||||
|
* and polls for readability against one overall deadline. The `used`
|
||||||
|
* accumulator reassembles a line that arrived in several write()s into a single
|
||||||
|
* line, so a split write is not misparsed as two entries.
|
||||||
|
*
|
||||||
|
* Returns SECRET_READ_LINE, SECRET_READ_EOF, or SECRET_READ_ERROR (err filled)
|
||||||
|
* on timeout or a genuine read error. */
|
||||||
|
static SecretReadResult secret_read_line(char* line, size_t cap, FILE* fp, const char* label,
|
||||||
|
const char* path, size_t* out_len, char* err,
|
||||||
|
size_t err_size) {
|
||||||
|
int fd = fileno(fp);
|
||||||
|
int64_t deadline = credential_monotonic_ms() + CREDENTIAL_FIFO_READ_TIMEOUT_MS;
|
||||||
|
size_t used = 0;
|
||||||
|
line[0] = '\0';
|
||||||
|
for (;;) {
|
||||||
|
errno = 0;
|
||||||
|
if (fgets(line + used, (int)(cap - used), fp)) {
|
||||||
|
used += strlen(line + used);
|
||||||
|
if (used > 0 && line[used - 1] == '\n') {
|
||||||
|
*out_len = used;
|
||||||
|
return SECRET_READ_LINE;
|
||||||
|
}
|
||||||
|
if (feof(fp)) {
|
||||||
|
*out_len = used; /* final unterminated line */
|
||||||
|
return SECRET_READ_LINE;
|
||||||
|
}
|
||||||
|
/* No newline and not EOF. A full buffer is the caller's over-long-line
|
||||||
|
* case; otherwise the line is only partially available (a nonblocking
|
||||||
|
* FIFO under a slow writer), so any genuine read error fails and anything
|
||||||
|
* else waits for the rest. */
|
||||||
|
if (used >= cap - 1) {
|
||||||
|
*out_len = used;
|
||||||
|
return SECRET_READ_LINE;
|
||||||
|
}
|
||||||
|
int e = ferror(fp) ? errno : 0;
|
||||||
|
if (e != 0 && e != EAGAIN && e != EWOULDBLOCK) {
|
||||||
|
set_error(err, err_size, "error reading %s '%s': %s", label, path, strerror(e));
|
||||||
|
return SECRET_READ_ERROR;
|
||||||
|
}
|
||||||
|
clearerr(fp);
|
||||||
|
if (!credential_wait_readable(fd, deadline, label, path, err, err_size))
|
||||||
|
return SECRET_READ_ERROR;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
/* fgets returned NULL: EOF, a not-yet-readable FIFO, or a real error. */
|
||||||
|
if (feof(fp)) {
|
||||||
|
*out_len = used;
|
||||||
|
return used > 0 ? SECRET_READ_LINE : SECRET_READ_EOF;
|
||||||
|
}
|
||||||
|
if (errno == EAGAIN || errno == EWOULDBLOCK) {
|
||||||
|
clearerr(fp);
|
||||||
|
if (!credential_wait_readable(fd, deadline, label, path, err, err_size))
|
||||||
|
return SECRET_READ_ERROR;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
set_error(err, err_size, "error reading %s '%s': %s", label, path,
|
||||||
|
errno != 0 ? strerror(errno) : "read failed");
|
||||||
|
return SECRET_READ_ERROR;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/* Trim leading/trailing ASCII space and tab in place; returns the new start. */
|
/* Trim leading/trailing ASCII space and tab in place; returns the new start. */
|
||||||
static char* trim_space(char* s) {
|
static char* trim_space(char* s) {
|
||||||
while (*s == ' ' || *s == '\t')
|
while (*s == ' ' || *s == '\t')
|
||||||
@@ -509,9 +673,17 @@ static CredentialStore* load_store_file(const char* path, char* err, size_t err_
|
|||||||
char line[CREDENTIAL_MAX_LINE + 2];
|
char line[CREDENTIAL_MAX_LINE + 2];
|
||||||
bool ok = true;
|
bool ok = true;
|
||||||
|
|
||||||
while (fgets(line, sizeof(line), fp)) {
|
for (;;) {
|
||||||
|
size_t len = 0;
|
||||||
|
SecretReadResult rr =
|
||||||
|
secret_read_line(line, sizeof(line), fp, "credential file", path, &len, err, err_size);
|
||||||
|
if (rr == SECRET_READ_EOF)
|
||||||
|
break;
|
||||||
|
if (rr == SECRET_READ_ERROR) {
|
||||||
|
ok = false;
|
||||||
|
break;
|
||||||
|
}
|
||||||
line_no++;
|
line_no++;
|
||||||
size_t len = strlen(line);
|
|
||||||
if (len == CREDENTIAL_MAX_LINE + 1 && line[len - 1] != '\n' && !feof(fp)) {
|
if (len == CREDENTIAL_MAX_LINE + 1 && line[len - 1] != '\n' && !feof(fp)) {
|
||||||
set_error(err, err_size, "credential file '%s' line %d exceeds the %d-byte limit", path,
|
set_error(err, err_size, "credential file '%s' line %d exceeds the %d-byte limit", path,
|
||||||
line_no, CREDENTIAL_MAX_LINE);
|
line_no, CREDENTIAL_MAX_LINE);
|
||||||
@@ -1148,9 +1320,17 @@ int credentials_hash_file(const char* path, uint32_t iters, FILE* out, char* err
|
|||||||
int line_no = 0;
|
int line_no = 0;
|
||||||
int result = 0;
|
int result = 0;
|
||||||
char line[CREDENTIAL_MAX_LINE + 2];
|
char line[CREDENTIAL_MAX_LINE + 2];
|
||||||
while (fgets(line, sizeof(line), fp)) {
|
for (;;) {
|
||||||
|
size_t len = 0;
|
||||||
|
SecretReadResult rr =
|
||||||
|
secret_read_line(line, sizeof(line), fp, "plaintext file", path, &len, err, err_size);
|
||||||
|
if (rr == SECRET_READ_EOF)
|
||||||
|
break;
|
||||||
|
if (rr == SECRET_READ_ERROR) {
|
||||||
|
result = -1;
|
||||||
|
break;
|
||||||
|
}
|
||||||
line_no++;
|
line_no++;
|
||||||
size_t len = strlen(line);
|
|
||||||
if (len == CREDENTIAL_MAX_LINE + 1 && line[len - 1] != '\n' && !feof(fp)) {
|
if (len == CREDENTIAL_MAX_LINE + 1 && line[len - 1] != '\n' && !feof(fp)) {
|
||||||
set_error(err, err_size, "plaintext file '%s' line %d exceeds the %d-byte limit", path,
|
set_error(err, err_size, "plaintext file '%s' line %d exceeds the %d-byte limit", path,
|
||||||
line_no, CREDENTIAL_MAX_LINE);
|
line_no, CREDENTIAL_MAX_LINE);
|
||||||
@@ -1228,9 +1408,15 @@ int credentials_read_secret_file(const char* path, char** user_out, char** passw
|
|||||||
char line[CREDENTIAL_MAX_LINE + 2];
|
char line[CREDENTIAL_MAX_LINE + 2];
|
||||||
int result = -1;
|
int result = -1;
|
||||||
|
|
||||||
while (fgets(line, sizeof(line), fp)) {
|
for (;;) {
|
||||||
|
size_t len = 0;
|
||||||
|
SecretReadResult rr =
|
||||||
|
secret_read_line(line, sizeof(line), fp, "password file", path, &len, err, err_size);
|
||||||
|
if (rr == SECRET_READ_EOF)
|
||||||
|
break;
|
||||||
|
if (rr == SECRET_READ_ERROR)
|
||||||
|
goto done;
|
||||||
line_no++;
|
line_no++;
|
||||||
size_t len = strlen(line);
|
|
||||||
if (len == CREDENTIAL_MAX_LINE + 1 && line[len - 1] != '\n' && !feof(fp)) {
|
if (len == CREDENTIAL_MAX_LINE + 1 && line[len - 1] != '\n' && !feof(fp)) {
|
||||||
set_error(err, err_size, "password file '%s' line %d exceeds the %d-byte limit", path,
|
set_error(err, err_size, "password file '%s' line %d exceeds the %d-byte limit", path,
|
||||||
line_no, CREDENTIAL_MAX_LINE);
|
line_no, CREDENTIAL_MAX_LINE);
|
||||||
|
|||||||
+217
-10
@@ -1,12 +1,12 @@
|
|||||||
#include "daemon_conf.h"
|
#include "daemon_conf.h"
|
||||||
#include "credentials.h"
|
#include "credentials.h"
|
||||||
|
#include "log.h"
|
||||||
#include "utils.h"
|
#include "utils.h"
|
||||||
#include <arpa/inet.h>
|
#include <arpa/inet.h>
|
||||||
#include <ctype.h>
|
#include <ctype.h>
|
||||||
#include <errno.h>
|
#include <errno.h>
|
||||||
#include <limits.h>
|
#include <limits.h>
|
||||||
#include <netinet/in.h>
|
#include <netinet/in.h>
|
||||||
#include <stdarg.h>
|
|
||||||
#include <stdint.h>
|
#include <stdint.h>
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
@@ -17,14 +17,7 @@
|
|||||||
/* helpers */
|
/* helpers */
|
||||||
/* ------------------------------------------------------------------ */
|
/* ------------------------------------------------------------------ */
|
||||||
|
|
||||||
static void set_error(char* err, size_t err_size, const char* fmt, ...) {
|
#define set_error utils_set_error
|
||||||
if (!err || err_size == 0)
|
|
||||||
return;
|
|
||||||
va_list args;
|
|
||||||
va_start(args, fmt);
|
|
||||||
vsnprintf(err, err_size, fmt, args);
|
|
||||||
va_end(args);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Trim leading and trailing ASCII space/tab in place; returns the new start. */
|
/* Trim leading and trailing ASCII space/tab in place; returns the new start. */
|
||||||
static char* trim_ws(char* s) {
|
static char* trim_ws(char* s) {
|
||||||
@@ -41,6 +34,158 @@ static bool key_equals(const char* key, const char* canonical) {
|
|||||||
return strcasecmp(key, canonical) == 0;
|
return strcasecmp(key, canonical) == 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* True when `key` matches one of the NUL-terminated names in `list`. */
|
||||||
|
static bool key_in_list(const char* key, const char* const* list, size_t count) {
|
||||||
|
for (size_t i = 0; i < count; i++) {
|
||||||
|
if (strcasecmp(key, list[i]) == 0)
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* rsync 3.4.1 rsyncd.conf GLOBAL keys accepted in the pre-module section that
|
||||||
|
* have no FastSync equivalent. They are recognized and documented as inert:
|
||||||
|
* accepting a real rsync config must not fail on a logging/process key, but a
|
||||||
|
* silently-reinterpreted key is never invented. `pidfile`/`logfile` are the
|
||||||
|
* compact --dparam spellings rsync documents. The same list is used by the
|
||||||
|
* `--dparam` dispatch (apply_global_key), so there is a single impl. */
|
||||||
|
static const char* const kRsyncInertGlobalKeys[] = {
|
||||||
|
"pid file",
|
||||||
|
"pidfile",
|
||||||
|
"log file",
|
||||||
|
"logfile",
|
||||||
|
"socket options",
|
||||||
|
"sockopts",
|
||||||
|
"listen backlog",
|
||||||
|
"syslog facility",
|
||||||
|
"syslog tag",
|
||||||
|
"log format",
|
||||||
|
"use chroot",
|
||||||
|
"uid",
|
||||||
|
"gid",
|
||||||
|
"timeout",
|
||||||
|
"max verbosity",
|
||||||
|
"min verbosity",
|
||||||
|
"lock file",
|
||||||
|
"transfer logging",
|
||||||
|
"strict modes",
|
||||||
|
"reverse lookup",
|
||||||
|
"forward lookup",
|
||||||
|
"ignore errors",
|
||||||
|
"ignore nonreadable",
|
||||||
|
"dont compress",
|
||||||
|
};
|
||||||
|
|
||||||
|
/* rsync 3.4.1 rsyncd.conf MODULE keys accepted in a [module] section that have
|
||||||
|
* no FastSync equivalent (accepted-and-documented inert). Keys with a FastSync
|
||||||
|
* meaning (`path`, `read only`, `write only`, `auth users`, `max connections`,
|
||||||
|
* `hosts allow`/`hosts deny`, `client owner`) are handled by apply_module_key
|
||||||
|
* before this list is consulted. Security-relevant keys (`exclude`, `filter`,
|
||||||
|
* `secrets file`, `refuse options`, ...) are inert, so a daemon-side filter or
|
||||||
|
* rsync secrets file is NOT enforced: each is loudly warned about at load time
|
||||||
|
* (see kRsyncUnenforcedModuleSecurityKeys) and documented as a residual in
|
||||||
|
* RSYNC_COMPAT.md. */
|
||||||
|
static const char* const kRsyncInertModuleKeys[] = {
|
||||||
|
"comment",
|
||||||
|
"use chroot",
|
||||||
|
"daemon chroot",
|
||||||
|
"uid",
|
||||||
|
"gid",
|
||||||
|
"daemon uid",
|
||||||
|
"daemon gid",
|
||||||
|
"exclude",
|
||||||
|
"include",
|
||||||
|
"exclude from",
|
||||||
|
"include from",
|
||||||
|
"filter",
|
||||||
|
"max verbosity",
|
||||||
|
"min verbosity",
|
||||||
|
"lock file",
|
||||||
|
"transfer logging",
|
||||||
|
"log file",
|
||||||
|
"log format",
|
||||||
|
"syslog facility",
|
||||||
|
"syslog tag",
|
||||||
|
"timeout",
|
||||||
|
"secrets file",
|
||||||
|
"auth digest",
|
||||||
|
"strict modes",
|
||||||
|
"numeric ids",
|
||||||
|
"fake super",
|
||||||
|
"munge symlinks",
|
||||||
|
"list",
|
||||||
|
"dont compress",
|
||||||
|
"charset",
|
||||||
|
"refuse options",
|
||||||
|
"incoming chmod",
|
||||||
|
"outgoing chmod",
|
||||||
|
"open noatime",
|
||||||
|
"max size",
|
||||||
|
"min size",
|
||||||
|
"temp dir",
|
||||||
|
"pre-xfer exec",
|
||||||
|
"post-xfer exec",
|
||||||
|
"name converter",
|
||||||
|
"proxy protocol",
|
||||||
|
"proxy protocol hosts",
|
||||||
|
"reverse lookup",
|
||||||
|
"forward lookup",
|
||||||
|
"ignore errors",
|
||||||
|
"ignore nonreadable",
|
||||||
|
};
|
||||||
|
|
||||||
|
/* Subset of the inert rsync keys whose intent is access control (data
|
||||||
|
* visibility, credential source, transfer hooks, daemon privilege), plus the
|
||||||
|
* global keys that shape the daemon's privilege/identity. These load for
|
||||||
|
* rsync-config compatibility, but because FastSync ignores them an operator
|
||||||
|
* migrating a hardened rsyncd.conf must not believe the restriction applies.
|
||||||
|
* The loader emits one LOG_LEVEL_WARNING per occurrence naming the key (and the
|
||||||
|
* module, for a module key). `write only` is deliberately absent: it is mapped
|
||||||
|
* onto writability instead (FastSync is push-only, so a write-only module is
|
||||||
|
* simply writable). */
|
||||||
|
static const char* const kRsyncUnenforcedModuleSecurityKeys[] = {
|
||||||
|
"secrets file",
|
||||||
|
"auth digest",
|
||||||
|
"refuse options",
|
||||||
|
"exclude",
|
||||||
|
"include",
|
||||||
|
"exclude from",
|
||||||
|
"include from",
|
||||||
|
"filter",
|
||||||
|
"max size",
|
||||||
|
"min size",
|
||||||
|
"pre-xfer exec",
|
||||||
|
"post-xfer exec",
|
||||||
|
"incoming chmod",
|
||||||
|
"outgoing chmod",
|
||||||
|
"name converter",
|
||||||
|
"use chroot",
|
||||||
|
"daemon chroot",
|
||||||
|
"uid",
|
||||||
|
"gid",
|
||||||
|
"daemon uid",
|
||||||
|
"daemon gid",
|
||||||
|
"munge symlinks",
|
||||||
|
"fake super",
|
||||||
|
"strict modes",
|
||||||
|
"proxy protocol",
|
||||||
|
"proxy protocol hosts",
|
||||||
|
};
|
||||||
|
|
||||||
|
static const char* const kRsyncUnenforcedGlobalSecurityKeys[] = {
|
||||||
|
"use chroot",
|
||||||
|
"uid",
|
||||||
|
"gid",
|
||||||
|
"strict modes",
|
||||||
|
};
|
||||||
|
|
||||||
|
#define kRsyncInertGlobalCount (sizeof(kRsyncInertGlobalKeys) / sizeof(kRsyncInertGlobalKeys[0]))
|
||||||
|
#define kRsyncInertModuleCount (sizeof(kRsyncInertModuleKeys) / sizeof(kRsyncInertModuleKeys[0]))
|
||||||
|
#define kRsyncUnenforcedModuleSecurityCount \
|
||||||
|
(sizeof(kRsyncUnenforcedModuleSecurityKeys) / sizeof(kRsyncUnenforcedModuleSecurityKeys[0]))
|
||||||
|
#define kRsyncUnenforcedGlobalSecurityCount \
|
||||||
|
(sizeof(kRsyncUnenforcedGlobalSecurityKeys) / sizeof(kRsyncUnenforcedGlobalSecurityKeys[0]))
|
||||||
|
|
||||||
static bool parse_bool_value(const char* value, bool* out) {
|
static bool parse_bool_value(const char* value, bool* out) {
|
||||||
if (strcasecmp(value, "yes") == 0 || strcasecmp(value, "true") == 0 || strcmp(value, "1") == 0) {
|
if (strcasecmp(value, "yes") == 0 || strcasecmp(value, "true") == 0 || strcmp(value, "1") == 0) {
|
||||||
*out = true;
|
*out = true;
|
||||||
@@ -258,6 +403,10 @@ DaemonConf* daemon_conf_create(void) {
|
|||||||
if (!conf)
|
if (!conf)
|
||||||
return NULL;
|
return NULL;
|
||||||
conf->global.port = DAEMON_CONF_DEFAULT_PORT;
|
conf->global.port = DAEMON_CONF_DEFAULT_PORT;
|
||||||
|
/* rsync modules are READ-ONLY unless `read only = no` (or `write only = yes`)
|
||||||
|
* is set, so FastSync must default the same way: a migrated rsyncd.conf that
|
||||||
|
* omits `read only` is served read-only, never writable. */
|
||||||
|
conf->global.read_only_default = true;
|
||||||
conf->global.max_connections = DAEMON_CONF_DEFAULT_MAX_CONNECTIONS;
|
conf->global.max_connections = DAEMON_CONF_DEFAULT_MAX_CONNECTIONS;
|
||||||
conf->global.auth_failure_delay_ms = DAEMON_CONF_DEFAULT_AUTH_FAILURE_DELAY_MS;
|
conf->global.auth_failure_delay_ms = DAEMON_CONF_DEFAULT_AUTH_FAILURE_DELAY_MS;
|
||||||
conf->global.max_connections_per_host = DAEMON_CONF_DEFAULT_MAX_CONNECTIONS_PER_HOST;
|
conf->global.max_connections_per_host = DAEMON_CONF_DEFAULT_MAX_CONNECTIONS_PER_HOST;
|
||||||
@@ -332,7 +481,7 @@ static bool apply_global_key(DaemonConf* conf, char* key, const char* value, boo
|
|||||||
char* err, size_t err_size) {
|
char* err, size_t err_size) {
|
||||||
if (key_equals(key, "port"))
|
if (key_equals(key, "port"))
|
||||||
return store_port(&conf->global.port, value, err, err_size);
|
return store_port(&conf->global.port, value, err, err_size);
|
||||||
if (key_equals(key, "motd file")) {
|
if (key_equals(key, "motd file") || key_equals(key, "motdfile")) {
|
||||||
if (!store_string(&conf->global.motd_file, value)) {
|
if (!store_string(&conf->global.motd_file, value)) {
|
||||||
set_error(err, err_size, "out of memory parsing 'motd file'");
|
set_error(err, err_size, "out of memory parsing 'motd file'");
|
||||||
return false;
|
return false;
|
||||||
@@ -346,6 +495,25 @@ static bool apply_global_key(DaemonConf* conf, char* key, const char* value, boo
|
|||||||
}
|
}
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
/* rsync allows the `read only` module key in the global section as the
|
||||||
|
* default for modules defined after it. Map it to that default (a later
|
||||||
|
* --dparam re-applies it to modules that did not set their own value) so a
|
||||||
|
* global `read only = yes` cannot be silently dropped into a writable
|
||||||
|
* default. */
|
||||||
|
if (key_equals(key, "read only")) {
|
||||||
|
bool parsed;
|
||||||
|
if (!parse_bool_value(value, &parsed)) {
|
||||||
|
set_error(err, err_size, "global 'read only' must be yes/no (or true/false/1/0), got '%s'",
|
||||||
|
value);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
conf->global.read_only_default = parsed;
|
||||||
|
for (int i = 0; i < conf->module_count; i++) {
|
||||||
|
if (!conf->modules[i].read_only_explicit)
|
||||||
|
conf->modules[i].read_only = parsed;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
if (key_equals(key, "max connections"))
|
if (key_equals(key, "max connections"))
|
||||||
return store_max_connections(&conf->global.max_connections, value, NULL, err, err_size);
|
return store_max_connections(&conf->global.max_connections, value, NULL, err, err_size);
|
||||||
if (key_equals(key, "max connections per host"))
|
if (key_equals(key, "max connections per host"))
|
||||||
@@ -368,6 +536,15 @@ static bool apply_global_key(DaemonConf* conf, char* key, const char* value, boo
|
|||||||
if (key_equals(key, "hosts deny"))
|
if (key_equals(key, "hosts deny"))
|
||||||
return store_host_list(&conf->global.hosts_deny, &conf->global.hosts_deny_count, value,
|
return store_host_list(&conf->global.hosts_deny, &conf->global.hosts_deny_count, value,
|
||||||
"hosts deny", NULL, replace_hosts, err, err_size);
|
"hosts deny", NULL, replace_hosts, err, err_size);
|
||||||
|
/* A recognized rsync global key with no FastSync equivalent loads inert. */
|
||||||
|
if (key_in_list(key, kRsyncInertGlobalKeys, kRsyncInertGlobalCount)) {
|
||||||
|
if (key_in_list(key, kRsyncUnenforcedGlobalSecurityKeys, kRsyncUnenforcedGlobalSecurityCount))
|
||||||
|
log_message(LOG_LEVEL_WARNING,
|
||||||
|
"daemon config: global key '%s' is accepted for rsync compatibility but is NOT "
|
||||||
|
"enforced by FastSync; the restriction it expresses will not be applied",
|
||||||
|
key);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
set_error(err, err_size, "unknown global key '%s'", key);
|
set_error(err, err_size, "unknown global key '%s'", key);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
@@ -396,6 +573,26 @@ static bool apply_module_key(DaemonModule* module, char* key, char* value, char*
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
module->read_only = parsed;
|
module->read_only = parsed;
|
||||||
|
module->read_only_explicit = true;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
/* rsync's `write only = yes` makes the module client-writable. FastSync has
|
||||||
|
* no read/pull path, so mapping it to writability is the exact
|
||||||
|
* security-relevant effect; set `read_only_explicit` so a global default
|
||||||
|
* cannot override the module's explicit choice. `write only = no` is the
|
||||||
|
* rsync default and leaves the module's read-only state untouched. */
|
||||||
|
if (key_equals(key, "write only")) {
|
||||||
|
bool parsed;
|
||||||
|
if (!parse_bool_value(value, &parsed)) {
|
||||||
|
set_error(err, err_size,
|
||||||
|
"module '%s': 'write only' must be yes/no (or true/false/1/0), got '%s'",
|
||||||
|
module->name, value);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (parsed) {
|
||||||
|
module->read_only = false;
|
||||||
|
module->read_only_explicit = true;
|
||||||
|
}
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
if (key_equals(key, "client owner")) {
|
if (key_equals(key, "client owner")) {
|
||||||
@@ -465,6 +662,15 @@ static bool apply_module_key(DaemonModule* module, char* key, char* value, char*
|
|||||||
if (key_equals(key, "hosts deny"))
|
if (key_equals(key, "hosts deny"))
|
||||||
return store_host_list(&module->hosts_deny, &module->hosts_deny_count, value, "hosts deny",
|
return store_host_list(&module->hosts_deny, &module->hosts_deny_count, value, "hosts deny",
|
||||||
module->name, false, err, err_size);
|
module->name, false, err, err_size);
|
||||||
|
/* A recognized rsync module key with no FastSync equivalent loads inert. */
|
||||||
|
if (key_in_list(key, kRsyncInertModuleKeys, kRsyncInertModuleCount)) {
|
||||||
|
if (key_in_list(key, kRsyncUnenforcedModuleSecurityKeys, kRsyncUnenforcedModuleSecurityCount))
|
||||||
|
log_message(LOG_LEVEL_WARNING,
|
||||||
|
"daemon config: module '%s' key '%s' is accepted for rsync compatibility but is "
|
||||||
|
"NOT enforced by FastSync; the restriction it expresses will not be applied",
|
||||||
|
module->name, key);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
set_error(err, err_size, "unknown key '%s' in module '%s'", key, module->name);
|
set_error(err, err_size, "unknown key '%s' in module '%s'", key, module->name);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
@@ -515,6 +721,7 @@ static int open_module(DaemonConf* conf, int* current_module, const char* name,
|
|||||||
}
|
}
|
||||||
conf->modules = grown;
|
conf->modules = grown;
|
||||||
memset(&conf->modules[conf->module_count], 0, sizeof(DaemonModule));
|
memset(&conf->modules[conf->module_count], 0, sizeof(DaemonModule));
|
||||||
|
conf->modules[conf->module_count].read_only = conf->global.read_only_default;
|
||||||
conf->modules[conf->module_count].name = str_dup(name);
|
conf->modules[conf->module_count].name = str_dup(name);
|
||||||
if (!conf->modules[conf->module_count].name) {
|
if (!conf->modules[conf->module_count].name) {
|
||||||
set_error(err, err_size, "out of memory adding module '%s'", name);
|
set_error(err, err_size, "out of memory adding module '%s'", name);
|
||||||
|
|||||||
+39
-13
@@ -17,7 +17,20 @@
|
|||||||
* DAEMON_CONF_MAX_LINE all fail the whole load with a clear, line-numbered
|
* DAEMON_CONF_MAX_LINE all fail the whole load with a clear, line-numbered
|
||||||
* error instead of being silently ignored. This keeps a typo from silently
|
* error instead of being silently ignored. This keeps a typo from silently
|
||||||
* changing what a module serves.
|
* changing what a module serves.
|
||||||
*/
|
*
|
||||||
|
* rsync compatibility: to reduce the divergence from rsync 3.4.1's rsyncd.conf
|
||||||
|
* grammar, the parser also ACCEPTS the common rsync GLOBAL and MODULE keys.
|
||||||
|
* Keys with a FastSync equivalent are mapped onto it (the native spellings are
|
||||||
|
* unchanged; `read only` defaults to yes like rsync, and `write only = yes`
|
||||||
|
* opts a module into writability). Keys with no FastSync equivalent are
|
||||||
|
* accepted and documented as inert (they load successfully but have no effect)
|
||||||
|
* rather than failing the whole config; the accepted inert set is listed in
|
||||||
|
* kRsyncInertGlobalKeys / kRsyncInertModuleKeys in daemon_conf.c and in
|
||||||
|
* RSYNC_COMPAT.md. Every inert key whose intent is access control is loudly
|
||||||
|
* warned about at load time (kRsyncUnenforced*SecurityKeys) so an operator
|
||||||
|
* migrating a hardened rsyncd.conf is never misled into believing the
|
||||||
|
* restriction is enforced. A key outside both the FastSync-native grammar and
|
||||||
|
* the recognized rsync subset is still rejected as unknown. */
|
||||||
|
|
||||||
/* A daemon module's configured root is used exactly like the standalone
|
/* A daemon module's configured root is used exactly like the standalone
|
||||||
* server's --destination-root: the daemon confines every connection that
|
* server's --destination-root: the daemon confines every connection that
|
||||||
@@ -42,15 +55,21 @@
|
|||||||
* store refuses (fail closed) rather than falling open; see server.c. Auth is
|
* store refuses (fail closed) rather than falling open; see server.c. Auth is
|
||||||
* never bypassed by ignoring the list. */
|
* never bypassed by ignoring the list. */
|
||||||
typedef struct DaemonModule {
|
typedef struct DaemonModule {
|
||||||
char* name; /* module name, as the client requests it */
|
char* name; /* module name, as the client requests it */
|
||||||
char* path; /* module root (daemon-side authorized root) */
|
char* path; /* module root (daemon-side authorized root) */
|
||||||
bool read_only; /* `read only = yes/no`; default no */
|
bool read_only; /* `read only = yes/no`; defaults to the global `read only`
|
||||||
bool client_owner; /* `client owner = yes/no`; default no. Per-module opt-in
|
default (rsync allows it in the global section), which is
|
||||||
that lets this module's clients choose ownership
|
itself default YES (rsync modules are read-only unless
|
||||||
(--numeric-ids/--chown/--usermap/--groupmap/--fake-super/
|
`read only = no` / `write only = yes` opts in) */
|
||||||
--copy-as) and request explicit --super super-user
|
bool read_only_explicit; /* set when this module set its own `read only` or
|
||||||
activities. Without it the daemon refuses all of them. */
|
`write only = yes`, so a later global default (from a
|
||||||
char** auth_users; /* `auth users = a,b`; Wave B credential list */
|
`--dparam read only=`) does not override it */
|
||||||
|
bool client_owner; /* `client owner = yes/no`; default no. Per-module opt-in
|
||||||
|
that lets this module's clients choose ownership
|
||||||
|
(--numeric-ids/--chown/--usermap/--groupmap/--fake-super/
|
||||||
|
--copy-as) and request explicit --super super-user
|
||||||
|
activities. Without it the daemon refuses all of them. */
|
||||||
|
char** auth_users; /* `auth users = a,b`; Wave B credential list */
|
||||||
int auth_user_count;
|
int auth_user_count;
|
||||||
/* `max connections = N` (optional per-module cap). 0 means unlimited. The
|
/* `max connections = N` (optional per-module cap). 0 means unlimited. The
|
||||||
* per-connection child records the selected module in the shared registry
|
* per-connection child records the selected module in the shared registry
|
||||||
@@ -69,6 +88,10 @@ typedef struct DaemonConfGlobals {
|
|||||||
int port; /* `port`, default DAEMON_CONF_DEFAULT_PORT (873) */
|
int port; /* `port`, default DAEMON_CONF_DEFAULT_PORT (873) */
|
||||||
char* motd_file; /* `motd file`, may be NULL */
|
char* motd_file; /* `motd file`, may be NULL */
|
||||||
char* address; /* `address` (optional bind address), may be NULL */
|
char* address; /* `address` (optional bind address), may be NULL */
|
||||||
|
bool read_only_default; /* global `read only` default for modules defined
|
||||||
|
after it (rsync allows the module key in the
|
||||||
|
global section); default YES to match rsync's
|
||||||
|
read-only modules */
|
||||||
int max_connections; /* `max connections`, default
|
int max_connections; /* `max connections`, default
|
||||||
DAEMON_CONF_DEFAULT_MAX_CONNECTIONS (100) */
|
DAEMON_CONF_DEFAULT_MAX_CONNECTIONS (100) */
|
||||||
int auth_failure_delay_ms; /* `auth failure delay`, milliseconds; default
|
int auth_failure_delay_ms; /* `auth failure delay`, milliseconds; default
|
||||||
@@ -152,10 +175,13 @@ const DaemonModule* daemon_conf_find_module(const DaemonConf* conf, const char*
|
|||||||
bool daemon_module_name_valid(const char* name);
|
bool daemon_module_name_valid(const char* name);
|
||||||
|
|
||||||
/* Parse one --dparam=KEY=VALUE (or "--dparam KEY=VALUE") override string and
|
/* Parse one --dparam=KEY=VALUE (or "--dparam KEY=VALUE") override string and
|
||||||
* apply it to the global keys only. Keys are case-insensitive and limited to
|
* apply it to the global keys only. Keys are case-insensitive and cover the
|
||||||
* the global keys defined by the grammar (port, motd file, address,
|
* global keys defined by the grammar (port, motd file, address, read only,
|
||||||
* max connections, max connections per host, auth failure delay,
|
* max connections, max connections per host, auth failure delay,
|
||||||
* auth lockout threshold, auth lockout duration, hosts allow, hosts deny).
|
* auth lockout threshold, auth lockout duration, hosts allow, hosts deny) plus
|
||||||
|
* the recognized inert rsync global keys and the compact rsync spellings
|
||||||
|
* (`motdfile`, `pidfile`, `logfile`). Applying `read only` sets the global
|
||||||
|
* default and re-applies it to every module that did not set its own value.
|
||||||
* Returns 0 on success, -1 on error (err filled). */
|
* Returns 0 on success, -1 on error (err filled). */
|
||||||
int daemon_conf_apply_dparam(DaemonConf* conf, const char* assignment, char* err, size_t err_size);
|
int daemon_conf_apply_dparam(DaemonConf* conf, const char* assignment, char* err, size_t err_size);
|
||||||
|
|
||||||
|
|||||||
@@ -264,6 +264,20 @@ static bool delay_publish_entry(DelayUpdatesContext* context, const Config* conf
|
|||||||
const StagedFileEntry* entry) {
|
const StagedFileEntry* entry) {
|
||||||
if (!delay_publish_backup(context, config, entry))
|
if (!delay_publish_backup(context, config, entry))
|
||||||
return false;
|
return false;
|
||||||
|
/* --force: an incoming regular file/symlink may replace a destination
|
||||||
|
DIRECTORY (possibly non-empty). The immediate-install path handles this in
|
||||||
|
file_receive; a --delay-updates run stages elsewhere and only discovers the
|
||||||
|
blocking directory here, so clear it before the rename (rsync's
|
||||||
|
"could not make way for new regular file" without --force). */
|
||||||
|
if (config && config->force_delete && file_directory_exists_secure(entry->final_path)) {
|
||||||
|
if (!file_remove_tree_secure(entry->final_path)) {
|
||||||
|
char* escaped = output_escape(entry->final_path, false);
|
||||||
|
log_message(LOG_LEVEL_ERROR, "could not remove destination directory blocking '%s': %s",
|
||||||
|
escaped ? escaped : "<allocation failed>", strerror(errno));
|
||||||
|
free(escaped);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
if (!file_rename_secure(entry->staged_path, entry->final_path)) {
|
if (!file_rename_secure(entry->staged_path, entry->final_path)) {
|
||||||
if (errno == EXDEV) {
|
if (errno == EXDEV) {
|
||||||
char* escaped = output_escape(entry->final_path, false);
|
char* escaped = output_escape(entry->final_path, false);
|
||||||
|
|||||||
@@ -0,0 +1,656 @@
|
|||||||
|
#include "delete.h"
|
||||||
|
|
||||||
|
#include "delay_updates.h"
|
||||||
|
#include "filter.h"
|
||||||
|
#include "log.h"
|
||||||
|
#include "utils.h"
|
||||||
|
#include <dirent.h>
|
||||||
|
#include <errno.h>
|
||||||
|
#include <fcntl.h>
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
|
#include <sys/stat.h>
|
||||||
|
#include <unistd.h>
|
||||||
|
|
||||||
|
/* Build the keep-set index from the exact manifest entries only. A lookup of
|
||||||
|
`rel` succeeds iff `rel` is a kept entry, a kept directory, or an ancestor
|
||||||
|
directory of kept content (the old is_dir_in_manifest predicate); the sorted
|
||||||
|
view answers "is an ancestor of kept content" without materializing any
|
||||||
|
per-component prefix copy, so the index is O(manifest size) memory. */
|
||||||
|
static bool build_keep_index(const ArrayList* manifest, PathIndex* index) {
|
||||||
|
if (!manifest || manifest->size <= 0)
|
||||||
|
return path_index_build(index, NULL, 0);
|
||||||
|
return path_index_build(index, (const char* const*)manifest->items, (size_t)manifest->size);
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool keep_is_dir(const PathIndex* index, const char* rel_path) {
|
||||||
|
return path_index_contains(index, rel_path) || path_index_has_descendant(index, rel_path);
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool keep_is_file(const PathIndex* index, const char* rel_path) {
|
||||||
|
return path_index_contains(index, rel_path);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* True when child_rel is, or lies below, a protected entry. A prefix "a"
|
||||||
|
therefore protects "a" and "a/b/c" but not "ab". Entries with top_level_only
|
||||||
|
set only protect DIRECT children of the receive root (at_root); nested
|
||||||
|
directories that share such a name stay ordinary destination content. */
|
||||||
|
bool path_under_skip_prefix(const char* child_rel, bool at_root, const DeleteSkipEntry* skips,
|
||||||
|
int skip_count) {
|
||||||
|
for (int i = 0; i < skip_count; i++) {
|
||||||
|
if (skips[i].top_level_only && !at_root)
|
||||||
|
continue;
|
||||||
|
size_t prefix_len = strlen(skips[i].prefix);
|
||||||
|
if (strncmp(child_rel, skips[i].prefix, prefix_len) == 0 &&
|
||||||
|
(child_rel[prefix_len] == '\0' || child_rel[prefix_len] == '/'))
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Per-run deletion budget and tallies. `max_delete` is the cap on the number
|
||||||
|
of entries the walker may remove (SIZE_MAX = unlimited); once it is reached
|
||||||
|
the remaining extras are counted in `skipped` and left in place, matching
|
||||||
|
rsync's partial --max-delete behavior. */
|
||||||
|
typedef struct {
|
||||||
|
size_t max_delete;
|
||||||
|
size_t deleted;
|
||||||
|
size_t skipped;
|
||||||
|
bool limit_hit;
|
||||||
|
} DeleteBudget;
|
||||||
|
|
||||||
|
/* True when direct children of the directory named by `rel` may be removed.
|
||||||
|
With no synchronization info (dirs == NULL) the whole tree is deletable; when
|
||||||
|
a dirs index is supplied only its exact entries are (the receive root is the
|
||||||
|
"." sentinel). */
|
||||||
|
static bool is_synced_dir(const PathIndex* dirs, const char* rel) {
|
||||||
|
if (!dirs)
|
||||||
|
return true;
|
||||||
|
return path_index_contains(dirs, rel[0] == '\0' ? "." : rel);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Unsigned byte-wise string compare, matching rsync's u_strcmp (a signed
|
||||||
|
strcmp would order bytes >= 0x80 differently). */
|
||||||
|
static int delete_name_cmp(const char* a, const char* b) {
|
||||||
|
const unsigned char* pa = (const unsigned char*)a;
|
||||||
|
const unsigned char* pb = (const unsigned char*)b;
|
||||||
|
while (*pa != '\0' && *pa == *pb) {
|
||||||
|
pa++;
|
||||||
|
pb++;
|
||||||
|
}
|
||||||
|
return (int)*pa - (int)*pb;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool delete_dir_entries_collect(int dirfd, DeleteDirEntry** out, size_t* count,
|
||||||
|
bool* operation_ok) {
|
||||||
|
*out = NULL;
|
||||||
|
*count = 0;
|
||||||
|
if (operation_ok)
|
||||||
|
*operation_ok = true;
|
||||||
|
int scanfd = openat(dirfd, ".", O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||||
|
if (scanfd < 0)
|
||||||
|
return false;
|
||||||
|
DIR* dir = fdopendir(scanfd);
|
||||||
|
if (!dir) {
|
||||||
|
close(scanfd);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
DeleteDirEntry* entries = NULL;
|
||||||
|
size_t used = 0;
|
||||||
|
size_t capacity = 0;
|
||||||
|
bool ok = true;
|
||||||
|
const struct dirent* entry;
|
||||||
|
while ((entry = readdir(dir)) != NULL) {
|
||||||
|
if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
|
||||||
|
continue;
|
||||||
|
struct stat st;
|
||||||
|
if (fstatat(dirfd, entry->d_name, &st, AT_SYMLINK_NOFOLLOW) != 0) {
|
||||||
|
if (errno != ENOENT && operation_ok)
|
||||||
|
*operation_ok = false;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (used == capacity) {
|
||||||
|
size_t next = capacity == 0 ? 16 : capacity * 2;
|
||||||
|
DeleteDirEntry* grown = realloc(entries, next * sizeof(*grown));
|
||||||
|
if (!grown) {
|
||||||
|
ok = false;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
entries = grown;
|
||||||
|
capacity = next;
|
||||||
|
}
|
||||||
|
entries[used].name = str_dup(entry->d_name);
|
||||||
|
if (!entries[used].name) {
|
||||||
|
ok = false;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
entries[used].is_dir = S_ISDIR(st.st_mode);
|
||||||
|
used++;
|
||||||
|
}
|
||||||
|
closedir(dir);
|
||||||
|
if (!ok) {
|
||||||
|
delete_dir_entries_free(entries, used);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
*out = entries;
|
||||||
|
*count = used;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
void delete_dir_entries_free(DeleteDirEntry* entries, size_t count) {
|
||||||
|
if (!entries)
|
||||||
|
return;
|
||||||
|
for (size_t i = 0; i < count; i++)
|
||||||
|
free(entries[i].name);
|
||||||
|
free(entries);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* rsync's extraneous-entry order: subdirectories before files, each group in
|
||||||
|
descending name order. */
|
||||||
|
int delete_dir_entry_cmp_desc(const void* a, const void* b) {
|
||||||
|
const DeleteDirEntry* ea = a;
|
||||||
|
const DeleteDirEntry* eb = b;
|
||||||
|
if (ea->is_dir != eb->is_dir)
|
||||||
|
return ea->is_dir ? -1 : 1;
|
||||||
|
return -delete_name_cmp(ea->name, eb->name);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* rsync's kept-subdirectory order: plain ascending name. */
|
||||||
|
int delete_dir_entry_cmp_asc(const void* a, const void* b) {
|
||||||
|
const DeleteDirEntry* ea = a;
|
||||||
|
const DeleteDirEntry* eb = b;
|
||||||
|
return delete_name_cmp(ea->name, eb->name);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* How the shared classification/descent walk disposes of an extra it has
|
||||||
|
identified. LIST records the destination-relative path without touching disk
|
||||||
|
(the -n/--dry-run would-delete enumeration); DELETE unlinks/rmdirs it, charges
|
||||||
|
the shared --max-delete budget and notifies the observer. Both modes classify
|
||||||
|
and traverse identically, so the dry-run enumeration and the real deletion
|
||||||
|
cannot drift. */
|
||||||
|
typedef enum { DELETE_WALK_MODE_DELETE, DELETE_WALK_MODE_LIST } DeleteWalkMode;
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
DeleteWalkMode mode;
|
||||||
|
DeleteBudget* budget; /* DELETE mode */
|
||||||
|
ArrayList* out; /* LIST mode: receives strdup'd relative paths */
|
||||||
|
size_t* recorded; /* LIST mode */
|
||||||
|
DeletePathObserver observer; /* DELETE mode */
|
||||||
|
void* observer_context; /* DELETE mode */
|
||||||
|
} DeleteWalkState;
|
||||||
|
|
||||||
|
/* Remove the extras directly inside the directory open on `dirfd` (DELETE mode)
|
||||||
|
or record the paths that WOULD be removed (LIST mode), recursing into every
|
||||||
|
child directory so kept content below a synchronized prefix is reached.
|
||||||
|
`all_removed` reports whether every child entry was removed (so the caller may
|
||||||
|
rmdir this directory). A child directory is never removed when it is itself a
|
||||||
|
synchronized directory or holds kept content; with a dirs index supplied,
|
||||||
|
direct children of a non-synchronized directory are never extras at all (they
|
||||||
|
are left in place but still descended into). Symlinks are unlinked like any
|
||||||
|
other non-directory extra (never followed).
|
||||||
|
|
||||||
|
Entries are processed in rsync's order (extraneous subdirectories in
|
||||||
|
descending name order, then extraneous files, then kept subdirectories in
|
||||||
|
ascending order) rather than readdir() order, so `--max-delete` leaves the
|
||||||
|
same survivors and the `--info=del`/dry-run line order matches rsync. */
|
||||||
|
static bool delete_walk_fd(int dirfd, const char* rel_path, const PathIndex* keep,
|
||||||
|
const PathIndex* dirs, DeleteWalkState* state,
|
||||||
|
const DeleteSkipEntry* skips, int skip_count,
|
||||||
|
const FilterRuleList* protect_rules, bool parent_deletable,
|
||||||
|
bool* all_removed) {
|
||||||
|
DeleteDirEntry* entries = NULL;
|
||||||
|
size_t count = 0;
|
||||||
|
bool collect_ok = true;
|
||||||
|
if (!delete_dir_entries_collect(dirfd, &entries, &count, &collect_ok))
|
||||||
|
return false;
|
||||||
|
bool operation_ok = collect_ok;
|
||||||
|
bool local_survives = false;
|
||||||
|
bool* shielded = calloc(count ? count : 1, sizeof(bool));
|
||||||
|
bool* is_extra = calloc(count ? count : 1, sizeof(bool));
|
||||||
|
if (!shielded || !is_extra) {
|
||||||
|
free(shielded);
|
||||||
|
free(is_extra);
|
||||||
|
delete_dir_entries_free(entries, count);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
/* A directory is deletable when it or ANY ancestor is synchronized; the
|
||||||
|
`parent_deletable` flag carries that down the recursion so dest-only
|
||||||
|
directories below a synchronized root are removed wholesale. */
|
||||||
|
bool deletable = parent_deletable || is_synced_dir(dirs, rel_path);
|
||||||
|
bool at_root = rel_path[0] == '\0';
|
||||||
|
|
||||||
|
/* Reproduce rsync's traversal order: extraneous subdirectories in descending
|
||||||
|
name order, then extraneous files in descending name order, and kept
|
||||||
|
subdirectories only afterwards (ascending). Sorting up front also fixes the
|
||||||
|
identity of the survivors under a partial --max-delete. */
|
||||||
|
if (count > 1)
|
||||||
|
qsort(entries, count, sizeof(*entries), delete_dir_entry_cmp_desc);
|
||||||
|
size_t dir_count = 0;
|
||||||
|
while (dir_count < count && entries[dir_count].is_dir)
|
||||||
|
dir_count++;
|
||||||
|
|
||||||
|
/* Classify every entry up front (the verdict does not depend on processing
|
||||||
|
order) so the ordered passes below can act on it. */
|
||||||
|
for (size_t i = 0; i < count; i++) {
|
||||||
|
char* child_rel = path_cat((char*)rel_path, entries[i].name);
|
||||||
|
if (!child_rel) {
|
||||||
|
operation_ok = false;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
/* A --delay-updates run keeps its staging directory as a direct child of
|
||||||
|
the receive root, and basis-dir snapshots live below it too. Their
|
||||||
|
contents are not manifest entries, so descending into them would delete
|
||||||
|
every staged / basis file as an "extra". Only the staging name (a
|
||||||
|
top-level-only prefix) and the basis prefixes are protected: a nested
|
||||||
|
destination directory that happens to be called .fastsync-stage is
|
||||||
|
ordinary content. */
|
||||||
|
if (path_under_skip_prefix(child_rel, at_root, skips, skip_count)) {
|
||||||
|
shielded[i] = true;
|
||||||
|
local_survives = true;
|
||||||
|
} else if (protect_rules &&
|
||||||
|
filter_rules_apply_side(protect_rules, child_rel, entries[i].name, entries[i].is_dir,
|
||||||
|
FILTER_SIDE_RECEIVER) == FILTER_ACTION_PROTECT) {
|
||||||
|
/* A first-match protect rule shields the extra; for a directory the whole
|
||||||
|
subtree is shielded (rsync prunes an excluded directory), so do not
|
||||||
|
descend. */
|
||||||
|
shielded[i] = true;
|
||||||
|
local_survives = true;
|
||||||
|
} else if (entries[i].is_dir) {
|
||||||
|
bool child_synced = dirs && path_index_contains(dirs, child_rel);
|
||||||
|
is_extra[i] = deletable && !child_synced && !keep_is_dir(keep, child_rel);
|
||||||
|
if (!is_extra[i])
|
||||||
|
local_survives = true;
|
||||||
|
} else {
|
||||||
|
is_extra[i] = deletable && !keep_is_file(keep, child_rel);
|
||||||
|
if (!is_extra[i])
|
||||||
|
local_survives = true;
|
||||||
|
}
|
||||||
|
free(child_rel);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Pass 1: extraneous subdirectories, descending. */
|
||||||
|
for (size_t i = 0; i < dir_count; i++) {
|
||||||
|
if (!is_extra[i])
|
||||||
|
continue;
|
||||||
|
char* child_rel = path_cat((char*)rel_path, entries[i].name);
|
||||||
|
if (!child_rel) {
|
||||||
|
operation_ok = false;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
int childfd = openat(dirfd, entries[i].name, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||||
|
bool child_all_removed = false;
|
||||||
|
if (childfd >= 0) {
|
||||||
|
if (!delete_walk_fd(childfd, child_rel, keep, dirs, state, skips, skip_count, protect_rules,
|
||||||
|
deletable, &child_all_removed))
|
||||||
|
operation_ok = false;
|
||||||
|
close(childfd);
|
||||||
|
} else if (errno != ENOENT) {
|
||||||
|
operation_ok = false;
|
||||||
|
}
|
||||||
|
if (child_all_removed && deletable) {
|
||||||
|
if (state->mode == DELETE_WALK_MODE_LIST) {
|
||||||
|
/* Record the directory with rsync's trailing slash. */
|
||||||
|
size_t len = strlen(child_rel);
|
||||||
|
char* copy = malloc(len + 2);
|
||||||
|
if (!copy) {
|
||||||
|
operation_ok = false;
|
||||||
|
} else {
|
||||||
|
memcpy(copy, child_rel, len);
|
||||||
|
copy[len] = '/';
|
||||||
|
copy[len + 1] = '\0';
|
||||||
|
if (!array_list_add(state->out, copy)) {
|
||||||
|
free(copy);
|
||||||
|
operation_ok = false;
|
||||||
|
} else {
|
||||||
|
(*state->recorded)++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if (state->budget->deleted >= state->budget->max_delete) {
|
||||||
|
state->budget->limit_hit = true;
|
||||||
|
state->budget->skipped++;
|
||||||
|
local_survives = true;
|
||||||
|
} else if (unlinkat(dirfd, entries[i].name, AT_REMOVEDIR) != 0) {
|
||||||
|
/* ENOENT: already gone (fine). ENOTEMPTY/EEXIST: the directory still
|
||||||
|
holds entries the walker leaves in place (a protected excluded
|
||||||
|
prefix, a kept file the manifest protects, a symlink); rsync leaves
|
||||||
|
such a directory behind, so this is not an error. Only genuine I/O
|
||||||
|
failures abort the deletion. */
|
||||||
|
if (errno != ENOENT && errno != ENOTEMPTY && errno != EEXIST)
|
||||||
|
operation_ok = false;
|
||||||
|
local_survives = true;
|
||||||
|
} else {
|
||||||
|
state->budget->deleted++;
|
||||||
|
/* rsync reports a removed directory with a trailing slash. */
|
||||||
|
if (state->observer) {
|
||||||
|
size_t len = strlen(child_rel);
|
||||||
|
char* with_slash = malloc(len + 2);
|
||||||
|
if (with_slash) {
|
||||||
|
memcpy(with_slash, child_rel, len);
|
||||||
|
with_slash[len] = '/';
|
||||||
|
with_slash[len + 1] = '\0';
|
||||||
|
state->observer(state->observer_context, with_slash);
|
||||||
|
free(with_slash);
|
||||||
|
} else {
|
||||||
|
state->observer(state->observer_context, child_rel);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
local_survives = true;
|
||||||
|
}
|
||||||
|
free(child_rel);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Pass 2: extraneous files, descending. */
|
||||||
|
for (size_t i = dir_count; i < count; i++) {
|
||||||
|
if (!is_extra[i])
|
||||||
|
continue;
|
||||||
|
if (state->mode == DELETE_WALK_MODE_LIST) {
|
||||||
|
char* child_rel = path_cat((char*)rel_path, entries[i].name);
|
||||||
|
if (!child_rel) {
|
||||||
|
operation_ok = false;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
char* copy = str_dup(child_rel);
|
||||||
|
if (!copy || !array_list_add(state->out, copy)) {
|
||||||
|
free(copy);
|
||||||
|
operation_ok = false;
|
||||||
|
} else {
|
||||||
|
(*state->recorded)++;
|
||||||
|
}
|
||||||
|
free(child_rel);
|
||||||
|
} else if (state->budget->deleted >= state->budget->max_delete) {
|
||||||
|
state->budget->limit_hit = true;
|
||||||
|
state->budget->skipped++;
|
||||||
|
local_survives = true;
|
||||||
|
} else if (unlinkat(dirfd, entries[i].name, 0) != 0) {
|
||||||
|
if (errno != ENOENT)
|
||||||
|
operation_ok = false;
|
||||||
|
local_survives = true;
|
||||||
|
} else {
|
||||||
|
state->budget->deleted++;
|
||||||
|
char* child_rel = path_cat((char*)rel_path, entries[i].name);
|
||||||
|
if (child_rel) {
|
||||||
|
if (state->observer)
|
||||||
|
state->observer(state->observer_context, child_rel);
|
||||||
|
char* escaped_path = output_escape(child_rel, log_get_8_bit_output());
|
||||||
|
fprintf(stderr, " Deleted: %s\n", escaped_path ? escaped_path : "<allocation failed>");
|
||||||
|
free(escaped_path);
|
||||||
|
}
|
||||||
|
free(child_rel);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Pass 3: kept subdirectories, ascending (rsync descends into these only
|
||||||
|
after the parent's own extras have been handled). */
|
||||||
|
for (size_t i = dir_count; i-- > 0;) {
|
||||||
|
if (is_extra[i] || shielded[i])
|
||||||
|
continue;
|
||||||
|
char* child_rel = path_cat((char*)rel_path, entries[i].name);
|
||||||
|
if (!child_rel) {
|
||||||
|
operation_ok = false;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
int childfd = openat(dirfd, entries[i].name, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||||
|
bool child_all_removed = false;
|
||||||
|
if (childfd >= 0) {
|
||||||
|
if (!delete_walk_fd(childfd, child_rel, keep, dirs, state, skips, skip_count, protect_rules,
|
||||||
|
deletable, &child_all_removed))
|
||||||
|
operation_ok = false;
|
||||||
|
close(childfd);
|
||||||
|
} else if (errno != ENOENT) {
|
||||||
|
operation_ok = false;
|
||||||
|
}
|
||||||
|
/* A kept/synchronized directory is never removed. */
|
||||||
|
local_survives = true;
|
||||||
|
free(child_rel);
|
||||||
|
}
|
||||||
|
|
||||||
|
free(shielded);
|
||||||
|
free(is_extra);
|
||||||
|
delete_dir_entries_free(entries, count);
|
||||||
|
*all_removed = !local_survives;
|
||||||
|
return operation_ok;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Open the receive root following the same authorized-root confinement the
|
||||||
|
walker uses, or dest_root directly when no authorized root is installed. */
|
||||||
|
static int open_destination_root(const char* dest_root) {
|
||||||
|
int root_fd = utils_get_authorized_root_fd();
|
||||||
|
if (root_fd >= 0) {
|
||||||
|
if (utils_get_authorized_root_path())
|
||||||
|
return utils_open_authorized_destination(dest_root);
|
||||||
|
if (dest_root == NULL)
|
||||||
|
return dup(root_fd);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
return open(dest_root, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool delete_extras_list(const char* dest_root, const ArrayList* manifest,
|
||||||
|
const ArrayList* synced_dirs, const DeleteSkipEntry* skips, int skip_count,
|
||||||
|
const FilterRuleList* protect_rules, ArrayList* out, size_t* count_out) {
|
||||||
|
if (count_out)
|
||||||
|
*count_out = 0;
|
||||||
|
if (!manifest || !out)
|
||||||
|
return false;
|
||||||
|
PathIndex keep;
|
||||||
|
if (!build_keep_index(manifest, &keep))
|
||||||
|
return false;
|
||||||
|
PathIndex dirs;
|
||||||
|
bool have_dirs = synced_dirs != NULL;
|
||||||
|
if (have_dirs &&
|
||||||
|
!path_index_build(&dirs, (const char* const*)synced_dirs->items, (size_t)synced_dirs->size)) {
|
||||||
|
path_index_free(&keep);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
int rootfd = open_destination_root(dest_root);
|
||||||
|
if (rootfd < 0) {
|
||||||
|
path_index_free(&keep);
|
||||||
|
if (have_dirs)
|
||||||
|
path_index_free(&dirs);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
bool all_removed = false;
|
||||||
|
size_t recorded = 0;
|
||||||
|
DeleteWalkState state = {.mode = DELETE_WALK_MODE_LIST,
|
||||||
|
.budget = NULL,
|
||||||
|
.out = out,
|
||||||
|
.recorded = &recorded,
|
||||||
|
.observer = NULL,
|
||||||
|
.observer_context = NULL};
|
||||||
|
bool ok = delete_walk_fd(rootfd, "", &keep, have_dirs ? &dirs : NULL, &state, skips, skip_count,
|
||||||
|
protect_rules, false, &all_removed);
|
||||||
|
if (close(rootfd) != 0)
|
||||||
|
ok = false;
|
||||||
|
path_index_free(&keep);
|
||||||
|
if (have_dirs)
|
||||||
|
path_index_free(&dirs);
|
||||||
|
if (count_out)
|
||||||
|
*count_out = recorded;
|
||||||
|
return ok;
|
||||||
|
}
|
||||||
|
|
||||||
|
DeleteWalkResult delete_extras_limited_observed(const char* dest_root, const ArrayList* manifest,
|
||||||
|
const ArrayList* synced_dirs, size_t max_delete,
|
||||||
|
const DeleteSkipEntry* skips, int skip_count,
|
||||||
|
const FilterRuleList* protect_rules,
|
||||||
|
size_t* deleted_out, size_t* skipped_out,
|
||||||
|
DeletePathObserver observer,
|
||||||
|
void* observer_context) {
|
||||||
|
if (deleted_out)
|
||||||
|
*deleted_out = 0;
|
||||||
|
if (skipped_out)
|
||||||
|
*skipped_out = 0;
|
||||||
|
if (!manifest)
|
||||||
|
return DELETE_WALK_ERROR;
|
||||||
|
/* Index the keep-set (and the synchronized-dir set, when supplied) once so
|
||||||
|
membership is answered in O(path length) instead of scanning every entry
|
||||||
|
for every destination entry. */
|
||||||
|
PathIndex keep;
|
||||||
|
if (!build_keep_index(manifest, &keep))
|
||||||
|
return DELETE_WALK_ERROR;
|
||||||
|
PathIndex dirs;
|
||||||
|
bool have_dirs = synced_dirs != NULL;
|
||||||
|
if (have_dirs &&
|
||||||
|
!path_index_build(&dirs, (const char* const*)synced_dirs->items, (size_t)synced_dirs->size)) {
|
||||||
|
path_index_free(&keep);
|
||||||
|
return DELETE_WALK_ERROR;
|
||||||
|
}
|
||||||
|
int rootfd = open_destination_root(dest_root);
|
||||||
|
if (rootfd < 0) {
|
||||||
|
path_index_free(&keep);
|
||||||
|
if (have_dirs)
|
||||||
|
path_index_free(&dirs);
|
||||||
|
return DELETE_WALK_ERROR;
|
||||||
|
}
|
||||||
|
DeleteBudget budget = {.max_delete = max_delete, .deleted = 0, .skipped = 0, .limit_hit = false};
|
||||||
|
bool all_removed = false;
|
||||||
|
DeleteWalkState state = {.mode = DELETE_WALK_MODE_DELETE,
|
||||||
|
.budget = &budget,
|
||||||
|
.out = NULL,
|
||||||
|
.recorded = NULL,
|
||||||
|
.observer = observer,
|
||||||
|
.observer_context = observer_context};
|
||||||
|
bool ok = delete_walk_fd(rootfd, "", &keep, have_dirs ? &dirs : NULL, &state, skips, skip_count,
|
||||||
|
protect_rules, false, &all_removed);
|
||||||
|
if (close(rootfd) != 0)
|
||||||
|
ok = false;
|
||||||
|
path_index_free(&keep);
|
||||||
|
if (have_dirs)
|
||||||
|
path_index_free(&dirs);
|
||||||
|
if (deleted_out)
|
||||||
|
*deleted_out = budget.deleted;
|
||||||
|
if (skipped_out)
|
||||||
|
*skipped_out = budget.skipped;
|
||||||
|
if (!ok)
|
||||||
|
return DELETE_WALK_ERROR;
|
||||||
|
return budget.limit_hit ? DELETE_WALK_LIMIT_REACHED : DELETE_WALK_OK;
|
||||||
|
}
|
||||||
|
|
||||||
|
DeleteWalkResult delete_extras_limited(const char* dest_root, const ArrayList* manifest,
|
||||||
|
const ArrayList* synced_dirs, size_t max_delete,
|
||||||
|
const DeleteSkipEntry* skips, int skip_count,
|
||||||
|
const FilterRuleList* protect_rules, size_t* deleted_out,
|
||||||
|
size_t* skipped_out) {
|
||||||
|
return delete_extras_limited_observed(dest_root, manifest, synced_dirs, max_delete, skips,
|
||||||
|
skip_count, protect_rules, deleted_out, skipped_out, NULL,
|
||||||
|
NULL);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool delete_extras(const char* dest_root, const ArrayList* manifest) {
|
||||||
|
return delete_extras_limited(dest_root, manifest, NULL, SIZE_MAX, NULL, 0, NULL, NULL, NULL) ==
|
||||||
|
DELETE_WALK_OK;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Build the delete-walk protection prefix for one basis directory. The walker
|
||||||
|
compares paths relative to the receive root, so a relative entry is already
|
||||||
|
in the right form; an absolute entry that lies below the root is converted to
|
||||||
|
its root-relative form, and one outside the root returns NULL (the walk
|
||||||
|
cannot reach it, and it is not protected data beneath the root). Exposed so
|
||||||
|
tests can exercise the root-of-"/" child mapping directly. */
|
||||||
|
char* delete_basis_relative(const Config* config, const char* path) {
|
||||||
|
if (!path)
|
||||||
|
return NULL;
|
||||||
|
if (path[0] != '/')
|
||||||
|
return str_dup(path);
|
||||||
|
const char* root = config->receive_root_directory;
|
||||||
|
if (!root || root[0] != '/')
|
||||||
|
return NULL;
|
||||||
|
size_t root_len = strlen(root);
|
||||||
|
while (root_len > 1 && root[root_len - 1] == '/')
|
||||||
|
root_len--;
|
||||||
|
if (strncmp(path, root, root_len) != 0)
|
||||||
|
return NULL;
|
||||||
|
if (root_len == 1) {
|
||||||
|
/* `root` is "/" (the only single-character absolute root): every absolute
|
||||||
|
path is below it, and the child relative form is everything after the
|
||||||
|
leading '/'. */
|
||||||
|
if (path[1] == '\0')
|
||||||
|
return NULL; /* identical to the root, not a child */
|
||||||
|
return str_dup(path + 1);
|
||||||
|
}
|
||||||
|
if (path[root_len] != '/')
|
||||||
|
return NULL; /* identical or a sibling sharing a name prefix */
|
||||||
|
return str_dup(path + root_len + 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool delete_skips_build(const Config* config, const ArrayList* protected_paths,
|
||||||
|
const ArrayList* size_skipped, bool basis_root_relative,
|
||||||
|
DeleteSkipSet* out) {
|
||||||
|
if (!out)
|
||||||
|
return false;
|
||||||
|
out->entries = NULL;
|
||||||
|
out->owned_prefixes = NULL;
|
||||||
|
out->count = 0;
|
||||||
|
out->owned_count = 0;
|
||||||
|
if (!config)
|
||||||
|
return false;
|
||||||
|
int protected_count = protected_paths ? protected_paths->size : 0;
|
||||||
|
int size_skipped_count = size_skipped ? size_skipped->size : 0;
|
||||||
|
int count =
|
||||||
|
(config->delay_updates ? 1 : 0) + config->basis_count + protected_count + size_skipped_count;
|
||||||
|
if (count == 0)
|
||||||
|
return true;
|
||||||
|
out->entries = calloc((size_t)count, sizeof(DeleteSkipEntry));
|
||||||
|
if (!out->entries)
|
||||||
|
return false;
|
||||||
|
if (basis_root_relative && config->basis_count > 0) {
|
||||||
|
out->owned_prefixes = calloc((size_t)config->basis_count, sizeof(char*));
|
||||||
|
if (!out->owned_prefixes) {
|
||||||
|
free(out->entries);
|
||||||
|
out->entries = NULL;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
out->owned_count = config->basis_count;
|
||||||
|
}
|
||||||
|
int idx = 0;
|
||||||
|
if (config->delay_updates) {
|
||||||
|
out->entries[idx].prefix = DELAY_UPDATES_STAGING_DIR;
|
||||||
|
out->entries[idx].top_level_only = true;
|
||||||
|
idx++;
|
||||||
|
}
|
||||||
|
for (int i = 0; i < config->basis_count; i++) {
|
||||||
|
const char* prefix = config->basis_dirs[i].path;
|
||||||
|
if (basis_root_relative) {
|
||||||
|
/* An absolute basis outside the receive root is unreachable by this walk,
|
||||||
|
so it contributes no protection prefix (and no slot). */
|
||||||
|
char* relative = delete_basis_relative(config, config->basis_dirs[i].path);
|
||||||
|
if (!relative)
|
||||||
|
continue;
|
||||||
|
out->owned_prefixes[i] = relative;
|
||||||
|
prefix = relative;
|
||||||
|
}
|
||||||
|
out->entries[idx].prefix = prefix;
|
||||||
|
out->entries[idx].top_level_only = false;
|
||||||
|
idx++;
|
||||||
|
}
|
||||||
|
for (int i = 0; i < protected_count; i++) {
|
||||||
|
out->entries[idx].prefix = (const char*)protected_paths->items[i];
|
||||||
|
out->entries[idx].top_level_only = false;
|
||||||
|
idx++;
|
||||||
|
}
|
||||||
|
for (int i = 0; i < size_skipped_count; i++) {
|
||||||
|
out->entries[idx].prefix = (const char*)size_skipped->items[i];
|
||||||
|
out->entries[idx].top_level_only = false;
|
||||||
|
idx++;
|
||||||
|
}
|
||||||
|
out->count = idx;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
void delete_skips_free(DeleteSkipSet* set) {
|
||||||
|
if (!set)
|
||||||
|
return;
|
||||||
|
if (set->owned_prefixes) {
|
||||||
|
for (int i = 0; i < set->owned_count; i++)
|
||||||
|
free(set->owned_prefixes[i]);
|
||||||
|
}
|
||||||
|
free(set->owned_prefixes);
|
||||||
|
free(set->entries);
|
||||||
|
set->entries = NULL;
|
||||||
|
set->owned_prefixes = NULL;
|
||||||
|
set->count = 0;
|
||||||
|
set->owned_count = 0;
|
||||||
|
}
|
||||||
@@ -0,0 +1,151 @@
|
|||||||
|
#ifndef DELETE_H
|
||||||
|
#define DELETE_H
|
||||||
|
|
||||||
|
#include "array_list.h"
|
||||||
|
#include "config.h"
|
||||||
|
#include <stdbool.h>
|
||||||
|
#include <stddef.h>
|
||||||
|
|
||||||
|
/* Delete engine.
|
||||||
|
*
|
||||||
|
* This module owns destination-relative delete traversal: the ordered directory
|
||||||
|
* walker that reproduces rsync's extraneous-entry order, the skip-prefix
|
||||||
|
* protection set shared by every delete pass, and the read-only enumeration
|
||||||
|
* that mirrors the walker for -n/--dry-run. The budgeted manifest commit
|
||||||
|
* (delete_commit.c) and the per-directory delete plans (delete_plan.c) are
|
||||||
|
* built on the primitives exported here. */
|
||||||
|
|
||||||
|
/* Result of a bounded extra-file deletion run. */
|
||||||
|
typedef enum {
|
||||||
|
/* Every extra entry was removed (or there were none). */
|
||||||
|
DELETE_WALK_OK = 0,
|
||||||
|
/* The numeric cap for this run was reached before every extra was removed.
|
||||||
|
The walker removed exactly the entries the cap allowed and skipped (without
|
||||||
|
removing) the rest, matching rsync's partial --max-delete behavior. */
|
||||||
|
DELETE_WALK_LIMIT_REACHED,
|
||||||
|
/* A traversal or unlink failure aborted the deletion (partial removal is
|
||||||
|
possible, mirroring the delete pass). */
|
||||||
|
DELETE_WALK_ERROR
|
||||||
|
} DeleteWalkResult;
|
||||||
|
|
||||||
|
/* One protected entry for the delete walker. When top_level_only is true the
|
||||||
|
prefix is skipped only as a DIRECT child of dest_root (the --delay-updates
|
||||||
|
staging directory, which must not hide genuine extras inside a nested
|
||||||
|
destination directory that happens to share the staging name); otherwise the
|
||||||
|
prefix is skipped at any depth (the --compare-dest/--copy-dest/--link-dest
|
||||||
|
basis trees, and the sender-side protected filter-excluded prefixes, which
|
||||||
|
are never destination content). */
|
||||||
|
typedef struct {
|
||||||
|
const char* prefix;
|
||||||
|
bool top_level_only;
|
||||||
|
} DeleteSkipEntry;
|
||||||
|
|
||||||
|
/* A built skip-prefix set. `entries`/`count` are what path_under_skip_prefix()
|
||||||
|
consumes. `owned_prefixes` holds any prefix strings the builder had to
|
||||||
|
allocate (root-relative basis-dir conversions); it is NULL when every prefix
|
||||||
|
is borrowed from the config or the caller's lists. Release with
|
||||||
|
delete_skips_free(). */
|
||||||
|
typedef struct {
|
||||||
|
DeleteSkipEntry* entries;
|
||||||
|
char** owned_prefixes;
|
||||||
|
int count;
|
||||||
|
int owned_count;
|
||||||
|
} DeleteSkipSet;
|
||||||
|
|
||||||
|
/* True when child_rel is, or lies below, one of the protected entries (a prefix
|
||||||
|
"a" protects "a" and "a/b/c" but not "ab"; top_level_only entries protect
|
||||||
|
only DIRECT children of the destination root, i.e. child_rel has no '/'). */
|
||||||
|
bool path_under_skip_prefix(const char* child_rel, bool at_root, const DeleteSkipEntry* skips,
|
||||||
|
int skip_count);
|
||||||
|
|
||||||
|
/* One destination-directory entry collected up front so the delete walkers can
|
||||||
|
reproduce rsync's traversal order instead of readdir() order. rsync processes
|
||||||
|
a directory's extraneous subdirectories first (descending name, depth-first),
|
||||||
|
then its extraneous files (descending name), and only afterwards descends into
|
||||||
|
its kept subdirectories (ascending name). */
|
||||||
|
typedef struct {
|
||||||
|
char* name;
|
||||||
|
bool is_dir;
|
||||||
|
} DeleteDirEntry;
|
||||||
|
/* Collect the entries of the directory open on `dirfd` (excluding "." and ".."),
|
||||||
|
stat'ing each with AT_SYMLINK_NOFOLLOW. On success *out is a malloc'd array of
|
||||||
|
*count entries whose names the caller frees with delete_dir_entries_free().
|
||||||
|
Returns false on an allocation/readdir failure; a vanished entry (ENOENT) is
|
||||||
|
skipped, any other stat failure is reported through *operation_ok while the
|
||||||
|
walk continues. */
|
||||||
|
bool delete_dir_entries_collect(int dirfd, DeleteDirEntry** out, size_t* count, bool* operation_ok);
|
||||||
|
void delete_dir_entries_free(DeleteDirEntry* entries, size_t count);
|
||||||
|
/* Sort comparators: `_desc` orders subdirectories before files and each group by
|
||||||
|
descending name (rsync's extraneous-entry order); `_asc` orders plain ascending
|
||||||
|
name (rsync's kept-subdirectory order). */
|
||||||
|
int delete_dir_entry_cmp_desc(const void* a, const void* b);
|
||||||
|
int delete_dir_entry_cmp_asc(const void* a, const void* b);
|
||||||
|
|
||||||
|
/* Remove files/dirs/symlinks under dest_root that are not listed in manifest
|
||||||
|
without ever descending into a protected prefix (see DeleteSkipEntry). When
|
||||||
|
`synced_dirs` is non-NULL, extras are only removed directly inside a directory
|
||||||
|
whose destination-relative path is an exact entry in that list (the receive
|
||||||
|
root is the "." sentinel); directories outside the synchronized set are still
|
||||||
|
descended into so kept content below a listed directory is preserved, but
|
||||||
|
nothing in them is removed. A NULL `synced_dirs` keeps the legacy behavior of
|
||||||
|
treating the whole destination tree as deletable. `max_delete` caps the
|
||||||
|
number of removed entries (SIZE_MAX = unlimited): the walker removes up to the
|
||||||
|
cap and returns DELETE_WALK_LIMIT_REACHED when more extras remained.
|
||||||
|
`deleted_out`/`skipped_out` optionally receive the number of entries removed
|
||||||
|
and the number skipped because of the cap. */
|
||||||
|
DeleteWalkResult delete_extras_limited(const char* dest_root, const ArrayList* manifest,
|
||||||
|
const ArrayList* synced_dirs, size_t max_delete,
|
||||||
|
const DeleteSkipEntry* skips, int skip_count,
|
||||||
|
const FilterRuleList* protect_rules, size_t* deleted_out,
|
||||||
|
size_t* skipped_out);
|
||||||
|
|
||||||
|
/* Optional per-deletion observer: called for each destination-relative path
|
||||||
|
actually removed (a file, symlink, or directory), in removal order, so the
|
||||||
|
receiver can stream rsync's `--info=del`/`--info=remove` lines. */
|
||||||
|
typedef void (*DeletePathObserver)(void* context, const char* rel_path);
|
||||||
|
|
||||||
|
/* `delete_extras_limited_observed` is delete_extras_limited with an optional
|
||||||
|
* observer; the observer is invoked only for entries truly removed. When
|
||||||
|
* `protect_rules` is non-NULL its receiver-side verdict is evaluated for every
|
||||||
|
* candidate extra: a first-match PROTECT leaves the entry (and, for a
|
||||||
|
* directory, its whole subtree) in place, while RISK/NONE fall through to the
|
||||||
|
* ordinary skip-prefix/keep-set logic. */
|
||||||
|
DeleteWalkResult delete_extras_limited_observed(const char* dest_root, const ArrayList* manifest,
|
||||||
|
const ArrayList* synced_dirs, size_t max_delete,
|
||||||
|
const DeleteSkipEntry* skips, int skip_count,
|
||||||
|
const FilterRuleList* protect_rules,
|
||||||
|
size_t* deleted_out, size_t* skipped_out,
|
||||||
|
DeletePathObserver observer,
|
||||||
|
void* observer_context);
|
||||||
|
/* Read-only companion to delete_extras_limited: walk the destination exactly as
|
||||||
|
the delete pass would and APPEND (strdup'd) destination-relative paths that
|
||||||
|
WOULD be removed, without touching disk. Used for -n/--dry-run --delete
|
||||||
|
would-delete reporting. Returns true on a clean walk; the caller owns the
|
||||||
|
strings appended to `out` and receives their count in *count_out. */
|
||||||
|
bool delete_extras_list(const char* dest_root, const ArrayList* manifest,
|
||||||
|
const ArrayList* synced_dirs, const DeleteSkipEntry* skips, int skip_count,
|
||||||
|
const FilterRuleList* protect_rules, ArrayList* out, size_t* count_out);
|
||||||
|
bool delete_extras(const char* dest_root, const ArrayList* manifest);
|
||||||
|
|
||||||
|
/* Build the delete walk's skip-prefix set from the config's --delay-updates
|
||||||
|
staging directory, its --compare-dest/--copy-dest/--link-dest basis dirs, and
|
||||||
|
the caller-supplied protection lists, in that order. `protected_paths` and
|
||||||
|
`size_skipped` are borrowed (may be NULL); every entry in them is protected at
|
||||||
|
any depth. The staging directory is protected only as a DIRECT child of the
|
||||||
|
receive root. `basis_root_relative` selects how a basis path becomes a
|
||||||
|
prefix: true converts an absolute path under the receive root to its
|
||||||
|
root-relative form (the whole-tree commit walk; an unreachable path
|
||||||
|
contributes no slot), false keeps the configured path verbatim (the
|
||||||
|
per-directory plan walk). On success the caller releases `*out` with
|
||||||
|
delete_skips_free(); returns false on allocation failure. */
|
||||||
|
bool delete_skips_build(const Config* config, const ArrayList* protected_paths,
|
||||||
|
const ArrayList* size_skipped, bool basis_root_relative,
|
||||||
|
DeleteSkipSet* out);
|
||||||
|
void delete_skips_free(DeleteSkipSet* set);
|
||||||
|
|
||||||
|
/* Convert one basis-directory path to the receive-root-relative protection
|
||||||
|
prefix the delete walker uses (NULL when it lies outside the root). Exposed
|
||||||
|
for unit tests of the root-of-"/" and normalization edge cases. */
|
||||||
|
char* delete_basis_relative(const Config* config, const char* path);
|
||||||
|
|
||||||
|
#endif
|
||||||
@@ -0,0 +1,488 @@
|
|||||||
|
#include <errno.h>
|
||||||
|
#include <ctype.h>
|
||||||
|
#include <dirent.h>
|
||||||
|
#include <fcntl.h>
|
||||||
|
#include <libgen.h>
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
|
#include <sys/stat.h>
|
||||||
|
#include <sys/sysmacros.h>
|
||||||
|
#include <unistd.h>
|
||||||
|
|
||||||
|
#include "array_list.h"
|
||||||
|
#include "charset.h"
|
||||||
|
#include "chmod.h"
|
||||||
|
#include "chunk.h"
|
||||||
|
#include "compression.h"
|
||||||
|
#include "config.h"
|
||||||
|
#include "data.h"
|
||||||
|
#include "delay_updates.h"
|
||||||
|
#include "delete_commit.h"
|
||||||
|
#include "delta.h"
|
||||||
|
#include "file.h"
|
||||||
|
#include "format.h"
|
||||||
|
#include "identity.h"
|
||||||
|
#include "log.h"
|
||||||
|
#include "metadata.h"
|
||||||
|
#include "protocol.h"
|
||||||
|
#include "utils.h"
|
||||||
|
#include "xattr.h"
|
||||||
|
|
||||||
|
#define MAX_SERVER_DELETE_COUNT 100000U
|
||||||
|
/* Retained cost of one delete-manifest entry beyond its path bytes: the
|
||||||
|
ArrayList pointer slot plus an approximate malloc header/rounding for the
|
||||||
|
heap copy. Charged against MAX_MANIFEST_BYTES so a frame full of tiny paths
|
||||||
|
cannot retain far more than the byte budget (B5). */
|
||||||
|
#define MANIFEST_ENTRY_OVERHEAD (sizeof(char*) + 16)
|
||||||
|
|
||||||
|
/* Read a delete-manifest frame (the STATUS_MANIFEST leading code has already
|
||||||
|
been consumed): a keep-set entry count followed by that many
|
||||||
|
destination-relative paths, then a protected-prefix count followed by that
|
||||||
|
many destination-relative prefixes, then a missing-args count followed by that
|
||||||
|
many destination-relative delete paths, then (protocol 2.23.0) a
|
||||||
|
synchronized-directory count followed by that many destination-relative
|
||||||
|
directory paths (the receive root is the "." sentinel). The frame is
|
||||||
|
self-delimiting (the counts are authoritative), so the caller decides what to
|
||||||
|
do next and continues reading the following STATUS_* frame. Every section is
|
||||||
|
validated identically: an entry must be non-empty, relative and traversal-free
|
||||||
|
and the aggregate length across ALL sections is capped by MAX_MANIFEST_BYTES
|
||||||
|
(so the missing-args deletion requests are confined like the rest of the
|
||||||
|
manifest). Returns an owned DeleteManifest, or NULL after sending STATUS_ERROR
|
||||||
|
when the frame is malformed (bad count, empty/absolute path, path traversal,
|
||||||
|
or an aggregate size beyond MAX_MANIFEST_BYTES). */
|
||||||
|
static bool receive_manifest_section(int fd, ArrayList* list, size_t* manifest_bytes,
|
||||||
|
size_t* manifest_entries) {
|
||||||
|
int count;
|
||||||
|
if (!receive_int(fd, &count)) {
|
||||||
|
send_status(fd, STATUS_ERROR);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (count < 0 || count > MAX_MANIFEST_ENTRIES ||
|
||||||
|
(size_t)count > MAX_MANIFEST_ENTRIES - *manifest_entries) {
|
||||||
|
send_status(fd, STATUS_ERROR);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
for (int i = 0; i < count; i++) {
|
||||||
|
char* s = receive_wire_str(fd);
|
||||||
|
size_t entry_size = s ? strlen(s) + MANIFEST_ENTRY_OVERHEAD : 0;
|
||||||
|
if (!s || s[0] == '\0' || s[0] == '/' || has_path_traversal(s) ||
|
||||||
|
entry_size > MAX_MANIFEST_BYTES - *manifest_bytes ||
|
||||||
|
(*manifest_bytes += entry_size) > MAX_MANIFEST_BYTES || !array_list_add(list, s)) {
|
||||||
|
free(s);
|
||||||
|
send_status(fd, STATUS_ERROR);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
*manifest_entries += (size_t)count;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
DeleteManifest* receive_manifest_entries(int fd) {
|
||||||
|
DeleteManifest* manifest = calloc(1, sizeof(DeleteManifest));
|
||||||
|
if (!manifest) {
|
||||||
|
send_status(fd, STATUS_ERROR);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
manifest->keeps = array_list_create(free);
|
||||||
|
manifest->protected = array_list_create(free);
|
||||||
|
manifest->missing = array_list_create(free);
|
||||||
|
manifest->dirs = array_list_create(free);
|
||||||
|
if (!manifest->keeps || !manifest->protected || !manifest->missing || !manifest->dirs) {
|
||||||
|
delete_manifest_free(manifest);
|
||||||
|
send_status(fd, STATUS_ERROR);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
size_t manifest_bytes = 0;
|
||||||
|
size_t manifest_entries = 0;
|
||||||
|
if (!receive_manifest_section(fd, manifest->keeps, &manifest_bytes, &manifest_entries) ||
|
||||||
|
!receive_manifest_section(fd, manifest->protected, &manifest_bytes, &manifest_entries) ||
|
||||||
|
!receive_manifest_section(fd, manifest->missing, &manifest_bytes, &manifest_entries) ||
|
||||||
|
!receive_manifest_section(fd, manifest->dirs, &manifest_bytes, &manifest_entries)) {
|
||||||
|
delete_manifest_free(manifest);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
return manifest;
|
||||||
|
}
|
||||||
|
|
||||||
|
void delete_manifest_free(DeleteManifest* manifest) {
|
||||||
|
if (!manifest)
|
||||||
|
return;
|
||||||
|
array_list_delete(manifest->keeps);
|
||||||
|
array_list_delete(manifest->protected);
|
||||||
|
array_list_delete(manifest->missing);
|
||||||
|
array_list_delete(manifest->dirs);
|
||||||
|
free(manifest);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Shared --max-delete budget for one receiver-side deletion commit. Both the
|
||||||
|
--delete-missing-args exact-path removals and the ordinary extras walk draw
|
||||||
|
from the same tally, matching rsync (whose --max-delete counts every deleted
|
||||||
|
file or directory). `max_delete` is SIZE_MAX for an unlimited budget. */
|
||||||
|
typedef struct {
|
||||||
|
size_t max_delete;
|
||||||
|
size_t deleted;
|
||||||
|
size_t skipped;
|
||||||
|
bool limit_hit;
|
||||||
|
} DeleteBudgetState;
|
||||||
|
|
||||||
|
/* Remove every destination entry under the receive root that is not in the
|
||||||
|
keep-set, bounded by the shared budget (a smaller client --max-delete=NUM
|
||||||
|
replaces the server hard bound; rsync deletes up to the bound and skips the
|
||||||
|
rest). With --delay-updates the not-yet-published staging directory is a
|
||||||
|
direct child of the receive root and must not be treated as a set of extras;
|
||||||
|
the manifest's protected prefixes (paths excluded on the source), the
|
||||||
|
size-pruned prefixes (--max-size/--min-size, always protected) and the
|
||||||
|
alternate basis directories are never destination content and are skipped at
|
||||||
|
any depth. Returns true unless a traversal/unlink error aborted the walk;
|
||||||
|
the budget's limit_hit/skipped fields report a cap-stopped run. */
|
||||||
|
static bool delete_extras_budgeted_observed(const Config* config, const DeleteManifest* manifest,
|
||||||
|
DeleteBudgetState* budget, DeletePathObserver observer,
|
||||||
|
void* observer_context) {
|
||||||
|
if (!config || !manifest || !manifest->keeps)
|
||||||
|
return false;
|
||||||
|
fprintf(stderr, "Deleting files not in manifest...\n");
|
||||||
|
/* Protected entries: the --delay-updates staging name (only as a DIRECT child
|
||||||
|
of the receive root), the alternate basis directories and the sender-side
|
||||||
|
protected prefixes (filter-excluded and size-pruned source mirrors), all at
|
||||||
|
any depth. See delete_skips_build(). */
|
||||||
|
DeleteSkipSet skips;
|
||||||
|
if (!delete_skips_build(config, manifest->protected, NULL, true, &skips))
|
||||||
|
return false;
|
||||||
|
/* Clamp rather than subtract: an accounting bug where deleted already exceeds
|
||||||
|
max_delete must never underflow into an effectively unlimited budget. */
|
||||||
|
size_t remaining;
|
||||||
|
if (budget->max_delete == SIZE_MAX)
|
||||||
|
remaining = SIZE_MAX;
|
||||||
|
else if (budget->deleted >= budget->max_delete)
|
||||||
|
remaining = 0;
|
||||||
|
else
|
||||||
|
remaining = budget->max_delete - budget->deleted;
|
||||||
|
size_t deleted = 0;
|
||||||
|
size_t skipped = 0;
|
||||||
|
DeleteWalkResult result = delete_extras_limited_observed(
|
||||||
|
config->receive_root_directory, manifest->keeps, manifest->dirs, remaining, skips.entries,
|
||||||
|
skips.count, config->protect_rules, &deleted, &skipped, observer, observer_context);
|
||||||
|
delete_skips_free(&skips);
|
||||||
|
budget->deleted += deleted;
|
||||||
|
budget->skipped += skipped;
|
||||||
|
if (result == DELETE_WALK_LIMIT_REACHED) {
|
||||||
|
budget->limit_hit = true;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (result != DELETE_WALK_OK) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "deletion failed while removing extraneous files");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool delete_extras_budgeted(const Config* config, const DeleteManifest* manifest,
|
||||||
|
DeleteBudgetState* budget) {
|
||||||
|
return delete_extras_budgeted_observed(config, manifest, budget, NULL, NULL);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Prefixes every observed path with a fixed subtree root, so a nested walk
|
||||||
|
(a recursively removed missing-arg directory) reports receive-root-relative
|
||||||
|
names like the rest of the delete output. */
|
||||||
|
typedef struct {
|
||||||
|
DeletePathObserver inner;
|
||||||
|
void* inner_context;
|
||||||
|
const char* prefix;
|
||||||
|
} PrefixedDeleteObserver;
|
||||||
|
|
||||||
|
static void prefixed_delete_observer(void* context, const char* rel) {
|
||||||
|
PrefixedDeleteObserver* prefixed = context;
|
||||||
|
if (!prefixed->inner || !rel)
|
||||||
|
return;
|
||||||
|
char* joined = path_cat((char*)prefixed->prefix, rel);
|
||||||
|
if (joined) {
|
||||||
|
prefixed->inner(prefixed->inner_context, joined);
|
||||||
|
free(joined);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* --delete-missing-args exact-path deletions: each destination mirror in
|
||||||
|
manifest->missing is an explicit user request, so it is removed even when the
|
||||||
|
ordinary extras walk (with its protected prefixes) would leave it alone. The
|
||||||
|
--delay-updates staging directory and basis snapshots are receiver artifacts
|
||||||
|
and stay protected exactly as in the extras walker. A regular file or
|
||||||
|
symlink is unlinked, an empty directory removed, and a NON-empty directory is
|
||||||
|
removed recursively only when --delete or --force is in effect (rsync parity:
|
||||||
|
the man page says a non-empty directory mirror is only deleted with --force
|
||||||
|
or --delete); otherwise it is left with a warning and the run continues. A
|
||||||
|
mirror that does not exist is a no-op. Each removal draws from the shared
|
||||||
|
--max-delete budget: once it is exhausted the remaining requests are skipped
|
||||||
|
and counted. Returns false only on a genuine error (a confinement failure on
|
||||||
|
a validated path or an I/O error), which fails the run. */
|
||||||
|
static bool delete_missing_args_budgeted_observed(const Config* config,
|
||||||
|
const DeleteManifest* manifest,
|
||||||
|
DeleteBudgetState* budget,
|
||||||
|
DeletePathObserver observer,
|
||||||
|
void* observer_context) {
|
||||||
|
if (!config || !manifest)
|
||||||
|
return false;
|
||||||
|
if (!manifest->missing || manifest->missing->size == 0)
|
||||||
|
return true;
|
||||||
|
fprintf(stderr, "Deleting destination mirrors of missing source arguments...\n");
|
||||||
|
/* The staging directory and basis snapshots stay protected exactly as in the
|
||||||
|
extras walker (the missing-args path overrides the ordinary protected
|
||||||
|
prefixes, so those are not passed here). */
|
||||||
|
DeleteSkipSet skips;
|
||||||
|
if (!delete_skips_build(config, NULL, NULL, true, &skips))
|
||||||
|
return false;
|
||||||
|
bool ok = true;
|
||||||
|
for (int i = 0; i < manifest->missing->size; i++) {
|
||||||
|
const char* rel = (const char*)manifest->missing->items[i];
|
||||||
|
if (!rel || *rel == '\0' || *rel == '/' || has_path_traversal(rel)) {
|
||||||
|
/* Defensive only: receive_manifest_entries already validated every
|
||||||
|
section identically, so a controlled peer never reaches this branch. */
|
||||||
|
log_message(LOG_LEVEL_ERROR, "invalid missing-args delete path");
|
||||||
|
ok = false;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
bool at_root = strchr(rel, '/') == NULL;
|
||||||
|
if (path_under_skip_prefix(rel, at_root, skips.entries, skips.count)) {
|
||||||
|
char* escaped = output_escape(rel, log_get_8_bit_output());
|
||||||
|
log_message(LOG_LEVEL_WARNING,
|
||||||
|
"missing-args path '%s' is protected (staging directory or basis snapshot); "
|
||||||
|
"not deleting",
|
||||||
|
escaped ? escaped : "<allocation failed>");
|
||||||
|
free(escaped);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
char* full = path_cat(config->receive_root_directory, rel);
|
||||||
|
if (!full) {
|
||||||
|
ok = false;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
char* leaf = NULL;
|
||||||
|
int parent_fd = file_open_secure_parent(full, &leaf, false);
|
||||||
|
if (parent_fd < 0) {
|
||||||
|
/* The mirror's parent directory may itself not exist on the destination
|
||||||
|
(a deeper missing entry whose leading directories were never created).
|
||||||
|
That is a no-op -- there is nothing to delete -- matching
|
||||||
|
file_remove_tree_secure's absent-path handling; only a genuine I/O
|
||||||
|
error (EACCES, a symlink loop, ...) fails the run. */
|
||||||
|
bool absent = errno == ENOENT || errno == ENOTDIR;
|
||||||
|
free(full);
|
||||||
|
free(leaf);
|
||||||
|
if (!absent)
|
||||||
|
ok = false;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
struct stat st;
|
||||||
|
if (fstatat(parent_fd, leaf, &st, AT_SYMLINK_NOFOLLOW) != 0) {
|
||||||
|
/* Already absent: nothing to delete (a no-op, not a deletion). */
|
||||||
|
if (errno != ENOENT)
|
||||||
|
ok = false;
|
||||||
|
close(parent_fd);
|
||||||
|
free(leaf);
|
||||||
|
free(full);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
/* An entry that exists is one deletion: skip it (and count it) when the
|
||||||
|
shared --max-delete budget is already exhausted. */
|
||||||
|
if (budget->deleted >= budget->max_delete) {
|
||||||
|
budget->limit_hit = true;
|
||||||
|
budget->skipped++;
|
||||||
|
close(parent_fd);
|
||||||
|
free(leaf);
|
||||||
|
free(full);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
bool removed = false;
|
||||||
|
if (S_ISDIR(st.st_mode)) {
|
||||||
|
if (unlinkat(parent_fd, leaf, AT_REMOVEDIR) == 0) {
|
||||||
|
removed = true;
|
||||||
|
} else if (errno == ENOTEMPTY || errno == EEXIST) {
|
||||||
|
close(parent_fd);
|
||||||
|
parent_fd = -1;
|
||||||
|
free(leaf);
|
||||||
|
leaf = NULL;
|
||||||
|
if (config->use_delete || config->force_delete) {
|
||||||
|
/* Remove the contents entry-by-entry through the budgeted extras
|
||||||
|
walker so every deleted file/dir counts toward --max-delete (rsync
|
||||||
|
parity); the now-empty directory itself costs one more. A run that
|
||||||
|
hits the cap leaves the remaining entries in place. */
|
||||||
|
ArrayList* no_keeps = array_list_create(free);
|
||||||
|
/* Never let an accounting slip (deleted > max_delete) underflow the
|
||||||
|
remaining budget into SIZE_MAX, which would grant unlimited
|
||||||
|
deletions. */
|
||||||
|
size_t remaining =
|
||||||
|
budget->deleted >= budget->max_delete ? 0 : budget->max_delete - budget->deleted;
|
||||||
|
size_t contents_deleted = 0;
|
||||||
|
size_t contents_skipped = 0;
|
||||||
|
PrefixedDeleteObserver nested = {observer, observer_context, rel};
|
||||||
|
DeleteWalkResult walk =
|
||||||
|
no_keeps ? delete_extras_limited_observed(full, no_keeps, NULL, remaining, NULL, 0,
|
||||||
|
NULL, &contents_deleted, &contents_skipped,
|
||||||
|
observer ? prefixed_delete_observer : NULL,
|
||||||
|
observer ? &nested : NULL)
|
||||||
|
: DELETE_WALK_ERROR;
|
||||||
|
if (no_keeps)
|
||||||
|
array_list_delete(no_keeps);
|
||||||
|
budget->deleted += contents_deleted;
|
||||||
|
budget->skipped += contents_skipped;
|
||||||
|
if (walk == DELETE_WALK_LIMIT_REACHED) {
|
||||||
|
budget->limit_hit = true;
|
||||||
|
} else if (walk != DELETE_WALK_OK) {
|
||||||
|
ok = false;
|
||||||
|
} else if (budget->deleted >= budget->max_delete) {
|
||||||
|
budget->limit_hit = true;
|
||||||
|
budget->skipped++;
|
||||||
|
} else if (file_remove_tree_secure(full)) {
|
||||||
|
/* The shared `if (removed)` tail charges this directory exactly
|
||||||
|
once; counting it here too would consume two budget units. */
|
||||||
|
removed = true;
|
||||||
|
} else {
|
||||||
|
ok = false;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
char* escaped = output_escape(rel, log_get_8_bit_output());
|
||||||
|
log_message(LOG_LEVEL_WARNING,
|
||||||
|
"missing-args destination '%s' is a non-empty directory; use --force or "
|
||||||
|
"--delete to remove it",
|
||||||
|
escaped ? escaped : "<allocation failed>");
|
||||||
|
free(escaped);
|
||||||
|
}
|
||||||
|
} else if (errno != ENOENT) {
|
||||||
|
ok = false;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if (unlinkat(parent_fd, leaf, 0) == 0) {
|
||||||
|
removed = true;
|
||||||
|
} else if (errno != ENOENT) {
|
||||||
|
ok = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (removed) {
|
||||||
|
budget->deleted++;
|
||||||
|
if (observer)
|
||||||
|
observer(observer_context, rel);
|
||||||
|
char* escaped = output_escape(rel, log_get_8_bit_output());
|
||||||
|
fprintf(stderr, " Deleted: %s\n", escaped ? escaped : "<allocation failed>");
|
||||||
|
free(escaped);
|
||||||
|
}
|
||||||
|
if (parent_fd >= 0)
|
||||||
|
close(parent_fd);
|
||||||
|
free(leaf);
|
||||||
|
free(full);
|
||||||
|
if (!ok)
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
delete_skips_free(&skips);
|
||||||
|
return ok;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Public wrappers used outside the commit path (and by unit tests): no
|
||||||
|
--max-delete budget. */
|
||||||
|
bool manifest_would_delete_list(const Config* config, const DeleteManifest* manifest,
|
||||||
|
ArrayList* out, size_t* count_out) {
|
||||||
|
if (count_out)
|
||||||
|
*count_out = 0;
|
||||||
|
if (!config || !manifest || !manifest->keeps || !out)
|
||||||
|
return false;
|
||||||
|
DeleteSkipSet skips;
|
||||||
|
if (!delete_skips_build(config, manifest->protected, NULL, true, &skips))
|
||||||
|
return false;
|
||||||
|
bool ok = delete_extras_list(config->receive_root_directory, manifest->keeps, manifest->dirs,
|
||||||
|
skips.entries, skips.count, config->protect_rules, out, count_out);
|
||||||
|
delete_skips_free(&skips);
|
||||||
|
return ok;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool manifest_delete_extras(const Config* config, const DeleteManifest* manifest) {
|
||||||
|
DeleteBudgetState budget = {
|
||||||
|
.max_delete = SIZE_MAX, .deleted = 0, .skipped = 0, .limit_hit = false};
|
||||||
|
return delete_extras_budgeted(config, manifest, &budget);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool manifest_delete_missing_args(const Config* config, const DeleteManifest* manifest) {
|
||||||
|
DeleteBudgetState budget = {
|
||||||
|
.max_delete = SIZE_MAX, .deleted = 0, .skipped = 0, .limit_hit = false};
|
||||||
|
return delete_missing_args_budgeted_observed(config, manifest, &budget, NULL, NULL);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool manifest_delete_missing_args_limited(const Config* config, const DeleteManifest* manifest,
|
||||||
|
size_t max_delete, size_t* deleted, size_t* skipped,
|
||||||
|
bool* limit_hit) {
|
||||||
|
return manifest_delete_missing_args_limited_observed(config, manifest, max_delete, deleted,
|
||||||
|
skipped, limit_hit, NULL, NULL);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool manifest_delete_missing_args_limited_observed(
|
||||||
|
const Config* config, const DeleteManifest* manifest, size_t max_delete, size_t* deleted,
|
||||||
|
size_t* skipped, bool* limit_hit, DeletePathObserver observer, void* observer_context) {
|
||||||
|
DeleteBudgetState budget = {
|
||||||
|
.max_delete = max_delete, .deleted = 0, .skipped = 0, .limit_hit = false};
|
||||||
|
bool ok =
|
||||||
|
delete_missing_args_budgeted_observed(config, manifest, &budget, observer, observer_context);
|
||||||
|
if (deleted)
|
||||||
|
*deleted = budget.deleted;
|
||||||
|
if (skipped)
|
||||||
|
*skipped = budget.skipped;
|
||||||
|
if (limit_hit)
|
||||||
|
*limit_hit = budget.limit_hit;
|
||||||
|
return ok;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Commit every deletion family the manifest carries. The --delete-missing-args
|
||||||
|
exact-path deletions run FIRST: they are explicit user requests and must not
|
||||||
|
be blocked by the extras walker's filter-exclusion protection (a protected
|
||||||
|
leftover inside a missing-argument directory must not make that user-requested
|
||||||
|
removal fail). The ordinary extras walk then runs when --delete is active.
|
||||||
|
Both draw from one --max-delete budget; the result reports a cap-stopped
|
||||||
|
(partial) commit distinctly so the client can exit 25 like rsync. */
|
||||||
|
DeleteCommitResult manifest_delete_all(const Config* config, const DeleteManifest* manifest) {
|
||||||
|
return manifest_delete_all_counted(config, manifest, NULL);
|
||||||
|
}
|
||||||
|
|
||||||
|
DeleteCommitResult manifest_delete_all_counted(const Config* config, const DeleteManifest* manifest,
|
||||||
|
size_t* deleted) {
|
||||||
|
return manifest_delete_all_observed(config, manifest, deleted, NULL, NULL);
|
||||||
|
}
|
||||||
|
|
||||||
|
DeleteCommitResult manifest_delete_all_observed(const Config* config,
|
||||||
|
const DeleteManifest* manifest, size_t* deleted,
|
||||||
|
DeletePathObserver observer,
|
||||||
|
void* observer_context) {
|
||||||
|
if (deleted)
|
||||||
|
*deleted = 0;
|
||||||
|
if (!config || !manifest)
|
||||||
|
return DELETE_COMMIT_ERROR;
|
||||||
|
/* Central no-mutation guard: a dry-run never deletes. No manifest is sent on
|
||||||
|
the dry-run path, but a hostile/buggy peer could; treat it as a no-op so
|
||||||
|
the receiver can never remove anything. */
|
||||||
|
if (config->dry_run)
|
||||||
|
return DELETE_COMMIT_OK;
|
||||||
|
/* A client --max-delete=NUM smaller than the server's hard bound replaces it
|
||||||
|
for this run; both still bound the commit. */
|
||||||
|
bool user_limited =
|
||||||
|
config->max_delete >= 0 && (size_t)config->max_delete < MAX_SERVER_DELETE_COUNT;
|
||||||
|
DeleteBudgetState budget = {.max_delete = user_limited ? (size_t)config->max_delete
|
||||||
|
: MAX_SERVER_DELETE_COUNT,
|
||||||
|
.deleted = 0,
|
||||||
|
.skipped = 0,
|
||||||
|
.limit_hit = false};
|
||||||
|
if (config->delete_missing_args &&
|
||||||
|
!delete_missing_args_budgeted_observed(config, manifest, &budget, observer, observer_context))
|
||||||
|
return DELETE_COMMIT_ERROR;
|
||||||
|
if (config->use_delete &&
|
||||||
|
!delete_extras_budgeted_observed(config, manifest, &budget, observer, observer_context))
|
||||||
|
return DELETE_COMMIT_ERROR;
|
||||||
|
if (deleted)
|
||||||
|
*deleted = budget.deleted;
|
||||||
|
if (budget.limit_hit) {
|
||||||
|
if (user_limited) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Deletions stopped due to --max-delete limit (%zu skipped)",
|
||||||
|
budget.skipped);
|
||||||
|
} else {
|
||||||
|
log_message(LOG_LEVEL_ERROR,
|
||||||
|
"Deletions stopped due to the server deletion limit of %u (%zu skipped)",
|
||||||
|
(unsigned)MAX_SERVER_DELETE_COUNT, budget.skipped);
|
||||||
|
}
|
||||||
|
return DELETE_COMMIT_LIMIT_REACHED;
|
||||||
|
}
|
||||||
|
return DELETE_COMMIT_OK;
|
||||||
|
}
|
||||||
@@ -0,0 +1,105 @@
|
|||||||
|
#ifndef DELETE_COMMIT_H
|
||||||
|
#define DELETE_COMMIT_H
|
||||||
|
|
||||||
|
#include "array_list.h"
|
||||||
|
#include "config.h"
|
||||||
|
#include "delete.h"
|
||||||
|
#include <stdbool.h>
|
||||||
|
|
||||||
|
/* Delete-commit module: delete-manifest receive plus the budgeted extras and
|
||||||
|
* --delete-missing-args walkers. These declarations are re-exported by the
|
||||||
|
* file_receive.h facade. */
|
||||||
|
|
||||||
|
/* A received delete-manifest frame: the keep-set (`keeps`, destination-relative
|
||||||
|
paths the sender transferred/keeps) plus `protected`, destination-relative
|
||||||
|
prefixes the sender asks the receiver never to delete (paths excluded on the
|
||||||
|
source, protected at any depth). When --delete-excluded is given the sender
|
||||||
|
transmits an empty protected list so excluded destination mirrors are treated
|
||||||
|
as ordinary extras. With --delete-missing-args a third section (`missing`)
|
||||||
|
carries the destination mirrors of explicitly-listed source entries that do
|
||||||
|
not exist: each is an exact deletion request, independent of the ordinary
|
||||||
|
extras walk (never blocked by the protected prefixes) and processed when the
|
||||||
|
manifest is committed. */
|
||||||
|
typedef struct DeleteManifest {
|
||||||
|
ArrayList* keeps;
|
||||||
|
ArrayList* protected;
|
||||||
|
ArrayList* missing;
|
||||||
|
/* Destination-relative paths of the directories the sender synchronized for
|
||||||
|
this run. The extras walker only removes entries directly inside one of
|
||||||
|
these (the receive root is the "." sentinel); `--files-from` runs therefore
|
||||||
|
leave untransmitted directories and the unlisted parts of listed ones
|
||||||
|
alone, matching rsync's "delete only in synchronized directories". */
|
||||||
|
ArrayList* dirs;
|
||||||
|
} DeleteManifest;
|
||||||
|
|
||||||
|
void delete_manifest_free(DeleteManifest* manifest);
|
||||||
|
/* Read a delete-manifest frame (protocol 2.23.0): keep count + keeps, then
|
||||||
|
protected count + protected prefixes, then missing count + missing paths,
|
||||||
|
then synchronized-directory count + directory paths (self-delimiting; the
|
||||||
|
leading STATUS_MANIFEST code has been consumed). Returns an owned
|
||||||
|
DeleteManifest, or NULL after signalling STATUS_ERROR on a malformed frame. */
|
||||||
|
DeleteManifest* receive_manifest_entries(int fd);
|
||||||
|
/* Remove destination entries under config->receive_root_directory that are not
|
||||||
|
in `manifest` (bounded, all-or-nothing walk; staging-dir, basis-dir and
|
||||||
|
protected-prefix skips). `--max-delete` and `--force` are honored here. The
|
||||||
|
caller decides WHEN to run it based on the negotiated delete timing. Returns
|
||||||
|
false (and the transfer fails) when the deletion cannot be committed. */
|
||||||
|
bool manifest_delete_extras(const Config* config, const DeleteManifest* manifest);
|
||||||
|
/* --delete-missing-args exact-path deletions: remove each destination mirror
|
||||||
|
in `manifest->missing` (never blocked by the protected prefixes, staging dir
|
||||||
|
and basis dirs excluded). A regular file/symlink is unlinked; an empty
|
||||||
|
directory is removed; a NON-empty directory is removed recursively only when
|
||||||
|
--delete or --force is in effect, otherwise it is left with a warning (rsync
|
||||||
|
parity). A missing path is a no-op. Returns false only on a genuine
|
||||||
|
confinement or I/O error (the run then fails); tolerated per-path cases are
|
||||||
|
reported and skipped. */
|
||||||
|
bool manifest_delete_missing_args(const Config* config, const DeleteManifest* manifest);
|
||||||
|
/* Budgeted form of manifest_delete_missing_args for the per-directory delete
|
||||||
|
session: each removed mirror draws from `max_delete` (SIZE_MAX = unlimited)
|
||||||
|
and the tallies are accumulated into `*deleted`/`*skipped`. `*limit_hit` is set
|
||||||
|
when the budget stopped the pass with entries left over. Returns false only
|
||||||
|
on a genuine deletion error. */
|
||||||
|
bool manifest_delete_missing_args_limited(const Config* config, const DeleteManifest* manifest,
|
||||||
|
size_t max_delete, size_t* deleted, size_t* skipped,
|
||||||
|
bool* limit_hit);
|
||||||
|
/* Observer-aware form of manifest_delete_missing_args_limited: `observer` (may
|
||||||
|
be NULL) is invoked for every destination-relative path truly removed. */
|
||||||
|
bool manifest_delete_missing_args_limited_observed(
|
||||||
|
const Config* config, const DeleteManifest* manifest, size_t max_delete, size_t* deleted,
|
||||||
|
size_t* skipped, bool* limit_hit, DeletePathObserver observer, void* observer_context);
|
||||||
|
/* Outcome of committing a delete manifest. LIMIT_REACHED reports rsync's
|
||||||
|
partial --max-delete result: the budget allowed some deletions and the rest
|
||||||
|
were skipped (the run still stores all file data but the client exits 25). */
|
||||||
|
typedef enum {
|
||||||
|
DELETE_COMMIT_OK = 0,
|
||||||
|
DELETE_COMMIT_LIMIT_REACHED,
|
||||||
|
DELETE_COMMIT_ERROR
|
||||||
|
} DeleteCommitResult;
|
||||||
|
|
||||||
|
/* Run every deletion family the manifest carries: the --delete-missing-args
|
||||||
|
exact-path deletions first (user requests are not blocked by exclusion
|
||||||
|
protection), then the ordinary extras walk when --delete is active. Both
|
||||||
|
share one --max-delete budget. Returns DELETE_COMMIT_OK when nothing was to
|
||||||
|
do or everything committed, DELETE_COMMIT_LIMIT_REACHED when the budget
|
||||||
|
stopped part of the work, or DELETE_COMMIT_ERROR on a genuine failure. */
|
||||||
|
DeleteCommitResult manifest_delete_all(const Config* config, const DeleteManifest* manifest);
|
||||||
|
/* Like manifest_delete_all, but reports how many destination entries the commit
|
||||||
|
removed (for the end-of-transfer wire stats). `deleted` may be NULL. */
|
||||||
|
DeleteCommitResult manifest_delete_all_counted(const Config* config, const DeleteManifest* manifest,
|
||||||
|
size_t* deleted);
|
||||||
|
/* Observer-aware form of manifest_delete_all_counted: `observer` (may be NULL)
|
||||||
|
is invoked for every destination-relative path truly removed. */
|
||||||
|
DeleteCommitResult manifest_delete_all_observed(const Config* config,
|
||||||
|
const DeleteManifest* manifest, size_t* deleted,
|
||||||
|
DeletePathObserver observer,
|
||||||
|
void* observer_context);
|
||||||
|
|
||||||
|
/* -n/--dry-run --delete would-delete reporting: walk the destination exactly as
|
||||||
|
the delete pass would and append (strdup'd) destination-relative paths that
|
||||||
|
WOULD be removed to `out`, without touching disk. Uses the same staging-dir,
|
||||||
|
basis-dir and protected-prefix skips as the real commit. Returns true on a
|
||||||
|
clean walk; `*count_out` receives the number of paths appended. */
|
||||||
|
bool manifest_would_delete_list(const Config* config, const DeleteManifest* manifest,
|
||||||
|
ArrayList* out, size_t* count_out);
|
||||||
|
|
||||||
|
#endif
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,106 @@
|
|||||||
|
#ifndef DELETE_PLAN_H
|
||||||
|
#define DELETE_PLAN_H
|
||||||
|
|
||||||
|
#include "array_list.h"
|
||||||
|
#include "config.h"
|
||||||
|
#include "delete.h"
|
||||||
|
#include "file_receive.h"
|
||||||
|
#include "protocol.h"
|
||||||
|
#include "utils.h"
|
||||||
|
#include <stdbool.h>
|
||||||
|
|
||||||
|
/* Per-directory delete plans (protocol 2.24.0).
|
||||||
|
*
|
||||||
|
* rsync's --delete-during removes a directory's extras while the generator
|
||||||
|
* processes that directory, and --delete-delay records the deletion list during
|
||||||
|
* the scan but applies it only after a fully-successful transfer. FastSync has
|
||||||
|
* no per-directory generator pass; instead the sender streams one plan per
|
||||||
|
* source directory, in directory order, and the receiver applies it when it
|
||||||
|
* arrives (during) or snapshots its extras and commits them at the end (delay).
|
||||||
|
*
|
||||||
|
* The sender side builds a plan set from the path-only pre-scan (it needs every
|
||||||
|
* directory's complete direct-child list before the first data byte of that
|
||||||
|
* directory). The receiver side is a session that carries the global protected
|
||||||
|
* prefixes (filter-excluded and size-skipped source mirrors), the
|
||||||
|
* --delete-missing-args exact deletions, the shared --max-delete budget and,
|
||||||
|
* for --delete-delay, the snapshotted extras. */
|
||||||
|
|
||||||
|
/* ---- Sender: plan builder ---- */
|
||||||
|
|
||||||
|
typedef struct DeletePlanSender DeletePlanSender;
|
||||||
|
|
||||||
|
DeletePlanSender* delete_plan_sender_create(void);
|
||||||
|
void delete_plan_sender_destroy(DeletePlanSender* sender);
|
||||||
|
/* Record one transmitted entry. `path` is the destination-relative wire path;
|
||||||
|
* is_dir marks an explicit directory entry (--dirs, a -x mount point). */
|
||||||
|
bool delete_plan_sender_add(DeletePlanSender* sender, const char* path, bool is_dir);
|
||||||
|
/* Drop plans for directories outside `synced_dirs` (the --files-from
|
||||||
|
* synchronization scope; pass NULL when a full recursive transfer synchronized
|
||||||
|
* every directory). The receive root is the "." sentinel.
|
||||||
|
*
|
||||||
|
* `walk_root` scopes a general -R transfer: when non-NULL it is the
|
||||||
|
* reconstructed destination prefix the run actually transferred, and only the
|
||||||
|
* plan for that prefix (and directories below it) is ever transmitted, so the
|
||||||
|
* prefix's parent-directory siblings are never walked. Pass NULL for a plain
|
||||||
|
* recursive transfer and for --files-from. */
|
||||||
|
void delete_plan_sender_finalize(DeletePlanSender* sender, const ArrayList* synced_dirs,
|
||||||
|
const char* walk_root);
|
||||||
|
/* True when no transmitted FILE entry was recorded (an ambiguous empty scan).
|
||||||
|
Directory keep entries do not count, so an I/O error that hid every file
|
||||||
|
still refuses to delete. */
|
||||||
|
bool delete_plan_sender_empty(const DeletePlanSender* sender);
|
||||||
|
/* Attach the global config sections advertised on the first plan frame. The
|
||||||
|
* block is always transmitted by delete_plan_send_root(), on a config-only
|
||||||
|
* carrier frame when the scope allows no directory plan. */
|
||||||
|
void delete_plan_sender_set_config(DeletePlanSender* sender, const ArrayList* protected_prefixes,
|
||||||
|
const ArrayList* size_skipped, const ArrayList* missing_args);
|
||||||
|
/* Send the root plan (even before any data, so root extras are handled like
|
||||||
|
* rsync's first generator directory), after transmitting the per-run config
|
||||||
|
* block on its own carrier frame. Returns -1 on I/O error. */
|
||||||
|
int delete_plan_send_root(int fd, DeletePlanSender* sender);
|
||||||
|
/* Send the plans for every ancestor of `path` (root-first) and, when is_dir,
|
||||||
|
* for `path` itself; already-sent plans are skipped. */
|
||||||
|
int delete_plan_send_for_path(int fd, DeletePlanSender* sender, const char* path, bool is_dir);
|
||||||
|
/* Send the plan for every directory in `dirs` that has not been transmitted
|
||||||
|
* yet. */
|
||||||
|
int delete_plan_send_remaining(int fd, DeletePlanSender* sender, const ArrayList* dirs);
|
||||||
|
/* Transmit the COMPLETE per-directory plan set in one pass, before any data
|
||||||
|
* frame: the root plan (with the one-shot per-run config block on its carrier
|
||||||
|
* frame) followed by every directory in `dirs`. Because the whole plan set is
|
||||||
|
* known from the path-only pre-scan, sending it all up front means a
|
||||||
|
* mid-transfer abort has already applied every planned removal, matching
|
||||||
|
* rsync's generator (which runs ahead of its throttled sender). A completed
|
||||||
|
* run is unaffected. `dirs` is the set of directories whose direct children
|
||||||
|
* were enumerated (the scanner's plan_dirs sink), so a merely listed but
|
||||||
|
* untraversed directory never gets a plan and its mirror is left intact.
|
||||||
|
* Returns -1 on I/O error. */
|
||||||
|
int delete_plan_send_all(int fd, DeletePlanSender* sender, const ArrayList* dirs);
|
||||||
|
|
||||||
|
/* ---- Receiver: delete session ---- */
|
||||||
|
|
||||||
|
typedef struct DeletePlanSession DeletePlanSession;
|
||||||
|
|
||||||
|
DeletePlanSession* delete_plan_session_create(const Config* config);
|
||||||
|
void delete_plan_session_destroy(DeletePlanSession* session);
|
||||||
|
/* Read one STATUS_DELETE_PLAN frame (the leading status already consumed) and
|
||||||
|
* act on it. Returns 0 on success (including a dry-run/disabled no-op) and -1
|
||||||
|
* after signalling STATUS_ERROR on a malformed frame or a deletion failure. */
|
||||||
|
int delete_plan_session_receive(DeletePlanSession* session, const Config* config, int fd);
|
||||||
|
/* Apply the deferred snapshot (--delete-delay) and the missing-args deletions.
|
||||||
|
* Safe to call once; returns the commit outcome. */
|
||||||
|
DeleteCommitResult delete_plan_session_commit(DeletePlanSession* session, const Config* config);
|
||||||
|
/* True once the shared --max-delete budget stopped part of a deletion. */
|
||||||
|
bool delete_plan_session_limit_reached(const DeletePlanSession* session);
|
||||||
|
/* Number of destination entries the session actually removed, for the
|
||||||
|
end-of-transfer stats. For --delete-delay this excludes a snapshotted entry
|
||||||
|
that survived (e.g. a refilled directory that failed ENOTEMPTY), even though
|
||||||
|
that entry already consumed --max-delete budget at snapshot time. */
|
||||||
|
size_t delete_plan_session_deleted(const DeletePlanSession* session);
|
||||||
|
/* Install an observer invoked for every destination-relative path the session
|
||||||
|
truly removes (including the deferred --delete-delay commit), so the receiver
|
||||||
|
can report rsync's `deleting PATH` lines through the terminal STATUS_STATS
|
||||||
|
record. Pass NULL/0 to clear. */
|
||||||
|
void delete_plan_session_set_delete_observer(DeletePlanSession* session,
|
||||||
|
DeletePathObserver observer, void* context);
|
||||||
|
|
||||||
|
#endif
|
||||||
+697
-122
File diff suppressed because it is too large
Load Diff
+90
-28
@@ -28,17 +28,36 @@ void file_metadata_destroy(void* metadata);
|
|||||||
/* --open-noatime process-wide sender policy; see file.c. */
|
/* --open-noatime process-wide sender policy; see file.c. */
|
||||||
void file_set_open_noatime(bool enable);
|
void file_set_open_noatime(bool enable);
|
||||||
bool file_get_open_noatime(void);
|
bool file_get_open_noatime(void);
|
||||||
|
/* Capture the process umask ONCE, before any threads are created. Call this at
|
||||||
|
* the very top of main() in both entry points so the cached value is read while
|
||||||
|
* the process is still single-threaded: reading the umask needs a get+set round
|
||||||
|
* trip (umask(0); umask(old)), which would race against receiver threads
|
||||||
|
* creating files if it happened during the first write. Idempotent and safe to
|
||||||
|
* call more than once. */
|
||||||
|
void file_umask_capture(void);
|
||||||
|
/* Process-wide umask, captured once (thread-safe). Used to derive the mode of
|
||||||
|
* a brand-new destination like rsync: source_mode & 0777 & ~umask. Falls back
|
||||||
|
* to file_umask_capture() (behind pthread_once) if capture was never called. */
|
||||||
|
unsigned file_process_umask(void);
|
||||||
/* Open `path` read-only for transfer, honouring --open-noatime when set. */
|
/* Open `path` read-only for transfer, honouring --open-noatime when set. */
|
||||||
int file_open_for_read(const char* path);
|
int file_open_for_read(const char* path);
|
||||||
bool file_write_to_disk(const char* path, const void* data, unsigned long long data_size,
|
bool file_write_to_disk(const char* path, const void* data, unsigned long long data_size,
|
||||||
bool inplace, bool sparse);
|
bool inplace, bool sparse);
|
||||||
|
|
||||||
/* Symlink trust-boundary helpers (Phase 4, symlink wave). --munge-links
|
/* Symlink trust-boundary helpers (Phase 4, symlink wave; rsync parity).
|
||||||
* sender-side marker: every transmitted symlink target is prefixed with this
|
* --munge-links is a RECEIVER-side rewrite: rsync prefixes every stored symlink
|
||||||
* while the flag is on; the receiver strips it to restore the real target. */
|
* target with this marker, making the link unusable while the referenced
|
||||||
#define SYMLINK_MUNGE_PREFIX "#SYMLINK/"
|
* directory does not exist. A SENDER receiving a munged source strips it back
|
||||||
|
* off before transmitting (so a munged tree round-trips through the receiver's
|
||||||
|
* re-munging). */
|
||||||
|
#define SYMLINK_MUNGE_PREFIX "/rsyncd-munged/"
|
||||||
|
|
||||||
char* file_symlink_munge(const char* target);
|
char* file_symlink_munge(const char* target);
|
||||||
|
/* rsync 3.4.1 unsafe_symlink(): true when `target` escapes the transfer tree
|
||||||
|
* rooted at `link_path` (the symlink's transfer-relative path incl. its name).
|
||||||
|
* Absolute/empty targets and targets climbing above the transfer root (via
|
||||||
|
* "..") are unsafe, as are internal "/../" components and trailing "/..". */
|
||||||
|
bool file_symlink_unsafe(const char* target, const char* link_path);
|
||||||
/* True when a lexical target is relative and contains no ".." component, so it
|
/* True when a lexical target is relative and contains no ".." component, so it
|
||||||
* can never escape the receive root once created beneath it. */
|
* can never escape the receive root once created beneath it. */
|
||||||
bool file_symlink_target_contained(const char* target);
|
bool file_symlink_target_contained(const char* target);
|
||||||
@@ -46,8 +65,9 @@ bool file_symlink_target_contained(const char* target);
|
|||||||
* returns true when a marker was removed. */
|
* returns true when a marker was removed. */
|
||||||
bool file_symlink_unmunge(char* target);
|
bool file_symlink_unmunge(char* target);
|
||||||
/* Create a symlink at `path` -> `target`, confined below the authorized root
|
/* Create a symlink at `path` -> `target`, confined below the authorized root
|
||||||
* (O_NOFOLLOW parent walk, symlinkat; the target is never followed). Returns
|
* (O_NOFOLLOW parent walk, symlinkat; the target is never followed). The link
|
||||||
* false when a directory already occupies `path`. */
|
* value is copied verbatim (rsync -l); only the placement path is confined.
|
||||||
|
* Returns false when a directory already occupies `path`. */
|
||||||
bool file_symlink_at_secure(const char* path, const char* target);
|
bool file_symlink_at_secure(const char* path, const char* target);
|
||||||
/* --keep-dirlinks (-K) receiver process-wide policy: allow an in-root existing
|
/* --keep-dirlinks (-K) receiver process-wide policy: allow an in-root existing
|
||||||
* symlink-to-directory to be followed as a directory. */
|
* symlink-to-directory to be followed as a directory. */
|
||||||
@@ -67,6 +87,11 @@ bool file_path_exists_secure(const char* path);
|
|||||||
bool file_stat_secure(const char* path, struct stat* st);
|
bool file_stat_secure(const char* path, struct stat* st);
|
||||||
bool file_destination_is_newer_secure(const char* path, const FileMetadata* metadata);
|
bool file_destination_is_newer_secure(const char* path, const FileMetadata* metadata);
|
||||||
int file_open_secure_parent(const char* path, char** leaf_out, bool create_dirs);
|
int file_open_secure_parent(const char* path, char** leaf_out, bool create_dirs);
|
||||||
|
/* Protocol 2.28.0 variant: also increments *dirs_created for every missing
|
||||||
|
* parent directory this walk creates that lies strictly below `count_floor`
|
||||||
|
* (a receive-root-relative path, or NULL to count all of them). */
|
||||||
|
int file_open_secure_parent_counted(const char* path, char** leaf_out, bool create_dirs,
|
||||||
|
unsigned* dirs_created, const char* count_floor);
|
||||||
bool file_ensure_directory_secure(const char* path);
|
bool file_ensure_directory_secure(const char* path);
|
||||||
bool file_directory_exists_secure(const char* path);
|
bool file_directory_exists_secure(const char* path);
|
||||||
bool file_rename_secure(const char* old_path, const char* new_path);
|
bool file_rename_secure(const char* old_path, const char* new_path);
|
||||||
@@ -75,37 +100,44 @@ bool file_rename_secure(const char* old_path, const char* new_path);
|
|||||||
regular file. See the .c for the exact success semantics. */
|
regular file. See the .c for the exact success semantics. */
|
||||||
bool file_remove_tree_secure(const char* path);
|
bool file_remove_tree_secure(const char* path);
|
||||||
/* Open a private 0700 directory (creating it on demand) that must live below
|
/* Open a private 0700 directory (creating it on demand) that must live below
|
||||||
the authorized root. Used for the --temp-dir scratch directory and the
|
the authorized root. Used for the --delay-updates staging directory. */
|
||||||
--delay-updates staging directory. */
|
|
||||||
int file_open_private_dir(const char* dir_path);
|
int file_open_private_dir(const char* dir_path);
|
||||||
|
|
||||||
|
/* Open an existing --temp-dir scratch directory (relative or absolute; no
|
||||||
|
creation). When an authorized receive root is configured the directory's
|
||||||
|
REAL path (symlinks resolved) must lie within it, so a client-planted
|
||||||
|
symlink cannot redirect receiver scratch files outside the sandbox; an
|
||||||
|
in-root symlink to another filesystem is still allowed for rsync's EXDEV
|
||||||
|
fallback. */
|
||||||
|
int file_open_temp_dir(const char* dir_path);
|
||||||
|
|
||||||
/* The file_to_disk_secure* variants write a temporary copy in the destination
|
/* The file_to_disk_secure* variants write a temporary copy in the destination
|
||||||
directory and atomically rename it over `path`. temp_dir is an absolute,
|
directory and atomically rename it over `path`. temp_dir is a scratch
|
||||||
root-confined scratch directory (already validated by the caller): when it
|
directory (an absolute path, or one the caller already resolved against the
|
||||||
is non-NULL the temporary copy is instead created there (with a name unique
|
destination root): when it is non-NULL the temporary copy is instead created
|
||||||
across the whole scratch directory) and atomically renamed into the
|
there (with a name unique across the whole scratch directory) and atomically
|
||||||
destination directory once fully written and fsynced. A rename across
|
renamed into the destination directory once fully written and fsynced. When
|
||||||
filesystems (EXDEV) fails the write with an error; the file is never
|
that rename/link fails with EXDEV (the scratch dir is on another filesystem)
|
||||||
silently copied into place. Pass NULL for the historical same-directory
|
the write falls back to a non-atomic copy directly in the destination
|
||||||
behavior. --inplace writes never use temp_dir. */
|
directory, matching rsync. Pass NULL for the same-directory behavior.
|
||||||
|
--inplace writes never use temp_dir. */
|
||||||
bool file_to_disk_secure(const char* path, const void* data, unsigned long long data_size,
|
bool file_to_disk_secure(const char* path, const void* data, unsigned long long data_size,
|
||||||
bool inplace, bool sparse, bool preallocate, const FileMetadata* metadata,
|
bool inplace, bool sparse, bool preallocate, const FileMetadata* metadata,
|
||||||
bool preserve_executability, const char* temp_dir);
|
FileAttrPolicy policy, const char* temp_dir);
|
||||||
bool file_to_disk_secure_with_fsync(const char* path, const void* data,
|
bool file_to_disk_secure_with_fsync(const char* path, const void* data,
|
||||||
unsigned long long data_size, bool inplace, bool sparse,
|
unsigned long long data_size, bool inplace, bool sparse,
|
||||||
bool preallocate, const FileMetadata* metadata,
|
bool preallocate, const FileMetadata* metadata,
|
||||||
bool preserve_executability, bool use_fsync,
|
FileAttrPolicy policy, bool use_fsync, const char* temp_dir);
|
||||||
const char* temp_dir);
|
|
||||||
/* With update enabled, an existing newer destination is left untouched. The
|
/* With update enabled, an existing newer destination is left untouched. The
|
||||||
check is descriptor-based for inplace writes; atomic replacement still has
|
check is descriptor-based for inplace writes; atomic replacement still has
|
||||||
an unavoidable final rename race without filesystem locking. */
|
an unavoidable final rename race without filesystem locking. */
|
||||||
bool file_to_disk_secure_update(const char* path, const void* data, unsigned long long data_size,
|
bool file_to_disk_secure_update(const char* path, const void* data, unsigned long long data_size,
|
||||||
bool inplace, bool sparse, bool preallocate,
|
bool inplace, bool sparse, bool preallocate,
|
||||||
const FileMetadata* metadata, bool preserve_executability,
|
const FileMetadata* metadata, FileAttrPolicy policy,
|
||||||
const char* temp_dir);
|
const char* temp_dir);
|
||||||
bool file_to_disk_secure_no_replace(const char* path, const void* data,
|
bool file_to_disk_secure_no_replace(const char* path, const void* data,
|
||||||
unsigned long long data_size, bool sparse, bool preallocate,
|
unsigned long long data_size, bool sparse, bool preallocate,
|
||||||
const FileMetadata* metadata, bool preserve_executability,
|
const FileMetadata* metadata, FileAttrPolicy policy,
|
||||||
const char* temp_dir);
|
const char* temp_dir);
|
||||||
/* Receiver write-path variant that also applies per-file xattrs (-X/-A) and the
|
/* Receiver write-path variant that also applies per-file xattrs (-X/-A) and the
|
||||||
* --fake-super stat xattr fd-relative before the final rename. `update` /
|
* --fake-super stat xattr fd-relative before the final rename. `update` /
|
||||||
@@ -113,10 +145,9 @@ bool file_to_disk_secure_no_replace(const char* path, const void* data,
|
|||||||
* enables --partial best-effort retention of a failed write's temp. */
|
* enables --partial best-effort retention of a failed write's temp. */
|
||||||
bool file_to_disk_secure_attrs(const char* path, const void* data, unsigned long long data_size,
|
bool file_to_disk_secure_attrs(const char* path, const void* data, unsigned long long data_size,
|
||||||
bool inplace, bool sparse, bool preallocate,
|
bool inplace, bool sparse, bool preallocate,
|
||||||
const FileMetadata* metadata, bool preserve_executability,
|
const FileMetadata* metadata, FileAttrPolicy policy, bool update,
|
||||||
bool update, bool no_replace, bool use_fsync,
|
bool no_replace, bool use_fsync, const FileXattrList* xattrs,
|
||||||
const FileXattrList* xattrs, bool fake_super, bool keep_partial,
|
bool fake_super, bool keep_partial, const char* temp_dir);
|
||||||
const char* temp_dir);
|
|
||||||
/* Atomic --link-dest install: replace `path` with a hard link to `basis_path`
|
/* Atomic --link-dest install: replace `path` with a hard link to `basis_path`
|
||||||
(via a temp name + rename); fall back to a byte-identical local copy from
|
(via a temp name + rename); fall back to a byte-identical local copy from
|
||||||
`data` when the link is impossible (EXDEV/EPERM/unsupported filesystem).
|
`data` when the link is impossible (EXDEV/EPERM/unsupported filesystem).
|
||||||
@@ -125,16 +156,47 @@ bool file_to_disk_secure_attrs(const char* path, const void* data, unsigned long
|
|||||||
never re-allocated). */
|
never re-allocated). */
|
||||||
bool file_to_disk_secure_link(const char* path, const char* basis_path, const void* data,
|
bool file_to_disk_secure_link(const char* path, const char* basis_path, const void* data,
|
||||||
unsigned long long data_size, bool preallocate,
|
unsigned long long data_size, bool preallocate,
|
||||||
const FileMetadata* metadata, bool preserve_executability,
|
const FileMetadata* metadata, FileAttrPolicy policy, bool use_fsync,
|
||||||
bool use_fsync, const char* temp_dir);
|
const char* temp_dir);
|
||||||
/* Like file_to_disk_secure_link, but the byte-copy fallback also applies the
|
/* Like file_to_disk_secure_link, but the byte-copy fallback also applies the
|
||||||
* per-file xattrs (-X/-A) and --fake-super stat xattr (fd-relative). On a
|
* per-file xattrs (-X/-A) and --fake-super stat xattr (fd-relative). On a
|
||||||
* successful hard link no attributes are applied (the shared inode already
|
* successful hard link no attributes are applied (the shared inode already
|
||||||
* carries the basis's). */
|
* carries the basis's). */
|
||||||
bool file_to_disk_secure_link_attrs(const char* path, const char* basis_path, const void* data,
|
bool file_to_disk_secure_link_attrs(const char* path, const char* basis_path, const void* data,
|
||||||
unsigned long long data_size, bool preallocate,
|
unsigned long long data_size, bool preallocate,
|
||||||
const FileMetadata* metadata, bool preserve_executability,
|
const FileMetadata* metadata, FileAttrPolicy policy,
|
||||||
bool use_fsync, const FileXattrList* xattrs, bool fake_super,
|
bool use_fsync, const FileXattrList* xattrs, bool fake_super,
|
||||||
const char* temp_dir);
|
const char* temp_dir);
|
||||||
|
/* Streaming --copy-dest install: atomically materialize `path` by copying the
|
||||||
|
* bytes of `basis_path` through a bounded buffer (no whole-file buffering, so
|
||||||
|
* an arbitrarily large basis works), applying the SOURCE metadata and the
|
||||||
|
* per-file xattrs / --fake-super record. `update` honors a newer destination;
|
||||||
|
* a --temp-dir scratch location falls back to a direct write on EXDEV. */
|
||||||
|
bool file_copy_basis_stream_attrs(const char* path, const char* basis_path,
|
||||||
|
unsigned long long expected_size, bool preallocate,
|
||||||
|
const FileMetadata* metadata, FileAttrPolicy policy, bool update,
|
||||||
|
bool use_fsync, const FileXattrList* xattrs, bool fake_super,
|
||||||
|
const char* temp_dir);
|
||||||
|
/* Protocol 2.28.0 receiver-stat variants: like the two above but additionally
|
||||||
|
* report through `dirs_created` (when non-NULL) how many parent directories the
|
||||||
|
* confined secure walk had to create that lie strictly below `count_floor` (a
|
||||||
|
* receive-root-relative prefix, or NULL for all). Used to reproduce rsync's
|
||||||
|
* `Number of created files` directory count on a fresh destination. */
|
||||||
|
bool file_to_disk_secure_attrs_counted(
|
||||||
|
const char* path, const void* data, unsigned long long data_size, bool inplace, bool sparse,
|
||||||
|
bool preallocate, const FileMetadata* metadata, FileAttrPolicy policy, bool update,
|
||||||
|
bool no_replace, bool use_fsync, const FileXattrList* xattrs, bool fake_super,
|
||||||
|
bool keep_partial, const char* temp_dir, unsigned* dirs_created, const char* count_floor,
|
||||||
|
uint32_t fake_super_rdev_major, uint32_t fake_super_rdev_minor);
|
||||||
|
bool file_to_disk_secure_link_attrs_counted(const char* path, const char* basis_path,
|
||||||
|
const void* data, unsigned long long data_size,
|
||||||
|
bool preallocate, const FileMetadata* metadata,
|
||||||
|
FileAttrPolicy policy, bool use_fsync,
|
||||||
|
const FileXattrList* xattrs, bool fake_super,
|
||||||
|
const char* temp_dir, unsigned* dirs_created,
|
||||||
|
const char* count_floor);
|
||||||
|
/* The logical transfer root expressed receive-root-relative, or NULL when the
|
||||||
|
* wire paths carry no mirror scaffolding above it. Caller frees non-NULL. */
|
||||||
|
char* file_transfer_root_floor(const Config* config);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
@@ -0,0 +1,44 @@
|
|||||||
|
#ifndef FILE_ATTR_H
|
||||||
|
#define FILE_ATTR_H
|
||||||
|
|
||||||
|
#include "config.h"
|
||||||
|
#include <stdbool.h>
|
||||||
|
#include <sys/stat.h>
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Per-attribute receiver policy for applying a transmitted FileMetadata. This
|
||||||
|
* is the split-out replacement for the former single use_metadata bundle: each
|
||||||
|
* flag is applied independently, matching rsync's -p/-t/-o/-g/-E/-U semantics.
|
||||||
|
* `use_metadata` remains the transport/presence gate (whether the metadata frame
|
||||||
|
* travelled at all); this struct decides which attributes are ACTUALLY applied.
|
||||||
|
*
|
||||||
|
* It lives in its own header (rather than metadata.h) because xattr.h's
|
||||||
|
* fake_super_restore_fd() takes one and metadata.h <-> file_types.h form an
|
||||||
|
* include cycle that must not be entered from xattr.h.
|
||||||
|
*
|
||||||
|
* The mode leg is: perms wins over executability; an exec-bits-only change is
|
||||||
|
* made only when perms is off; when neither is set the receiver deliberately
|
||||||
|
* sets no source mode. file.c then substitutes the pre-existing destination
|
||||||
|
* mode for a brand-new destination with metadata it uses the sanitized
|
||||||
|
* source-mode-&-umask base (S_IWGRP|S_IWOTH cleared), and the fixed 0644
|
||||||
|
* default only when no metadata is available at all, so a no--p overwrite
|
||||||
|
* does not lose the destination's perms.
|
||||||
|
*/
|
||||||
|
typedef struct FileAttrPolicy {
|
||||||
|
bool perms; /* config->preserve_perms: apply the source mode bits */
|
||||||
|
bool times; /* config->preserve_times: apply the source mtime */
|
||||||
|
bool atimes; /* config->preserve_atimes (-U): apply the source atime */
|
||||||
|
bool executability; /* config->use_executability (-E): exec-bits-only mode */
|
||||||
|
/* privilege_super_mode_permitted(): when false (SUPER_MODE_OFF / --no-super,
|
||||||
|
or a daemon that did not grant `client owner = yes`), the setuid/setgid/
|
||||||
|
sticky bits are stripped from every applied mode (source mode and any
|
||||||
|
--chmod result) even under --perms. When true, rsync's exact semantics are
|
||||||
|
preserved: -p copies the special bits and the kernel decides. */
|
||||||
|
bool super_permitted;
|
||||||
|
} FileAttrPolicy;
|
||||||
|
|
||||||
|
/* Build the per-attribute policy from a connection's Config. A NULL config
|
||||||
|
* yields the all-off policy (no attribute application). */
|
||||||
|
FileAttrPolicy file_attr_policy_from_config(const Config* config);
|
||||||
|
|
||||||
|
#endif
|
||||||
@@ -240,3 +240,29 @@ bool file_list_affects(const FileListSet* set, const char* rel) {
|
|||||||
entry (binary search for the first entry at or after `rel` + '/'). */
|
entry (binary search for the first entry at or after `rel` + '/'). */
|
||||||
return path_index_has_descendant(&set->index, rel);
|
return path_index_has_descendant(&set->index, rel);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool file_list_dir_in_scope(const FileListSet* set, const char* rel) {
|
||||||
|
if (!set || set->whole_tree)
|
||||||
|
return true;
|
||||||
|
if (!rel || rel[0] == '\0')
|
||||||
|
return false;
|
||||||
|
/* `rel` itself is listed, or one of its ancestor prefixes is an exact listed
|
||||||
|
directory (a listed prefix of a directory path is necessarily a
|
||||||
|
directory). */
|
||||||
|
size_t len = strlen(rel);
|
||||||
|
while (len > 0) {
|
||||||
|
const char* slash = NULL;
|
||||||
|
for (size_t i = len; i-- > 0;) {
|
||||||
|
if (rel[i] == '/') {
|
||||||
|
slash = rel + i;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!slash)
|
||||||
|
break;
|
||||||
|
len = (size_t)(slash - rel);
|
||||||
|
if (path_index_contains_n(&set->index, rel, len))
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return path_index_contains(&set->index, rel);
|
||||||
|
}
|
||||||
|
|||||||
@@ -40,4 +40,14 @@ void file_list_destroy(FileListSet* set);
|
|||||||
* this returns true, files are transferred only when it returns true. */
|
* this returns true, files are transferred only when it returns true. */
|
||||||
bool file_list_affects(const FileListSet* set, const char* rel);
|
bool file_list_affects(const FileListSet* set, const char* rel);
|
||||||
|
|
||||||
|
/* True when the DIRECTORY `rel` (path relative to the source root) is inside a
|
||||||
|
* listed directory subtree: `rel` itself is a listed entry, or one of `rel`'s
|
||||||
|
* ancestor directory prefixes is an exact listed entry. Unlike
|
||||||
|
* file_list_affects this does NOT treat an ancestor of a listed entry as
|
||||||
|
* affected, so an implied parent directory of a listed file is not synchronized
|
||||||
|
* (rsync deletes nothing in it). With no set or a whole-tree set every
|
||||||
|
* directory is in scope. This is the delete-walker's "synchronized directory"
|
||||||
|
* predicate. */
|
||||||
|
bool file_list_dir_in_scope(const FileListSet* set, const char* rel);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
+137
-2651
File diff suppressed because it is too large
Load Diff
+34
-78
@@ -2,10 +2,19 @@
|
|||||||
#define FILE_RECEIVE_H
|
#define FILE_RECEIVE_H
|
||||||
|
|
||||||
#include "config.h"
|
#include "config.h"
|
||||||
|
#include "delete_commit.h"
|
||||||
|
#include "file_save.h"
|
||||||
#include "file_types.h"
|
#include "file_types.h"
|
||||||
|
#include "incremental_check.h"
|
||||||
|
#include "utils.h"
|
||||||
#include <stdbool.h>
|
#include <stdbool.h>
|
||||||
|
|
||||||
/* Server-side file receive/save path. */
|
/* Server-side file receive/save path.
|
||||||
|
*
|
||||||
|
* This header is the public facade for the file_receive module family: the
|
||||||
|
* wire receive dispatch (this file) plus the save-to-disk (file_save.h), the
|
||||||
|
* incremental check (incremental_check.h) and the delete-commit
|
||||||
|
* (delete_commit.h) modules. */
|
||||||
|
|
||||||
/* Cumulative caps for the deferred directory-time accumulator. The sender may
|
/* Cumulative caps for the deferred directory-time accumulator. The sender may
|
||||||
* legitimately split a large tree across repeated STATUS_DIR_TIMES frames, so a
|
* legitimately split a large tree across repeated STATUS_DIR_TIMES frames, so a
|
||||||
@@ -22,15 +31,6 @@ File* file_receive_dir_time(int file_descriptor, const Config* config);
|
|||||||
File* file_receive_hardlink(int file_descriptor);
|
File* file_receive_hardlink(int file_descriptor);
|
||||||
File* file_receive_symlink(int file_descriptor, const Config* config);
|
File* file_receive_symlink(int file_descriptor, const Config* config);
|
||||||
File* file_receive_special(int file_descriptor);
|
File* file_receive_special(int file_descriptor);
|
||||||
bool file_special_rdev_valid(int32_t major, int32_t minor, mode_t mode);
|
|
||||||
File* receive_incremental_check(int fd, const Config* config, bool* skipped);
|
|
||||||
/* Extended variant used by the receiver. `would_transfer` (may be NULL) is set
|
|
||||||
* true only on the server-contacting --dry-run path when the file is not up to
|
|
||||||
* date: the receiver has already sent STATUS_DRY_RUN_TRANSFER and returns NULL
|
|
||||||
* without storing anything. On that path `*skipped` is true for an up-to-date
|
|
||||||
* (STATUS_OK) file and both flags are false for a genuine error. */
|
|
||||||
File* receive_incremental_check_ex(int fd, const Config* config, bool* skipped,
|
|
||||||
bool* would_transfer);
|
|
||||||
|
|
||||||
/* P7 Wave D directory-time accumulator. The receiver collects the metadata of
|
/* P7 Wave D directory-time accumulator. The receiver collects the metadata of
|
||||||
* every directory it creates/receives (STATUS_MKDIR with metadata and/or the
|
* every directory it creates/receives (STATUS_MKDIR with metadata and/or the
|
||||||
@@ -40,82 +40,38 @@ File* receive_incremental_check_ex(int fd, const Config* config, bool* skipped,
|
|||||||
* parent's mtime). -O/--omit-dir-times skips the application entirely. The
|
* parent's mtime). -O/--omit-dir-times skips the application entirely. The
|
||||||
* list owns deep copies of the paths and metadata; freed on every path. */
|
* list owns deep copies of the paths and metadata; freed on every path. */
|
||||||
typedef struct {
|
typedef struct {
|
||||||
char** paths; /* owned, destination-relative wire paths */
|
char** paths; /* owned, destination-relative wire paths */
|
||||||
FileMetadata* entries; /* owned, parallel to paths */
|
FileMetadata* entries; /* owned, parallel to paths */
|
||||||
|
FileXattrList** xattrs; /* owned, parallel to paths; NULL when none */
|
||||||
size_t count;
|
size_t count;
|
||||||
size_t capacity;
|
size_t capacity;
|
||||||
size_t bytes; /* cumulative strlen of every retained path */
|
size_t bytes; /* cumulative strlen of every retained path */
|
||||||
} DirTimeList;
|
} DirTimeList;
|
||||||
|
|
||||||
/* Capture gate shared by the sender-side and receiver-side sinks: directory
|
/* Capture gate shared by the sender-side and receiver-side sinks: directory
|
||||||
* metadata is accumulated only when --times/--metadata is in effect and
|
* metadata is accumulated only when a directory attribute is requested
|
||||||
* -O/--omit-dir-times does not suppress it. Kept here, next to the accumulator
|
* (-p/--perms for directory modes, or -t/--times for directory mtimes with
|
||||||
* it guards, so both call sites express the same condition. */
|
* -O/--omit-dir-times not suppressing them) and metadata rides the wire. Kept
|
||||||
bool dir_times_should_capture(const Config* config);
|
* here, next to the accumulator it guards, so both call sites express the same
|
||||||
|
* condition. */
|
||||||
|
bool dir_metadata_should_capture(const Config* config);
|
||||||
|
|
||||||
void dir_time_list_init(DirTimeList* list);
|
void dir_time_list_init(DirTimeList* list);
|
||||||
void dir_time_list_free(DirTimeList* list);
|
void dir_time_list_free(DirTimeList* list);
|
||||||
/* Deep-copy one directory's path + metadata into the list. Returns false on
|
/* Deep-copy one directory's path + metadata (and, when non-NULL, its captured
|
||||||
* allocation failure OR when the cumulative entry/byte caps would be exceeded
|
* xattr/ACL block) into the list. Returns false on allocation failure OR when
|
||||||
* (the caller fails the transfer). */
|
* the cumulative entry/byte caps would be exceeded (the caller fails the
|
||||||
bool dir_time_list_add(DirTimeList* list, const char* wire_path, const FileMetadata* metadata);
|
* transfer). */
|
||||||
/* Apply every accumulated directory's mtime (and atime when captured) beneath
|
bool dir_time_list_add(DirTimeList* list, const char* wire_path, const FileMetadata* metadata,
|
||||||
* `root_directory`, confined fd-relative. Best-effort per entry: an absent
|
const FileXattrList* xattrs);
|
||||||
* directory (an empty/pruned source dir that was deliberately not created) or a
|
/* Apply every accumulated directory's metadata beneath `root_directory`,
|
||||||
* non-directory at the path is skipped QUIETLY, an unreachable one with a
|
* confined fd-relative: ownership through the negotiated identity policy,
|
||||||
* warning, and never fatal. */
|
* times (mtime, plus atime when -U captured one under -t), the mode (through
|
||||||
void dir_time_list_apply(const DirTimeList* list, const char* root_directory);
|
* --chmod when configured, under -p), and the captured xattrs/ACLs (under
|
||||||
|
* -X/-A). Best-effort per entry: an absent directory (an empty/pruned source
|
||||||
/* A received delete-manifest frame: the keep-set (`keeps`, destination-relative
|
* dir that was deliberately not created) or a non-directory at the path is
|
||||||
paths the sender transferred/keeps) plus `protected`, destination-relative
|
* skipped QUIETLY, an unreachable one with a warning, and never fatal. */
|
||||||
prefixes the sender asks the receiver never to delete (paths excluded on the
|
void dir_metadata_list_apply(const DirTimeList* list, const char* root_directory,
|
||||||
source, protected at any depth). When --delete-excluded is given the sender
|
const Config* config);
|
||||||
transmits an empty protected list so excluded destination mirrors are treated
|
|
||||||
as ordinary extras. With --delete-missing-args a third section (`missing`)
|
|
||||||
carries the destination mirrors of explicitly-listed source entries that do
|
|
||||||
not exist: each is an exact deletion request, independent of the ordinary
|
|
||||||
extras walk (never blocked by the protected prefixes) and processed when the
|
|
||||||
manifest is committed. */
|
|
||||||
typedef struct DeleteManifest {
|
|
||||||
ArrayList* keeps;
|
|
||||||
ArrayList* protected;
|
|
||||||
ArrayList* missing;
|
|
||||||
} DeleteManifest;
|
|
||||||
|
|
||||||
void delete_manifest_free(DeleteManifest* manifest);
|
|
||||||
/* Read a delete-manifest frame: keep count + keeps, then protected count +
|
|
||||||
protected prefixes, then missing count + missing paths (self-delimiting; the
|
|
||||||
leading STATUS_MANIFEST code has been consumed). Returns an owned
|
|
||||||
DeleteManifest, or NULL after signalling STATUS_ERROR on a malformed frame. */
|
|
||||||
DeleteManifest* receive_manifest_entries(int fd);
|
|
||||||
/* Remove destination entries under config->receive_root_directory that are not
|
|
||||||
in `manifest` (bounded, all-or-nothing walk; staging-dir, basis-dir and
|
|
||||||
protected-prefix skips). `--max-delete` and `--force` are honored here. The
|
|
||||||
caller decides WHEN to run it based on the negotiated delete timing. Returns
|
|
||||||
false (and the transfer fails) when the deletion cannot be committed. */
|
|
||||||
bool manifest_delete_extras(const Config* config, DeleteManifest* manifest);
|
|
||||||
/* --delete-missing-args exact-path deletions: remove each destination mirror
|
|
||||||
in `manifest->missing` (never blocked by the protected prefixes, staging dir
|
|
||||||
and basis dirs excluded). A regular file/symlink is unlinked; an empty
|
|
||||||
directory is removed; a NON-empty directory is removed recursively only when
|
|
||||||
--delete or --force is in effect, otherwise it is left with a warning (rsync
|
|
||||||
parity). A missing path is a no-op. Returns false only on a genuine
|
|
||||||
confinement or I/O error (the run then fails); tolerated per-path cases are
|
|
||||||
reported and skipped. */
|
|
||||||
bool manifest_delete_missing_args(const Config* config, DeleteManifest* manifest);
|
|
||||||
/* Run every deletion family the manifest carries: the --delete-missing-args
|
|
||||||
exact-path deletions first (user requests are not blocked by exclusion
|
|
||||||
protection), then the ordinary extras walk when --delete is active. Returns
|
|
||||||
true when nothing to do or everything committed. */
|
|
||||||
bool manifest_delete_all(const Config* config, DeleteManifest* manifest);
|
|
||||||
|
|
||||||
/* Outcome of a single file_save_to_disk operation. The receiver needs to
|
|
||||||
distinguish "written" from "skipped" so --remove-source-files can be told
|
|
||||||
which sources were actually stored. */
|
|
||||||
typedef enum { FILE_SAVE_ERROR = 0, FILE_SAVE_WRITTEN = 1, FILE_SAVE_SKIPPED = 2 } FileSaveResult;
|
|
||||||
|
|
||||||
FileSaveResult file_save_to_disk_full(const char* root_directory, const File* file,
|
|
||||||
const Config* config);
|
|
||||||
bool file_save_to_disk(const char* root_directory, const File* file, const Config* config);
|
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,48 @@
|
|||||||
|
#ifndef FILE_SAVE_H
|
||||||
|
#define FILE_SAVE_H
|
||||||
|
|
||||||
|
#include "config.h"
|
||||||
|
#include "file_types.h"
|
||||||
|
#include "format.h"
|
||||||
|
#include <stdbool.h>
|
||||||
|
|
||||||
|
/* Save-to-disk module: regular-file/symlink/hardlink/special install, xattr
|
||||||
|
* application, --fake-super and the --delay-updates staging path. These
|
||||||
|
* declarations are re-exported by the file_receive.h facade. */
|
||||||
|
|
||||||
|
/* Outcome of a single file_save_to_disk operation. The receiver needs to
|
||||||
|
distinguish "written" from "skipped" so --remove-source-files can be told
|
||||||
|
which sources were actually stored. FILE_SAVE_FAILED is a per-entry failure
|
||||||
|
(for example a device node that mknodat() refused with EPERM/EACCES): it is
|
||||||
|
logged and counted by the receiver but does NOT abort the transfer, matching
|
||||||
|
rsync's continue-and-exit-partial behavior. */
|
||||||
|
typedef enum {
|
||||||
|
FILE_SAVE_ERROR = 0,
|
||||||
|
FILE_SAVE_WRITTEN = 1,
|
||||||
|
FILE_SAVE_SKIPPED = 2,
|
||||||
|
FILE_SAVE_FAILED = 3
|
||||||
|
} FileSaveResult;
|
||||||
|
|
||||||
|
bool file_special_rdev_valid(int32_t major, int32_t minor, mode_t mode);
|
||||||
|
|
||||||
|
FileSaveResult file_save_to_disk_full(const char* root_directory, const File* file,
|
||||||
|
const Config* config);
|
||||||
|
/* Protocol 2.28.0 variant: also reports through `created` (when non-NULL)
|
||||||
|
* whether the destination entry did not exist before this save, and through
|
||||||
|
* `created_dirs` how many parent directories the confined walk created, so the
|
||||||
|
* receiver can build rsync's `Number of created files` breakdown. The plain
|
||||||
|
* file_save_to_disk_full() is this with both out-params NULL. */
|
||||||
|
FileSaveResult file_save_to_disk_full_ex(const char* root_directory, const File* file,
|
||||||
|
const Config* config, bool* created,
|
||||||
|
unsigned* created_dirs);
|
||||||
|
bool file_save_to_disk(const char* root_directory, const File* file, const Config* config);
|
||||||
|
|
||||||
|
/* Protocol 2.28.0 receiver counter accumulator: fold one successfully saved
|
||||||
|
* entry into `stats`, adding its receiver-observed literal bytes and, when
|
||||||
|
* `created`, the matching created-by-type counter (regular file / symlink /
|
||||||
|
* special) plus `created_dirs` implicitly-created parent directories.
|
||||||
|
* Non-first hardlink siblings contribute no literal bytes. */
|
||||||
|
void receiver_stats_note_saved(ReceiverStats* stats, const File* file, bool created,
|
||||||
|
unsigned created_dirs);
|
||||||
|
|
||||||
|
#endif
|
||||||
+28
-12
@@ -124,8 +124,12 @@ bool file_send_sendfile_with_skip(File* file, int file_descriptor, bool use_meta
|
|||||||
}
|
}
|
||||||
|
|
||||||
/* sendfile cannot encrypt TLS records. Keep the framing identical but
|
/* sendfile cannot encrypt TLS records. Keep the framing identical but
|
||||||
route encrypted transfers through the deadline-aware IO layer. */
|
route encrypted transfers through the deadline-aware IO layer. Resolve
|
||||||
if (io_get_ssl() != NULL) {
|
the transport from the bound session, not the thread-local io_ssl: a
|
||||||
|
worker thread running a TLS transfer has its SSL only on the session it
|
||||||
|
bound, so io_get_ssl() would be NULL there and the raw sendfile() path
|
||||||
|
would be taken on an encrypted socket. */
|
||||||
|
if (protocol_current_ssl() != NULL) {
|
||||||
unsigned char buffer[64 * 1024];
|
unsigned char buffer[64 * 1024];
|
||||||
unsigned long long remaining = file_size;
|
unsigned long long remaining = file_size;
|
||||||
bool ok = true;
|
bool ok = true;
|
||||||
@@ -143,21 +147,31 @@ bool file_send_sendfile_with_skip(File* file, int file_descriptor, bool use_meta
|
|||||||
}
|
}
|
||||||
|
|
||||||
off_t offset = 0;
|
off_t offset = 0;
|
||||||
|
/* A non-positive --timeout disables the deadline: poll blocks until the
|
||||||
|
* socket is writable (rsync's --timeout=0 default). */
|
||||||
|
int io_timeout_sec = protocol_get_io_timeout_sec();
|
||||||
struct timespec deadline;
|
struct timespec deadline;
|
||||||
clock_gettime(CLOCK_MONOTONIC, &deadline);
|
if (io_timeout_sec > 0) {
|
||||||
deadline.tv_sec += protocol_get_io_timeout_sec();
|
clock_gettime(CLOCK_MONOTONIC, &deadline);
|
||||||
|
deadline.tv_sec += io_timeout_sec;
|
||||||
|
}
|
||||||
while ((unsigned long long)offset < file_size) {
|
while ((unsigned long long)offset < file_size) {
|
||||||
struct timespec now;
|
int timeout = -1;
|
||||||
clock_gettime(CLOCK_MONOTONIC, &now);
|
if (io_timeout_sec > 0) {
|
||||||
long long remaining = (long long)(deadline.tv_sec - now.tv_sec) * 1000LL +
|
struct timespec now;
|
||||||
(deadline.tv_nsec - now.tv_nsec) / 1000000LL;
|
clock_gettime(CLOCK_MONOTONIC, &now);
|
||||||
if (remaining <= 0) {
|
long long remaining = (long long)(deadline.tv_sec - now.tv_sec) * 1000LL +
|
||||||
close(fd);
|
(deadline.tv_nsec - now.tv_nsec) / 1000000LL;
|
||||||
return false;
|
if (remaining <= 0) {
|
||||||
|
close(fd);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
timeout = remaining > INT_MAX ? INT_MAX : (int)remaining;
|
||||||
}
|
}
|
||||||
struct pollfd pfd = {.fd = file_descriptor, .events = POLLOUT};
|
struct pollfd pfd = {.fd = file_descriptor, .events = POLLOUT};
|
||||||
int timeout = remaining > INT_MAX ? INT_MAX : (int)remaining;
|
|
||||||
int polled = poll(&pfd, 1, timeout);
|
int polled = poll(&pfd, 1, timeout);
|
||||||
|
if (polled < 0 && errno == EINTR)
|
||||||
|
continue;
|
||||||
if (polled <= 0 || (pfd.revents & (POLLERR | POLLHUP | POLLNVAL))) {
|
if (polled <= 0 || (pfd.revents & (POLLERR | POLLHUP | POLLNVAL))) {
|
||||||
close(fd);
|
close(fd);
|
||||||
return false;
|
return false;
|
||||||
@@ -174,6 +188,8 @@ bool file_send_sendfile_with_skip(File* file, int file_descriptor, bool use_meta
|
|||||||
close(fd);
|
close(fd);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
protocol_note_bytes_written((unsigned long long)sent);
|
||||||
|
protocol_throttle_bytes(file_descriptor, (size_t)sent);
|
||||||
}
|
}
|
||||||
|
|
||||||
close(fd);
|
close(fd);
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
#define FILE_TYPES_H
|
#define FILE_TYPES_H
|
||||||
|
|
||||||
#include "data.h"
|
#include "data.h"
|
||||||
|
#include "format.h"
|
||||||
#include "xattr.h"
|
#include "xattr.h"
|
||||||
#include <stdbool.h>
|
#include <stdbool.h>
|
||||||
#include <sys/stat.h>
|
#include <sys/stat.h>
|
||||||
@@ -56,6 +57,11 @@ typedef struct {
|
|||||||
* equals the incoming file, and `data` is kept as the cross-filesystem
|
* equals the incoming file, and `data` is kept as the cross-filesystem
|
||||||
* fallback (a local copy) if the hard link cannot be created. */
|
* fallback (a local copy) if the hard link cannot be created. */
|
||||||
char* basis_link;
|
char* basis_link;
|
||||||
|
/* Receiver-only, --copy-dest: when set (and basis_link is NULL), stream the
|
||||||
|
* basis file's bytes into the destination instead of `data`/`data->size`.
|
||||||
|
* This lets a basis larger than any whole-file bound materialize without
|
||||||
|
* buffering it; the source metadata on `metadata` is applied afterwards. */
|
||||||
|
char* basis_copy;
|
||||||
/* --hard-links (-H), sender + receiver wire state. link_group is a run-local
|
/* --hard-links (-H), sender + receiver wire state. link_group is a run-local
|
||||||
* id shared by every member of one source inode (0 = not part of a group).
|
* id shared by every member of one source inode (0 = not part of a group).
|
||||||
* The FIRST member (link_first == true) carries its data on the wire and is
|
* The FIRST member (link_first == true) carries its data on the wire and is
|
||||||
@@ -86,6 +92,21 @@ typedef struct {
|
|||||||
* Receiver: parsed off the wire, attached here, and applied fd-relative on
|
* Receiver: parsed off the wire, attached here, and applied fd-relative on
|
||||||
* the written file. NULL/0 == the file carries no xattrs. */
|
* the written file. NULL/0 == the file carries no xattrs. */
|
||||||
FileXattrList* xattrs;
|
FileXattrList* xattrs;
|
||||||
|
/* Sender-side output-parity state (never serialized): the receiver-reported
|
||||||
|
* pre-transfer destination snapshot for this entry, filled by the per-file
|
||||||
|
* STATUS_CHECK exchange when report_dest_info is set. `known` is false when
|
||||||
|
* no report was requested/received, in which case -i/--out-format treats the
|
||||||
|
* entry conservatively as newly created. */
|
||||||
|
OutputDestState dest_state;
|
||||||
|
/* Receiver-only wire-stats tally: the number of bytes reconstructed from the
|
||||||
|
* basis file (matched delta blocks) for this entry. 0 when the file was sent
|
||||||
|
* whole. Accumulated into ReceiverStats.matched_data by the receiver sink. */
|
||||||
|
unsigned long long matched_bytes;
|
||||||
|
/* Receiver-only (protocol 2.28.0) wire-stats tally: the literal delta fragment
|
||||||
|
* bytes this entry carried (DELTA_INSTR_LITERAL). 0 when the file was sent
|
||||||
|
* whole; the sink then falls back to the whole payload size. Accumulated
|
||||||
|
* into ReceiverStats.literal_bytes. */
|
||||||
|
unsigned long long literal_bytes;
|
||||||
} File;
|
} File;
|
||||||
|
|
||||||
/* The path that should be sent on the wire and used for the receiver-side
|
/* The path that should be sent on the wire and used for the receiver-side
|
||||||
|
|||||||
+669
-226
File diff suppressed because it is too large
Load Diff
+99
-44
@@ -4,80 +4,135 @@
|
|||||||
#include <stdbool.h>
|
#include <stdbool.h>
|
||||||
#include <stddef.h>
|
#include <stddef.h>
|
||||||
|
|
||||||
/* rsync-style filter rule engine (client-side file selection).
|
/* rsync-style filter rule engine (client-side file selection and the
|
||||||
|
* receiver-side protection set it feeds).
|
||||||
*
|
*
|
||||||
* Supported rule syntax (documented subset):
|
* Rule syntax (see the rsync man page FILTER RULES section):
|
||||||
* [+|-] [anchored '/' prefix] pattern [trailing '/' for dir-only]
|
* RULE [PATTERN_OR_FILENAME]
|
||||||
*
|
* RULE,MODIFIERS [PATTERN_OR_FILENAME]
|
||||||
* "+ PATTERN" include rule (first match wins)
|
* Short RULE names may attach MODIFIERS directly ("-sr foo"); the long name
|
||||||
* "- PATTERN" exclude rule
|
* form requires the comma. The pattern/filename is separated from the rule by
|
||||||
* "PATTERN" implicit exclude rule (rsync default)
|
* one space or underscore. Rule names:
|
||||||
* "include PATTERN" / "exclude PATTERN" word forms
|
* exclude/- exclude (by default both sender-hide and receiver-protect)
|
||||||
* leading '/' after the +/- anchors the pattern to its owner directory
|
* include/+ include (by default both sender-show and receiver-risk)
|
||||||
* (the transfer root for command-line/-C rules, the directory that
|
* hide/H sender-only exclude
|
||||||
* contains a .rsync-filter file for per-directory rules)
|
* show/S sender-only include
|
||||||
* a trailing '/' makes the rule match directories only
|
* protect/P receiver-only exclude (protect from deletion)
|
||||||
*
|
* risk/R receiver-only include (allow deletion)
|
||||||
* Rejected explicitly (no silent no-ops): the rsync merge/dir-merge/list-clear
|
* merge/. read a client-side merge file for more rules
|
||||||
* shorthands written as a rule that starts with ':' or '.' or '!', the
|
* dir-merge/: per-directory merge file (registered for the scanner)
|
||||||
* merge/dir-merge/hide/show/protect/risk/clear words, and every include/exclude
|
* clear/! clear the current rule list (takes no argument)
|
||||||
* rule modifier other than '/' (! C s r p x). The pattern must be separated
|
* Modifiers: '/' absolute anchor, '!' negate match, 'C' inject CVS defaults,
|
||||||
* from +/- by a space (or a single '/' anchor), exactly like rsync's
|
* 's' sender side, 'r' receiver side, 'p' perishable. The rsync 'x'
|
||||||
* "-s foo"/"-p ..." modifier syntax is refused.
|
* (xattr-name) modifier is not implemented and is rejected explicitly
|
||||||
|
* everywhere. The merge-file modifiers 'e' (exclude the merge file itself),
|
||||||
|
* 'n' (do not inherit the merge file), 'w' (word-split the merge file) and '-'
|
||||||
|
* (do not transfer the merge file) are accepted and consumed only on merge/
|
||||||
|
* dir-merge rules (rejected on every other rule, matching rsync); their
|
||||||
|
* semantics are not implemented and they are otherwise ignored.
|
||||||
|
* A trailing '/' makes a pattern match directories only. A leading '/' anchors
|
||||||
|
* the pattern to its owner directory.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
typedef enum {
|
typedef enum {
|
||||||
FILTER_ACTION_NONE = 0, /* no rule matched */
|
FILTER_ACTION_NONE = 0, /* no rule matched */
|
||||||
FILTER_ACTION_EXCLUDE = -1,
|
FILTER_ACTION_EXCLUDE = -1,
|
||||||
FILTER_ACTION_INCLUDE = 1
|
FILTER_ACTION_INCLUDE = 1,
|
||||||
|
/* Receiver-side-only verdicts: the entry is transferred but its destination
|
||||||
|
* mirror is protected from --delete (PROTECT) or explicitly left at risk
|
||||||
|
* (RISK). */
|
||||||
|
FILTER_ACTION_PROTECT = 2,
|
||||||
|
FILTER_ACTION_RISK = 3,
|
||||||
} FilterAction;
|
} FilterAction;
|
||||||
|
|
||||||
typedef struct {
|
#define FILTER_SIDE_SENDER 1u
|
||||||
FilterAction action;
|
#define FILTER_SIDE_RECEIVER 2u
|
||||||
bool anchored; /* pattern anchored to the rule's owner directory */
|
|
||||||
bool dir_only; /* pattern had a trailing '/': matches directories only */
|
|
||||||
char* owner; /* owning directory rel path ("" == transfer root) */
|
|
||||||
char* pattern; /* cleaned glob pattern (no leading '/', no trailing '/') */
|
|
||||||
} FilterRule;
|
|
||||||
|
|
||||||
typedef struct {
|
typedef struct {
|
||||||
|
FilterAction action; /* EXCLUDE or INCLUDE (the base pattern action) */
|
||||||
|
unsigned sides; /* FILTER_SIDE_SENDER | FILTER_SIDE_RECEIVER */
|
||||||
|
bool anchored; /* pattern anchored to the rule's owner directory */
|
||||||
|
bool dir_only; /* pattern had a trailing '/': matches directories only */
|
||||||
|
bool negate; /* '!' modifier: match succeeds when the pattern does not */
|
||||||
|
bool perishable; /* 'p' modifier (ignored in deleted directories) */
|
||||||
|
char* owner; /* owning directory rel path ("" == transfer root) */
|
||||||
|
char* pattern; /* cleaned glob pattern (no leading '/', no trailing '/') */
|
||||||
|
} FilterRule;
|
||||||
|
|
||||||
|
typedef struct FilterRuleList {
|
||||||
FilterRule** items; /* owned array of rule pointers */
|
FilterRule** items; /* owned array of rule pointers */
|
||||||
int count;
|
int count;
|
||||||
int capacity;
|
int capacity;
|
||||||
|
/* Per-directory merge-file basenames registered by "dir-merge NAME"/": NAME"
|
||||||
|
* or by -F (.rsync-filter). Owned strings; the scanner reads each name in
|
||||||
|
* every directory it traverses. */
|
||||||
|
char** dir_merge_names;
|
||||||
|
int dir_merge_count;
|
||||||
|
int dir_merge_capacity;
|
||||||
} FilterRuleList;
|
} FilterRuleList;
|
||||||
|
|
||||||
|
/* Context needed while parsing a rule list (merge files, --delete-excluded). */
|
||||||
|
typedef struct {
|
||||||
|
bool delete_excluded; /* --delete-excluded: default sides become sender-only */
|
||||||
|
bool cvs_exclude; /* -C: expand the CVS default excludes */
|
||||||
|
} FilterParseOptions;
|
||||||
|
|
||||||
/* Parse a single filter-rule line (no trailing newline required). Returns an
|
/* Parse a single filter-rule line (no trailing newline required). Returns an
|
||||||
* owned rule, or NULL on unsupported/invalid syntax with a message in `err`. */
|
* owned rule, or NULL on unsupported/invalid syntax with a message in `err`.
|
||||||
FilterRule* filter_rule_parse(const char* line, char* err, size_t err_size);
|
* `opts` may be NULL (no merge expansion / no delete-excluded). */
|
||||||
|
FilterRule* filter_rule_parse(const char* line, const FilterParseOptions* opts, char* err,
|
||||||
|
size_t err_size);
|
||||||
void filter_rule_free(FilterRule* rule);
|
void filter_rule_free(FilterRule* rule);
|
||||||
|
|
||||||
FilterRuleList* filter_rule_list_create(void);
|
FilterRuleList* filter_rule_list_create(void);
|
||||||
/* Append a fully-parsed rule (takes ownership). Returns false on OOM. */
|
/* Append a fully-parsed rule (takes ownership). Returns false on OOM. */
|
||||||
bool filter_rule_list_add(FilterRuleList* list, FilterRule* rule);
|
bool filter_rule_list_add(FilterRuleList* list, FilterRule* rule);
|
||||||
/* Parse `line` and append it. Returns false and fills `err` on bad syntax. */
|
/* Register a per-directory merge-file basename (idempotent). Returns false on
|
||||||
bool filter_rule_list_parse_append(FilterRuleList* list, const char* line, char* err,
|
* OOM. Used by the scanner to read custom "dir-merge" files. */
|
||||||
size_t err_size);
|
bool filter_rule_list_add_dir_merge(FilterRuleList* list, const char* name);
|
||||||
|
/* Parse `line` and append it. Handles "clear"/"!" (resets the list), "merge
|
||||||
|
* FILE"/". FILE" (splices the file's rules) and "dir-merge NAME"/": NAME"
|
||||||
|
* (registers a per-directory filename). Returns false and fills `err` on bad
|
||||||
|
* syntax or an unreadable merge file. `merge_base_dir` resolves a relative
|
||||||
|
* merge-file path (NULL means the process working directory). */
|
||||||
|
bool filter_rule_list_parse_append(FilterRuleList* list, const char* line,
|
||||||
|
const FilterParseOptions* opts, const char* merge_base_dir,
|
||||||
|
char* err, size_t err_size);
|
||||||
void filter_rule_list_free(FilterRuleList* list);
|
void filter_rule_list_free(FilterRuleList* list);
|
||||||
|
|
||||||
/* Build the command-line filter set: `rule_texts` (--filter=RULE in the order
|
/* Build the command-line filter set: `rule_texts` (--filter=RULE in the order
|
||||||
* given, 0..rule_count) followed by the -C CVS default excludes when
|
* given, 0..rule_count) followed by the -C CVS default excludes when
|
||||||
* cvs_exclude is true. All rules are owned by "" (the transfer root).
|
* cvs_exclude is true. All rules are owned by "" (the transfer root).
|
||||||
* Returns NULL on unsupported rule text (message in `err`). */
|
* Returns NULL on unsupported rule text (message in `err`). */
|
||||||
FilterRuleList* filter_base_build(const char* const* rule_texts, int rule_count, bool cvs_exclude,
|
FilterRuleList* filter_base_build(const char* const* rule_texts, int rule_count, bool cvs_exclude,
|
||||||
char* err, size_t err_size);
|
bool delete_excluded, char* err, size_t err_size);
|
||||||
|
|
||||||
/* Read "<dir_path>/.rsync-filter" and return its rules, each owned by
|
/* Read "<dir_path>/<name>" and return its rules, each owned by `owner_rel`. A
|
||||||
* `owner_rel`. A missing file yields an empty list with *exists=false; an
|
* missing file yields an empty list with *exists=false; an unreadable file is
|
||||||
* unreadable file is treated as missing. Returns NULL only on parse or
|
* treated as missing. Returns NULL only on parse or allocation failure
|
||||||
* allocation failure (message in `err`). */
|
* (message in `err`). `opts` may be NULL. */
|
||||||
|
FilterRuleList* filter_file_read_named(const char* dir_path, const char* name,
|
||||||
|
const char* owner_rel, const FilterParseOptions* opts,
|
||||||
|
bool* exists, char* err, size_t err_size);
|
||||||
|
|
||||||
|
/* Append the rules of "<dir_path>/<name>" into an existing list (each owned by
|
||||||
|
* `owner_rel`). A missing file yields *exists=false and no error. Returns
|
||||||
|
* false only on parse/allocation failure (message in `err`). */
|
||||||
|
bool filter_file_append(FilterRuleList* list, const char* dir_path, const char* name,
|
||||||
|
const char* owner_rel, const FilterParseOptions* opts, bool* exists,
|
||||||
|
char* err, size_t err_size);
|
||||||
|
|
||||||
|
/* filter_file_read_named with the default ".rsync-filter" name. */
|
||||||
FilterRuleList* filter_file_read(const char* dir_path, const char* owner_rel, bool* exists,
|
FilterRuleList* filter_file_read(const char* dir_path, const char* owner_rel, bool* exists,
|
||||||
char* err, size_t err_size);
|
char* err, size_t err_size);
|
||||||
|
|
||||||
/* Evaluate an entry against one ordered rule list. Returns FILTER_ACTION_NONE
|
/* Evaluate an entry against one ordered rule list for one side. Returns
|
||||||
* when no rule matched, otherwise the first matching rule's action.
|
* FILTER_ACTION_NONE when no rule matched, otherwise the first matching rule's
|
||||||
* `rel_path` is the entry's path relative to the transfer root ("" == root),
|
* action (for the receiver side an EXCLUDE is reported as
|
||||||
* `leaf` its final name, `is_dir` whether it is a directory. */
|
* FILTER_ACTION_PROTECT and an INCLUDE as FILTER_ACTION_RISK). `rel_path` is
|
||||||
FilterAction filter_rules_apply(const FilterRuleList* list, const char* rel_path, const char* leaf,
|
* the entry's path relative to the transfer root ("" == root), `leaf` its final
|
||||||
bool is_dir);
|
* name, `is_dir` whether it is a directory. */
|
||||||
|
FilterAction filter_rules_apply_side(const FilterRuleList* list, const char* rel_path,
|
||||||
|
const char* leaf, bool is_dir, unsigned side);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
@@ -0,0 +1,131 @@
|
|||||||
|
#include "format.h"
|
||||||
|
#include "protocol.h"
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <string.h>
|
||||||
|
|
||||||
|
bool format_human_size_decimal(unsigned long long bytes, char* buffer, size_t buffer_size) {
|
||||||
|
if (!buffer || buffer_size == 0)
|
||||||
|
return false;
|
||||||
|
if (bytes < 1000ULL) {
|
||||||
|
int written = snprintf(buffer, buffer_size, "%llu", bytes);
|
||||||
|
return written >= 0 && (size_t)written < buffer_size;
|
||||||
|
}
|
||||||
|
static const char units[] = "KMGTPE";
|
||||||
|
double value = (double)bytes;
|
||||||
|
size_t divisions = 0;
|
||||||
|
while (value >= 1000.0 && divisions < sizeof(units) - 1) {
|
||||||
|
value /= 1000.0;
|
||||||
|
divisions++;
|
||||||
|
}
|
||||||
|
int written = snprintf(buffer, buffer_size, "%.2f%c", value, units[divisions - 1]);
|
||||||
|
return written >= 0 && (size_t)written < buffer_size;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool format_big_num(unsigned long long value, bool human_readable, char* buffer,
|
||||||
|
size_t buffer_size) {
|
||||||
|
if (human_readable)
|
||||||
|
return format_human_size_decimal(value, buffer, buffer_size);
|
||||||
|
char digits[32];
|
||||||
|
int written = snprintf(digits, sizeof(digits), "%llu", value);
|
||||||
|
if (written < 0 || (size_t)written >= sizeof(digits))
|
||||||
|
return false;
|
||||||
|
size_t len = (size_t)written;
|
||||||
|
size_t separators = len > 1 ? (len - 1) / 3 : 0;
|
||||||
|
size_t total = len + separators;
|
||||||
|
if (total + 1 > buffer_size)
|
||||||
|
return false;
|
||||||
|
size_t out = total;
|
||||||
|
buffer[out] = '\0';
|
||||||
|
size_t digits_since_sep = 0;
|
||||||
|
for (size_t i = len; i > 0; i--) {
|
||||||
|
buffer[--out] = digits[i - 1];
|
||||||
|
digits_since_sep++;
|
||||||
|
if (digits_since_sep == 3 && i > 1) {
|
||||||
|
buffer[--out] = ',';
|
||||||
|
digits_since_sep = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool format_rsync_datetime(time_t when, bool dash, char* buffer, size_t buffer_size) {
|
||||||
|
if (!buffer || buffer_size == 0)
|
||||||
|
return false;
|
||||||
|
struct tm broken_down;
|
||||||
|
if (localtime_r(&when, &broken_down) == NULL)
|
||||||
|
return false;
|
||||||
|
const char* format = dash ? "%Y/%m/%d-%H:%M:%S" : "%Y/%m/%d %H:%M:%S";
|
||||||
|
return strftime(buffer, buffer_size, format, &broken_down) != 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool format_dest_state_send(int fd, const OutputDestState* state) {
|
||||||
|
if (!state)
|
||||||
|
return false;
|
||||||
|
int32_t has_old = state->existed ? 1 : 0;
|
||||||
|
uint64_t size = (uint64_t)state->size;
|
||||||
|
int64_t mtime = (int64_t)state->mtime_sec;
|
||||||
|
int64_t mtime_nsec = state->mtime_nsec;
|
||||||
|
uint32_t mode = state->mode;
|
||||||
|
int32_t uid = state->uid;
|
||||||
|
int32_t gid = state->gid;
|
||||||
|
return send_n_data(fd, &has_old, sizeof(has_old)) && send_n_data(fd, &size, sizeof(size)) &&
|
||||||
|
send_n_data(fd, &mtime, sizeof(mtime)) &&
|
||||||
|
send_n_data(fd, &mtime_nsec, sizeof(mtime_nsec)) && send_n_data(fd, &mode, sizeof(mode)) &&
|
||||||
|
send_n_data(fd, &uid, sizeof(uid)) && send_n_data(fd, &gid, sizeof(gid));
|
||||||
|
}
|
||||||
|
|
||||||
|
bool format_dest_state_receive(int fd, OutputDestState* state) {
|
||||||
|
if (!state)
|
||||||
|
return false;
|
||||||
|
int32_t has_old = 0;
|
||||||
|
uint64_t size = 0;
|
||||||
|
int64_t mtime = 0;
|
||||||
|
int64_t mtime_nsec = 0;
|
||||||
|
uint32_t mode = 0;
|
||||||
|
int32_t uid = 0;
|
||||||
|
int32_t gid = 0;
|
||||||
|
if (!receive_n_data(fd, &has_old, sizeof(has_old)) || !receive_n_data(fd, &size, sizeof(size)) ||
|
||||||
|
!receive_n_data(fd, &mtime, sizeof(mtime)) ||
|
||||||
|
!receive_n_data(fd, &mtime_nsec, sizeof(mtime_nsec)) ||
|
||||||
|
!receive_n_data(fd, &mode, sizeof(mode)) || !receive_n_data(fd, &uid, sizeof(uid)) ||
|
||||||
|
!receive_n_data(fd, &gid, sizeof(gid)))
|
||||||
|
return false;
|
||||||
|
memset(state, 0, sizeof(*state));
|
||||||
|
state->known = true;
|
||||||
|
state->existed = has_old != 0;
|
||||||
|
state->size = size;
|
||||||
|
state->mtime_sec = mtime;
|
||||||
|
state->mtime_nsec = mtime_nsec;
|
||||||
|
state->mode = mode;
|
||||||
|
state->uid = uid;
|
||||||
|
state->gid = gid;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool format_stats_send(int fd, const ReceiverStats* stats) {
|
||||||
|
if (!stats)
|
||||||
|
return false;
|
||||||
|
unsigned long long fields[8] = {
|
||||||
|
stats->matched_data, stats->deleted_files, stats->would_delete_count, stats->literal_bytes,
|
||||||
|
stats->created_reg, stats->created_dir, stats->created_link, stats->created_special,
|
||||||
|
};
|
||||||
|
return send_n_data(fd, fields, sizeof(fields));
|
||||||
|
}
|
||||||
|
|
||||||
|
bool format_stats_receive(int fd, ReceiverStats* stats) {
|
||||||
|
if (!stats)
|
||||||
|
return false;
|
||||||
|
unsigned long long fields[8] = {0};
|
||||||
|
if (!receive_n_data(fd, fields, sizeof(fields)))
|
||||||
|
return false;
|
||||||
|
memset(stats, 0, sizeof(*stats));
|
||||||
|
stats->matched_data = fields[0];
|
||||||
|
stats->deleted_files = fields[1];
|
||||||
|
stats->would_delete_count = fields[2];
|
||||||
|
stats->literal_bytes = fields[3];
|
||||||
|
stats->created_reg = fields[4];
|
||||||
|
stats->created_dir = fields[5];
|
||||||
|
stats->created_link = fields[6];
|
||||||
|
stats->created_special = fields[7];
|
||||||
|
return true;
|
||||||
|
}
|
||||||
@@ -0,0 +1,111 @@
|
|||||||
|
#ifndef FORMAT_H
|
||||||
|
#define FORMAT_H
|
||||||
|
|
||||||
|
#include <stdbool.h>
|
||||||
|
#include <stddef.h>
|
||||||
|
#include <stdint.h>
|
||||||
|
#include <time.h>
|
||||||
|
|
||||||
|
/* Low-level output-formatting primitives shared by the change-event model
|
||||||
|
* (change_list.c) and the transfer driver (client_send.c).
|
||||||
|
*
|
||||||
|
* The functions here are pure/string-level except for the STATUS_DEST_INFO
|
||||||
|
* codec, which lets the receiver report the pre-transfer destination entry so
|
||||||
|
* the sender can render rsync-accurate --itemize-changes / --out-format
|
||||||
|
* columns (see protocol.h). */
|
||||||
|
|
||||||
|
/* Pre-transfer destination snapshot, reported by the receiver when the wire
|
||||||
|
* config carries report_dest_info. `known` distinguishes "no report was
|
||||||
|
* requested/received" from "the destination did not exist" (`existed == false`
|
||||||
|
* with `known == true`). */
|
||||||
|
typedef struct {
|
||||||
|
bool known;
|
||||||
|
bool existed;
|
||||||
|
unsigned long long size;
|
||||||
|
long long mtime_sec;
|
||||||
|
long long mtime_nsec;
|
||||||
|
uint32_t mode;
|
||||||
|
int32_t uid;
|
||||||
|
int32_t gid;
|
||||||
|
} OutputDestState;
|
||||||
|
|
||||||
|
/* rsync's -h/--human-readable size (decimal, base 1000): integers below 1000
|
||||||
|
* print verbatim; larger values use the largest unit that keeps the value
|
||||||
|
* below 1000 (K/M/G/T/P/E) with exactly two decimals, so 1500000 -> "1.50M"
|
||||||
|
* and 999999 -> "1000.00K" (matching rsync's human_num). Returns false when
|
||||||
|
* the buffer is too small (nothing is written). */
|
||||||
|
bool format_human_size_decimal(unsigned long long bytes, char* buffer, size_t buffer_size);
|
||||||
|
|
||||||
|
/* rsync's general number formatting (big_num). When `human_readable` is true
|
||||||
|
* this is format_human_size_decimal; otherwise the integer is rendered with a
|
||||||
|
* ',' thousands separator every three digits (rsync's separator in the C
|
||||||
|
* locale). Returns false on an undersized buffer. */
|
||||||
|
bool format_big_num(unsigned long long value, bool human_readable, char* buffer,
|
||||||
|
size_t buffer_size);
|
||||||
|
|
||||||
|
/* rsync's %M/%t timestamp. When `dash` is true the separator between the date
|
||||||
|
* and the time is '-' (the %M form: "YYYY/MM/DD-HH:MM:SS"); otherwise it is a
|
||||||
|
* space (the %t form: "YYYY/MM/DD HH:MM:SS"). Local time. Returns false on a
|
||||||
|
* bad time or an undersized buffer. */
|
||||||
|
bool format_rsync_datetime(time_t when, bool dash, char* buffer, size_t buffer_size);
|
||||||
|
|
||||||
|
/* Fixed-width STATUS_DEST_INFO record codec (int32 has_old, uint64 size,
|
||||||
|
* int64 mtime, int64 mtime_nsec, uint32 mode, int32 uid, int32 gid). The
|
||||||
|
* status frame itself is sent/received by the caller. Returns false on I/O
|
||||||
|
* failure. */
|
||||||
|
bool format_dest_state_send(int fd, const OutputDestState* state);
|
||||||
|
bool format_dest_state_receive(int fd, OutputDestState* state);
|
||||||
|
|
||||||
|
/* End-of-transfer receiver counters reported through STATUS_STATS (protocol
|
||||||
|
* 2.25.0, extended in 2.28.0) when the wire config carries report_stats.
|
||||||
|
* `would_delete_count` is the number of destination-relative paths the receiver
|
||||||
|
* would have deleted in a -n/--dry-run --delete run; that many wire strings
|
||||||
|
* immediately follow the fixed record (sent/read by the caller).
|
||||||
|
*
|
||||||
|
* Protocol 2.28.0 adds the receiver-observed counters the sender cannot see:
|
||||||
|
* `literal_bytes` is the file data the receiver actually stored literally
|
||||||
|
* (whole files plus the literal fragments of a delta) and the four `created_*`
|
||||||
|
* counters split the destination entries the receiver newly created by type,
|
||||||
|
* reproducing rsync's `Number of created files` breakdown and an exact
|
||||||
|
* `Literal data` for a delta run. */
|
||||||
|
typedef struct {
|
||||||
|
unsigned long long matched_data;
|
||||||
|
unsigned long long deleted_files;
|
||||||
|
unsigned long long would_delete_count;
|
||||||
|
unsigned long long literal_bytes;
|
||||||
|
unsigned long long created_reg;
|
||||||
|
unsigned long long created_dir;
|
||||||
|
unsigned long long created_link;
|
||||||
|
unsigned long long created_special;
|
||||||
|
} ReceiverStats;
|
||||||
|
|
||||||
|
/* Fixed-width STATUS_STATS counter record. The status frame and the optional
|
||||||
|
* would-delete path list are sent/received by the caller. Returns false on I/O
|
||||||
|
* failure. */
|
||||||
|
bool format_stats_send(int fd, const ReceiverStats* stats);
|
||||||
|
bool format_stats_receive(int fd, ReceiverStats* stats);
|
||||||
|
|
||||||
|
/* Sender-side file-list accounting for rsync's `--stats` block. Filled while
|
||||||
|
* the scan/send loops walk each entry: the flist counters describe every
|
||||||
|
* scanned source entry (transferred or skipped), while the transferred/literal
|
||||||
|
* counters describe only the regular files the receiver actually stored. The
|
||||||
|
* type split lets the client print rsync's `Number of files` breakdown; the
|
||||||
|
* receiver-only counters (matched data, deleted, created) come from
|
||||||
|
* STATUS_STATS. */
|
||||||
|
typedef struct {
|
||||||
|
unsigned long long flist_reg;
|
||||||
|
unsigned long long flist_dir;
|
||||||
|
unsigned long long flist_link;
|
||||||
|
unsigned long long flist_special;
|
||||||
|
unsigned long long total_file_size; /* sum of entry sizes (link target len) */
|
||||||
|
unsigned long long transferred_regular; /* regular files actually stored */
|
||||||
|
unsigned long long transferred_file_size; /* source size of those files */
|
||||||
|
/* Whole-file accuracy: the `--stats` "Literal data" row. The sender counts
|
||||||
|
* the source size of every stored file, so a whole-file transfer matches
|
||||||
|
* rsync. A delta run actually ships only the literal fragments of the diff
|
||||||
|
* (the rest is matched/copied), so here the value is an upper bound, not
|
||||||
|
* rsync's literal-byte total; see RSYNC_COMPAT.md's `--stats` row. */
|
||||||
|
unsigned long long literal_data;
|
||||||
|
} TransferStats;
|
||||||
|
|
||||||
|
#endif
|
||||||
+631
-136
@@ -3,6 +3,7 @@
|
|||||||
#include "utils.h"
|
#include "utils.h"
|
||||||
#include <errno.h>
|
#include <errno.h>
|
||||||
#include <fcntl.h>
|
#include <fcntl.h>
|
||||||
|
#include <fnmatch.h>
|
||||||
#include <grp.h>
|
#include <grp.h>
|
||||||
#include <limits.h>
|
#include <limits.h>
|
||||||
#include <pwd.h>
|
#include <pwd.h>
|
||||||
@@ -12,6 +13,8 @@
|
|||||||
#include <sys/stat.h>
|
#include <sys/stat.h>
|
||||||
#include <unistd.h>
|
#include <unistd.h>
|
||||||
|
|
||||||
|
static bool identity_id_fits_int32(unsigned long id);
|
||||||
|
|
||||||
/* The active identity snapshot lives in a per-process global. The TCP server
|
/* The active identity snapshot lives in a per-process global. The TCP server
|
||||||
* forks one child process per connection, so a connection never shares this
|
* forks one child process per connection, so a connection never shares this
|
||||||
* with another; within a connection the multithreaded receiver reads it without
|
* with another; within a connection the multithreaded receiver reads it without
|
||||||
@@ -36,14 +39,34 @@ typedef struct {
|
|||||||
bool copy_as_set;
|
bool copy_as_set;
|
||||||
int32_t copy_as_uid;
|
int32_t copy_as_uid;
|
||||||
int32_t copy_as_gid;
|
int32_t copy_as_gid;
|
||||||
|
/* -o/--owner and -g/--group: preserve the source owner/group through the
|
||||||
|
* normal name/identity resolution path. Split out of the former
|
||||||
|
* use_metadata bundle; unlike --numeric-ids/--chown/--usermap/--groupmap/-a
|
||||||
|
* these are a preserve-source request, not an arbitrary client-chosen owner,
|
||||||
|
* so they are tracked separately from the explicit ownership gate. */
|
||||||
|
bool preserve_owner;
|
||||||
|
bool preserve_group;
|
||||||
|
/* --fake-super: when active the receiver must only RECORD the (resolved)
|
||||||
|
* ownership in the reserved xattr, never perform a real chown. Snapshotted
|
||||||
|
* so the fd-relative ownership helpers can suppress the chown without a
|
||||||
|
* Config argument. */
|
||||||
|
bool fake_super;
|
||||||
bool set;
|
bool set;
|
||||||
} IdentityActive;
|
} IdentityActive;
|
||||||
|
|
||||||
static IdentityActive g_identity;
|
static IdentityActive g_identity;
|
||||||
|
|
||||||
static void identity_active_reset(void) {
|
static void identity_active_reset(void) {
|
||||||
free(g_identity.usermap);
|
if (g_identity.usermap) {
|
||||||
free(g_identity.groupmap);
|
for (int i = 0; i < g_identity.usermap_count; i++)
|
||||||
|
free(g_identity.usermap[i].to_name);
|
||||||
|
free(g_identity.usermap);
|
||||||
|
}
|
||||||
|
if (g_identity.groupmap) {
|
||||||
|
for (int i = 0; i < g_identity.groupmap_count; i++)
|
||||||
|
free(g_identity.groupmap[i].to_name);
|
||||||
|
free(g_identity.groupmap);
|
||||||
|
}
|
||||||
g_identity.usermap = NULL;
|
g_identity.usermap = NULL;
|
||||||
g_identity.groupmap = NULL;
|
g_identity.groupmap = NULL;
|
||||||
g_identity.usermap_count = 0;
|
g_identity.usermap_count = 0;
|
||||||
@@ -57,6 +80,9 @@ static void identity_active_reset(void) {
|
|||||||
g_identity.copy_as_set = false;
|
g_identity.copy_as_set = false;
|
||||||
g_identity.copy_as_uid = 0;
|
g_identity.copy_as_uid = 0;
|
||||||
g_identity.copy_as_gid = 0;
|
g_identity.copy_as_gid = 0;
|
||||||
|
g_identity.preserve_owner = false;
|
||||||
|
g_identity.preserve_group = false;
|
||||||
|
g_identity.fake_super = false;
|
||||||
g_identity.set = false;
|
g_identity.set = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -77,32 +103,57 @@ bool identity_set_active(const Config* config) {
|
|||||||
g_identity.copy_as_set = config->copy_as_set;
|
g_identity.copy_as_set = config->copy_as_set;
|
||||||
g_identity.copy_as_uid = config->copy_as_uid;
|
g_identity.copy_as_uid = config->copy_as_uid;
|
||||||
g_identity.copy_as_gid = config->copy_as_gid;
|
g_identity.copy_as_gid = config->copy_as_gid;
|
||||||
|
g_identity.preserve_owner = config->preserve_owner;
|
||||||
|
g_identity.preserve_group = config->preserve_group;
|
||||||
|
g_identity.fake_super = config->fake_super;
|
||||||
if (config->usermap_count > 0) {
|
if (config->usermap_count > 0) {
|
||||||
g_identity.usermap = calloc((size_t)config->usermap_count, sizeof(IdentityMap));
|
g_identity.usermap = calloc((size_t)config->usermap_count, sizeof(IdentityMap));
|
||||||
if (!g_identity.usermap)
|
if (!g_identity.usermap)
|
||||||
goto alloc_failed;
|
goto alloc_failed;
|
||||||
memcpy(g_identity.usermap, config->usermap,
|
for (int i = 0; i < config->usermap_count; i++) {
|
||||||
(size_t)config->usermap_count * sizeof(IdentityMap));
|
g_identity.usermap[i] = config->usermap[i];
|
||||||
|
g_identity.usermap[i].to_name =
|
||||||
|
config->usermap[i].to_name ? str_dup(config->usermap[i].to_name) : NULL;
|
||||||
|
if (config->usermap[i].to_name && !g_identity.usermap[i].to_name) {
|
||||||
|
g_identity.usermap_count = i; /* free only the entries already duplicated */
|
||||||
|
goto alloc_failed;
|
||||||
|
}
|
||||||
|
}
|
||||||
g_identity.usermap_count = config->usermap_count;
|
g_identity.usermap_count = config->usermap_count;
|
||||||
}
|
}
|
||||||
if (config->groupmap_count > 0) {
|
if (config->groupmap_count > 0) {
|
||||||
g_identity.groupmap = calloc((size_t)config->groupmap_count, sizeof(IdentityMap));
|
g_identity.groupmap = calloc((size_t)config->groupmap_count, sizeof(IdentityMap));
|
||||||
if (!g_identity.groupmap)
|
if (!g_identity.groupmap)
|
||||||
goto alloc_failed;
|
goto alloc_failed;
|
||||||
memcpy(g_identity.groupmap, config->groupmap,
|
for (int i = 0; i < config->groupmap_count; i++) {
|
||||||
(size_t)config->groupmap_count * sizeof(IdentityMap));
|
g_identity.groupmap[i] = config->groupmap[i];
|
||||||
|
g_identity.groupmap[i].to_name =
|
||||||
|
config->groupmap[i].to_name ? str_dup(config->groupmap[i].to_name) : NULL;
|
||||||
|
if (config->groupmap[i].to_name && !g_identity.groupmap[i].to_name) {
|
||||||
|
g_identity.groupmap_count = i;
|
||||||
|
goto alloc_failed;
|
||||||
|
}
|
||||||
|
}
|
||||||
g_identity.groupmap_count = config->groupmap_count;
|
g_identity.groupmap_count = config->groupmap_count;
|
||||||
}
|
}
|
||||||
g_identity.set = true;
|
g_identity.set = true;
|
||||||
/* A root receiver would honor any client-supplied ownership request (a
|
/* A root receiver would honor any client-supplied ownership request (a
|
||||||
--usermap/--groupmap/--chown/--copy-as, or raw ids under --numeric-ids).
|
--usermap/--groupmap/--chown/--copy-as, or raw ids under --numeric-ids)
|
||||||
Surface that prominently; a privileged daemon applying arbitrary client
|
ONLY when super-user activities are permitted. --no-super (or a daemon
|
||||||
ownership is a deliberate, opt-in choice the operator should be aware of. */
|
veto that forced SUPER_MODE_OFF) forbids the chown even for root, so do
|
||||||
if (geteuid() == 0)
|
not claim the ownership will be honored in that case. */
|
||||||
log_message(LOG_LEVEL_WARNING,
|
if (geteuid() == 0) {
|
||||||
"identity mapping active and running as root: client-supplied "
|
if (privilege_super_mode_permitted(g_identity.super_mode))
|
||||||
"ownership (usermap/groupmap/chown/numeric-ids) will be honored; "
|
log_message(LOG_LEVEL_WARNING,
|
||||||
"run the daemon as an unprivileged user unless intended");
|
"identity mapping active and running as root: client-supplied "
|
||||||
|
"ownership (usermap/groupmap/chown/numeric-ids) will be honored; "
|
||||||
|
"run the daemon as an unprivileged user unless intended");
|
||||||
|
else
|
||||||
|
log_message(LOG_LEVEL_WARNING,
|
||||||
|
"identity mapping active and running as root, but super-user activities are "
|
||||||
|
"disabled (--no-super): requested ownership will NOT be applied; run the "
|
||||||
|
"daemon as an unprivileged user unless intended");
|
||||||
|
}
|
||||||
/* --super explicitly requests super-user activities, but FastSync never
|
/* --super explicitly requests super-user activities, but FastSync never
|
||||||
elevates privileges: when the receiver is not already root the kernel will
|
elevates privileges: when the receiver is not already root the kernel will
|
||||||
refuse those confined attempts and each is skipped per entry. Warn exactly
|
refuse those confined attempts and each is skipped per entry. Warn exactly
|
||||||
@@ -138,25 +189,55 @@ bool privilege_super_mode_permitted(SuperMode mode) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
bool identity_active_enabled(void) {
|
bool identity_active_enabled(void) {
|
||||||
/* numeric_ids is included: this set only gates identity_apply_ownership,
|
/* --numeric-ids is deliberately NOT included: it is a mapping MODIFIER (use
|
||||||
which runs only when metadata is present (a -M/--preserve transfer). A
|
* the transmitted numeric id raw instead of a name lookup), not a request to
|
||||||
standalone --numeric-ids (no ownership-affecting flag) carries no
|
* change ownership. rsync's --numeric-ids on its own never chowns anything;
|
||||||
metadata, never reaches identity_apply_ownership, and therefore correctly
|
* it only changes how an already-requested -o/-g/map resolves. Ownership is
|
||||||
stays inert; combined with -M it activates raw-id application. --super /
|
* activated only by an explicit request: --chown/--usermap/--groupmap/
|
||||||
--no-super does NOT enable ownership: it only permits or forbids the
|
* --copy-as or a preserve-source -o/--owner / -g/--group. --super/--no-super
|
||||||
already-requested super-user activities, so a --super with no explicit
|
* likewise does NOT enable ownership: it only permits or forbids the
|
||||||
identity flag must never silently apply client-chosen ownership. */
|
* already-requested super-user activities. */
|
||||||
return g_identity.set &&
|
return g_identity.set &&
|
||||||
(g_identity.numeric_ids || g_identity.chown_uid_set || g_identity.chown_gid_set ||
|
(g_identity.chown_uid_set || g_identity.chown_gid_set || g_identity.usermap_count > 0 ||
|
||||||
g_identity.usermap_count > 0 || g_identity.groupmap_count > 0 || g_identity.copy_as_set);
|
g_identity.groupmap_count > 0 || g_identity.copy_as_set || g_identity.preserve_owner ||
|
||||||
|
g_identity.preserve_group);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool identity_owner_requested(void) {
|
||||||
|
return g_identity.set && (g_identity.copy_as_set || g_identity.chown_uid_set ||
|
||||||
|
g_identity.preserve_owner || g_identity.usermap_count > 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool identity_group_requested(void) {
|
||||||
|
return g_identity.set && (g_identity.copy_as_set || g_identity.chown_gid_set ||
|
||||||
|
g_identity.preserve_group || g_identity.groupmap_count > 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
bool identity_ownership_requested(const Config* config) {
|
bool identity_ownership_requested(const Config* config) {
|
||||||
if (!config)
|
if (!config)
|
||||||
return false;
|
return false;
|
||||||
/* Every value that makes the receiver act on a client-chosen owner, plus an
|
/* General-awareness predicate: every value that makes the receiver act on a
|
||||||
* explicit --super (super-user device-node activities). Pure config, so the
|
* client-chosen owner, plus an explicit --super (super-user device-node
|
||||||
* daemon gate can evaluate it before identity_set_active(). */
|
* activities) and the preserve-source -o/-g requests. Pure config, so callers
|
||||||
|
* can evaluate it before identity_set_active(). The daemon module gate uses
|
||||||
|
* the narrower identity_explicit_ownership_requested() below, which treats a
|
||||||
|
* plain -o/-g/-a as a preserve-source request rather than arbitrary
|
||||||
|
* client-chosen ownership. */
|
||||||
|
return config->numeric_ids || config->chown_uid_set || config->chown_gid_set ||
|
||||||
|
config->usermap_count > 0 || config->groupmap_count > 0 || config->copy_as_set ||
|
||||||
|
config->preserve_owner || config->preserve_group || config->fake_super ||
|
||||||
|
config->super_mode == SUPER_MODE_ON;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool identity_explicit_ownership_requested(const Config* config) {
|
||||||
|
if (!config)
|
||||||
|
return false;
|
||||||
|
/* The narrow set the daemon gate refuses for a non-opted module: a request
|
||||||
|
* that lets the CLIENT choose an arbitrary owner/group (rather than preserve
|
||||||
|
* the source's own). Deliberately EXCLUDES preserve_owner/preserve_group so a
|
||||||
|
* plain -a/-o/-g push is not refused; for those the gate instead forces
|
||||||
|
* super-user ownership activity off (no chown happens) unless the module has
|
||||||
|
* `client owner = yes`. */
|
||||||
return config->numeric_ids || config->chown_uid_set || config->chown_gid_set ||
|
return config->numeric_ids || config->chown_uid_set || config->chown_gid_set ||
|
||||||
config->usermap_count > 0 || config->groupmap_count > 0 || config->copy_as_set ||
|
config->usermap_count > 0 || config->groupmap_count > 0 || config->copy_as_set ||
|
||||||
config->fake_super || config->super_mode == SUPER_MODE_ON;
|
config->fake_super || config->super_mode == SUPER_MODE_ON;
|
||||||
@@ -177,6 +258,29 @@ bool identity_copy_as_refused(const Config* config) {
|
|||||||
return geteuid() != 0 || config->super_mode == SUPER_MODE_OFF;
|
return geteuid() != 0 || config->super_mode == SUPER_MODE_OFF;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Validate one received FROM:TO map rule. `from` is a single id, the LOW end
|
||||||
|
* of an inclusive range, IDENTITY_MATCH_ANY, or IDENTITY_MATCH_UNNAMED; a
|
||||||
|
* sentinel FROM must carry the same value in from_hi. `to` is a non-negative
|
||||||
|
* id, IDENTITY_CURRENT, or ignored when a bounded receiver-resolved `to_name`
|
||||||
|
* is present. */
|
||||||
|
static bool identity_wire_map_valid(const IdentityMap* map) {
|
||||||
|
if (!map)
|
||||||
|
return false;
|
||||||
|
if (map->from < IDENTITY_MATCH_UNNAMED)
|
||||||
|
return false;
|
||||||
|
if (map->from < 0) {
|
||||||
|
if (map->from_hi != map->from)
|
||||||
|
return false;
|
||||||
|
} else if (map->from_hi < map->from) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (map->to < IDENTITY_CURRENT)
|
||||||
|
return false;
|
||||||
|
if (map->to_name && strlen(map->to_name) > 255)
|
||||||
|
return false;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
bool identity_wire_valid(const Config* config) {
|
bool identity_wire_valid(const Config* config) {
|
||||||
if (!config)
|
if (!config)
|
||||||
return false;
|
return false;
|
||||||
@@ -188,11 +292,11 @@ bool identity_wire_valid(const Config* config) {
|
|||||||
if (config->chown_gid_set && config->chown_gid < IDENTITY_MATCH_ANY)
|
if (config->chown_gid_set && config->chown_gid < IDENTITY_MATCH_ANY)
|
||||||
return false;
|
return false;
|
||||||
for (int i = 0; i < config->usermap_count; i++) {
|
for (int i = 0; i < config->usermap_count; i++) {
|
||||||
if (config->usermap[i].from < IDENTITY_MATCH_ANY || config->usermap[i].to < IDENTITY_CURRENT)
|
if (!identity_wire_map_valid(&config->usermap[i]))
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
for (int i = 0; i < config->groupmap_count; i++) {
|
for (int i = 0; i < config->groupmap_count; i++) {
|
||||||
if (config->groupmap[i].from < IDENTITY_MATCH_ANY || config->groupmap[i].to < IDENTITY_CURRENT)
|
if (!identity_wire_map_valid(&config->groupmap[i]))
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
/* Defense-in-depth: a --copy-as block must never carry a negative (sentinel)
|
/* Defense-in-depth: a --copy-as block must never carry a negative (sentinel)
|
||||||
@@ -250,19 +354,266 @@ static int identity_resolve_token(const char* token, bool is_group, int32_t* out
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
static int identity_append_rule(IdentityMap** map, int* count, int32_t from, int32_t to) {
|
static bool identity_all_digits(const char* token) {
|
||||||
|
if (!token || *token == '\0')
|
||||||
|
return false;
|
||||||
|
for (const char* p = token; *p; p++)
|
||||||
|
if (*p < '0' || *p > '9')
|
||||||
|
return false;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool identity_token_has_glob(const char* token) {
|
||||||
|
return token && (strchr(token, '*') || strchr(token, '?') || strchr(token, '['));
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Parse a --usermap/--groupmap FROM token into a matcher (from/from_hi). rsync
|
||||||
|
* accepts a name, a numeric id, an inclusive LOW-HIGH range, '*' (any id), or an
|
||||||
|
* empty token (ids with no name on the sender). Returns 0 on success, -1 on a
|
||||||
|
* malformed token or an unresolvable sender-side name. */
|
||||||
|
static int identity_parse_from(const char* token, bool is_group, int32_t* out_from,
|
||||||
|
int32_t* out_hi) {
|
||||||
|
if (token[0] == '\0') {
|
||||||
|
*out_from = IDENTITY_MATCH_UNNAMED;
|
||||||
|
*out_hi = IDENTITY_MATCH_UNNAMED;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
if (strcmp(token, "*") == 0) {
|
||||||
|
*out_from = IDENTITY_MATCH_ANY;
|
||||||
|
*out_hi = IDENTITY_MATCH_ANY;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
const char* num = token[0] == '@' ? token + 1 : token;
|
||||||
|
if (identity_all_digits(num)) {
|
||||||
|
int32_t id;
|
||||||
|
if (identity_resolve_token(token, is_group, &id) != 0)
|
||||||
|
return -1;
|
||||||
|
*out_from = id;
|
||||||
|
*out_hi = id;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
/* An inclusive LOW-HIGH numeric range. */
|
||||||
|
const char* dash = strchr(num, '-');
|
||||||
|
if (dash && dash != num && dash[1] != '\0' && strchr(dash + 1, '-') == NULL) {
|
||||||
|
size_t lo_len = (size_t)(dash - num);
|
||||||
|
size_t hi_len = strlen(dash + 1);
|
||||||
|
char low[16];
|
||||||
|
char high[16];
|
||||||
|
if (lo_len < sizeof(low) && hi_len < sizeof(high)) {
|
||||||
|
memcpy(low, num, lo_len);
|
||||||
|
low[lo_len] = '\0';
|
||||||
|
memcpy(high, dash + 1, hi_len);
|
||||||
|
high[hi_len] = '\0';
|
||||||
|
if (identity_all_digits(low) && identity_all_digits(high)) {
|
||||||
|
char* endptr = NULL;
|
||||||
|
errno = 0;
|
||||||
|
long lo = strtol(low, &endptr, 10);
|
||||||
|
if (errno != 0 || !endptr || *endptr != '\0')
|
||||||
|
return -1;
|
||||||
|
errno = 0;
|
||||||
|
long hi = strtol(high, &endptr, 10);
|
||||||
|
if (errno != 0 || !endptr || *endptr != '\0' || hi < lo || hi > INT32_MAX)
|
||||||
|
return -1;
|
||||||
|
*out_from = (int32_t)lo;
|
||||||
|
*out_hi = (int32_t)hi;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
/* Not a numeric LOW-HIGH range: fall through and treat as a name (a
|
||||||
|
* hyphenated account name like "wayne-smith" must still resolve). */
|
||||||
|
}
|
||||||
|
/* A sender-side name. A FROM name wildcard other than the bare '*' is handled
|
||||||
|
* by identity_expand_from_glob() in the caller (it expands against the
|
||||||
|
* sender's account database at CLI-parse time), so this function only sees the
|
||||||
|
* bare '*' or a literal name here. */
|
||||||
|
int32_t id;
|
||||||
|
if (identity_resolve_token(token, is_group, &id) != 0)
|
||||||
|
return -1;
|
||||||
|
*out_from = id;
|
||||||
|
*out_hi = id;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Parse a --usermap/--groupmap TO token. '*', a bare numeric id, or an @N id is
|
||||||
|
* stored numerically; every other non-empty token is a NAME resolved on the
|
||||||
|
* RECEIVER at apply time (rsync resolves TO names against the receiving side).
|
||||||
|
* Returns 0 on success, -1 on an empty/malformed token. */
|
||||||
|
static int identity_parse_to(const char* token, bool is_group, int32_t* out_to, char** out_name) {
|
||||||
|
if (token[0] == '\0') {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "%smap TO value is missing", is_group ? "--group" : "--user");
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
if (strcmp(token, "*") == 0) {
|
||||||
|
*out_to = IDENTITY_CURRENT;
|
||||||
|
*out_name = NULL;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
const char* num = token[0] == '@' ? token + 1 : token;
|
||||||
|
if (identity_all_digits(num)) {
|
||||||
|
int32_t id;
|
||||||
|
if (identity_resolve_token(token, is_group, &id) != 0)
|
||||||
|
return -1;
|
||||||
|
*out_to = id;
|
||||||
|
*out_name = NULL;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
if (identity_token_has_glob(token)) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "%smap TO '%s' may not contain a wildcard",
|
||||||
|
is_group ? "--group" : "--user", token);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
char* name = str_dup(token);
|
||||||
|
if (!name)
|
||||||
|
return -1;
|
||||||
|
*out_to = 0;
|
||||||
|
*out_name = name;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
static int identity_append_rule(IdentityMap** map, int* count, const IdentityMap* rule) {
|
||||||
if (*count >= MAX_IDENTITY_MAP)
|
if (*count >= MAX_IDENTITY_MAP)
|
||||||
return -1;
|
return -1;
|
||||||
IdentityMap* grown = realloc(*map, (size_t)(*count + 1) * sizeof(IdentityMap));
|
IdentityMap* grown = realloc(*map, (size_t)(*count + 1) * sizeof(IdentityMap));
|
||||||
if (!grown)
|
if (!grown)
|
||||||
return -1;
|
return -1;
|
||||||
*map = grown;
|
*map = grown;
|
||||||
(*map)[*count].from = from;
|
(*map)[*count] = *rule;
|
||||||
(*map)[*count].to = to;
|
|
||||||
(*count)++;
|
(*count)++;
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* True when `lo` and `hi` are adjacent ids (no overflow at INT32_MAX). */
|
||||||
|
static bool identity_ids_adjacent(int32_t lo, int32_t hi) {
|
||||||
|
return lo < INT32_MAX && hi == lo + 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
static int identity_id_cmp(const void* a, const void* b) {
|
||||||
|
int32_t x = *(const int32_t*)a;
|
||||||
|
int32_t y = *(const int32_t*)b;
|
||||||
|
return (x > y) - (x < y);
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool identity_ids_push(int32_t** ids, size_t* count, size_t* cap, int32_t id) {
|
||||||
|
if (*count == *cap) {
|
||||||
|
size_t grown_cap = *cap ? *cap * 2 : 16;
|
||||||
|
int32_t* grown = realloc(*ids, grown_cap * sizeof(int32_t));
|
||||||
|
if (!grown)
|
||||||
|
return false;
|
||||||
|
*ids = grown;
|
||||||
|
*cap = grown_cap;
|
||||||
|
}
|
||||||
|
(*ids)[(*count)++] = id;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Expand a FROM name wildcard (rsync's match against sender-side account names)
|
||||||
|
* into one rule per contiguous run of matching numeric ids, all sharing the same
|
||||||
|
* TO side. FastSync transmits numeric ids only, so the wildcard must be
|
||||||
|
* resolved here -- at CLI-parse time -- against the SENDER's passwd/group
|
||||||
|
* database; the receiver has no sender names to match. Contiguous matched ids
|
||||||
|
* are collapsed into a single LOW-HIGH range (a range of adjacent ids contains
|
||||||
|
* exactly the ids it spans, so this is semantically exact). Returns 0 on
|
||||||
|
* success, -1 on an allocation failure, a wildcard that matches no sender
|
||||||
|
* account, or an expansion that would push the map past MAX_IDENTITY_MAP. */
|
||||||
|
static int identity_expand_from_glob(Config* config, const char* glob, bool is_group,
|
||||||
|
const IdentityMap* to_rule) {
|
||||||
|
const char* optname = is_group ? "--groupmap" : "--usermap";
|
||||||
|
size_t cap = 0;
|
||||||
|
size_t n = 0;
|
||||||
|
int32_t* ids = NULL;
|
||||||
|
bool alloc_failed = false;
|
||||||
|
|
||||||
|
if (is_group) {
|
||||||
|
setgrent();
|
||||||
|
struct group* gr;
|
||||||
|
while ((gr = getgrent()) != NULL) {
|
||||||
|
if (fnmatch(glob, gr->gr_name, 0) != 0)
|
||||||
|
continue;
|
||||||
|
if (!identity_id_fits_int32((unsigned long)gr->gr_gid))
|
||||||
|
continue;
|
||||||
|
if (!identity_ids_push(&ids, &n, &cap, (int32_t)gr->gr_gid)) {
|
||||||
|
alloc_failed = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
endgrent();
|
||||||
|
} else {
|
||||||
|
setpwent();
|
||||||
|
struct passwd* pw;
|
||||||
|
while ((pw = getpwent()) != NULL) {
|
||||||
|
if (fnmatch(glob, pw->pw_name, 0) != 0)
|
||||||
|
continue;
|
||||||
|
if (!identity_id_fits_int32((unsigned long)pw->pw_uid))
|
||||||
|
continue;
|
||||||
|
if (!identity_ids_push(&ids, &n, &cap, (int32_t)pw->pw_uid)) {
|
||||||
|
alloc_failed = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
endpwent();
|
||||||
|
}
|
||||||
|
|
||||||
|
if (alloc_failed) {
|
||||||
|
free(ids);
|
||||||
|
log_message(LOG_LEVEL_ERROR, "%s: memory allocation failed expanding FROM '%s'", optname, glob);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
if (n == 0) {
|
||||||
|
free(ids);
|
||||||
|
log_message(LOG_LEVEL_ERROR, "%s FROM '%s': no source account name matches the wildcard",
|
||||||
|
optname, glob);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
qsort(ids, n, sizeof(int32_t), identity_id_cmp);
|
||||||
|
size_t unique = 0;
|
||||||
|
for (size_t i = 0; i < n; i++) {
|
||||||
|
if (unique == 0 || ids[unique - 1] != ids[i])
|
||||||
|
ids[unique++] = ids[i];
|
||||||
|
}
|
||||||
|
n = unique;
|
||||||
|
|
||||||
|
int runs = 0;
|
||||||
|
for (size_t i = 0; i < n; i++) {
|
||||||
|
if (i == 0 || !identity_ids_adjacent(ids[i - 1], ids[i]))
|
||||||
|
runs++;
|
||||||
|
}
|
||||||
|
|
||||||
|
IdentityMap** map = is_group ? &config->groupmap : &config->usermap;
|
||||||
|
int* count = is_group ? &config->groupmap_count : &config->usermap_count;
|
||||||
|
if (*count > MAX_IDENTITY_MAP - runs) {
|
||||||
|
log_message(LOG_LEVEL_ERROR,
|
||||||
|
"%s FROM '%s': the name wildcard expands to %d rule(s), which would exceed "
|
||||||
|
"the maximum of %d map rules",
|
||||||
|
optname, glob, runs, MAX_IDENTITY_MAP);
|
||||||
|
free(ids);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (size_t i = 0; i < n;) {
|
||||||
|
size_t j = i;
|
||||||
|
while (j + 1 < n && identity_ids_adjacent(ids[j], ids[j + 1]))
|
||||||
|
j++;
|
||||||
|
IdentityMap rule;
|
||||||
|
rule.from = ids[i];
|
||||||
|
rule.from_hi = ids[j];
|
||||||
|
rule.to = to_rule->to;
|
||||||
|
rule.to_name = to_rule->to_name ? str_dup(to_rule->to_name) : NULL;
|
||||||
|
if (to_rule->to_name && !rule.to_name) {
|
||||||
|
free(ids);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
if (identity_append_rule(map, count, &rule) != 0) {
|
||||||
|
free(rule.to_name);
|
||||||
|
free(ids);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
i = j + 1;
|
||||||
|
}
|
||||||
|
free(ids);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
int identity_parse_map(Config* config, const char* value, bool is_group) {
|
int identity_parse_map(Config* config, const char* value, bool is_group) {
|
||||||
if (!config || !value || *value == '\0') {
|
if (!config || !value || *value == '\0') {
|
||||||
log_message(LOG_LEVEL_ERROR, "%smap requires a value", is_group ? "--group" : "--user");
|
log_message(LOG_LEVEL_ERROR, "%smap requires a value", is_group ? "--group" : "--user");
|
||||||
@@ -275,7 +626,7 @@ int identity_parse_map(Config* config, const char* value, bool is_group) {
|
|||||||
char* saveptr = NULL;
|
char* saveptr = NULL;
|
||||||
for (char* rule = strtok_r(list, ",", &saveptr); rule; rule = strtok_r(NULL, ",", &saveptr)) {
|
for (char* rule = strtok_r(list, ",", &saveptr); rule; rule = strtok_r(NULL, ",", &saveptr)) {
|
||||||
char* colon = strchr(rule, ':');
|
char* colon = strchr(rule, ':');
|
||||||
if (!colon || colon == rule) {
|
if (!colon) {
|
||||||
/* Log before freeing: `rule` points into the str_dup'd list. */
|
/* Log before freeing: `rule` points into the str_dup'd list. */
|
||||||
log_message(LOG_LEVEL_ERROR, "%s rules must be FROM:TO (got '%s')", optname, rule);
|
log_message(LOG_LEVEL_ERROR, "%s rules must be FROM:TO (got '%s')", optname, rule);
|
||||||
free(list);
|
free(list);
|
||||||
@@ -284,25 +635,49 @@ int identity_parse_map(Config* config, const char* value, bool is_group) {
|
|||||||
*colon = '\0';
|
*colon = '\0';
|
||||||
char* from_token = rule;
|
char* from_token = rule;
|
||||||
char* to_token = colon + 1;
|
char* to_token = colon + 1;
|
||||||
if (*to_token == '\0') {
|
IdentityMap parsed;
|
||||||
|
memset(&parsed, 0, sizeof(parsed));
|
||||||
|
/* A FROM name wildcard (anything with a glob metacharacter other than the
|
||||||
|
* bare '*') is expanded against the sender's account database here, while
|
||||||
|
* the sender's passwd/group DB is still available; the resulting numeric
|
||||||
|
* rules travel on the wire like an explicit list. The TO side is parsed
|
||||||
|
* first so every expanded rule shares it. */
|
||||||
|
if (strcmp(from_token, "*") != 0 && identity_token_has_glob(from_token)) {
|
||||||
|
if (identity_parse_to(to_token, is_group, &parsed.to, &parsed.to_name) != 0) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "%s could not parse TO '%s' in '%s'", optname, to_token,
|
||||||
|
value);
|
||||||
|
free(list);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
if (identity_expand_from_glob(config, from_token, is_group, &parsed) != 0) {
|
||||||
|
free(parsed.to_name);
|
||||||
|
free(list);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
/* Every rule emitted by the expansion took its own str_dup of the name,
|
||||||
|
* so the parse-time copy is unreachable on success: release it here (the
|
||||||
|
* failure path above already does). `parsed.to_name` is NULL for a
|
||||||
|
* numeric TO. */
|
||||||
|
free(parsed.to_name);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (identity_parse_from(from_token, is_group, &parsed.from, &parsed.from_hi) != 0) {
|
||||||
|
log_message(LOG_LEVEL_ERROR,
|
||||||
|
"%s could not resolve FROM '%s' in '%s' (a name must exist on the "
|
||||||
|
"source; use @N for a numeric id)",
|
||||||
|
optname, from_token, value);
|
||||||
free(list);
|
free(list);
|
||||||
log_message(LOG_LEVEL_ERROR, "%s rule 'FROM:' is missing the TO value (got '%s')", optname,
|
|
||||||
value);
|
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
int32_t from_id, to_id;
|
if (identity_parse_to(to_token, is_group, &parsed.to, &parsed.to_name) != 0) {
|
||||||
if (identity_resolve_token(from_token, is_group, &from_id) != 0 ||
|
log_message(LOG_LEVEL_ERROR, "%s could not parse TO '%s' in '%s'", optname, to_token, value);
|
||||||
identity_resolve_token(to_token, is_group, &to_id) != 0) {
|
|
||||||
free(list);
|
free(list);
|
||||||
log_message(LOG_LEVEL_ERROR,
|
|
||||||
"%s could not resolve '%s' (name must exist on the source; use "
|
|
||||||
"@N for a numeric id)",
|
|
||||||
optname, value);
|
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
if (identity_append_rule(is_group ? &config->groupmap : &config->usermap,
|
if (identity_append_rule(is_group ? &config->groupmap : &config->usermap,
|
||||||
is_group ? &config->groupmap_count : &config->usermap_count, from_id,
|
is_group ? &config->groupmap_count : &config->usermap_count,
|
||||||
to_id) != 0) {
|
&parsed) != 0) {
|
||||||
|
free(parsed.to_name);
|
||||||
free(list);
|
free(list);
|
||||||
log_message(LOG_LEVEL_ERROR, "%s has too many rules (max %d)", optname, MAX_IDENTITY_MAP);
|
log_message(LOG_LEVEL_ERROR, "%s has too many rules (max %d)", optname, MAX_IDENTITY_MAP);
|
||||||
return -1;
|
return -1;
|
||||||
@@ -366,6 +741,67 @@ static int identity_split_chown(const char* value, char** puser, char** pgroup)
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* --chown is rsync's shorthand for "--usermap=*:USER --groupmap=*:GROUP", so a
|
||||||
|
* name TO value must be resolved on the RECEIVER, not on the sender. Append the
|
||||||
|
* equivalent map rule (FROM matches every id). The numeric/'*' forms are stored
|
||||||
|
* numerically exactly as rsync's id_parse/user_to_uid would. Returns 0 on
|
||||||
|
* success, -1 on a malformed numeric token or allocation failure. */
|
||||||
|
static int identity_append_chown_rule(Config* config, bool is_group, const char* token) {
|
||||||
|
IdentityMap rule;
|
||||||
|
memset(&rule, 0, sizeof(rule));
|
||||||
|
rule.from = IDENTITY_MATCH_ANY;
|
||||||
|
rule.from_hi = IDENTITY_MATCH_ANY;
|
||||||
|
if (strcmp(token, "*") == 0) {
|
||||||
|
rule.to = IDENTITY_CURRENT;
|
||||||
|
} else if (identity_all_digits(token[0] == '@' ? token + 1 : token)) {
|
||||||
|
if (identity_resolve_token(token, is_group, &rule.to) != 0) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "--chown numeric id is out of range: %s", token);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
rule.to = 0;
|
||||||
|
rule.to_name = str_dup(token);
|
||||||
|
if (!rule.to_name)
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
if (identity_append_rule(is_group ? &config->groupmap : &config->usermap,
|
||||||
|
is_group ? &config->groupmap_count : &config->usermap_count,
|
||||||
|
&rule) != 0) {
|
||||||
|
free(rule.to_name);
|
||||||
|
log_message(LOG_LEVEL_ERROR, "--chown has too many rules (max %d)", MAX_IDENTITY_MAP);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Resolve/record one --chown side. The source-side numeric value is kept in
|
||||||
|
* chown_uid/chown_gid purely as a fallback (the appended map rule resolves the
|
||||||
|
* name on the receiver and wins); a name that does not exist on the sender is
|
||||||
|
* accepted and left to receiver-side resolution, matching rsync. */
|
||||||
|
static int identity_parse_chown_side(Config* config, bool is_group, const char* token) {
|
||||||
|
if (identity_append_chown_rule(config, is_group, token) != 0)
|
||||||
|
return -1;
|
||||||
|
bool numeric = identity_all_digits(token[0] == '@' ? token + 1 : token);
|
||||||
|
int32_t resolved;
|
||||||
|
if (identity_resolve_token(token, is_group, &resolved) == 0) {
|
||||||
|
if (is_group) {
|
||||||
|
config->chown_gid = resolved;
|
||||||
|
config->chown_gid_set = true;
|
||||||
|
} else {
|
||||||
|
config->chown_uid = resolved;
|
||||||
|
config->chown_uid_set = true;
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
if (numeric) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "--chown could not resolve numeric id '%s'", token);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
/* Unknown sender-side name: rsync accepts it and resolves it (or warns) on
|
||||||
|
* the receiver; do the same instead of failing the whole run. */
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
int identity_parse_chown(Config* config, const char* value) {
|
int identity_parse_chown(Config* config, const char* value) {
|
||||||
if (!config || !value || *value == '\0') {
|
if (!config || !value || *value == '\0') {
|
||||||
log_message(LOG_LEVEL_ERROR, "--chown requires a value (USER:GROUP, USER, or :GROUP)");
|
log_message(LOG_LEVEL_ERROR, "--chown requires a value (USER:GROUP, USER, or :GROUP)");
|
||||||
@@ -404,32 +840,18 @@ int identity_parse_chown(Config* config, const char* value) {
|
|||||||
if (*user == '\0') {
|
if (*user == '\0') {
|
||||||
log_message(LOG_LEVEL_ERROR, "--chown requires a user or group (got '%s')", value);
|
log_message(LOG_LEVEL_ERROR, "--chown requires a user or group (got '%s')", value);
|
||||||
ret = -1;
|
ret = -1;
|
||||||
} else if (identity_resolve_token(user, false, &config->chown_uid) != 0) {
|
} else if (identity_parse_chown_side(config, false, user) != 0) {
|
||||||
log_message(LOG_LEVEL_ERROR,
|
|
||||||
"--chown could not resolve user '%s' (use a name that exists "
|
|
||||||
"on the source, '*', or @N)",
|
|
||||||
value);
|
|
||||||
ret = -1;
|
ret = -1;
|
||||||
} else {
|
|
||||||
config->chown_uid_set = true;
|
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
/* --chown=USER:GROUP, --chown=:GROUP, --chown=USER: */
|
/* --chown=USER:GROUP, --chown=:GROUP, --chown=USER: */
|
||||||
if (*user != '\0') {
|
if (*user != '\0' && identity_parse_chown_side(config, false, user) != 0) {
|
||||||
if (identity_resolve_token(user, false, &config->chown_uid) != 0) {
|
ret = -1;
|
||||||
log_message(LOG_LEVEL_ERROR, "--chown could not resolve user '%s'", value);
|
goto done;
|
||||||
ret = -1;
|
|
||||||
goto done;
|
|
||||||
}
|
|
||||||
config->chown_uid_set = true;
|
|
||||||
}
|
}
|
||||||
if (*group != '\0') {
|
if (*group != '\0' && identity_parse_chown_side(config, true, group) != 0) {
|
||||||
if (identity_resolve_token(group, true, &config->chown_gid) != 0) {
|
ret = -1;
|
||||||
log_message(LOG_LEVEL_ERROR, "--chown could not resolve group '%s'", value);
|
goto done;
|
||||||
ret = -1;
|
|
||||||
goto done;
|
|
||||||
}
|
|
||||||
config->chown_gid_set = true;
|
|
||||||
}
|
}
|
||||||
if (!*user && !*group) {
|
if (!*user && !*group) {
|
||||||
log_message(LOG_LEVEL_ERROR, "--chown must set a user, a group, or both (got '%s')", value);
|
log_message(LOG_LEVEL_ERROR, "--chown must set a user, a group, or both (got '%s')", value);
|
||||||
@@ -567,9 +989,6 @@ int identity_parse_copy_as(Config* config, const char* value) {
|
|||||||
config->copy_as_set = true;
|
config->copy_as_set = true;
|
||||||
config->copy_as_uid = uid;
|
config->copy_as_uid = uid;
|
||||||
config->copy_as_gid = gid;
|
config->copy_as_gid = gid;
|
||||||
/* Ownership application needs the metadata path (the source uid/gid must be
|
|
||||||
* transmitted); imply it exactly like --chown/--usermap/--groupmap. */
|
|
||||||
config->use_metadata = true;
|
|
||||||
ret = 0;
|
ret = 0;
|
||||||
|
|
||||||
done:
|
done:
|
||||||
@@ -580,34 +999,120 @@ done:
|
|||||||
|
|
||||||
/* ---- Receiver-side ownership application ---- */
|
/* ---- Receiver-side ownership application ---- */
|
||||||
|
|
||||||
static bool identity_map_lookup(const IdentityMap* map, int count, int32_t source_id,
|
/* True when a map rule's FROM matcher accepts `id`. A sentinel FROM never
|
||||||
|
* carries a range. IDENTITY_MATCH_UNNAMED mirrors rsync's empty FROM: it
|
||||||
|
* matches only ids that have no name in the account database (rsync matches the
|
||||||
|
* sender's names; FastSync transmits numeric ids only, so it approximates this
|
||||||
|
* with the receiver's database -- documented in RSYNC_COMPAT.md). */
|
||||||
|
static bool identity_map_from_matches(const IdentityMap* map, int32_t id, bool is_group) {
|
||||||
|
if (map->from == IDENTITY_MATCH_ANY)
|
||||||
|
return true;
|
||||||
|
if (map->from == IDENTITY_MATCH_UNNAMED)
|
||||||
|
return is_group ? (getgrgid((gid_t)id) == NULL) : (getpwuid((uid_t)id) == NULL);
|
||||||
|
return id >= map->from && id <= map->from_hi;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* First matching rule wins. A rule whose TO is a receiver-side name resolves it
|
||||||
|
* against the receiver's account database here; an unresolvable TO name is
|
||||||
|
* skipped with a warning and the next rule is considered (rsync prints "Unknown
|
||||||
|
* --usermap name on receiver" and leaves the id unmapped rather than aborting). */
|
||||||
|
static bool identity_map_lookup(const IdentityMap* map, int count, int32_t source_id, bool is_group,
|
||||||
int32_t* out_to) {
|
int32_t* out_to) {
|
||||||
for (int i = 0; i < count; i++) {
|
for (int i = 0; i < count; i++) {
|
||||||
if (map[i].from == IDENTITY_MATCH_ANY || map[i].from == source_id) {
|
if (!identity_map_from_matches(&map[i], source_id, is_group))
|
||||||
|
continue;
|
||||||
|
if (map[i].to_name) {
|
||||||
|
if (is_group) {
|
||||||
|
struct group* gr = getgrnam(map[i].to_name);
|
||||||
|
if (!gr) {
|
||||||
|
log_message(LOG_LEVEL_WARNING, "Unknown --groupmap name on receiver: %s", map[i].to_name);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
*out_to = (int32_t)gr->gr_gid;
|
||||||
|
} else {
|
||||||
|
struct passwd* pw = getpwnam(map[i].to_name);
|
||||||
|
if (!pw) {
|
||||||
|
log_message(LOG_LEVEL_WARNING, "Unknown --usermap name on receiver: %s", map[i].to_name);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
*out_to = (int32_t)pw->pw_uid;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
*out_to = map[i].to;
|
*out_to = map[i].to;
|
||||||
return true;
|
|
||||||
}
|
}
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Resolve the owner side from the negotiated policy. Sets *out and returns
|
||||||
|
* true when an owner-affecting request is active (a usermap, --chown USER, or
|
||||||
|
* -o/--owner); returns false (leaving *out untouched) when the owner side is
|
||||||
|
* not requested, so callers can pass (uid_t)-1 to fchown and leave it as-is.
|
||||||
|
* --numeric-ids only changes the RESOLUTION (raw id instead of a name lookup);
|
||||||
|
* it never makes the side requested. */
|
||||||
|
static bool identity_resolve_owner(int32_t source_uid, uid_t* out) {
|
||||||
|
if (!(g_identity.chown_uid_set || g_identity.preserve_owner || g_identity.usermap_count > 0))
|
||||||
|
return false;
|
||||||
|
int32_t target;
|
||||||
|
if (identity_map_lookup(g_identity.usermap, g_identity.usermap_count, source_uid, false,
|
||||||
|
&target)) {
|
||||||
|
*out = target == IDENTITY_CURRENT ? geteuid() : (uid_t)target;
|
||||||
|
} else if (g_identity.chown_uid_set) {
|
||||||
|
*out = g_identity.chown_uid == IDENTITY_CURRENT ? geteuid() : (uid_t)g_identity.chown_uid;
|
||||||
|
} else if (g_identity.numeric_ids) {
|
||||||
|
*out = (uid_t)source_uid;
|
||||||
|
} else {
|
||||||
|
/* Best-effort name mapping against the receiver's own database. When the
|
||||||
|
* transmitted (numeric) id has no name here, fall back to the raw numeric id
|
||||||
|
* so -o still preserves the source owner. */
|
||||||
|
struct passwd* pw = getpwuid((uid_t)source_uid);
|
||||||
|
if (pw) {
|
||||||
|
const struct passwd* mapped = getpwnam(pw->pw_name);
|
||||||
|
*out = mapped ? mapped->pw_uid : (uid_t)source_uid;
|
||||||
|
} else {
|
||||||
|
*out = (uid_t)source_uid;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Group-side counterpart of identity_resolve_owner(). */
|
||||||
|
static bool identity_resolve_group(int32_t source_gid, gid_t* out) {
|
||||||
|
if (!(g_identity.chown_gid_set || g_identity.preserve_group || g_identity.groupmap_count > 0))
|
||||||
|
return false;
|
||||||
|
int32_t target;
|
||||||
|
if (identity_map_lookup(g_identity.groupmap, g_identity.groupmap_count, source_gid, true,
|
||||||
|
&target)) {
|
||||||
|
*out = target == IDENTITY_CURRENT ? getegid() : (gid_t)target;
|
||||||
|
} else if (g_identity.chown_gid_set) {
|
||||||
|
*out = g_identity.chown_gid == IDENTITY_CURRENT ? getegid() : (gid_t)g_identity.chown_gid;
|
||||||
|
} else if (g_identity.numeric_ids) {
|
||||||
|
*out = (gid_t)source_gid;
|
||||||
|
} else {
|
||||||
|
struct group* gr = getgrgid((gid_t)source_gid);
|
||||||
|
if (gr) {
|
||||||
|
const struct group* mapped = getgrnam(gr->gr_name);
|
||||||
|
*out = mapped ? mapped->gr_gid : (gid_t)source_gid;
|
||||||
|
} else {
|
||||||
|
*out = (gid_t)source_gid;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
/* Resolve the target ownership from the negotiated policy against the entry's
|
/* Resolve the target ownership from the negotiated policy against the entry's
|
||||||
* current stat. Shared by the fd (regular file) and no-follow (symlink) apply
|
* current stat. Shared by the fd (regular file) and no-follow (symlink) apply
|
||||||
* paths. Returns false when no side is to be changed. */
|
* paths. Returns false when no side is to be changed. */
|
||||||
static bool identity_resolve_targets(const struct stat* st, int32_t source_uid, int32_t source_gid,
|
static bool identity_resolve_targets(const struct stat* st, int32_t source_uid, int32_t source_gid,
|
||||||
uid_t* out_uid, gid_t* out_gid) {
|
uid_t* out_uid, gid_t* out_gid) {
|
||||||
bool set_uid = false;
|
|
||||||
bool set_gid = false;
|
|
||||||
uid_t uid = 0;
|
|
||||||
gid_t gid = 0;
|
|
||||||
|
|
||||||
/* --copy-as (P7 Wave E) has the highest priority: it forces BOTH the owner
|
/* --copy-as (P7 Wave E) has the highest priority: it forces BOTH the owner
|
||||||
* and group of every written entry to the requested ids, beating usermap /
|
* and group of every written entry to the requested ids, beating usermap /
|
||||||
* groupmap / --chown / --numeric-ids and the best-effort name lookup. Only
|
* groupmap / --chown / --numeric-ids and the best-effort name lookup. Only
|
||||||
* skip when the entry already carries exactly those ids. */
|
* skip when the entry already carries exactly those ids. */
|
||||||
if (g_identity.copy_as_set) {
|
if (g_identity.copy_as_set) {
|
||||||
uid = (uid_t)g_identity.copy_as_uid;
|
uid_t uid = (uid_t)g_identity.copy_as_uid;
|
||||||
gid = (gid_t)g_identity.copy_as_gid;
|
gid_t gid = (gid_t)g_identity.copy_as_gid;
|
||||||
if (st->st_uid == uid && st->st_gid == gid)
|
if (st->st_uid == uid && st->st_gid == gid)
|
||||||
return false;
|
return false;
|
||||||
*out_uid = uid;
|
*out_uid = uid;
|
||||||
@@ -615,67 +1120,52 @@ static bool identity_resolve_targets(const struct stat* st, int32_t source_uid,
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
int32_t target;
|
/* Each side is resolved independently: -o/-g and the explicit identity flags
|
||||||
if (identity_map_lookup(g_identity.usermap, g_identity.usermap_count, source_uid, &target)) {
|
* request the owner/group respectively, and a side that is NOT requested must
|
||||||
uid = target == IDENTITY_CURRENT ? geteuid() : (uid_t)target;
|
* be left exactly as it is (`-1` to fchown on that side). This is what lets
|
||||||
set_uid = true;
|
* plain -g change only the group, or -o only the owner. */
|
||||||
} else if (g_identity.chown_uid_set) {
|
uid_t uid = (uid_t)-1;
|
||||||
uid = g_identity.chown_uid == IDENTITY_CURRENT ? geteuid() : (uid_t)g_identity.chown_uid;
|
gid_t gid = (gid_t)-1;
|
||||||
set_uid = true;
|
bool owner_requested = identity_resolve_owner(source_uid, &uid);
|
||||||
} else if (g_identity.numeric_ids) {
|
bool group_requested = identity_resolve_group(source_gid, &gid);
|
||||||
uid = (uid_t)source_uid;
|
if (!owner_requested && !group_requested)
|
||||||
set_uid = true;
|
|
||||||
} else {
|
|
||||||
/* Best-effort name mapping against the receiver's own database: if the
|
|
||||||
* transmitted (numeric) id resolves to a name present on this machine,
|
|
||||||
* re-resolve it. On a shared-account host this is the identity operation;
|
|
||||||
* when the id has no name here, the user side is left alone. */
|
|
||||||
struct passwd* pw = getpwuid((uid_t)source_uid);
|
|
||||||
if (pw) {
|
|
||||||
const struct passwd* mapped = getpwnam(pw->pw_name);
|
|
||||||
if (mapped) {
|
|
||||||
uid = mapped->pw_uid;
|
|
||||||
set_uid = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (identity_map_lookup(g_identity.groupmap, g_identity.groupmap_count, source_gid, &target)) {
|
|
||||||
gid = target == IDENTITY_CURRENT ? getegid() : (gid_t)target;
|
|
||||||
set_gid = true;
|
|
||||||
} else if (g_identity.chown_gid_set) {
|
|
||||||
gid = g_identity.chown_gid == IDENTITY_CURRENT ? getegid() : (gid_t)g_identity.chown_gid;
|
|
||||||
set_gid = true;
|
|
||||||
} else if (g_identity.numeric_ids) {
|
|
||||||
gid = (gid_t)source_gid;
|
|
||||||
set_gid = true;
|
|
||||||
} else {
|
|
||||||
struct group* gr = getgrgid((gid_t)source_gid);
|
|
||||||
if (gr) {
|
|
||||||
const struct group* mapped = getgrnam(gr->gr_name);
|
|
||||||
if (mapped) {
|
|
||||||
gid = mapped->gr_gid;
|
|
||||||
set_gid = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!set_uid && !set_gid)
|
|
||||||
return false;
|
return false;
|
||||||
/* An unset side keeps the file's current id so the other side can change. */
|
|
||||||
if (!set_uid)
|
/* Only change ownership when a requested side actually differs (avoid
|
||||||
uid = st->st_uid;
|
* needless syscalls and any chance of clearing setuid/setgid on an
|
||||||
if (!set_gid)
|
* already-correct entry). */
|
||||||
gid = st->st_gid;
|
bool changed = (owner_requested && uid != st->st_uid) || (group_requested && gid != st->st_gid);
|
||||||
/* Only change ownership when the target differs (avoid needless syscalls and
|
if (!changed)
|
||||||
* any chance of clearing setuid/setgid on an already-correct entry). */
|
|
||||||
if (st->st_uid == uid && st->st_gid == gid)
|
|
||||||
return false;
|
return false;
|
||||||
*out_uid = uid;
|
*out_uid = uid;
|
||||||
*out_gid = gid;
|
*out_gid = gid;
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* --fake-super storage resolution: the receiver records the ownership it WOULD
|
||||||
|
* have applied. A requested side uses the resolved mapping (--copy-as /
|
||||||
|
* usermap / --chown / -o/-g, with --numeric-ids as the raw-id modifier); a side
|
||||||
|
* that was not requested keeps the source's own id, so a plain --fake-super run
|
||||||
|
* records the source owner untouched. */
|
||||||
|
void identity_resolve_storage_ids(int32_t source_uid, int32_t source_gid, uint32_t* out_uid,
|
||||||
|
uint32_t* out_gid) {
|
||||||
|
if (g_identity.copy_as_set) {
|
||||||
|
*out_uid = (uint32_t)g_identity.copy_as_uid;
|
||||||
|
*out_gid = (uint32_t)g_identity.copy_as_gid;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
uid_t uid = (uid_t)source_uid;
|
||||||
|
gid_t gid = (gid_t)source_gid;
|
||||||
|
uid_t resolved_uid;
|
||||||
|
gid_t resolved_gid;
|
||||||
|
if (identity_resolve_owner(source_uid, &resolved_uid))
|
||||||
|
uid = resolved_uid;
|
||||||
|
if (identity_resolve_group(source_gid, &resolved_gid))
|
||||||
|
gid = resolved_gid;
|
||||||
|
*out_uid = (uint32_t)uid;
|
||||||
|
*out_gid = (uint32_t)gid;
|
||||||
|
}
|
||||||
|
|
||||||
static void identity_log_chown_failure(const char* what, uid_t uid, gid_t gid) {
|
static void identity_log_chown_failure(const char* what, uid_t uid, gid_t gid) {
|
||||||
/* EPERM/EACCES are expected when the receiver is not privileged (e.g. the CI
|
/* EPERM/EACCES are expected when the receiver is not privileged (e.g. the CI
|
||||||
* `nobody` user): warn and continue, never abort the transfer. Any other
|
* `nobody` user): warn and continue, never abort the transfer. Any other
|
||||||
@@ -710,8 +1200,12 @@ bool identity_apply_ownership(int fd, int32_t source_uid, int32_t source_gid) {
|
|||||||
/* Ownership application is OFF unless the client requested an identity flag.
|
/* Ownership application is OFF unless the client requested an identity flag.
|
||||||
* This is the controlled gate: a default (or plain -M) transfer never changes
|
* This is the controlled gate: a default (or plain -M) transfer never changes
|
||||||
* ownership, byte-for-byte preserving FastSync's existing behavior. --no-super
|
* ownership, byte-for-byte preserving FastSync's existing behavior. --no-super
|
||||||
* additionally forbids it even when the receiver is root. */
|
* additionally forbids it even when the receiver is root. --fake-super never
|
||||||
if (!identity_active_enabled() || !privilege_super_permitted() || fd < 0)
|
* performs a REAL chown: that would defeat the point of the flag (record the
|
||||||
|
* source ownership on an unprivileged receiver for a later privileged
|
||||||
|
* restore); the resolved ownership is stored in the reserved xattr instead by
|
||||||
|
* fake_super_store_fd(). */
|
||||||
|
if (!identity_active_enabled() || g_identity.fake_super || !privilege_super_permitted() || fd < 0)
|
||||||
return true;
|
return true;
|
||||||
struct stat st;
|
struct stat st;
|
||||||
if (fstat(fd, &st) != 0)
|
if (fstat(fd, &st) != 0)
|
||||||
@@ -731,7 +1225,8 @@ bool identity_apply_ownership(int fd, int32_t source_uid, int32_t source_gid) {
|
|||||||
|
|
||||||
bool identity_apply_ownership_link(int parent_fd, const char* leaf, int32_t source_uid,
|
bool identity_apply_ownership_link(int parent_fd, const char* leaf, int32_t source_uid,
|
||||||
int32_t source_gid) {
|
int32_t source_gid) {
|
||||||
if (!identity_active_enabled() || !privilege_super_permitted() || parent_fd < 0 || !leaf)
|
if (!identity_active_enabled() || g_identity.fake_super || !privilege_super_permitted() ||
|
||||||
|
parent_fd < 0 || !leaf)
|
||||||
return true;
|
return true;
|
||||||
struct stat st;
|
struct stat st;
|
||||||
if (fstatat(parent_fd, leaf, &st, AT_SYMLINK_NOFOLLOW) != 0)
|
if (fstatat(parent_fd, leaf, &st, AT_SYMLINK_NOFOLLOW) != 0)
|
||||||
|
|||||||
+43
-8
@@ -25,8 +25,14 @@
|
|||||||
|
|
||||||
/* Parse one --usermap= / --groupmap= value (comma-separated FROM:TO rules,
|
/* Parse one --usermap= / --groupmap= value (comma-separated FROM:TO rules,
|
||||||
* first match wins) into config->usermap / config->groupmap. is_group selects
|
* first match wins) into config->usermap / config->groupmap. is_group selects
|
||||||
* the group tables and name databases. Returns 0 on success, -1 on a
|
* the group tables and name databases. A FROM name wildcard (containing `*`,
|
||||||
* malformed spec or an unresolvable name (never a silent no-op). */
|
* `?` or `[...]`, but not the bare `*`) is expanded against the SENDER's
|
||||||
|
* account database at parse time into one or more numeric id/range rules
|
||||||
|
* (contiguous ids collapse to a range) sharing the same TO, because only
|
||||||
|
* numeric ids cross the wire; the expansion is capped at MAX_IDENTITY_MAP and a
|
||||||
|
* wildcard matching no account is an error. Returns 0 on success, -1 on a
|
||||||
|
* malformed spec, an unresolvable name, an unmatched wildcard, or a map that
|
||||||
|
* would exceed MAX_IDENTITY_MAP (never a silent no-op). */
|
||||||
int identity_parse_map(Config* config, const char* value, bool is_group);
|
int identity_parse_map(Config* config, const char* value, bool is_group);
|
||||||
|
|
||||||
/* Parse --chown=USER:GROUP. Supports USER:GROUP, USER (owner only), :GROUP
|
/* Parse --chown=USER:GROUP. Supports USER:GROUP, USER (owner only), :GROUP
|
||||||
@@ -80,16 +86,37 @@ void identity_clear_active(void);
|
|||||||
* snapshot. Ownership stays OFF ("do not apply") for every transfer that
|
* snapshot. Ownership stays OFF ("do not apply") for every transfer that
|
||||||
* requests none of them, preserving FastSync's existing behavior. --super /
|
* requests none of them, preserving FastSync's existing behavior. --super /
|
||||||
* --no-super alone does NOT enable ownership; an explicit identity flag
|
* --no-super alone does NOT enable ownership; an explicit identity flag
|
||||||
* (--numeric-ids / --chown / --usermap / --groupmap / --copy-as) is required. */
|
* (--numeric-ids / --chown / --usermap / --groupmap / --copy-as) or a
|
||||||
|
* preserve-source -o/--owner / -g/--group request is required. */
|
||||||
bool identity_active_enabled(void);
|
bool identity_active_enabled(void);
|
||||||
|
|
||||||
/* Pure, config-only predicate: true when the client requested ANY
|
/* Per-side predicates over the ACTIVE per-connection snapshot (call
|
||||||
* client-chosen ownership or super-user activity (--numeric-ids, --chown,
|
* identity_set_active() first). They mirror the owner_requested /
|
||||||
* --usermap/--groupmap, --copy-as, --fake-super, or an explicit --super). Used
|
* group_requested conditions inside identity_resolve_targets() exactly, so
|
||||||
* by the daemon module gate to decide whether a module's per-module opt-in is
|
* callers that must apply only one side (e.g. the --fake-super owner replay)
|
||||||
* required; it never reads the per-connection snapshot. */
|
* can pass (uid_t)-1 / (gid_t)-1 for the side that was NOT requested and leave
|
||||||
|
* it untouched. The owner side is requested by --copy-as, --chown USER,
|
||||||
|
* --numeric-ids, -o/--owner, or a non-empty --usermap; the group side by
|
||||||
|
* --copy-as, --chown :GROUP, --numeric-ids, -g/--group, or a non-empty
|
||||||
|
* --groupmap. */
|
||||||
|
bool identity_owner_requested(void);
|
||||||
|
bool identity_group_requested(void);
|
||||||
|
|
||||||
|
/* Pure, config-only predicate: true when the client requested ANY client-chosen
|
||||||
|
* ownership or super-user activity (--numeric-ids, --chown, --usermap/--groupmap,
|
||||||
|
* --copy-as, --fake-super, an explicit --super, or a preserve-source -o/-g).
|
||||||
|
* General awareness only; the daemon module gate uses the narrower
|
||||||
|
* identity_explicit_ownership_requested() below. Never reads the snapshot. */
|
||||||
bool identity_ownership_requested(const Config* config);
|
bool identity_ownership_requested(const Config* config);
|
||||||
|
|
||||||
|
/* Pure, config-only predicate for the narrow set that lets the CLIENT choose an
|
||||||
|
* arbitrary owner/group: --numeric-ids, --chown, --usermap/--groupmap,
|
||||||
|
* --copy-as, --fake-super, or an explicit --super. Deliberately EXCLUDES a
|
||||||
|
* plain -o/--owner / -g/--group (or -a) preserve-source request, which the
|
||||||
|
* daemon gate handles by forcing super-user ownership activity off rather than
|
||||||
|
* refusing the whole transfer. Never reads the snapshot. */
|
||||||
|
bool identity_explicit_ownership_requested(const Config* config);
|
||||||
|
|
||||||
/* Apply the negotiated ownership to an already-written file descriptor.
|
/* Apply the negotiated ownership to an already-written file descriptor.
|
||||||
* source_uid/source_gid are the transmitted numeric ids. Resolution order:
|
* source_uid/source_gid are the transmitted numeric ids. Resolution order:
|
||||||
* --copy-as (highest priority, forces both ids), then a matching
|
* --copy-as (highest priority, forces both ids), then a matching
|
||||||
@@ -106,6 +133,14 @@ bool identity_ownership_requested(const Config* config);
|
|||||||
* is active returns true. */
|
* is active returns true. */
|
||||||
bool identity_apply_ownership(int fd, int32_t source_uid, int32_t source_gid);
|
bool identity_apply_ownership(int fd, int32_t source_uid, int32_t source_gid);
|
||||||
|
|
||||||
|
/* Resolve the ownership that --fake-super should RECORD in the reserved xattr
|
||||||
|
* (rather than chown for real). A requested side (--copy-as / usermap /
|
||||||
|
* --chown / -o / -g, with --numeric-ids as the raw-id modifier) yields the
|
||||||
|
* resolved target; a side that was not requested keeps the transmitted source
|
||||||
|
* id. Must be called after identity_set_active(). */
|
||||||
|
void identity_resolve_storage_ids(int32_t source_uid, int32_t source_gid, uint32_t* out_uid,
|
||||||
|
uint32_t* out_gid);
|
||||||
|
|
||||||
/* P7 Wave D: the no-follow (symlink) counterpart. Resolves the same
|
/* P7 Wave D: the no-follow (symlink) counterpart. Resolves the same
|
||||||
* usermap/groupmap/chown/numeric-ids/copy-as policy but applies it with
|
* usermap/groupmap/chown/numeric-ids/copy-as policy but applies it with
|
||||||
* fchownat(..., AT_SYMLINK_NOFOLLOW) so a symlink's own ownership is changed
|
* fchownat(..., AT_SYMLINK_NOFOLLOW) so a symlink's own ownership is changed
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,41 @@
|
|||||||
|
#ifndef INCREMENTAL_CHECK_H
|
||||||
|
#define INCREMENTAL_CHECK_H
|
||||||
|
|
||||||
|
#include "config.h"
|
||||||
|
#include "file_types.h"
|
||||||
|
#include "protocol.h"
|
||||||
|
#include <stdbool.h>
|
||||||
|
|
||||||
|
/* Incremental-check module: the per-file STATUS_CHECK state machine, the
|
||||||
|
* incremental delta / alternate-basis / fuzzy matching helpers and the shared
|
||||||
|
* xattr receive helper. These declarations are re-exported by the
|
||||||
|
* file_receive.h facade. */
|
||||||
|
|
||||||
|
/* Whole-file payload bound shared by the plain receive path and the
|
||||||
|
* incremental check paths. */
|
||||||
|
#define MAX_FILE_DATA_SIZE MAX_RECEIVE_WHOLE_FILE_SIZE
|
||||||
|
|
||||||
|
/* Receive a file's xattr block (when the config enables xattr transport) and
|
||||||
|
* attach it to `file`. Returns false on a malformed/oversized frame. */
|
||||||
|
bool receive_file_xattrs(File* file, int fd, const Config* config);
|
||||||
|
|
||||||
|
File* receive_incremental_check(int fd, const Config* config, bool* skipped);
|
||||||
|
/* Extended variant used by the receiver. `would_transfer` (may be NULL) is set
|
||||||
|
* true only on the server-contacting --dry-run path when the file is not up to
|
||||||
|
* date: the receiver has already sent STATUS_DRY_RUN_TRANSFER and returns NULL
|
||||||
|
* without storing anything. On that path `*skipped` is true for an up-to-date
|
||||||
|
* (STATUS_OK) file and both flags are false for a genuine error. */
|
||||||
|
File* receive_incremental_check_ex(int fd, const Config* config, bool* skipped,
|
||||||
|
bool* would_transfer);
|
||||||
|
|
||||||
|
/* Testable basis quick-check / verification policy. file_basis_quick_match is
|
||||||
|
* rsync's metadata quick-check for a basis candidate (equal size is required
|
||||||
|
* separately by the caller; this adds the --size-only / mtime / --modify-window
|
||||||
|
* leg). file_basis_content_required reports whether a hit must ALSO be
|
||||||
|
* confirmed by a whole-file content digest (--verify-basis; false is the
|
||||||
|
* default rsync-parity behavior). */
|
||||||
|
bool file_basis_quick_match(const Config* config, const struct stat* st, time_t check_mtime,
|
||||||
|
long check_mtime_nsec);
|
||||||
|
bool file_basis_content_required(const Config* config);
|
||||||
|
|
||||||
|
#endif
|
||||||
+49
-5
@@ -5,6 +5,15 @@
|
|||||||
#include <stdbool.h>
|
#include <stdbool.h>
|
||||||
#include <stdint.h>
|
#include <stdint.h>
|
||||||
|
|
||||||
|
/* Ask the compiler to type-check the printf-style arguments of the variadic
|
||||||
|
* logging helpers. Only enabled for GNU-compatible compilers (gcc/clang). */
|
||||||
|
#if defined(__GNUC__)
|
||||||
|
#define LOG_PRINTF_ATTR(fmt_idx, first_vararg_idx) \
|
||||||
|
__attribute__((format(printf, fmt_idx, first_vararg_idx)))
|
||||||
|
#else
|
||||||
|
#define LOG_PRINTF_ATTR(fmt_idx, first_vararg_idx)
|
||||||
|
#endif
|
||||||
|
|
||||||
typedef enum { LOG_LEVEL_DEBUG, LOG_LEVEL_INFO, LOG_LEVEL_WARNING, LOG_LEVEL_ERROR } LogLevel;
|
typedef enum { LOG_LEVEL_DEBUG, LOG_LEVEL_INFO, LOG_LEVEL_WARNING, LOG_LEVEL_ERROR } LogLevel;
|
||||||
typedef enum { LOG_STDERR_ERRORS, LOG_STDERR_ALL } LogStderrMode;
|
typedef enum { LOG_STDERR_ERRORS, LOG_STDERR_ALL } LogStderrMode;
|
||||||
|
|
||||||
@@ -13,7 +22,19 @@ typedef enum {
|
|||||||
LOG_DEBUG_PROTO = 1u << 1,
|
LOG_DEBUG_PROTO = 1u << 1,
|
||||||
LOG_DEBUG_PACK = 1u << 2,
|
LOG_DEBUG_PACK = 1u << 2,
|
||||||
LOG_DEBUG_UTIL = 1u << 3,
|
LOG_DEBUG_UTIL = 1u << 3,
|
||||||
LOG_DEBUG_ALL = (1u << 4) - 1,
|
/* rsync --debug categories that now map to a natural FastSync event:
|
||||||
|
* flist (file-list scan progress), del (deletions), hash/deltasum
|
||||||
|
* (whole-file hashing and delta-sum generation), recv (receiver
|
||||||
|
* responses/signatures), filter (selection/exclusion decisions) and send
|
||||||
|
* (files handed to the sender). Only emitted when the category is
|
||||||
|
* explicitly enabled; a normal run stays silent. */
|
||||||
|
LOG_DEBUG_FLIST = 1u << 4,
|
||||||
|
LOG_DEBUG_DEL = 1u << 5,
|
||||||
|
LOG_DEBUG_HASH = 1u << 6,
|
||||||
|
LOG_DEBUG_RECV = 1u << 7,
|
||||||
|
LOG_DEBUG_FILTER = 1u << 8,
|
||||||
|
LOG_DEBUG_SEND = 1u << 9,
|
||||||
|
LOG_DEBUG_ALL = (1u << 10) - 1,
|
||||||
} LogDebugFlag;
|
} LogDebugFlag;
|
||||||
|
|
||||||
typedef enum {
|
typedef enum {
|
||||||
@@ -21,10 +42,33 @@ typedef enum {
|
|||||||
LOG_INFO_MISC = 1u << 1,
|
LOG_INFO_MISC = 1u << 1,
|
||||||
LOG_INFO_SKIP = 1u << 2,
|
LOG_INFO_SKIP = 1u << 2,
|
||||||
LOG_INFO_STATS = 1u << 3,
|
LOG_INFO_STATS = 1u << 3,
|
||||||
LOG_INFO_ALL = LOG_INFO_COPY | LOG_INFO_MISC | LOG_INFO_SKIP | LOG_INFO_STATS,
|
/* rsync categories that map to a FastSync event (emitted in rsync's line
|
||||||
|
* format): del (deletions), remove (sender-side source removal), name
|
||||||
|
* (transferred entry names), flist (file-list header), nonreg (skipped
|
||||||
|
* non-regular files), progress (per-file progress). rsync's `backup`
|
||||||
|
* category is accepted for CLI parity but stays silent: the receiver does the
|
||||||
|
* backing-up and FastSync has no backup event to report from the sender. */
|
||||||
|
LOG_INFO_DEL = 1u << 4,
|
||||||
|
LOG_INFO_REMOVE = 1u << 5,
|
||||||
|
LOG_INFO_NAME = 1u << 6,
|
||||||
|
LOG_INFO_FLIST = 1u << 7,
|
||||||
|
LOG_INFO_NONREG = 1u << 8,
|
||||||
|
LOG_INFO_PROGRESS = 1u << 9,
|
||||||
|
/* Marker for `--info=name2` and higher: also print rsync's
|
||||||
|
"NAME is uptodate" line for entries the receiver already has. It rides in
|
||||||
|
the info_level bitset (there is no separate Config field) and is never set
|
||||||
|
by --info=all (which selects level 1). */
|
||||||
|
LOG_INFO_NAME_UPTODATE = 1u << 10,
|
||||||
|
/* --info=mount: print rsync's `[sender] skipping mount-point dir NAME` when
|
||||||
|
* -xx/--one-file-system drops a mount-point directory (FastSync's client is
|
||||||
|
* the sender). */
|
||||||
|
LOG_INFO_MOUNT = 1u << 11,
|
||||||
|
LOG_INFO_ALL = LOG_INFO_COPY | LOG_INFO_MISC | LOG_INFO_SKIP | LOG_INFO_STATS | LOG_INFO_DEL |
|
||||||
|
LOG_INFO_REMOVE | LOG_INFO_NAME | LOG_INFO_FLIST | LOG_INFO_NONREG |
|
||||||
|
LOG_INFO_PROGRESS | LOG_INFO_MOUNT,
|
||||||
} LogInfoFlag;
|
} LogInfoFlag;
|
||||||
|
|
||||||
void log_message(LogLevel log_level, const char* message, ...);
|
void log_message(LogLevel log_level, const char* message, ...) LOG_PRINTF_ATTR(2, 3);
|
||||||
void log_perror(const char* context);
|
void log_perror(const char* context);
|
||||||
void set_log_level(LogLevel level);
|
void set_log_level(LogLevel level);
|
||||||
void set_log_debug_flags(uint32_t flags);
|
void set_log_debug_flags(uint32_t flags);
|
||||||
@@ -33,10 +77,10 @@ uint32_t get_log_debug_flags(void);
|
|||||||
* the debug log level is enabled AND the flag is selected. Hot paths use this
|
* the debug log level is enabled AND the flag is selected. Hot paths use this
|
||||||
* to skip expensive message formatting/escaping when the line is filtered. */
|
* to skip expensive message formatting/escaping when the line is filtered. */
|
||||||
bool log_debug_enabled(LogDebugFlag flag);
|
bool log_debug_enabled(LogDebugFlag flag);
|
||||||
void log_debug_message(LogDebugFlag flag, const char* message, ...);
|
void log_debug_message(LogDebugFlag flag, const char* message, ...) LOG_PRINTF_ATTR(2, 3);
|
||||||
void set_log_info_flags(uint32_t flags);
|
void set_log_info_flags(uint32_t flags);
|
||||||
uint32_t get_log_info_flags(void);
|
uint32_t get_log_info_flags(void);
|
||||||
void log_info_message(LogInfoFlag flag, const char* message, ...);
|
void log_info_message(LogInfoFlag flag, const char* message, ...) LOG_PRINTF_ATTR(2, 3);
|
||||||
void log_set_file(FILE* fp);
|
void log_set_file(FILE* fp);
|
||||||
void log_set_8_bit_output(bool enabled);
|
void log_set_8_bit_output(bool enabled);
|
||||||
bool log_get_8_bit_output(void);
|
bool log_get_8_bit_output(void);
|
||||||
|
|||||||
+124
-52
@@ -209,22 +209,71 @@ FileMetadata* metadata_receive(int file_descriptor, int* ok) {
|
|||||||
return m;
|
return m;
|
||||||
}
|
}
|
||||||
|
|
||||||
static mode_t metadata_mode(const FileMetadata* metadata, mode_t current_mode,
|
bool metadata_mode_for_policy(mode_t source_mode, mode_t current_mode, FileAttrPolicy policy,
|
||||||
bool preserve_executability) {
|
mode_t* out_mode) {
|
||||||
|
const mode_t special_bits = (mode_t)(S_ISUID | S_ISGID | S_ISVTX);
|
||||||
const mode_t execute_bits = S_IXUSR | S_IXGRP | S_IXOTH;
|
const mode_t execute_bits = S_IXUSR | S_IXGRP | S_IXOTH;
|
||||||
if (preserve_executability)
|
if (policy.perms) {
|
||||||
return (current_mode & 0777 & ~execute_bits) | (metadata->mode & execute_bits);
|
/* rsync --perms copies the source's permission and special bits exactly,
|
||||||
return metadata->mode & 0777 & ~(S_IWGRP | S_IWOTH);
|
* including group/other write and setuid/setgid/sticky. The kernel may
|
||||||
|
* still clear setgid when the receiver is not in the file's group; the
|
||||||
|
* caller logs a failed chmod rather than silently masking the bits here.
|
||||||
|
* Setuid/setgid/sticky are super-user activities: when the connection did
|
||||||
|
* not permit them (SUPER_MODE_OFF / --no-super) they are stripped, so a
|
||||||
|
* client can never install a privileged bit on a receiver that forbade
|
||||||
|
* super-user activities. This also covers bits introduced by --chmod,
|
||||||
|
* whose result is fed in as source_mode. */
|
||||||
|
mode_t bits = source_mode & (mode_t)(special_bits | 0777);
|
||||||
|
if (!policy.super_permitted)
|
||||||
|
bits &= ~special_bits;
|
||||||
|
*out_mode = bits;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (policy.executability) {
|
||||||
|
/* -E/--executability (rsync 3.4 rule): do NOT copy the source's execute
|
||||||
|
* bits per class. If the source is executable at all, derive the execute
|
||||||
|
* bits from the DESTINATION's own read bits (so a class that can read may
|
||||||
|
* execute); otherwise clear every execute bit. This runs on the
|
||||||
|
* destination-derived base (pre-existing dest mode, or source&~umask for a
|
||||||
|
* new file), and leaves the special bits untouched. --perms wins when both
|
||||||
|
* are set (handled above). The destination's own special bits survive
|
||||||
|
* unless super-user activities are forbidden. */
|
||||||
|
mode_t base = current_mode & (mode_t)(special_bits | 0777);
|
||||||
|
if (!policy.super_permitted)
|
||||||
|
base &= ~special_bits;
|
||||||
|
if (source_mode & 0111)
|
||||||
|
*out_mode = base | ((base & 0444) >> 2);
|
||||||
|
else
|
||||||
|
*out_mode = base & ~execute_bits;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
/* Neither requested: no source mode is applied at all. */
|
||||||
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
void file_restore_metadata(const char* path, const FileMetadata* metadata,
|
FileAttrPolicy file_attr_policy_from_config(const Config* config) {
|
||||||
bool preserve_executability) {
|
FileAttrPolicy policy = {0};
|
||||||
|
if (config) {
|
||||||
|
policy.perms = config->preserve_perms;
|
||||||
|
policy.times = config->preserve_times;
|
||||||
|
policy.atimes = config->preserve_atimes;
|
||||||
|
policy.executability = config->use_executability;
|
||||||
|
policy.super_permitted = privilege_super_mode_permitted(config->super_mode);
|
||||||
|
}
|
||||||
|
return policy;
|
||||||
|
}
|
||||||
|
|
||||||
|
void file_restore_metadata(const char* path, const FileMetadata* metadata, FileAttrPolicy policy) {
|
||||||
if (metadata == NULL)
|
if (metadata == NULL)
|
||||||
return;
|
return;
|
||||||
struct stat current;
|
bool apply_mode = false;
|
||||||
mode_t current_mode = stat(path, ¤t) == 0 ? current.st_mode : 0;
|
mode_t safe_mode = 0;
|
||||||
mode_t safe_mode = metadata_mode(metadata, current_mode, preserve_executability);
|
if (policy.perms || policy.executability) {
|
||||||
if (chmod(path, safe_mode) != 0) {
|
struct stat current;
|
||||||
|
mode_t current_mode = stat(path, ¤t) == 0 ? current.st_mode : 0;
|
||||||
|
apply_mode = metadata_mode_for_policy(metadata->mode, current_mode, policy, &safe_mode);
|
||||||
|
}
|
||||||
|
if (apply_mode && chmod(path, safe_mode) != 0) {
|
||||||
char* escaped_path = output_escape(path, log_get_8_bit_output());
|
char* escaped_path = output_escape(path, log_get_8_bit_output());
|
||||||
log_message(LOG_LEVEL_WARNING, "Failed to chmod %s: %s",
|
log_message(LOG_LEVEL_WARNING, "Failed to chmod %s: %s",
|
||||||
escaped_path ? escaped_path : "<allocation failed>", strerror(errno));
|
escaped_path ? escaped_path : "<allocation failed>", strerror(errno));
|
||||||
@@ -232,14 +281,23 @@ void file_restore_metadata(const char* path, const FileMetadata* metadata,
|
|||||||
}
|
}
|
||||||
/* Never apply client-supplied ownership. The descriptor API below is the
|
/* Never apply client-supplied ownership. The descriptor API below is the
|
||||||
receiver write path; retain this legacy API only for compatibility. */
|
receiver write path; retain this legacy API only for compatibility. */
|
||||||
struct timespec times[2];
|
if (policy.times || (policy.atimes && metadata->atime_valid)) {
|
||||||
times[0].tv_sec = 0;
|
struct timespec times[2] = {{.tv_sec = 0, .tv_nsec = UTIME_OMIT},
|
||||||
times[0].tv_nsec = UTIME_OMIT;
|
{.tv_sec = 0, .tv_nsec = UTIME_OMIT}};
|
||||||
times[1].tv_sec = metadata->mtime_sec;
|
if (policy.times) {
|
||||||
times[1].tv_nsec = metadata->mtime_nsec;
|
times[1].tv_sec = metadata->mtime_sec;
|
||||||
if (metadata->atime_valid) {
|
times[1].tv_nsec = metadata->mtime_nsec;
|
||||||
times[0].tv_sec = metadata->atime_sec;
|
}
|
||||||
times[0].tv_nsec = metadata->atime_nsec;
|
if (policy.atimes && metadata->atime_valid) {
|
||||||
|
times[0].tv_sec = metadata->atime_sec;
|
||||||
|
times[0].tv_nsec = metadata->atime_nsec;
|
||||||
|
}
|
||||||
|
if (utimensat(AT_FDCWD, path, times, 0) != 0) {
|
||||||
|
char* escaped_path = output_escape(path, log_get_8_bit_output());
|
||||||
|
log_message(LOG_LEVEL_WARNING, "Failed to set timestamps on %s: %s",
|
||||||
|
escaped_path ? escaped_path : "<allocation failed>", strerror(errno));
|
||||||
|
free(escaped_path);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if (metadata->crtime_valid) {
|
if (metadata->crtime_valid) {
|
||||||
log_message(LOG_LEVEL_DEBUG,
|
log_message(LOG_LEVEL_DEBUG,
|
||||||
@@ -247,16 +305,10 @@ void file_restore_metadata(const char* path, const FileMetadata* metadata,
|
|||||||
"setter exists",
|
"setter exists",
|
||||||
(long long)metadata->crtime_sec, metadata->crtime_nsec, path);
|
(long long)metadata->crtime_sec, metadata->crtime_nsec, path);
|
||||||
}
|
}
|
||||||
if (utimensat(AT_FDCWD, path, times, 0) != 0) {
|
|
||||||
char* escaped_path = output_escape(path, log_get_8_bit_output());
|
|
||||||
log_message(LOG_LEVEL_WARNING, "Failed to set timestamps on %s: %s",
|
|
||||||
escaped_path ? escaped_path : "<allocation failed>", strerror(errno));
|
|
||||||
free(escaped_path);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
bool file_restore_symlink_metadata(const char* path, const FileMetadata* metadata,
|
bool file_restore_symlink_metadata(const char* path, const FileMetadata* metadata,
|
||||||
bool omit_link_times) {
|
FileAttrPolicy policy, bool omit_link_times) {
|
||||||
if (path == NULL || metadata == NULL)
|
if (path == NULL || metadata == NULL)
|
||||||
return !identity_copy_as_active();
|
return !identity_copy_as_active();
|
||||||
char* leaf = NULL;
|
char* leaf = NULL;
|
||||||
@@ -269,18 +321,27 @@ bool file_restore_symlink_metadata(const char* path, const FileMetadata* metadat
|
|||||||
best-effort. */
|
best-effort. */
|
||||||
bool owned = identity_apply_ownership_link(parent_fd, leaf, (int32_t)metadata->uid,
|
bool owned = identity_apply_ownership_link(parent_fd, leaf, (int32_t)metadata->uid,
|
||||||
(int32_t)metadata->gid);
|
(int32_t)metadata->gid);
|
||||||
/* Symlink mode: not settable on Linux (fchmodat AT_SYMLINK_NOFOLLOW returns
|
/* Symlink mode: only when -p is in effect. It is not settable on Linux
|
||||||
EOPNOTSUPP/ENOTSUP); attempt it for platforms that support it and quietly
|
(fchmodat AT_SYMLINK_NOFOLLOW returns EOPNOTSUPP/ENOTSUP); attempt it for
|
||||||
ignore the unsupported case so the transfer never fails over it. */
|
platforms that support it and quietly ignore the unsupported case so the
|
||||||
mode_t link_mode = metadata->mode & 0777;
|
transfer never fails over it. */
|
||||||
if (fchmodat(parent_fd, leaf, link_mode, AT_SYMLINK_NOFOLLOW) != 0 && errno != EOPNOTSUPP &&
|
if (policy.perms) {
|
||||||
errno != ENOTSUP && errno != ENOSYS) {
|
mode_t link_mode = metadata->mode & (mode_t)(S_ISUID | S_ISGID | S_ISVTX | 0777);
|
||||||
log_message(LOG_LEVEL_DEBUG, "Could not set symlink mode on %s: %s", path, strerror(errno));
|
if (!policy.super_permitted)
|
||||||
|
link_mode &= ~(mode_t)(S_ISUID | S_ISGID | S_ISVTX);
|
||||||
|
if (fchmodat(parent_fd, leaf, link_mode, AT_SYMLINK_NOFOLLOW) != 0 && errno != EOPNOTSUPP &&
|
||||||
|
errno != ENOTSUP && errno != ENOSYS) {
|
||||||
|
log_message(LOG_LEVEL_DEBUG, "Could not set symlink mode on %s: %s", path, strerror(errno));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if (!omit_link_times) {
|
if (!omit_link_times && (policy.times || (policy.atimes && metadata->atime_valid))) {
|
||||||
struct timespec times[2] = {{.tv_sec = 0, .tv_nsec = UTIME_OMIT},
|
struct timespec times[2] = {{.tv_sec = 0, .tv_nsec = UTIME_OMIT},
|
||||||
{.tv_sec = metadata->mtime_sec, .tv_nsec = metadata->mtime_nsec}};
|
{.tv_sec = 0, .tv_nsec = UTIME_OMIT}};
|
||||||
if (metadata->atime_valid) {
|
if (policy.times) {
|
||||||
|
times[1].tv_sec = metadata->mtime_sec;
|
||||||
|
times[1].tv_nsec = metadata->mtime_nsec;
|
||||||
|
}
|
||||||
|
if (policy.atimes && metadata->atime_valid) {
|
||||||
times[0].tv_sec = metadata->atime_sec;
|
times[0].tv_sec = metadata->atime_sec;
|
||||||
times[0].tv_nsec = metadata->atime_nsec;
|
times[0].tv_nsec = metadata->atime_nsec;
|
||||||
}
|
}
|
||||||
@@ -296,19 +357,13 @@ bool file_restore_symlink_metadata(const char* path, const FileMetadata* metadat
|
|||||||
return owned;
|
return owned;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool file_restore_metadata_fd(int fd, const FileMetadata* metadata, bool preserve_executability) {
|
bool file_restore_metadata_fd(int fd, const FileMetadata* metadata, FileAttrPolicy policy) {
|
||||||
if (fd < 0 || metadata == NULL)
|
if (fd < 0 || metadata == NULL)
|
||||||
return metadata == NULL;
|
return metadata == NULL;
|
||||||
bool ok = true;
|
bool ok = true;
|
||||||
struct stat current;
|
|
||||||
if (fstat(fd, ¤t) != 0)
|
|
||||||
return false;
|
|
||||||
mode_t safe_mode = metadata_mode(metadata, current.st_mode, preserve_executability);
|
|
||||||
if (fchmod(fd, safe_mode) != 0)
|
|
||||||
ok = false;
|
|
||||||
/* Client uid/gid values are deliberately not authoritative UNLESS the client
|
/* Client uid/gid values are deliberately not authoritative UNLESS the client
|
||||||
explicitly opted in with an identity flag (--numeric-ids / --usermap /
|
explicitly opted in with an identity flag (--numeric-ids / --usermap /
|
||||||
--groupmap / --chown). identity_apply_ownership is the controlled,
|
--groupmap / --chown / -o/-g). identity_apply_ownership is the controlled,
|
||||||
privilege-gated path: it consults the negotiated policy, resolves the
|
privilege-gated path: it consults the negotiated policy, resolves the
|
||||||
target ids, and applies them via an fd-relative fchown() that is confined
|
target ids, and applies them via an fd-relative fchown() that is confined
|
||||||
to the just-written file (EPERM/EACCES are logged, never fatal) -- EXCEPT
|
to the just-written file (EPERM/EACCES are logged, never fatal) -- EXCEPT
|
||||||
@@ -316,14 +371,19 @@ bool file_restore_metadata_fd(int fd, const FileMetadata* metadata, bool preserv
|
|||||||
marks this entry as failed instead of reporting a wrong-owner write as
|
marks this entry as failed instead of reporting a wrong-owner write as
|
||||||
success. With no identity flag set it is a no-op, so a default or plain -M
|
success. With no identity flag set it is a no-op, so a default or plain -M
|
||||||
transfer keeps FastSync's existing behavior of never applying client
|
transfer keeps FastSync's existing behavior of never applying client
|
||||||
ownership. */
|
ownership. Ownership runs BEFORE the mode because a chown clears
|
||||||
|
setuid/setgid; rsync likewise chowns first and then restores the source
|
||||||
|
mode (including its special bits). */
|
||||||
if (!identity_apply_ownership(fd, (int32_t)metadata->uid, (int32_t)metadata->gid))
|
if (!identity_apply_ownership(fd, (int32_t)metadata->uid, (int32_t)metadata->gid))
|
||||||
ok = false;
|
ok = false;
|
||||||
struct timespec times[2] = {{.tv_sec = 0, .tv_nsec = UTIME_OMIT},
|
if (policy.perms || policy.executability) {
|
||||||
{.tv_sec = metadata->mtime_sec, .tv_nsec = metadata->mtime_nsec}};
|
struct stat current;
|
||||||
if (metadata->atime_valid) {
|
if (fstat(fd, ¤t) != 0)
|
||||||
times[0].tv_sec = metadata->atime_sec;
|
return false;
|
||||||
times[0].tv_nsec = metadata->atime_nsec;
|
mode_t safe_mode = 0;
|
||||||
|
bool apply_mode = metadata_mode_for_policy(metadata->mode, current.st_mode, policy, &safe_mode);
|
||||||
|
if (apply_mode && fchmod(fd, safe_mode) != 0)
|
||||||
|
ok = false;
|
||||||
}
|
}
|
||||||
/* --crtimes captures and transmits the source birth time, but there is no
|
/* --crtimes captures and transmits the source birth time, but there is no
|
||||||
* portable way to set a birth time (utimensat can only set atime/mtime), so
|
* portable way to set a birth time (utimensat can only set atime/mtime), so
|
||||||
@@ -335,7 +395,19 @@ bool file_restore_metadata_fd(int fd, const FileMetadata* metadata, bool preserv
|
|||||||
"crtime (birth time) %lld.%09ld transmitted but not applied: no portable setter",
|
"crtime (birth time) %lld.%09ld transmitted but not applied: no portable setter",
|
||||||
(long long)metadata->crtime_sec, metadata->crtime_nsec);
|
(long long)metadata->crtime_sec, metadata->crtime_nsec);
|
||||||
}
|
}
|
||||||
if (futimens(fd, times) != 0)
|
if (policy.times || (policy.atimes && metadata->atime_valid)) {
|
||||||
ok = false;
|
struct timespec times[2] = {{.tv_sec = 0, .tv_nsec = UTIME_OMIT},
|
||||||
|
{.tv_sec = 0, .tv_nsec = UTIME_OMIT}};
|
||||||
|
if (policy.times) {
|
||||||
|
times[1].tv_sec = metadata->mtime_sec;
|
||||||
|
times[1].tv_nsec = metadata->mtime_nsec;
|
||||||
|
}
|
||||||
|
if (policy.atimes && metadata->atime_valid) {
|
||||||
|
times[0].tv_sec = metadata->atime_sec;
|
||||||
|
times[0].tv_nsec = metadata->atime_nsec;
|
||||||
|
}
|
||||||
|
if (futimens(fd, times) != 0)
|
||||||
|
ok = false;
|
||||||
|
}
|
||||||
return ok;
|
return ok;
|
||||||
}
|
}
|
||||||
|
|||||||
+20
-7
@@ -2,6 +2,7 @@
|
|||||||
#define METADATA_H
|
#define METADATA_H
|
||||||
|
|
||||||
#include "file.h"
|
#include "file.h"
|
||||||
|
#include "file_attr.h"
|
||||||
#include <stdbool.h>
|
#include <stdbool.h>
|
||||||
#include <stddef.h>
|
#include <stddef.h>
|
||||||
#include <stdint.h>
|
#include <stdint.h>
|
||||||
@@ -49,19 +50,31 @@ void metadata_to_buf(char** buf, const FileMetadata* m);
|
|||||||
FileMetadata* metadata_from_buf(const uint8_t* buf, size_t len);
|
FileMetadata* metadata_from_buf(const uint8_t* buf, size_t len);
|
||||||
bool metadata_send(int file_descriptor, const FileMetadata* m);
|
bool metadata_send(int file_descriptor, const FileMetadata* m);
|
||||||
FileMetadata* metadata_receive(int file_descriptor, int* ok);
|
FileMetadata* metadata_receive(int file_descriptor, int* ok);
|
||||||
void file_restore_metadata(const char* path, const FileMetadata* metadata,
|
void file_restore_metadata(const char* path, const FileMetadata* metadata, FileAttrPolicy policy);
|
||||||
bool preserve_executability);
|
bool file_restore_metadata_fd(int fd, const FileMetadata* metadata, FileAttrPolicy policy);
|
||||||
bool file_restore_metadata_fd(int fd, const FileMetadata* metadata, bool preserve_executability);
|
|
||||||
|
/* Shared mode-policy helper: the single source of truth for the receiver's
|
||||||
|
* mode rule. Given a source mode and the destination's CURRENT mode, returns
|
||||||
|
* true and stores the exact mode to apply in *out_mode when `policy` requests
|
||||||
|
* a change, or false when it requests neither --perms nor --executability (the
|
||||||
|
* caller then leaves the destination mode alone). --perms wins over -E; the
|
||||||
|
* -E rule derives exec bits from the destination's read bits (rsync 3.4);
|
||||||
|
* group/other write is never granted from a client-supplied mode. Shared by
|
||||||
|
* file_restore_metadata_fd() and the --fake-super replay so the two cannot
|
||||||
|
* diverge. */
|
||||||
|
bool metadata_mode_for_policy(mode_t source_mode, mode_t current_mode, FileAttrPolicy policy,
|
||||||
|
mode_t* out_mode);
|
||||||
/* P7 Wave D: apply a SYMLINK's own metadata using no-follow primitives only
|
/* P7 Wave D: apply a SYMLINK's own metadata using no-follow primitives only
|
||||||
* (utimensat/lchown/fchmodat with AT_SYMLINK_NOFOLLOW), confined fd-relative
|
* (utimensat/lchown/fchmodat with AT_SYMLINK_NOFOLLOW), confined fd-relative
|
||||||
* under the authorized root. `omit_link_times` (-J/--omit-link-times)
|
* under the authorized root. The link's mode is applied only when policy.perms;
|
||||||
* suppresses the timestamps; the link's mode/ownership are still attempted
|
* policy.times (further suppressed by `omit_link_times` for -J) applies the
|
||||||
* (ownership stays gated by the identity policy and by default is not applied).
|
* mtime with policy.atimes controlling the atime slot; ownership stays gated by
|
||||||
|
* the identity policy and by default is not applied.
|
||||||
* A null metadata or an unfollowable parent is a harmless no-op. Returns false
|
* A null metadata or an unfollowable parent is a harmless no-op. Returns false
|
||||||
* only when a REQUIRED --copy-as ownership application failed, so the caller can
|
* only when a REQUIRED --copy-as ownership application failed, so the caller can
|
||||||
* report the entry as failed instead of claiming a wrong-owner success. */
|
* report the entry as failed instead of claiming a wrong-owner success. */
|
||||||
bool file_restore_symlink_metadata(const char* path, const FileMetadata* metadata,
|
bool file_restore_symlink_metadata(const char* path, const FileMetadata* metadata,
|
||||||
bool omit_link_times);
|
FileAttrPolicy policy, bool omit_link_times);
|
||||||
|
|
||||||
/* Compare timestamps using rsync's whole-second modification window. */
|
/* Compare timestamps using rsync's whole-second modification window. */
|
||||||
bool metadata_mtime_matches(time_t left_sec, long left_nsec, time_t right_sec, long right_nsec,
|
bool metadata_mtime_matches(time_t left_sec, long left_nsec, time_t right_sec, long right_nsec,
|
||||||
|
|||||||
@@ -30,20 +30,29 @@ PipelineContextSender* pipeline_context_sender_create(Config* config, Queue* que
|
|||||||
context->max_queue_bytes = 0;
|
context->max_queue_bytes = 0;
|
||||||
context->manifest = NULL;
|
context->manifest = NULL;
|
||||||
context->excluded_paths = NULL;
|
context->excluded_paths = NULL;
|
||||||
|
context->size_skipped_paths = NULL;
|
||||||
|
context->synced_dirs = NULL;
|
||||||
|
context->plan_dirs = NULL;
|
||||||
context->missing_args = NULL;
|
context->missing_args = NULL;
|
||||||
context->scan_had_io_error = false;
|
context->scan_had_io_error = false;
|
||||||
context->remove_source_files = NULL;
|
context->remove_source_files = NULL;
|
||||||
context->early_delete = false;
|
context->early_delete = false;
|
||||||
|
context->prescan_chunks = NULL;
|
||||||
|
context->delete_plans = NULL;
|
||||||
|
context->delete_suppressed = false;
|
||||||
context->scan_stopped_early = false;
|
context->scan_stopped_early = false;
|
||||||
context->total_files = 0;
|
context->total_files = 0;
|
||||||
context->progress_bytes = 0;
|
context->progress_bytes = 0;
|
||||||
context->total_bytes = 0;
|
context->total_bytes = 0;
|
||||||
|
memset(&context->stats, 0, sizeof(context->stats));
|
||||||
context->sender_done = false;
|
context->sender_done = false;
|
||||||
atomic_init(&context->cancelled, false);
|
atomic_init(&context->cancelled, false);
|
||||||
protocol_session_init(&context->allocation_session, -1, -1);
|
protocol_session_init(&context->allocation_session, -1, -1);
|
||||||
protocol_session_set_max_alloc(&context->allocation_session, config->max_alloc);
|
protocol_session_set_max_alloc(&context->allocation_session, config->max_alloc);
|
||||||
context->dir_entries = NULL;
|
context->dir_entries = NULL;
|
||||||
context->dir_entries_mutex_init = false;
|
context->dir_entries_mutex_init = false;
|
||||||
|
atomic_init(&context->dir_count, 0);
|
||||||
|
context->delete_limit = false;
|
||||||
int init = 0;
|
int init = 0;
|
||||||
if (config->use_metadata) {
|
if (config->use_metadata) {
|
||||||
context->dir_entries = array_list_create(file_destroy);
|
context->dir_entries = array_list_create(file_destroy);
|
||||||
@@ -78,11 +87,13 @@ PipelineContextSender* pipeline_context_sender_create(Config* config, Queue* que
|
|||||||
return context;
|
return context;
|
||||||
|
|
||||||
fail:
|
fail:
|
||||||
log_perror("Error initializing synchronization objects");
|
log_message(LOG_LEVEL_ERROR, "%s", "Error initializing synchronization objects");
|
||||||
if (context->dir_entries_mutex_init)
|
if (context->dir_entries_mutex_init)
|
||||||
mtx_destroy(&context->dir_entries_mutex);
|
mtx_destroy(&context->dir_entries_mutex);
|
||||||
if (context->dir_entries)
|
if (context->dir_entries)
|
||||||
array_list_delete(context->dir_entries);
|
array_list_delete(context->dir_entries);
|
||||||
|
if (init >= 7)
|
||||||
|
mtx_destroy(&context->mutex_progress);
|
||||||
if (init >= 6)
|
if (init >= 6)
|
||||||
cnd_destroy(&context->condition_not_empty_loader);
|
cnd_destroy(&context->condition_not_empty_loader);
|
||||||
if (init >= 5)
|
if (init >= 5)
|
||||||
@@ -184,8 +195,18 @@ void pipeline_context_sender_destroy(PipelineContextSender* context) {
|
|||||||
if (context->manifest) {
|
if (context->manifest) {
|
||||||
array_list_delete(context->manifest);
|
array_list_delete(context->manifest);
|
||||||
}
|
}
|
||||||
|
if (context->prescan_chunks)
|
||||||
|
array_list_delete(context->prescan_chunks);
|
||||||
|
if (context->delete_plans)
|
||||||
|
delete_plan_sender_destroy(context->delete_plans);
|
||||||
if (context->excluded_paths)
|
if (context->excluded_paths)
|
||||||
array_list_delete(context->excluded_paths);
|
array_list_delete(context->excluded_paths);
|
||||||
|
if (context->size_skipped_paths)
|
||||||
|
array_list_delete(context->size_skipped_paths);
|
||||||
|
if (context->synced_dirs)
|
||||||
|
array_list_delete(context->synced_dirs);
|
||||||
|
if (context->plan_dirs)
|
||||||
|
array_list_delete(context->plan_dirs);
|
||||||
if (context->missing_args)
|
if (context->missing_args)
|
||||||
array_list_delete(context->missing_args);
|
array_list_delete(context->missing_args);
|
||||||
if (context->remove_source_files)
|
if (context->remove_source_files)
|
||||||
|
|||||||
@@ -7,7 +7,9 @@
|
|||||||
#include "array_list.h"
|
#include "array_list.h"
|
||||||
#include "chunk.h"
|
#include "chunk.h"
|
||||||
#include "config.h"
|
#include "config.h"
|
||||||
|
#include "delete_plan.h"
|
||||||
#include "file.h"
|
#include "file.h"
|
||||||
|
#include "format.h"
|
||||||
#include "protocol.h"
|
#include "protocol.h"
|
||||||
#include "queue.h"
|
#include "queue.h"
|
||||||
#include "stop_condition.h"
|
#include "stop_condition.h"
|
||||||
@@ -42,6 +44,23 @@ typedef struct {
|
|||||||
scanner's exclusion sink) or, in the early modes, by the path-only pre-scan
|
scanner's exclusion sink) or, in the early modes, by the path-only pre-scan
|
||||||
on the calling thread before the pipeline starts. */
|
on the calling thread before the pipeline starts. */
|
||||||
ArrayList* excluded_paths;
|
ArrayList* excluded_paths;
|
||||||
|
/* --max-size/--min-size pruned source paths. These are ALWAYS sent as
|
||||||
|
protected prefixes (even with --delete-excluded), so the destination
|
||||||
|
mirrors of size-skipped files survive --delete like rsync. Populated by
|
||||||
|
the scanner thread (workers append under mutex_scanner) or, in the early
|
||||||
|
modes, by the path-only pre-scan on the calling thread. */
|
||||||
|
ArrayList* size_skipped_paths;
|
||||||
|
/* Destination-relative paths of the directories the source scan synchronized
|
||||||
|
for this run (the receive root is the "." sentinel). Sent with the
|
||||||
|
manifest so the receiver confines its extras walk to them, matching rsync's
|
||||||
|
"delete only in synchronized directories" (notably for --files-from).
|
||||||
|
Populated by the scanner thread or the early pre-scan. */
|
||||||
|
ArrayList* synced_dirs;
|
||||||
|
/* Destination-relative paths of every traversed source directory, for the
|
||||||
|
per-directory delete plan keep set (so an empty source directory survives
|
||||||
|
--delete rather than being removed as an extra). Prebuilt by the path-only
|
||||||
|
pre-scan on the calling thread. */
|
||||||
|
ArrayList* plan_dirs;
|
||||||
/* --delete-missing-args: the destination-relative mirrors of the --files-from
|
/* --delete-missing-args: the destination-relative mirrors of the --files-from
|
||||||
entries that are missing under the source. Computed by the preflight on
|
entries that are missing under the source. Computed by the preflight on
|
||||||
the calling thread before the pipeline starts; the sender thread transmits
|
the calling thread before the pipeline starts; the sender thread transmits
|
||||||
@@ -54,15 +73,39 @@ typedef struct {
|
|||||||
--ignore-errors kept the run going. */
|
--ignore-errors kept the run going. */
|
||||||
bool scan_had_io_error;
|
bool scan_had_io_error;
|
||||||
ArrayList* remove_source_files;
|
ArrayList* remove_source_files;
|
||||||
/* True when --delete-before/--delete-during require the keep-set manifest to
|
/* True when --delete-before requires the whole-tree keep-set manifest to be
|
||||||
be transmitted before any file data: context->manifest is then prebuilt by
|
transmitted before any file data: context->manifest is then prebuilt by a
|
||||||
a path-only pre-scan on the calling thread and the pipeline scanner must
|
path-only pre-scan on the calling thread and the pipeline scanner must not
|
||||||
not append to it. Set once before the worker threads start. */
|
append to it. Set once before the worker threads start. */
|
||||||
bool early_delete;
|
bool early_delete;
|
||||||
|
/* --delete-before: the path-only pre-scan that built the early keep-set,
|
||||||
|
retained as the pipeline's file list (owning Chunk*; consumed and NULLed by
|
||||||
|
the scanner thread) so the data pass replays rsync's single file list
|
||||||
|
instead of re-reading the source. NULL in every other mode, where the
|
||||||
|
scanner thread scans normally. Set once before the worker threads start
|
||||||
|
and freed with the context. */
|
||||||
|
ArrayList* prescan_chunks;
|
||||||
|
/* Non-NULL for --delete-during/--delete-delay: the per-directory plan set
|
||||||
|
prebuilt by the path-only pre-scan on the calling thread. The sender
|
||||||
|
thread transmits the root plan before any data and the remaining plans
|
||||||
|
alongside the chunks. Set once before the worker threads start. */
|
||||||
|
DeletePlanSender* delete_plans;
|
||||||
|
/* A scan I/O error without --ignore-errors suppressed deletion: the prebuilt
|
||||||
|
keep-set/plans were dropped, and the streaming scanner must not build a
|
||||||
|
fresh manifest or re-send the per-directory plans. Set once before the
|
||||||
|
worker threads start. */
|
||||||
|
bool delete_suppressed;
|
||||||
mtx_t mutex_progress;
|
mtx_t mutex_progress;
|
||||||
int total_files;
|
int total_files;
|
||||||
unsigned long long progress_bytes;
|
unsigned long long progress_bytes;
|
||||||
unsigned long long total_bytes;
|
unsigned long long total_bytes;
|
||||||
|
/* Per-type flist / transferred accounting for the rsync --stats breakdown and
|
||||||
|
the progress `to-chk` denominator. Owned by the sender thread: it is the
|
||||||
|
only writer (the entry/transfer notes in send_chunks_multithreaded) and it
|
||||||
|
reads the totals in its completion tail, so no lock is needed. This is NOT
|
||||||
|
guarded by mutex_progress (which covers total_files/progress_bytes/
|
||||||
|
total_bytes/sender_done). */
|
||||||
|
TransferStats stats;
|
||||||
bool sender_done;
|
bool sender_done;
|
||||||
atomic_bool cancelled;
|
atomic_bool cancelled;
|
||||||
ProtocolSession allocation_session;
|
ProtocolSession allocation_session;
|
||||||
@@ -83,6 +126,14 @@ typedef struct {
|
|||||||
ArrayList* dir_entries;
|
ArrayList* dir_entries;
|
||||||
mtx_t dir_entries_mutex;
|
mtx_t dir_entries_mutex;
|
||||||
bool dir_entries_mutex_init;
|
bool dir_entries_mutex_init;
|
||||||
|
/* --stats directory accounting for a `-r` scan (no directory metadata):
|
||||||
|
shared by the parallel scanner workers, read by the sender thread once the
|
||||||
|
scanner is done. See ScannerOptions.dir_count. */
|
||||||
|
atomic_ullong dir_count;
|
||||||
|
/* Set by the sender thread when the receiver reported a --max-delete-capped
|
||||||
|
deletion (STATUS_DELETE_LIMIT): the transfer succeeded and the process must
|
||||||
|
exit 25 like rsync. Read by the caller after the sender thread is joined. */
|
||||||
|
bool delete_limit;
|
||||||
} PipelineContextSender;
|
} PipelineContextSender;
|
||||||
|
|
||||||
/* `config` is borrowed and must outlive the context: destroy does NOT free it,
|
/* `config` is borrowed and must outlive the context: destroy does NOT free it,
|
||||||
|
|||||||
+301
-99
@@ -12,8 +12,7 @@
|
|||||||
#include <time.h>
|
#include <time.h>
|
||||||
#include <unistd.h>
|
#include <unistd.h>
|
||||||
|
|
||||||
#define RECEIVE_TIMEOUT_SEC 60 /* 60 second per-message timeout */
|
#define RECEIVE_TIMEOUT_SEC 60 /* built-in fallback for explicit -timed calls only */
|
||||||
#define SEND_TIMEOUT_SEC 60
|
|
||||||
|
|
||||||
static __thread int io_read_fd = -1;
|
static __thread int io_read_fd = -1;
|
||||||
static __thread int io_write_fd = -1;
|
static __thread int io_write_fd = -1;
|
||||||
@@ -30,8 +29,138 @@ static unsigned long long io_bwlimit = 0;
|
|||||||
static mtx_t bw_mutex;
|
static mtx_t bw_mutex;
|
||||||
static once_flag bw_mutex_once = ONCE_FLAG_INIT;
|
static once_flag bw_mutex_once = ONCE_FLAG_INIT;
|
||||||
|
|
||||||
|
/* Process-wide wire byte counters, used by the client to render rsync's
|
||||||
|
* --stats/--progress totals and the --out-format %b/%c tokens. The zero-copy
|
||||||
|
* sendfile path bypasses protocol_send_n_data, so it reports its bytes through
|
||||||
|
* protocol_note_bytes_written. */
|
||||||
|
static atomic_ullong io_bytes_written = 0;
|
||||||
|
static atomic_ullong io_bytes_read = 0;
|
||||||
|
|
||||||
static unsigned long long global_bwlimit(void);
|
static unsigned long long global_bwlimit(void);
|
||||||
|
|
||||||
|
/* ------------------------------------------------------------------------- *
|
||||||
|
* Transport vtable implementations.
|
||||||
|
*
|
||||||
|
* Each op performs exactly one transfer attempt. WANT_READ/WANT_WRITE and an
|
||||||
|
* EINTR-interrupted syscall are reported as PROTOCOL_IO_RETRY (with
|
||||||
|
* *wait_events set to the poll event the caller must wait on); a clean peer
|
||||||
|
* close is PROTOCOL_IO_CLOSED and anything else is PROTOCOL_IO_ERROR. This
|
||||||
|
* keeps every WANT_READ/WANT_WRITE and EINTR retry exactly where it was before
|
||||||
|
* the vtable was introduced, just moved behind the function pointer.
|
||||||
|
* ------------------------------------------------------------------------- */
|
||||||
|
|
||||||
|
static ssize_t plain_io_send(ProtocolSession* session, const void* data, size_t size,
|
||||||
|
short* wait_events) {
|
||||||
|
ssize_t written = write(session->write_fd, data, size);
|
||||||
|
if (written < 0) {
|
||||||
|
if (errno == EINTR)
|
||||||
|
return PROTOCOL_IO_RETRY;
|
||||||
|
return PROTOCOL_IO_ERROR;
|
||||||
|
}
|
||||||
|
if (written == 0)
|
||||||
|
return PROTOCOL_IO_ERROR;
|
||||||
|
*wait_events = POLLOUT;
|
||||||
|
return written;
|
||||||
|
}
|
||||||
|
|
||||||
|
static ssize_t plain_io_recv(ProtocolSession* session, void* data, size_t size,
|
||||||
|
short* wait_events) {
|
||||||
|
ssize_t received = read(session->read_fd, data, size);
|
||||||
|
if (received < 0) {
|
||||||
|
if (errno == EINTR)
|
||||||
|
return PROTOCOL_IO_RETRY;
|
||||||
|
return PROTOCOL_IO_ERROR;
|
||||||
|
}
|
||||||
|
if (received == 0)
|
||||||
|
return PROTOCOL_IO_CLOSED;
|
||||||
|
*wait_events = POLLIN;
|
||||||
|
return received;
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool plain_io_has_pending(const ProtocolSession* session) {
|
||||||
|
(void)session;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
static ssize_t tls_io_send(ProtocolSession* session, const void* data, size_t size,
|
||||||
|
short* wait_events) {
|
||||||
|
/* SSL_write takes an int length; clamp a >INT_MAX request into chunks so the
|
||||||
|
* size_t downcast can never truncate into a negative/partial write. */
|
||||||
|
size_t chunk = size > (size_t)INT_MAX ? (size_t)INT_MAX : size;
|
||||||
|
ssize_t written = SSL_write(session->ssl, data, (int)chunk);
|
||||||
|
if (written <= 0) {
|
||||||
|
int ssl_err = SSL_get_error(session->ssl, (int)written);
|
||||||
|
if (ssl_err == SSL_ERROR_WANT_WRITE) {
|
||||||
|
*wait_events = POLLOUT;
|
||||||
|
return PROTOCOL_IO_RETRY;
|
||||||
|
}
|
||||||
|
if (ssl_err == SSL_ERROR_WANT_READ) {
|
||||||
|
*wait_events = POLLIN;
|
||||||
|
return PROTOCOL_IO_RETRY;
|
||||||
|
}
|
||||||
|
/* A signal (e.g. Ctrl-C) interrupts the blocking TLS write: retry so the
|
||||||
|
* send loop can observe the abort flag at the next checkpoint. Only an
|
||||||
|
* actual negative return is an interrupted syscall; a 0-byte SSL_write is
|
||||||
|
* not a valid EINTR retry. */
|
||||||
|
if (written < 0 && ssl_err == SSL_ERROR_SYSCALL && errno == EINTR)
|
||||||
|
return PROTOCOL_IO_RETRY;
|
||||||
|
return PROTOCOL_IO_ERROR;
|
||||||
|
}
|
||||||
|
*wait_events = POLLOUT;
|
||||||
|
return written;
|
||||||
|
}
|
||||||
|
|
||||||
|
static ssize_t tls_io_recv(ProtocolSession* session, void* data, size_t size, short* wait_events) {
|
||||||
|
/* SSL_read takes an int length; clamp a >INT_MAX request into chunks
|
||||||
|
* (mirrors the send path) so the size_t downcast can never truncate into a
|
||||||
|
* negative/partial read. */
|
||||||
|
size_t chunk = size > (size_t)INT_MAX ? (size_t)INT_MAX : size;
|
||||||
|
ssize_t received = SSL_read(session->ssl, data, (int)chunk);
|
||||||
|
if (received <= 0) {
|
||||||
|
int ssl_err = SSL_get_error(session->ssl, (int)received);
|
||||||
|
if (ssl_err == SSL_ERROR_WANT_WRITE) {
|
||||||
|
*wait_events = POLLOUT;
|
||||||
|
return PROTOCOL_IO_RETRY;
|
||||||
|
}
|
||||||
|
if (ssl_err == SSL_ERROR_WANT_READ) {
|
||||||
|
*wait_events = POLLIN;
|
||||||
|
return PROTOCOL_IO_RETRY;
|
||||||
|
}
|
||||||
|
/* A signal interrupts the blocking TLS read: retry (mirrors the send path)
|
||||||
|
* so the loop reaches its next abort/deadline checkpoint. Only an actual
|
||||||
|
* negative return is an interrupted syscall: a 0-byte SSL_read is an
|
||||||
|
* unexpected EOF (the peer closed without close_notify), which OpenSSL also
|
||||||
|
* reports as SSL_ERROR_SYSCALL with errno possibly still EINTR from an
|
||||||
|
* earlier interrupted poll/read. Retrying that would busy-spin the
|
||||||
|
* status-read loop until its deadline, so classify it as closed instead. */
|
||||||
|
if (received < 0 && ssl_err == SSL_ERROR_SYSCALL && errno == EINTR)
|
||||||
|
return PROTOCOL_IO_RETRY;
|
||||||
|
/* A zero-length SSL_read is the peer's clean close_notify (or EOF without
|
||||||
|
* one); report it distinctly so the caller can log it as a close. */
|
||||||
|
if (received == 0)
|
||||||
|
return PROTOCOL_IO_CLOSED;
|
||||||
|
return PROTOCOL_IO_ERROR;
|
||||||
|
}
|
||||||
|
*wait_events = POLLIN;
|
||||||
|
return received;
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool tls_io_has_pending(const ProtocolSession* session) {
|
||||||
|
return session->ssl != NULL && SSL_pending(session->ssl) > 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
static const ProtocolIoOps plain_io_ops = {
|
||||||
|
.send = plain_io_send,
|
||||||
|
.recv = plain_io_recv,
|
||||||
|
.has_pending = plain_io_has_pending,
|
||||||
|
};
|
||||||
|
|
||||||
|
static const ProtocolIoOps tls_io_ops = {
|
||||||
|
.send = tls_io_send,
|
||||||
|
.recv = tls_io_recv,
|
||||||
|
.has_pending = tls_io_has_pending,
|
||||||
|
};
|
||||||
|
|
||||||
static bool protocol_reserve_memory(ProtocolSession* session, size_t charge) {
|
static bool protocol_reserve_memory(ProtocolSession* session, size_t charge) {
|
||||||
unsigned long long allocated = atomic_load(&session->total_allocated_bytes);
|
unsigned long long allocated = atomic_load(&session->total_allocated_bytes);
|
||||||
while (true) {
|
while (true) {
|
||||||
@@ -73,6 +202,7 @@ void io_set_fds(int read_fd, int write_fd) {
|
|||||||
legacy_io_session.read_fd = read_fd;
|
legacy_io_session.read_fd = read_fd;
|
||||||
legacy_io_session.write_fd = write_fd;
|
legacy_io_session.write_fd = write_fd;
|
||||||
legacy_io_session.ssl = NULL;
|
legacy_io_session.ssl = NULL;
|
||||||
|
legacy_io_session.ops = &plain_io_ops;
|
||||||
legacy_io_session.eight_bit_output = false;
|
legacy_io_session.eight_bit_output = false;
|
||||||
atomic_store(&legacy_io_session.total_allocated_bytes, 0);
|
atomic_store(&legacy_io_session.total_allocated_bytes, 0);
|
||||||
legacy_io_session.max_alloc = DEFAULT_MAX_ALLOC;
|
legacy_io_session.max_alloc = DEFAULT_MAX_ALLOC;
|
||||||
@@ -85,6 +215,7 @@ void protocol_session_init(ProtocolSession* session, int read_fd, int write_fd)
|
|||||||
memset(session, 0, sizeof(*session));
|
memset(session, 0, sizeof(*session));
|
||||||
session->read_fd = read_fd;
|
session->read_fd = read_fd;
|
||||||
session->write_fd = write_fd;
|
session->write_fd = write_fd;
|
||||||
|
session->ops = &plain_io_ops;
|
||||||
session->max_alloc = DEFAULT_MAX_ALLOC;
|
session->max_alloc = DEFAULT_MAX_ALLOC;
|
||||||
session->io_timeout_sec = RECEIVE_TIMEOUT_SEC;
|
session->io_timeout_sec = RECEIVE_TIMEOUT_SEC;
|
||||||
atomic_init(&session->total_allocated_bytes, 0);
|
atomic_init(&session->total_allocated_bytes, 0);
|
||||||
@@ -99,8 +230,14 @@ void protocol_session_set_io_timeout(ProtocolSession* session, int sec) {
|
|||||||
|
|
||||||
int protocol_get_io_timeout_sec(void) {
|
int protocol_get_io_timeout_sec(void) {
|
||||||
const ProtocolSession* session = bound_session ? bound_session : &legacy_io_session;
|
const ProtocolSession* session = bound_session ? bound_session : &legacy_io_session;
|
||||||
int sec = session->io_timeout_sec;
|
/* 0 (or negative) means the session timeout is disabled, matching rsync's
|
||||||
return sec > 0 ? sec : RECEIVE_TIMEOUT_SEC;
|
* --timeout=0 default. Callers must treat a non-positive result as "wait
|
||||||
|
* without a deadline" instead of substituting a built-in window. */
|
||||||
|
return session->io_timeout_sec > 0 ? session->io_timeout_sec : 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
int protocol_server_io_timeout_sec(int client_timeout) {
|
||||||
|
return client_timeout > 0 ? client_timeout : SERVER_IO_TIMEOUT_SEC;
|
||||||
}
|
}
|
||||||
|
|
||||||
void protocol_session_set_max_alloc(ProtocolSession* session, unsigned long long max_alloc) {
|
void protocol_session_set_max_alloc(ProtocolSession* session, unsigned long long max_alloc) {
|
||||||
@@ -110,7 +247,8 @@ void protocol_session_set_max_alloc(ProtocolSession* session, unsigned long long
|
|||||||
}
|
}
|
||||||
|
|
||||||
static bool allocation_allowed(const ProtocolSession* session, size_t size) {
|
static bool allocation_allowed(const ProtocolSession* session, size_t size) {
|
||||||
return (unsigned long long)size <= session->max_alloc;
|
/* max_alloc == 0 is rsync's --max-alloc=0 "no limit". */
|
||||||
|
return session->max_alloc == 0 || (unsigned long long)size <= session->max_alloc;
|
||||||
}
|
}
|
||||||
|
|
||||||
static void* protocol_alloc_for_session(const ProtocolSession* session, size_t size) {
|
static void* protocol_alloc_for_session(const ProtocolSession* session, size_t size) {
|
||||||
@@ -145,8 +283,12 @@ void protocol_session_unbind(void) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
void protocol_session_set_ssl(ProtocolSession* session, SSL* ssl) {
|
void protocol_session_set_ssl(ProtocolSession* session, SSL* ssl) {
|
||||||
if (session)
|
if (!session)
|
||||||
session->ssl = ssl;
|
return;
|
||||||
|
session->ssl = ssl;
|
||||||
|
/* Select the transport dispatch once, here, instead of branching on the SSL
|
||||||
|
* pointer inside every I/O loop. */
|
||||||
|
session->ops = ssl ? &tls_io_ops : &plain_io_ops;
|
||||||
}
|
}
|
||||||
|
|
||||||
static void bw_mutex_init(void) {
|
static void bw_mutex_init(void) {
|
||||||
@@ -170,12 +312,28 @@ void io_set_bwlimit(unsigned long long bytes_per_sec) {
|
|||||||
mtx_unlock(&bw_mutex);
|
mtx_unlock(&bw_mutex);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
unsigned long long io_get_bwlimit(void) {
|
||||||
|
return global_bwlimit();
|
||||||
|
}
|
||||||
|
|
||||||
|
/* rsync's throttle (io.c sleep_for_bwlimit) sleeps once its unslept debt
|
||||||
|
* reaches ~100 ms of bandwidth, so its effective initial burst is about 0.1 s
|
||||||
|
* worth of bytes, not a full second. FastSync models the same with a token
|
||||||
|
* bucket whose capacity is bwlimit/10, so a throttled run paces like rsync
|
||||||
|
* instead of sending a full second's worth up front. */
|
||||||
|
static long long bw_burst_capacity(unsigned long long bwlimit) {
|
||||||
|
if (bwlimit == 0)
|
||||||
|
return 0;
|
||||||
|
long long burst = (long long)(bwlimit / 10);
|
||||||
|
return burst > 0 ? burst : 1;
|
||||||
|
}
|
||||||
|
|
||||||
void protocol_session_set_bwlimit(ProtocolSession* session, unsigned long long bytes_per_sec) {
|
void protocol_session_set_bwlimit(ProtocolSession* session, unsigned long long bytes_per_sec) {
|
||||||
if (!session)
|
if (!session)
|
||||||
return;
|
return;
|
||||||
session->bwlimit =
|
session->bwlimit =
|
||||||
bytes_per_sec > (unsigned long long)LLONG_MAX ? (unsigned long long)LLONG_MAX : bytes_per_sec;
|
bytes_per_sec > (unsigned long long)LLONG_MAX ? (unsigned long long)LLONG_MAX : bytes_per_sec;
|
||||||
session->bw_tokens = (long long)session->bwlimit;
|
session->bw_tokens = bw_burst_capacity(session->bwlimit);
|
||||||
struct timespec now;
|
struct timespec now;
|
||||||
clock_gettime(CLOCK_MONOTONIC, &now);
|
clock_gettime(CLOCK_MONOTONIC, &now);
|
||||||
session->bw_last_refill_sec = now.tv_sec;
|
session->bw_last_refill_sec = now.tv_sec;
|
||||||
@@ -209,8 +367,9 @@ static void bw_throttle_session(ProtocolSession* session, size_t bytes_written)
|
|||||||
|
|
||||||
long long tokens_to_add = (long long)((double)session->bwlimit * elapsed_ns / 1000000000.0);
|
long long tokens_to_add = (long long)((double)session->bwlimit * elapsed_ns / 1000000000.0);
|
||||||
session->bw_tokens += tokens_to_add;
|
session->bw_tokens += tokens_to_add;
|
||||||
if (session->bw_tokens > (long long)session->bwlimit)
|
long long burst = bw_burst_capacity(session->bwlimit);
|
||||||
session->bw_tokens = (long long)session->bwlimit;
|
if (session->bw_tokens > burst)
|
||||||
|
session->bw_tokens = burst;
|
||||||
|
|
||||||
session->bw_tokens -= bytes_written;
|
session->bw_tokens -= bytes_written;
|
||||||
|
|
||||||
@@ -221,7 +380,11 @@ static void bw_throttle_session(ProtocolSession* session, size_t bytes_written)
|
|||||||
poll(NULL, 0, (int)(deficit_us / 1000));
|
poll(NULL, 0, (int)(deficit_us / 1000));
|
||||||
else
|
else
|
||||||
usleep((useconds_t)deficit_us);
|
usleep((useconds_t)deficit_us);
|
||||||
|
/* Reset the bucket AFTER the sleep: crediting the sleep duration as elapsed
|
||||||
|
refill time would cancel half the throttle (the next call would see the
|
||||||
|
whole sleep as refill and immediately grant a fresh burst). */
|
||||||
session->bw_tokens = 0;
|
session->bw_tokens = 0;
|
||||||
|
clock_gettime(CLOCK_MONOTONIC, &now);
|
||||||
session->bw_last_refill_sec = now.tv_sec;
|
session->bw_last_refill_sec = now.tv_sec;
|
||||||
session->bw_last_refill_nsec = now.tv_nsec;
|
session->bw_last_refill_nsec = now.tv_nsec;
|
||||||
}
|
}
|
||||||
@@ -236,6 +399,32 @@ SSL* io_get_ssl(void) {
|
|||||||
return io_ssl;
|
return io_ssl;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
SSL* protocol_current_ssl(void) {
|
||||||
|
/* The bound session is the authoritative transport for a worker thread: it
|
||||||
|
* was explicitly handed to protocol_session_bind() and carries its own SSL,
|
||||||
|
* whereas io_ssl is thread-local and NULL in a thread that never performed
|
||||||
|
* the handshake. Only a session whose selected dispatch is TLS may supply
|
||||||
|
* the SSL: a bound plaintext session has ssl == NULL and must not shadow a
|
||||||
|
* live thread-local io_ssl, or file_send.c would take the raw sendfile(2)
|
||||||
|
* path on a socket this thread is driving with TLS. With no TLS session
|
||||||
|
* bound (plaintext session, or the fd-shim path), fall back to io_ssl. */
|
||||||
|
if (bound_session && bound_session->ops == &tls_io_ops && bound_session->ssl)
|
||||||
|
return bound_session->ssl;
|
||||||
|
return io_ssl;
|
||||||
|
}
|
||||||
|
|
||||||
|
unsigned long long protocol_bytes_written(void) {
|
||||||
|
return atomic_load(&io_bytes_written);
|
||||||
|
}
|
||||||
|
|
||||||
|
unsigned long long protocol_bytes_read(void) {
|
||||||
|
return atomic_load(&io_bytes_read);
|
||||||
|
}
|
||||||
|
|
||||||
|
void protocol_note_bytes_written(unsigned long long bytes) {
|
||||||
|
atomic_fetch_add(&io_bytes_written, bytes);
|
||||||
|
}
|
||||||
|
|
||||||
static ProtocolSession* legacy_session(int read_fd, int write_fd) {
|
static ProtocolSession* legacy_session(int read_fd, int write_fd) {
|
||||||
if (bound_session)
|
if (bound_session)
|
||||||
return bound_session;
|
return bound_session;
|
||||||
@@ -252,9 +441,21 @@ static ProtocolSession* legacy_session(int read_fd, int write_fd) {
|
|||||||
protocol_session_set_bwlimit(&legacy_io_session, global_bwlimit());
|
protocol_session_set_bwlimit(&legacy_io_session, global_bwlimit());
|
||||||
}
|
}
|
||||||
legacy_io_session.ssl = io_ssl;
|
legacy_io_session.ssl = io_ssl;
|
||||||
|
legacy_io_session.ops = io_ssl ? &tls_io_ops : &plain_io_ops;
|
||||||
return &legacy_io_session;
|
return &legacy_io_session;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Pace an out-of-band write that bypassed protocol_send_n_data (the plaintext
|
||||||
|
* sendfile fast path). The bound/legacy session is resolved exactly as the
|
||||||
|
* preceding send_n_data(fd, ...) resolved it, so the same token-bucket state is
|
||||||
|
* throttled and the TLS and plaintext transports share identical --bwlimit
|
||||||
|
* semantics. Passing the wire fd (rather than -1) is essential: the sendfile
|
||||||
|
* send left legacy_io_session.write_fd bound to it, so resolving with -1 would
|
||||||
|
* mismatch, re-initialize the session and hand out a second first-call burst. */
|
||||||
|
void protocol_throttle_bytes(int file_descriptor, size_t bytes) {
|
||||||
|
bw_throttle_session(legacy_session(-1, file_descriptor), bytes);
|
||||||
|
}
|
||||||
|
|
||||||
bool send_n_data(int file_descriptor, const void* data, size_t data_size) {
|
bool send_n_data(int file_descriptor, const void* data, size_t data_size) {
|
||||||
return protocol_send_n_data(legacy_session(-1, file_descriptor), data, data_size);
|
return protocol_send_n_data(legacy_session(-1, file_descriptor), data, data_size);
|
||||||
}
|
}
|
||||||
@@ -278,13 +479,18 @@ bool protocol_send_n_data(ProtocolSession* session, const void* data, size_t dat
|
|||||||
if (!data && data_size != 0)
|
if (!data && data_size != 0)
|
||||||
return false;
|
return false;
|
||||||
log_debug_message(LOG_DEBUG_IO, " Sending n Data: %zu", data_size);
|
log_debug_message(LOG_DEBUG_IO, " Sending n Data: %zu", data_size);
|
||||||
if (!session)
|
if (!session || !session->ops)
|
||||||
return false;
|
return false;
|
||||||
int timeout_sec = session->io_timeout_sec > 0 ? session->io_timeout_sec : SEND_TIMEOUT_SEC;
|
const ProtocolIoOps* ops = session->ops;
|
||||||
|
/* A non-positive session timeout disables the deadline entirely (rsync's
|
||||||
|
* --timeout=0 default); poll then blocks until the socket becomes writable. */
|
||||||
|
int timeout_sec = session->io_timeout_sec > 0 ? session->io_timeout_sec : 0;
|
||||||
int fd = session->write_fd;
|
int fd = session->write_fd;
|
||||||
struct timespec deadline;
|
struct timespec deadline;
|
||||||
clock_gettime(CLOCK_MONOTONIC, &deadline);
|
if (timeout_sec > 0) {
|
||||||
deadline.tv_sec += timeout_sec;
|
clock_gettime(CLOCK_MONOTONIC, &deadline);
|
||||||
|
deadline.tv_sec += timeout_sec;
|
||||||
|
}
|
||||||
short wait_events = POLLOUT;
|
short wait_events = POLLOUT;
|
||||||
ssize_t total_bytes_send = 0;
|
ssize_t total_bytes_send = 0;
|
||||||
while ((size_t)total_bytes_send < data_size) {
|
while ((size_t)total_bytes_send < data_size) {
|
||||||
@@ -292,7 +498,7 @@ bool protocol_send_n_data(ProtocolSession* session, const void* data, size_t dat
|
|||||||
if (session->bwlimit > 0 && chunk > 65536)
|
if (session->bwlimit > 0 && chunk > 65536)
|
||||||
chunk = 65536;
|
chunk = 65536;
|
||||||
struct pollfd pfd = {.fd = fd, .events = wait_events};
|
struct pollfd pfd = {.fd = fd, .events = wait_events};
|
||||||
int poll_result = poll(&pfd, 1, deadline_remaining_ms(&deadline));
|
int poll_result = poll(&pfd, 1, timeout_sec > 0 ? deadline_remaining_ms(&deadline) : -1);
|
||||||
if (poll_result == 0 || (poll_result < 0 && errno != EINTR)) {
|
if (poll_result == 0 || (poll_result < 0 && errno != EINTR)) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Send timeout or poll failure");
|
log_message(LOG_LEVEL_ERROR, "Send timeout or poll failure");
|
||||||
return false;
|
return false;
|
||||||
@@ -301,49 +507,39 @@ bool protocol_send_n_data(ProtocolSession* session, const void* data, size_t dat
|
|||||||
continue;
|
continue;
|
||||||
if (pfd.revents & (POLLERR | POLLNVAL))
|
if (pfd.revents & (POLLERR | POLLNVAL))
|
||||||
return false;
|
return false;
|
||||||
ssize_t bytes_send;
|
ssize_t bytes_send =
|
||||||
if (session->ssl) {
|
ops->send(session, (const char*)data + total_bytes_send, chunk, &wait_events);
|
||||||
/* SSL_write takes an int length; clamp a >INT_MAX request into chunks so
|
if (bytes_send == PROTOCOL_IO_RETRY)
|
||||||
* the size_t downcast can never truncate into a negative/partial write. */
|
continue;
|
||||||
size_t ssl_chunk = chunk > (size_t)INT_MAX ? (size_t)INT_MAX : chunk;
|
|
||||||
bytes_send = SSL_write(session->ssl, (const char*)data + total_bytes_send, (int)ssl_chunk);
|
|
||||||
} else {
|
|
||||||
bytes_send = write(fd, (const char*)data + total_bytes_send, chunk);
|
|
||||||
}
|
|
||||||
if (bytes_send <= 0) {
|
if (bytes_send <= 0) {
|
||||||
if (session->ssl) {
|
|
||||||
int ssl_err = SSL_get_error(session->ssl, (int)bytes_send);
|
|
||||||
if (ssl_err == SSL_ERROR_WANT_WRITE || ssl_err == SSL_ERROR_WANT_READ) {
|
|
||||||
wait_events = ssl_err == SSL_ERROR_WANT_WRITE ? POLLOUT : POLLIN;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
/* A signal (e.g. Ctrl-C) interrupts the blocking TLS write: retry so
|
|
||||||
the send loop can observe the abort flag at the next checkpoint. */
|
|
||||||
if (ssl_err == SSL_ERROR_SYSCALL && errno == EINTR)
|
|
||||||
continue;
|
|
||||||
} else if (errno == EINTR) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
log_message(LOG_LEVEL_ERROR, "Could not send data");
|
log_message(LOG_LEVEL_ERROR, "Could not send data");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
bw_throttle_session(session, (size_t)bytes_send);
|
bw_throttle_session(session, (size_t)bytes_send);
|
||||||
total_bytes_send += bytes_send;
|
total_bytes_send += bytes_send;
|
||||||
if (session->ssl)
|
|
||||||
wait_events = POLLOUT;
|
|
||||||
}
|
}
|
||||||
log_debug_message(LOG_DEBUG_IO, " Send n Data: %zu", total_bytes_send);
|
log_debug_message(LOG_DEBUG_IO, " Send n Data: %zd", total_bytes_send);
|
||||||
|
atomic_fetch_add(&io_bytes_written, (unsigned long long)total_bytes_send);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool protocol_receive_n_data_timed(ProtocolSession* session, void* data, size_t data_size,
|
bool protocol_receive_n_data_timed(ProtocolSession* session, void* data, size_t data_size,
|
||||||
int timeout_sec);
|
int timeout_sec);
|
||||||
|
static bool protocol_receive_n_data_until(ProtocolSession* session, void* data, size_t data_size,
|
||||||
|
const struct timespec* deadline);
|
||||||
|
|
||||||
bool protocol_receive_n_data(ProtocolSession* session, void* data, size_t data_size) {
|
bool protocol_receive_n_data(ProtocolSession* session, void* data, size_t data_size) {
|
||||||
/* Honor the session's configured deadline; protocol_receive_n_data_timed
|
/* Honor the session's configured deadline. A non-positive value disables the
|
||||||
* re-applies the built-in 60 s default when the value is <= 0. */
|
* deadline (rsync's --timeout=0 default): wait without a poll timeout. The
|
||||||
int timeout_sec = session ? session->io_timeout_sec : 0;
|
* explicit _timed variants keep their own 0 -> built-in-default contract. */
|
||||||
return protocol_receive_n_data_timed(session, data, data_size, timeout_sec);
|
if (!session)
|
||||||
|
return false;
|
||||||
|
if (session->io_timeout_sec <= 0)
|
||||||
|
return protocol_receive_n_data_until(session, data, data_size, NULL);
|
||||||
|
struct timespec deadline;
|
||||||
|
clock_gettime(CLOCK_MONOTONIC, &deadline);
|
||||||
|
deadline.tv_sec += session->io_timeout_sec;
|
||||||
|
return protocol_receive_n_data_until(session, data, data_size, &deadline);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Read exactly `data_size` bytes from `session` before `deadline` elapses
|
/* Read exactly `data_size` bytes from `session` before `deadline` elapses
|
||||||
@@ -353,16 +549,18 @@ bool protocol_receive_n_data(ProtocolSession* session, void* data, size_t data_s
|
|||||||
static bool protocol_receive_n_data_until(ProtocolSession* session, void* data, size_t data_size,
|
static bool protocol_receive_n_data_until(ProtocolSession* session, void* data, size_t data_size,
|
||||||
const struct timespec* deadline) {
|
const struct timespec* deadline) {
|
||||||
log_debug_message(LOG_DEBUG_IO, " Receiving n Data: %zu", data_size);
|
log_debug_message(LOG_DEBUG_IO, " Receiving n Data: %zu", data_size);
|
||||||
if (!session || !deadline)
|
if (!session || !session->ops)
|
||||||
return false;
|
return false;
|
||||||
|
const ProtocolIoOps* ops = session->ops;
|
||||||
int fd = session->read_fd;
|
int fd = session->read_fd;
|
||||||
|
|
||||||
size_t total_bytes_received = 0;
|
size_t total_bytes_received = 0;
|
||||||
short wait_events = POLLIN;
|
short wait_events = POLLIN;
|
||||||
while (total_bytes_received < data_size) {
|
while (total_bytes_received < data_size) {
|
||||||
if (!session->ssl || SSL_pending(session->ssl) == 0) {
|
if (!ops->has_pending(session)) {
|
||||||
struct pollfd pfd = {.fd = fd, .events = wait_events};
|
struct pollfd pfd = {.fd = fd, .events = wait_events};
|
||||||
int poll_result = poll(&pfd, 1, deadline_remaining_ms(deadline));
|
/* A NULL deadline means "wait indefinitely" (timeout disabled). */
|
||||||
|
int poll_result = poll(&pfd, 1, deadline ? deadline_remaining_ms(deadline) : -1);
|
||||||
if (poll_result == 0) {
|
if (poll_result == 0) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Receive timeout");
|
log_message(LOG_LEVEL_ERROR, "Receive timeout");
|
||||||
return false;
|
return false;
|
||||||
@@ -377,39 +575,22 @@ static bool protocol_receive_n_data_until(ProtocolSession* session, void* data,
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
ssize_t bytes_received;
|
ssize_t bytes_received = ops->recv(session, (char*)data + total_bytes_received,
|
||||||
if (session->ssl)
|
data_size - total_bytes_received, &wait_events);
|
||||||
bytes_received = SSL_read(session->ssl, (char*)data + total_bytes_received,
|
if (bytes_received == PROTOCOL_IO_RETRY)
|
||||||
data_size - total_bytes_received);
|
continue;
|
||||||
else
|
if (bytes_received == PROTOCOL_IO_CLOSED) {
|
||||||
bytes_received =
|
log_message(LOG_LEVEL_ERROR, "Connection closed while receiving data");
|
||||||
read(fd, (char*)data + total_bytes_received, data_size - total_bytes_received);
|
return false;
|
||||||
|
}
|
||||||
if (bytes_received <= 0) {
|
if (bytes_received <= 0) {
|
||||||
if (session->ssl) {
|
log_message(LOG_LEVEL_ERROR, "Could not receive bytes");
|
||||||
int ssl_err = SSL_get_error(session->ssl, (int)bytes_received);
|
|
||||||
if (ssl_err == SSL_ERROR_WANT_WRITE || ssl_err == SSL_ERROR_WANT_READ) {
|
|
||||||
wait_events = ssl_err == SSL_ERROR_WANT_WRITE ? POLLOUT : POLLIN;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
/* A signal interrupts the blocking TLS read: retry (mirrors the send
|
|
||||||
path and protocol_read_status_until) so the loop reaches its next
|
|
||||||
abort/deadline checkpoint instead of failing spuriously. */
|
|
||||||
if (ssl_err == SSL_ERROR_SYSCALL && errno == EINTR)
|
|
||||||
continue;
|
|
||||||
} else if (errno == EINTR) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (bytes_received == 0)
|
|
||||||
log_message(LOG_LEVEL_ERROR, "Connection closed while receiving data");
|
|
||||||
else
|
|
||||||
log_message(LOG_LEVEL_ERROR, "Could not receive bytes");
|
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
total_bytes_received += (size_t)bytes_received;
|
total_bytes_received += (size_t)bytes_received;
|
||||||
if (session->ssl)
|
|
||||||
wait_events = POLLIN;
|
|
||||||
}
|
}
|
||||||
log_debug_message(LOG_DEBUG_IO, " Received n Data: %zu", total_bytes_received);
|
log_debug_message(LOG_DEBUG_IO, " Received n Data: %zu", total_bytes_received);
|
||||||
|
atomic_fetch_add(&io_bytes_read, (unsigned long long)total_bytes_received);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -479,11 +660,24 @@ static const char* status_to_string(Status status) {
|
|||||||
return "ERROR_DETAIL";
|
return "ERROR_DETAIL";
|
||||||
case STATUS_DRY_RUN_TRANSFER:
|
case STATUS_DRY_RUN_TRANSFER:
|
||||||
return "DRY_RUN_TRANSFER";
|
return "DRY_RUN_TRANSFER";
|
||||||
|
case STATUS_DELETE_LIMIT:
|
||||||
|
return "DELETE_LIMIT";
|
||||||
|
case STATUS_DEST_INFO:
|
||||||
|
return "DEST_INFO";
|
||||||
default:
|
default:
|
||||||
return "UNKNOWN";
|
return "UNKNOWN";
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Reject a raw wire status outside the known enum range before it is handed to
|
||||||
|
* callers, so an unknown/corrupt frame fails as a protocol error instead of
|
||||||
|
* being silently interpreted as an unexpected-but-valid verdict. STATUS_OK is
|
||||||
|
* the first enumerator and STATUS_STATS the last, so the range check accepts
|
||||||
|
* every status the protocol defines. */
|
||||||
|
static bool status_is_valid(Status status) {
|
||||||
|
return status >= STATUS_OK && status <= STATUS_STATS;
|
||||||
|
}
|
||||||
|
|
||||||
/* Shared string send/receive implementation. `redact` selects whether the
|
/* Shared string send/receive implementation. `redact` selects whether the
|
||||||
* payload body is written to the LOG_DEBUG_PROTO debug log: daemon auth material
|
* payload body is written to the LOG_DEBUG_PROTO debug log: daemon auth material
|
||||||
* (the username and the proof/signature fields) sets it so a --verbose log never
|
* (the username and the proof/signature fields) sets it so a --verbose log never
|
||||||
@@ -567,7 +761,7 @@ bool protocol_send_data(ProtocolSession* session, const Data* data) {
|
|||||||
return false;
|
return false;
|
||||||
if (!protocol_send_n_data(session, data->data, data_size))
|
if (!protocol_send_n_data(session, data->data, data_size))
|
||||||
return false;
|
return false;
|
||||||
log_debug_message(LOG_DEBUG_PROTO, "Send %lld data", data_size);
|
log_debug_message(LOG_DEBUG_PROTO, "Send %llu data", data_size);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -601,7 +795,7 @@ Data* protocol_receive_data_limited(ProtocolSession* session, unsigned long long
|
|||||||
protocol_release_memory_for_session(session, allocation_size);
|
protocol_release_memory_for_session(session, allocation_size);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
log_debug_message(LOG_DEBUG_PROTO, "Received %lld data", size);
|
log_debug_message(LOG_DEBUG_PROTO, "Received %llu data", size);
|
||||||
Data* result = data_create(data, (size_t)size);
|
Data* result = data_create(data, (size_t)size);
|
||||||
if (!result) {
|
if (!result) {
|
||||||
protocol_release_memory_for_session(session, allocation_size);
|
protocol_release_memory_for_session(session, allocation_size);
|
||||||
@@ -702,13 +896,20 @@ static bool protocol_capture_error_detail(ProtocolSession* session, Status* stat
|
|||||||
bool protocol_receive_status(ProtocolSession* session, Status* status) {
|
bool protocol_receive_status(ProtocolSession* session, Status* status) {
|
||||||
if (!session || !status)
|
if (!session || !status)
|
||||||
return false;
|
return false;
|
||||||
int timeout_sec = session->io_timeout_sec > 0 ? session->io_timeout_sec : RECEIVE_TIMEOUT_SEC;
|
|
||||||
struct timespec deadline;
|
struct timespec deadline;
|
||||||
clock_gettime(CLOCK_MONOTONIC, &deadline);
|
const struct timespec* deadline_ptr = NULL;
|
||||||
deadline.tv_sec += timeout_sec;
|
if (session->io_timeout_sec > 0) {
|
||||||
if (!protocol_receive_n_data_until(session, status, sizeof(Status), &deadline))
|
clock_gettime(CLOCK_MONOTONIC, &deadline);
|
||||||
|
deadline.tv_sec += session->io_timeout_sec;
|
||||||
|
deadline_ptr = &deadline;
|
||||||
|
}
|
||||||
|
if (!protocol_receive_n_data_until(session, status, sizeof(Status), deadline_ptr))
|
||||||
return false;
|
return false;
|
||||||
if (!protocol_capture_error_detail(session, status, &deadline, NULL))
|
if (!status_is_valid(*status)) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Received unknown protocol status %d", *status);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (!protocol_capture_error_detail(session, status, deadline_ptr, NULL))
|
||||||
return false;
|
return false;
|
||||||
log_debug_message(LOG_DEBUG_PROTO, "Received Status: %s", status_to_string(*status));
|
log_debug_message(LOG_DEBUG_PROTO, "Received Status: %s", status_to_string(*status));
|
||||||
return true;
|
return true;
|
||||||
@@ -729,6 +930,10 @@ bool protocol_receive_status_timed(ProtocolSession* session, Status* status, int
|
|||||||
deadline.tv_sec += timeout_sec;
|
deadline.tv_sec += timeout_sec;
|
||||||
if (!protocol_receive_n_data_until(session, status, sizeof(Status), &deadline))
|
if (!protocol_receive_n_data_until(session, status, sizeof(Status), &deadline))
|
||||||
return false;
|
return false;
|
||||||
|
if (!status_is_valid(*status)) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Received unknown protocol status %d", *status);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
if (!protocol_capture_error_detail(session, status, &deadline, NULL))
|
if (!protocol_capture_error_detail(session, status, &deadline, NULL))
|
||||||
return false;
|
return false;
|
||||||
log_debug_message(LOG_DEBUG_PROTO, "Received Status: %s", status_to_string(*status));
|
log_debug_message(LOG_DEBUG_PROTO, "Received Status: %s", status_to_string(*status));
|
||||||
@@ -742,13 +947,16 @@ bool protocol_receive_status_timed(ProtocolSession* session, Status* status, int
|
|||||||
* reply across a frame boundary. Returns false on timeout/EOF/error. */
|
* reply across a frame boundary. Returns false on timeout/EOF/error. */
|
||||||
static bool protocol_read_status_until(ProtocolSession* session, Status* status,
|
static bool protocol_read_status_until(ProtocolSession* session, Status* status,
|
||||||
const struct timespec* deadline) {
|
const struct timespec* deadline) {
|
||||||
|
if (!session || !session->ops)
|
||||||
|
return false;
|
||||||
|
const ProtocolIoOps* ops = session->ops;
|
||||||
Status received = STATUS_ERROR;
|
Status received = STATUS_ERROR;
|
||||||
size_t got = 0;
|
size_t got = 0;
|
||||||
short wait_events = POLLIN;
|
short wait_events = POLLIN;
|
||||||
while (got < sizeof(Status)) {
|
while (got < sizeof(Status)) {
|
||||||
if (!session->ssl || SSL_pending(session->ssl) == 0) {
|
if (!ops->has_pending(session)) {
|
||||||
int remaining_ms = deadline_remaining_ms(deadline);
|
int remaining_ms = deadline ? deadline_remaining_ms(deadline) : -1;
|
||||||
if (remaining_ms <= 0) {
|
if (remaining_ms == 0) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Receive timeout while reading status");
|
log_message(LOG_LEVEL_ERROR, "Receive timeout while reading status");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
@@ -766,21 +974,11 @@ static bool protocol_read_status_until(ProtocolSession* session, Status* status,
|
|||||||
if (pfd.revents & (POLLERR | POLLNVAL))
|
if (pfd.revents & (POLLERR | POLLNVAL))
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
ssize_t bytes_received;
|
ssize_t bytes_received =
|
||||||
if (session->ssl)
|
ops->recv(session, (char*)&received + got, sizeof(Status) - got, &wait_events);
|
||||||
bytes_received = SSL_read(session->ssl, (char*)&received + got, sizeof(Status) - got);
|
if (bytes_received == PROTOCOL_IO_RETRY)
|
||||||
else
|
continue;
|
||||||
bytes_received = read(session->read_fd, (char*)&received + got, sizeof(Status) - got);
|
|
||||||
if (bytes_received <= 0) {
|
if (bytes_received <= 0) {
|
||||||
if (session->ssl) {
|
|
||||||
int ssl_err = SSL_get_error(session->ssl, (int)bytes_received);
|
|
||||||
if (ssl_err == SSL_ERROR_WANT_READ || ssl_err == SSL_ERROR_WANT_WRITE) {
|
|
||||||
wait_events = ssl_err == SSL_ERROR_WANT_WRITE ? POLLOUT : POLLIN;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (bytes_received < 0 && errno == EINTR)
|
|
||||||
continue;
|
|
||||||
log_message(LOG_LEVEL_ERROR, "Connection closed while receiving status");
|
log_message(LOG_LEVEL_ERROR, "Connection closed while receiving status");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
@@ -809,7 +1007,7 @@ bool protocol_receive_status_keepalive(ProtocolSession* session, Status* status,
|
|||||||
while (true) {
|
while (true) {
|
||||||
if (abort_check && abort_check())
|
if (abort_check && abort_check())
|
||||||
return false;
|
return false;
|
||||||
if (!session->ssl || SSL_pending(session->ssl) == 0) {
|
if (!session->ops || !session->ops->has_pending(session)) {
|
||||||
int remaining_ms = deadline_remaining_ms(&deadline);
|
int remaining_ms = deadline_remaining_ms(&deadline);
|
||||||
if (remaining_ms <= 0) {
|
if (remaining_ms <= 0) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Receive timeout after %ds", timeout_sec);
|
log_message(LOG_LEVEL_ERROR, "Receive timeout after %ds", timeout_sec);
|
||||||
@@ -842,6 +1040,10 @@ bool protocol_receive_status_keepalive(ProtocolSession* session, Status* status,
|
|||||||
Status received;
|
Status received;
|
||||||
if (!protocol_read_status_until(session, &received, &deadline))
|
if (!protocol_read_status_until(session, &received, &deadline))
|
||||||
return false;
|
return false;
|
||||||
|
if (!status_is_valid(received)) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Received unknown protocol status %d", received);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
if (!protocol_capture_error_detail(session, &received, &deadline, abort_check))
|
if (!protocol_capture_error_detail(session, &received, &deadline, abort_check))
|
||||||
return false;
|
return false;
|
||||||
if (received == STATUS_KEEPALIVE) {
|
if (received == STATUS_KEEPALIVE) {
|
||||||
|
|||||||
+136
-14
@@ -28,12 +28,21 @@
|
|||||||
|
|
||||||
/* Maximum chunk size (64 MB) — prevents unbounded allocation from the wire */
|
/* Maximum chunk size (64 MB) — prevents unbounded allocation from the wire */
|
||||||
#define MAX_CHUNK_SIZE (64ULL * 1024 * 1024)
|
#define MAX_CHUNK_SIZE (64ULL * 1024 * 1024)
|
||||||
|
/* Files larger than this are not kept fully in memory while loading: the
|
||||||
|
* loader skips them so the sender streams from the path, and file_checksum
|
||||||
|
* hashes them from disk in bounded buffers instead of forcing a full load. */
|
||||||
|
#define STREAM_THRESHOLD (64ULL * 1024 * 1024)
|
||||||
#define MAX_MANIFEST_ENTRIES (1024 * 1024)
|
#define MAX_MANIFEST_ENTRIES (1024 * 1024)
|
||||||
/* Aggregate bytes retained by one received deletion manifest. */
|
/* Aggregate bytes retained by one received deletion manifest. */
|
||||||
#define MAX_MANIFEST_BYTES (16ULL * 1024 * 1024)
|
#define MAX_MANIFEST_BYTES (16ULL * 1024 * 1024)
|
||||||
#define DEFAULT_MAX_ALLOC (1ULL * 1024 * 1024 * 1024)
|
#define DEFAULT_MAX_ALLOC (1ULL * 1024 * 1024 * 1024)
|
||||||
/* Server policy ceiling for a client-provided allocation limit. */
|
/* Server policy ceiling for a client-provided allocation limit. */
|
||||||
#define MAX_SERVER_ALLOC (256ULL * 1024 * 1024)
|
#define MAX_SERVER_ALLOC (256ULL * 1024 * 1024)
|
||||||
|
/* Server-owned floor for the per-message I/O deadline. A client --timeout=0
|
||||||
|
(rsync's default) disables the client's own deadlines, but a server session
|
||||||
|
must never be held open forever by a silent peer (slow-loris), so the server
|
||||||
|
floors the effective deadline at this value. */
|
||||||
|
#define SERVER_IO_TIMEOUT_SEC 60
|
||||||
/* Bounded cumulative per-connection receive budget. In-flight wire buffers,
|
/* Bounded cumulative per-connection receive budget. In-flight wire buffers,
|
||||||
decompression buffers and queued (not yet written) file payloads for a
|
decompression buffers and queued (not yet written) file payloads for a
|
||||||
connection must stay within this ceiling. */
|
connection must stay within this ceiling. */
|
||||||
@@ -41,16 +50,48 @@
|
|||||||
|
|
||||||
typedef struct ssl_st SSL;
|
typedef struct ssl_st SSL;
|
||||||
|
|
||||||
|
typedef struct ProtocolSession ProtocolSession;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Transport vtable: the per-session set of I/O primitives the three protocol
|
||||||
|
* loops (send, receive, status-read) dispatch through. The ops are selected
|
||||||
|
* once, when the session is initialized or its SSL is installed, so the loops
|
||||||
|
* never branch on the transport at runtime. A plaintext session uses the
|
||||||
|
* read()/write() ops; a TLS session uses the SSL_read()/SSL_write() ops.
|
||||||
|
*
|
||||||
|
* `send`/`recv` attempt exactly one transfer and return:
|
||||||
|
* > 0 bytes transferred,
|
||||||
|
* PROTOCOL_IO_RETRY no progress; poll on *wait_events and retry,
|
||||||
|
* PROTOCOL_IO_CLOSED peer closed the stream,
|
||||||
|
* PROTOCOL_IO_ERROR fatal transport error.
|
||||||
|
* `has_pending` reports bytes already buffered by the transport (a TLS record
|
||||||
|
* residue); the receive loops skip the poll() gate when it is true.
|
||||||
|
*/
|
||||||
|
typedef struct ProtocolIoOps {
|
||||||
|
ssize_t (*send)(ProtocolSession* session, const void* data, size_t size, short* wait_events);
|
||||||
|
ssize_t (*recv)(ProtocolSession* session, void* data, size_t size, short* wait_events);
|
||||||
|
bool (*has_pending)(const ProtocolSession* session);
|
||||||
|
} ProtocolIoOps;
|
||||||
|
|
||||||
|
/* Negative sentinels returned by ProtocolIoOps.send/recv (see above). */
|
||||||
|
enum {
|
||||||
|
PROTOCOL_IO_RETRY = -1,
|
||||||
|
PROTOCOL_IO_CLOSED = -2,
|
||||||
|
PROTOCOL_IO_ERROR = -3,
|
||||||
|
};
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Explicit owner of protocol I/O. A session does not own the descriptors or
|
* Explicit owner of protocol I/O. A session does not own the descriptors or
|
||||||
* SSL object; it only describes the transport used by a transfer. This makes
|
* SSL object; it only describes the transport used by a transfer. This makes
|
||||||
* it safe to pass the transport to a worker without relying on inherited
|
* it safe to pass the transport to a worker without relying on inherited
|
||||||
* thread-local state.
|
* thread-local state.
|
||||||
*/
|
*/
|
||||||
typedef struct ProtocolSession {
|
struct ProtocolSession {
|
||||||
int read_fd;
|
int read_fd;
|
||||||
int write_fd;
|
int write_fd;
|
||||||
SSL* ssl;
|
SSL* ssl;
|
||||||
|
/* Transport dispatch selected by protocol_session_init()/set_ssl(). */
|
||||||
|
const ProtocolIoOps* ops;
|
||||||
unsigned long long bwlimit;
|
unsigned long long bwlimit;
|
||||||
long long bw_tokens;
|
long long bw_tokens;
|
||||||
long long bw_last_refill_sec;
|
long long bw_last_refill_sec;
|
||||||
@@ -59,12 +100,14 @@ typedef struct ProtocolSession {
|
|||||||
bool eight_bit_output;
|
bool eight_bit_output;
|
||||||
unsigned long long max_alloc;
|
unsigned long long max_alloc;
|
||||||
/* Per-session deadline (seconds) applied to every protocol send/receive by
|
/* Per-session deadline (seconds) applied to every protocol send/receive by
|
||||||
* protocol_send_n_data / protocol_receive_n_data. Defaults to the built-in
|
* protocol_send_n_data / protocol_receive_n_data. The initialized default is
|
||||||
* 60 s window; a value <= 0 falls back to that default. Set from the
|
* the built-in 60 s window; a value <= 0 disables the deadline (rsync's
|
||||||
* negotiated Config->timeout so --timeout is honored by the poll()-driven
|
* --timeout=0). Set from the negotiated Config->timeout so --timeout is
|
||||||
* protocol I/O, not just the socket SO_RCVTIMEO/SO_SNDTIMEO. */
|
* honored by the poll()-driven protocol I/O, not just the socket
|
||||||
|
* SO_RCVTIMEO/SO_SNDTIMEO. The server does not propagate a client 0 here: it
|
||||||
|
* installs protocol_server_io_timeout_sec() so its sessions keep a floor. */
|
||||||
int io_timeout_sec;
|
int io_timeout_sec;
|
||||||
} ProtocolSession;
|
};
|
||||||
|
|
||||||
typedef int Status;
|
typedef int Status;
|
||||||
enum NET_STATUS {
|
enum NET_STATUS {
|
||||||
@@ -155,13 +198,84 @@ enum NET_STATUS {
|
|||||||
* (the receiver reads none in dry-run). STATUS_OK keeps its meaning in this
|
* (the receiver reads none in dry-run). STATUS_OK keeps its meaning in this
|
||||||
* path ("already up to date / nothing to do"). Appended after
|
* path ("already up to date / nothing to do"). Appended after
|
||||||
* STATUS_ERROR_DETAIL so no existing status is renumbered. */
|
* STATUS_ERROR_DETAIL so no existing status is renumbered. */
|
||||||
STATUS_DRY_RUN_TRANSFER
|
STATUS_DRY_RUN_TRANSFER,
|
||||||
|
/* --max-delete budget exhausted (protocol 2.23.0). Sent by the receiver as
|
||||||
|
* the terminal success status INSTEAD of STATUS_OK when a --delete/
|
||||||
|
* --delete-missing-args commit removed up to the --max-delete bound but had
|
||||||
|
* to skip further extras. The transfer itself succeeded and all file data is
|
||||||
|
* stored; the sender maps this to rsync's exit code 25 ("the --max-delete
|
||||||
|
* limit stopped deletions"). Appended after STATUS_DRY_RUN_TRANSFER so no
|
||||||
|
* existing status is renumbered. */
|
||||||
|
STATUS_DELETE_LIMIT,
|
||||||
|
/* Destination-state report for output parity (protocol 2.23.0). When the
|
||||||
|
* wire config carries report_dest_info=true, the receiver answers every
|
||||||
|
* per-file STATUS_CHECK request with STATUS_DEST_INFO FIRST, followed by a
|
||||||
|
* fixed record describing the pre-transfer destination entry
|
||||||
|
* (int32 has_old; uint64 size; int64 mtime; int64 mtime_nsec; uint32 mode;
|
||||||
|
* int32 uid; int32 gid). The ordinary STATUS_OK/STATUS_NEXT/... verdict
|
||||||
|
* follows, so the sender can render rsync-accurate -i/--out-format columns
|
||||||
|
* (new vs modified, and which of size/time/perms/owner/group differ) without
|
||||||
|
* changing the transfer decision itself. Appended after
|
||||||
|
* STATUS_DELETE_LIMIT so no existing status is renumbered. */
|
||||||
|
STATUS_DEST_INFO,
|
||||||
|
/* Per-directory delete plan (protocol 2.24.0). The sender of a
|
||||||
|
* --delete-during/--delete-delay transfer streams one frame per source
|
||||||
|
* directory in directory order instead of a single whole-tree keep-set
|
||||||
|
* manifest. The receiver applies the plan when it arrives
|
||||||
|
* (--delete-during removes that directory's extras immediately) or records
|
||||||
|
* the extras and applies them only after the whole transfer succeeded
|
||||||
|
* (--delete-delay). Payload: an int32 has_config flag (1 on the first plan
|
||||||
|
* of the run, 0 afterwards); when set, the three global config sections
|
||||||
|
* (protected-prefix count+paths, size-skipped count+paths, missing-args
|
||||||
|
* count+paths); then an int32 apply flag (1 for a real plan, 0 for a
|
||||||
|
* config-only carrier frame that must not walk a directory); then the
|
||||||
|
* destination-relative directory path wire string
|
||||||
|
* ("." for the receive root); then the child-directory count + names and the
|
||||||
|
* child-file count + names that must be kept. Appended after
|
||||||
|
* STATUS_DEST_INFO so no existing status is renumbered. */
|
||||||
|
STATUS_DELETE_PLAN,
|
||||||
|
/* End-of-transfer receiver counter report (protocol 2.25.0). When the wire
|
||||||
|
* config carries report_stats=true, the receiver sends this status once,
|
||||||
|
* immediately before its terminal success status, followed by a fixed stats
|
||||||
|
* record (see format_stats_send/receive in format.h) and, when the run is a
|
||||||
|
* --dry-run with --delete, the would-delete path list. Appended after
|
||||||
|
* STATUS_DELETE_PLAN so no existing status is renumbered. */
|
||||||
|
STATUS_STATS
|
||||||
};
|
};
|
||||||
|
|
||||||
void io_set_fds(int read_fd, int write_fd);
|
void io_set_fds(int read_fd, int write_fd);
|
||||||
void io_set_bwlimit(unsigned long long bytes_per_sec);
|
void io_set_bwlimit(unsigned long long bytes_per_sec);
|
||||||
|
unsigned long long io_get_bwlimit(void);
|
||||||
void io_set_ssl(SSL* ssl);
|
void io_set_ssl(SSL* ssl);
|
||||||
SSL* io_get_ssl(void);
|
SSL* io_get_ssl(void);
|
||||||
|
/* SSL object of the transport in effect on this thread: the currently bound
|
||||||
|
* session's SSL when a TLS session is bound, otherwise the legacy thread-local
|
||||||
|
* io_ssl. NULL for a plaintext transport. Unlike io_get_ssl(), this resolves
|
||||||
|
* worker threads that bound a TLS session via protocol_session_set_ssl()/
|
||||||
|
* protocol_session_bind() but never called io_set_ssl() themselves (C11
|
||||||
|
* _Thread_local state is not inherited by a new thread). A bound session only
|
||||||
|
* wins when its selected dispatch is TLS; a bound plaintext session (ssl ==
|
||||||
|
* NULL) falls back to io_ssl so it can never mask a live encrypted transport.
|
||||||
|
* Callers that must choose a TLS-only code path (e.g. file_send.c's sendfile
|
||||||
|
* fallback) must use this instead of io_get_ssl(). */
|
||||||
|
SSL* protocol_current_ssl(void);
|
||||||
|
|
||||||
|
/* Process-wide wire byte counters. protocol_send_n_data/protocol_receive_n_data
|
||||||
|
* update them; the zero-copy sendfile path reports through
|
||||||
|
* protocol_note_bytes_written. Used by the client to render rsync's
|
||||||
|
* --stats/--progress totals and the --out-format %b/%c tokens. */
|
||||||
|
unsigned long long protocol_bytes_written(void);
|
||||||
|
unsigned long long protocol_bytes_read(void);
|
||||||
|
void protocol_note_bytes_written(unsigned long long bytes);
|
||||||
|
/* Apply --bwlimit pacing to bytes written outside protocol_send_n_data (the
|
||||||
|
* plaintext zero-copy sendfile fast path). `file_descriptor` is the wire fd
|
||||||
|
* the bytes were written to, so the legacy session is resolved exactly as the
|
||||||
|
* preceding send_n_data call resolved it (the bound TLS session still wins when
|
||||||
|
* set); resolving with the same fd avoids re-initializing the legacy session
|
||||||
|
* and granting a second first-call burst. Runs the same token-bucket throttle,
|
||||||
|
* so the sendfile transport is paced identically to the buffered/TLS paths. A
|
||||||
|
* no-op when the effective session has no bandwidth limit. */
|
||||||
|
void protocol_throttle_bytes(int file_descriptor, size_t bytes);
|
||||||
|
|
||||||
void protocol_session_init(ProtocolSession* session, int read_fd, int write_fd);
|
void protocol_session_init(ProtocolSession* session, int read_fd, int write_fd);
|
||||||
/* Transitional bridge for helpers whose signatures still carry only an fd. */
|
/* Transitional bridge for helpers whose signatures still carry only an fd. */
|
||||||
@@ -170,15 +284,20 @@ void protocol_session_unbind(void);
|
|||||||
void protocol_session_set_ssl(ProtocolSession* session, SSL* ssl);
|
void protocol_session_set_ssl(ProtocolSession* session, SSL* ssl);
|
||||||
void protocol_session_set_bwlimit(ProtocolSession* session, unsigned long long bytes_per_sec);
|
void protocol_session_set_bwlimit(ProtocolSession* session, unsigned long long bytes_per_sec);
|
||||||
void protocol_session_set_max_alloc(ProtocolSession* session, unsigned long long max_alloc);
|
void protocol_session_set_max_alloc(ProtocolSession* session, unsigned long long max_alloc);
|
||||||
/* Override the per-message send/receive deadline for this session.
|
/* Override the per-message send/receive deadline for this session. The value
|
||||||
* `sec` <= 0 restores the built-in 60 s default (used for --timeout=0/unset).
|
* is stored verbatim: a positive value sets the deadline, `sec` <= 0 disables
|
||||||
* An explicit long deadline (e.g. the delete-ack wait) is applied per-call by
|
* it (rsync's --timeout=0). An explicit long deadline (e.g. the delete-ack
|
||||||
* protocol_receive_status_timed and is unaffected by this setter. */
|
* wait) is applied per-call by protocol_receive_status_timed and is unaffected
|
||||||
|
* by this setter. */
|
||||||
void protocol_session_set_io_timeout(ProtocolSession* session, int sec);
|
void protocol_session_set_io_timeout(ProtocolSession* session, int sec);
|
||||||
/* Effective per-message I/O deadline (seconds) for the currently-bound session,
|
/* Effective per-message I/O deadline (seconds) for the currently-bound session.
|
||||||
* falling back to the built-in default. Used by the plaintext sendfile path
|
* Zero means the deadline is disabled (rsync's --timeout=0). Used by the
|
||||||
* which bypasses the protocol send primitive. */
|
* plaintext sendfile path which bypasses the protocol send primitive. */
|
||||||
int protocol_get_io_timeout_sec(void);
|
int protocol_get_io_timeout_sec(void);
|
||||||
|
/* The server-side effective deadline for a client-requested timeout: a positive
|
||||||
|
* client value is honored, otherwise the SERVER_IO_TIMEOUT_SEC floor applies so
|
||||||
|
* a silent peer can never hold a session open forever. */
|
||||||
|
int protocol_server_io_timeout_sec(int client_timeout);
|
||||||
void* protocol_alloc(size_t size);
|
void* protocol_alloc(size_t size);
|
||||||
void* protocol_realloc(void* ptr, size_t size);
|
void* protocol_realloc(void* ptr, size_t size);
|
||||||
void protocol_session_set_8_bit_output(ProtocolSession* session, bool enabled);
|
void protocol_session_set_8_bit_output(ProtocolSession* session, bool enabled);
|
||||||
@@ -200,6 +319,9 @@ bool protocol_send_int(ProtocolSession* session, int data);
|
|||||||
bool protocol_receive_int(ProtocolSession* session, int* data);
|
bool protocol_receive_int(ProtocolSession* session, int* data);
|
||||||
bool protocol_send_status(ProtocolSession* session, Status status);
|
bool protocol_send_status(ProtocolSession* session, Status status);
|
||||||
bool protocol_receive_status(ProtocolSession* session, Status* status);
|
bool protocol_receive_status(ProtocolSession* session, Status* status);
|
||||||
|
/* As protocol_receive_status, but with an explicit per-message deadline
|
||||||
|
* (seconds) instead of the session's configured io_timeout_sec. */
|
||||||
|
bool protocol_receive_status_timed(ProtocolSession* session, Status* status, int timeout_sec);
|
||||||
bool send_n_data(int file_descriptor, const void* data, size_t data_size);
|
bool send_n_data(int file_descriptor, const void* data, size_t data_size);
|
||||||
bool receive_n_data(int file_descriptor, void* data, size_t data_size);
|
bool receive_n_data(int file_descriptor, void* data, size_t data_size);
|
||||||
|
|
||||||
|
|||||||
+18
-1
@@ -128,7 +128,7 @@ bool queue_enqueue_multithreaded_cancel(Queue* queue, void* item, mtx_t* mutex,
|
|||||||
|
|
||||||
void* queue_dequeue(Queue* queue) {
|
void* queue_dequeue(Queue* queue) {
|
||||||
if (queue == NULL || queue_is_empty(queue)) {
|
if (queue == NULL || queue_is_empty(queue)) {
|
||||||
log_perror("ERROR: Could not dequeue from null or empty queue.");
|
log_message(LOG_LEVEL_ERROR, "%s", "ERROR: Could not dequeue from null or empty queue.");
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -139,6 +139,23 @@ void* queue_dequeue(Queue* queue) {
|
|||||||
return item;
|
return item;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool queue_push(Queue* queue, void* item) {
|
||||||
|
return queue_enqueue(queue, item);
|
||||||
|
}
|
||||||
|
|
||||||
|
void* queue_pop(Queue* queue) {
|
||||||
|
if (queue == NULL || queue_is_empty(queue)) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "%s", "ERROR: Could not pop from null or empty queue.");
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
queue->rear = (queue->rear - 1 + queue->capacity) % queue->capacity;
|
||||||
|
void* item = queue->items[queue->rear];
|
||||||
|
queue->items[queue->rear] = NULL;
|
||||||
|
queue->size--;
|
||||||
|
return item;
|
||||||
|
}
|
||||||
|
|
||||||
void* queue_dequeue_multithreaded(Queue* queue, mtx_t* mutex, cnd_t* condition_not_empty,
|
void* queue_dequeue_multithreaded(Queue* queue, mtx_t* mutex, cnd_t* condition_not_empty,
|
||||||
cnd_t* condition_not_full, const bool* other_thread_done) {
|
cnd_t* condition_not_full, const bool* other_thread_done) {
|
||||||
mtx_lock(mutex);
|
mtx_lock(mutex);
|
||||||
|
|||||||
@@ -28,4 +28,11 @@ void* queue_dequeue(Queue* queue);
|
|||||||
void* queue_dequeue_multithreaded(Queue* queue, mtx_t* mutex, cnd_t* condition_not_empty,
|
void* queue_dequeue_multithreaded(Queue* queue, mtx_t* mutex, cnd_t* condition_not_empty,
|
||||||
cnd_t* condition_not_full, const bool* other_thread_done);
|
cnd_t* condition_not_full, const bool* other_thread_done);
|
||||||
|
|
||||||
|
/* LIFO stack operations over the same ring buffer. queue_push() is the enqueue
|
||||||
|
primitive; queue_pop() removes from the rear, so a sequence of pushes is
|
||||||
|
returned in reverse order. Used by the sequential scanner's depth-first
|
||||||
|
traversal. */
|
||||||
|
bool queue_push(Queue* queue, void* item);
|
||||||
|
void* queue_pop(Queue* queue);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
+181
-21
@@ -44,6 +44,181 @@ static bool parse_two_digits(const char* s, int* out) {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* True when the current character of the cursor is a decimal digit. */
|
||||||
|
static bool is_digit(const char* cp) {
|
||||||
|
return *cp >= '0' && *cp <= '9';
|
||||||
|
}
|
||||||
|
|
||||||
|
/* rsync 3.4.1's flexible --stop-at date parser (ported from
|
||||||
|
* options.c:parse_time). Returns a time_t, or (time_t)-1 on a malformed value.
|
||||||
|
* Accepted forms include Y-M-DTh:m, Y/M/DTh:m, Y-M-D, M-D, D, h:m, :m and
|
||||||
|
* "T h:m"; a 1- or 2-digit year and omitted fields are resolved to the next
|
||||||
|
* matching point in time in the local timezone. Seconds are NOT accepted
|
||||||
|
* (rsync rejects them too); FastSync keeps its own HH:MM:SS spelling as an
|
||||||
|
* extension handled by the caller. `now` is passed in so tests are
|
||||||
|
* deterministic; production passes time(NULL). */
|
||||||
|
static time_t parse_time_rsync(const char* value, time_t now) {
|
||||||
|
const char* cp;
|
||||||
|
time_t val;
|
||||||
|
struct tm today;
|
||||||
|
if (!localtime_r(&now, &today))
|
||||||
|
return (time_t)-1;
|
||||||
|
struct tm t;
|
||||||
|
int in_date, old_mday, n;
|
||||||
|
|
||||||
|
memset(&t, 0, sizeof t);
|
||||||
|
t.tm_year = t.tm_mon = t.tm_mday = -1;
|
||||||
|
t.tm_hour = t.tm_min = t.tm_isdst = -1;
|
||||||
|
cp = value;
|
||||||
|
if (*cp == 'T' || *cp == 't' || *cp == ':') {
|
||||||
|
in_date = *cp == ':' ? 0 : -1;
|
||||||
|
cp++;
|
||||||
|
} else
|
||||||
|
in_date = 1;
|
||||||
|
for (;; cp++) {
|
||||||
|
if (!is_digit(cp))
|
||||||
|
return (time_t)-1;
|
||||||
|
n = 0;
|
||||||
|
do {
|
||||||
|
n = n * 10 + *cp++ - '0';
|
||||||
|
} while (is_digit(cp));
|
||||||
|
if (*cp == ':')
|
||||||
|
in_date = 0;
|
||||||
|
if (in_date > 0) {
|
||||||
|
if (t.tm_year != -1)
|
||||||
|
return (time_t)-1;
|
||||||
|
t.tm_year = t.tm_mon;
|
||||||
|
t.tm_mon = t.tm_mday;
|
||||||
|
t.tm_mday = n;
|
||||||
|
if (!*cp)
|
||||||
|
break;
|
||||||
|
if (*cp == 'T' || *cp == 't') {
|
||||||
|
if (!cp[1])
|
||||||
|
break;
|
||||||
|
in_date = -1;
|
||||||
|
} else if (*cp != '-' && *cp != '/')
|
||||||
|
return (time_t)-1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (t.tm_hour != -1)
|
||||||
|
return (time_t)-1;
|
||||||
|
t.tm_hour = t.tm_min;
|
||||||
|
t.tm_min = n;
|
||||||
|
if (!*cp) {
|
||||||
|
if (in_date < 0)
|
||||||
|
return (time_t)-1;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if (*cp != ':')
|
||||||
|
return (time_t)-1;
|
||||||
|
in_date = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
in_date = 0;
|
||||||
|
if (t.tm_year < 0) {
|
||||||
|
t.tm_year = today.tm_year;
|
||||||
|
in_date = 1;
|
||||||
|
} else if (t.tm_year < 100) {
|
||||||
|
while (t.tm_year < today.tm_year)
|
||||||
|
t.tm_year += 100;
|
||||||
|
} else
|
||||||
|
t.tm_year -= 1900;
|
||||||
|
if (t.tm_mon < 0) {
|
||||||
|
t.tm_mon = today.tm_mon;
|
||||||
|
in_date = 2;
|
||||||
|
} else
|
||||||
|
t.tm_mon--;
|
||||||
|
if (t.tm_mday < 0) {
|
||||||
|
t.tm_mday = today.tm_mday;
|
||||||
|
in_date = 3;
|
||||||
|
}
|
||||||
|
|
||||||
|
n = 0;
|
||||||
|
if (t.tm_min < 0) {
|
||||||
|
t.tm_hour = t.tm_min = 0;
|
||||||
|
} else if (t.tm_hour < 0) {
|
||||||
|
if (in_date != 3)
|
||||||
|
return (time_t)-1;
|
||||||
|
in_date = 0;
|
||||||
|
t.tm_hour = today.tm_hour;
|
||||||
|
n = 60 * 60;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* mktime() may roll a too-large tm_mday into the following month; undo that
|
||||||
|
* in the "next match" loop below. */
|
||||||
|
old_mday = t.tm_mday;
|
||||||
|
if (t.tm_hour > 23 || t.tm_min > 59 || t.tm_mon < 0 || t.tm_mon >= 12 || t.tm_mday < 1 ||
|
||||||
|
t.tm_mday > 31 || (val = mktime(&t)) == (time_t)-1)
|
||||||
|
return (time_t)-1;
|
||||||
|
|
||||||
|
while (in_date && (val <= now || t.tm_mday < old_mday)) {
|
||||||
|
switch (in_date) {
|
||||||
|
case 3:
|
||||||
|
old_mday = ++t.tm_mday;
|
||||||
|
break;
|
||||||
|
case 2:
|
||||||
|
if (t.tm_mday < old_mday)
|
||||||
|
t.tm_mday = old_mday; /* the month already got bumped forward */
|
||||||
|
else if (++t.tm_mon == 12) {
|
||||||
|
t.tm_mon = 0;
|
||||||
|
t.tm_year++;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
case 1:
|
||||||
|
if (t.tm_mday < old_mday) {
|
||||||
|
/* mon==1 mday==29 got bumped to mon==2 */
|
||||||
|
if (t.tm_mon != 2 || old_mday != 29)
|
||||||
|
return (time_t)-1;
|
||||||
|
t.tm_mon = 1;
|
||||||
|
t.tm_mday = 29;
|
||||||
|
}
|
||||||
|
t.tm_year++;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if ((val = mktime(&t)) == (time_t)-1) {
|
||||||
|
if (in_date != 3 || t.tm_mday <= 28)
|
||||||
|
return (time_t)-1;
|
||||||
|
t.tm_mday = old_mday = 1;
|
||||||
|
in_date = 2;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (n) {
|
||||||
|
while (val <= now)
|
||||||
|
val += n;
|
||||||
|
}
|
||||||
|
return val;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* FastSync's HH:MM or HH:MM:SS spelling on the current local day. rsync's own
|
||||||
|
* --stop-at accepts only HH:MM, so this is a strict superset extension. */
|
||||||
|
static bool parse_clock_time(const char* value, time_t now, time_t* out_deadline) {
|
||||||
|
size_t len = strlen(value);
|
||||||
|
if (len != 5 && len != 8)
|
||||||
|
return false;
|
||||||
|
if (value[2] != ':' || (len == 8 && value[5] != ':'))
|
||||||
|
return false;
|
||||||
|
int hh, mm, ss = 0;
|
||||||
|
if (!parse_two_digits(value, &hh) || !parse_two_digits(value + 3, &mm))
|
||||||
|
return false;
|
||||||
|
if (len == 8 && !parse_two_digits(value + 6, &ss))
|
||||||
|
return false;
|
||||||
|
if (hh > 23 || mm > 59 || ss > 59)
|
||||||
|
return false;
|
||||||
|
|
||||||
|
struct tm today;
|
||||||
|
if (!localtime_r(&now, &today))
|
||||||
|
return false;
|
||||||
|
today.tm_hour = hh;
|
||||||
|
today.tm_min = mm;
|
||||||
|
today.tm_sec = ss;
|
||||||
|
today.tm_isdst = -1;
|
||||||
|
time_t deadline = mktime(&today);
|
||||||
|
if (deadline == (time_t)-1)
|
||||||
|
return false;
|
||||||
|
*out_deadline = deadline;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
bool stop_parse_at_time(const char* value, time_t now, time_t* out_deadline) {
|
bool stop_parse_at_time(const char* value, time_t now, time_t* out_deadline) {
|
||||||
if (!value || !out_deadline)
|
if (!value || !out_deadline)
|
||||||
return false;
|
return false;
|
||||||
@@ -91,28 +266,13 @@ bool stop_parse_at_time(const char* value, time_t now, time_t* out_deadline) {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* HH:MM or HH:MM:SS on the current local day. */
|
/* HH:MM or HH:MM:SS on the current local day (FastSync extension). */
|
||||||
size_t len = strlen(value);
|
if (parse_clock_time(value, now, out_deadline))
|
||||||
if (len != 5 && len != 8)
|
return true;
|
||||||
return false;
|
|
||||||
if (value[2] != ':' || (len == 8 && value[5] != ':'))
|
|
||||||
return false;
|
|
||||||
int hh, mm, ss = 0;
|
|
||||||
if (!parse_two_digits(value, &hh) || !parse_two_digits(value + 3, &mm))
|
|
||||||
return false;
|
|
||||||
if (len == 8 && !parse_two_digits(value + 6, &ss))
|
|
||||||
return false;
|
|
||||||
if (hh > 23 || mm > 59 || ss > 59)
|
|
||||||
return false;
|
|
||||||
|
|
||||||
struct tm today;
|
/* rsync's full/partial date-and-time form (e.g. 2000-12-31T23:59, 12-31,
|
||||||
if (!localtime_r(&now, &today))
|
* 14:00, :59, 1, 1-30). */
|
||||||
return false;
|
time_t deadline = parse_time_rsync(value, now);
|
||||||
today.tm_hour = hh;
|
|
||||||
today.tm_min = mm;
|
|
||||||
today.tm_sec = ss;
|
|
||||||
today.tm_isdst = -1;
|
|
||||||
time_t deadline = mktime(&today);
|
|
||||||
if (deadline == (time_t)-1)
|
if (deadline == (time_t)-1)
|
||||||
return false;
|
return false;
|
||||||
*out_deadline = deadline;
|
*out_deadline = deadline;
|
||||||
|
|||||||
@@ -87,7 +87,7 @@ static int parse_remote_dest(const char* dest, RemoteDest* r) {
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
char* ssh_build_remote_command(const char* server_path, bool old_args, char* const* remote_options,
|
char* ssh_build_remote_command(const char* server_path, char* const* remote_options,
|
||||||
int remote_option_count) {
|
int remote_option_count) {
|
||||||
const char* path = server_path ? server_path : "fastsync-server";
|
const char* path = server_path ? server_path : "fastsync-server";
|
||||||
const char* suffix = " --stdio";
|
const char* suffix = " --stdio";
|
||||||
@@ -105,9 +105,7 @@ char* ssh_build_remote_command(const char* server_path, bool old_args, char* con
|
|||||||
shell word (remote options below reuse the same escaping), then
|
shell word (remote options below reuse the same escaping), then
|
||||||
" --stdio". Quoting the path is the only injection-safe construction: an
|
" --stdio". Quoting the path is the only injection-safe construction: an
|
||||||
unquoted path would carry shell metacharacters straight into the remote
|
unquoted path would carry shell metacharacters straight into the remote
|
||||||
shell command. --old-args is kept for CLI/ABI compatibility but no longer
|
shell command. (rsync's --old-args no longer disables that protection.) */
|
||||||
disables that protection. */
|
|
||||||
(void)old_args;
|
|
||||||
size_t quote_count = 0;
|
size_t quote_count = 0;
|
||||||
for (const char* p = path; *p; p++)
|
for (const char* p = path; *p; p++)
|
||||||
if (*p == '\'')
|
if (*p == '\'')
|
||||||
@@ -296,8 +294,8 @@ void ssh_free_client_argv(char** argv) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
Client* client_connect_ssh(const char* destination, int port, const char* server_path,
|
Client* client_connect_ssh(const char* destination, int port, const char* server_path,
|
||||||
bool old_args, const char* rsh_command, bool blocking_io,
|
const char* rsh_command, bool blocking_io, char* const* remote_options,
|
||||||
char* const* remote_options, int remote_option_count) {
|
int remote_option_count) {
|
||||||
RemoteDest r;
|
RemoteDest r;
|
||||||
if (parse_remote_dest(destination, &r) != 0) {
|
if (parse_remote_dest(destination, &r) != 0) {
|
||||||
char* escaped = output_escape(destination, false);
|
char* escaped = output_escape(destination, false);
|
||||||
@@ -376,7 +374,7 @@ Client* client_connect_ssh(const char* destination, int port, const char* server
|
|||||||
snprintf(ssh_user, ssh_user_len, "%s", r.host);
|
snprintf(ssh_user, ssh_user_len, "%s", r.host);
|
||||||
|
|
||||||
char* remote_command =
|
char* remote_command =
|
||||||
ssh_build_remote_command(server_path, old_args, remote_options, remote_option_count);
|
ssh_build_remote_command(server_path, remote_options, remote_option_count);
|
||||||
if (!remote_command)
|
if (!remote_command)
|
||||||
ssh_child_setup_failed(exec_pipe[1]);
|
ssh_child_setup_failed(exec_pipe[1]);
|
||||||
char** ssh_argv = ssh_build_client_argv(rsh_command, port, ssh_user, remote_command);
|
char** ssh_argv = ssh_build_client_argv(rsh_command, port, ssh_user, remote_command);
|
||||||
|
|||||||
@@ -4,17 +4,17 @@
|
|||||||
#include "transport_tcp.h"
|
#include "transport_tcp.h"
|
||||||
|
|
||||||
Client* client_connect_ssh(const char* destination, int port, const char* server_path,
|
Client* client_connect_ssh(const char* destination, int port, const char* server_path,
|
||||||
bool old_args, const char* rsh_command, bool blocking_io,
|
const char* rsh_command, bool blocking_io, char* const* remote_options,
|
||||||
char* const* remote_options, int remote_option_count);
|
int remote_option_count);
|
||||||
/* Build the escaped remote-shell command string (the server program path always
|
/* Build the escaped remote-shell command string (the server program path always
|
||||||
* quoted as one remote-shell word, followed by ` --stdio` and each
|
* quoted as one remote-shell word, followed by ` --stdio` and each
|
||||||
* --remote-option value appended as an individually single-quoted shell word).
|
* --remote-option value appended as an individually single-quoted shell word).
|
||||||
* `old_args` is accepted for CLI/ABI compatibility but no longer disables
|
* The path is always escaped so a metacharacter-bearing --rsync-path can never
|
||||||
* quoting: the path is always escaped so a metacharacter-bearing
|
* be interpreted by the remote shell (the --old-args no-op does not disable
|
||||||
* --rsync-path can never be interpreted by the remote shell. Every
|
* quoting). Every --remote-option value is individually escaped with the '\''
|
||||||
* --remote-option value is individually escaped with the '\'' sequence and
|
* sequence and values with empty/control characters are rejected at the CLI
|
||||||
* values with empty/control characters are rejected at the CLI parse layer. */
|
* parse layer. */
|
||||||
char* ssh_build_remote_command(const char* server_path, bool old_args, char* const* remote_options,
|
char* ssh_build_remote_command(const char* server_path, char* const* remote_options,
|
||||||
int remote_option_count);
|
int remote_option_count);
|
||||||
/* Build the NULL-terminated child argv for the remote-shell client (argv[0] is
|
/* Build the NULL-terminated child argv for the remote-shell client (argv[0] is
|
||||||
* the exec/execvp program). rsh_command is whitespace-split into leading argv
|
* the exec/execvp program). rsh_command is whitespace-split into leading argv
|
||||||
|
|||||||
+19
-11
@@ -289,14 +289,14 @@ void server_accept_loop(Server* server, void (*child_fn)(int, void*), void* chil
|
|||||||
accept_loop(server, child_fn, child_ctx, log_fmt);
|
accept_loop(server, child_fn, child_ctx, log_fmt);
|
||||||
}
|
}
|
||||||
|
|
||||||
static int g_timeout_sec = 30;
|
/* rsync defaults: --timeout=0 (disabled) and --contimeout=60. A non-positive
|
||||||
static int g_contimeout_sec = 10;
|
* value means "no timeout" rather than "leave the built-in value in place". */
|
||||||
|
static int g_timeout_sec = 0;
|
||||||
|
static int g_contimeout_sec = 60;
|
||||||
|
|
||||||
void tcp_set_timeouts(int timeout_sec, int contimeout_sec) {
|
void tcp_set_timeouts(int timeout_sec, int contimeout_sec) {
|
||||||
if (timeout_sec > 0)
|
g_timeout_sec = timeout_sec > 0 ? timeout_sec : 0;
|
||||||
g_timeout_sec = timeout_sec;
|
g_contimeout_sec = contimeout_sec > 0 ? contimeout_sec : 0;
|
||||||
if (contimeout_sec > 0)
|
|
||||||
g_contimeout_sec = contimeout_sec;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
int tcp_get_contimeout_sec(void) {
|
int tcp_get_contimeout_sec(void) {
|
||||||
@@ -308,6 +308,10 @@ int tcp_get_timeout_sec(void) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
static void tcp_apply_socket_timeout(int fd) {
|
static void tcp_apply_socket_timeout(int fd) {
|
||||||
|
/* timeout 0 means no timeout: leave the socket in its default (blocking)
|
||||||
|
* mode instead of installing a zero SO_RCVTIMEO/SO_SNDTIMEO. */
|
||||||
|
if (g_timeout_sec <= 0)
|
||||||
|
return;
|
||||||
struct timeval tv;
|
struct timeval tv;
|
||||||
tv.tv_sec = g_timeout_sec;
|
tv.tv_sec = g_timeout_sec;
|
||||||
tv.tv_usec = 0;
|
tv.tv_usec = 0;
|
||||||
@@ -483,11 +487,15 @@ bool tcp_connect_socket_ex(Client* client, const char* host, int port,
|
|||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
struct timeval ct;
|
/* --contimeout=0 disables the connect timeout: skip the pre-connect socket
|
||||||
ct.tv_sec = g_contimeout_sec;
|
* timeouts entirely. */
|
||||||
ct.tv_usec = 0;
|
if (g_contimeout_sec > 0) {
|
||||||
setsockopt(client->file_descriptor, SOL_SOCKET, SO_RCVTIMEO, &ct, sizeof(ct));
|
struct timeval ct;
|
||||||
setsockopt(client->file_descriptor, SOL_SOCKET, SO_SNDTIMEO, &ct, sizeof(ct));
|
ct.tv_sec = g_contimeout_sec;
|
||||||
|
ct.tv_usec = 0;
|
||||||
|
setsockopt(client->file_descriptor, SOL_SOCKET, SO_RCVTIMEO, &ct, sizeof(ct));
|
||||||
|
setsockopt(client->file_descriptor, SOL_SOCKET, SO_SNDTIMEO, &ct, sizeof(ct));
|
||||||
|
}
|
||||||
|
|
||||||
if (bind_addr_family != 0) {
|
if (bind_addr_family != 0) {
|
||||||
if (rp->ai_family != bind_addr_family) {
|
if (rp->ai_family != bind_addr_family) {
|
||||||
|
|||||||
+78
-298
@@ -2,10 +2,12 @@
|
|||||||
#include "array_list.h"
|
#include "array_list.h"
|
||||||
#include "log.h"
|
#include "log.h"
|
||||||
#include <arpa/inet.h>
|
#include <arpa/inet.h>
|
||||||
|
#include <ctype.h>
|
||||||
#include <dirent.h>
|
#include <dirent.h>
|
||||||
#include <errno.h>
|
#include <errno.h>
|
||||||
#include <fcntl.h>
|
#include <fcntl.h>
|
||||||
#include <netinet/in.h>
|
#include <netinet/in.h>
|
||||||
|
#include <stdarg.h>
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
@@ -47,11 +49,45 @@ const char* utils_get_authorized_root_path(void) {
|
|||||||
return authorized_root_path;
|
return authorized_root_path;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void utils_set_error(char* err, size_t err_size, const char* fmt, ...) {
|
||||||
|
if (!err || err_size == 0)
|
||||||
|
return;
|
||||||
|
va_list args;
|
||||||
|
va_start(args, fmt);
|
||||||
|
vsnprintf(err, err_size, fmt, args);
|
||||||
|
va_end(args);
|
||||||
|
}
|
||||||
|
|
||||||
bool path_is_within_root(const char* root, const char* path) {
|
bool path_is_within_root(const char* root, const char* path) {
|
||||||
size_t root_len = strlen(root);
|
size_t root_len = strlen(root);
|
||||||
return strncmp(root, path, root_len) == 0 && (path[root_len] == '\0' || path[root_len] == '/');
|
return strncmp(root, path, root_len) == 0 && (path[root_len] == '\0' || path[root_len] == '/');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Borrowed transfer-relative view of `path`: strip any leading '/' and then a
|
||||||
|
* `root` prefix (its own leading/trailing slashes tolerated), returning a
|
||||||
|
* pointer into `path`. Non-allocating, so it is safe on the hot scan/print
|
||||||
|
* paths. A NULL/empty root, or a path not under `root`, leaves only the
|
||||||
|
* leading-slash strip. `path` must be NUL-terminated and live in the caller. */
|
||||||
|
const char* utils_strip_transfer_root(const char* path, const char* root) {
|
||||||
|
if (path == NULL)
|
||||||
|
return NULL;
|
||||||
|
const char* rel = path;
|
||||||
|
while (*rel == '/')
|
||||||
|
rel++;
|
||||||
|
if (root == NULL)
|
||||||
|
return rel;
|
||||||
|
while (*root == '/')
|
||||||
|
root++;
|
||||||
|
size_t root_len = strlen(root);
|
||||||
|
while (root_len > 0 && root[root_len - 1] == '/')
|
||||||
|
root_len--;
|
||||||
|
if (root_len == 0)
|
||||||
|
return rel;
|
||||||
|
if (strncmp(rel, root, root_len) == 0 && (rel[root_len] == '/' || rel[root_len] == '\0'))
|
||||||
|
return rel + root_len + (rel[root_len] == '/' ? 1 : 0);
|
||||||
|
return rel;
|
||||||
|
}
|
||||||
|
|
||||||
/* Open the destination root directory itself, confined to the authorized root.
|
/* Open the destination root directory itself, confined to the authorized root.
|
||||||
* NOTE (do not merge with file_open_secure_parent): this walk opens dest_root
|
* NOTE (do not merge with file_open_secure_parent): this walk opens dest_root
|
||||||
* (a directory that must already exist) and returns its fd, whereas
|
* (a directory that must already exist) and returns its fd, whereas
|
||||||
@@ -60,7 +96,7 @@ bool path_is_within_root(const char* root, const char* path) {
|
|||||||
* two differ in create-vs-no-create, in what path component they stop at, and
|
* two differ in create-vs-no-create, in what path component they stop at, and
|
||||||
* in the extra receiver policies they apply, so they are intentionally kept
|
* in the extra receiver policies they apply, so they are intentionally kept
|
||||||
* separate. Both rely on the shared lexical path_is_within_root check. */
|
* separate. Both rely on the shared lexical path_is_within_root check. */
|
||||||
static int open_authorized_destination(const char* dest_root) {
|
int utils_open_authorized_destination(const char* dest_root) {
|
||||||
int root_fd = utils_get_authorized_root_fd();
|
int root_fd = utils_get_authorized_root_fd();
|
||||||
const char* root_path = utils_get_authorized_root_path();
|
const char* root_path = utils_get_authorized_root_path();
|
||||||
if (root_fd < 0 || !root_path || !dest_root || !path_is_within_root(root_path, dest_root))
|
if (root_fd < 0 || !root_path || !dest_root || !path_is_within_root(root_path, dest_root))
|
||||||
@@ -117,6 +153,47 @@ char* str_dup(const char* string) {
|
|||||||
return new_string;
|
return new_string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
int env_choice_first(const char* env_name, int (*resolve)(const char*), bool* specified) {
|
||||||
|
if (specified)
|
||||||
|
*specified = false;
|
||||||
|
if (!env_name || !resolve)
|
||||||
|
return -1;
|
||||||
|
const char* env = getenv(env_name);
|
||||||
|
if (!env)
|
||||||
|
return -1;
|
||||||
|
|
||||||
|
bool saw_nonblank = false;
|
||||||
|
const char* p = env;
|
||||||
|
while (*p) {
|
||||||
|
if (*p == '&')
|
||||||
|
break;
|
||||||
|
if (isspace((unsigned char)*p)) {
|
||||||
|
p++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
saw_nonblank = true;
|
||||||
|
char token[64];
|
||||||
|
size_t len = 0;
|
||||||
|
while (*p && *p != '&' && !isspace((unsigned char)*p)) {
|
||||||
|
if (len < sizeof(token) - 1)
|
||||||
|
token[len++] = *p;
|
||||||
|
p++;
|
||||||
|
}
|
||||||
|
token[len] = '\0';
|
||||||
|
if (len > 0) {
|
||||||
|
int id = resolve(token);
|
||||||
|
if (id >= 0) {
|
||||||
|
if (specified)
|
||||||
|
*specified = true;
|
||||||
|
return id;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (specified)
|
||||||
|
*specified = saw_nonblank;
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
#define STR_HASH_SET_MIN_CAPACITY 16
|
#define STR_HASH_SET_MIN_CAPACITY 16
|
||||||
|
|
||||||
static size_t str_hash_set_hash(const char* key, size_t len) {
|
static size_t str_hash_set_hash(const char* key, size_t len) {
|
||||||
@@ -548,303 +625,6 @@ bool format_human_bytes(unsigned long long bytes, char* buffer, size_t buffer_si
|
|||||||
return written >= 0 && (size_t)written < buffer_size;
|
return written >= 0 && (size_t)written < buffer_size;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Build the keep-set index from the exact manifest entries only. A lookup of
|
|
||||||
`rel` succeeds iff `rel` is a kept entry, a kept directory, or an ancestor
|
|
||||||
directory of kept content (the old is_dir_in_manifest predicate); the sorted
|
|
||||||
view answers "is an ancestor of kept content" without materializing any
|
|
||||||
per-component prefix copy, so the index is O(manifest size) memory. */
|
|
||||||
static bool build_keep_index(const ArrayList* manifest, PathIndex* index) {
|
|
||||||
if (!manifest || manifest->size <= 0)
|
|
||||||
return path_index_build(index, NULL, 0);
|
|
||||||
return path_index_build(index, (const char* const*)manifest->items, (size_t)manifest->size);
|
|
||||||
}
|
|
||||||
|
|
||||||
static bool keep_is_dir(const PathIndex* index, const char* rel_path) {
|
|
||||||
return path_index_contains(index, rel_path) || path_index_has_descendant(index, rel_path);
|
|
||||||
}
|
|
||||||
|
|
||||||
static bool keep_is_file(const PathIndex* index, const char* rel_path) {
|
|
||||||
return path_index_contains(index, rel_path);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* True when child_rel is, or lies below, a protected entry. A prefix "a"
|
|
||||||
therefore protects "a" and "a/b/c" but not "ab". Entries with top_level_only
|
|
||||||
set only protect DIRECT children of the receive root (at_root); nested
|
|
||||||
directories that share such a name stay ordinary destination content. */
|
|
||||||
bool path_under_skip_prefix(const char* child_rel, bool at_root, const DeleteSkipEntry* skips,
|
|
||||||
int skip_count) {
|
|
||||||
for (int i = 0; i < skip_count; i++) {
|
|
||||||
if (skips[i].top_level_only && !at_root)
|
|
||||||
continue;
|
|
||||||
size_t prefix_len = strlen(skips[i].prefix);
|
|
||||||
if (strncmp(child_rel, skips[i].prefix, prefix_len) == 0 &&
|
|
||||||
(child_rel[prefix_len] == '\0' || child_rel[prefix_len] == '/'))
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* All-or-nothing max-delete needs to know BEFORE any unlink whether the run
|
|
||||||
would delete more than max_delete entries. This rehearsal pass walks the
|
|
||||||
destination with the same decisions as the delete pass but never touches the
|
|
||||||
filesystem: it counts every regular file the delete pass would unlink and
|
|
||||||
every directory it would rmdir (a directory is removed only once every entry
|
|
||||||
below it has been removed and nothing the walker leaves in place survives).
|
|
||||||
Entries the walker never removes (symlinks, manifest-listed files, protected
|
|
||||||
prefixes) mark the enclosing directory as surviving, exactly as they would
|
|
||||||
make a real rmdir fail with ENOTEMPTY. Stops early once *count reaches the
|
|
||||||
cap (sets *exceeds). Returns false on a traversal error. */
|
|
||||||
static bool count_extras_fd(int dirfd, const char* rel_path, const PathIndex* keep, size_t cap,
|
|
||||||
size_t* count, bool* exceeds, const DeleteSkipEntry* skips,
|
|
||||||
int skip_count, bool* survives) {
|
|
||||||
/* openat(dirfd, ".") opens an independent file description: a dup() would
|
|
||||||
share dirfd's file offset, and a prior rehearsal pass must not have drained
|
|
||||||
this directory's stream before the delete pass reads it again. */
|
|
||||||
int scanfd = openat(dirfd, ".", O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
|
||||||
if (scanfd < 0)
|
|
||||||
return false;
|
|
||||||
DIR* dir = fdopendir(scanfd);
|
|
||||||
if (!dir) {
|
|
||||||
close(scanfd);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
bool operation_ok = true;
|
|
||||||
bool local_survives = false;
|
|
||||||
bool at_root = rel_path[0] == '\0';
|
|
||||||
const struct dirent* entry;
|
|
||||||
while ((entry = readdir(dir)) != NULL) {
|
|
||||||
if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
|
|
||||||
continue;
|
|
||||||
if (*exceeds)
|
|
||||||
break;
|
|
||||||
char* child_rel = path_cat((char*)rel_path, entry->d_name);
|
|
||||||
if (!child_rel) {
|
|
||||||
operation_ok = false;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (path_under_skip_prefix(child_rel, at_root, skips, skip_count)) {
|
|
||||||
local_survives = true;
|
|
||||||
free(child_rel);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
struct stat st;
|
|
||||||
if (fstatat(dirfd, entry->d_name, &st, AT_SYMLINK_NOFOLLOW) != 0) {
|
|
||||||
if (errno != ENOENT)
|
|
||||||
operation_ok = false;
|
|
||||||
free(child_rel);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (S_ISLNK(st.st_mode)) {
|
|
||||||
local_survives = true;
|
|
||||||
free(child_rel);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (S_ISDIR(st.st_mode)) {
|
|
||||||
int childfd = openat(dirfd, entry->d_name, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
|
||||||
bool child_ok = true;
|
|
||||||
bool child_survives = true;
|
|
||||||
if (childfd >= 0) {
|
|
||||||
child_ok = count_extras_fd(childfd, child_rel, keep, cap, count, exceeds, skips, skip_count,
|
|
||||||
&child_survives);
|
|
||||||
close(childfd);
|
|
||||||
} else if (errno != ENOENT) {
|
|
||||||
operation_ok = false;
|
|
||||||
}
|
|
||||||
if (!child_ok)
|
|
||||||
operation_ok = false;
|
|
||||||
if (keep_is_dir(keep, child_rel)) {
|
|
||||||
/* A directory with kept content below it is never removed. */
|
|
||||||
local_survives = true;
|
|
||||||
} else if (child_survives) {
|
|
||||||
/* The directory still holds entries the walker leaves in place, so an
|
|
||||||
rmdir would fail with ENOTEMPTY; the delete pass leaves it behind
|
|
||||||
rather than reporting an error (matching rsync). */
|
|
||||||
local_survives = true;
|
|
||||||
} else {
|
|
||||||
if (*count >= cap) {
|
|
||||||
*exceeds = true;
|
|
||||||
} else {
|
|
||||||
(*count)++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
bool found = keep_is_file(keep, child_rel);
|
|
||||||
if (!found) {
|
|
||||||
if (*count >= cap) {
|
|
||||||
*exceeds = true;
|
|
||||||
} else {
|
|
||||||
(*count)++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
free(child_rel);
|
|
||||||
}
|
|
||||||
closedir(dir);
|
|
||||||
*survives = local_survives;
|
|
||||||
return operation_ok;
|
|
||||||
}
|
|
||||||
|
|
||||||
static bool delete_extras_fd(int dirfd, const char* rel_path, const PathIndex* keep,
|
|
||||||
size_t max_delete, size_t* deleted_count, const DeleteSkipEntry* skips,
|
|
||||||
int skip_count) {
|
|
||||||
/* Independent file description (see count_extras_fd). */
|
|
||||||
int scanfd = openat(dirfd, ".", O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
|
||||||
if (scanfd < 0)
|
|
||||||
return false;
|
|
||||||
DIR* dir = fdopendir(scanfd);
|
|
||||||
if (!dir) {
|
|
||||||
close(scanfd);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
bool operation_ok = true;
|
|
||||||
const struct dirent* entry;
|
|
||||||
while ((entry = readdir(dir)) != NULL) {
|
|
||||||
if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
|
|
||||||
continue;
|
|
||||||
char* child_rel = path_cat((char*)rel_path, entry->d_name);
|
|
||||||
if (!child_rel) {
|
|
||||||
operation_ok = false;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
/* A --delay-updates run keeps its staging directory as a direct child of
|
|
||||||
the receive root, and basis-dir snapshots live below it too. Their
|
|
||||||
contents are not manifest entries, so descending into them would delete
|
|
||||||
every staged / basis file as an "extra". Only the staging name (a
|
|
||||||
top-level-only prefix) and the basis prefixes are protected: a nested
|
|
||||||
destination directory that happens to be called .fastsync-stage is
|
|
||||||
ordinary content. */
|
|
||||||
if (path_under_skip_prefix(child_rel, rel_path[0] == '\0', skips, skip_count)) {
|
|
||||||
free(child_rel);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
struct stat st;
|
|
||||||
if (fstatat(dirfd, entry->d_name, &st, AT_SYMLINK_NOFOLLOW) != 0) {
|
|
||||||
if (errno != ENOENT)
|
|
||||||
operation_ok = false;
|
|
||||||
free(child_rel);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
// Skip symlinks to prevent following them outside the destination tree
|
|
||||||
if (S_ISLNK(st.st_mode)) {
|
|
||||||
free(child_rel);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (S_ISDIR(st.st_mode)) {
|
|
||||||
int childfd = openat(dirfd, entry->d_name, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
|
||||||
bool child_removed = false;
|
|
||||||
if (childfd >= 0) {
|
|
||||||
child_removed = delete_extras_fd(childfd, child_rel, keep, max_delete, deleted_count, skips,
|
|
||||||
skip_count);
|
|
||||||
if (!child_removed)
|
|
||||||
operation_ok = false;
|
|
||||||
close(childfd);
|
|
||||||
} else if (errno != ENOENT) {
|
|
||||||
operation_ok = false;
|
|
||||||
}
|
|
||||||
if (child_removed && !keep_is_dir(keep, child_rel)) {
|
|
||||||
if (*deleted_count >= max_delete) {
|
|
||||||
operation_ok = false;
|
|
||||||
} else {
|
|
||||||
if (unlinkat(dirfd, entry->d_name, AT_REMOVEDIR) != 0) {
|
|
||||||
/* ENOENT: already gone (fine). ENOTEMPTY/EEXIST: the directory
|
|
||||||
still holds entries the walker leaves in place (a protected
|
|
||||||
excluded prefix, a kept file the manifest protects, a symlink);
|
|
||||||
rsync leaves such a directory behind, so this is not an error.
|
|
||||||
Only genuine I/O failures abort the deletion. */
|
|
||||||
if (errno != ENOENT && errno != ENOTEMPTY && errno != EEXIST)
|
|
||||||
operation_ok = false;
|
|
||||||
} else {
|
|
||||||
(*deleted_count)++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// Check if relative path is in manifest
|
|
||||||
bool found = keep_is_file(keep, child_rel);
|
|
||||||
if (!found) {
|
|
||||||
if (*deleted_count >= max_delete) {
|
|
||||||
operation_ok = false;
|
|
||||||
free(child_rel);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (unlinkat(dirfd, entry->d_name, 0) != 0) {
|
|
||||||
if (errno != ENOENT)
|
|
||||||
operation_ok = false;
|
|
||||||
} else {
|
|
||||||
(*deleted_count)++;
|
|
||||||
}
|
|
||||||
char* escaped_path = output_escape(child_rel, log_get_8_bit_output());
|
|
||||||
fprintf(stderr, " Deleted: %s\n", escaped_path ? escaped_path : "<allocation failed>");
|
|
||||||
free(escaped_path);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
free(child_rel);
|
|
||||||
}
|
|
||||||
closedir(dir);
|
|
||||||
return operation_ok;
|
|
||||||
}
|
|
||||||
|
|
||||||
DeleteWalkResult delete_extras_limited(const char* dest_root, const ArrayList* manifest,
|
|
||||||
size_t max_delete, const DeleteSkipEntry* skips,
|
|
||||||
int skip_count, size_t* deleted_out) {
|
|
||||||
if (deleted_out)
|
|
||||||
*deleted_out = 0;
|
|
||||||
if (!manifest)
|
|
||||||
return DELETE_WALK_ERROR;
|
|
||||||
/* Index the keep-set once so both passes answer membership in O(path length)
|
|
||||||
instead of scanning every manifest entry for every destination entry. */
|
|
||||||
PathIndex keep;
|
|
||||||
if (!build_keep_index(manifest, &keep))
|
|
||||||
return DELETE_WALK_ERROR;
|
|
||||||
int rootfd;
|
|
||||||
int root_fd = utils_get_authorized_root_fd();
|
|
||||||
if (root_fd >= 0) {
|
|
||||||
if (utils_get_authorized_root_path())
|
|
||||||
rootfd = open_authorized_destination(dest_root);
|
|
||||||
else if (dest_root == NULL)
|
|
||||||
rootfd = dup(root_fd);
|
|
||||||
else
|
|
||||||
rootfd = -1;
|
|
||||||
} else {
|
|
||||||
rootfd = open(dest_root, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
|
||||||
}
|
|
||||||
if (rootfd < 0) {
|
|
||||||
path_index_free(&keep);
|
|
||||||
return DELETE_WALK_ERROR;
|
|
||||||
}
|
|
||||||
if (max_delete != SIZE_MAX) {
|
|
||||||
/* Rehearse the deletion first so a run that would exceed the cap removes
|
|
||||||
nothing (rsync's all-or-nothing --max-delete contract). */
|
|
||||||
size_t count = 0;
|
|
||||||
bool exceeds = false;
|
|
||||||
bool survives = false;
|
|
||||||
bool counted_ok = count_extras_fd(rootfd, "", &keep, max_delete, &count, &exceeds, skips,
|
|
||||||
skip_count, &survives);
|
|
||||||
if (!counted_ok) {
|
|
||||||
close(rootfd);
|
|
||||||
path_index_free(&keep);
|
|
||||||
return DELETE_WALK_ERROR;
|
|
||||||
}
|
|
||||||
if (exceeds) {
|
|
||||||
close(rootfd);
|
|
||||||
path_index_free(&keep);
|
|
||||||
return DELETE_WALK_LIMIT_EXCEEDED;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
size_t deleted_count = 0;
|
|
||||||
bool ok = delete_extras_fd(rootfd, "", &keep, max_delete, &deleted_count, skips, skip_count);
|
|
||||||
if (close(rootfd) != 0)
|
|
||||||
ok = false;
|
|
||||||
path_index_free(&keep);
|
|
||||||
if (deleted_out)
|
|
||||||
*deleted_out = deleted_count;
|
|
||||||
return ok ? DELETE_WALK_OK : DELETE_WALK_ERROR;
|
|
||||||
}
|
|
||||||
|
|
||||||
bool delete_extras(const char* dest_root, const ArrayList* manifest) {
|
|
||||||
return delete_extras_limited(dest_root, manifest, SIZE_MAX, NULL, 0, NULL) == DELETE_WALK_OK;
|
|
||||||
}
|
|
||||||
|
|
||||||
bool has_path_traversal(const char* path) {
|
bool has_path_traversal(const char* path) {
|
||||||
if (!path)
|
if (!path)
|
||||||
return true;
|
return true;
|
||||||
|
|||||||
+28
-42
@@ -2,6 +2,7 @@
|
|||||||
#define UTILS_H
|
#define UTILS_H
|
||||||
|
|
||||||
#include "array_list.h"
|
#include "array_list.h"
|
||||||
|
#include "filter.h"
|
||||||
#include <stddef.h>
|
#include <stddef.h>
|
||||||
#include <stdbool.h>
|
#include <stdbool.h>
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
@@ -81,6 +82,17 @@ bool path_index_has_descendant(const PathIndex* index, const char* path);
|
|||||||
|
|
||||||
char* str_dup(const char* string);
|
char* str_dup(const char* string);
|
||||||
char* output_escape(const char* string, bool eight_bit_output);
|
char* output_escape(const char* string, bool eight_bit_output);
|
||||||
|
|
||||||
|
/* Resolve the first supported name from a rsync algorithm-preference
|
||||||
|
* environment variable (RSYNC_COMPRESS_LIST / RSYNC_CHECKSUM_LIST). `resolve`
|
||||||
|
* maps a case-insensitive name to an algorithm id (>= 0) or -1 for an unknown
|
||||||
|
* name. rsync's syntax is a whitespace-separated list (comma/colon are NOT
|
||||||
|
* separators); the client-side half ends at '&'. Unknown entries are skipped
|
||||||
|
* and the first resolvable one wins. *specified is set true when the variable
|
||||||
|
* holds at least one non-blank character. Returns the first resolvable id, or
|
||||||
|
* -1 when the variable is unset/blank or names no supported algorithm. */
|
||||||
|
int env_choice_first(const char* env_name, int (*resolve)(const char*), bool* specified);
|
||||||
|
|
||||||
/* Upper bound on one line/token read from a local list file (--files-from,
|
/* Upper bound on one line/token read from a local list file (--files-from,
|
||||||
* --exclude-from/--include-from, .rsync-filter). Mirrors MAX_STRING_SIZE and
|
* --exclude-from/--include-from, .rsync-filter). Mirrors MAX_STRING_SIZE and
|
||||||
* stops a hostile multi-gigabyte line from forcing unbounded allocation. */
|
* stops a hostile multi-gigabyte line from forcing unbounded allocation. */
|
||||||
@@ -94,48 +106,12 @@ char* output_escape(const char* string, bool eight_bit_output);
|
|||||||
ssize_t utils_getdelim_bounded(FILE* stream, char** line, size_t* cap, int delim, size_t max_len);
|
ssize_t utils_getdelim_bounded(FILE* stream, char** line, size_t* cap, int delim, size_t max_len);
|
||||||
char* path_cat(const char* path1, const char* path2);
|
char* path_cat(const char* path1, const char* path2);
|
||||||
bool glob_match(const char* pattern, const char* str);
|
bool glob_match(const char* pattern, const char* str);
|
||||||
/* Result of a bounded extra-file deletion run. */
|
|
||||||
typedef enum {
|
/* Open the existing destination directory at `dest_root`, confined to the
|
||||||
/* Every extra entry was removed (or there were none). */
|
authorized root with an O_NOFOLLOW component walk (the same confinement the
|
||||||
DELETE_WALK_OK = 0,
|
deletion walker uses for its root). Returns a new fd the caller owns, or -1
|
||||||
/* The destination holds more extras than the numeric cap for this run. With
|
on error (including a destination that does not exist). */
|
||||||
the all-or-nothing max-delete semantics NOTHING was removed (the walker
|
int utils_open_authorized_destination(const char* dest_root);
|
||||||
counts first and refuses to start when the run would exceed the limit). */
|
|
||||||
DELETE_WALK_LIMIT_EXCEEDED,
|
|
||||||
/* A traversal or unlink failure aborted the deletion (partial removal is
|
|
||||||
possible, mirroring the delete pass). */
|
|
||||||
DELETE_WALK_ERROR
|
|
||||||
} DeleteWalkResult;
|
|
||||||
/* One protected entry for the delete walker. When top_level_only is true the
|
|
||||||
prefix is skipped only as a DIRECT child of dest_root (the --delay-updates
|
|
||||||
staging directory, which must not hide genuine extras inside a nested
|
|
||||||
destination directory that happens to share the staging name); otherwise the
|
|
||||||
prefix is skipped at any depth (the --compare-dest/--copy-dest/--link-dest
|
|
||||||
basis trees, and the sender-side protected filter-excluded prefixes, which
|
|
||||||
are never destination content). */
|
|
||||||
typedef struct {
|
|
||||||
const char* prefix;
|
|
||||||
bool top_level_only;
|
|
||||||
} DeleteSkipEntry;
|
|
||||||
/* True when child_rel is, or lies below, one of the protected entries (a prefix
|
|
||||||
"a" protects "a" and "a/b/c" but not "ab"; top_level_only entries protect
|
|
||||||
only DIRECT children of the destination root, i.e. child_rel has no '/'). */
|
|
||||||
bool path_under_skip_prefix(const char* child_rel, bool at_root, const DeleteSkipEntry* skips,
|
|
||||||
int skip_count);
|
|
||||||
/* Remove files/dirs under dest_root that are not listed in manifest without
|
|
||||||
ever descending into a protected prefix (see DeleteSkipEntry). When
|
|
||||||
max_delete is not SIZE_MAX the run is all-or-nothing: extras are counted
|
|
||||||
first and DELETE_WALK_LIMIT_EXCEEDED is returned (with nothing removed) when
|
|
||||||
the count would exceed the cap. `deleted_out` optionally receives the number
|
|
||||||
of entries actually removed. The all-or-nothing guarantee holds only while
|
|
||||||
the destination tree is not being concurrently modified: the rehearsal pass
|
|
||||||
and the delete pass are two separate walks, so a concurrent change between
|
|
||||||
them (another process adding/removing entries) can make the second pass
|
|
||||||
delete a different set than the first one counted. */
|
|
||||||
DeleteWalkResult delete_extras_limited(const char* dest_root, const ArrayList* manifest,
|
|
||||||
size_t max_delete, const DeleteSkipEntry* skips,
|
|
||||||
int skip_count, size_t* deleted_out);
|
|
||||||
bool delete_extras(const char* dest_root, const ArrayList* manifest);
|
|
||||||
bool utils_set_authorized_root(int fd, const char* canonical_path);
|
bool utils_set_authorized_root(int fd, const char* canonical_path);
|
||||||
/* The fd-only compatibility form is fail-closed for path-based operations;
|
/* The fd-only compatibility form is fail-closed for path-based operations;
|
||||||
* callers should use utils_set_authorized_root with the canonical identity. */
|
* callers should use utils_set_authorized_root with the canonical identity. */
|
||||||
@@ -155,12 +131,22 @@ void utils_set_authorized_root_fd(int fd);
|
|||||||
* threads spawn; see utils.c). */
|
* threads spawn; see utils.c). */
|
||||||
int utils_get_authorized_root_fd(void);
|
int utils_get_authorized_root_fd(void);
|
||||||
const char* utils_get_authorized_root_path(void);
|
const char* utils_get_authorized_root_path(void);
|
||||||
|
/* Write a diagnostic message into a caller-supplied buffer, mirroring
|
||||||
|
* vsnprintf. A NULL `err` or a zero `err_size` is a no-op, so a caller that
|
||||||
|
* only needs the boolean status may safely pass NULL. Returns nothing; the
|
||||||
|
* buffer is always NUL-terminated by vsnprintf when err_size > 0. */
|
||||||
|
void utils_set_error(char* err, size_t err_size, const char* fmt, ...);
|
||||||
/* True when `path` is `root` itself or lies directly beneath it: a lexical
|
/* True when `path` is `root` itself or lies directly beneath it: a lexical
|
||||||
* prefix test requiring the byte after `root` to be '\0' or '/'. Both `root`
|
* prefix test requiring the byte after `root` to be '\0' or '/'. Both `root`
|
||||||
* and `path` must be absolute canonical paths free of "."/".." components (the
|
* and `path` must be absolute canonical paths free of "."/".." components (the
|
||||||
* callers guarantee this); this is containment by string, not by resolved
|
* callers guarantee this); this is containment by string, not by resolved
|
||||||
* symlinks. Shared by the utils and file secure-walk root confinement. */
|
* symlinks. Shared by the utils and file secure-walk root confinement. */
|
||||||
bool path_is_within_root(const char* root, const char* path);
|
bool path_is_within_root(const char* root, const char* path);
|
||||||
|
/* Non-allocating transfer-relative view of `path`: strip any leading '/' and
|
||||||
|
* then a `root` prefix (leading/trailing slashes tolerated), returning a
|
||||||
|
* borrowed pointer into `path`. A NULL/empty root, or a path not under
|
||||||
|
* `root`, yields just the leading-slash strip. `path`/`root` must stay alive. */
|
||||||
|
const char* utils_strip_transfer_root(const char* path, const char* root);
|
||||||
/* True when `path` contains a ".." component. This is a purely lexical
|
/* True when `path` contains a ".." component. This is a purely lexical
|
||||||
* dot-dot check: an absolute path is NOT rejected here, because default
|
* dot-dot check: an absolute path is NOT rejected here, because default
|
||||||
* (non-relative) transfers legitimately put the sender's absolute source path
|
* (non-relative) transfers legitimately put the sender's absolute source path
|
||||||
|
|||||||
+194
-61
@@ -2,10 +2,12 @@
|
|||||||
#include "xattr.h"
|
#include "xattr.h"
|
||||||
#include "identity.h"
|
#include "identity.h"
|
||||||
#include "log.h"
|
#include "log.h"
|
||||||
|
#include "metadata.h"
|
||||||
#include "protocol.h"
|
#include "protocol.h"
|
||||||
#include "utils.h"
|
#include "utils.h"
|
||||||
#include "file_types.h"
|
#include "file_types.h"
|
||||||
#include <errno.h>
|
#include <errno.h>
|
||||||
|
#include <limits.h>
|
||||||
#include <stdint.h>
|
#include <stdint.h>
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
@@ -37,6 +39,22 @@ void xattr_list_free(FileXattrList* list) {
|
|||||||
free(list);
|
free(list);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
FileXattrList* xattr_list_clone(const FileXattrList* list) {
|
||||||
|
if (!list)
|
||||||
|
return NULL;
|
||||||
|
FileXattrList* clone = xattr_list_new();
|
||||||
|
if (!clone)
|
||||||
|
return NULL;
|
||||||
|
for (int i = 0; i < list->count; i++) {
|
||||||
|
if (!xattr_list_append(clone, list->items[i].name, list->items[i].value,
|
||||||
|
list->items[i].value_len)) {
|
||||||
|
xattr_list_free(clone);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return clone;
|
||||||
|
}
|
||||||
|
|
||||||
bool xattr_list_append(FileXattrList* list, const char* name, const void* value, size_t value_len) {
|
bool xattr_list_append(FileXattrList* list, const char* name, const void* value, size_t value_len) {
|
||||||
if (!list || !name || (!value && value_len != 0))
|
if (!list || !name || (!value && value_len != 0))
|
||||||
return false;
|
return false;
|
||||||
@@ -116,16 +134,23 @@ static bool xattr_name_is_posix_acl(const char* name) {
|
|||||||
|
|
||||||
/* ---- SENDER: capture ---- */
|
/* ---- SENDER: capture ---- */
|
||||||
|
|
||||||
FileXattrList* xattr_capture_path(const char* path, bool preserve_acls) {
|
/* The two syscall families differ only in whether the FINAL component is
|
||||||
|
* followed (`listxattr`/`getxattr` follow; `llistxattr`/`lgetxattr` do not), so
|
||||||
|
* one common implementation backs both public entry points. */
|
||||||
|
typedef ssize_t (*XattrListFn)(const char* path, char* list, size_t size);
|
||||||
|
typedef ssize_t (*XattrGetFn)(const char* path, const char* name, void* value, size_t size);
|
||||||
|
|
||||||
|
static FileXattrList* xattr_capture_common(const char* path, bool preserve_acls,
|
||||||
|
XattrListFn list_fn, XattrGetFn get_fn) {
|
||||||
if (!path)
|
if (!path)
|
||||||
return NULL;
|
return NULL;
|
||||||
ssize_t list_size = listxattr(path, NULL, 0);
|
ssize_t list_size = list_fn(path, NULL, 0);
|
||||||
if (list_size <= 0)
|
if (list_size <= 0)
|
||||||
return NULL; /* no xattrs, ENOTSUP, or error: nothing appliable */
|
return NULL; /* no xattrs, ENOTSUP, or error: nothing appliable */
|
||||||
char* names = malloc((size_t)list_size);
|
char* names = malloc((size_t)list_size);
|
||||||
if (!names)
|
if (!names)
|
||||||
return NULL;
|
return NULL;
|
||||||
ssize_t got = listxattr(path, names, (size_t)list_size);
|
ssize_t got = list_fn(path, names, (size_t)list_size);
|
||||||
if (got < 0) {
|
if (got < 0) {
|
||||||
free(names);
|
free(names);
|
||||||
return NULL;
|
return NULL;
|
||||||
@@ -148,7 +173,7 @@ FileXattrList* xattr_capture_path(const char* path, bool preserve_acls) {
|
|||||||
negotiated. Without it a plain -X capture never carries an ACL. */
|
negotiated. Without it a plain -X capture never carries an ACL. */
|
||||||
if (!xattr_name_appliable(name, preserve_acls))
|
if (!xattr_name_appliable(name, preserve_acls))
|
||||||
continue;
|
continue;
|
||||||
ssize_t value_size = getxattr(path, name, NULL, 0);
|
ssize_t value_size = get_fn(path, name, NULL, 0);
|
||||||
if (value_size < 0)
|
if (value_size < 0)
|
||||||
continue;
|
continue;
|
||||||
if (value_size > XATTR_VALUE_MAX)
|
if (value_size > XATTR_VALUE_MAX)
|
||||||
@@ -161,7 +186,7 @@ FileXattrList* xattr_capture_path(const char* path, bool preserve_acls) {
|
|||||||
free(names);
|
free(names);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
ssize_t read_len = getxattr(path, name, buffer, (size_t)value_size);
|
ssize_t read_len = get_fn(path, name, buffer, (size_t)value_size);
|
||||||
if (read_len < 0 || read_len != value_size) {
|
if (read_len < 0 || read_len != value_size) {
|
||||||
free(buffer);
|
free(buffer);
|
||||||
continue;
|
continue;
|
||||||
@@ -183,6 +208,14 @@ FileXattrList* xattr_capture_path(const char* path, bool preserve_acls) {
|
|||||||
return list;
|
return list;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
FileXattrList* xattr_capture_path(const char* path, bool preserve_acls) {
|
||||||
|
return xattr_capture_common(path, preserve_acls, listxattr, getxattr);
|
||||||
|
}
|
||||||
|
|
||||||
|
FileXattrList* xattr_capture_path_nofollow(const char* path, bool preserve_acls) {
|
||||||
|
return xattr_capture_common(path, preserve_acls, llistxattr, lgetxattr);
|
||||||
|
}
|
||||||
|
|
||||||
/* ---- WIRE ---- */
|
/* ---- WIRE ---- */
|
||||||
|
|
||||||
bool xattr_send(int fd, const FileXattrList* list) {
|
bool xattr_send(int fd, const FileXattrList* list) {
|
||||||
@@ -346,16 +379,74 @@ bool xattr_apply_fd(int fd, const FileXattrList* list) {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* ---- --fake-super: park ownership/mode/mtime in a reserved xattr ---- */
|
/* Symlink counterpart of xattr_apply_fd(): target the link ITSELF, never its
|
||||||
|
* referent. fsetxattr cannot be used (no *at xattr syscall exists, and the
|
||||||
|
* kernel rejects xattr syscalls on an O_PATH descriptor), so the already-open,
|
||||||
|
* confinement-checked parent directory is addressed through /proc/self/fd and
|
||||||
|
* the final component is applied with lsetxattr, which does not follow it.
|
||||||
|
*
|
||||||
|
* The list is trusted to come from xattr_receive() (already whitelisted), but
|
||||||
|
* every name is re-validated here so this path-based primitive is confined on
|
||||||
|
* its own -- this is the only apply primitive that addresses a path, and the
|
||||||
|
* header promises a whitelisted apply. The apply is best-effort: if /proc is
|
||||||
|
* not mounted (the anchor cannot be formed) or the kernel refuses the set, the
|
||||||
|
* failure is skipped and never fails the transfer. See xattr.h for the bounded
|
||||||
|
* residual TOCTOU between link creation and lsetxattr. */
|
||||||
|
bool xattr_apply_path_nofollow(int parent_fd, const char* leaf, const FileXattrList* list,
|
||||||
|
bool preserve_acls) {
|
||||||
|
if (parent_fd < 0 || !leaf || leaf[0] == '\0' || strchr(leaf, '/') != NULL || !list)
|
||||||
|
return false;
|
||||||
|
if (list->count == 0)
|
||||||
|
return true;
|
||||||
|
char prefix[64];
|
||||||
|
int prefix_len = snprintf(prefix, sizeof(prefix), "/proc/self/fd/%d/", parent_fd);
|
||||||
|
if (prefix_len < 0 || (size_t)prefix_len >= sizeof(prefix))
|
||||||
|
return false;
|
||||||
|
size_t leaf_len = strlen(leaf);
|
||||||
|
char* path = malloc((size_t)prefix_len + leaf_len + 1);
|
||||||
|
if (!path)
|
||||||
|
return false;
|
||||||
|
memcpy(path, prefix, (size_t)prefix_len);
|
||||||
|
memcpy(path + prefix_len, leaf, leaf_len + 1);
|
||||||
|
bool warned = false;
|
||||||
|
int first_errno = 0;
|
||||||
|
for (int i = 0; i < list->count; i++) {
|
||||||
|
const FileXattr* xa = &list->items[i];
|
||||||
|
/* Defense in depth: re-validate against the receiver's full whitelist, so a
|
||||||
|
hand-crafted list can never apply a privileged namespace or the reserved
|
||||||
|
--fake-super key through this path-based primitive. */
|
||||||
|
if (!xattr_name_appliable(xa->name, preserve_acls))
|
||||||
|
continue;
|
||||||
|
if (lsetxattr(path, xa->name, xa->value, xa->value_len, 0) != 0) {
|
||||||
|
if (!warned) {
|
||||||
|
warned = true;
|
||||||
|
first_errno = errno;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (warned)
|
||||||
|
log_message(LOG_LEVEL_WARNING,
|
||||||
|
"could not set one or more xattrs on the destination symlink: %s",
|
||||||
|
strerror(first_errno));
|
||||||
|
free(path);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
void fake_super_store_fd(int fd, uint32_t uid, uint32_t gid, uint32_t mode, int64_t mtime_sec,
|
/* ---- --fake-super: park ownership/mode/rdev in a reserved xattr ---- */
|
||||||
int64_t mtime_nsec) {
|
|
||||||
|
void fake_super_store_fd(int fd, uint32_t uid, uint32_t gid, uint32_t mode, uint32_t rdev_major,
|
||||||
|
uint32_t rdev_minor) {
|
||||||
if (fd < 0)
|
if (fd < 0)
|
||||||
return;
|
return;
|
||||||
char record[128];
|
/* rsync 3.4.1's exact grammar: "<octal full st_mode> <rdev_major>,<rdev_minor>
|
||||||
int len =
|
* <uid>:<gid>". The octal mode carries the S_IFMT bits (e.g. 0104711 for a
|
||||||
snprintf(record, sizeof(record), "%lu:%lu:%03o:%lld:%ld", (unsigned long)uid,
|
* setuid regular file, 020644 for a char device); the rdev pair is 0,0 for a
|
||||||
(unsigned long)gid, (unsigned)mode & 0777U, (long long)mtime_sec, (long)mtime_nsec);
|
* non-device. No mtime field: rsync leaves the file's own timestamp in
|
||||||
|
* charge of mtime. This value is what rsync reads back to restore a
|
||||||
|
* fake-super tree, so the field order and separators must not change. */
|
||||||
|
char record[96];
|
||||||
|
int len = snprintf(record, sizeof(record), "%o %u,%u %u:%u", (unsigned)mode, (unsigned)rdev_major,
|
||||||
|
(unsigned)rdev_minor, (unsigned)uid, (unsigned)gid);
|
||||||
if (len <= 0 || (size_t)len >= sizeof(record))
|
if (len <= 0 || (size_t)len >= sizeof(record))
|
||||||
return;
|
return;
|
||||||
if (fsetxattr(fd, FAKESUPER_XATTR, record, (size_t)len, 0) != 0) {
|
if (fsetxattr(fd, FAKESUPER_XATTR, record, (size_t)len, 0) != 0) {
|
||||||
@@ -364,26 +455,65 @@ void fake_super_store_fd(int fd, uint32_t uid, uint32_t gid, uint32_t mode, int6
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/* --fake-super replay: read the freshly-stored record and re-apply the source
|
/* Parse rsync's `user.rsync.%stat` grammar strictly:
|
||||||
* stat fd-relative. A privileged (root) run can actually change the owner;
|
* "<octal st_mode> <rdev_major>,<rdev_minor> <uid>:<gid>"
|
||||||
* a non-root run silently skips the fchown on EPERM/EACCES (never fatal,
|
* Every field is parsed with strtoul() so an out-of-range value is a clean
|
||||||
* mirroring the normal metadata identity path; other errors are logged) and
|
* rejection rather than the undefined behavior sscanf("%u") exhibited, each
|
||||||
* still applies mode/mtime where permitted.
|
* field is range-checked against the same bounds the wire validator uses, and
|
||||||
*
|
* the whole record must be consumed (only trailing whitespace is tolerated) so
|
||||||
* The OWNER leg additionally honors three policies:
|
* trailing garbage is refused. Returns false on any malformed input. */
|
||||||
* - an explicit ownership identity policy must be active (numeric-ids /
|
static bool fake_super_parse_stat(const char* record, unsigned* mode_out, unsigned* rdev_major_out,
|
||||||
* chown / usermap / groupmap / copy-as). --fake-super on its own only
|
unsigned* rdev_minor_out, unsigned* uid_out, unsigned* gid_out) {
|
||||||
* RECORDS the source owner; replaying that owner as a live chown without an
|
if (!record)
|
||||||
* explicit ownership opt-in would be an un-gated client-chosen-ownership
|
return false;
|
||||||
* primitive.
|
char* end = NULL;
|
||||||
* - --no-super (privilege_super_permitted() false) suppresses it even for a
|
const char* p = record;
|
||||||
* root receiver, exactly like the normal metadata identity path.
|
errno = 0;
|
||||||
* - an active --copy-as is AUTHORITATIVE: the identity path already forced the
|
unsigned long mode = strtoul(p, &end, 8);
|
||||||
* target owner, so replaying the recorded source owner here would silently
|
if (errno != 0 || end == p || mode > (unsigned long)UINT_MAX || *end != ' ')
|
||||||
* override it. The xattr record is still stored/replayed for a later
|
return false;
|
||||||
* privileged restore; only the live chown is skipped. Mode/mtime remain
|
p = end + 1;
|
||||||
* applied either way so unprivileged --fake-super still works. */
|
errno = 0;
|
||||||
bool fake_super_restore_fd(int fd) {
|
unsigned long rdev_major = strtoul(p, &end, 10);
|
||||||
|
if (errno != 0 || end == p || rdev_major > 0xffffUL || *end != ',')
|
||||||
|
return false;
|
||||||
|
p = end + 1;
|
||||||
|
errno = 0;
|
||||||
|
unsigned long rdev_minor = strtoul(p, &end, 10);
|
||||||
|
if (errno != 0 || end == p || rdev_minor > 0x00ffffffUL || *end != ' ')
|
||||||
|
return false;
|
||||||
|
p = end + 1;
|
||||||
|
errno = 0;
|
||||||
|
unsigned long uid = strtoul(p, &end, 10);
|
||||||
|
if (errno != 0 || end == p || uid > (unsigned long)UINT_MAX || *end != ':')
|
||||||
|
return false;
|
||||||
|
p = end + 1;
|
||||||
|
errno = 0;
|
||||||
|
unsigned long gid = strtoul(p, &end, 10);
|
||||||
|
if (errno != 0 || end == p || gid > (unsigned long)UINT_MAX)
|
||||||
|
return false;
|
||||||
|
p = end;
|
||||||
|
while (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r')
|
||||||
|
p++;
|
||||||
|
if (*p != '\0')
|
||||||
|
return false;
|
||||||
|
*mode_out = (unsigned)mode;
|
||||||
|
*rdev_major_out = (unsigned)rdev_major;
|
||||||
|
*rdev_minor_out = (unsigned)rdev_minor;
|
||||||
|
*uid_out = (unsigned)uid;
|
||||||
|
*gid_out = (unsigned)gid;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* --fake-super replay: read the freshly-stored record and re-apply its
|
||||||
|
* permission bits fd-relative. The recorded uid/gid are retained for a later
|
||||||
|
* privileged restore but are NEVER chowned here: --fake-super only RECORDS
|
||||||
|
* ownership, it must not real-chown the recorded (resolved) owner. The
|
||||||
|
* recorded rdev is likewise parsed for grammar compatibility but is not acted
|
||||||
|
* on (device recreation is a separate, privilege-gated path). mtime is not in
|
||||||
|
* the record: the normal metadata path applies it (policy.times), exactly as
|
||||||
|
* rsync relies on the file's own timestamp. */
|
||||||
|
bool fake_super_restore_fd(int fd, FileAttrPolicy policy) {
|
||||||
if (fd < 0)
|
if (fd < 0)
|
||||||
return false;
|
return false;
|
||||||
char record[128];
|
char record[128];
|
||||||
@@ -391,35 +521,38 @@ bool fake_super_restore_fd(int fd) {
|
|||||||
if (len < 0)
|
if (len < 0)
|
||||||
return false; /* absent or filesystem without xattrs: silent no-op */
|
return false; /* absent or filesystem without xattrs: silent no-op */
|
||||||
record[len] = '\0';
|
record[len] = '\0';
|
||||||
unsigned long ul_uid, ul_gid, ul_mode;
|
unsigned ul_mode, rdev_major, rdev_minor, ul_uid, ul_gid;
|
||||||
long long mtime_sec;
|
if (!fake_super_parse_stat(record, &ul_mode, &rdev_major, &rdev_minor, &ul_uid, &ul_gid))
|
||||||
long mtime_nsec;
|
|
||||||
if (sscanf(record, "%lu:%lu:%lo:%lld:%ld", &ul_uid, &ul_gid, &ul_mode, &mtime_sec, &mtime_nsec) !=
|
|
||||||
5)
|
|
||||||
return false; /* malformed record: skip, never fatal */
|
return false; /* malformed record: skip, never fatal */
|
||||||
|
|
||||||
/* Owner is applied best-effort only: a non-root process cannot chown and
|
/* --fake-super NEVER performs a real chown: that would defeat the whole point
|
||||||
must not abort the transfer for that reason (FastSync identity philosophy).
|
of the flag (record privileged ownership on an unprivileged receiver for a
|
||||||
EPERM/EACCES (expected for a non-root receiver) are skipped silently; a
|
later privileged restore). The uid/gid parsed above are retained in the
|
||||||
genuine EINVAL (an impossible stored id) is logged so the corruption is
|
record for that later restore, but no ownership change happens here. The
|
||||||
not hidden. --no-super suppresses the owner leg even for root, and an
|
rdev is retained for the same reason. */
|
||||||
active --copy-as is authoritative so its forced owner must not be
|
(void)rdev_major;
|
||||||
overwritten by the recorded source owner. */
|
(void)rdev_minor;
|
||||||
if (identity_active_enabled() && privilege_super_permitted() && !identity_copy_as_active() &&
|
(void)ul_uid;
|
||||||
fchown(fd, (uid_t)ul_uid, (gid_t)ul_gid) != 0 && errno != EPERM && errno != EACCES)
|
(void)ul_gid;
|
||||||
log_message(LOG_LEVEL_WARNING, "--fake-super: could not restore owner on destination file: %s",
|
/* Mode is applied only when the per-attribute policy asks for it, through the
|
||||||
strerror(errno));
|
SAME shared helper the normal metadata path uses (metadata_mode_for_policy).
|
||||||
/* Mode is applied through the same sanitization the normal metadata path
|
The recorded special bits are stripped first: rsync's fake-super receiver
|
||||||
uses (metadata_mode): group/other write bits are never granted, so a
|
stores the full mode in the xattr but never installs setuid/setgid/sticky on
|
||||||
recorded source mode of 0666 restores as 0644 — identical to a non-fake-
|
the real file, so only the 0777 permission bits may be replayed. The -E
|
||||||
super --preserve run, never a privilege-granting regression. */
|
rule then derives exec bits from the destination's read bits exactly like
|
||||||
if (fchmod(fd, (mode_t)(ul_mode & 0777U & ~(S_IWGRP | S_IWOTH))) != 0)
|
file_restore_metadata_fd. */
|
||||||
log_message(LOG_LEVEL_WARNING, "--fake-super: could not restore mode on destination file: %s",
|
if (policy.perms || policy.executability) {
|
||||||
strerror(errno));
|
struct stat cur;
|
||||||
struct timespec times[2] = {{.tv_sec = 0, .tv_nsec = UTIME_OMIT},
|
mode_t want = 0;
|
||||||
{.tv_sec = (time_t)mtime_sec, .tv_nsec = mtime_nsec}};
|
if (fstat(fd, &cur) != 0) {
|
||||||
if (futimens(fd, times) != 0)
|
log_message(LOG_LEVEL_WARNING, "--fake-super: could not read destination mode: %s",
|
||||||
log_message(LOG_LEVEL_WARNING, "--fake-super: could not restore mtime on destination file: %s",
|
strerror(errno));
|
||||||
strerror(errno));
|
} else if (metadata_mode_for_policy((mode_t)(ul_mode & 0777U), cur.st_mode, policy, &want)) {
|
||||||
|
if (fchmod(fd, want) != 0)
|
||||||
|
log_message(LOG_LEVEL_WARNING,
|
||||||
|
"--fake-super: could not restore mode on destination file: %s",
|
||||||
|
strerror(errno));
|
||||||
|
}
|
||||||
|
}
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|||||||
+77
-22
@@ -1,6 +1,7 @@
|
|||||||
#ifndef XATTR_H
|
#ifndef XATTR_H
|
||||||
#define XATTR_H
|
#define XATTR_H
|
||||||
|
|
||||||
|
#include "file_attr.h"
|
||||||
#include <stdbool.h>
|
#include <stdbool.h>
|
||||||
#include <stddef.h>
|
#include <stddef.h>
|
||||||
#include <stdint.h>
|
#include <stdint.h>
|
||||||
@@ -25,16 +26,22 @@
|
|||||||
* and total bytes) on BOTH ends to prevent OOM/memory abuse; an oversized
|
* and total bytes) on BOTH ends to prevent OOM/memory abuse; an oversized
|
||||||
* or malformed frame is a clean protocol rejection, never an allocation
|
* or malformed frame is a clean protocol rejection, never an allocation
|
||||||
* blowup.
|
* blowup.
|
||||||
* * Application is confined to the exact destination file descriptor
|
* * Application is confined to the exact destination entry: fsetxattr on the
|
||||||
* (fsetxattr on the just-written fd), never a caller-controlled path.
|
* just-written fd for regular files/directories, and for a symlink an
|
||||||
|
* lsetxattr on "/proc/self/fd/<parent_fd>/<leaf>" reached through the
|
||||||
|
* already-opened, confinement-checked parent directory -- never a
|
||||||
|
* caller-controlled path, and never following the link.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/* Reserved key used by --fake-super to park the source's privileged ownership
|
/* Reserved key used by --fake-super to park the source's privileged ownership
|
||||||
* / mode / mtime on the destination file as an unprivileged user.* xattr, so a
|
* / mode / rdev on the destination file as an unprivileged user.* xattr, so the
|
||||||
* later privileged restore could re-apply them. Exact documented format:
|
* tree is interoperable with rsync 3.4.1 and a later privileged restore can
|
||||||
* uid:gid:mode:mtime_sec:mtime_nsec (decimal, decimal, octal, dec, dec)
|
* re-apply them. This is rsync's own key and value grammar exactly:
|
||||||
* e.g. "1000:1000:644:1765238400:0". */
|
* <octal st_mode with S_IFMT> <rdev_major>,<rdev_minor> <uid>:<gid>
|
||||||
#define FAKESUPER_XATTR "user.fastsync.stat"
|
* e.g. "104711 0,0 1234:5678" for a setuid regular file owned by 1234:5678,
|
||||||
|
* or "20644 1,3 111:222" for a char device. mtime is deliberately NOT part of
|
||||||
|
* the record: exactly like rsync, the file's own timestamp carries it. */
|
||||||
|
#define FAKESUPER_XATTR "user.rsync.%stat"
|
||||||
|
|
||||||
/* --- bounds --- */
|
/* --- bounds --- */
|
||||||
#define XATTR_NAME_MAX 255 /* xattr names are limited to 255 bytes */
|
#define XATTR_NAME_MAX 255 /* xattr names are limited to 255 bytes */
|
||||||
@@ -55,6 +62,8 @@ typedef struct {
|
|||||||
|
|
||||||
FileXattrList* xattr_list_new(void);
|
FileXattrList* xattr_list_new(void);
|
||||||
void xattr_list_free(FileXattrList* list);
|
void xattr_list_free(FileXattrList* list);
|
||||||
|
/* Deep-copy `list` (NULL in, NULL out). Returns NULL on allocation failure. */
|
||||||
|
FileXattrList* xattr_list_clone(const FileXattrList* list);
|
||||||
/* Append one entry (deep copy). Returns false on allocation failure. */
|
/* Append one entry (deep copy). Returns false on allocation failure. */
|
||||||
bool xattr_list_append(FileXattrList* list, const char* name, const void* value, size_t value_len);
|
bool xattr_list_append(FileXattrList* list, const char* name, const void* value, size_t value_len);
|
||||||
|
|
||||||
@@ -73,6 +82,17 @@ bool xattr_name_appliable(const char* name, bool preserve_acls);
|
|||||||
* distinct from NULL. */
|
* distinct from NULL. */
|
||||||
FileXattrList* xattr_capture_path(const char* path, bool preserve_acls);
|
FileXattrList* xattr_capture_path(const char* path, bool preserve_acls);
|
||||||
|
|
||||||
|
/* Sender: like xattr_capture_path() but reads the xattrs of `path` ITSELF,
|
||||||
|
* never following a final symlink (llistxattr/lgetxattr). A symlink entry must
|
||||||
|
* use this so the scanner never captures the REFERENT's attributes onto the
|
||||||
|
* link (the path-following variant would). On Linux the VFS refuses to
|
||||||
|
* associate xattrs with symlinks at all, so this normally returns NULL; it is
|
||||||
|
* still correct and portable for a filesystem/platform that supports them.
|
||||||
|
* The same whitelist/bounds as xattr_capture_path() apply. Returns NULL when
|
||||||
|
* the link has no appliable xattrs (or the filesystem does not support them);
|
||||||
|
* an empty-but-valid list is never returned distinct from NULL. */
|
||||||
|
FileXattrList* xattr_capture_path_nofollow(const char* path, bool preserve_acls);
|
||||||
|
|
||||||
/* Wire: bounded serialization. xattr_send returns false on write failure; an
|
/* Wire: bounded serialization. xattr_send returns false on write failure; an
|
||||||
* empty/NULL list transmits a zero-count block. xattr_receive returns NULL and
|
* empty/NULL list transmits a zero-count block. xattr_receive returns NULL and
|
||||||
* sets *ok = 0 on any malformed / oversized / non-whitelisted entry. When
|
* sets *ok = 0 on any malformed / oversized / non-whitelisted entry. When
|
||||||
@@ -88,22 +108,57 @@ FileXattrList* xattr_receive(int fd, int* ok, bool preserve_acls);
|
|||||||
* true when apply was attempted (allowing callers to treat it as best-effort). */
|
* true when apply was attempted (allowing callers to treat it as best-effort). */
|
||||||
bool xattr_apply_fd(int fd, const FileXattrList* list);
|
bool xattr_apply_fd(int fd, const FileXattrList* list);
|
||||||
|
|
||||||
/* --fake-super: write the source uid/gid/mode/mtime record into the reserved
|
/* Receiver: apply every entry to the symlink named by (parent_fd, leaf) WITHOUT
|
||||||
* FAKESUPER_XATTR on `fd`. Best-effort (logged, never fatal). Only meaningful
|
* following it, via lsetxattr() on the confined path
|
||||||
* when metadata was transmitted so the values exist. */
|
* "/proc/self/fd/<parent_fd>/<leaf>". Every incoming name is independently
|
||||||
void fake_super_store_fd(int fd, uint32_t uid, uint32_t gid, uint32_t mode, int64_t mtime_sec,
|
* re-validated against xattr_name_appliable() with `preserve_acls`, exactly like
|
||||||
int64_t mtime_nsec);
|
* xattr_apply_fd(): a non-whitelisted namespace (including the reserved
|
||||||
|
* --fake-super key) is skipped, so this primitive stays confined even if handed
|
||||||
|
* a hand-crafted list. A symlink cannot be targeted by the fd-relative
|
||||||
|
* fsetxattr() path: there is no *at() xattr syscall and the kernel rejects
|
||||||
|
* xattr syscalls on an O_PATH descriptor, so the already-opened,
|
||||||
|
* confinement-checked parent directory is the anchor and only the final
|
||||||
|
* component is the (no-follow) link. `leaf` must be a single path component.
|
||||||
|
*
|
||||||
|
* Portability: the "/proc/self/fd/<parent_fd>" anchor requires a mounted /proc.
|
||||||
|
* Where /proc is unavailable (or the fd cannot be addressed that way) the
|
||||||
|
* lsetxattr simply fails and is skipped -- the apply is best-effort exactly like
|
||||||
|
* xattr_apply_fd(), so no error is propagated and the transfer continues. A
|
||||||
|
* per-attribute failure (on Linux every set on a symlink fails with EPERM) is
|
||||||
|
* logged once and skipped, never fatal. Returns false only for an invalid
|
||||||
|
* anchor/list; true when an apply was attempted.
|
||||||
|
*
|
||||||
|
* Residual TOCTOU: `leaf` is a caller-supplied name resolved by path in the
|
||||||
|
* parent, so a local writer could replace the just-created symlink between its
|
||||||
|
* creation and lsetxattr(). This is bounded: it requires write access to the
|
||||||
|
* confinement-checked destination directory (already trusted), can only install
|
||||||
|
* a whitelisted user namespace or POSIX-ACL name, and never follows the link (a
|
||||||
|
* replacement symlink is still applied to as the final, no-follow component). */
|
||||||
|
bool xattr_apply_path_nofollow(int parent_fd, const char* leaf, const FileXattrList* list,
|
||||||
|
bool preserve_acls);
|
||||||
|
|
||||||
|
/* --fake-super: write the source uid/gid/mode/rdev record into the reserved
|
||||||
|
* FAKESUPER_XATTR on `fd`, using rsync 3.4.1's exact grammar (see the key
|
||||||
|
* comment above). `mode` is the full st_mode including its S_IFMT bits.
|
||||||
|
* Best-effort (logged, never fatal). Only meaningful when metadata was
|
||||||
|
* transmitted so the values exist. */
|
||||||
|
void fake_super_store_fd(int fd, uint32_t uid, uint32_t gid, uint32_t mode, uint32_t rdev_major,
|
||||||
|
uint32_t rdev_minor);
|
||||||
|
|
||||||
/* --fake-super replay: parse the FAKESUPER_XATTR record previously written on
|
/* --fake-super replay: parse the FAKESUPER_XATTR record previously written on
|
||||||
* `fd` by fake_super_store_fd and re-apply uid/gid/mode/mtime fd-relative.
|
* `fd` by fake_super_store_fd and re-apply the recorded permission bits
|
||||||
* Best-effort: absence of the xattr or a malformed record is a silent no-op
|
* fd-relative. The recorded uid/gid are deliberately NOT chowned for real:
|
||||||
* that never fails the transfer. The OWNER leg is applied only when an explicit
|
* --fake-super only RECORDS ownership (the caller stores the resolved mapping
|
||||||
* ownership identity policy is active (numeric-ids/chown/usermap/groupmap/
|
* via identity_resolve_storage_ids), it never performs a real chown. The
|
||||||
* copy-as), when super-user activities are permitted, and when --copy-as is not
|
* recorded rdev is retained for a later privileged restore but is not acted on
|
||||||
* authoritative; a non-root EPERM/EACCES is skipped silently, matching
|
* here. Best-effort: absence of the xattr or a malformed record is a silent
|
||||||
* FastSync's identity philosophy. The mode is sanitized exactly like the normal
|
* no-op that never fails the transfer. The MODE leg is applied only when
|
||||||
* metadata path (group/other write bits never granted). Returns true when the
|
* policy.perms||policy.executability, and the recorded special bits
|
||||||
* xattr was present and parsed. */
|
* (setuid/setgid/sticky) are NOT applied to the real file -- exactly like
|
||||||
bool fake_super_restore_fd(int fd);
|
* rsync's fake-super receiver, which stores the full mode in the xattr but
|
||||||
|
* strips the special bits on disk. mtime is not part of the record; the normal
|
||||||
|
* metadata path carries it (policy.times) exactly as rsync sets the file's own
|
||||||
|
* timestamp. Returns true when the xattr was present and parsed. */
|
||||||
|
bool fake_super_restore_fd(int fd, FileAttrPolicy policy);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
@@ -115,7 +115,16 @@ static void build_canonical_frame(void) {
|
|||||||
if (cfg->usermap) {
|
if (cfg->usermap) {
|
||||||
cfg->usermap_count = 1;
|
cfg->usermap_count = 1;
|
||||||
cfg->usermap[0].from = MAP_FROM;
|
cfg->usermap[0].from = MAP_FROM;
|
||||||
|
cfg->usermap[0].from_hi = MAP_FROM;
|
||||||
cfg->usermap[0].to = MAP_TO;
|
cfg->usermap[0].to = MAP_TO;
|
||||||
|
cfg->usermap[0].to_name = NULL;
|
||||||
|
}
|
||||||
|
/* Force a non-empty receiver delete-protection block so the fuzzer mutates
|
||||||
|
* its rule count, action/sides codes and pattern strings. */
|
||||||
|
cfg->filters = array_list_create(free);
|
||||||
|
if (cfg->filters) {
|
||||||
|
array_list_add(cfg->filters, str_dup("P *.log"));
|
||||||
|
array_list_add(cfg->filters, str_dup("+r keep/**"));
|
||||||
}
|
}
|
||||||
if (!cfg->send_directory || !cfg->receive_root_directory || !cfg->usermap) {
|
if (!cfg->send_directory || !cfg->receive_root_directory || !cfg->usermap) {
|
||||||
config_delete(cfg);
|
config_delete(cfg);
|
||||||
|
|||||||
@@ -0,0 +1,80 @@
|
|||||||
|
# Integration tests
|
||||||
|
|
||||||
|
The integration suite drives the built `build/server` and `build/client`
|
||||||
|
against local corpora. Unit tests live in `tests/` (the custom C framework);
|
||||||
|
the Python suite here covers the full transfer pipeline, transports, features,
|
||||||
|
and rsync parity.
|
||||||
|
|
||||||
|
## Running
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Full suite (excludes privilege-dependent tests on CI runners)
|
||||||
|
python3 -m pytest tests/integration/ -n 4 --dist=load -m "not setpriv"
|
||||||
|
|
||||||
|
# Fast PR subset only
|
||||||
|
python3 -m pytest tests/integration/ -n 4 --dist=load -m ci
|
||||||
|
```
|
||||||
|
|
||||||
|
The tests expect `build/server` and `build/client` (configure/build with CMake
|
||||||
|
first); `common.py` derives `BUILD_DIR` from the repository root.
|
||||||
|
|
||||||
|
## Differential rsync-parity gate
|
||||||
|
|
||||||
|
`test_differential_parity.py` runs the **same** transfer with real
|
||||||
|
`rsync 3.4.1` and with FastSync over separate destinations, then compares:
|
||||||
|
|
||||||
|
- the destination trees — relative paths, file content hashes, symlink
|
||||||
|
targets, modes (where the case is about perms), and hard-link grouping;
|
||||||
|
- the normalized stdout for output-oriented flags (`-i`,
|
||||||
|
`--out-format=...`, `--stats`), after stripping volatile fields
|
||||||
|
(timings, rates, wire byte counts) and directory-only itemize lines that
|
||||||
|
FastSync's recursive scanner documents as absent.
|
||||||
|
|
||||||
|
FastSync mirrors the absolute source path under its receive root (see
|
||||||
|
`get_dest_received_dir`); the harness normalizes that layout (and the
|
||||||
|
`-R`/`--files-from` layouts) before comparing.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Fast subset that guards the ✅ surface on pull requests
|
||||||
|
python3 -m pytest tests/integration/test_differential_parity.py -n 4 --dist=load -m parity_ci
|
||||||
|
|
||||||
|
# Full differential case table (`_CASES`): every row is marked `parity`, and a
|
||||||
|
# case with an allowlisted residual in `parity_caveats.py` is included too.
|
||||||
|
python3 -m pytest tests/integration/test_differential_parity.py -n 4 --dist=load -m parity
|
||||||
|
```
|
||||||
|
|
||||||
|
`-m parity` selects only the `_CASES` table in this module. Differential
|
||||||
|
coverage for options outside that table (`--temp-dir`, `--delay-updates`,
|
||||||
|
`--dry-run`, `--fuzzy`, the basis-dir options, `-M` over daemon/TCP, and
|
||||||
|
receiver filter-protect) lives in dedicated modules (`test_option_parity.py`,
|
||||||
|
`test_parity_blockers.py`, `test_parity_quickwins.py`, ...) and is not part of
|
||||||
|
this gate. The suite skips cleanly when `rsync` is not installed.
|
||||||
|
|
||||||
|
## Allowlist (`parity_caveats.py`)
|
||||||
|
|
||||||
|
`parity_caveats.py` is the single data-driven allowlist of known differences.
|
||||||
|
Each entry maps a case id to the aspects that may differ (`tree`, `stdout`,
|
||||||
|
`extra`, `rc`) and cites the governing row in `RSYNC_COMPAT.md`:
|
||||||
|
|
||||||
|
```python
|
||||||
|
CAVEATS = {
|
||||||
|
# no known residuals at present -- the burn-down reached zero
|
||||||
|
# "some_case_id": {"tree": "documented residual ... ref: RSYNC_COMPAT.md ..."},
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
A differential mismatch in an aspect that is **not** listed fails the gate with
|
||||||
|
a readable tree/stdout diff.
|
||||||
|
|
||||||
|
If a case is allowlisted but now matches rsync, the gate emits a loud warning
|
||||||
|
naming the stale entry — that is the parity burn-down signal. Run with
|
||||||
|
`FASTSYNC_PARITY_STRICT=1` to make stale entries fail instead (the full CI
|
||||||
|
parity job sets this). To add a residual:
|
||||||
|
|
||||||
|
1. Reproduce it with `-m parity` and read the failure's tree/stdout diff.
|
||||||
|
2. Confirm it is a documented `⚠️`/`❌` residual (or get the `✅` row
|
||||||
|
reclassified) and cite the row.
|
||||||
|
3. Add the case id and aspect(s) to `CAVEATS`, keeping the reason concise.
|
||||||
|
|
||||||
|
Do not allowlist an undocumented divergence from a `✅` row — fix it or get the
|
||||||
|
row reclassified first.
|
||||||
+73
-17
@@ -20,6 +20,12 @@ CLIENT_CMD = [os.path.join(BUILD_DIR, "client")]
|
|||||||
_WORKER = os.environ.get("PYTEST_XDIST_WORKER")
|
_WORKER = os.environ.get("PYTEST_XDIST_WORKER")
|
||||||
TEST_DATA_DIR = os.path.join(PROJECT_ROOT, f"test_data-{_WORKER}" if _WORKER else "test_data")
|
TEST_DATA_DIR = os.path.join(PROJECT_ROOT, f"test_data-{_WORKER}" if _WORKER else "test_data")
|
||||||
|
|
||||||
|
# Default wall-clock budget for a short-lived client invocation. Every client
|
||||||
|
# is expected to finish well within this; the bound exists so a hung client
|
||||||
|
# fails the test instead of stalling the whole CI run indefinitely. Callers
|
||||||
|
# that legitimately need longer can pass an explicit ``timeout``.
|
||||||
|
CLIENT_TIMEOUT = 180
|
||||||
|
|
||||||
|
|
||||||
class ServerManager:
|
class ServerManager:
|
||||||
"""Manages a long-lived server process. Reuses across test cases."""
|
"""Manages a long-lived server process. Reuses across test cases."""
|
||||||
@@ -101,9 +107,15 @@ class CountingProxy:
|
|||||||
return
|
return
|
||||||
counter[0] += len(data)
|
counter[0] += len(data)
|
||||||
|
|
||||||
def run(self, cmd):
|
def run(self, cmd, join_timeout=20):
|
||||||
"""Forward one client run (the full command list) to the real server and
|
"""Forward one client run (the full command list) to the real server and
|
||||||
return the CompletedProcess after the counts have settled."""
|
return the CompletedProcess after the counts have settled.
|
||||||
|
|
||||||
|
``join_timeout`` bounds how long to wait for the forwarding threads. The
|
||||||
|
client->server count is published as soon as the client side reaches EOF
|
||||||
|
(i.e. once the client process has exited), so callers that only need that
|
||||||
|
count can pass a small value instead of waiting for the server to close
|
||||||
|
its idle socket."""
|
||||||
|
|
||||||
def serve():
|
def serve():
|
||||||
try:
|
try:
|
||||||
@@ -119,19 +131,52 @@ class CountingProxy:
|
|||||||
a.start()
|
a.start()
|
||||||
b.start()
|
b.start()
|
||||||
a.join()
|
a.join()
|
||||||
b.join()
|
|
||||||
self.client_to_server = c2s[0]
|
self.client_to_server = c2s[0]
|
||||||
|
b.join()
|
||||||
self.server_to_client = s2c[0]
|
self.server_to_client = s2c[0]
|
||||||
self._listener.close()
|
self._listener.close()
|
||||||
|
|
||||||
thread = threading.Thread(target=serve)
|
thread = threading.Thread(target=serve, daemon=True)
|
||||||
thread.start()
|
thread.start()
|
||||||
result = subprocess.run(cmd, capture_output=True, text=True, timeout=180)
|
result = subprocess.run(cmd, capture_output=True, text=True, timeout=180)
|
||||||
thread.join(20)
|
thread.join(join_timeout)
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
def run_client(source_dir, dest_dir, flags=None, port=None, extra_args=None):
|
def _run_client_cmd(cmd, timeout):
|
||||||
|
"""Run one client command, returning ``(result, duration)``.
|
||||||
|
|
||||||
|
On timeout the client is killed and a result-like ``CompletedProcess`` with
|
||||||
|
a non-zero returncode is returned instead of raising, so callers keep the
|
||||||
|
established ``(result, duration)`` contract and the failure carries the
|
||||||
|
command plus whatever output was captured for diagnosis.
|
||||||
|
"""
|
||||||
|
start = time.monotonic()
|
||||||
|
try:
|
||||||
|
result = subprocess.run(cmd, text=True, capture_output=True, timeout=timeout)
|
||||||
|
except subprocess.TimeoutExpired as exc:
|
||||||
|
duration = time.monotonic() - start
|
||||||
|
stdout = exc.stdout or ""
|
||||||
|
stderr = exc.stderr or ""
|
||||||
|
if isinstance(stdout, bytes):
|
||||||
|
stdout = stdout.decode(errors="replace")
|
||||||
|
if isinstance(stderr, bytes):
|
||||||
|
stderr = stderr.decode(errors="replace")
|
||||||
|
diagnostic = (
|
||||||
|
f"client timed out after {timeout}s\n"
|
||||||
|
f"command: {cmd!r}\n"
|
||||||
|
f"--- captured stdout ---\n{stdout}\n"
|
||||||
|
f"--- captured stderr ---\n{stderr}"
|
||||||
|
)
|
||||||
|
result = subprocess.CompletedProcess(cmd, returncode=-1,
|
||||||
|
stdout=stdout, stderr=diagnostic)
|
||||||
|
return result, duration
|
||||||
|
duration = time.monotonic() - start
|
||||||
|
return result, duration
|
||||||
|
|
||||||
|
|
||||||
|
def run_client(source_dir, dest_dir, flags=None, port=None, extra_args=None,
|
||||||
|
timeout=CLIENT_TIMEOUT):
|
||||||
"""Run the client and return (result, duration)."""
|
"""Run the client and return (result, duration)."""
|
||||||
cmd = CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir, "--save-to-disk"]
|
cmd = CLIENT_CMD + ["--source-dir", source_dir, "--dest-dir", dest_dir, "--save-to-disk"]
|
||||||
if port:
|
if port:
|
||||||
@@ -140,23 +185,18 @@ def run_client(source_dir, dest_dir, flags=None, port=None, extra_args=None):
|
|||||||
cmd += flags
|
cmd += flags
|
||||||
if extra_args:
|
if extra_args:
|
||||||
cmd += extra_args
|
cmd += extra_args
|
||||||
start = time.monotonic()
|
return _run_client_cmd(cmd, timeout)
|
||||||
result = subprocess.run(cmd, text=True, capture_output=True)
|
|
||||||
duration = time.monotonic() - start
|
|
||||||
return result, duration
|
|
||||||
|
|
||||||
|
|
||||||
def run_client_posix(source_dir, dest_dir, flags=None, port=None):
|
def run_client_posix(source_dir, dest_dir, flags=None, port=None,
|
||||||
|
timeout=CLIENT_TIMEOUT):
|
||||||
"""Run the client with positional args (rsync-style)."""
|
"""Run the client with positional args (rsync-style)."""
|
||||||
cmd = CLIENT_CMD + [source_dir, dest_dir, "--save-to-disk"]
|
cmd = CLIENT_CMD + [source_dir, dest_dir, "--save-to-disk"]
|
||||||
if port:
|
if port:
|
||||||
cmd += ["--server-port", str(port)]
|
cmd += ["--server-port", str(port)]
|
||||||
if flags:
|
if flags:
|
||||||
cmd += flags
|
cmd += flags
|
||||||
start = time.monotonic()
|
return _run_client_cmd(cmd, timeout)
|
||||||
result = subprocess.run(cmd, text=True, capture_output=True)
|
|
||||||
duration = time.monotonic() - start
|
|
||||||
return result, duration
|
|
||||||
|
|
||||||
|
|
||||||
def generate_test_files(source_dir, full=False):
|
def generate_test_files(source_dir, full=False):
|
||||||
@@ -232,8 +272,17 @@ def make_result(name, success, duration=None, error=""):
|
|||||||
|
|
||||||
|
|
||||||
def get_dest_received_dir(dest_dir, source_dir):
|
def get_dest_received_dir(dest_dir, source_dir):
|
||||||
"""Get the path where received files land inside dest_dir."""
|
"""Get the path where received files land inside dest_dir.
|
||||||
return os.path.join(dest_dir, os.path.abspath(source_dir).lstrip(os.sep))
|
|
||||||
|
FastSync mirrors the absolute source path below the receive root with the
|
||||||
|
leading root separator removed. Strip that separator explicitly rather
|
||||||
|
than with ``str.lstrip(os.sep)``: ``lstrip`` removes a *set* of characters
|
||||||
|
rather than a path prefix, which is not the same operation.
|
||||||
|
"""
|
||||||
|
abs_source = os.path.abspath(source_dir)
|
||||||
|
if abs_source.startswith(os.sep):
|
||||||
|
abs_source = abs_source[len(os.sep):]
|
||||||
|
return os.path.join(dest_dir, abs_source)
|
||||||
|
|
||||||
|
|
||||||
def _find_free_port():
|
def _find_free_port():
|
||||||
@@ -254,6 +303,13 @@ def _wait_for_port(port, timeout=5):
|
|||||||
|
|
||||||
|
|
||||||
def _wait_proc(proc, timeout=5):
|
def _wait_proc(proc, timeout=5):
|
||||||
|
"""Stop a long-lived subprocess promptly. The server installs a SIGTERM
|
||||||
|
handler, so signal first and only escalate to SIGKILL if it does not exit;
|
||||||
|
waiting without signalling would burn the full timeout on every stop."""
|
||||||
|
if proc.poll() is not None:
|
||||||
|
proc.wait()
|
||||||
|
return
|
||||||
|
proc.terminate()
|
||||||
try:
|
try:
|
||||||
proc.wait(timeout=timeout)
|
proc.wait(timeout=timeout)
|
||||||
except subprocess.TimeoutExpired:
|
except subprocess.TimeoutExpired:
|
||||||
|
|||||||
@@ -0,0 +1,34 @@
|
|||||||
|
"""Data-driven allowlist for the differential rsync-parity gate.
|
||||||
|
|
||||||
|
Every entry maps a case id (see ``test_differential_parity.py``) to the aspects
|
||||||
|
that are *known* to differ from ``rsync 3.4.1`` and the documented reason. A
|
||||||
|
differential mismatch in an aspect that is **not** listed here fails the gate.
|
||||||
|
|
||||||
|
Aspect keys
|
||||||
|
-----------
|
||||||
|
``tree`` destination tree differs (paths, file hashes, symlink targets,
|
||||||
|
modes, hardlink grouping)
|
||||||
|
``stdout`` normalized output for ``-i`` / ``--stats`` / ``--out-format``
|
||||||
|
``extra`` a case-specific assertion differs (basis/inode checks, ...)
|
||||||
|
``rc`` exit status differs
|
||||||
|
|
||||||
|
Burn-down
|
||||||
|
---------
|
||||||
|
If a case is listed here but now matches rsync, the gate emits a loud
|
||||||
|
``pytest`` warning naming the stale entry: delete the entry (and, when the
|
||||||
|
underlying row in ``RSYNC_COMPAT.md`` is now parity, update that row). Set
|
||||||
|
``FASTSYNC_PARITY_STRICT=1`` to turn stale entries into failures in CI.
|
||||||
|
|
||||||
|
Keep the values concise but cite the governing row so the entry can be
|
||||||
|
re-triaged when the row moves.
|
||||||
|
"""
|
||||||
|
|
||||||
|
# case id -> {aspect: "reason (ref: RSYNC_COMPAT.md ...)"}
|
||||||
|
CAVEATS = {}
|
||||||
|
|
||||||
|
# Accepted aspect names (guards against typos in this file).
|
||||||
|
ASPECTS = ("tree", "stdout", "extra", "rc")
|
||||||
|
|
||||||
|
|
||||||
|
def caveat_for(case_id: str) -> dict:
|
||||||
|
return CAVEATS.get(case_id, {})
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user