Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
919a729206 | ||
|
|
8cd2b550d9 | ||
|
|
99c0fd8016 | ||
|
|
a2200f039a | ||
|
|
1437c6dc6b | ||
|
|
38356ecc1e | ||
|
|
7badac7f97 | ||
|
|
6ccf16b650 | ||
|
|
1174993d6b | ||
|
|
d5fcfa2c5c | ||
|
|
e79d2b47b0 | ||
|
|
7c24a365cf | ||
|
|
5893de4a34 | ||
|
|
825ba69753 | ||
|
|
34abaadb9a | ||
|
|
10c4ffebdf | ||
|
|
dfa2a42028 | ||
|
|
5b0ec5880d | ||
|
|
1a26bde2d4 | ||
|
|
58a28334b8 | ||
|
|
e48f19ee2b | ||
|
|
a2370433b2 | ||
|
|
9da5a0a9ed | ||
|
|
80c1ff321c | ||
|
|
551c187005 | ||
|
|
0d6c1f784f | ||
|
|
f75a69f96a | ||
|
|
df887c73b1 | ||
|
|
5d39619a8a | ||
|
|
6269ae54e5 | ||
|
|
07f7555c1d | ||
|
|
5b8aca5799 | ||
|
|
a1eaa93357 | ||
|
|
99df0a8a6d | ||
|
|
334fc5b3e8 | ||
|
|
88aee6ce94 | ||
|
|
f6e8b6ddc4 | ||
|
|
6f974eff19 | ||
|
|
72cccaa256 | ||
|
|
eb71b29d1c | ||
|
|
4d5befedfe | ||
|
|
f00844cf9a | ||
|
|
2ec17e821c | ||
|
|
6d47d93fd7 | ||
|
|
6ea966781f | ||
|
|
0a7f5faea6 | ||
|
|
25909110ac | ||
|
|
bd43448af2 | ||
|
|
264964411c | ||
|
|
18d1b84246 | ||
|
|
0f95f48899 | ||
|
|
c78a21de57 | ||
|
|
5c8970c64f | ||
|
|
4e918a1b69 | ||
|
|
87f6cb0243 | ||
|
|
e1f8f75e7c | ||
|
|
4c17122b00 | ||
|
|
0abaa62193 | ||
|
|
5334397b81 | ||
|
|
6968ff6734 | ||
|
|
5aca91ab22 | ||
|
|
3260a39ab4 | ||
|
|
5d3c43305e | ||
|
|
9242e86772 | ||
|
|
0155902d95 | ||
|
|
3499baf80b | ||
|
|
83eacf3151 | ||
|
|
c2df0347ef | ||
|
|
dd44537b44 | ||
|
|
2854a9d149 | ||
|
|
1fa2fbd266 | ||
|
|
10b18ab2d2 | ||
|
|
dcc78c14c5 | ||
|
|
5a829adb85 | ||
|
|
42c72030fb | ||
|
|
99f8045105 | ||
|
|
eea66a7848 | ||
|
|
c944787e03 | ||
|
|
57ce6d04f0 | ||
|
|
3cf2e2c91f | ||
|
|
301cb0dbaf | ||
|
|
a4f4110397 | ||
|
|
24fe8c5583 | ||
|
|
d274bdff4e | ||
|
|
6c636a19e6 | ||
|
|
1a83e284c4 | ||
|
|
317d5d081a | ||
|
|
69fe7f3c9f | ||
|
|
ddc71a7df5 | ||
|
|
ffa1d24625 | ||
|
|
b16349b81e | ||
|
|
4bc84fe954 | ||
|
|
fc560246c1 | ||
|
|
dff6609976 | ||
|
|
1acb66628d | ||
|
|
ba1c7a369f | ||
|
|
d28489d83c | ||
|
|
312ed05170 | ||
|
|
fecbe2c90c | ||
|
|
c8f5d80fcb | ||
|
|
8147ff7b50 | ||
|
|
b7fbb56289 | ||
|
|
08063b6d73 | ||
|
|
ea2f76cd7a | ||
|
|
76eeba1773 | ||
|
|
b72ab298ab | ||
|
|
446a714ef8 | ||
|
|
a90e234eb3 | ||
|
|
f8252cf3e7 | ||
|
|
4557924972 | ||
|
|
59ce174d22 | ||
|
|
2a8941ee5c | ||
|
|
37037a6ee7 | ||
|
|
061e9ad43f | ||
|
|
f928879755 | ||
|
|
84b7cb0de3 | ||
|
|
921472b8b3 | ||
|
|
082ac2645d | ||
|
|
eefbd1e849 | ||
|
|
1fd462cca8 | ||
|
|
4ac37c4d8a | ||
|
|
08af945bd6 |
@@ -12,7 +12,7 @@ jobs:
|
|||||||
container: gitea.tap-tap.win/taptap/fastsync-ci:v10
|
container: gitea.tap-tap.win/taptap/fastsync-ci:v10
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||||
|
|
||||||
- name: clang-format check
|
- name: clang-format check
|
||||||
run: find src/ tests/ -name '*.c' -o -name '*.h' | xargs clang-format --dry-run --Werror
|
run: find src/ tests/ -name '*.c' -o -name '*.h' | xargs clang-format --dry-run --Werror
|
||||||
@@ -30,7 +30,7 @@ jobs:
|
|||||||
needs: lint
|
needs: lint
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||||
|
|
||||||
- name: Configure
|
- name: Configure
|
||||||
run: cmake -B build -S . -DSTRICT_WARNINGS=ON
|
run: cmake -B build -S . -DSTRICT_WARNINGS=ON
|
||||||
@@ -59,7 +59,7 @@ jobs:
|
|||||||
sanitizer: [address, undefined]
|
sanitizer: [address, undefined]
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||||
|
|
||||||
- name: Configure
|
- name: Configure
|
||||||
run: cmake -B build-${{ matrix.sanitizer }} -S . -DSANITIZER=${{ matrix.sanitizer }}
|
run: cmake -B build-${{ matrix.sanitizer }} -S . -DSANITIZER=${{ matrix.sanitizer }}
|
||||||
@@ -77,7 +77,7 @@ jobs:
|
|||||||
if: github.event_name == 'push'
|
if: github.event_name == 'push'
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||||
|
|
||||||
- name: Configure (clang + fuzz)
|
- name: Configure (clang + fuzz)
|
||||||
run: CC=clang CXX=clang++ cmake -B build-fuzz -S . -DENABLE_FUZZ=ON
|
run: CC=clang CXX=clang++ cmake -B build-fuzz -S . -DENABLE_FUZZ=ON
|
||||||
@@ -99,7 +99,7 @@ jobs:
|
|||||||
if: github.event_name == 'push'
|
if: github.event_name == 'push'
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||||
|
|
||||||
- name: Configure
|
- name: Configure
|
||||||
run: cmake -B build -S . -DENABLE_COVERAGE=ON
|
run: cmake -B build -S . -DENABLE_COVERAGE=ON
|
||||||
@@ -123,7 +123,7 @@ jobs:
|
|||||||
if: github.event_name == 'push'
|
if: github.event_name == 'push'
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
||||||
|
|
||||||
- name: Configure
|
- name: Configure
|
||||||
run: cmake -B build -S . -DSTRICT_WARNINGS=ON
|
run: cmake -B build -S . -DSTRICT_WARNINGS=ON
|
||||||
|
|||||||
@@ -8,3 +8,7 @@ build-*/
|
|||||||
build2/
|
build2/
|
||||||
build3/
|
build3/
|
||||||
build_docker2/
|
build_docker2/
|
||||||
|
|
||||||
|
# Test/run artifacts
|
||||||
|
root/
|
||||||
|
test_partial_install_tmp/
|
||||||
|
|||||||
@@ -128,7 +128,7 @@ Do not wait for the user to tell you CI failed — check proactively. The user s
|
|||||||
|
|
||||||
## Branch Strategy
|
## Branch Strategy
|
||||||
|
|
||||||
Never push directly to `main`. All changes must be developed on a feature branch and merged via a pull request. Always create a new branch (`git checkout -b <branch-name>`) before making changes, push it, and open a PR with `gh pr create --fill`. Wait for CI to pass before merging.
|
Never push directly to `dev` or `main`. All changes must be developed on a feature branch and merged via a pull request targeting `dev`. Create a branch (`git checkout -b <branch-name>`), push it, and open the PR with `tea pr create --repo TapTap/FastSync --base dev --head <branch-name>`. Wait for CI to pass before merging.
|
||||||
|
|
||||||
## Dependency Installation
|
## Dependency Installation
|
||||||
|
|
||||||
|
|||||||
@@ -92,7 +92,7 @@ When using `tea` (the task execution agent) to run CI or tests, always set a suf
|
|||||||
|
|
||||||
## Branch Strategy
|
## Branch Strategy
|
||||||
|
|
||||||
Never push directly to `main`. All changes must be developed on a feature branch and merged via a pull request. Always create a new branch (`git checkout -b <branch-name>`) before making changes, push it, and open a PR with `gh pr create --fill`. Wait for CI to pass before merging.
|
Never push directly to `dev` or `main`. All changes must be developed on a feature branch and merged via a pull request targeting `dev`. Create a branch (`git checkout -b <branch-name>`), push it, and open the PR with `tea pr create --repo TapTap/FastSync --base dev --head <branch-name>`. Wait for CI to pass before merging.
|
||||||
|
|
||||||
## Dependency Installation
|
## Dependency Installation
|
||||||
|
|
||||||
|
|||||||
@@ -27,16 +27,19 @@ FetchContent_Declare(xxhash GIT_REPOSITORY https://github.com/Cyan4973/xxHash GI
|
|||||||
FetchContent_MakeAvailable(xxhash)
|
FetchContent_MakeAvailable(xxhash)
|
||||||
|
|
||||||
# Sanitizer option
|
# Sanitizer option
|
||||||
set(SANITIZER "none" CACHE STRING "Sanitizer to enable (address, thread, none)")
|
set(SANITIZER "none" CACHE STRING "Sanitizer to enable (address, thread, undefined, none)")
|
||||||
set_property(CACHE SANITIZER PROPERTY STRINGS address thread none)
|
set_property(CACHE SANITIZER PROPERTY STRINGS address thread undefined none)
|
||||||
if(SANITIZER STREQUAL "address")
|
if(SANITIZER STREQUAL "address")
|
||||||
add_compile_options(-fsanitize=address -fno-omit-frame-pointer -g)
|
add_compile_options(-fsanitize=address -fno-omit-frame-pointer -g)
|
||||||
add_link_options(-fsanitize=address)
|
add_link_options(-fsanitize=address)
|
||||||
elseif(SANITIZER STREQUAL "thread")
|
elseif(SANITIZER STREQUAL "thread")
|
||||||
add_compile_options(-fsanitize=thread -fno-omit-frame-pointer -g)
|
add_compile_options(-fsanitize=thread -fno-omit-frame-pointer -g)
|
||||||
add_link_options(-fsanitize=thread)
|
add_link_options(-fsanitize=thread)
|
||||||
|
elseif(SANITIZER STREQUAL "undefined")
|
||||||
|
add_compile_options(-fsanitize=undefined -fno-omit-frame-pointer -g)
|
||||||
|
add_link_options(-fsanitize=undefined)
|
||||||
elseif(NOT SANITIZER STREQUAL "none")
|
elseif(NOT SANITIZER STREQUAL "none")
|
||||||
message(FATAL_ERROR "Unknown sanitizer: ${SANITIZER}. Supported values: address, thread, none")
|
message(FATAL_ERROR "Unknown sanitizer: ${SANITIZER}. Supported values: address, thread, undefined, none")
|
||||||
endif()
|
endif()
|
||||||
|
|
||||||
option(STRICT_WARNINGS "Enable strict warnings" OFF)
|
option(STRICT_WARNINGS "Enable strict warnings" OFF)
|
||||||
@@ -84,7 +87,7 @@ tests/integration/ — Python pytest integration tests
|
|||||||
### Dependencies
|
### Dependencies
|
||||||
- **zstd** — found via `find_library(ZSTD_LIBRARY zstd)`
|
- **zstd** — found via `find_library(ZSTD_LIBRARY zstd)`
|
||||||
- **OpenSSL** — found via `find_package(OpenSSL REQUIRED)` (TLS 1.2+ transport)
|
- **OpenSSL** — found via `find_package(OpenSSL REQUIRED)` (TLS 1.2+ transport)
|
||||||
- **xxHash** — fetched via `FetchContent` from GitHub (delta transfer hashing, v0.8.3)
|
- **xxHash** — fetched via `FetchContent` from the upstream repository (delta transfer hashing, v0.8.3)
|
||||||
- **pthreads** — found via `find_package(Threads REQUIRED)`
|
- **pthreads** — found via `find_package(Threads REQUIRED)`
|
||||||
- **C11 standard** — required
|
- **C11 standard** — required
|
||||||
- **CMake 3.22+** — minimum version
|
- **CMake 3.22+** — minimum version
|
||||||
@@ -94,7 +97,7 @@ tests/integration/ — Python pytest integration tests
|
|||||||
- Use `file(GLOB ...)` for source collection (existing pattern).
|
- Use `file(GLOB ...)` for source collection (existing pattern).
|
||||||
- All targets link `Threads::Threads`, `${ZSTD_LIBRARY}`, `OpenSSL::SSL`, `OpenSSL::Crypto`, and `xxhash`.
|
- All targets link `Threads::Threads`, `${ZSTD_LIBRARY}`, `OpenSSL::SSL`, `OpenSSL::Crypto`, and `xxhash`.
|
||||||
- Include directories: `src/shared`, `src/server`, `src/client`, `tests` (for test target).
|
- Include directories: `src/shared`, `src/server`, `src/client`, `tests` (for test target).
|
||||||
- Sanitizer support: pass `-DSANITIZER=address` or `-DSANITIZER=thread` to cmake (live option in CMakeLists.txt).
|
- Sanitizer support: pass `-DSANITIZER=address`, `-DSANITIZER=thread`, or `-DSANITIZER=undefined` to cmake (live option in CMakeLists.txt).
|
||||||
- Build with `cmake -B build -S . && cmake --build build -j$(nproc)`.
|
- Build with `cmake -B build -S . && cmake --build build -j$(nproc)`.
|
||||||
- For CI, dependencies are provided by the project's custom Docker image (repo-root `Dockerfile`, same image CI uses). For local development, use `nix-shell`. Never add `apt-get install` / `pip install` to CI workflows. See `AGENTS.md`.
|
- For CI, dependencies are provided by the project's custom Docker image (repo-root `Dockerfile`, same image CI uses). For local development, use `nix-shell`. Never add `apt-get install` / `pip install` to CI workflows. See `AGENTS.md`.
|
||||||
|
|
||||||
@@ -105,7 +108,7 @@ tests/integration/ — Python pytest integration tests
|
|||||||
3. Add new dependencies with `find_package` or `find_library`.
|
3. Add new dependencies with `find_package` or `find_library`.
|
||||||
4. When adding a new executable target, follow the pattern of existing targets.
|
4. When adding a new executable target, follow the pattern of existing targets.
|
||||||
5. When adding a new library (static/shared), use `add_library` and follow the project's naming.
|
5. When adding a new library (static/shared), use `add_library` and follow the project's naming.
|
||||||
6. For sanitizer builds, pass `-DSANITIZER=address` or `-DSANITIZER=thread` to cmake (matching CI's matrix strategy).
|
6. For sanitizer builds, pass `-DSANITIZER=address`, `-DSANITIZER=thread`, or `-DSANITIZER=undefined` to cmake (matching CI's matrix strategy).
|
||||||
7. Always verify the build compiles after changes.
|
7. Always verify the build compiles after changes.
|
||||||
|
|
||||||
## Sanitizer Configurations
|
## Sanitizer Configurations
|
||||||
@@ -119,11 +122,9 @@ cmake -B build -S . -DSANITIZER=thread # ThreadSanitizer (race conditions)
|
|||||||
cmake --build build -j$(nproc)
|
cmake --build build -j$(nproc)
|
||||||
```
|
```
|
||||||
|
|
||||||
For UndefinedBehaviorSanitizer (no `-DSANITIZER=undefined` option in CMakeLists.txt yet), use the manual flag approach:
|
UndefinedBehaviorSanitizer uses the same built-in option:
|
||||||
```bash
|
```bash
|
||||||
cmake -B build -S . \
|
cmake -B build -S . -DSANITIZER=undefined
|
||||||
-DCMAKE_C_FLAGS="-fsanitize=undefined -fno-omit-frame-pointer -g" \
|
|
||||||
-DCMAKE_EXE_LINKER_FLAGS="-fsanitize=undefined"
|
|
||||||
cmake --build build -j$(nproc)
|
cmake --build build -j$(nproc)
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -159,7 +160,7 @@ cmake -B build -S . -DCMAKE_BUILD_TYPE=RelWithDebInfo
|
|||||||
```bash
|
```bash
|
||||||
cmake -B build -S .
|
cmake -B build -S .
|
||||||
cmake --build build -j$(nproc)
|
cmake --build build -j$(nproc)
|
||||||
./build/server
|
./build/server -p 8080 --allow-unauthenticated
|
||||||
./build/client
|
./build/client
|
||||||
./build/tests
|
./build/tests
|
||||||
```
|
```
|
||||||
@@ -187,7 +188,7 @@ When using `tea` (the task execution agent) to run CI or tests, always set a suf
|
|||||||
|
|
||||||
## Branch Strategy
|
## Branch Strategy
|
||||||
|
|
||||||
Never push directly to `main`. All changes must be developed on a feature branch and merged via a pull request. Always create a new branch (`git checkout -b <branch-name>`) before making changes, push it, and open a PR with `gh pr create --fill`. Wait for CI to pass before merging.
|
Never push directly to `dev` or `main`. All changes must be developed on a feature branch and merged via a pull request targeting `dev`. Create a branch (`git checkout -b <branch-name>`), push it, and open the PR with `tea pr create --repo TapTap/FastSync --base dev --head <branch-name>`. Wait for CI to pass before merging.
|
||||||
|
|
||||||
## Dependency Installation
|
## Dependency Installation
|
||||||
|
|
||||||
|
|||||||
@@ -27,7 +27,7 @@ FastSync is a file synchronization tool (like rsync, but faster). It transfers f
|
|||||||
cmake -B build -S . && cmake --build build -j$(nproc)
|
cmake -B build -S . && cmake --build build -j$(nproc)
|
||||||
|
|
||||||
# Server (TCP mode)
|
# Server (TCP mode)
|
||||||
./build/server
|
./build/server -p 8080 --allow-unauthenticated
|
||||||
|
|
||||||
# Client (TCP mode)
|
# Client (TCP mode)
|
||||||
./build/client --source-dir /path/to/send --dest-dir /path/to/receive --save-to-disk
|
./build/client --source-dir /path/to/send --dest-dir /path/to/receive --save-to-disk
|
||||||
@@ -37,13 +37,13 @@ cmake -B build -S . && cmake --build build -j$(nproc)
|
|||||||
|
|
||||||
# Run tests
|
# Run tests
|
||||||
./build/tests # unit tests
|
./build/tests # unit tests
|
||||||
python3 test.py # integration tests
|
python3 -m pytest tests/integration/ -n 4 --dist=load -m "not setpriv" # integration tests
|
||||||
```
|
```
|
||||||
|
|
||||||
## Code Walkthrough
|
## Code Walkthrough
|
||||||
|
|
||||||
### Client Entry Point (`src/client/client_cli.c`)
|
### Client Entry Point (`src/client/client_cli.c`)
|
||||||
- Parses CLI arguments using `getopt_long`
|
- Parses CLI arguments using a custom option-table parser (`OPTION_TABLE` in `src/client/client_cli.c`); there is no `getopt*` usage
|
||||||
- Creates `Config` struct with all options
|
- Creates `Config` struct with all options
|
||||||
- Detects SSH destinations (contains `:`)
|
- Detects SSH destinations (contains `:`)
|
||||||
- Calls into `client_send.c` for the actual transfer
|
- Calls into `client_send.c` for the actual transfer
|
||||||
@@ -109,7 +109,7 @@ Collection of files for batch transfer. Serialized with file count, then per-fil
|
|||||||
zstd streaming compression via `ZSTD_compressStream2`/`ZSTD_decompressStream`. Compression happens per-chunk in the sender stage. Level 1-22 (default 5). Streaming means memory usage stays bounded regardless of file size.
|
zstd streaming compression via `ZSTD_compressStream2`/`ZSTD_decompressStream`. Compression happens per-chunk in the sender stage. Level 1-22 (default 5). Streaming means memory usage stays bounded regardless of file size.
|
||||||
|
|
||||||
### "How does sendfile() work?"
|
### "How does sendfile() work?"
|
||||||
On Linux, `sendfile()` copies data directly from kernel file buffer to socket, bypassing userspace. ~2x faster for large files. Enabled with `-f` flag. Only works with TCP (not SSH, not compression).
|
On Linux, `sendfile()` copies data directly from kernel file buffer to socket, bypassing userspace. ~2x faster for large files. Enabled with `--sendfile` (long form only). Only works with TCP (not SSH, not compression).
|
||||||
|
|
||||||
### "How does incremental sync work?"
|
### "How does incremental sync work?"
|
||||||
Client sends file metadata (path, size, mtime) to server. Server checks if destination file has same size+mtime. If match, server responds `STATUS_OK` (skip). If mismatch, server responds `STATUS_NEXT` (send).
|
Client sends file metadata (path, size, mtime) to server. Server checks if destination file has same size+mtime. If match, server responds `STATUS_OK` (skip). If mismatch, server responds `STATUS_NEXT` (send).
|
||||||
@@ -138,7 +138,7 @@ When using `tea` (the task execution agent) to run CI or tests, always set a suf
|
|||||||
|
|
||||||
## Branch Strategy
|
## Branch Strategy
|
||||||
|
|
||||||
Never push directly to `main`. All changes must be developed on a feature branch and merged via a pull request. Always create a new branch (`git checkout -b <branch-name>`) before making changes, push it, and open a PR with `gh pr create --fill`. Wait for CI to pass before merging.
|
Never push directly to `dev` or `main`. All changes must be developed on a feature branch and merged via a pull request targeting `dev`. Create a branch (`git checkout -b <branch-name>`), push it, and open the PR with `tea pr create --repo TapTap/FastSync --base dev --head <branch-name>`. Wait for CI to pass before merging.
|
||||||
|
|
||||||
## Dependency Installation
|
## Dependency Installation
|
||||||
|
|
||||||
|
|||||||
@@ -316,7 +316,7 @@ When using `tea` (the task execution agent) to run CI or tests, always set a suf
|
|||||||
|
|
||||||
## Branch Strategy
|
## Branch Strategy
|
||||||
|
|
||||||
Never push directly to `main`. All changes must be developed on a feature branch and merged via a pull request. Always create a new branch (`git checkout -b <branch-name>`) before making changes, push it, and open a PR with `gh pr create --fill`. Wait for CI to pass before merging.
|
Never push directly to `dev` or `main`. All changes must be developed on a feature branch and merged via a pull request targeting `dev`. Create a branch (`git checkout -b <branch-name>`), push it, and open the PR with `tea pr create --repo TapTap/FastSync --base dev --head <branch-name>`. Wait for CI to pass before merging.
|
||||||
|
|
||||||
## Dependency Installation
|
## Dependency Installation
|
||||||
|
|
||||||
|
|||||||
@@ -14,10 +14,9 @@ Diagnose crashes, memory errors, hangs, and logic bugs. You use structured debug
|
|||||||
### Memory Errors
|
### Memory Errors
|
||||||
```bash
|
```bash
|
||||||
# AddressSanitizer (fast, recommended first)
|
# AddressSanitizer (fast, recommended first)
|
||||||
cmake -B build -S . -DCMAKE_C_FLAGS="-fsanitize=address -fno-omit-frame-pointer" \
|
cmake -B build-asan -S . -DSANITIZER=address
|
||||||
-DCMAKE_EXE_LINKER_FLAGS="-fsanitize=address"
|
cmake --build build-asan -j$(nproc)
|
||||||
cmake --build build -j$(nproc)
|
./build-asan/client # or ./build-asan/server -p 8080 --allow-unauthenticated
|
||||||
./build/client # or ./build/server
|
|
||||||
|
|
||||||
# Valgrind (slower, more thorough)
|
# Valgrind (slower, more thorough)
|
||||||
valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes \
|
valgrind --leak-check=full --show-leak-kinds=all --track-origins=yes \
|
||||||
@@ -32,10 +31,9 @@ valgrind --tool=drd ./build/client ...
|
|||||||
|
|
||||||
### Thread Sanitizer
|
### Thread Sanitizer
|
||||||
```bash
|
```bash
|
||||||
cmake -B build -S . -DCMAKE_C_FLAGS="-fsanitize=thread" \
|
cmake -B build-tsan -S . -DSANITIZER=thread
|
||||||
-DCMAKE_EXE_LINKER_FLAGS="-fsanitize=thread"
|
cmake --build build-tsan -j$(nproc)
|
||||||
cmake --build build -j$(nproc)
|
./build-tsan/tests
|
||||||
./build/tests
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### GDB
|
### GDB
|
||||||
@@ -143,7 +141,7 @@ gprof ./build/client gmon.out
|
|||||||
|
|
||||||
### Step 5: Verify
|
### Step 5: Verify
|
||||||
- Run `./build/tests` (unit tests)
|
- Run `./build/tests` (unit tests)
|
||||||
- Run `python3 test.py` (integration tests)
|
- Run `python3 -m pytest tests/integration/ -n 4 --dist=load -m "not setpriv"` (integration tests)
|
||||||
- Run under valgrind again to confirm clean
|
- Run under valgrind again to confirm clean
|
||||||
- Test under ASan again
|
- Test under ASan again
|
||||||
|
|
||||||
@@ -162,7 +160,7 @@ When using `tea` (the task execution agent) to run CI or tests, always set a suf
|
|||||||
|
|
||||||
## Branch Strategy
|
## Branch Strategy
|
||||||
|
|
||||||
Never push directly to `main`. All changes must be developed on a feature branch and merged via a pull request. Always create a new branch (`git checkout -b <branch-name>`) before making changes, push it, and open a PR with `gh pr create --fill`. Wait for CI to pass before merging.
|
Never push directly to `dev` or `main`. All changes must be developed on a feature branch and merged via a pull request targeting `dev`. Create a branch (`git checkout -b <branch-name>`), push it, and open the PR with `tea pr create --repo TapTap/FastSync --base dev --head <branch-name>`. Wait for CI to pass before merging.
|
||||||
|
|
||||||
## Dependency Installation
|
## Dependency Installation
|
||||||
|
|
||||||
|
|||||||
@@ -96,7 +96,7 @@ When using `tea` (the task execution agent) to run CI or tests, always set a suf
|
|||||||
|
|
||||||
## Branch Strategy
|
## Branch Strategy
|
||||||
|
|
||||||
Never push directly to `main`. All changes must be developed on a feature branch and merged via a pull request. Always create a new branch (`git checkout -b <branch-name>`) before making changes, push it, and open a PR with `gh pr create --fill`. Wait for CI to pass before merging.
|
Never push directly to `dev` or `main`. All changes must be developed on a feature branch and merged via a pull request targeting `dev`. Create a branch (`git checkout -b <branch-name>`), push it, and open the PR with `tea pr create --repo TapTap/FastSync --base dev --head <branch-name>`. Wait for CI to pass before merging.
|
||||||
|
|
||||||
## Dependency Installation
|
## Dependency Installation
|
||||||
|
|
||||||
|
|||||||
@@ -16,12 +16,18 @@ Scan the codebase for patterns that suggest new feature opportunities. You ident
|
|||||||
### Module Map
|
### Module Map
|
||||||
```
|
```
|
||||||
src/client/ Client-side: CLI parsing, scanning, sending
|
src/client/ Client-side: CLI parsing, scanning, sending
|
||||||
client_cli.c Entry point, argument parsing, config setup
|
client_cli.c Entry point, OPTION_TABLE parser, config setup
|
||||||
|
usage.c Usage/help text (authoritative CLI flag list)
|
||||||
client_send.c Transfer orchestration, pipeline management
|
client_send.c Transfer orchestration, pipeline management
|
||||||
|
client_validation.c Destination/CLI validation
|
||||||
scanner.c BFS directory traversal, chunk building
|
scanner.c BFS directory traversal, chunk building
|
||||||
|
change_list.c File change-list bookkeeping
|
||||||
|
|
||||||
src/server/ Server-side: listening, receiving, writing
|
src/server/ Server-side: listening, receiving, writing
|
||||||
server.c TCP accept loop, per-connection handling
|
server.c TCP accept loop, per-connection handling
|
||||||
|
server_cli.c Server option-table CLI parsing
|
||||||
|
receiver.c Receiver-side file handling
|
||||||
|
receiver_pipeline.c Receiver worker pipeline
|
||||||
|
|
||||||
src/shared/ Shared libraries (used by both client and server)
|
src/shared/ Shared libraries (used by both client and server)
|
||||||
protocol.c/h Wire protocol: status codes, send/receive primitives
|
protocol.c/h Wire protocol: status codes, send/receive primitives
|
||||||
@@ -32,40 +38,63 @@ src/shared/ Shared libraries (used by both client and server)
|
|||||||
data.c/h Generic buffer type (Data)
|
data.c/h Generic buffer type (Data)
|
||||||
metadata.c/h File metadata (mode, uid, gid, mtime)
|
metadata.c/h File metadata (mode, uid, gid, mtime)
|
||||||
file.c/h File representation
|
file.c/h File representation
|
||||||
|
file_send.c/h Sender-side file transfer
|
||||||
|
file_receive.c/h Receiver-side file transfer
|
||||||
|
file_list.c/h File list model
|
||||||
|
file_store.c/h Destination file store
|
||||||
array_list.c/h Dynamic array
|
array_list.c/h Dynamic array
|
||||||
|
delta.c/h Delta transfer algorithm
|
||||||
|
checksum.c/h Whole-file/block checksums (xxHash, md5)
|
||||||
|
filter.c/h rsync-style filter rules
|
||||||
|
batch.c/h Batch files (--write-batch/--read-batch)
|
||||||
|
charset.c/h Filename charset conversion (--iconv)
|
||||||
|
chmod.c/h Permission modification (--chmod)
|
||||||
|
xattr.c/h Extended attributes
|
||||||
|
hardlink.c/h Hard-link handling
|
||||||
|
identity.c/h uid/gid mapping (--usermap/--groupmap/--chown)
|
||||||
|
credentials.c/h Daemon credentials
|
||||||
|
daemon_conf.c/h Daemon module configuration
|
||||||
|
motd.c/h Daemon MOTD
|
||||||
|
delay_updates.c/h Delayed update staging
|
||||||
|
stop_condition.c/h Stop-after/stop-at handling
|
||||||
transport_tcp.c/h TCP client/server with sendfile() zero-copy
|
transport_tcp.c/h TCP client/server with sendfile() zero-copy
|
||||||
transport_ssh.c/h SSH transport with ControlMaster
|
transport_ssh.c/h SSH transport with ControlMaster
|
||||||
transport_tls.c/h TLS encryption via OpenSSL
|
transport_tls.c/h TLS encryption via OpenSSL
|
||||||
multiprocessing.c/h Fork-based concurrency
|
multiprocessing.c/h Fork-based concurrency
|
||||||
log.c/h Logging utilities
|
log.c/h Logging utilities
|
||||||
utils.c/h Shared utilities
|
utils.c/h Shared utilities
|
||||||
|
file_types.h Shared file type definitions
|
||||||
```
|
```
|
||||||
|
|
||||||
### Existing CLI Flags (from client_cli.c)
|
### Existing CLI Flags (authoritative source: `src/client/usage.c`)
|
||||||
```
|
```
|
||||||
--source-dir <dir> Source directory to sync (required)
|
--source-dir <dir> Source directory
|
||||||
--dest-dir <dir> Destination directory on server (required)
|
--dest-dir <dir> Destination directory on server
|
||||||
--host <host> Server hostname/IP (required)
|
--server-host <ip> Server IP address (default: 127.0.0.1)
|
||||||
--port <port> Server TCP port
|
--server-port <n> Server port (default: 8080); --port is an alias
|
||||||
--server-mode Listen as server
|
-c, --checksum Verify content by checksum instead of size+mtime
|
||||||
--use-compression, -c Enable zstd compression
|
-z, --compress [level] Enable compression (level 1-22, default 5)
|
||||||
--use-multithreading, -m Enable multithreaded transfer
|
-j, --threads[=N] Enable multithreaded scanner/loader/sender pipeline
|
||||||
--use-sendfile, -s Use sendfile() zero-copy TCP
|
--chunk-serialization Enable chunk serialization (long form only)
|
||||||
--use-ssh, -S Use SSH transport
|
--sendfile sendfile() zero-copy (TCP only; long form only)
|
||||||
--use-tls, -T Enable TLS encryption
|
-s, --secluded-args Protect-args compatibility option (no effect)
|
||||||
--cert <file> TLS certificate file
|
--tls Enable TLS encryption; --cert/--key/--ca give PEMs
|
||||||
--key <file> TLS key file
|
--bwlimit <KB/s> Bandwidth limit in kilobytes per second
|
||||||
--ca <file> TLS CA certificate file
|
--delete Delete files on receiver not in source
|
||||||
--insecure Skip TLS verification
|
--incremental Skip files unchanged since last transfer
|
||||||
--bwlimit <bytes/s> Bandwidth limit
|
--delta Delta transfer for changed files (needs --incremental)
|
||||||
--delete Delete files not in source
|
-f, --filter=RULE rsync-style filter rule (+/- include/exclude)
|
||||||
--include <pattern> Include filter pattern
|
--exclude <pattern> Exclude files matching pattern
|
||||||
--exclude <pattern> Exclude filter pattern
|
--include <pattern> Only include files matching pattern
|
||||||
--dry-run Print what would be transferred
|
-m, --prune-empty-dirs Do not transfer empty directory entries
|
||||||
--save-to-disk Save transferred files to disk (for server tests)
|
-n, --dry-run Show what would be transferred
|
||||||
|
--save-to-disk Write received files to disk
|
||||||
--version Print version and exit
|
--version Print version and exit
|
||||||
--help Print help
|
--help Show help
|
||||||
```
|
```
|
||||||
|
> Always confirm the current flags with `./build/client --help`; the table above
|
||||||
|
> is a representative subset. `src/client/usage.c` is the authoritative list and
|
||||||
|
> `OPTION_TABLE` in `src/client/client_cli.c` is the parser (there is no `getopt*`).
|
||||||
|
|
||||||
## Feature Scout Checklist
|
## Feature Scout Checklist
|
||||||
|
|
||||||
@@ -288,7 +317,7 @@ When using `tea` (the task execution agent) to run CI or tests, always set a suf
|
|||||||
|
|
||||||
## Branch Strategy
|
## Branch Strategy
|
||||||
|
|
||||||
Never push directly to `main`. All changes must be developed on a feature branch and merged via a pull request. Always create a new branch (`git checkout -b <branch-name>`) before making changes, push it, and open a PR with `gh pr create --fill`. Wait for CI to pass before merging.
|
Never push directly to `dev` or `main`. All changes must be developed on a feature branch and merged via a pull request targeting `dev`. Create a branch (`git checkout -b <branch-name>`), push it, and open the PR with `tea pr create --repo TapTap/FastSync --base dev --head <branch-name>`. Wait for CI to pass before merging.
|
||||||
|
|
||||||
## Dependency Installation
|
## Dependency Installation
|
||||||
|
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ Design integration tests that verify the full transfer pipeline works end-to-end
|
|||||||
- Multiple configurations (TCP, SSH, TLS, compression, multithreading)
|
- Multiple configurations (TCP, SSH, TLS, compression, multithreading)
|
||||||
- Network shaping (LAN, WAN profiles)
|
- Network shaping (LAN, WAN profiles)
|
||||||
- Feature tests (dry run, archive, exclude, delete, incremental, bandwidth limit)
|
- Feature tests (dry run, archive, exclude, delete, incremental, bandwidth limit)
|
||||||
- Run: `python3 -m pytest tests/ -v --tb=short`
|
- Run: `python3 -m pytest tests/integration/ -n 4 --dist=load -m "not setpriv"`
|
||||||
|
|
||||||
### 3. New: Focused Integration Tests
|
### 3. New: Focused Integration Tests
|
||||||
When adding new features or fixing bugs, write targeted integration tests.
|
When adding new features or fixing bugs, write targeted integration tests.
|
||||||
@@ -35,13 +35,14 @@ mkdir -p /tmp/fastsync_test/src
|
|||||||
echo "test content" > /tmp/fastsync_test/src/file.txt
|
echo "test content" > /tmp/fastsync_test/src/file.txt
|
||||||
|
|
||||||
# Start server
|
# Start server
|
||||||
./build/server &
|
./build/server -p 8080 --allow-unauthenticated &
|
||||||
SERVER_PID=$!
|
SERVER_PID=$!
|
||||||
sleep 0.5
|
sleep 0.5
|
||||||
|
|
||||||
# Run client
|
# Run client
|
||||||
./build/client --source-dir /tmp/fastsync_test/src \
|
./build/client --source-dir /tmp/fastsync_test/src \
|
||||||
--dest-dir /tmp/fastsync_test/dst \
|
--dest-dir /tmp/fastsync_test/dst \
|
||||||
|
--server-port 8080 \
|
||||||
--save-to-disk
|
--save-to-disk
|
||||||
|
|
||||||
# Verify
|
# Verify
|
||||||
@@ -76,7 +77,7 @@ openssl req -x509 -newkey rsa:2048 -keyout /tmp/key.pem -out /tmp/cert.pem \
|
|||||||
### Pattern 4: Incremental Sync
|
### Pattern 4: Incremental Sync
|
||||||
```bash
|
```bash
|
||||||
# First sync
|
# First sync
|
||||||
./build/client --source-dir /tmp/src --dest-dir /tmp/dst --save-to-disk -M
|
./build/client --source-dir /tmp/src --dest-dir /tmp/dst --save-to-disk
|
||||||
|
|
||||||
# Modify source
|
# Modify source
|
||||||
echo "updated" >> /tmp/src/file.txt
|
echo "updated" >> /tmp/src/file.txt
|
||||||
@@ -89,14 +90,14 @@ echo "updated" >> /tmp/src/file.txt
|
|||||||
### Pattern 5: Delete Verification
|
### Pattern 5: Delete Verification
|
||||||
```bash
|
```bash
|
||||||
# Initial sync
|
# Initial sync
|
||||||
./build/client --source-dir /tmp/src --dest-dir /tmp/dst --save-to-disk -M
|
./build/client --source-dir /tmp/src --dest-dir /tmp/dst --save-to-disk
|
||||||
|
|
||||||
# Add extra file to dest
|
# Add extra file to dest
|
||||||
echo "extra" > /tmp/dst/.../extra.txt
|
echo "extra" > /tmp/dst/.../extra.txt
|
||||||
|
|
||||||
# Sync with --delete
|
# Sync with --delete
|
||||||
./build/client --source-dir /tmp/src --dest-dir /tmp/dst \
|
./build/client --source-dir /tmp/src --dest-dir /tmp/dst \
|
||||||
--save-to-disk --delete -M
|
--save-to-disk --delete
|
||||||
|
|
||||||
# Verify extra.txt is gone
|
# Verify extra.txt is gone
|
||||||
test ! -f /tmp/dst/.../extra.txt
|
test ! -f /tmp/dst/.../extra.txt
|
||||||
@@ -115,7 +116,7 @@ The project uses Gitea Actions. Key jobs:
|
|||||||
jobs:
|
jobs:
|
||||||
new-job:
|
new-job:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
container: gitea.tap-tap.win/taptap/fastsync-ci:v7
|
container: gitea.tap-tap.win/taptap/fastsync-ci:v10
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
- name: Configure
|
- name: Configure
|
||||||
@@ -127,7 +128,7 @@ jobs:
|
|||||||
- name: Unit Tests
|
- name: Unit Tests
|
||||||
run: ./build-${{ matrix.sanitizer }}/tests
|
run: ./build-${{ matrix.sanitizer }}/tests
|
||||||
- name: Integration Tests
|
- name: Integration Tests
|
||||||
run: LSAN_OPTIONS=suppressions=.lsan-suppressions.txt python3 -m pytest tests/ -v --tb=short
|
run: LSAN_OPTIONS=suppressions=.lsan-suppressions.txt python3 -m pytest tests/integration/ -n 4 --dist=load -m "not setpriv"
|
||||||
```
|
```
|
||||||
The symlink step is required because `tests/conftest.py` expects `./build` to exist.
|
The symlink step is required because `tests/conftest.py` expects `./build` to exist.
|
||||||
|
|
||||||
@@ -135,7 +136,7 @@ The symlink step is required because `tests/conftest.py` expects `./build` to ex
|
|||||||
|
|
||||||
After any code change:
|
After any code change:
|
||||||
- [ ] Unit tests pass: `./build/tests`
|
- [ ] Unit tests pass: `./build/tests`
|
||||||
- [ ] Integration tests pass: `python3 -m pytest tests/ -v --tb=short`
|
- [ ] Integration tests pass: `python3 -m pytest tests/integration/ -n 4 --dist=load -m "not setpriv"`
|
||||||
- [ ] Build clean: no warnings with `-Wall`
|
- [ ] Build clean: no warnings with `-Wall`
|
||||||
- [ ] No memory errors: ASan clean
|
- [ ] No memory errors: ASan clean
|
||||||
- [ ] No thread errors: TSan clean (if threading involved)
|
- [ ] No thread errors: TSan clean (if threading involved)
|
||||||
@@ -156,7 +157,7 @@ When using `tea` (the task execution agent) to run CI or tests, always set a suf
|
|||||||
|
|
||||||
## Branch Strategy
|
## Branch Strategy
|
||||||
|
|
||||||
Never push directly to `main`. All changes must be developed on a feature branch and merged via a pull request. Always create a new branch (`git checkout -b <branch-name>`) before making changes, push it, and open a PR with `gh pr create --fill`. Wait for CI to pass before merging.
|
Never push directly to `dev` or `main`. All changes must be developed on a feature branch and merged via a pull request targeting `dev`. Create a branch (`git checkout -b <branch-name>`), push it, and open the PR with `tea pr create --repo TapTap/FastSync --base dev --head <branch-name>`. Wait for CI to pass before merging.
|
||||||
|
|
||||||
## Dependency Installation
|
## Dependency Installation
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
---
|
---
|
||||||
description: Top-level orchestrator that analyzes the FastSync codebase by delegating to specialized sub-agents and creates GitHub issues from their findings.
|
description: Top-level orchestrator that analyzes the FastSync codebase by delegating to specialized sub-agents and creates Gitea issues from their findings.
|
||||||
mode: subagent
|
mode: subagent
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -12,7 +12,7 @@ You are the primary orchestrator agent. Your job is to:
|
|||||||
2. Decide which specialized sub-agents to dispatch for analysis
|
2. Decide which specialized sub-agents to dispatch for analysis
|
||||||
3. Delegate analysis work using the task tool
|
3. Delegate analysis work using the task tool
|
||||||
4. Receive structured findings from sub-agents
|
4. Receive structured findings from sub-agents
|
||||||
5. Create GitHub issues from those findings using `gh issue create`
|
5. Create Gitea issues from those findings using `tea issues create`
|
||||||
6. Coordinate the overall analysis workflow end-to-end
|
6. Coordinate the overall analysis workflow end-to-end
|
||||||
|
|
||||||
> **Environment rule:** for CI, dependency installation must use the project's custom Docker image (repo-root `Dockerfile`, same as CI). For local development, use `nix-shell` (see `README.md`). See `AGENTS.md`.
|
> **Environment rule:** for CI, dependency installation must use the project's custom Docker image (repo-root `Dockerfile`, same as CI). For local development, use `nix-shell` (see `README.md`). See `AGENTS.md`.
|
||||||
@@ -97,7 +97,7 @@ First, read the repository structure to understand what exists:
|
|||||||
### Phase 2: Determine Analysis Scope
|
### Phase 2: Determine Analysis Scope
|
||||||
Based on what the user requests or what needs attention:
|
Based on what the user requests or what needs attention:
|
||||||
- **New features wanted?** → Dispatch `feature-scout` sub-agent
|
- **New features wanted?** → Dispatch `feature-scout` sub-agent
|
||||||
- **Security audit needed?** → Dispatch `security-screener` sub-agent
|
- **Security audit needed?** → Dispatch `security-auditor` sub-agent
|
||||||
- **Code quality review?** → Dispatch `code-quality-guardian` sub-agent
|
- **Code quality review?** → Dispatch `code-quality-guardian` sub-agent
|
||||||
- **All of the above?** → Run all three in parallel
|
- **All of the above?** → Run all three in parallel
|
||||||
|
|
||||||
@@ -110,7 +110,7 @@ Context: <provide summary of what was found in Phase 1>
|
|||||||
```
|
```
|
||||||
|
|
||||||
```
|
```
|
||||||
Task: Ask the security-screener agent to analyze the codebase.
|
Task: Ask the security-auditor agent to analyze the codebase.
|
||||||
Context: <provide summary of what was found in Phase 1>
|
Context: <provide summary of what was found in Phase 1>
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -138,14 +138,14 @@ Each sub-agent returns findings in this structured format:
|
|||||||
- **Labels**: comma-separated labels for the issue
|
- **Labels**: comma-separated labels for the issue
|
||||||
```
|
```
|
||||||
|
|
||||||
### Phase 5: Create GitHub Issues
|
### Phase 5: Create Gitea Issues
|
||||||
For each finding, create a GitHub issue:
|
For each finding, create a Gitea issue:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
gh issue create \
|
tea issues create --repo TapTap/FastSync \
|
||||||
--title "<Finding Title>" \
|
--title "<Finding Title>" \
|
||||||
--label "<labels>" \
|
--labels "<labels>" \
|
||||||
--body "## Description
|
--description "## Description
|
||||||
<description>
|
<description>
|
||||||
|
|
||||||
## Location
|
## Location
|
||||||
@@ -175,11 +175,13 @@ _This issue was automatically generated by the issue-creator agent._"
|
|||||||
|
|
||||||
### Duplicate Detection
|
### Duplicate Detection
|
||||||
Before creating an issue:
|
Before creating an issue:
|
||||||
1. Check existing open issues: `gh issue list --state open --label "<label>"`
|
1. Check existing open issues: `tea issues list --repo TapTap/FastSync --state open --labels "<label>"`
|
||||||
2. Search for similar titles using `gh issue list --search "<keywords>"`
|
2. Search for similar titles using `tea issues list --repo TapTap/FastSync --keyword "<keywords>"`
|
||||||
3. If a similar issue exists, add a comment instead of creating a duplicate:
|
3. If a similar issue exists, add a comment instead of creating a duplicate:
|
||||||
```bash
|
```bash
|
||||||
gh issue comment <issue-number> --body "Additional finding from automated analysis: <details>"
|
tea comment --repo TapTap/FastSync <issue-number> "Additional finding from automated analysis: <details>"
|
||||||
|
# or POST to the Gitea API:
|
||||||
|
# POST https://gitea.tap-tap.win/api/v1/repos/TapTap/FastSync/issues/<n>/comments
|
||||||
```
|
```
|
||||||
|
|
||||||
## Sub-Agent Reference
|
## Sub-Agent Reference
|
||||||
@@ -189,13 +191,12 @@ Before creating an issue:
|
|||||||
| Agent | File | Purpose |
|
| Agent | File | Purpose |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| feature-scout | `.opencode/agents/feature-scout.md` | Scans for feature opportunities |
|
| feature-scout | `.opencode/agents/feature-scout.md` | Scans for feature opportunities |
|
||||||
| security-screener | `.opencode/agents/security-screener.md` | Scans for security vulnerabilities |
|
| security-auditor | `.opencode/agents/security-auditor.md` | Security audits and vulnerability scans |
|
||||||
| code-quality-guardian | `.opencode/agents/code-quality-guardian.md` | Scans for code quality improvements |
|
| code-quality-guardian | `.opencode/agents/code-quality-guardian.md` | Scans for code quality improvements |
|
||||||
| architect | `.opencode/agents/architect.md` | Architecture reviews |
|
| architect | `.opencode/agents/architect.md` | Architecture reviews |
|
||||||
| c-reviewer | `.opencode/agents/c-reviewer.md` | C code correctness reviews |
|
| c-reviewer | `.opencode/agents/c-reviewer.md` | C code correctness reviews |
|
||||||
| debugger | `.opencode/agents/debugger.md` | Bug diagnosis |
|
| debugger | `.opencode/agents/debugger.md` | Bug diagnosis |
|
||||||
| refactorer | `.opencode/agents/refactorer.md` | Code refactoring |
|
| refactorer | `.opencode/agents/refactorer.md` | Code refactoring |
|
||||||
| security-auditor | `.opencode/agents/security-auditor.md` | Security audits |
|
|
||||||
| test-writer | `.opencode/agents/test-writer.md` | Test development |
|
| test-writer | `.opencode/agents/test-writer.md` | Test development |
|
||||||
| perf-analyst | `.opencode/agents/perf-analyst.md` | Performance analysis |
|
| perf-analyst | `.opencode/agents/perf-analyst.md` | Performance analysis |
|
||||||
| protocol-designer | `.opencode/agents/protocol-designer.md` | Protocol design |
|
| protocol-designer | `.opencode/agents/protocol-designer.md` | Protocol design |
|
||||||
@@ -242,7 +243,7 @@ tests/test_file.c — File tests
|
|||||||
tests/test_transport_tcp.c — TCP transport tests
|
tests/test_transport_tcp.c — TCP transport tests
|
||||||
tests/test_transport_tls.c — TLS transport tests
|
tests/test_transport_tls.c — TLS transport tests
|
||||||
tests/test_array_list.c — Array list tests
|
tests/test_array_list.c — Array list tests
|
||||||
tests/pytest/ — Python integration tests
|
tests/integration/ — Python pytest integration tests
|
||||||
```
|
```
|
||||||
|
|
||||||
### Build & Config Files
|
### Build & Config Files
|
||||||
@@ -259,7 +260,7 @@ When using `tea` (the task execution agent) to run CI or tests, always set a suf
|
|||||||
|
|
||||||
## Branch Strategy
|
## Branch Strategy
|
||||||
|
|
||||||
Never push directly to `main`. All changes must be developed on a feature branch and merged via a pull request. Always create a new branch (`git checkout -b <branch-name>`) before making changes, push it, and open a PR with `gh pr create --fill`. Wait for CI to pass before merging.
|
Never push directly to `dev` or `main`. All changes must be developed on a feature branch and merged via a pull request targeting `dev`. Create a branch (`git checkout -b <branch-name>`), push it, and open the PR with `tea pr create --repo TapTap/FastSync --base dev --head <branch-name>`. Wait for CI to pass before merging.
|
||||||
|
|
||||||
## Dependency Installation
|
## Dependency Installation
|
||||||
|
|
||||||
|
|||||||
@@ -56,9 +56,11 @@ DirectoryScanner → Queue(Scanner→Loader) → ChunkBuilder → Queue(Loader
|
|||||||
|
|
||||||
### Benchmark Context
|
### Benchmark Context
|
||||||
|
|
||||||
From README benchmarks (25MB mixed files, localhost):
|
Use the maintained benchmark tool — do not cite stale README numbers:
|
||||||
- Best config: `-m -c` (multithread + compression) → 0.20s, 11.2× faster than rsync
|
- `python3 benchmark/bench.py` runs the repeatable throughput benchmark.
|
||||||
- `sendfile()` bypasses userspace → ~2× faster on localhost
|
- The real flags are `-j` (multithreading) and `-z` (compression); a fast loopback
|
||||||
|
config combines `-j -z`.
|
||||||
|
- `sendfile()` (via `--sendfile`) bypasses userspace → ~2× faster on localhost
|
||||||
- Compression reduces wire data enough that transfer becomes latency-bound on WAN
|
- Compression reduces wire data enough that transfer becomes latency-bound on WAN
|
||||||
|
|
||||||
## Output Format
|
## Output Format
|
||||||
@@ -120,6 +122,7 @@ time ./build/client [args...]
|
|||||||
|
|
||||||
# High precision
|
# High precision
|
||||||
perf stat -e task-clock ./build/client [args...]
|
perf stat -e task-clock ./build/client [args...]
|
||||||
|
```
|
||||||
|
|
||||||
## CI & Task Execution
|
## CI & Task Execution
|
||||||
|
|
||||||
@@ -127,9 +130,8 @@ When using `tea` (the task execution agent) to run CI or tests, always set a suf
|
|||||||
|
|
||||||
## Branch Strategy
|
## Branch Strategy
|
||||||
|
|
||||||
Never push directly to `main`. All changes must be developed on a feature branch and merged via a pull request. Always create a new branch (`git checkout -b <branch-name>`) before making changes, push it, and open a PR with `gh pr create --fill`. Wait for CI to pass before merging.
|
Never push directly to `dev` or `main`. All changes must be developed on a feature branch and merged via a pull request targeting `dev`. Create a branch (`git checkout -b <branch-name>`), push it, and open the PR with `tea pr create --repo TapTap/FastSync --base dev --head <branch-name>`. Wait for CI to pass before merging.
|
||||||
|
|
||||||
## Dependency Installation
|
## Dependency Installation
|
||||||
|
|
||||||
**CI rule:** never add `apt-get install` / `pip install` steps to CI workflows — use the custom Docker image instead. **Host rule:** for local development, use `nix-shell` (see `README.md`) which provides zstd, OpenSSL, CMake, and gcc. See `AGENTS.md` for details.
|
**CI rule:** never add `apt-get install` / `pip install` steps to CI workflows — use the custom Docker image instead. **Host rule:** for local development, use `nix-shell` (see `README.md`) which provides zstd, OpenSSL, CMake, and gcc. See `AGENTS.md` for details.
|
||||||
```
|
|
||||||
|
|||||||
@@ -91,7 +91,7 @@ When using `tea` (the task execution agent) to run CI or tests, always set a suf
|
|||||||
|
|
||||||
## Branch Strategy
|
## Branch Strategy
|
||||||
|
|
||||||
Never push directly to `main`. All changes must be developed on a feature branch and merged via a pull request. Always create a new branch (`git checkout -b <branch-name>`) before making changes, push it, and open a PR with `gh pr create --fill`. Wait for CI to pass before merging.
|
Never push directly to `dev` or `main`. All changes must be developed on a feature branch and merged via a pull request targeting `dev`. Create a branch (`git checkout -b <branch-name>`), push it, and open the PR with `tea pr create --repo TapTap/FastSync --base dev --head <branch-name>`. Wait for CI to pass before merging.
|
||||||
|
|
||||||
## Dependency Installation
|
## Dependency Installation
|
||||||
|
|
||||||
|
|||||||
@@ -160,7 +160,7 @@ When using `tea` (the task execution agent) to run CI or tests, always set a suf
|
|||||||
|
|
||||||
## Branch Strategy
|
## Branch Strategy
|
||||||
|
|
||||||
Never push directly to `main`. All changes must be developed on a feature branch and merged via a pull request. Always create a new branch (`git checkout -b <branch-name>`) before making changes, push it, and open a PR with `gh pr create --fill`. Wait for CI to pass before merging.
|
Never push directly to `dev` or `main`. All changes must be developed on a feature branch and merged via a pull request targeting `dev`. Create a branch (`git checkout -b <branch-name>`), push it, and open the PR with `tea pr create --repo TapTap/FastSync --base dev --head <branch-name>`. Wait for CI to pass before merging.
|
||||||
|
|
||||||
## Dependency Installation
|
## Dependency Installation
|
||||||
|
|
||||||
|
|||||||
@@ -3,25 +3,64 @@ description: Audits FastSync for security vulnerabilities — TLS config, input
|
|||||||
mode: subagent
|
mode: subagent
|
||||||
---
|
---
|
||||||
|
|
||||||
You are a security auditor for the FastSync project — a high-performance file synchronization system written in C11 with TCP, SSH, and TLS transport.
|
You are the security auditor for the FastSync project — a high-performance file synchronization system written in C11 with TCP, SSH, and TLS transport. This is the single canonical security agent.
|
||||||
|
|
||||||
## Your Role
|
## Your Role
|
||||||
|
|
||||||
Audit the codebase for security vulnerabilities. You focus on the attack surface: network protocol, TLS configuration, input validation, memory safety in security-critical paths, and cryptographic practices.
|
Audit the codebase for security vulnerabilities. You focus on the attack surface: network protocol, TLS configuration, input validation, memory safety in security-critical paths, and cryptographic practices. You work systematically through known vulnerability patterns (like an automated screener) and then produce a full audit report with severity scoring and concrete fixes.
|
||||||
|
|
||||||
## Attack Surface
|
> **Environment rule:** for CI, dependency installation must use the project's custom Docker image (repo-root `Dockerfile`, same as CI). For local development, use `nix-shell` (see `README.md`). See `AGENTS.md`.
|
||||||
|
|
||||||
### Network Input Points
|
## Project Architecture
|
||||||
1. **TCP server** (`src/server/server.c`) — accepts connections from any client
|
|
||||||
2. **SSH transport** (`src/shared/transport_ssh.c`) — receives data via stdio pipe
|
|
||||||
3. **Protocol parsing** (`src/shared/protocol.c`) — deserializes all incoming data
|
|
||||||
4. **Config deserialization** (`src/shared/config.c`) — receives remote config
|
|
||||||
5. **Chunk deserialization** (`src/shared/chunk.c`) — receives file batches
|
|
||||||
|
|
||||||
### TLS Configuration
|
### Module Map
|
||||||
- OpenSSL TLS 1.2+ via `src/shared/transport_tls.c`
|
```
|
||||||
- Certificate/key loading, CA verification
|
src/client/ Client-side: CLI parsing, scanning, sending
|
||||||
- SSL context setup, cipher suite selection
|
client_cli.c Entry point, argument parsing, config setup
|
||||||
|
client_send.c Transfer orchestration, pipeline management
|
||||||
|
client_validation.c Destination/CLI validation
|
||||||
|
scanner.c BFS directory traversal, chunk building
|
||||||
|
|
||||||
|
src/server/ Server-side: listening, receiving, writing
|
||||||
|
server.c TCP accept loop, per-connection handling
|
||||||
|
receiver.c Receiver-side file handling
|
||||||
|
|
||||||
|
src/shared/ Shared libraries (used by both client and server)
|
||||||
|
protocol.c/h Wire protocol: status codes, send/receive primitives
|
||||||
|
compression.c/h zstd streaming compression/decompression
|
||||||
|
chunk.c/h File grouping and batch serialization
|
||||||
|
queue.c/h Thread-safe bounded queue (producer-consumer)
|
||||||
|
config.c/h Runtime configuration, serialization, parsing
|
||||||
|
data.c/h Generic buffer type (Data)
|
||||||
|
metadata.c/h File metadata (mode, uid, gid, mtime)
|
||||||
|
file.c/h File representation
|
||||||
|
file_receive.c/h Receiver-side file transfer
|
||||||
|
file_store.c/h Destination file store
|
||||||
|
delta.c/h Delta transfer algorithm
|
||||||
|
checksum.c/h Whole-file/block checksums (xxHash, md5)
|
||||||
|
filter.c/h rsync-style filter rules
|
||||||
|
xattr.c/h Extended attributes
|
||||||
|
identity.c/h uid/gid mapping
|
||||||
|
credentials.c/h Daemon credentials
|
||||||
|
transport_tcp.c/h TCP client/server with sendfile() zero-copy
|
||||||
|
transport_ssh.c/h SSH transport with ControlMaster
|
||||||
|
transport_tls.c/h TLS encryption via OpenSSL
|
||||||
|
multiprocessing.c/h Fork-based concurrency
|
||||||
|
log.c/h Logging utilities
|
||||||
|
utils.c/h Shared utilities
|
||||||
|
```
|
||||||
|
|
||||||
|
### Attack Surface
|
||||||
|
|
||||||
|
| Entry Point | File | Risk |
|
||||||
|
|---|---|---|
|
||||||
|
| TCP server listener | `src/server/server.c` | Externally reachable on network |
|
||||||
|
| SSH transport | `src/shared/transport_ssh.c` | Accepts data via stdio pipe |
|
||||||
|
| Protocol parser | `src/shared/protocol.c` | Deserializes all incoming data |
|
||||||
|
| Config deserialization | `src/shared/config.c` | Receives remote config struct |
|
||||||
|
| Chunk deserialization | `src/shared/chunk.c` | Receives file batches |
|
||||||
|
| TLS handshake | `src/shared/transport_tls.c` | SSL context and cert validation |
|
||||||
|
| File writer | `src/server/server.c` / `receiver.c` | Writes received files to disk |
|
||||||
|
|
||||||
## Security Audit Checklist
|
## Security Audit Checklist
|
||||||
|
|
||||||
@@ -33,51 +72,182 @@ Audit the codebase for security vulnerabilities. You focus on the attack surface
|
|||||||
- [ ] Chunk count and file count validated before allocation
|
- [ ] Chunk count and file count validated before allocation
|
||||||
- [ ] Config field lengths bounded
|
- [ ] Config field lengths bounded
|
||||||
|
|
||||||
### 2. Buffer Safety
|
### 2. Buffer Overflow Risks
|
||||||
- [ ] No `strcpy` — use `snprintf` or `strncpy` with null termination
|
|
||||||
- [ ] `malloc` size calculations don't overflow (e.g., `count * sizeof(...)`)
|
|
||||||
- [ ] No fixed-size stack buffers for unbounded input
|
|
||||||
- [ ] `receive_n_data` always checks return value
|
|
||||||
- [ ] Off-by-one in path concatenation
|
|
||||||
|
|
||||||
### 3. Memory Safety in Error Paths
|
Search for these dangerous patterns in all `.c` and `.h` files:
|
||||||
- [ ] All error paths free allocated resources
|
|
||||||
- [ ] No use-after-free on error paths
|
|
||||||
- [ ] No double-free on error paths
|
|
||||||
- [ ] Partial reads handled (don't use incomplete data)
|
|
||||||
|
|
||||||
### 4. TLS/SSL Security
|
- [ ] **Fixed-size stack buffers** used for unbounded or network-provided data
|
||||||
- [ ] TLS 1.2 minimum enforced (no SSLv3, TLS 1.0, TLS 1.1)
|
```c
|
||||||
- [ ] Certificate verification enabled when CA provided
|
char path[PATH_MAX]; // OK if PATH_MAX is used, bad if size is arbitrary
|
||||||
- [ ] Certificate verification disabled only with explicit warning
|
char buf[1024]; // SUSPICIOUS — what limits the input to 1024?
|
||||||
- [ ] Private key file permissions checked
|
char line[4096]; // SUSPICIOUS — what limits the line length?
|
||||||
- [ ] No hardcoded certificates or keys
|
```
|
||||||
- [ ] Cipher suites restricted to strong algorithms
|
- [ ] **`strcpy` / `strcat` / `sprintf` calls** — all should be `snprintf` or equivalent
|
||||||
- [ ] SSL error codes checked after `SSL_read`/`SSL_write`
|
```bash
|
||||||
|
grep -rn '\bstrcpy\b\|\bstrcat\b\|\bsprintf\b' src/ --include="*.c" --include="*.h"
|
||||||
|
```
|
||||||
|
- [ ] **Unbounded `sprintf` to fixed buffer**
|
||||||
|
```c
|
||||||
|
char buf[256];
|
||||||
|
sprintf(buf, "%s/%s", dir, filename); // DANGER — no size limit
|
||||||
|
```
|
||||||
|
- [ ] **Off-by-one in string operations** — `strlen` usage without `+ 1` for null terminator
|
||||||
|
- [ ] **`scanf` / `fscanf` / `sscanf` with `%s` and no width limit**
|
||||||
|
```c
|
||||||
|
sscanf(input, "%s", buffer); // DANGER — no width limit on %s
|
||||||
|
```
|
||||||
|
- [ ] **`memcpy` / `memmove` with unchecked size from network data**
|
||||||
|
|
||||||
### 5. Authentication & Authorization
|
### 3. Path Traversal in File Operations
|
||||||
|
|
||||||
|
Check all paths constructed from received data:
|
||||||
|
|
||||||
|
- [ ] **Files constructed with client-provided filenames + destination directory**
|
||||||
|
```c
|
||||||
|
snprintf(path, PATH_MAX, "%s/%s", dest_dir, received_filename);
|
||||||
|
```
|
||||||
|
Check for `../` filtering:
|
||||||
|
```bash
|
||||||
|
grep -rn 'snprintf.*%s.*%s.*path\|snprintf.*dest_dir\|snprintf.*base_dir' src/ --include="*.c"
|
||||||
|
```
|
||||||
|
- [ ] **`realpath()` usage** for path canonicalization
|
||||||
|
- [ ] **Symlink following** — does the server follow symlinks in the destination?
|
||||||
|
- [ ] **Null byte injection** — received filenames with embedded `\0`
|
||||||
|
|
||||||
|
### 4. Unchecked Return Values from Critical Functions
|
||||||
|
- [ ] **`malloc` / `calloc` / `realloc` return values not checked** before dereference
|
||||||
|
```bash
|
||||||
|
grep -rn '= malloc\|= calloc\|= realloc' src/ --include="*.c"
|
||||||
|
```
|
||||||
|
For each match, verify NULL check exists before use.
|
||||||
|
- [ ] **`send_n_data` / `receive_n_data` return values** not checked
|
||||||
|
- [ ] **`SSL_read` / `SSL_write`** error codes not checked
|
||||||
|
- [ ] **`write()` / `read()` syscall** return values not checked (short writes/reads)
|
||||||
|
- [ ] **`fopen()` / `open()`** return values not checked
|
||||||
|
- [ ] **`snprintf` / `vsnprintf`** negative return not handled
|
||||||
|
|
||||||
|
### 5. TLS / SSL Security
|
||||||
|
- [ ] **TLS version not restricted** — server allows SSLv3, TLS 1.0, or TLS 1.1
|
||||||
|
```c
|
||||||
|
SSL_CTX_set_min_proto_version(ctx, TLS1_2_VERSION); // REQUIRED
|
||||||
|
```
|
||||||
|
- [ ] **Certificate verification disabled** without explicit `--ca`/warning
|
||||||
|
- [ ] **`SSL_CTX_set_verify` not called** — default is no verification
|
||||||
|
- [ ] **Weak cipher suites allowed** — need to call `SSL_CTX_set_cipher_list()`
|
||||||
|
- [ ] **Private key file permissions** not checked before loading
|
||||||
|
- [ ] **Hostname verification** not performed on server certificate
|
||||||
|
- [ ] **Session renegotiation** not limited (DoS vector)
|
||||||
|
- [ ] **TLS certificate/key paths from untrusted input** — can client specify arbitrary paths?
|
||||||
|
- [ ] **No hardcoded certificates or keys**
|
||||||
|
- [ ] **SSL error codes checked after `SSL_read`/`SSL_write`**
|
||||||
|
|
||||||
|
### 6. Memory Safety Issues
|
||||||
|
- [ ] **Use-after-free** — object freed but pointer still used later
|
||||||
|
- [ ] **Double-free** — `free()` called twice on same pointer
|
||||||
|
- [ ] **Memory leaks** on error paths — allocated but not freed before return
|
||||||
|
- [ ] **Integer overflow** in allocation size computation
|
||||||
|
```c
|
||||||
|
// DANGER: count * sizeof(Type) can overflow
|
||||||
|
void *arr = malloc(count * sizeof(Element));
|
||||||
|
|
||||||
|
// SAFE:
|
||||||
|
if (count > SIZE_MAX / sizeof(Element)) return NULL;
|
||||||
|
void *arr = malloc(count * sizeof(Element));
|
||||||
|
```
|
||||||
|
- [ ] **`realloc` return value** not saved to temporary pointer (leak on failure)
|
||||||
|
```c
|
||||||
|
// BAD: leaks original pointer on failure
|
||||||
|
buf = realloc(buf, new_size);
|
||||||
|
|
||||||
|
// GOOD:
|
||||||
|
void *tmp = realloc(buf, new_size);
|
||||||
|
if (!tmp) { free(buf); return NULL; }
|
||||||
|
buf = tmp;
|
||||||
|
```
|
||||||
|
- [ ] **All error paths free allocated resources** (no leaks / UAF / double-free)
|
||||||
|
- [ ] **Partial reads handled** (don't use incomplete data)
|
||||||
|
|
||||||
|
### 7. Integer Overflow in Allocation
|
||||||
|
|
||||||
|
Check all size calculations:
|
||||||
|
|
||||||
|
- [ ] Allocations where count comes from network data (chunk count, file count, etc.)
|
||||||
|
- [ ] Allocations where size is multiplied by count
|
||||||
|
```bash
|
||||||
|
grep -rn 'malloc.*\*.*sizeof\|calloc(.*sizeof' src/ --include="*.c"
|
||||||
|
```
|
||||||
|
- [ ] Loop counters that could wrap (unsigned underflow)
|
||||||
|
- [ ] Signed integer overflow in size checks
|
||||||
|
|
||||||
|
### 8. Format String Vulnerabilities
|
||||||
|
- [ ] User-controlled data passed as format string
|
||||||
|
```c
|
||||||
|
printf(user_input); // VULNERABLE
|
||||||
|
fprintf(stderr, user_input); // VULNERABLE
|
||||||
|
syslog(LOG_INFO, user_input); // VULNERABLE
|
||||||
|
|
||||||
|
printf("%s", user_input); // SAFE
|
||||||
|
```
|
||||||
|
```bash
|
||||||
|
grep -rn 'printf(\|fprintf(\|syslog(\|snprintf(' src/ --include="*.c" | grep -v '"[^"]*%'
|
||||||
|
```
|
||||||
|
|
||||||
|
### 9. Authentication & Authorization
|
||||||
- [ ] SSH transport relies on SSH authentication (not custom auth)
|
- [ ] SSH transport relies on SSH authentication (not custom auth)
|
||||||
- [ ] No password/credential storage in plaintext
|
- [ ] No password/credential storage in plaintext
|
||||||
- [ ] Server doesn't trust client-supplied paths blindly
|
- [ ] Server doesn't trust client-supplied paths blindly
|
||||||
- [ ] Destination directory validated before writing
|
- [ ] Destination directory validated before writing
|
||||||
|
|
||||||
### 6. Denial of Service
|
### 10. TOCTOU Race Conditions
|
||||||
- [ ] Bounded memory allocation (can't OOM server with huge chunk)
|
- [ ] File existence check followed by open (Time-of-check to Time-of-use)
|
||||||
- [ ] Timeout on connections (no indefinite blocking)
|
```c
|
||||||
- [ ] Maximum connection limit or rate limiting
|
if (access(path, F_OK) == 0) { // CHECK
|
||||||
- [ ] Malformed protocol messages handled gracefully (no crash)
|
fd = open(path, O_RDWR); // USE — file could have changed
|
||||||
|
}
|
||||||
|
```
|
||||||
|
- [ ] `stat()` followed by `open()` with different permissions
|
||||||
|
- [ ] Temporary file creation with predictable names
|
||||||
|
|
||||||
### 7. Cryptographic Practices
|
### 11. Insecure Temporary File Usage
|
||||||
- [ ] No custom crypto — uses OpenSSL only
|
- [ ] `mktemp` / `tmpnam` — use `mkstemp` instead
|
||||||
- [ ] No hardcoded keys, IVs, or salts
|
- [ ] Temporary files created in world-writable directories
|
||||||
- [ ] Random data from `/dev/urandom` or OpenSSL `RAND_bytes`
|
- [ ] Temporary files not cleaned up on error paths
|
||||||
|
- [ ] Predictable temp file names (race + symlink attack)
|
||||||
|
|
||||||
### 8. File System Security
|
### 12. Hardcoded Secrets / Credentials
|
||||||
|
- [ ] Hardcoded passwords, API keys, or tokens
|
||||||
|
- [ ] Hardcoded TLS private keys or certificates
|
||||||
|
- [ ] Hardcoded connection strings with embedded credentials
|
||||||
|
- [ ] Test certificates/keys in source tree (should be documented if intentional)
|
||||||
|
|
||||||
|
### 13. Denial of Service Vectors
|
||||||
|
- [ ] **Unbounded memory allocation** — can client request huge allocation that OOMs server?
|
||||||
|
- Check `chunk.c` for chunk count limits
|
||||||
|
- Check `protocol.c` for message size limits
|
||||||
|
- Check `config.c` for config field size limits
|
||||||
|
- [ ] **No connection limits** — server doesn't cap concurrent connections
|
||||||
|
- [ ] **No timeouts** — connections can hang indefinitely
|
||||||
|
- [ ] **Recursive parsing** — could cause stack overflow with crafted input
|
||||||
|
- [ ] **Repeated slow reads** — slow loris style attack
|
||||||
|
- [ ] **Fork bomb** — server forks per connection without limit
|
||||||
|
|
||||||
|
### 14. Information Disclosure
|
||||||
|
- [ ] Server sends detailed error messages to client (path disclosure, version info)
|
||||||
|
- [ ] Debug logging enabled in production
|
||||||
|
- [ ] Stack traces leaked to users
|
||||||
|
- [ ] Timing side channels in authentication or comparison
|
||||||
|
|
||||||
|
### 15. File System Security
|
||||||
- [ ] Received file permissions validated (no SUID/SGID injection)
|
- [ ] Received file permissions validated (no SUID/SGID injection)
|
||||||
- [ ] Symlink attack prevention (don't follow symlinks in destination)
|
- [ ] Symlink attack prevention (don't follow symlinks in destination)
|
||||||
- [ ] Race conditions in file creation (TOCTOU)
|
- [ ] Race conditions in file creation (TOCTOU)
|
||||||
- [ ] Temporary file security (if any)
|
- [ ] Temporary file security (if any)
|
||||||
|
|
||||||
|
### 16. Cryptographic Practices
|
||||||
|
- [ ] No custom crypto — uses OpenSSL only
|
||||||
|
- [ ] No hardcoded keys, IVs, or salts
|
||||||
|
- [ ] Random data from `/dev/urandom` or OpenSSL `RAND_bytes`
|
||||||
|
|
||||||
## Common Vulnerability Patterns
|
## Common Vulnerability Patterns
|
||||||
|
|
||||||
### Format String Bugs
|
### Format String Bugs
|
||||||
@@ -118,9 +288,59 @@ receive_n_data(fd, buffer, expected_size);
|
|||||||
if (!receive_n_data(fd, buffer, expected_size)) { /* handle error */ }
|
if (!receive_n_data(fd, buffer, expected_size)) { /* handle error */ }
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## How to Scan
|
||||||
|
|
||||||
|
### Automated Pattern Search
|
||||||
|
Run these searches across the codebase:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Buffer overflow risks
|
||||||
|
grep -rn '\bstrcpy\b\|\bstrcat\b\|\bsprintf\b' src/ --include="*.c"
|
||||||
|
|
||||||
|
# Fixed size stack buffers
|
||||||
|
grep -rn 'char [a-z_]*\[[0-9]*\];' src/ --include="*.c" --include="*.h"
|
||||||
|
|
||||||
|
# Format string risks
|
||||||
|
grep -rn 'printf(\|fprintf(\|syslog(' src/ --include="*.c" | grep -v '"[^"]*%'
|
||||||
|
|
||||||
|
# Malloc without null check pattern
|
||||||
|
grep -rn '= malloc\|= calloc\|= realloc' src/ --include="*.c"
|
||||||
|
|
||||||
|
# Integer overflow in allocation
|
||||||
|
grep -rn 'malloc.*\*\|calloc.*<' src/ --include="*.c"
|
||||||
|
|
||||||
|
# Path construction
|
||||||
|
grep -rn 'snprintf.*path\|snprintf.*dir' src/ --include="*.c"
|
||||||
|
```
|
||||||
|
|
||||||
|
### Manual Code Review
|
||||||
|
After automated scanning, manually review high-risk files:
|
||||||
|
1. `src/shared/protocol.c` — all receive paths
|
||||||
|
2. `src/shared/config.c` — deserialization logic
|
||||||
|
3. `src/shared/chunk.c` — chunk parsing
|
||||||
|
4. `src/shared/transport_tls.c` — TLS configuration
|
||||||
|
5. `src/server/server.c` — file writing and connection handling
|
||||||
|
|
||||||
## Output Format
|
## Output Format
|
||||||
|
|
||||||
For each vulnerability found:
|
Return findings in this structured format, one per vulnerability:
|
||||||
|
|
||||||
|
```
|
||||||
|
## Finding: <Short descriptive title>
|
||||||
|
- **Severity**: critical/high/medium/low
|
||||||
|
- **Category**: security
|
||||||
|
- **Location**: file:line range
|
||||||
|
- **Description**: what the vulnerability is, including:
|
||||||
|
- How it can be triggered
|
||||||
|
- What the impact is (RCE, DoS, info leak, etc.)
|
||||||
|
- Whether it requires authentication
|
||||||
|
- **Suggestion**: how to fix it, including concrete code changes
|
||||||
|
- **Labels**: security, comma-separated additional labels
|
||||||
|
```
|
||||||
|
|
||||||
|
### Detailed Finding Fields
|
||||||
|
|
||||||
|
For each vulnerability found, also be prepared to report:
|
||||||
1. **Location** — file:line
|
1. **Location** — file:line
|
||||||
2. **Severity** — critical / high / medium / low / informational
|
2. **Severity** — critical / high / medium / low / informational
|
||||||
3. **Category** — input-validation / buffer / memory / tls / auth / dos / crypto / fs
|
3. **Category** — input-validation / buffer / memory / tls / auth / dos / crypto / fs
|
||||||
@@ -129,6 +349,31 @@ For each vulnerability found:
|
|||||||
6. **Fix** — concrete code change
|
6. **Fix** — concrete code change
|
||||||
7. **CVSS estimate** — rough severity score if exploitable
|
7. **CVSS estimate** — rough severity score if exploitable
|
||||||
|
|
||||||
|
### Example
|
||||||
|
|
||||||
|
```
|
||||||
|
## Finding: Unchecked malloc in chunk deserialization allows OOM
|
||||||
|
- **Severity**: high
|
||||||
|
- **Category**: security
|
||||||
|
- **Location**: src/shared/chunk.c:45-50
|
||||||
|
- **Description**: `chunk_deserialize()` calls `malloc(count * sizeof(File))`
|
||||||
|
where `count` comes directly from the network. An attacker can send a crafted
|
||||||
|
chunk header with an extremely large count (e.g., UINT32_MAX), causing malloc
|
||||||
|
to either fail (crash if unchecked) or allocate enormous memory (OOM).
|
||||||
|
No authentication needed — the attack works on the initial connection.
|
||||||
|
- **Suggestion**: Add bounds checking before allocation:
|
||||||
|
```c
|
||||||
|
if (count > MAX_CHUNK_FILES || count > SIZE_MAX / sizeof(File)) {
|
||||||
|
log_error("Invalid chunk file count: %u", count);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
Define `MAX_CHUNK_FILES` as a reasonable limit (e.g., 100000).
|
||||||
|
- **Labels**: security, dos
|
||||||
|
```
|
||||||
|
|
||||||
|
### Audit Summary
|
||||||
|
|
||||||
Also provide a summary:
|
Also provide a summary:
|
||||||
```
|
```
|
||||||
=== SECURITY AUDIT SUMMARY ===
|
=== SECURITY AUDIT SUMMARY ===
|
||||||
@@ -140,13 +385,30 @@ Low: <count>
|
|||||||
Informational: <count>
|
Informational: <count>
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### No Findings
|
||||||
|
If no security issues are found, return:
|
||||||
|
```
|
||||||
|
## No security findings
|
||||||
|
The codebase appears clean in the areas checked. No vulnerabilities found at this time.
|
||||||
|
```
|
||||||
|
|
||||||
|
## Severity Guidelines
|
||||||
|
|
||||||
|
| Severity | Definition | Example |
|
||||||
|
|---|---|---|
|
||||||
|
| **critical** | Remote code execution, unauthenticated compromise | Buffer overflow on network input |
|
||||||
|
| **high** | Significant impact but requires specific conditions | DoS via unbounded allocation, path traversal |
|
||||||
|
| **medium** | Limited impact, requires auth or other conditions | TOCTOU race in file operations |
|
||||||
|
| **low** | Minor issues, defense in depth | Missing null check that's unlikely to trigger |
|
||||||
|
| **informational** | Not exploitable but violates best practice | Hardcoded value that could be configurable |
|
||||||
|
|
||||||
## CI & Task Execution
|
## CI & Task Execution
|
||||||
|
|
||||||
When using `tea` (the task execution agent) to run CI or tests, always set a sufficient timeout (e.g., 600000ms) to allow the workflow to finish. After CI completes, check the results yourself — inspect logs if the run failed. Never assume success.
|
When using `tea` (the task execution agent) to run CI or tests, always set a sufficient timeout (e.g., 600000ms) to allow the workflow to finish. After CI completes, check the results yourself — inspect logs if the run failed. Never assume success.
|
||||||
|
|
||||||
## Branch Strategy
|
## Branch Strategy
|
||||||
|
|
||||||
Never push directly to `main`. All changes must be developed on a feature branch and merged via a pull request. Always create a new branch (`git checkout -b <branch-name>`) before making changes, push it, and open a PR with `gh pr create --fill`. Wait for CI to pass before merging.
|
Never push directly to `dev` or `main`. All changes must be developed on a feature branch and merged via a pull request targeting `dev`. Create a branch (`git checkout -b <branch-name>`), push it, and open the PR with `tea pr create --repo TapTap/FastSync --base dev --head <branch-name>`. Wait for CI to pass before merging.
|
||||||
|
|
||||||
## Dependency Installation
|
## Dependency Installation
|
||||||
|
|
||||||
|
|||||||
@@ -1,310 +0,0 @@
|
|||||||
---
|
|
||||||
description: Scans the FastSync codebase for security vulnerabilities — buffer overflows, path traversal, TLS issues, memory safety, and cryptographic hygiene.
|
|
||||||
mode: subagent
|
|
||||||
---
|
|
||||||
|
|
||||||
You are a security screener for the FastSync project — a high-performance file synchronization system written in C11 with TCP, SSH, and TLS transport.
|
|
||||||
|
|
||||||
## Your Role
|
|
||||||
|
|
||||||
Scan the codebase for security vulnerabilities. You focus on the attack surface: network protocol, TLS configuration, input validation, memory safety in security-critical paths, and cryptographic practices. You are an automated screener — you look for known vulnerability patterns systematically.
|
|
||||||
|
|
||||||
> **Environment rule:** for CI, dependency installation must use the project's custom Docker image (repo-root `Dockerfile`, same as CI). For local development, use `nix-shell` (see `README.md`). See `AGENTS.md`.
|
|
||||||
|
|
||||||
## Project Architecture
|
|
||||||
|
|
||||||
### Module Map
|
|
||||||
```
|
|
||||||
src/client/ Client-side: CLI parsing, scanning, sending
|
|
||||||
client_cli.c Entry point, argument parsing, config setup
|
|
||||||
client_send.c Transfer orchestration, pipeline management
|
|
||||||
scanner.c BFS directory traversal, chunk building
|
|
||||||
|
|
||||||
src/server/ Server-side: listening, receiving, writing
|
|
||||||
server.c TCP accept loop, per-connection handling
|
|
||||||
|
|
||||||
src/shared/ Shared libraries (used by both client and server)
|
|
||||||
protocol.c/h Wire protocol: status codes, send/receive primitives
|
|
||||||
compression.c/h zstd streaming compression/decompression
|
|
||||||
chunk.c/h File grouping and batch serialization
|
|
||||||
queue.c/h Thread-safe bounded queue (producer-consumer)
|
|
||||||
config.c/h Runtime configuration, serialization, parsing
|
|
||||||
data.c/h Generic buffer type (Data)
|
|
||||||
metadata.c/h File metadata (mode, uid, gid, mtime)
|
|
||||||
file.c/h File representation
|
|
||||||
array_list.c/h Dynamic array
|
|
||||||
transport_tcp.c/h TCP client/server with sendfile() zero-copy
|
|
||||||
transport_ssh.c/h SSH transport with ControlMaster
|
|
||||||
transport_tls.c/h TLS encryption via OpenSSL
|
|
||||||
multiprocessing.c/h Fork-based concurrency
|
|
||||||
log.c/h Logging utilities
|
|
||||||
utils.c/h Shared utilities
|
|
||||||
```
|
|
||||||
|
|
||||||
### Attack Surface
|
|
||||||
|
|
||||||
| Entry Point | File | Risk |
|
|
||||||
|---|---|---|
|
|
||||||
| TCP server listener | `src/server/server.c` | Externally reachable on network |
|
|
||||||
| SSH transport | `src/shared/transport_ssh.c` | Accepts data via stdio pipe |
|
|
||||||
| Protocol parser | `src/shared/protocol.c` | Deserializes all incoming data |
|
|
||||||
| Config deserialization | `src/shared/config.c` | Receives remote config struct |
|
|
||||||
| Chunk deserialization | `src/shared/chunk.c` | Receives file batches |
|
|
||||||
| TLS handshake | `src/shared/transport_tls.c` | SSL context and cert validation |
|
|
||||||
| File writer | `src/server/server.c` | Writes received files to disk |
|
|
||||||
|
|
||||||
## Security Screener Checklist
|
|
||||||
|
|
||||||
### 1. Buffer Overflow Risks
|
|
||||||
Search for these dangerous patterns in all `.c` and `.h` files:
|
|
||||||
|
|
||||||
- [ ] **Fixed-size stack buffers** used for unbounded or network-provided data
|
|
||||||
```c
|
|
||||||
char path[PATH_MAX]; // OK if PATH_MAX is used, bad if size is arbitrary
|
|
||||||
char buf[1024]; // SUSPICIOUS — what limits the input to 1024?
|
|
||||||
char line[4096]; // SUSPICIOUS — what limits the line length?
|
|
||||||
```
|
|
||||||
- [ ] **`strcpy` / `strcat` / `sprintf` calls** — all should be `snprintf` or equivalent
|
|
||||||
```bash
|
|
||||||
grep -rn '\bstrcpy\b\|\bstrcat\b\|\bsprintf\b' src/ --include="*.c" --include="*.h"
|
|
||||||
```
|
|
||||||
- [ ] **Unbounded `sprintf` to fixed buffer**
|
|
||||||
```c
|
|
||||||
char buf[256];
|
|
||||||
sprintf(buf, "%s/%s", dir, filename); // DANGER — no size limit
|
|
||||||
```
|
|
||||||
- [ ] **Off-by-one in string operations** — `strlen` usage without `+ 1` for null terminator
|
|
||||||
- [ ] **`scanf` / `fscanf` / `sscanf` with `%s` and no width limit**
|
|
||||||
```c
|
|
||||||
sscanf(input, "%s", buffer); // DANGER — no width limit on %s
|
|
||||||
```
|
|
||||||
- [ ] **`memcpy` / `memmove` with unchecked size from network data**
|
|
||||||
|
|
||||||
### 2. Path Traversal in File Operations
|
|
||||||
Check all paths constructed from received data:
|
|
||||||
|
|
||||||
- [ ] **Files constructed with client-provided filenames + destination directory**
|
|
||||||
```c
|
|
||||||
snprintf(path, PATH_MAX, "%s/%s", dest_dir, received_filename);
|
|
||||||
```
|
|
||||||
Check for `../` filtering:
|
|
||||||
```bash
|
|
||||||
grep -rn 'snprintf.*%s.*%s.*path\|snprintf.*dest_dir\|snprintf.*base_dir' src/ --include="*.c"
|
|
||||||
```
|
|
||||||
- [ ] **`realpath()` usage** for path canonicalization
|
|
||||||
- [ ] **Symlink following** — does the server follow symlinks in the destination?
|
|
||||||
- [ ] **Null byte injection** — received filenames with embedded `\0`
|
|
||||||
|
|
||||||
### 3. Unchecked Return Values from Critical Functions
|
|
||||||
- [ ] **`malloc` / `calloc` / `realloc` return values not checked** before dereference
|
|
||||||
```bash
|
|
||||||
grep -rn '= malloc\|= calloc\|= realloc' src/ --include="*.c"
|
|
||||||
```
|
|
||||||
For each match, verify NULL check exists before use.
|
|
||||||
- [ ] **`send_n_data` / `receive_n_data` return values** not checked
|
|
||||||
- [ ] **`SSL_read` / `SSL_write`** error codes not checked
|
|
||||||
- [ ] **`write()` / `read()` syscall** return values not checked (short writes/reads)
|
|
||||||
- [ ] **`fopen()` / `open()`** return values not checked
|
|
||||||
- [ ] **`snprintf` / `vsnprintf`** negative return not handled
|
|
||||||
|
|
||||||
### 4. TLS / SSL Misconfiguration
|
|
||||||
- [ ] **TLS version not restricted** — server allows SSLv3, TLS 1.0, or TLS 1.1
|
|
||||||
```c
|
|
||||||
SSL_CTX_set_min_proto_version(ctx, TLS1_2_VERSION); // REQUIRED
|
|
||||||
```
|
|
||||||
- [ ] **Certificate verification disabled** without explicit `--insecure` flag
|
|
||||||
- [ ] **`SSL_CTX_set_verify` not called** — default is no verification
|
|
||||||
- [ ] **Weak cipher suites allowed** — need to call `SSL_CTX_set_cipher_list()`
|
|
||||||
- [ ] **Private key file permissions** not checked before loading
|
|
||||||
- [ ] **Hostname verification** not performed on server certificate
|
|
||||||
- [ ] **Session renegotiation** not limited (DoS vector)
|
|
||||||
- [ ] **TLS certificate/key paths from untrusted input** — can client specify arbitrary paths?
|
|
||||||
|
|
||||||
### 5. Memory Safety Issues
|
|
||||||
- [ ] **Use-after-free** — object freed but pointer still used later
|
|
||||||
- [ ] **Double-free** — `free()` called twice on same pointer
|
|
||||||
- [ ] **Memory leaks** on error paths — allocated but not freed before return
|
|
||||||
- [ ] **Integer overflow** in allocation size computation
|
|
||||||
```c
|
|
||||||
// DANGER: count * sizeof(Type) can overflow
|
|
||||||
void *arr = malloc(count * sizeof(Element));
|
|
||||||
|
|
||||||
// SAFE:
|
|
||||||
if (count > SIZE_MAX / sizeof(Element)) return NULL;
|
|
||||||
void *arr = malloc(count * sizeof(Element));
|
|
||||||
```
|
|
||||||
- [ ] **`realloc` return value** not saved to temporary pointer (leak on failure)
|
|
||||||
```c
|
|
||||||
// BAD: leaks original pointer on failure
|
|
||||||
buf = realloc(buf, new_size);
|
|
||||||
|
|
||||||
// GOOD:
|
|
||||||
void *tmp = realloc(buf, new_size);
|
|
||||||
if (!tmp) { free(buf); return NULL; }
|
|
||||||
buf = tmp;
|
|
||||||
```
|
|
||||||
|
|
||||||
### 6. Integer Overflow in Allocation
|
|
||||||
Check all size calculations:
|
|
||||||
|
|
||||||
- [ ] Allocations where count comes from network data (chunk count, file count, etc.)
|
|
||||||
- [ ] Allocations where size is multiplied by count
|
|
||||||
```bash
|
|
||||||
grep -rn 'malloc.*\*.*sizeof\|calloc(.*sizeof' src/ --include="*.c"
|
|
||||||
```
|
|
||||||
- [ ] Loop counters that could wrap (unsigned underflow)
|
|
||||||
- [ ] Signed integer overflow in size checks
|
|
||||||
|
|
||||||
### 7. Format String Vulnerabilities
|
|
||||||
- [ ] User-controlled data passed as format string
|
|
||||||
```c
|
|
||||||
printf(user_input); // VULNERABLE
|
|
||||||
fprintf(stderr, user_input); // VULNERABLE
|
|
||||||
syslog(LOG_INFO, user_input); // VULNERABLE
|
|
||||||
|
|
||||||
printf("%s", user_input); // SAFE
|
|
||||||
```
|
|
||||||
```bash
|
|
||||||
grep -rn 'printf(\|fprintf(\|syslog(\|snprintf(' src/ --include="*.c" | grep -v '"[^"]*%'
|
|
||||||
```
|
|
||||||
|
|
||||||
### 8. TOCTOU Race Conditions
|
|
||||||
- [ ] File existence check followed by open (Time-of-check to Time-of-use)
|
|
||||||
```c
|
|
||||||
if (access(path, F_OK) == 0) { // CHECK
|
|
||||||
fd = open(path, O_RDWR); // USE — file could have changed
|
|
||||||
}
|
|
||||||
```
|
|
||||||
- [ ] `stat()` followed by `open()` with different permissions
|
|
||||||
- [ ] Temporary file creation with predictable names
|
|
||||||
|
|
||||||
### 9. Insecure Temporary File Usage
|
|
||||||
- [ ] `mktemp` / `tmpnam` — use `mkstemp` instead
|
|
||||||
- [ ] Temporary files created in world-writable directories
|
|
||||||
- [ ] Temporary files not cleaned up on error paths
|
|
||||||
- [ ] Predictable temp file names (race + symlink attack)
|
|
||||||
|
|
||||||
### 10. Hardcoded Secrets / Credentials
|
|
||||||
- [ ] Hardcoded passwords, API keys, or tokens
|
|
||||||
- [ ] Hardcoded TLS private keys or certificates
|
|
||||||
- [ ] Hardcoded connection strings with embedded credentials
|
|
||||||
- [ ] Test certificates/keys in source tree (should be documented if intentional)
|
|
||||||
|
|
||||||
### 11. Denial of Service Vectors
|
|
||||||
- [ ] **Unbounded memory allocation** — can client request huge allocation that OOMs server?
|
|
||||||
- Check `chunk.c` for chunk count limits
|
|
||||||
- Check `protocol.c` for message size limits
|
|
||||||
- Check `config.c` for config field size limits
|
|
||||||
- [ ] **No connection limits** — server doesn't cap concurrent connections
|
|
||||||
- [ ] **No timeouts** — connections can hang indefinitely
|
|
||||||
- [ ] **Recursive parsing** — could cause stack overflow with crafted input
|
|
||||||
- [ ] **Repeated slow reads** — slow loris style attack
|
|
||||||
- [ ] **Fork bomb** — server forks per connection without limit
|
|
||||||
|
|
||||||
### 12. Information Disclosure
|
|
||||||
- [ ] Server sends detailed error messages to client (path disclosure, version info)
|
|
||||||
- [ ] Debug logging enabled in production
|
|
||||||
- [ ] Stack traces leaked to users
|
|
||||||
- [ ] Timing side channels in authentication or comparison
|
|
||||||
|
|
||||||
## How to Scan
|
|
||||||
|
|
||||||
### Automated Pattern Search
|
|
||||||
Run these searches across the codebase:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Buffer overflow risks
|
|
||||||
grep -rn '\bstrcpy\b\|\bstrcat\b\|\bsprintf\b' src/ --include="*.c"
|
|
||||||
|
|
||||||
# Fixed size stack buffers
|
|
||||||
grep -rn 'char [a-z_]*\[[0-9]*\];' src/ --include="*.c" --include="*.h"
|
|
||||||
|
|
||||||
# Format string risks
|
|
||||||
grep -rn 'printf(\|fprintf(\|syslog(' src/ --include="*.c" | grep -v '"[^"]*%'
|
|
||||||
|
|
||||||
# Malloc without null check pattern
|
|
||||||
grep -rn '= malloc\|= calloc\|= realloc' src/ --include="*.c"
|
|
||||||
|
|
||||||
# Integer overflow in allocation
|
|
||||||
grep -rn 'malloc.*\*\|calloc.*<' src/ --include="*.c"
|
|
||||||
|
|
||||||
# Path construction
|
|
||||||
grep -rn 'snprintf.*path\|snprintf.*dir' src/ --include="*.c"
|
|
||||||
```
|
|
||||||
|
|
||||||
### Manual Code Review
|
|
||||||
After automated scanning, manually review high-risk files:
|
|
||||||
1. `src/shared/protocol.c` — all receive paths
|
|
||||||
2. `src/shared/config.c` — deserialization logic
|
|
||||||
3. `src/shared/chunk.c` — chunk parsing
|
|
||||||
4. `src/shared/transport_tls.c` — TLS configuration
|
|
||||||
5. `src/server/server.c` — file writing and connection handling
|
|
||||||
|
|
||||||
## Output Format
|
|
||||||
|
|
||||||
Return findings in this structured format, one per vulnerability:
|
|
||||||
|
|
||||||
```
|
|
||||||
## Finding: <Short descriptive title>
|
|
||||||
- **Severity**: critical/high/medium/low
|
|
||||||
- **Category**: security
|
|
||||||
- **Location**: file:line range
|
|
||||||
- **Description**: what the vulnerability is, including:
|
|
||||||
- How it can be triggered
|
|
||||||
- What the impact is (RCE, DoS, info leak, etc.)
|
|
||||||
- Whether it requires authentication
|
|
||||||
- **Suggestion**: how to fix it, including concrete code changes
|
|
||||||
- **Labels**: security, comma-separated additional labels
|
|
||||||
```
|
|
||||||
|
|
||||||
### Example
|
|
||||||
|
|
||||||
```
|
|
||||||
## Finding: Unchecked malloc in chunk deserialization allows OOM
|
|
||||||
- **Severity**: high
|
|
||||||
- **Category**: security
|
|
||||||
- **Location**: src/shared/chunk.c:45-50
|
|
||||||
- **Description**: `chunk_deserialize()` calls `malloc(count * sizeof(File))`
|
|
||||||
where `count` comes directly from the network. An attacker can send a crafted
|
|
||||||
chunk header with an extremely large count (e.g., UINT32_MAX), causing malloc
|
|
||||||
to either fail (crash if unchecked) or allocate enormous memory (OOM).
|
|
||||||
No authentication needed — the attack works on the initial connection.
|
|
||||||
- **Suggestion**: Add bounds checking before allocation:
|
|
||||||
```c
|
|
||||||
if (count > MAX_CHUNK_FILES || count > SIZE_MAX / sizeof(File)) {
|
|
||||||
log_error("Invalid chunk file count: %u", count);
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
```
|
|
||||||
Define `MAX_CHUNK_FILES` as a reasonable limit (e.g., 100000).
|
|
||||||
- **Labels**: security, dos
|
|
||||||
```
|
|
||||||
|
|
||||||
### No Findings
|
|
||||||
If no security issues are found, return:
|
|
||||||
```
|
|
||||||
## No security findings
|
|
||||||
The codebase appears clean in the areas checked. No vulnerabilities found at this time.
|
|
||||||
```
|
|
||||||
|
|
||||||
## Severity Guidelines
|
|
||||||
|
|
||||||
| Severity | Definition | Example |
|
|
||||||
|---|---|---|
|
|
||||||
| **critical** | Remote code execution, unauthenticated compromise | Buffer overflow on network input |
|
|
||||||
| **high** | Significant impact but requires specific conditions | DoS via unbounded allocation, path traversal |
|
|
||||||
| **medium** | Limited impact, requires auth or other conditions | TOCTOU race in file operations |
|
|
||||||
| **low** | Minor issues, defense in depth | Missing null check that's unlikely to trigger |
|
|
||||||
| **informational** | Not exploitable but violates best practice | Hardcoded value that could be configurable |
|
|
||||||
|
|
||||||
## CI & Task Execution
|
|
||||||
|
|
||||||
When using `tea` (the task execution agent) to run CI or tests, always set a sufficient timeout (e.g., 600000ms) to allow the workflow to finish. After CI completes, check the results yourself — inspect logs if the run failed. Never assume success.
|
|
||||||
|
|
||||||
## Branch Strategy
|
|
||||||
|
|
||||||
Never push directly to `main`. All changes must be developed on a feature branch and merged via a pull request. Always create a new branch (`git checkout -b <branch-name>`) before making changes, push it, and open a PR with `gh pr create --fill`. Wait for CI to pass before merging.
|
|
||||||
|
|
||||||
## Dependency Installation
|
|
||||||
|
|
||||||
**CI rule:** never add `apt-get install` / `pip install` steps to CI workflows — use the custom Docker image instead. **Host rule:** for local development, use `nix-shell` (see `README.md`) which provides zstd, OpenSSL, CMake, and gcc. See `AGENTS.md` for details.
|
|
||||||
@@ -138,11 +138,9 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
|
|||||||
|
|
||||||
Build for fuzzing:
|
Build for fuzzing:
|
||||||
```bash
|
```bash
|
||||||
cmake -B build-fuzz -S . \
|
CC=clang CXX=clang++ cmake -B build-fuzz -S . -DENABLE_FUZZ=ON
|
||||||
-DCMAKE_C_FLAGS="-fsanitize=fuzzer,address,undefined -g" \
|
|
||||||
-DCMAKE_EXE_LINKER_FLAGS="-fsanitize=fuzzer,address,undefined"
|
|
||||||
cmake --build build-fuzz -j$(nproc)
|
cmake --build build-fuzz -j$(nproc)
|
||||||
./build-fuzz/tests/fuzz_chunk_deserialize corpus/ -max_len=1048576
|
./build-fuzz/fuzz_chunk_deserialize corpus/ -max_len=1048576
|
||||||
```
|
```
|
||||||
|
|
||||||
### AFL++ Harness
|
### AFL++ Harness
|
||||||
@@ -216,7 +214,7 @@ When using `tea` (the task execution agent) to run CI or tests, always set a suf
|
|||||||
|
|
||||||
## Branch Strategy
|
## Branch Strategy
|
||||||
|
|
||||||
Never push directly to `main`. All changes must be developed on a feature branch and merged via a pull request. Always create a new branch (`git checkout -b <branch-name>`) before making changes, push it, and open a PR with `gh pr create --fill`. Wait for CI to pass before merging.
|
Never push directly to `dev` or `main`. All changes must be developed on a feature branch and merged via a pull request targeting `dev`. Create a branch (`git checkout -b <branch-name>`), push it, and open the PR with `tea pr create --repo TapTap/FastSync --base dev --head <branch-name>`. Wait for CI to pass before merging.
|
||||||
|
|
||||||
## Dependency Installation
|
## Dependency Installation
|
||||||
|
|
||||||
|
|||||||
@@ -38,16 +38,20 @@ dd if=/dev/urandom of=/tmp/fastsync_bench/src/large.bin bs=1M count=10 2>/dev/nu
|
|||||||
Test each configuration 3 times, record median:
|
Test each configuration 3 times, record median:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
# Real FastSync flags: -z=compression, -j=multithreading,
|
||||||
|
# --chunk-serialization, --sendfile (long form only). The old rsync-style
|
||||||
|
# spellings -c/-m/-s/-f are NOT the same options (-c=--checksum,
|
||||||
|
# -m=--prune-empty-dirs, -s=--secluded-args, -f=--filter) and must not be used.
|
||||||
CONFIGS=(
|
CONFIGS=(
|
||||||
"Standard|"
|
"Standard|"
|
||||||
"Compression|-c"
|
"Compression|-z"
|
||||||
"Multithreading|-m"
|
"Multithreading|-j"
|
||||||
"MT+Compression|-m -c"
|
"MT+Compression|-j -z"
|
||||||
"Chunk Serialization|-s"
|
"Chunk Serialization|-j -z --chunk-serialization"
|
||||||
"MT+Compression+Chunk|-m -c -s"
|
"Sendfile|--sendfile"
|
||||||
"Sendfile|-f"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
PORT=18080
|
||||||
for config in "${CONFIGS[@]}"; do
|
for config in "${CONFIGS[@]}"; do
|
||||||
IFS='|' read -r name flags <<< "$config"
|
IFS='|' read -r name flags <<< "$config"
|
||||||
echo "=== $name ==="
|
echo "=== $name ==="
|
||||||
@@ -55,13 +59,14 @@ for config in "${CONFIGS[@]}"; do
|
|||||||
rm -rf /tmp/fastsync_bench/dst
|
rm -rf /tmp/fastsync_bench/dst
|
||||||
mkdir -p /tmp/fastsync_bench/dst
|
mkdir -p /tmp/fastsync_bench/dst
|
||||||
|
|
||||||
./build/server &
|
./build/server -p "$PORT" --allow-unauthenticated &
|
||||||
SERVER_PID=$!
|
SERVER_PID=$!
|
||||||
sleep 0.5
|
sleep 0.5
|
||||||
|
|
||||||
START=$(date +%s%N)
|
START=$(date +%s%N)
|
||||||
./build/client --source-dir /tmp/fastsync_bench/src \
|
./build/client --source-dir /tmp/fastsync_bench/src \
|
||||||
--dest-dir /tmp/fastsync_bench/dst \
|
--dest-dir /tmp/fastsync_bench/dst \
|
||||||
|
--server-port "$PORT" \
|
||||||
--save-to-disk $flags
|
--save-to-disk $flags
|
||||||
END=$(date +%s%N)
|
END=$(date +%s%N)
|
||||||
|
|
||||||
@@ -74,14 +79,21 @@ for config in "${CONFIGS[@]}"; do
|
|||||||
done
|
done
|
||||||
```
|
```
|
||||||
|
|
||||||
### Step 4: Full Integration Benchmark (Optional)
|
### Step 4: Full Benchmark Tool (Preferred)
|
||||||
|
|
||||||
|
The maintained benchmark tool is `benchmark/bench.py`. It handles building,
|
||||||
|
data generation, network shaping (LAN/WAN profiles or custom `--delay`/`--jitter`/
|
||||||
|
`--throughput`/`--loss`), rsync comparison, and JSON/table reporting:
|
||||||
|
|
||||||
For comprehensive benchmarking with network shaping:
|
|
||||||
```bash
|
```bash
|
||||||
python3 test.py --full
|
python3 benchmark/bench.py --help
|
||||||
|
python3 benchmark/bench.py --runs 5 --profiles unlimited
|
||||||
|
python3 benchmark/bench.py --size-mb 100 --random-ratio 0.5 --output json
|
||||||
|
python3 benchmark/bench.py --delay 50ms --jitter 10ms --throughput 100mbit
|
||||||
```
|
```
|
||||||
|
|
||||||
This tests LAN/WAN profiles, SSH, TLS, and compares against rsync.
|
Network shaping needs root (`tc`/`netem` on `lo`). SSH and TLS coverage lives in
|
||||||
|
the pytest integration suite, not the benchmark tool.
|
||||||
|
|
||||||
### Step 5: Report Results
|
### Step 5: Report Results
|
||||||
|
|
||||||
@@ -92,13 +104,14 @@ Platform: <OS, CPU, network>
|
|||||||
|
|
||||||
Configuration | Run 1 | Run 2 | Run 3 | Median
|
Configuration | Run 1 | Run 2 | Run 3 | Median
|
||||||
-----------------------|---------|---------|---------|--------
|
-----------------------|---------|---------|---------|--------
|
||||||
Standard | 0.12s | 0.11s | 0.12s | 0.12s
|
Standard | 0.12s | 0.11s | 0.12s | 0.12s
|
||||||
Compression (-c) | 0.09s | 0.08s | 0.09s | 0.09s
|
Compression (-z) | 0.09s | 0.08s | 0.09s | 0.09s
|
||||||
Multithreading (-m) | 0.07s | 0.07s | 0.08s | 0.07s
|
Multithreading (-j) | 0.07s | 0.07s | 0.08s | 0.07s
|
||||||
MT+Compression (-m -c) | 0.05s | 0.05s | 0.06s | 0.05s
|
MT+Compression (-j -z) | 0.05s | 0.05s | 0.06s | 0.05s
|
||||||
Sendfile (-f) | 0.04s | 0.04s | 0.04s | 0.04s
|
Chunk Serialization (--chunk-serialization) | 0.05s | 0.04s | 0.05s | 0.05s
|
||||||
|
Sendfile (--sendfile) | 0.04s | 0.04s | 0.04s | 0.04s
|
||||||
|
|
||||||
Best configuration: MT+Compression (-m -c)
|
Best configuration: Sendfile (--sendfile)
|
||||||
Throughput: <X> MB/s
|
Throughput: <X> MB/s
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|||||||
@@ -32,23 +32,19 @@ Try to reproduce the issue with the exact command the user provides.
|
|||||||
|
|
||||||
**Memory errors (first priority):**
|
**Memory errors (first priority):**
|
||||||
```bash
|
```bash
|
||||||
rm -rf build
|
rm -rf build-asan
|
||||||
cmake -B build -S . \
|
cmake -B build-asan -S . -DSANITIZER=address
|
||||||
-DCMAKE_C_FLAGS="-fsanitize=address -fno-omit-frame-pointer -g" \
|
cmake --build build-asan -j$(nproc)
|
||||||
-DCMAKE_EXE_LINKER_FLAGS="-fsanitize=address"
|
./build-asan/tests
|
||||||
cmake --build build -j$(nproc)
|
|
||||||
./build/tests
|
|
||||||
# or run the failing command
|
# or run the failing command
|
||||||
```
|
```
|
||||||
|
|
||||||
**Thread errors:**
|
**Thread errors:**
|
||||||
```bash
|
```bash
|
||||||
rm -rf build
|
rm -rf build-tsan
|
||||||
cmake -B build -S . \
|
cmake -B build-tsan -S . -DSANITIZER=thread
|
||||||
-DCMAKE_C_FLAGS="-fsanitize=thread -g" \
|
cmake --build build-tsan -j$(nproc)
|
||||||
-DCMAKE_EXE_LINKER_FLAGS="-fsanitize=thread"
|
./build-tsan/tests
|
||||||
cmake --build build -j$(nproc)
|
|
||||||
./build/tests
|
|
||||||
```
|
```
|
||||||
|
|
||||||
**Valgrind (if ASan doesn't find it):**
|
**Valgrind (if ASan doesn't find it):**
|
||||||
@@ -108,13 +104,12 @@ cmake -B build -S . && cmake --build build -j$(nproc)
|
|||||||
./build/tests
|
./build/tests
|
||||||
|
|
||||||
# If integration test needed
|
# If integration test needed
|
||||||
python3 test.py
|
python3 -m pytest tests/integration/ -n 4 --dist=load -m "not setpriv"
|
||||||
|
|
||||||
# Re-run under sanitizer to confirm fix
|
# Re-run under sanitizer to confirm fix
|
||||||
rm -rf build
|
rm -rf build-asan
|
||||||
cmake -B build -S . -DCMAKE_C_FLAGS="-fsanitize=address -fno-omit-frame-pointer" \
|
cmake -B build-asan -S . -DSANITIZER=address
|
||||||
-DCMAKE_EXE_LINKER_FLAGS="-fsanitize=address"
|
cmake --build build-asan -j$(nproc)
|
||||||
cmake --build build -j$(nproc)
|
|
||||||
# reproduce the original failing command
|
# reproduce the original failing command
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ tea pr checkout <number>
|
|||||||
If already on a PR branch, verify with:
|
If already on a PR branch, verify with:
|
||||||
```bash
|
```bash
|
||||||
git branch --show-current
|
git branch --show-current
|
||||||
git log main..HEAD --oneline
|
git log dev..HEAD --oneline
|
||||||
```
|
```
|
||||||
|
|
||||||
### Step 2: Clean build
|
### Step 2: Clean build
|
||||||
@@ -39,17 +39,13 @@ If the PR touches threading, memory management, or network code, also build with
|
|||||||
```bash
|
```bash
|
||||||
# AddressSanitizer
|
# AddressSanitizer
|
||||||
rm -rf build-asan
|
rm -rf build-asan
|
||||||
cmake -B build-asan -S . \
|
cmake -B build-asan -S . -DSANITIZER=address
|
||||||
-DCMAKE_C_FLAGS="-fsanitize=address -fno-omit-frame-pointer -g" \
|
|
||||||
-DCMAKE_EXE_LINKER_FLAGS="-fsanitize=address"
|
|
||||||
cmake --build build-asan -j$(nproc)
|
cmake --build build-asan -j$(nproc)
|
||||||
./build-asan/tests
|
./build-asan/tests
|
||||||
|
|
||||||
# ThreadSanitizer (if threading changes)
|
# ThreadSanitizer (if threading changes)
|
||||||
rm -rf build-tsan
|
rm -rf build-tsan
|
||||||
cmake -B build-tsan -S . \
|
cmake -B build-tsan -S . -DSANITIZER=thread
|
||||||
-DCMAKE_C_FLAGS="-fsanitize=thread -g" \
|
|
||||||
-DCMAKE_EXE_LINKER_FLAGS="-fsanitize=thread"
|
|
||||||
cmake --build build-tsan -j$(nproc)
|
cmake --build build-tsan -j$(nproc)
|
||||||
./build-tsan/tests
|
./build-tsan/tests
|
||||||
```
|
```
|
||||||
@@ -91,10 +87,10 @@ If tests fail:
|
|||||||
### Step 6: Run integration tests (optional)
|
### Step 6: Run integration tests (optional)
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
python3 test.py
|
python3 -m pytest tests/integration/ -n 4 --dist=load -m "not setpriv"
|
||||||
```
|
```
|
||||||
|
|
||||||
This runs the integration + benchmark suite. It takes longer — only run if the user asks or if unit tests pass.
|
This runs the integration suite (benchmarking is `benchmark/bench.py`). It takes longer — only run if the user asks or if unit tests pass.
|
||||||
|
|
||||||
### Step 7: Fix and commit
|
### Step 7: Fix and commit
|
||||||
|
|
||||||
|
|||||||
@@ -19,13 +19,13 @@ tea pr checkout <number>
|
|||||||
If already on a PR branch, verify with:
|
If already on a PR branch, verify with:
|
||||||
```bash
|
```bash
|
||||||
git branch --show-current
|
git branch --show-current
|
||||||
git log main..HEAD --oneline
|
git log dev..HEAD --oneline
|
||||||
```
|
```
|
||||||
|
|
||||||
### Step 2: Get changed files
|
### Step 2: Get changed files
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
git diff main --name-only -- '*.c' '*.h'
|
git diff dev --name-only -- '*.c' '*.h'
|
||||||
```
|
```
|
||||||
|
|
||||||
This gives the list of C source and header files changed in the PR.
|
This gives the list of C source and header files changed in the PR.
|
||||||
@@ -125,7 +125,7 @@ STYLE: <count>
|
|||||||
|
|
||||||
If the user wants to post the review as a PR comment:
|
If the user wants to post the review as a PR comment:
|
||||||
```bash
|
```bash
|
||||||
tea pr comment <number> --comment "<review report>"
|
tea comment --repo TapTap/FastSync <number> "<review report>"
|
||||||
```
|
```
|
||||||
|
|
||||||
## Rules
|
## Rules
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ Ask the user or determine from context:
|
|||||||
- **Minor** (x.Y.0) — new features, backward compatible
|
- **Minor** (x.Y.0) — new features, backward compatible
|
||||||
- **Patch** (x.y.Z) — bug fixes, no protocol changes
|
- **Patch** (x.y.Z) — bug fixes, no protocol changes
|
||||||
|
|
||||||
Current version: `PROTOCOL_VERSION "1.1.0"` in `src/shared/config.h`
|
Current version: `PROTOCOL_VERSION "2.21.0"` in `src/shared/config.h`
|
||||||
|
|
||||||
### Step 2: Check Protocol Version
|
### Step 2: Check Protocol Version
|
||||||
|
|
||||||
@@ -37,7 +37,7 @@ rm -rf build
|
|||||||
cmake -B build -S .
|
cmake -B build -S .
|
||||||
cmake --build build -j$(nproc)
|
cmake --build build -j$(nproc)
|
||||||
./build/tests
|
./build/tests
|
||||||
python3 test.py
|
python3 -m pytest tests/integration/ -n 4 --dist=load -m "not setpriv"
|
||||||
```
|
```
|
||||||
|
|
||||||
ALL tests must pass before release.
|
ALL tests must pass before release.
|
||||||
@@ -46,12 +46,10 @@ ALL tests must pass before release.
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
# ASan
|
# ASan
|
||||||
rm -rf build
|
rm -rf build-asan
|
||||||
cmake -B build -S . \
|
cmake -B build-asan -S . -DSANITIZER=address
|
||||||
-DCMAKE_C_FLAGS="-fsanitize=address -fno-omit-frame-pointer" \
|
cmake --build build-asan -j$(nproc)
|
||||||
-DCMAKE_EXE_LINKER_FLAGS="-fsanitize=address"
|
./build-asan/tests
|
||||||
cmake --build build -j$(nproc)
|
|
||||||
./build/tests
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### Step 5: Update README (If Needed)
|
### Step 5: Update README (If Needed)
|
||||||
@@ -79,12 +77,23 @@ git commit -m "Release vX.Y.Z
|
|||||||
git tag -a vX.Y.Z -m "Release vX.Y.Z"
|
git tag -a vX.Y.Z -m "Release vX.Y.Z"
|
||||||
```
|
```
|
||||||
|
|
||||||
### Step 8: Push
|
### Step 8: Push and Open dev → main PR
|
||||||
|
|
||||||
|
`main` is protected and only receives changes via `dev` → `main` PRs (see AGENTS.md). Never push directly to `main`.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
git push origin main --tags
|
# Push the release commit and tag to dev
|
||||||
|
git push origin dev
|
||||||
|
git push origin vX.Y.Z
|
||||||
|
|
||||||
|
# Open the dev → main release PR for review + CI
|
||||||
|
tea pr create --repo TapTap/FastSync --head dev --base main \
|
||||||
|
--title "Release vX.Y.Z" \
|
||||||
|
--description "Release vX.Y.Z"
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Then wait for the full CI to pass and request review before the PR is merged to `main`.
|
||||||
|
|
||||||
### Step 9: Report
|
### Step 9: Report
|
||||||
|
|
||||||
```
|
```
|
||||||
|
|||||||
@@ -102,9 +102,9 @@ Informational: <count>
|
|||||||
...
|
...
|
||||||
|
|
||||||
=== VERDICT ===
|
=== VERDICT ===
|
||||||
[PASS] No critical/high issues found
|
[PASS] No critical/high-severity issues found
|
||||||
— or —
|
— or —
|
||||||
[FAIL] <N> critical/high issues must be fixed
|
[FAIL] <N> critical/high-severity issues must be fixed
|
||||||
```
|
```
|
||||||
|
|
||||||
## Rules
|
## Rules
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ docker run --rm --user "$(id -u):$(id -g)" -v "$PWD:/workspace" \
|
|||||||
sh -c 'cmake -B build -S . && cmake --build build -j$(nproc) && ./build/tests && python3 -m pytest tests/integration/ -n 4 --dist=load'
|
sh -c 'cmake -B build -S . && cmake --build build -j$(nproc) && ./build/tests && python3 -m pytest tests/integration/ -n 4 --dist=load'
|
||||||
```
|
```
|
||||||
|
|
||||||
> **Note:** The first `cmake configure` (`cmake -B build -S .`) fetches xxHash from GitHub via `FetchContent` — network access is required. Subsequent reconfigures reuse the cached source.
|
> **Note:** The first `cmake configure` (`cmake -B build -S .`) fetches xxHash via `FetchContent` — network access is required. Subsequent reconfigures reuse the cached source.
|
||||||
|
|
||||||
If a dependency is missing from the CI image, add it to the `Dockerfile` (and rebuild) rather than adding an install step to the CI workflow.
|
If a dependency is missing from the CI image, add it to the `Dockerfile` (and rebuild) rather than adding an install step to the CI workflow.
|
||||||
|
|
||||||
@@ -59,7 +59,7 @@ python3 -m pytest tests/integration/ -n 4 --dist=load -m ci # PR-gate subset o
|
|||||||
|
|
||||||
## CI Workflow — Waiting for Results
|
## CI Workflow — Waiting for Results
|
||||||
|
|
||||||
When running the CI workflow via `tea` (the task execution agent), always set a sufficient timeout (e.g., 600000ms) to allow CI to finish. After CI completes, check the results yourself — do not assume success. Use `gh run watch` or similar to monitor CI status, then inspect logs on failure.
|
When running the CI workflow via `tea` (the task execution agent), always set a sufficient timeout (e.g., 600000ms) to allow CI to finish. After CI completes, check the results yourself — do not assume success. Monitor CI status via the Gitea API (see below) or `tea actions`, then inspect logs on failure.
|
||||||
|
|
||||||
## CI Troubleshooting
|
## CI Troubleshooting
|
||||||
|
|
||||||
@@ -80,7 +80,7 @@ docker run --rm -v "$PWD:/workspace" -w /workspace gitea.tap-tap.win/taptap/fast
|
|||||||
### If integration tests fail
|
### If integration tests fail
|
||||||
Run locally before pushing:
|
Run locally before pushing:
|
||||||
```bash
|
```bash
|
||||||
python3 -m pytest tests/ -v --tb=short
|
python3 -m pytest tests/integration/ -n 4 --dist=load -m "not setpriv"
|
||||||
```
|
```
|
||||||
|
|
||||||
## Branch Strategy
|
## Branch Strategy
|
||||||
@@ -165,7 +165,7 @@ This can be cron'd locally if desired (e.g., `crontab -e` with `opencode run`).
|
|||||||
## Is opencode a good option?
|
## Is opencode a good option?
|
||||||
|
|
||||||
**Yes, for FastSync's needs.** The hybrid model works well:
|
**Yes, for FastSync's needs.** The hybrid model works well:
|
||||||
- opencode's 17 specialized agents handle deep code analysis, fixes, tests, and reviews
|
- opencode's 16 specialized agents handle deep code analysis, fixes, tests, and reviews
|
||||||
- The assistant orchestrates subagents, merges branches, and iterates on CI
|
- The assistant orchestrates subagents, merges branches, and iterates on CI
|
||||||
- You only review the final output
|
- You only review the final output
|
||||||
|
|
||||||
|
|||||||
+134
@@ -4,6 +4,140 @@ All notable changes to FastSync are documented here. Versions match
|
|||||||
`PROTOCOL_VERSION` (printed by `fastsync --version`); the client and server must
|
`PROTOCOL_VERSION` (printed by `fastsync --version`); the client and server must
|
||||||
run the same version because the handshake is strict.
|
run the same version because the handshake is strict.
|
||||||
|
|
||||||
|
## [2.21.0] - 2026-09-14
|
||||||
|
|
||||||
|
### Added
|
||||||
|
|
||||||
|
- Optional server→client rejection detail (protocol 2.21.0). A rejected
|
||||||
|
operation may now carry a bounded human-readable reason via
|
||||||
|
`STATUS_ERROR_DETAIL` instead of a bare `STATUS_ERROR`, so the client can
|
||||||
|
report *why* the server refused (daemon module gate, config validation,
|
||||||
|
receiver-side path/node validation). `receive_status()` transparently maps the
|
||||||
|
new status back to `STATUS_ERROR` for every existing call site and captures
|
||||||
|
the reason into a thread-local buffer exposed by `protocol_last_error()`. The
|
||||||
|
detail body is always consumed, so the stream cannot desynchronize, and
|
||||||
|
messages are sliced to `MAX_ERROR_DETAIL_BYTES` (4096) on send.
|
||||||
|
- **Server-contacting `--dry-run` (protocol 2.21.0).** `--dry-run` now performs
|
||||||
|
a real handshake with a remote/daemon receiver and reports exactly what WOULD
|
||||||
|
change based on receiver state (existing destination files, mtimes, checksums,
|
||||||
|
basis dirs). The wire config carries the dry-run intent (`Config.dry_run`) and
|
||||||
|
the receiver answers each per-file check with `STATUS_DRY_RUN_TRANSFER` (would
|
||||||
|
transfer) or `STATUS_OK` (already up to date); the sender prints the
|
||||||
|
would-transfer set and its trailer without sending any file data. The receiver
|
||||||
|
performs the normal read-only incremental decision but mutates nothing: no temp
|
||||||
|
files, writes, renames, deletes, metadata/xattr/chown, or directory creation.
|
||||||
|
A plain local destination (no explicit `--server-port`/remote) keeps the
|
||||||
|
original client-side dry-run. Would-delete reporting for `--delete*` is
|
||||||
|
deferred to a follow-up; dry-run never deletes.
|
||||||
|
- Daemon `max connections per host` (per-source-IP concurrent cap, default 0 =
|
||||||
|
unlimited), `auth lockout threshold` (default 10; 0 disables) and
|
||||||
|
`auth lockout duration` (default 300 s) config keys.
|
||||||
|
- `fastsync-server --allow-super` opt-in for a privileged standalone TCP server;
|
||||||
|
without it a root standalone receiver forces super-user activities off (device
|
||||||
|
nodes, `--write-devices`, ownership). The `--stdio` SSH argv is client-composed,
|
||||||
|
so super activities always stay off there.
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- Config wire fields are now declared once in an X-macro table
|
||||||
|
(`CONFIG_WIRE_FIELDS` in `src/shared/config.h`) that generates the struct
|
||||||
|
members, defaults, and the send/receive sequence, removing the manual
|
||||||
|
six-site field sync. Wire bytes and `PROTOCOL_VERSION` are unchanged.
|
||||||
|
- `receive_incremental_check()` (the per-file `STATUS_CHECK` fast path) is split
|
||||||
|
into small static helpers with a short linear orchestrator. Pure refactor: the
|
||||||
|
wire byte stream and all cleanup are unchanged.
|
||||||
|
- `authorized_root` state has a single owner (`utils.c`) with read accessors; the
|
||||||
|
duplicated statics in `file.c` and the server were removed.
|
||||||
|
- `Data` records its owning `ProtocolSession` so its memory charge is returned to
|
||||||
|
the session that reserved it, regardless of the destroying thread.
|
||||||
|
- The receiver pipeline moved out of `shared` into `server/receiver_pipeline.[ch]`;
|
||||||
|
the build now uses explicit `fastsync_shared` / `fastsync_client_core` /
|
||||||
|
`fastsync_server_core` targets instead of a GLOB, and the client no longer links
|
||||||
|
server code.
|
||||||
|
- The benchmark tool generates the requested random/compressible data mix
|
||||||
|
accurately, verifies each transfer before recording it, computes correct
|
||||||
|
percentiles, adds a MB/s column, handles `tc`/netem without requiring `sudo`
|
||||||
|
when already root, builds into a dedicated `build-bench/` directory, and adds a
|
||||||
|
`--warm` incremental-transfer mode.
|
||||||
|
- The `nix-shell` dev environment provides the full toolchain (clang-format,
|
||||||
|
cppcheck, pytest-xdist, OpenSSH, rsync, iproute2, valgrind, lcov) and no longer
|
||||||
|
builds on entry.
|
||||||
|
|
||||||
|
### Security
|
||||||
|
|
||||||
|
- Enforce the daemon's per-module `max connections` cap (0 = unlimited) and add
|
||||||
|
the shared per-source `max connections per host` cap plus a cross-process
|
||||||
|
`auth lockout`. Because the listener forks one child per connection, the
|
||||||
|
counters live in an anonymous shared mapping created before the accept loop and
|
||||||
|
reclaimed by the parent's `SIGCHLD` handler, so the per-module, per-source and
|
||||||
|
auth-failure state is shared across every child (including after `SIGKILL`). The
|
||||||
|
per-source table has a bounded lifetime (expired/idle entries are reclaimed,
|
||||||
|
with a rate-limited warning when genuinely full), and the occupancy counters are
|
||||||
|
re-derived from the shared slot table on every child exit. Trusted loopback
|
||||||
|
peers are exempt (they share one address); clients behind a shared NAT/proxy
|
||||||
|
share a single per-host budget and lockout, which is documented.
|
||||||
|
- Hardening from a full security audit:
|
||||||
|
- Fail a truncated zstd frame instead of spinning forever (remote DoS).
|
||||||
|
- Open receiver destination/basis/hard-link entries `O_NONBLOCK` so a
|
||||||
|
client-planted FIFO cannot block a worker indefinitely.
|
||||||
|
- Require a regular file before `--inplace` writes, closing a FIFO-hang and a
|
||||||
|
raw-device write that bypassed the `--write-devices` gate.
|
||||||
|
- Reject SSH destinations whose user/host begins with `-` and insert `--` before
|
||||||
|
the host token, closing `-o ProxyCommand=…` argument injection (RCE).
|
||||||
|
- Gate client `--force` recursive removal behind the server `--allow-delete`
|
||||||
|
policy.
|
||||||
|
- Reject empty `hosts allow`/`hosts deny`/`auth users` values instead of
|
||||||
|
silently meaning "unrestricted".
|
||||||
|
- Restrict TLS 1.2 to AEAD suites and set server cipher preference; load the
|
||||||
|
private key TOCTOU-safely from an `O_NOFOLLOW` fd; verify IP literals against
|
||||||
|
IP SANs; guard client-cert CN truncation.
|
||||||
|
- Make `--dry-run` content-blind: it neither reads destination files nor
|
||||||
|
hashes basis files, removing a 1-bit content oracle against `read only`
|
||||||
|
modules.
|
||||||
|
- Bound glob matching (iterative DP, no exponential backtracking) and bound
|
||||||
|
line reads for filter/`--files-from`/pattern files.
|
||||||
|
- Gate `system.posix_acl_*` xattrs on `--acls` and charge decompression/chunk
|
||||||
|
allocations against the per-connection memory budget.
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
|
||||||
|
- Pre-auth NULL dereference in `config_delete()` when an over-long
|
||||||
|
`basis_count` (and the analogous count fields) was received and then failed
|
||||||
|
validation; received counts are now validated before being published.
|
||||||
|
- Leaked inherited `Data` in the forked compression-truncation unit test
|
||||||
|
(valgrind definite leak).
|
||||||
|
- `receive_status()` no longer loses a captured rejection reason when owed
|
||||||
|
keepalives are drained.
|
||||||
|
|
||||||
|
## [2.20.0] - 2026-09-13
|
||||||
|
|
||||||
|
### Security
|
||||||
|
|
||||||
|
- Cap cumulative `DirTimeList` growth and bound pre-auth config-string memory
|
||||||
|
(remote memory-exhaustion DoS).
|
||||||
|
- Daemon host access control (`hosts allow`/`hosts deny`, IPv4/IPv6/CIDR),
|
||||||
|
configurable global `max connections`, connection audit logging, and a
|
||||||
|
bounded `auth failure delay` throttle. IPv4-mapped peers are normalized and
|
||||||
|
invalid patterns are rejected at parse time (no silent fail-open).
|
||||||
|
- Honor `--timeout` for protocol I/O and bound idle/session time to defeat
|
||||||
|
keepalive slowloris; child-safe signal handling in the forked daemon.
|
||||||
|
- Compiler/linker hardening (`_FORTIFY_SOURCE`, stack protector, PIE, RELRO)
|
||||||
|
and pinned build dependencies.
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
|
||||||
|
- Use-after-free in the basis-dir oversize preflight.
|
||||||
|
- Placeholder `Data` leaks, `missing_args` leak, scanner chunk leak.
|
||||||
|
- Thread-safe logging; single fd owner and cleanup epilogue in the server
|
||||||
|
handler.
|
||||||
|
|
||||||
|
### Performance
|
||||||
|
|
||||||
|
- Metadata now crosses the wire as one packed frame (protocol 2.20.0).
|
||||||
|
- Delete keep-set and `--files-from` lookups indexed (O(n*m) → O(n)).
|
||||||
|
- Reused per-thread zstd contexts; `TCP_NODELAY` by default.
|
||||||
|
- Byte-bounded sender queues; removed a redundant scanner `stat()`.
|
||||||
|
|
||||||
## [2.19.0] - 2026-09-12
|
## [2.19.0] - 2026-09-12
|
||||||
|
|
||||||
### Security
|
### Security
|
||||||
|
|||||||
+199
-26
@@ -1,6 +1,6 @@
|
|||||||
cmake_minimum_required(VERSION 3.22)
|
cmake_minimum_required(VERSION 3.22)
|
||||||
|
|
||||||
project(FastFileTransfer VERSION 2.19.0)
|
project(FastFileTransfer VERSION 2.21.0)
|
||||||
|
|
||||||
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
|
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
|
||||||
set(CMAKE_C_STANDARD 11)
|
set(CMAKE_C_STANDARD 11)
|
||||||
@@ -38,11 +38,24 @@ if(ENABLE_COVERAGE)
|
|||||||
add_link_options(--coverage)
|
add_link_options(--coverage)
|
||||||
endif()
|
endif()
|
||||||
|
|
||||||
|
# --- Build hardening option ---
|
||||||
|
# Production hardening is applied to the shipping server/client binaries only,
|
||||||
|
# and only when no sanitizer or coverage instrumentation is active: sanitizers
|
||||||
|
# carry their own instrumentation, and _FORTIFY_SOURCE requires an optimising
|
||||||
|
# build (never the -O0 used for coverage).
|
||||||
|
option(ENABLE_HARDENING "Enable compiler/linker hardening for production targets" ON)
|
||||||
|
set(HARDENING_ACTIVE OFF)
|
||||||
|
if(ENABLE_HARDENING AND SANITIZER STREQUAL "none" AND NOT ENABLE_COVERAGE)
|
||||||
|
set(HARDENING_ACTIVE ON)
|
||||||
|
endif()
|
||||||
|
|
||||||
include(FetchContent)
|
include(FetchContent)
|
||||||
FetchContent_Declare(
|
FetchContent_Declare(
|
||||||
xxhash
|
xxhash
|
||||||
GIT_REPOSITORY https://github.com/Cyan4973/xxHash
|
GIT_REPOSITORY https://github.com/Cyan4973/xxHash
|
||||||
GIT_TAG v0.8.3
|
# v0.8.3 is a lightweight tag pointing at this exact commit (no ^{} peel
|
||||||
|
# entry); pin the commit SHA instead of the mutable tag.
|
||||||
|
GIT_TAG e626a72bc2321cd320e953a0ccf1584cad60f363 # v0.8.3
|
||||||
SOURCE_SUBDIR cmake_unofficial
|
SOURCE_SUBDIR cmake_unofficial
|
||||||
)
|
)
|
||||||
FetchContent_MakeAvailable(xxhash)
|
FetchContent_MakeAvailable(xxhash)
|
||||||
@@ -57,35 +70,179 @@ endif()
|
|||||||
|
|
||||||
find_package(OpenSSL REQUIRED)
|
find_package(OpenSSL REQUIRED)
|
||||||
|
|
||||||
file(GLOB SHARED_SRCS "src/shared/*.c")
|
# --- Explicit source lists ---
|
||||||
set(FILE_STORE_SRCS "${CMAKE_CURRENT_SOURCE_DIR}/src/shared/file_store.c")
|
# The shared library is self-contained: it must never depend on the client or
|
||||||
list(REMOVE_ITEM SHARED_SRCS ${FILE_STORE_SRCS})
|
# server modules. In particular, the receiver pipeline (receive_thread /
|
||||||
file(GLOB SERVER_SRCS "src/server/*.c")
|
# write_thread) lives under src/server, not here, so the client executable can
|
||||||
set(SERVER_RECEIVER_SRCS src/server/receiver.c)
|
# link the shared library without pulling in any server code.
|
||||||
file(GLOB CLIENT_SRCS "src/client/*.c")
|
set(SHARED_SRCS
|
||||||
|
src/shared/array_list.c
|
||||||
|
src/shared/batch.c
|
||||||
|
src/shared/charset.c
|
||||||
|
src/shared/checksum.c
|
||||||
|
src/shared/chmod.c
|
||||||
|
src/shared/chunk.c
|
||||||
|
src/shared/compression.c
|
||||||
|
src/shared/config.c
|
||||||
|
src/shared/credentials.c
|
||||||
|
src/shared/daemon_conf.c
|
||||||
|
src/shared/daemon_limits.c
|
||||||
|
src/shared/data.c
|
||||||
|
src/shared/delay_updates.c
|
||||||
|
src/shared/delta.c
|
||||||
|
src/shared/file.c
|
||||||
|
src/shared/file_list.c
|
||||||
|
src/shared/file_receive.c
|
||||||
|
src/shared/file_send.c
|
||||||
|
src/shared/file_store.c
|
||||||
|
src/shared/filter.c
|
||||||
|
src/shared/hardlink.c
|
||||||
|
src/shared/identity.c
|
||||||
|
src/shared/log.c
|
||||||
|
src/shared/metadata.c
|
||||||
|
src/shared/motd.c
|
||||||
|
src/shared/multiprocessing.c
|
||||||
|
src/shared/protocol.c
|
||||||
|
src/shared/queue.c
|
||||||
|
src/shared/stop_condition.c
|
||||||
|
src/shared/transport_ssh.c
|
||||||
|
src/shared/transport_tcp.c
|
||||||
|
src/shared/transport_tls.c
|
||||||
|
src/shared/utils.c
|
||||||
|
src/shared/xattr.c
|
||||||
|
)
|
||||||
|
|
||||||
|
# Server implementation (no main): the receiver read/write pipeline plus the
|
||||||
|
# CLI parser. The server executable adds its own main (server.c).
|
||||||
|
set(SERVER_CORE_SRCS
|
||||||
|
src/server/receiver.c
|
||||||
|
src/server/receiver_pipeline.c
|
||||||
|
src/server/server_cli.c
|
||||||
|
)
|
||||||
|
set(SERVER_MAIN_SRCS src/server/server.c)
|
||||||
|
|
||||||
|
# Client implementation (no main): everything except the CLI entry point.
|
||||||
|
set(CLIENT_CORE_SRCS
|
||||||
|
src/client/change_list.c
|
||||||
|
src/client/client_send.c
|
||||||
|
src/client/client_validation.c
|
||||||
|
src/client/scanner.c
|
||||||
|
src/client/usage.c
|
||||||
|
)
|
||||||
|
set(CLIENT_MAIN_SRCS src/client/client_cli.c)
|
||||||
|
|
||||||
|
# --- Library targets ---
|
||||||
|
add_library(fastsync_shared STATIC ${SHARED_SRCS})
|
||||||
|
target_include_directories(fastsync_shared PUBLIC src/shared)
|
||||||
|
target_link_libraries(fastsync_shared PUBLIC Threads::Threads ${ZSTD_LIBRARY} OpenSSL::SSL
|
||||||
|
OpenSSL::Crypto xxhash)
|
||||||
|
|
||||||
|
add_library(fastsync_client_core STATIC ${CLIENT_CORE_SRCS})
|
||||||
|
target_include_directories(fastsync_client_core PUBLIC src/client)
|
||||||
|
target_link_libraries(fastsync_client_core PUBLIC fastsync_shared)
|
||||||
|
|
||||||
|
add_library(fastsync_server_core STATIC ${SERVER_CORE_SRCS})
|
||||||
|
target_include_directories(fastsync_server_core PUBLIC src/server)
|
||||||
|
target_link_libraries(fastsync_server_core PUBLIC fastsync_shared)
|
||||||
|
|
||||||
# --- Main executables ---
|
# --- Main executables ---
|
||||||
add_executable(server ${SERVER_SRCS} ${SHARED_SRCS} ${FILE_STORE_SRCS})
|
# The client links only the shared library and its own core; it deliberately
|
||||||
target_include_directories(server PRIVATE src/shared src/server src/client)
|
# does NOT get src/server on its include path nor compile receiver.c.
|
||||||
target_link_libraries(server PRIVATE Threads::Threads ${ZSTD_LIBRARY} OpenSSL::SSL OpenSSL::Crypto xxhash)
|
add_executable(server ${SERVER_MAIN_SRCS})
|
||||||
|
target_link_libraries(server PRIVATE fastsync_server_core)
|
||||||
|
|
||||||
add_executable(client ${CLIENT_SRCS} ${SHARED_SRCS} ${FILE_STORE_SRCS} ${SERVER_RECEIVER_SRCS})
|
add_executable(client ${CLIENT_MAIN_SRCS})
|
||||||
target_include_directories(client PRIVATE src/shared src/server src/client)
|
target_link_libraries(client PRIVATE fastsync_client_core)
|
||||||
target_link_libraries(client PRIVATE Threads::Threads ${ZSTD_LIBRARY} OpenSSL::SSL OpenSSL::Crypto xxhash)
|
|
||||||
|
# --- Production hardening ---
|
||||||
|
# Each compile flag is probed so a compiler/architecture that lacks it still
|
||||||
|
# configures cleanly. _FORTIFY_SOURCE is guarded separately because it only
|
||||||
|
# works in an optimising build. xxHash is a static archive built by
|
||||||
|
# FetchContent, so it must be position-independent for the -pie link; the same
|
||||||
|
# applies to the first-party static libraries linked into the -pie binaries.
|
||||||
|
if(HARDENING_ACTIVE)
|
||||||
|
set_target_properties(xxhash fastsync_shared fastsync_server_core fastsync_client_core
|
||||||
|
PROPERTIES POSITION_INDEPENDENT_CODE ON)
|
||||||
|
include(CheckCCompilerFlag)
|
||||||
|
foreach(flag -fstack-protector-strong -fstack-clash-protection -fPIE)
|
||||||
|
string(MAKE_C_IDENTIFIER "HARDEN_${flag}" _harden_var)
|
||||||
|
check_c_compiler_flag("${flag}" ${_harden_var})
|
||||||
|
endforeach()
|
||||||
|
check_c_compiler_flag("-D_FORTIFY_SOURCE=2" HARDEN_FORTIFY_SOURCE)
|
||||||
|
foreach(target fastsync_shared fastsync_server_core fastsync_client_core server client)
|
||||||
|
foreach(flag -fstack-protector-strong -fstack-clash-protection -fPIE)
|
||||||
|
string(MAKE_C_IDENTIFIER "HARDEN_${flag}" _harden_var)
|
||||||
|
if(${_harden_var})
|
||||||
|
target_compile_options(${target} PRIVATE ${flag})
|
||||||
|
endif()
|
||||||
|
endforeach()
|
||||||
|
if(HARDEN_FORTIFY_SOURCE)
|
||||||
|
target_compile_options(${target} PRIVATE -D_FORTIFY_SOURCE=2)
|
||||||
|
endif()
|
||||||
|
endforeach()
|
||||||
|
foreach(target server client)
|
||||||
|
target_link_options(${target} PRIVATE -pie -Wl,-z,relro -Wl,-z,now -Wl,-z,noexecstack)
|
||||||
|
endforeach()
|
||||||
|
endif()
|
||||||
|
|
||||||
# --- Testing ---
|
# --- Testing ---
|
||||||
enable_testing()
|
enable_testing()
|
||||||
|
|
||||||
# Common test libraries
|
# --- Unit tests ---
|
||||||
set(TEST_LIBS Threads::Threads ${ZSTD_LIBRARY} OpenSSL::SSL OpenSSL::Crypto xxhash)
|
# The monolithic test binary exercises both client and server code, so it is
|
||||||
set(TEST_INCLUDES tests src/shared src/server src/client)
|
# the one place that legitimately sees both include directories and links both
|
||||||
|
# core libraries. client_cli.c is compiled here directly (with the test build
|
||||||
|
# define) rather than linked from fastsync_client_core so its test-only shims
|
||||||
|
# and the absence of main() are preserved.
|
||||||
|
set(TEST_SRCS
|
||||||
|
tests/runner.c
|
||||||
|
tests/test_array_list.c
|
||||||
|
tests/test_batch.c
|
||||||
|
tests/test_change_list.c
|
||||||
|
tests/test_checksum.c
|
||||||
|
tests/test_chunk.c
|
||||||
|
tests/test_client_cli.c
|
||||||
|
tests/test_compression.c
|
||||||
|
tests/test_config.c
|
||||||
|
tests/test_credentials.c
|
||||||
|
tests/test_daemon_conf.c
|
||||||
|
tests/test_daemon_limits.c
|
||||||
|
tests/test_data.c
|
||||||
|
tests/test_delay_updates.c
|
||||||
|
tests/test_delta.c
|
||||||
|
tests/test_file.c
|
||||||
|
tests/test_file_list.c
|
||||||
|
tests/test_file_sendfile.c
|
||||||
|
tests/test_fuzz_smoke.c
|
||||||
|
tests/test_glob.c
|
||||||
|
tests/test_hardlink.c
|
||||||
|
tests/test_iconv.c
|
||||||
|
tests/test_log.c
|
||||||
|
tests/test_metadata.c
|
||||||
|
tests/test_motd.c
|
||||||
|
tests/test_multiprocessing.c
|
||||||
|
tests/test_property.c
|
||||||
|
tests/test_protocol.c
|
||||||
|
tests/test_protocol_error.c
|
||||||
|
tests/test_queue.c
|
||||||
|
tests/test_receiver_timeout.c
|
||||||
|
tests/test_robustness.c
|
||||||
|
tests/test_scanner.c
|
||||||
|
tests/test_server.c
|
||||||
|
tests/test_server_cli.c
|
||||||
|
tests/test_shared_utils.c
|
||||||
|
tests/test_stop.c
|
||||||
|
tests/test_stress.c
|
||||||
|
tests/test_transport_ssh.c
|
||||||
|
tests/test_transport_tcp.c
|
||||||
|
tests/test_transport_tls.c
|
||||||
|
tests/test_xattr.c
|
||||||
|
)
|
||||||
|
|
||||||
# Monolithic test binary (backward compatible)
|
add_executable(tests ${TEST_SRCS} src/client/client_cli.c)
|
||||||
file(GLOB TEST_SRCS "tests/test_*.c" "tests/runner.c")
|
target_include_directories(tests PRIVATE tests)
|
||||||
add_executable(tests ${TEST_SRCS} ${SHARED_SRCS} ${FILE_STORE_SRCS} ${SERVER_RECEIVER_SRCS} src/client/scanner.c src/client/change_list.c src/client/client_cli.c src/client/client_validation.c src/client/usage.c src/server/server_cli.c)
|
|
||||||
target_include_directories(tests PRIVATE ${TEST_INCLUDES})
|
|
||||||
target_compile_definitions(tests PRIVATE FASTSYNC_TEST_BUILD)
|
target_compile_definitions(tests PRIVATE FASTSYNC_TEST_BUILD)
|
||||||
target_link_libraries(tests PRIVATE ${TEST_LIBS})
|
target_link_libraries(tests PRIVATE fastsync_server_core fastsync_client_core)
|
||||||
add_test(NAME unit_all COMMAND tests)
|
add_test(NAME unit_all COMMAND tests)
|
||||||
|
|
||||||
# --- Fuzz targets (requires clang) ---
|
# --- Fuzz targets (requires clang) ---
|
||||||
@@ -94,13 +251,29 @@ if(ENABLE_FUZZ)
|
|||||||
if(NOT CMAKE_C_COMPILER_ID MATCHES "Clang")
|
if(NOT CMAKE_C_COMPILER_ID MATCHES "Clang")
|
||||||
message(FATAL_ERROR "ENABLE_FUZZ requires Clang (compiler is ${CMAKE_C_COMPILER_ID})")
|
message(FATAL_ERROR "ENABLE_FUZZ requires Clang (compiler is ${CMAKE_C_COMPILER_ID})")
|
||||||
endif()
|
endif()
|
||||||
file(GLOB FUZZ_SRCS "tests/fuzz/*.c")
|
set(FUZZ_SRCS
|
||||||
|
tests/fuzz/fuzz_chunk_deserialize.c
|
||||||
|
tests/fuzz/fuzz_compress_decompress.c
|
||||||
|
tests/fuzz/fuzz_config_receive.c
|
||||||
|
tests/fuzz/fuzz_delta_deserialize.c
|
||||||
|
tests/fuzz/fuzz_delta_signature_deserialize.c
|
||||||
|
tests/fuzz/fuzz_glob_match.c
|
||||||
|
tests/fuzz/fuzz_identity_parse.c
|
||||||
|
tests/fuzz/fuzz_manifest.c
|
||||||
|
tests/fuzz/fuzz_metadata_from_buf.c
|
||||||
|
tests/fuzz/fuzz_protocol_framing.c
|
||||||
|
tests/fuzz/fuzz_xattr_block.c
|
||||||
|
)
|
||||||
|
# Compile the sources under test directly so libFuzzer's coverage
|
||||||
|
# instrumentation sees them (static libraries would be uninstrumented).
|
||||||
|
set(FUZZ_CORE_SRCS ${SHARED_SRCS} src/server/receiver.c src/server/receiver_pipeline.c)
|
||||||
foreach(FUZZ_SRC ${FUZZ_SRCS})
|
foreach(FUZZ_SRC ${FUZZ_SRCS})
|
||||||
get_filename_component(FUZZ_NAME ${FUZZ_SRC} NAME_WE)
|
get_filename_component(FUZZ_NAME ${FUZZ_SRC} NAME_WE)
|
||||||
add_executable(${FUZZ_NAME} ${FUZZ_SRC} ${SHARED_SRCS} ${FILE_STORE_SRCS} ${SERVER_RECEIVER_SRCS})
|
add_executable(${FUZZ_NAME} ${FUZZ_SRC} ${FUZZ_CORE_SRCS})
|
||||||
target_include_directories(${FUZZ_NAME} PRIVATE ${TEST_INCLUDES})
|
target_include_directories(${FUZZ_NAME} PRIVATE tests src/shared src/server)
|
||||||
target_compile_options(${FUZZ_NAME} PRIVATE -fsanitize=fuzzer,address,undefined -fno-omit-frame-pointer)
|
target_compile_options(${FUZZ_NAME} PRIVATE -fsanitize=fuzzer,address,undefined -fno-omit-frame-pointer)
|
||||||
target_link_options(${FUZZ_NAME} PRIVATE -fsanitize=fuzzer,address,undefined)
|
target_link_options(${FUZZ_NAME} PRIVATE -fsanitize=fuzzer,address,undefined)
|
||||||
target_link_libraries(${FUZZ_NAME} PRIVATE ${TEST_LIBS})
|
target_link_libraries(${FUZZ_NAME} PRIVATE Threads::Threads ${ZSTD_LIBRARY} OpenSSL::SSL
|
||||||
|
OpenSSL::Crypto xxhash)
|
||||||
endforeach()
|
endforeach()
|
||||||
endif()
|
endif()
|
||||||
|
|||||||
@@ -96,6 +96,8 @@ partial, alternate, and planned behavior.
|
|||||||
|
|
||||||
### Build
|
### Build
|
||||||
|
|
||||||
|
`compile_commands.json` is a symlink to `build/compile_commands.json` and is used by clangd/editor tooling; its target is generated by the build, so it dangles until the first build.
|
||||||
|
|
||||||
### Client
|
### Client
|
||||||
|
|
||||||
| Argument | Description |
|
| Argument | Description |
|
||||||
@@ -104,7 +106,7 @@ partial, alternate, and planned behavior.
|
|||||||
| `-c, --checksum` | Verify content by checksum instead of size+mtime |
|
| `-c, --checksum` | Verify content by checksum instead of size+mtime |
|
||||||
| `-z, --compress [level]` | Enable streaming zstd compression (level 1–22, default 5) |
|
| `-z, --compress [level]` | Enable streaming zstd compression (level 1–22, default 5) |
|
||||||
| `-a, --archive` | rsync archive mode (`-rlptgoD`): links, metadata, devices and specials (not compression/multithreading) |
|
| `-a, --archive` | rsync archive mode (`-rlptgoD`): links, metadata, devices and specials (not compression/multithreading) |
|
||||||
| `-j, --threads` | Multithreading mode |
|
| `-j, --threads[=N]` | Multithreading mode; `N` (1–256) sets the parallel scanner worker count, bare `-j`/`--threads` uses the default |
|
||||||
| `-m` | rsync `--prune-empty-dirs` (short form now rsync-parity) |
|
| `-m` | rsync `--prune-empty-dirs` (short form now rsync-parity) |
|
||||||
| `--chunk-serialization` | Chunk serialization (batch all files per chunk; long form only) |
|
| `--chunk-serialization` | Chunk serialization (batch all files per chunk; long form only) |
|
||||||
| `-s` | rsync `--secluded-args` compatibility no-op (remote SSH argv is already injection-safe) |
|
| `-s` | rsync `--secluded-args` compatibility no-op (remote SSH argv is already injection-safe) |
|
||||||
@@ -132,7 +134,7 @@ partial, alternate, and planned behavior.
|
|||||||
| `--existing` | Skip files not already present at the destination; update existing files normally. |
|
| `--existing` | Skip files not already present at the destination; update existing files normally. |
|
||||||
| `--bwlimit <KB/s>` | Bandwidth limit in kilobytes per second |
|
| `--bwlimit <KB/s>` | Bandwidth limit in kilobytes per second |
|
||||||
| `--chunk-size <n>` | Chunk size in bytes (default: 10485760) |
|
| `--chunk-size <n>` | Chunk size in bytes (default: 10485760) |
|
||||||
| `--timeout <sec>` | I/O timeout in seconds (default: 30) |
|
| `--timeout <sec>` | Positive I/O timeout in seconds, applied to both the socket (`SO_RCVTIMEO`/`SO_SNDTIMEO`, built-in default 30 s) and the per-message protocol poll deadline (built-in default 60 s). Omit the option to keep both built-ins; `0` is rejected. The server side keeps the built-in 60 s protocol window (the value is not sent on the wire). |
|
||||||
| `--contimeout <sec>` | Connection timeout in seconds (default: 10) |
|
| `--contimeout <sec>` | Connection timeout in seconds (default: 10) |
|
||||||
| `--backup` | Backup existing destination files before overwriting |
|
| `--backup` | Backup existing destination files before overwriting |
|
||||||
| `--backup-dir <dir>` | Target directory for backups (requires `--backup`) |
|
| `--backup-dir <dir>` | Target directory for backups (requires `--backup`) |
|
||||||
@@ -151,6 +153,19 @@ partial, alternate, and planned behavior.
|
|||||||
| `--ca <path>` | TLS CA certificate file for verification (PEM) |
|
| `--ca <path>` | TLS CA certificate file for verification (PEM) |
|
||||||
| `--client-cn <name>` | TLS client certificate common name; mandatory with `--tls` (a TLS connection always verifies the client CN) |
|
| `--client-cn <name>` | TLS client certificate common name; mandatory with `--tls` (a TLS connection always verifies the client CN) |
|
||||||
|
|
||||||
|
**Per-message vs. connection timeouts.** `--timeout` bounds each individual protocol
|
||||||
|
send/receive (the `poll()` deadline), so a peer that stops mid-frame is dropped. It
|
||||||
|
does not, by itself, stop a peer that keeps sending well-formed frames forever. The
|
||||||
|
receiver therefore also enforces two wall-clock (`CLOCK_MONOTONIC`) bounds on a
|
||||||
|
connection: a **1 hour** idle limit and a **24 hour** overall session cap. Only
|
||||||
|
frames that move real work (not `STATUS_KEEPALIVE`/`STATUS_ABORT` and not an
|
||||||
|
empty `STATUS_CHECK_BATCH`/`STATUS_DIR_TIMES`) refresh the idle timestamp, so a
|
||||||
|
peer cannot hold a connection slot by emitting cheap empty frames; a peer that
|
||||||
|
fabricates minimal non-empty frames can still occupy a slot until the 24 hour
|
||||||
|
cap, since no bound can require actual payload without risking a legitimate
|
||||||
|
long operation. Both are deliberately generous so a legitimate long-running
|
||||||
|
transfer is never aborted.
|
||||||
|
|
||||||
### Server
|
### Server
|
||||||
|
|
||||||
| Argument | Description |
|
| Argument | Description |
|
||||||
@@ -163,6 +178,7 @@ partial, alternate, and planned behavior.
|
|||||||
| `--ca <path>` | TLS CA certificate file for verification (PEM) |
|
| `--ca <path>` | TLS CA certificate file for verification (PEM) |
|
||||||
| `--destination-root <path>` | Authorized destination root (default: `.`) |
|
| `--destination-root <path>` | Authorized destination root (default: `.`) |
|
||||||
| `--allow-delete` | Permit manifest deletion |
|
| `--allow-delete` | Permit manifest deletion |
|
||||||
|
| `--allow-super` | Standalone TCP listener only: keep super-user activities enabled for a **root** receiver. Without it a root standalone server forces `SUPER_MODE_OFF`, so client `--devices`/`--write-devices`/`--super` and client-chosen ownership requests are skipped/refused. **Rejected with `--stdio`** (the SSH remote argv is client-composed, so a client could otherwise pass it and defeat the secure default; operators exposing `fastsync-server --stdio` over SSH must use a forced command if the default must hold). No effect when not root. |
|
||||||
| `--allow-unauthenticated` | Permit plaintext TCP clients. For an `auth users` module this opts in **loopback plaintext only**; remote auth still requires verified TLS, so the flag never permits remote plaintext auth. |
|
| `--allow-unauthenticated` | Permit plaintext TCP clients. For an `auth users` module this opts in **loopback plaintext only**; remote auth still requires verified TLS, so the flag never permits remote plaintext auth. |
|
||||||
| `-v, --verbose` | Enable debug logging |
|
| `-v, --verbose` | Enable debug logging |
|
||||||
| `--help` | Show help |
|
| `--help` | Show help |
|
||||||
@@ -189,7 +205,7 @@ partial, alternate, and planned behavior.
|
|||||||
3. **FileMetadata** — `mode`, `uid`, `gid`, `mtime_sec`, `mtime_nsec`;
|
3. **FileMetadata** — `mode`, `uid`, `gid`, `mtime_sec`, `mtime_nsec`;
|
||||||
uid / gid are advisory wire fields and are never applied by the receiver;
|
uid / gid are advisory wire fields and are never applied by the receiver;
|
||||||
atime is unsupported
|
atime is unsupported
|
||||||
4. **Config** — runtime parameters (transported over wire, TLS settings excluded). Includes `timeout`, `contimeout`, `quiet`, `backup`, `backup_dir`, `stats`, `max_depth`, `log_file`, `queue_size`.
|
4. **Config** — runtime parameters (transported over wire, TLS settings excluded). Includes `timeout`, `contimeout`, `quiet`, `backup`, `backup_dir`, `stats`, `max_depth`, `log_file`.
|
||||||
5. **Queue** — thread-safe bounded queue with condition variables
|
5. **Queue** — thread-safe bounded queue with condition variables
|
||||||
6. **DirectoryScanner** — recursive BFS traversal with exclude and include pattern support, max-depth enforcement
|
6. **DirectoryScanner** — recursive BFS traversal with exclude and include pattern support, max-depth enforcement
|
||||||
|
|
||||||
@@ -289,17 +305,28 @@ The remote host must have `fastsync-server` available in `PATH`, or use
|
|||||||
working directory, so use a destination below that directory unless the
|
working directory, so use a destination below that directory unless the
|
||||||
remote server is otherwise configured with a matching authorized root.
|
remote server is otherwise configured with a matching authorized root.
|
||||||
|
|
||||||
|
The remote `--stdio` server argv is composed by the client, so it must never
|
||||||
|
be trusted to opt a root receiver into super-user activities: `--allow-super`
|
||||||
|
is rejected with `--stdio` and super stays off on that path. Operators
|
||||||
|
exposing `fastsync-server --stdio` over SSH must use a forced command (e.g. an
|
||||||
|
`authorized_keys` `command=` entry) if the default must hold.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
ssh user@host 'mkdir -p destination'
|
ssh user@host 'mkdir -p destination'
|
||||||
./build/client /path/to/source user@host:destination
|
./build/client /path/to/source user@host:destination
|
||||||
```
|
```
|
||||||
|
|
||||||
|
FastSync is **push-only**: the source (first argument) is always a local
|
||||||
|
directory and only the destination may be remote. A remote source such as
|
||||||
|
`client user@host:src ./local` (a "pull") is intentionally not supported; see
|
||||||
|
[RSYNC_COMPAT.md](RSYNC_COMPAT.md#direction).
|
||||||
|
|
||||||
### TCP transfer
|
### TCP transfer
|
||||||
|
|
||||||
Start the FastSync server:
|
Start the FastSync server:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
./build/server --destination-root /path/to -p 8080
|
./build/server --destination-root /path/to -p 8080 --allow-unauthenticated
|
||||||
```
|
```
|
||||||
|
|
||||||
Then run the client:
|
Then run the client:
|
||||||
@@ -351,7 +378,7 @@ FastSync-native are optional performance or transport extensions.
|
|||||||
./build/client --incremental --checksum /source/ user@host:destination/
|
./build/client --incremental --checksum /source/ user@host:destination/
|
||||||
|
|
||||||
#Preserve supported mode and timestamp metadata
|
#Preserve supported mode and timestamp metadata
|
||||||
./build/client -M /source/ user@host:destination/
|
./build/client --preserve /source/ user@host:destination/
|
||||||
|
|
||||||
#Keep backups of overwritten destination files
|
#Keep backups of overwritten destination files
|
||||||
./build/client --backup --backup-dir backups \
|
./build/client --backup --backup-dir backups \
|
||||||
@@ -365,7 +392,7 @@ features without changing the meaning of ordinary compatibility options.
|
|||||||
|
|
||||||
| Option | Purpose |
|
| Option | Purpose |
|
||||||
|---|---|
|
|---|---|
|
||||||
| `-j`, `--threads` | Enable the multithreaded scanner/loader/sender pipeline. |
|
| `-j`, `--threads[=N]` | Enable the multithreaded scanner/loader/sender pipeline. `N` (1–256) sets the parallel scanner worker count; bare `-j`/`--threads` uses the default. |
|
||||||
| `-z [level]`, `--compress [level]` | Enable streaming zstd compression, levels 1-22. |
|
| `-z [level]`, `--compress [level]` | Enable streaming zstd compression, levels 1-22. |
|
||||||
| `--compress-level <n>` | Set the zstd compression level. |
|
| `--compress-level <n>` | Set the zstd compression level. |
|
||||||
| `--zc <alg>` | Alias for `--compress-choice`. FastSync supports `zstd` and `none`. |
|
| `--zc <alg>` | Alias for `--compress-choice`. FastSync supports `zstd` and `none`. |
|
||||||
@@ -379,12 +406,12 @@ features without changing the meaning of ordinary compatibility options.
|
|||||||
| `--delta-block <bytes>` | Set the FastSync delta block size (`--block-size` is an alias). |
|
| `--delta-block <bytes>` | Set the FastSync delta block size (`--block-size` is an alias). |
|
||||||
| `--delta-max <bytes>` | Limit files eligible for FastSync delta transfer. |
|
| `--delta-max <bytes>` | Limit files eligible for FastSync delta transfer. |
|
||||||
| `--server-host <host>` | Select the TCP server host. |
|
| `--server-host <host>` | Select the TCP server host. |
|
||||||
| `--server-port <port>` | Select the TCP server port. |
|
| `--server-port <port>` | Select the TCP server port (`--port <port>` and `--port=<port>` are rsync-friendly aliases). |
|
||||||
| `--tls` | Enable TLS for TCP transport. |
|
| `--tls` | Enable TLS for TCP transport. |
|
||||||
| `--bwlimit <KB/s>` | Apply token-bucket bandwidth limiting. |
|
| `--bwlimit <KB/s>` | Apply token-bucket bandwidth limiting. |
|
||||||
| `--progress` | Show transfer progress and throughput. |
|
| `--progress` | Show transfer progress and throughput. |
|
||||||
| `--stats` | Print transfer statistics. |
|
| `--stats` | Print transfer statistics. |
|
||||||
| `--timeout <seconds>` | Set I/O timeout. |
|
| `--timeout <seconds>` | Set the socket **and** per-message protocol I/O timeout (positive seconds). Omit to keep the built-in 30 s socket / 60 s protocol defaults. |
|
||||||
| `--contimeout <seconds>` | Set connection timeout. |
|
| `--contimeout <seconds>` | Set connection timeout. |
|
||||||
|
|
||||||
Short-option conflicts with rsync have been resolved for the CLI namespace
|
Short-option conflicts with rsync have been resolved for the CLI namespace
|
||||||
@@ -465,7 +492,7 @@ link-target transfer remains incomplete. |
|
|||||||
| `--dest-dir <path>` | Set the destination directory explicitly. |
|
| `--dest-dir <path>` | Set the destination directory explicitly. |
|
||||||
| `--save-to-disk` | Enable server-side disk persistence. |
|
| `--save-to-disk` | Enable server-side disk persistence. |
|
||||||
| `--server-host <host>` | TCP server address. |
|
| `--server-host <host>` | TCP server address. |
|
||||||
| `--server-port <port>` | TCP server port. |
|
| `--server-port <port>` | TCP server port. `--port <port>` / `--port=<port>` is an alias. |
|
||||||
| `--tls` | Enable TLS. Requires `--cert` and `--key`. |
|
| `--tls` | Enable TLS. Requires `--cert` and `--key`. |
|
||||||
| `--cert <path>` | TLS certificate file. |
|
| `--cert <path>` | TLS certificate file. |
|
||||||
| `--key <path>` | TLS private key file. |
|
| `--key <path>` | TLS private key file. |
|
||||||
@@ -483,10 +510,67 @@ link-target transfer remains incomplete. |
|
|||||||
| `--ca <path>` | CA file for peer verification. |
|
| `--ca <path>` | CA file for peer verification. |
|
||||||
| `--destination-root <path>` | Confine received files to this server-side root;
|
| `--destination-root <path>` | Confine received files to this server-side root;
|
||||||
defaults to the current directory. |
|
defaults to the current directory. |
|
||||||
| `--allow-delete` | Permit client delete manifests. Deletion is refused by default. |
|
| `--allow-delete` | Permit client delete manifests. Deletion is refused by default. This also gates `--force` (which can recursively replace/remove a destination directory tree). |
|
||||||
|
| `--allow-super` | Standalone TCP listener only: keep super-user activities enabled for a **root** receiver. Without it a root standalone server forces `SUPER_MODE_OFF`, so client `--devices`/`--write-devices`/`--super` and client-chosen ownership requests are skipped/refused. Rejected with `--stdio` (the SSH remote argv is client-composed; use a forced command if the default must hold). No effect when not root. Daemon modules opt in per module with `client owner = yes`. |
|
||||||
| `-v`, `--verbose` | Enable debug logging. |
|
| `-v`, `--verbose` | Enable debug logging. |
|
||||||
| `--help` | Print server usage. |
|
| `--help` | Print server usage. |
|
||||||
|
|
||||||
|
### Daemon configuration
|
||||||
|
|
||||||
|
`fastsync-server --daemon --config FILE` reads a line-based module config (an
|
||||||
|
implicit global section, then `[module]` sections). Besides `port`, `motd file`,
|
||||||
|
and `address`, the global section accepts:
|
||||||
|
|
||||||
|
- `max connections = N` — global cap on concurrent connections, default 100. The
|
||||||
|
listener enforces it; `0`, negative, and non-numeric values are parse errors.
|
||||||
|
- `max connections per host = N` — cap on concurrent connections from a single
|
||||||
|
source IP, default 0 (unlimited). Enforced across all forked connection
|
||||||
|
children through a shared registry.
|
||||||
|
- `auth failure delay = MS` — milliseconds to sleep after a failed
|
||||||
|
authentication, default 500. `0` disables it and the value is capped at 5000,
|
||||||
|
so online password guessing is rate-limited per connection. Successful auths
|
||||||
|
are never delayed.
|
||||||
|
- `auth lockout threshold = N` — number of failed authentications from one source
|
||||||
|
IP before that source is locked out, default 10; `0` disables the lockout. The
|
||||||
|
failure counter is shared across every connection child, so the lockout holds
|
||||||
|
even when the next attempt is handled by a different forked child.
|
||||||
|
- `auth lockout duration = SECONDS` — how long a locked-out source is refused
|
||||||
|
(default 300). A locked-out client is refused before any SCRAM challenge is
|
||||||
|
sent; a successful authentication clears the counter.
|
||||||
|
- `hosts allow` / `hosts deny` — comma- and/or whitespace-separated host access
|
||||||
|
patterns.
|
||||||
|
|
||||||
|
A `[module]` may also set `max connections` (0 = unlimited; enforced per module
|
||||||
|
across all connection children) and its own `hosts allow`/`hosts deny`.
|
||||||
|
|
||||||
|
The per-host cap and the shared auth lockout identify a source by its numeric
|
||||||
|
peer IP. **Loopback peers (127.0.0.0/8, IPv6 `::1`) are exempt**: every local
|
||||||
|
client shares that one address, so counting or locking them out would let one
|
||||||
|
local process deny service to all the others. The per-module and global
|
||||||
|
`max connections` caps still apply to loopback. Because the key is the peer IP,
|
||||||
|
`max connections per host` and `auth lockout` also cannot distinguish clients
|
||||||
|
behind the same NAT, proxy, or reverse-proxy address — they share one budget and
|
||||||
|
one lockout counter, so an over-aggressive lockout can affect unrelated users
|
||||||
|
behind that address. Prefer TLS client certificates (`--client-cn`) plus
|
||||||
|
`hosts allow`/`hosts deny` for per-client policy when clients share an address,
|
||||||
|
and size `auth lockout threshold` accordingly.
|
||||||
|
|
||||||
|
The shared per-source table has a bounded lifetime: an entry with no live
|
||||||
|
connection is reclaimed once its lockout has expired, or after it has been idle
|
||||||
|
(300 s). If every entry is still live or locked, a new source is admitted without
|
||||||
|
per-host accounting (fail open) and a rate-limited warning is logged; the
|
||||||
|
per-module cap and host ACLs still apply. The occupancy counters are re-derived
|
||||||
|
from the shared slot table after every child exit, so a child killed mid-transfer
|
||||||
|
(or mid-registration) cannot leak a slot or an occupancy count.
|
||||||
|
|
||||||
|
Host patterns are `*` (match all), IPv4/IPv6 literals, or IPv4/IPv6 CIDR
|
||||||
|
(`10.0.0.0/8`, `2001:db8::/32`). Hostnames are not resolved, so hostname globs
|
||||||
|
are rejected at parse time rather than silently never matching. A matching
|
||||||
|
`hosts deny` rejects; if any `hosts allow` entries exist, a peer matching none of
|
||||||
|
them is rejected; deny takes precedence over allow. The global list is checked
|
||||||
|
before the module list, before authentication, and the connecting peer address
|
||||||
|
(IPv4 or IPv6) appears in the connection and authentication audit log lines.
|
||||||
|
|
||||||
## Architecture
|
## Architecture
|
||||||
|
|
||||||
### Client
|
### Client
|
||||||
@@ -511,7 +595,7 @@ defaults to the current directory. |
|
|||||||
|
|
||||||
## Protocol and Security
|
## Protocol and Security
|
||||||
|
|
||||||
FastSync protocol version `2.19.0` is shared by the client and server. The
|
FastSync protocol version `2.21.0` is shared by the client and server. The
|
||||||
current protocol is sender-driven and includes configuration negotiation,
|
current protocol is sender-driven and includes configuration negotiation,
|
||||||
including the maximum allocation limit, incremental checks, checksums,
|
including the maximum allocation limit, incremental checks, checksums,
|
||||||
manifests, keep-alives, abort handling, per-file remove-source results, and
|
manifests, keep-alives, abort handling, per-file remove-source results, and
|
||||||
@@ -601,7 +685,7 @@ Run the unit test binary:
|
|||||||
Run the Python integration suite:
|
Run the Python integration suite:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
python3 -m pytest tests/
|
python3 -m pytest tests/integration/ -n 4 --dist=load -m "not setpriv"
|
||||||
```
|
```
|
||||||
|
|
||||||
For stricter local validation:
|
For stricter local validation:
|
||||||
|
|||||||
+33
-13
@@ -93,7 +93,7 @@ This document maps rsync's full feature set to FastSync's current implementation
|
|||||||
|
|
||||||
| Flag | Rsync Description | FastSync Status | Notes |
|
| Flag | Rsync Description | FastSync Status | Notes |
|
||||||
|------|-------------------|-----------------|-------|
|
|------|-------------------|-----------------|-------|
|
||||||
| `-n`, `--dry-run` | Trial run with no changes | ✅ Implemented | `dry_run` config field |
|
| `-n`, `--dry-run` | Trial run with no changes | ✅ Implemented | Server-contacting since protocol 2.21.0. The final routing predicate is `dry_run_targets_server()` in `src/client/client_send.c`: any target a real run would reach over the wire selects the server-contacting path — an SSH transport, a daemon `host::module` destination, an explicit `--server-host` or `--server-port`/`--port`, TLS, or a source-bind `--address` — and the client handshakes with the receiver, which runs the normal read-only per-file check and answers `STATUS_DRY_RUN_TRANSFER`/`STATUS_OK` without mutating anything. A plain local destination (none of those) keeps the original client-side manifest that never dials the default `127.0.0.1:8080`. Would-delete reporting for `--delete*` is deferred (dry-run never deletes). |
|
||||||
| `-b`, `--backup` | Make backups of overwritten files | ✅ Implemented | Backup before overwrite |
|
| `-b`, `--backup` | Make backups of overwritten files | ✅ Implemented | Backup before overwrite |
|
||||||
| `--backup-dir=DIR` | Backup directory hierarchy | ✅ Implemented | `backup_dir` config field |
|
| `--backup-dir=DIR` | Backup directory hierarchy | ✅ Implemented | `backup_dir` config field |
|
||||||
| `--suffix=SUFFIX` | Backup suffix (default ~) | ✅ Implemented | `suffix` config field |
|
| `--suffix=SUFFIX` | Backup suffix (default ~) | ✅ Implemented | `suffix` config field |
|
||||||
@@ -257,14 +257,14 @@ why plain `--append` works on the normal atomic path, not only with `--inplace`.
|
|||||||
| `-N`, `--crtimes` | Preserve create times | ⛔ Impossible/Divergence | Birth-times cannot be set by any portable filesystem call (`utimensat`/`futimens` only set atime/mtime), so this row is an explicit **Impossible/Divergence** (Phase 7 Wave B). Capture + transmit stays: `statx(STATX_BTIME)` on Linux records the source birth time as a wire field; the receiver logs a debug note that it cannot be applied and continues — never failing the transfer and never pretending it worked. On platforms without `statx` it parses as a documented no-op (flag accepted; nothing is captured). Implies metadata transmission. Wire: new `crtime` fields + a `preserve_crtimes` config boolean; `PROTOCOL_VERSION` bumped **2.11.0 → 2.12.0** (see the Phase-4 metadata-time notes) |
|
| `-N`, `--crtimes` | Preserve create times | ⛔ Impossible/Divergence | Birth-times cannot be set by any portable filesystem call (`utimensat`/`futimens` only set atime/mtime), so this row is an explicit **Impossible/Divergence** (Phase 7 Wave B). Capture + transmit stays: `statx(STATX_BTIME)` on Linux records the source birth time as a wire field; the receiver logs a debug note that it cannot be applied and continues — never failing the transfer and never pretending it worked. On platforms without `statx` it parses as a documented no-op (flag accepted; nothing is captured). Implies metadata transmission. Wire: new `crtime` fields + a `preserve_crtimes` config boolean; `PROTOCOL_VERSION` bumped **2.11.0 → 2.12.0** (see the Phase-4 metadata-time notes) |
|
||||||
| `-O`, `--omit-dir-times` | Omit dirs from --times | ✅ Implemented | Real modifier now that FastSync preserves directory times. With metadata on, the scanner captures every traversed source directory's mtime (and atime under `-U`) and the sender transmits them in trailing `STATUS_DIR_TIMES` frame(s) **after all file data and the optional delete manifest** (chunked at the receiver's `MAX_MANIFEST_ENTRIES` per-frame cap); a dir-time entry only RECORDS metadata and never creates the directory, so empty source directories stay untransferred. The receiver defers applying them until its delete / `--delay-updates` publication phases have committed, so writing or removing a child never clobbers a parent directory's mtime (rsync applies directory times at the end for exactly this reason). When `-O` is set (the boolean crosses the wire) the receiver does not apply any of them; without `-O` an `-a`/`--preserve` transfer now restores directory times (reversing the old "never preserves dir times" divergence). Wire change: the terminal `STATUS_DIR_TIMES` frame; `PROTOCOL_VERSION` bumped **2.16.0 → 2.17.0** |
|
| `-O`, `--omit-dir-times` | Omit dirs from --times | ✅ Implemented | Real modifier now that FastSync preserves directory times. With metadata on, the scanner captures every traversed source directory's mtime (and atime under `-U`) and the sender transmits them in trailing `STATUS_DIR_TIMES` frame(s) **after all file data and the optional delete manifest** (chunked at the receiver's `MAX_MANIFEST_ENTRIES` per-frame cap); a dir-time entry only RECORDS metadata and never creates the directory, so empty source directories stay untransferred. The receiver defers applying them until its delete / `--delay-updates` publication phases have committed, so writing or removing a child never clobbers a parent directory's mtime (rsync applies directory times at the end for exactly this reason). When `-O` is set (the boolean crosses the wire) the receiver does not apply any of them; without `-O` an `-a`/`--preserve` transfer now restores directory times (reversing the old "never preserves dir times" divergence). Wire change: the terminal `STATUS_DIR_TIMES` frame; `PROTOCOL_VERSION` bumped **2.16.0 → 2.17.0** |
|
||||||
| `-J`, `--omit-link-times` | Omit symlinks from --times | ✅ Implemented | Real modifier now that FastSync preserves symlink times. Symlink entries already carried their metadata on `STATUS_SYMLINK`; the receiver now applies it with **no-follow primitives only** (`utimensat(..., AT_SYMLINK_NOFOLLOW)`, plus best-effort `fchmodat(..., AT_SYMLINK_NOFOLLOW)` and policy-gated `fchownat(..., AT_SYMLINK_NOFOLLOW)`), so the link itself is stamped without ever dereferencing it, confined fd-relative below the authorized receive root. A symlink has no children, so the times are applied immediately at creation. When `-J` is set (the boolean crosses the wire) the receiver skips the timestamps (mode/ownership are unaffected); without `-J` an `-a`/`-l` transfer restores symlink mtimes. Wire change alongside `-O`: the shared `STATUS_DIR_TIMES` frame; `PROTOCOL_VERSION` bumped **2.16.0 → 2.17.0** |
|
| `-J`, `--omit-link-times` | Omit symlinks from --times | ✅ Implemented | Real modifier now that FastSync preserves symlink times. Symlink entries already carried their metadata on `STATUS_SYMLINK`; the receiver now applies it with **no-follow primitives only** (`utimensat(..., AT_SYMLINK_NOFOLLOW)`, plus best-effort `fchmodat(..., AT_SYMLINK_NOFOLLOW)` and policy-gated `fchownat(..., AT_SYMLINK_NOFOLLOW)`), so the link itself is stamped without ever dereferencing it, confined fd-relative below the authorized receive root. A symlink has no children, so the times are applied immediately at creation. When `-J` is set (the boolean crosses the wire) the receiver skips the timestamps (mode/ownership are unaffected); without `-J` an `-a`/`-l` transfer restores symlink mtimes. Wire change alongside `-O`: the shared `STATUS_DIR_TIMES` frame; `PROTOCOL_VERSION` bumped **2.16.0 → 2.17.0** |
|
||||||
| `--super` | Receiver attempts super-user activities | ✅ Implemented | Phase 7 Wave E: receiver-side **safe-subset + clear-refusal** privilege model, tri-state `super_mode` (auto/on/off). `--super` **permits** the receiver to attempt super-user activities — ownership application and char/block device-node creation — that are already confined fd-relative below the authorized receive root; `--no-super` **forbids** them even when the receiver is root; the default (`auto`) preserves the pre-existing **best-effort** behavior of *attempting* them (not only when already root: an unprivileged attempt is refused by the kernel and skipped per entry, matching FastSync's history). The server additionally accepts an operator-level `--no-super` veto that forces `OFF` for every connection it accepts (so it also refuses any client `--copy-as`/`--super`); the `--fake-super` owner replay and the `--write-devices` write path are gated by the same policy. **FastSync never elevates**: no `setuid`/`seteuid`/`setgid` is ever called, and `--super` never bypasses the confinement floor (`file_open_secure_parent`, `O_NOFOLLOW`, root checks) — it only permits an attempt that is already confined. `--super` does **not** imply `--numeric-ids` and never enables client-chosen ownership on its own: ownership is applied only when an explicit identity policy (`--usermap`/`--groupmap`/`--chown`/`--numeric-ids`/`--copy-as`) is also given. A non-root receiver given `--super` logs exactly one warning at activation and each confined attempt is then refused by the kernel and skipped per entry (never aborts); `--no-super` suppresses ownership, char/block `mknod`, `--write-devices` and the fake-super owner replay, while unprivileged FIFO creation is unaffected. Wire: one trailing `super_mode` int on the config frame (validated 0..2), sent **before** the `--copy-as` block (fixed order: super int, then copy-as presence int + ids); `PROTOCOL_VERSION` bumped **2.17.0 → 2.18.0**. **Documented divergence from rsync:** rsync's `--super` runs the receiver with elevated privilege; FastSync only permits a confined attempt and never elevates |
|
| `--super` | Receiver attempts super-user activities | ✅ Implemented | Phase 7 Wave E: receiver-side **safe-subset + clear-refusal** privilege model, tri-state `super_mode` (auto/on/off). `--super` **permits** the receiver to attempt super-user activities — ownership application and char/block device-node creation — that are already confined fd-relative below the authorized receive root; `--no-super` **forbids** them even when the receiver is root; the default (`auto`) preserves the pre-existing **best-effort** behavior of *attempting* them (not only when already root: an unprivileged attempt is refused by the kernel and skipped per entry, matching FastSync's history). The server additionally accepts an operator-level `--no-super` veto that forces `OFF` for every connection it accepts (so it also refuses any client `--copy-as`/`--super`); a **privileged (root) standalone TCP listener now also defaults to `OFF`** unless the operator opts in with the new server-only `--allow-super` flag (the flag is **rejected with `--stdio`**, whose remote argv is composed by the client and must never defeat the secure default; operators exposing `fastsync-server --stdio` over SSH need a forced command if the default must hold. An unprivileged receiver is unchanged, since the kernel refuses the confined attempts anyway; the `--daemon` path keeps its per-module `client owner = yes` opt-in); the `--fake-super` owner replay and the `--write-devices` write path are gated by the same policy. **FastSync never elevates**: no `setuid`/`seteuid`/`setgid` is ever called, and `--super` never bypasses the confinement floor (`file_open_secure_parent`, `O_NOFOLLOW`, root checks) — it only permits an attempt that is already confined. `--super` does **not** imply `--numeric-ids` and never enables client-chosen ownership on its own: ownership is applied only when an explicit identity policy (`--usermap`/`--groupmap`/`--chown`/`--numeric-ids`/`--copy-as`) is also given. A non-root receiver given `--super` logs exactly one warning at activation and each confined attempt is then refused by the kernel and skipped per entry (never aborts); `--no-super` suppresses ownership, char/block `mknod`, `--write-devices` and the fake-super owner replay, while unprivileged FIFO creation is unaffected. Wire: one trailing `super_mode` int on the config frame (validated 0..2), sent **before** the `--copy-as` block (fixed order: super int, then copy-as presence int + ids); `PROTOCOL_VERSION` bumped **2.17.0 → 2.18.0**. **Documented divergence from rsync:** rsync's `--super` runs the receiver with elevated privilege; FastSync only permits a confined attempt and never elevates |
|
||||||
| `--fake-super` | Store/recover privileged attrs via xattrs | ✅ Implemented | Phase 7 Wave B: full record **and replay**. The receiver writes the source `uid:gid:mode:mtime_sec:mtime_nsec` into a reserved `user.fastsync.stat` xattr on each written file (best-effort, fd-relative, format unchanged), then immediately re-applies it via `fake_super_restore_fd`: `fchown` (only where privileged — a non-root EPERM/EACCES is skipped silently, matching FastSync's identity philosophy), `fchmod`, and `futimens`. The OWNER leg is additionally skipped unless an explicit ownership identity policy (`--numeric-ids`/`--usermap`/`--groupmap`/`--chown`/`--copy-as`) is active — `--fake-super` on its own only *records* the source owner and must not act as an un-gated chown primitive — when `--no-super` forbids super-user activities (even for root), or when an active `--copy-as` is authoritative, so the recorded source owner can never override a forced `--copy-as` owner; the xattr record is still stored/replayed for a later privileged restore and mode/mtime still apply, so unprivileged `--fake-super` keeps working. The restored mode goes through the same sanitization as the normal metadata path (group/other write bits are never granted, so a recorded 0666 restores as 0644), so fake-super replay can never grant group/other-write that plain `--preserve` would refuse. Absence or a malformed record is a silent no-op, never fatal. The recording format diverges from rsync's `user.rsync.%stat%`; no cross-tool conversion is attempted. Implies metadata transmission so the source uid/gid/mode/mtime are available. Both it and `-X`/`-A` are incompatible with `-s` (chunk serialization), rejected up front |
|
| `--fake-super` | Store/recover privileged attrs via xattrs | ✅ Implemented | Phase 7 Wave B: full record **and replay**. The receiver writes the source `uid:gid:mode:mtime_sec:mtime_nsec` into a reserved `user.fastsync.stat` xattr on each written file (best-effort, fd-relative, format unchanged), then immediately re-applies it via `fake_super_restore_fd`: `fchown` (only where privileged — a non-root EPERM/EACCES is skipped silently, matching FastSync's identity philosophy), `fchmod`, and `futimens`. The OWNER leg is additionally skipped unless an explicit ownership identity policy (`--numeric-ids`/`--usermap`/`--groupmap`/`--chown`/`--copy-as`) is active — `--fake-super` on its own only *records* the source owner and must not act as an un-gated chown primitive — when `--no-super` forbids super-user activities (even for root), or when an active `--copy-as` is authoritative, so the recorded source owner can never override a forced `--copy-as` owner; the xattr record is still stored/replayed for a later privileged restore and mode/mtime still apply, so unprivileged `--fake-super` keeps working. The restored mode goes through the same sanitization as the normal metadata path (group/other write bits are never granted, so a recorded 0666 restores as 0644), so fake-super replay can never grant group/other-write that plain `--preserve` would refuse. Absence or a malformed record is a silent no-op, never fatal. The recording format diverges from rsync's `user.rsync.%stat%`; no cross-tool conversion is attempted. Implies metadata transmission so the source uid/gid/mode/mtime are available. Both it and `-X`/`-A` are incompatible with `-s` (chunk serialization), rejected up front |
|
||||||
| `--open-noatime` | Avoid changing access time when opening files | ✅ Implemented | Sender-side policy: the sender opens source files with `O_NOATIME` (Linux) when reading them for transfer, so the open/read does NOT bump the source's on-disk access time. Degrades safely when `O_NOATIME` is unavailable (not defined) or refused (`EPERM`, since it needs `CAP_FOWNER` or file ownership): the code falls back to a normal open, so the data always transfers — only the atime-bump is skipped. It does not itself capture/preserve atime; it only avoids modifying it. **Client-only, never crosses the wire.** Exposed as `file_open_for_read()` and applied to both the buffered data path and the sendfile path |
|
| `--open-noatime` | Avoid changing access time when opening files | ✅ Implemented | Sender-side policy: the sender opens source files with `O_NOATIME` (Linux) when reading them for transfer, so the open/read does NOT bump the source's on-disk access time. Degrades safely when `O_NOATIME` is unavailable (not defined) or refused (`EPERM`, since it needs `CAP_FOWNER` or file ownership): the code falls back to a normal open, so the data always transfers — only the atime-bump is skipped. It does not itself capture/preserve atime; it only avoids modifying it. **Client-only, never crosses the wire.** Exposed as `file_open_for_read()` and applied to both the buffered data path and the sendfile path |
|
||||||
| `--numeric-ids` | Do not map uid/gid by name | ✅ Implemented | Ownership is applied through FastSync's opt-in identity path (see the Phase-4 identity notes below). `--numeric-ids` is a mapping-policy modifier: when applying ownership it uses the transmitted numeric uid/gid directly, skipping the name lookup. Without an ownership-affecting option it is inert (FastSync only applies ownership when the user opts in). It does not need `-M` to be parsed, but ownership is only applied when metadata (hence the source uid/gid) is actually transmitted (see the notes) |
|
| `--numeric-ids` | Do not map uid/gid by name | ✅ Implemented | Ownership is applied through FastSync's opt-in identity path (see the Phase-4 identity notes below). `--numeric-ids` is a mapping-policy modifier: when applying ownership it uses the transmitted numeric uid/gid directly, skipping the name lookup. Without an ownership-affecting option it is inert (FastSync only applies ownership when the user opts in). It does not need `-M` to be parsed, but ownership is only applied when metadata (hence the source uid/gid) is actually transmitted (see the notes) |
|
||||||
| `--usermap=STRING` | Map usernames | ✅ Implemented | Opt-in ownership application. rsync subset implemented: comma-separated `FROM:TO` rules evaluated in order, first match wins; `FROM`/`TO` are group/user names (resolved on the SOURCE machine at parse time), `*` (FROM matches any id / TO = the receiving process's current euid), and an `@N` or bare `N` numeric id. Rules are carried over the wire as resolved numeric id pairs; the receiver applies a matching rule (else falls back to `--chown`, `--numeric-ids`, then a best-effort name lookup) via an fd-relative `fchown`. Malformed/unresolvable specs are rejected with a clear error, never a silent no-op. Implies metadata preservation so the source uid/gid travel. Only effective when the receiver can actually change ownership (root or membership); otherwise it warns and continues |
|
| `--usermap=STRING` | Map usernames | ✅ Implemented | Opt-in ownership application. rsync subset implemented: comma-separated `FROM:TO` rules evaluated in order, first match wins; `FROM`/`TO` are group/user names (resolved on the SOURCE machine at parse time), `*` (FROM matches any id / TO = the receiving process's current euid), and an `@N` or bare `N` numeric id. Rules are carried over the wire as resolved numeric id pairs; the receiver applies a matching rule (else falls back to `--chown`, `--numeric-ids`, then a best-effort name lookup) via an fd-relative `fchown`. Malformed/unresolvable specs are rejected with a clear error, never a silent no-op. Implies metadata preservation so the source uid/gid travel. Only effective when the receiver can actually change ownership (root or membership); otherwise it warns and continues |
|
||||||
| `--groupmap=STRING` | Map group names | ✅ Implemented | Same rsync subset and semantics as `--usermap` but for the group (gid) side and the group databases. See the Phase-4 identity notes |
|
| `--groupmap=STRING` | Map group names | ✅ Implemented | Same rsync subset and semantics as `--usermap` but for the group (gid) side and the group databases. See the Phase-4 identity notes |
|
||||||
| `--chown=USER:GROUP` | Map owner and group | ✅ Implemented | Opt-in ownership override applied receiver-side. Forms: `USER:GROUP`, `USER` (owner only), `:GROUP` (group only); a `*` for USER/GROUP means the current/root user or group as appropriate; an `@N`/bare `N` numeric id is accepted. A `:` inside a name may be escaped as `\:`. Equivalent to a trailing `*:*` usermap+groupmap rule (so an explicit `--usermap`/`--groupmap` match wins). Malformed or unresolvable specs are clear parse errors. Implies metadata preservation. Only effective when the receiver has permission to chown; otherwise it warns and continues (rsync parity) |
|
| `--chown=USER:GROUP` | Map owner and group | ✅ Implemented | Opt-in ownership override applied receiver-side. Forms: `USER:GROUP`, `USER` (owner only), `:GROUP` (group only); a `*` for USER/GROUP means the current/root user or group as appropriate; an `@N`/bare `N` numeric id is accepted. A `:` inside a name may be escaped as `\:`. Equivalent to a trailing `*:*` usermap+groupmap rule (so an explicit `--usermap`/`--groupmap` match wins). Malformed or unresolvable specs are clear parse errors. Implies metadata preservation. Only effective when the receiver has permission to chown; otherwise it warns and continues (rsync parity) |
|
||||||
| `--copy-as=USER[:GROUP]` | Perform the copy as another user/group | ✅ Implemented | Safe-subset implementation, an explicit divergence from rsync's **real identity switching**. rsync makes the receiving process actually assume USER/GROUP (setuid/setgid); FastSync's receiver is multithreaded, so a real credential drop would be unsafe and is never attempted — FastSync never calls `setuid`/`seteuid`/`setgid`. Instead the receiver FORCES the ownership of every entry it writes to `copy_as_uid`/`copy_as_gid` through the existing confined, fd-relative identity path (the same `fchown`/`fchownat` mechanism as `--chown`/`--usermap`/`--groupmap`; symlinks use `fchownat(..., AT_SYMLINK_NOFOLLOW)`, and directories — including intermediate parents created implicitly while writing a nested file — and char/block/FIFO nodes are owned no-follow too, so a directory never keeps the receiver's owner while its children get the target owner), with `--copy-as` at the **highest priority** — it beats usermap/groupmap/`--chown`/`--numeric-ids` and the best-effort name lookup. This REQUIRES a privileged (root) receiver: an unprivileged receiver REFUSES the whole transfer up front at the config handshake (`server_module_gate`, running inside `config_receive_with_validate` before the `STATUS_OK` ack) with a clear error and no file data exchanged — never a silent wrong-ownership result. A server running with an operator `--no-super` veto also refuses it, and a **daemon** refuses `--copy-as`, like every other client-chosen-ownership request (`--numeric-ids`/`--chown`/`--usermap`/`--groupmap`/`--fake-super`/explicit `--super`), unless the selected module opts in with `client owner = yes`; without that per-module opt-in a daemon must not honor an arbitrary client-selected owner (the standalone listener and SSH `--stdio` server keep honoring these for their single operator-authorized root). `--fake-super` interaction: `--copy-as` is authoritative, so the recorded source owner is never replayed over the forced target owner. If the ownership apply still fails with EPERM/EACCES (capability-restricted root, root-squash, read-only mount) the failure is logged at ERROR and the **entry is reported as failed** rather than written with the wrong owner, which fails the transfer (fail-fast) so overall success is never reported with the wrong owner. USER is resolved on the client against the user database (a name, an `@N`/bare `N` numeric id, or `*` meaning the client's current euid); when `:GROUP` is present it is resolved against the group database (`*` meaning the client's egid). **Group-default rule:** when the group is omitted FastSync uses the user's primary gid (`getpwuid(uid)->pw_gid`); a numeric id with no local passwd entry has no primary gid to look up, so `gid` falls back to `uid` (documented divergence). Malformed/empty/unresolvable specs are clear parse errors, never a silent no-op. Never elevates privileges and never bypasses the confined receive root. Implies metadata preservation (the source uid/gid must be transmitted). Wire: a new trailing config-frame block **sent after** the `--super` int (presence int, then the two int32 ids, both validated `>= 0` on receive; the ids are also rejected if they do not fit int32 at CLI parse time); `PROTOCOL_VERSION` bumped **2.17.0 → 2.18.0** |
|
| `--copy-as=USER[:GROUP]` | Perform the copy as another user/group | ✅ Implemented | Safe-subset implementation, an explicit divergence from rsync's **real identity switching**. rsync makes the receiving process actually assume USER/GROUP (setuid/setgid); FastSync's receiver is multithreaded, so a real credential drop would be unsafe and is never attempted — FastSync never calls `setuid`/`seteuid`/`setgid`. Instead the receiver FORCES the ownership of every entry it writes to `copy_as_uid`/`copy_as_gid` through the existing confined, fd-relative identity path (the same `fchown`/`fchownat` mechanism as `--chown`/`--usermap`/`--groupmap`; symlinks use `fchownat(..., AT_SYMLINK_NOFOLLOW)`, and directories — including intermediate parents created implicitly while writing a nested file — and char/block/FIFO nodes are owned no-follow too, so a directory never keeps the receiver's owner while its children get the target owner), with `--copy-as` at the **highest priority** — it beats usermap/groupmap/`--chown`/`--numeric-ids` and the best-effort name lookup. This REQUIRES a privileged (root) receiver: an unprivileged receiver REFUSES the whole transfer up front at the config handshake (`server_module_gate`, running inside `config_receive_with_validate` before the `STATUS_OK` ack) with a clear error and no file data exchanged — never a silent wrong-ownership result. A server running with an operator `--no-super` veto also refuses it; a privileged (root) standalone TCP listener refuses it by default too and only honors it after the operator passes `--allow-super` (the flag is rejected with `--stdio`, where the client-composed remote argv could otherwise defeat the default; a forced command is required if the default must hold), and a **daemon** refuses `--copy-as`, like every other client-chosen-ownership request (`--numeric-ids`/`--chown`/`--usermap`/`--groupmap`/`--fake-super`/explicit `--super`), unless the selected module opts in with `client owner = yes`; without that per-module opt-in a daemon must not honor an arbitrary client-selected owner (a root standalone listener honors these for its single operator-authorized root only when started with `--allow-super`). `--fake-super` interaction: `--copy-as` is authoritative, so the recorded source owner is never replayed over the forced target owner. If the ownership apply still fails with EPERM/EACCES (capability-restricted root, root-squash, read-only mount) the failure is logged at ERROR and the **entry is reported as failed** rather than written with the wrong owner, which fails the transfer (fail-fast) so overall success is never reported with the wrong owner. USER is resolved on the client against the user database (a name, an `@N`/bare `N` numeric id, or `*` meaning the client's current euid); when `:GROUP` is present it is resolved against the group database (`*` meaning the client's egid). **Group-default rule:** when the group is omitted FastSync uses the user's primary gid (`getpwuid(uid)->pw_gid`); a numeric id with no local passwd entry has no primary gid to look up, so `gid` falls back to `uid` (documented divergence). Malformed/empty/unresolvable specs are clear parse errors, never a silent no-op. Never elevates privileges and never bypasses the confined receive root. Implies metadata preservation (the source uid/gid must be transmitted). Wire: a new trailing config-frame block **sent after** the `--super` int (presence int, then the two int32 ids, both validated `>= 0` on receive; the ids are also rejected if they do not fit int32 at CLI parse time); `PROTOCOL_VERSION` bumped **2.17.0 → 2.18.0** |
|
||||||
|
|
||||||
**Phase-4 metadata-time notes:** `-U/--atimes`, `-N/--crtimes`,
|
**Phase-4 metadata-time notes:** `-U/--atimes`, `-N/--crtimes`,
|
||||||
`-O/--omit-dir-times`, `-J/--omit-link-times`, and `--open-noatime` are new.
|
`-O/--omit-dir-times`, `-J/--omit-link-times`, and `--open-noatime` are new.
|
||||||
@@ -612,7 +612,7 @@ now transmits targets (the prior behavior was broken/partial); its status moved
|
|||||||
|------|-------------------|-----------------|-------|
|
|------|-------------------|-----------------|-------|
|
||||||
| `-e`, `--rsh=COMMAND` | Remote shell to use | ✅ Implemented | `-e`/`--rsh` (and `--rsh=COMMAND`) select the remote-shell program used to build the SSH child argv, overriding the default `ssh`. The command is whitespace-split into the leading argv words so rsync's `-e "ssh -p 2222"` works; the standard `-o` family, an optional `-p` port, `user@host` and the quoted remote command (`fastsync-server --stdio`) follow. Stored in the `rsh_command` config field. **Client-only, never crosses the wire** (it is a launch concern, not a handshake property) |
|
| `-e`, `--rsh=COMMAND` | Remote shell to use | ✅ Implemented | `-e`/`--rsh` (and `--rsh=COMMAND`) select the remote-shell program used to build the SSH child argv, overriding the default `ssh`. The command is whitespace-split into the leading argv words so rsync's `-e "ssh -p 2222"` works; the standard `-o` family, an optional `-p` port, `user@host` and the quoted remote command (`fastsync-server --stdio`) follow. Stored in the `rsh_command` config field. **Client-only, never crosses the wire** (it is a launch concern, not a handshake property) |
|
||||||
| `--rsync-path=PROGRAM` | rsync binary on remote | ✅ Implemented | Alias for `--fastsync-server-path`: both write the `fastsync_server_path` config field used as the remote-side server program (always quoted as one remote-shell word), which CROSSES the wire as before. Kept separate from `--rsh`, which names the local connecting program |
|
| `--rsync-path=PROGRAM` | rsync binary on remote | ✅ Implemented | Alias for `--fastsync-server-path`: both write the `fastsync_server_path` config field used as the remote-side server program (always quoted as one remote-shell word), which CROSSES the wire as before. Kept separate from `--rsh`, which names the local connecting program |
|
||||||
| `--port=PORT` | Alternate daemon port | ✅ Implemented | rsync's daemon-port flag maps to the client-side `server_port` config field: a client connects to a TCP/TLS server (incl. `host::module/path` daemon destinations) with `--server-port`, and the `fastsync-server --daemon` listener's port is taken from its config's `port` key (default 873) or overridden by `--dparam port=` / `-p` |
|
| `--port=PORT`, `--port PORT` | Alternate daemon port | ✅ Implemented | rsync's daemon-port flag is an alias for `--server-port`: both spellings (and `--server-port=PORT`) map to the client-side `server_port` config field. The client connects to a TCP/TLS server (incl. `host::module/path` daemon destinations) on that port, and the `fastsync-server --daemon` listener's port is taken from its config's `port` key (default 873) or overridden by `--dparam port=` / `-p` |
|
||||||
| `--sockopts=OPTIONS` | Custom TCP options | ✅ Implemented | Comma-separated allowlist of `OPT=VAL` applied via `setsockopt` after `socket()` before `connect()`/`bind()`. Only `TCP_NODELAY`, `SO_KEEPALIVE`, `SO_REUSEADDR` (0/1) and `SO_RCVBUF`/`SO_SNDBUF` (byte count) are accepted; an unknown option name or a bad value is rejected up front, never silently ignored. A value is required for every option (`OPT=VAL`; a bare name is an error). Applied to the outgoing TCP and TLS client socket; absent by default. `SockOptEntry`/`sockopts` config fields. Local socket concern: never crosses the wire |
|
| `--sockopts=OPTIONS` | Custom TCP options | ✅ Implemented | Comma-separated allowlist of `OPT=VAL` applied via `setsockopt` after `socket()` before `connect()`/`bind()`. Only `TCP_NODELAY`, `SO_KEEPALIVE`, `SO_REUSEADDR` (0/1) and `SO_RCVBUF`/`SO_SNDBUF` (byte count) are accepted; an unknown option name or a bad value is rejected up front, never silently ignored. A value is required for every option (`OPT=VAL`; a bare name is an error). Applied to the outgoing TCP and TLS client socket; absent by default. `SockOptEntry`/`sockopts` config fields. Local socket concern: never crosses the wire |
|
||||||
| `--blocking-io` | Use blocking I/O for remote shell | ✅ Implemented | With `--blocking-io` the SSH-transport socketpair socket is left without `SO_RCVTIMEO`/`SO_SNDTIMEO`, so the transfer blocks naturally; by default it gets the same read/write timeout as the TCP transport (see `--timeout`). `blocking_io` config bool. **Client-only, never crosses the wire** |
|
| `--blocking-io` | Use blocking I/O for remote shell | ✅ Implemented | With `--blocking-io` the SSH-transport socketpair socket is left without `SO_RCVTIMEO`/`SO_SNDTIMEO`, so the transfer blocks naturally; by default it gets the same read/write timeout as the TCP transport (see `--timeout`). `blocking_io` config bool. **Client-only, never crosses the wire** |
|
||||||
| `--outbuf=N\|L\|B` | Set output buffering | ✅ Implemented | `N` (none/unbuffered) → `_IONBF`, `L` (line) → `_IOLBF`, `B` (block, the default) → `_IOFBF` via `setvbuf` on stdout and stderr. Garbage values are rejected. `outbuf` config field (`OutbufMode`). **Client-only, never crosses the wire** |
|
| `--outbuf=N\|L\|B` | Set output buffering | ✅ Implemented | `N` (none/unbuffered) → `_IONBF`, `L` (line) → `_IOLBF`, `B` (block, the default) → `_IOFBF` via `setvbuf` on stdout and stderr. Garbage values are rejected. `outbuf` config field (`OutbufMode`). **Client-only, never crosses the wire** |
|
||||||
@@ -627,7 +627,7 @@ now transmits targets (the prior behavior was broken/partial); its status moved
|
|||||||
|------|-------------------|-----------------|-------|
|
|------|-------------------|-----------------|-------|
|
||||||
| `--daemon` | Run as rsync daemon | ✅ Implemented | Wave A: a real persistent listener. `fastsync-server --daemon --config FILE` (plus `--no-detach` to stay foreground; without it the listener detaches to the background after binding) reads a FastSync-native module config file and serves each connection confined to the requested module's `path` root (never a client-chosen root; every client-chosen-ownership/super-user request (`--numeric-ids`/`--chown`/`--usermap`/`--groupmap`/`--fake-super`/`--copy-as`/explicit `--super`) is refused unless the module opts in with `client owner = yes`, and the operator `--no-super` veto is honored). TCP/TLS via the existing `--tls` stack; plaintext still requires `--allow-unauthenticated` (same secure default as the standalone server). Client destinations use rsync's `host::module/path` form. Wire/protocol: the config frame gained a trailing daemon-module string and `PROTOCOL_VERSION` was bumped **2.14.0 → 2.15.0** (see the Daemon Mode notes below). Daemon mode is built in FastSync's own protocol/config grammar, not rsync's SMB/daemon option encoding |
|
| `--daemon` | Run as rsync daemon | ✅ Implemented | Wave A: a real persistent listener. `fastsync-server --daemon --config FILE` (plus `--no-detach` to stay foreground; without it the listener detaches to the background after binding) reads a FastSync-native module config file and serves each connection confined to the requested module's `path` root (never a client-chosen root; every client-chosen-ownership/super-user request (`--numeric-ids`/`--chown`/`--usermap`/`--groupmap`/`--fake-super`/`--copy-as`/explicit `--super`) is refused unless the module opts in with `client owner = yes`, and the operator `--no-super` veto is honored). TCP/TLS via the existing `--tls` stack; plaintext still requires `--allow-unauthenticated` (same secure default as the standalone server). Client destinations use rsync's `host::module/path` form. Wire/protocol: the config frame gained a trailing daemon-module string and `PROTOCOL_VERSION` was bumped **2.14.0 → 2.15.0** (see the Daemon Mode notes below). Daemon mode is built in FastSync's own protocol/config grammar, not rsync's SMB/daemon option encoding |
|
||||||
| `--config=FILE` | Alternate rsyncd.conf file | ✅ Implemented | Wave A: selects the daemon config file. Default when omitted (in `--daemon` mode): `~/.config/fastsync/fastsyncd.conf` if it exists, else `/etc/fastsyncd.conf`. The grammar is FastSync-native (documented in the Daemon Mode notes below) and strictly rejects unknown keys so a typo can never silently change what a module serves; requires `--daemon` |
|
| `--config=FILE` | Alternate rsyncd.conf file | ✅ Implemented | Wave A: selects the daemon config file. Default when omitted (in `--daemon` mode): `~/.config/fastsync/fastsyncd.conf` if it exists, else `/etc/fastsyncd.conf`. The grammar is FastSync-native (documented in the Daemon Mode notes below) and strictly rejects unknown keys so a typo can never silently change what a module serves; requires `--daemon` |
|
||||||
| `--dparam=OVERRIDE` | Override global daemon config | ✅ Implemented | Wave A: overrides one global scalar from the command line (`--dparam port=8734` and `--dparam=KEY=VALUE` both work). Limited to the global scalar keys the grammar defines (`port`, `motd file`, `address`); keys are case-insensitive and unknown keys/invalid values are rejected. Requires `--daemon` |
|
| `--dparam=OVERRIDE` | Override global daemon config | ✅ Implemented | Wave A: overrides one global scalar from the command line (`--dparam port=8734` and `--dparam=KEY=VALUE` both work). Limited to the global keys the grammar defines (`port`, `motd file`, `address`, `max connections`, `max connections per host`, `auth failure delay`, `auth lockout threshold`, `auth lockout duration`, `hosts allow`, `hosts deny`); keys are case-insensitive and unknown keys/invalid values are rejected. Requires `--daemon` |
|
||||||
| `--no-detach` | Don't detach from parent | ✅ Implemented | Wave A: with `--daemon`, keeps the listener in the foreground (what integration tests use). Without it the daemonizes (fork/setsid, stdio redirected to /dev/null) after the listening socket is bound. Requires `--daemon` |
|
| `--no-detach` | Don't detach from parent | ✅ Implemented | Wave A: with `--daemon`, keeps the listener in the foreground (what integration tests use). Without it the daemonizes (fork/setsid, stdio redirected to /dev/null) after the listening socket is bound. Requires `--daemon` |
|
||||||
| `--password-file=FILE` | Read daemon password from file | ✅ Implemented | A7 daemon auth. Client: `--password-file` supplies `user:password` for a `host::module/path` destination (the username is taken from this file, so `user@host::module` stays rejected); the literal password is held client-side only for the SCRAM handshake and wiped at teardown. Server (`fastsync-server --daemon --password-file FILE`): the salted-PBKDF2 verifier store that modules with `auth users` are verified against. **Neither the password nor any replayable bearer value crosses the wire or is stored server-side** — the store holds a per-user salt plus derived keys, and the daemon proves the secret with a per-connection nonce challenge. The file must be private to its owner: both the client and server verify the exact inode they read (open-then-`fstat`, so the check cannot be raced) and refuse a `--password-file`/`--early-input` that is not owned by the current user or grants any group/other permission bit (mode 0600), mirroring the TLS private-key check. A process-substitution pipe (`--early-input <(vault ...)`) is still accepted when it satisfies those checks. See the Daemon Mode notes below for the file formats and the plaintext/TLS caveat |
|
| `--password-file=FILE` | Read daemon password from file | ✅ Implemented | A7 daemon auth. Client: `--password-file` supplies `user:password` for a `host::module/path` destination (the username is taken from this file, so `user@host::module` stays rejected); the literal password is held client-side only for the SCRAM handshake and wiped at teardown. Server (`fastsync-server --daemon --password-file FILE`): the salted-PBKDF2 verifier store that modules with `auth users` are verified against. **Neither the password nor any replayable bearer value crosses the wire or is stored server-side** — the store holds a per-user salt plus derived keys, and the daemon proves the secret with a per-connection nonce challenge. The file must be private to its owner: both the client and server verify the exact inode they read (open-then-`fstat`, so the check cannot be raced) and refuse a `--password-file`/`--early-input` that is not owned by the current user or grants any group/other permission bit (mode 0600), mirroring the TLS private-key check. A process-substitution pipe (`--early-input <(vault ...)`) is still accepted when it satisfies those checks. See the Daemon Mode notes below for the file formats and the plaintext/TLS caveat |
|
||||||
| `--early-input=FILE` | Use FILE for daemon early exec | ✅ Implemented | Server-only (requires `--daemon`): a second credential-store file, same new-format grammar as `--password-file`, read before the listener accepts connections (a secrets-manager / process-substitution source). Its entries layer over `--password-file`: byte-identical verifiers dedupe, a conflicting verifier for the same user is a startup error. A daemon whose modules declare `auth users` must be given at least one of the two, or it refuses to start (fail closed) |
|
| `--early-input=FILE` | Use FILE for daemon early exec | ✅ Implemented | Server-only (requires `--daemon`): a second credential-store file, same new-format grammar as `--password-file`, read before the listener accepts connections (a secrets-manager / process-substitution source). Its entries layer over `--password-file`: byte-identical verifiers dedupe, a conflicting verifier for the same user is a startup error. A daemon whose modules declare `auth users` must be given at least one of the two, or it refuses to start (fail closed) |
|
||||||
@@ -635,10 +635,13 @@ now transmits targets (the prior behavior was broken/partial); its status moved
|
|||||||
|
|
||||||
**Daemon Mode notes (Wave A protocol 2.15.0; A7 auth protocol 2.19.0; MOTD no bump):** FastSync daemon mode is supported in FastSync's own protocol/config grammar, not rsync's SMB/daemon option encoding.
|
**Daemon Mode notes (Wave A protocol 2.15.0; A7 auth protocol 2.19.0; MOTD no bump):** FastSync daemon mode is supported in FastSync's own protocol/config grammar, not rsync's SMB/daemon option encoding.
|
||||||
|
|
||||||
- **Config grammar** (`fastsyncd.conf`): line-based; an implicit global section first, then `[module]` sections. Keys are case-insensitive, values are trimmed and may be wrapped in one layer of double quotes (`path = "/srv/my dir"`). `#` and `;` at the start of a line (after leading whitespace) are full-line comments; inline comments and `\` continuations are not supported. Lines are bounded (4096 chars). Global keys: `port` (default 873), `motd file` (the daemon sends its bounded, escaped content to a client after the module gate/auth accepts, unless the client passes `--no-motd`), `address` (optional bind address). Module keys: `path` (required; the daemon-side authorized root for that module), `read only` (yes/no/true/false/1/0, default no), `client owner` (yes/no/true/false/1/0, default no; opts the module into client-chosen ownership — see below), `auth users` (comma list). **Unknown keys and malformed lines are parse-and-reject errors** (never silently ignored), so a typo cannot change what a module serves.
|
- **Config grammar** (`fastsyncd.conf`): line-based; an implicit global section first, then `[module]` sections. Keys are case-insensitive, values are trimmed and may be wrapped in one layer of double quotes (`path = "/srv/my dir"`). `#` and `;` at the start of a line (after leading whitespace) are full-line comments; inline comments and `\` continuations are not supported. Lines are bounded (4096 chars), and at most 256 `[module]` sections are accepted. Global keys: `port` (default 873), `motd file` (the daemon sends its bounded, escaped content to a client after the module gate/auth accepts, unless the client passes `--no-motd`), `address` (optional bind address), `max connections` (positive integer cap on concurrent connections, default 100; 0/negative/garbage is a parse error), `max connections per host` (concurrent-connection cap per source IP, default 0 = unlimited), `auth failure delay` (milliseconds to sleep after a failed authentication, default 500; 0 disables, capped at 5000), `auth lockout threshold` (failed authentications from one source before lockout, default 10; 0 disables), `auth lockout duration` (seconds a locked-out source is refused, default 300), `hosts allow` and `hosts deny` (comma- and/or whitespace-separated host access patterns — see the host access control note below). Module keys: `path` (required; the daemon-side authorized root for that module), `read only` (yes/no/true/false/1/0, default no), `client owner` (yes/no/true/false/1/0, default no; opts the module into client-chosen ownership — see below), `auth users` (comma list), `max connections` (optional per-module cap, 0 = unlimited; enforced across all connection children), `hosts allow`/`hosts deny` (per-module host access lists). **Unknown keys and malformed lines are parse-and-reject errors** (never silently ignored), so a typo cannot change what a module serves.
|
||||||
|
- **Host access control (`hosts allow`/`hosts deny`):** both keys accept a comma- and/or whitespace-separated list of patterns and may appear globally and/or per module (multiple config-file lines append; a `--dparam` override replaces). Supported patterns are `*` (match all), an IPv4 or IPv6 literal (`10.0.0.1`, `2001:db8::1`), and an IPv4/IPv6 CIDR (`10.0.0.0/8`, `2001:db8::/32`). Hostname patterns are **not** supported: because the peer is always a numeric address and no reverse DNS is performed, a hostname/glob pattern would silently never match, so it is rejected at load time (fail-closed) instead of being accepted as a dead rule. An IPv4 peer on a dual-stack IPv6 listener is normalized from its `::ffff:a.b.c.d` form so IPv4 patterns match it. rsync-like semantics: a matching `hosts deny` rejects; if any `hosts allow` entries exist, a peer matching none of them is rejected; deny takes precedence over allow. The daemon enforces the global list first, then the selected module's list, **before authentication** in `server_module_gate`, with an audit log line naming the peer, the module and the outcome. The numeric peer address is obtained with `getpeername`+`inet_ntop` (`utils_fd_peer_ip`, handling both address families); when it cannot be obtained a module with any ACL fails closed (refused), while an ACL-free module continues and logs at debug. A malformed pattern (e.g. an out-of-range CIDR prefix) is a parse error at load time.
|
||||||
|
- **Connection caps, shared registry and auth lockout:** the global `max connections` key (default 100) is plumbed into the listener (`transport_tcp.c`), which rejects a connection once the accept-loop parent's active-child count reaches it; the IPv4/IPv6 peer is logged for every accepted connection. Because the listener forks one child per connection, the per-module `max connections` cap, the global `max connections per host` cap, and the auth-failure counter live in a fixed-size registry carved from an anonymous shared mapping (`daemon_limits.c`, `mmap(MAP_SHARED|MAP_ANONYMOUS)`) created by the parent before the accept loop, so every forked child shares the same counters (C11 atomics only — never a pthread lock, which can deadlock in a forked child). The parent reserves a registry slot per accepted connection and the child records the selected module and source IP once known; the parent's `SIGCHLD` handler reclaims the slot when the child dies (including `SIGKILL`) and re-derives the per-module and per-source occupancy counts from the surviving REGISTERED slots, so a child killed mid-registration cannot leak a count. The per-source table has a bounded lifetime: an entry with no live connection is reclaimed after its lockout expires or it has been idle (300 s); if the table is genuinely full the per-source cap/lockout fails open for new sources (per-module cap and ACLs still apply) with a rate-limited warning. The per-module cap (0 = unlimited) is enforced after the module lookup and before auth; per-source identity reuses the normalized numeric peer address (`utils_fd_peer_ip`, IPv4-mapped IPv6 collapsed to IPv4), and a trusted loopback peer (127.0.0.0/8 / `::1`, `utils_fd_peer_is_local`) is exempt from the per-source cap and the auth lockout because all local clients share one address (the per-module/global caps still apply). Clients behind a shared NAT/proxy address likewise share one per-source budget and lockout counter. A failed authentication increments the shared per-source failure count and, once `auth lockout threshold` (default 10; 0 disables) is reached, the source is refused for `auth lockout duration` seconds (default 300) before any challenge is sent, even when the next attempt is handled by a different forked child; a successful authentication clears the counter. On a failed authentication the per-connection child still sleeps the global `auth failure delay` (default 500 ms, 0 disables, capped at 5000) via `nanosleep`, rate-limiting online guessing without delaying a success. A missing registry (allocation failure) degrades to the global cap and host ACLs rather than refusing to start.
|
||||||
- **Module selection & confinement:** the client requests a module with an rsync-style `host::module[/path]` destination. The module name crosses the wire as a trailing string on the config frame (bumping `PROTOCOL_VERSION` 2.14.0 → 2.15.0; the bump is required because the config-frame layout changed and the strict same-version handshake is what prevents a peer from desynchronizing on the new trailing field). The daemon looks the module up in ITS OWN config and uses the module's `path` as the authorized root through the exact same `configure_authorization` confinement the standalone server applies to `--destination-root` (`file_open_secure_parent`, `has_path_traversal`, `path_is_within`); the client never supplies the root, every client-chosen-ownership/super-user request is refused unless the module declares `client owner = yes` (the daemon's per-module opt-in, see below), and the operator `--no-super` veto forces super-user activities off for every daemon connection. The client's `/path` part is relative inside the module and is rejected if absolute or if it contains `..`. Unknown modules are refused before any data moves (the run fails cleanly at the config handshake). An absolute destination and a module request against a non-daemon server are also refused.
|
- **Module selection & confinement:** the client requests a module with an rsync-style `host::module[/path]` destination. The module name crosses the wire as a trailing string on the config frame (bumping `PROTOCOL_VERSION` 2.14.0 → 2.15.0; the bump is required because the config-frame layout changed and the strict same-version handshake is what prevents a peer from desynchronizing on the new trailing field). The daemon looks the module up in ITS OWN config and uses the module's `path` as the authorized root through the exact same `configure_authorization` confinement the standalone server applies to `--destination-root` (`file_open_secure_parent`, `has_path_traversal`, `path_is_within`); the client never supplies the root, every client-chosen-ownership/super-user request is refused unless the module declares `client owner = yes` (the daemon's per-module opt-in, see below), and the operator `--no-super` veto forces super-user activities off for every daemon connection. The client's `/path` part is relative inside the module and is rejected if absolute or if it contains `..`. Unknown modules are refused before any data moves (the run fails cleanly at the config handshake). An absolute destination and a module request against a non-daemon server are also refused.
|
||||||
- **`client owner` (client-chosen-ownership opt-in):** by default a daemon module refuses every request that would let the client pick an owner or ask for super-user activities — `--numeric-ids`, `--chown`, `--usermap`/`--groupmap`, `--fake-super`, `--copy-as`, and an explicit `--super` — at the config handshake (before `STATUS_OK`), because a daemon has no per-module opt-in for client-chosen ownership and any anonymous client could otherwise force arbitrary owner ids inside the module root. `client owner = yes` opts a single module in, allowing those requests within that module's root (the standalone listener and the SSH `--stdio` server always honor them for their single operator-authorized root). Without the opt-in the daemon also forces super-user **device** activity off for that connection — char/block device-node creation (`--devices`) and `--write-devices` — even under the default `AUTO` mode, so a non-opted module can never be made to `mknod` or write a raw device; those entries are skipped (not refused) so an ordinary `-a` push still succeeds without device nodes. The opt-in does **not** lift the privilege requirement: `--copy-as` still needs a root receiver, and the operator `--no-super` veto still forces super-user activities off for every connection. The daemon logs a prominent startup warning for each `client owner = yes` module so the operator's deliberate choice is visible.
|
- **`client owner` (client-chosen-ownership opt-in):** by default a daemon module refuses every request that would let the client pick an owner or ask for super-user activities — `--numeric-ids`, `--chown`, `--usermap`/`--groupmap`, `--fake-super`, `--copy-as`, and an explicit `--super` — at the config handshake (before `STATUS_OK`), because a daemon has no per-module opt-in for client-chosen ownership and any anonymous client could otherwise force arbitrary owner ids inside the module root. `client owner = yes` opts a single module in, allowing those requests within that module's root (a root standalone TCP listener honors them for its single operator-authorized root only when started with `--allow-super`; the flag is rejected with `--stdio`, whose client-composed remote argv must never opt back into super mode). Without the opt-in the daemon also forces super-user **device** activity off for that connection — char/block device-node creation (`--devices`) and `--write-devices` — even under the default `AUTO` mode, so a non-opted module can never be made to `mknod` or write a raw device; those entries are skipped (not refused) so an ordinary `-a` push still succeeds without device nodes. The opt-in does **not** lift the privilege requirement: `--copy-as` still needs a root receiver, and the operator `--no-super` veto still forces super-user activities off for every connection. The daemon logs a prominent startup warning for each `client owner = yes` module so the operator's deliberate choice is visible.
|
||||||
- **`read only` safe default:** every network transfer FastSync currently supports is a push that writes under the module root, so a `read only` module refuses the connection (clear server log "module is read only"; the client exits non-zero, nothing is transferred). A future pull/list operation can be opened up when it exists; the knob is already stored.
|
- **`read only` safe default:** every network transfer FastSync currently supports is a push that writes under the module root, so a `read only` module refuses the connection (clear server log "module is read only"; the client exits non-zero, nothing is transferred). A future pull/list operation can be opened up when it exists; the knob is already stored.
|
||||||
|
- **Direction — remote source / pull is intentionally unsupported:** FastSync is push-only. The first positional argument is always a **local** source directory and the second is the destination; only the destination is parsed for remote syntax (`user@host:path` SSH, `host::module[/path]` daemon). A remote source such as `fastsync user@host:src ./local` is deliberately **not** implemented: rsync has no pull flag (direction is positional), so supporting a remote source is an optional feature rather than a compatibility requirement, and it would require a protocol role reversal (server as sender, client as receiver) across both transports. FastSync documents this as an intentional limitation rather than a missing rsync option. <a id="direction"></a>
|
||||||
- **`auth users` (A7 SCRAM-SHA-256 authentication):** a module that declares `auth users` requires the client to present credentials. The config frame carries ONLY the username; the daemon answers an auth-required module with `STATUS_AUTH_CHALLENGE` (PBKDF2 iteration count, 16-byte salt, 32-byte server nonce), the client answers with `STATUS_AUTH_RESPONSE` (fresh 32-byte client nonce + a 32-byte ClientProof), and the daemon accepts only when the proof verifies **and** the username is **on the module's `auth users` list** and has a store entry, replying `STATUS_AUTH_OK` with a 32-byte ServerSignature the client verifies before proceeding. Verification is constant-time over fixed 32-byte keys (the compare runs even for a miss), username membership uses a constant-time full-length scan, and an unknown/off-list user still receives a challenge and runs the same math against a dummy verifier: a deterministic per-username salt (`HMAC-SHA256(store dummy key, username)`), the store-wide uniform iteration count and dummy keys. Re-probing the same unknown username therefore yields an identical salt and iteration count while a different username yields a different salt, so there is no user-enumeration or timing oracle. The daemon logs the username but **never the password, proof or keys**. A module WITHOUT `auth users` stays open (legitimate rsync configuration); credentials sent to such a module are ignored. Read-only is orthogonal: even a correctly authenticated push to a `read only` module is still refused (all FastSync network transfers write). Fail-closed policy: a daemon whose config declares `auth users` on any module refuses to start unless a credential store was given (`--password-file` and/or `--early-input`); a missing or empty store is never silently treated as "open". A failed handshake (missing credentials, unknown/off-list user, wrong proof or malformed data) yields a single generic `STATUS_AUTH_FAILED` and the daemon closes before any data moves. The dummy key is persisted in an owner-only `<store_path>.dummykey` sidecar (auto-created on first load, mode 0600) so the dummy salt stays stable across daemon restarts, closing the restart-gated enumeration channel. The sidecar is secret material and must be protected like the credential store (owner-only 0600, included with the store in backups and rotation). It must be preserved across restarts for that guarantee; if it cannot be created (a process-substitution/FIFO store path such as `/dev/fd/N`, a read-only filesystem, a missing directory, or a create/write/fsync/link/fchmod failure), the daemon logs a warning and uses a transient per-run key, so unknown-user challenges change across restarts and the cross-restart guarantee does not hold for that deployment. One residual is accepted: the store iteration count is observable pre-auth by design, since the miss path must match a hit. **Transport policy (hardening A7-3/S1):** an auth-required module accepts credentials only when either (a) the connection is an encrypted, verified TLS connection whose client certificate matches `--client-cn`, or (b) the connection is plaintext from a loopback TCP peer **and** the operator explicitly passed `--allow-unauthenticated`. A remote plaintext peer, and a loopback plaintext peer without that flag, are refused at the config gate before any challenge is sent; `--allow-unauthenticated` never permits remote plaintext auth (remote peers still require verified TLS). Daemon modules are a `--daemon`-only feature — the SSH `--stdio` path never loads a daemon config and is not an auth transport for them. Because the loopback allowance trusts whichever peer the kernel reports as `127.0.0.1`, it assumes nothing relays remote connections to the daemon: a local TCP forwarder or TLS-terminating proxy in front of an auth-module listener makes remote clients appear as loopback and bypasses the mutual-TLS identity check, so do not front an auth-module listener with such a relay.
|
- **`auth users` (A7 SCRAM-SHA-256 authentication):** a module that declares `auth users` requires the client to present credentials. The config frame carries ONLY the username; the daemon answers an auth-required module with `STATUS_AUTH_CHALLENGE` (PBKDF2 iteration count, 16-byte salt, 32-byte server nonce), the client answers with `STATUS_AUTH_RESPONSE` (fresh 32-byte client nonce + a 32-byte ClientProof), and the daemon accepts only when the proof verifies **and** the username is **on the module's `auth users` list** and has a store entry, replying `STATUS_AUTH_OK` with a 32-byte ServerSignature the client verifies before proceeding. Verification is constant-time over fixed 32-byte keys (the compare runs even for a miss), username membership uses a constant-time full-length scan, and an unknown/off-list user still receives a challenge and runs the same math against a dummy verifier: a deterministic per-username salt (`HMAC-SHA256(store dummy key, username)`), the store-wide uniform iteration count and dummy keys. Re-probing the same unknown username therefore yields an identical salt and iteration count while a different username yields a different salt, so there is no user-enumeration or timing oracle. The daemon logs the username but **never the password, proof or keys**. A module WITHOUT `auth users` stays open (legitimate rsync configuration); credentials sent to such a module are ignored. Read-only is orthogonal: even a correctly authenticated push to a `read only` module is still refused (all FastSync network transfers write). Fail-closed policy: a daemon whose config declares `auth users` on any module refuses to start unless a credential store was given (`--password-file` and/or `--early-input`); a missing or empty store is never silently treated as "open". A failed handshake (missing credentials, unknown/off-list user, wrong proof or malformed data) yields a single generic `STATUS_AUTH_FAILED` and the daemon closes before any data moves. The dummy key is persisted in an owner-only `<store_path>.dummykey` sidecar (auto-created on first load, mode 0600) so the dummy salt stays stable across daemon restarts, closing the restart-gated enumeration channel. The sidecar is secret material and must be protected like the credential store (owner-only 0600, included with the store in backups and rotation). It must be preserved across restarts for that guarantee; if it cannot be created (a process-substitution/FIFO store path such as `/dev/fd/N`, a read-only filesystem, a missing directory, or a create/write/fsync/link/fchmod failure), the daemon logs a warning and uses a transient per-run key, so unknown-user challenges change across restarts and the cross-restart guarantee does not hold for that deployment. One residual is accepted: the store iteration count is observable pre-auth by design, since the miss path must match a hit. **Transport policy (hardening A7-3/S1):** an auth-required module accepts credentials only when either (a) the connection is an encrypted, verified TLS connection whose client certificate matches `--client-cn`, or (b) the connection is plaintext from a loopback TCP peer **and** the operator explicitly passed `--allow-unauthenticated`. A remote plaintext peer, and a loopback plaintext peer without that flag, are refused at the config gate before any challenge is sent; `--allow-unauthenticated` never permits remote plaintext auth (remote peers still require verified TLS). Daemon modules are a `--daemon`-only feature — the SSH `--stdio` path never loads a daemon config and is not an auth transport for them. Because the loopback allowance trusts whichever peer the kernel reports as `127.0.0.1`, it assumes nothing relays remote connections to the daemon: a local TCP forwarder or TLS-terminating proxy in front of an auth-module listener makes remote clients appear as loopback and bypasses the mutual-TLS identity check, so do not front an auth-module listener with such a relay.
|
||||||
- **Credential store format:** server `--password-file`/`--early-input` files are line-based `user:$fastsync$1$pbkdf2-sha256$<iters>$<salt_b64>$<stored_key_b64>$<server_key_b64>`, one per line (standard base64; 16-byte salt, 32-byte keys; `iters` in `[100000, 10000000]`, default 600000). Every entry in the resulting store must agree on `iters` (a store whose entries disagree, or where a layered `--early-input` disagrees with `--password-file`, is rejected). Generate lines with `fastsync-server --hash-credentials FILE [--iterations N]`; the emitted lines are secret material, so redirect them to an owner-only (mode 0600) file (the tool warns on stderr if stdout is a group/other-accessible regular file). Blank lines and lines starting with `#`/`;` are comments; the parser is strict (a malformed line fails the whole load, so a typo can never let a different set of users in). **The legacy `user:SHA256HEX` form is hard-rejected** with an actionable "legacy" error; there is no auto-upgrade, so a replayable bearer digest can never be loaded by a 2.19.0 daemon. The client `--password-file` holds `user:password` on its first meaningful line (the literal password, used only for the handshake then burned); keep both files readable only by their owner (mode 0600). Per-username wire length is bounded (256 chars) and every decoded salt/key length is validated. Loading the store also maintains an owner-only `<store_path>.dummykey` sidecar (auto-created, mode 0600, exactly 32 bytes) holding the store-wide dummy key that shapes unknown-user challenges; persist it across daemon restarts so those challenges stay stable, and treat a sidecar with the wrong owner, a mode other than exactly 0600, the wrong size or the wrong type as a fatal load error (fail closed). If the sidecar cannot be created (e.g. a process-substitution store path such as `/dev/fd/N`, a read-only filesystem, a missing directory, or a create/write/fsync/link/fchmod failure), the daemon logs a warning and uses a transient per-run key, so the cross-restart stability guarantee does not hold there.
|
- **Credential store format:** server `--password-file`/`--early-input` files are line-based `user:$fastsync$1$pbkdf2-sha256$<iters>$<salt_b64>$<stored_key_b64>$<server_key_b64>`, one per line (standard base64; 16-byte salt, 32-byte keys; `iters` in `[100000, 10000000]`, default 600000). Every entry in the resulting store must agree on `iters` (a store whose entries disagree, or where a layered `--early-input` disagrees with `--password-file`, is rejected). Generate lines with `fastsync-server --hash-credentials FILE [--iterations N]`; the emitted lines are secret material, so redirect them to an owner-only (mode 0600) file (the tool warns on stderr if stdout is a group/other-accessible regular file). Blank lines and lines starting with `#`/`;` are comments; the parser is strict (a malformed line fails the whole load, so a typo can never let a different set of users in). **The legacy `user:SHA256HEX` form is hard-rejected** with an actionable "legacy" error; there is no auto-upgrade, so a replayable bearer digest can never be loaded by a 2.19.0 daemon. The client `--password-file` holds `user:password` on its first meaningful line (the literal password, used only for the handshake then burned); keep both files readable only by their owner (mode 0600). Per-username wire length is bounded (256 chars) and every decoded salt/key length is validated. Loading the store also maintains an owner-only `<store_path>.dummykey` sidecar (auto-created, mode 0600, exactly 32 bytes) holding the store-wide dummy key that shapes unknown-user challenges; persist it across daemon restarts so those challenges stay stable, and treat a sidecar with the wrong owner, a mode other than exactly 0600, the wrong size or the wrong type as a fatal load error (fail closed). If the sidecar cannot be created (e.g. a process-substitution store path such as `/dev/fd/N`, a read-only filesystem, a missing directory, or a create/write/fsync/link/fchmod failure), the daemon logs a warning and uses a transient per-run key, so the cross-restart stability guarantee does not hold there.
|
||||||
- **Plaintext caveat:** an auth-required module is refused, **before any challenge is sent**, unless the connection is encrypted and verified TLS whose client certificate matches the server's `--client-cn`, or it is plaintext from a loopback TCP peer **and** the operator passed `--allow-unauthenticated`. A remote plaintext peer, and a loopback plaintext peer without that flag, never receive a challenge, and `--allow-unauthenticated` never permits remote plaintext auth (remote peers still require verified TLS). On the loopback plaintext transport that remains permitted, a local sniffer could still read the challenge and response and mount an **offline dictionary attack** against a weak password, so use `--tls` for any real deployment. `--client-cn` matches the certificate CN only (not a subjectAltName), which is acceptable for a private CA. Clients sending daemon credentials with `--password-file` to a non-loopback daemon must use `--tls`; the client rejects such a destination before any network I/O. Unlike the old challenge-less exchange there is **no replay**: the proof is bound to the fresh per-connection server nonce, so a captured `STATUS_AUTH_RESPONSE` cannot be reused on another connection (an integration test proxies the daemon and proves this). TLS client-CN (`--client-cn`) is an independent transport identity check and composes with password auth; because `--tls` already mandates `--client-cn`, a TLS auth connection always verifies the client CN, so both checks necessarily apply together on such a connection.
|
- **Plaintext caveat:** an auth-required module is refused, **before any challenge is sent**, unless the connection is encrypted and verified TLS whose client certificate matches the server's `--client-cn`, or it is plaintext from a loopback TCP peer **and** the operator passed `--allow-unauthenticated`. A remote plaintext peer, and a loopback plaintext peer without that flag, never receive a challenge, and `--allow-unauthenticated` never permits remote plaintext auth (remote peers still require verified TLS). On the loopback plaintext transport that remains permitted, a local sniffer could still read the challenge and response and mount an **offline dictionary attack** against a weak password, so use `--tls` for any real deployment. `--client-cn` matches the certificate CN only (not a subjectAltName), which is acceptable for a private CA. Clients sending daemon credentials with `--password-file` to a non-loopback daemon must use `--tls`; the client rejects such a destination before any network I/O. Unlike the old challenge-less exchange there is **no replay**: the proof is bound to the fresh per-connection server nonce, so a captured `STATUS_AUTH_RESPONSE` cannot be reused on another connection (an integration test proxies the daemon and proves this). TLS client-CN (`--client-cn`) is an independent transport identity check and composes with password auth; because `--tls` already mandates `--client-cn`, a TLS auth connection always verifies the client CN, so both checks necessarily apply together on such a connection.
|
||||||
@@ -660,7 +663,7 @@ now transmits targets (the prior behavior was broken/partial); its status moved
|
|||||||
| `--trust-sender` | Trust remote sender's file list | ✅ Implemented | Long-form-only, receiver-local policy that never crosses the wire. The receiver skips its redundant up-front re-validation of the incoming file list (empty/`..` path rejection and the escaping-symlink-target containment), trusting the sender instead of double-checking (fewer checks, faster, potentially unsafe, matching rsync). Off by default. The low-level fd-relative confinement primitives (`file_open_secure_parent`, the O_NOFOLLOW parent walk, leaf/destination confinement) are deliberately KEPT even under `--trust-sender`, so a hostile sender still cannot write or link outside the authorized root (see Phase-5 notes below) |
|
| `--trust-sender` | Trust remote sender's file list | ✅ Implemented | Long-form-only, receiver-local policy that never crosses the wire. The receiver skips its redundant up-front re-validation of the incoming file list (empty/`..` path rejection and the escaping-symlink-target containment), trusting the sender instead of double-checking (fewer checks, faster, potentially unsafe, matching rsync). Off by default. The low-level fd-relative confinement primitives (`file_open_secure_parent`, the O_NOFOLLOW parent walk, leaf/destination confinement) are deliberately KEPT even under `--trust-sender`, so a hostile sender still cannot write or link outside the authorized root (see Phase-5 notes below) |
|
||||||
| `--old-args` | Disable modern arg protection | ✅ Implemented | SSH-only; accepted for CLI compatibility but is now a **documented no-op**: FastSync always single-quote-escapes the remote server path and each `--remote-option` value (`ssh_build_remote_command`), so a metacharacter-bearing `--rsync-path` can never be interpreted by the remote shell. The flag no longer disables that quoting (the old raw-construction behavior was an injection foot-gun and is removed); the safety-relevant behavior is identical either way |
|
| `--old-args` | Disable modern arg protection | ✅ Implemented | SSH-only; accepted for CLI compatibility but is now a **documented no-op**: FastSync always single-quote-escapes the remote server path and each `--remote-option` value (`ssh_build_remote_command`), so a metacharacter-bearing `--rsync-path` can never be interpreted by the remote shell. The flag no longer disables that quoting (the old raw-construction behavior was an injection foot-gun and is removed); the safety-relevant behavior is identical either way |
|
||||||
| `--ignore-missing-args` | Ignore missing source args | ✅ Implemented | FastSync has a single source-root argument (which always exists), so the "explicitly requested source arguments" are the `--files-from` entries and the flags only ever apply there (inert without `--files-from`, like `-R`). Without the flag a listed-but-missing entry stays a hard pre-transfer error (nothing is transferred). With it each missing entry is skipped: nothing is sent for it, it never enters the keep-set, and the run succeeds for the rest — an all-missing non-empty list succeeds transferring nothing, matching rsync. `--dirs` + `--files-from` missing entries are skipped the same way. Every skipped entry is logged and a per-run warning names the count, so the handling is never a silent no-op. Divergences: an EMPTY `--files-from` file stays a hard error in every mode (no argument was requested at all; rsync likewise reports "no source files specified"); missing-arg skipping only applies to the pre-transfer list validation, so an entry that is present at preflight and vanishes mid-transfer still fails (matching rsync, whose flag "does not affect subsequent vanished-file errors"); `--no-ignore-missing-args` is not a supported negation |
|
| `--ignore-missing-args` | Ignore missing source args | ✅ Implemented | FastSync has a single source-root argument (which always exists), so the "explicitly requested source arguments" are the `--files-from` entries and the flags only ever apply there (inert without `--files-from`, like `-R`). Without the flag a listed-but-missing entry stays a hard pre-transfer error (nothing is transferred). With it each missing entry is skipped: nothing is sent for it, it never enters the keep-set, and the run succeeds for the rest — an all-missing non-empty list succeeds transferring nothing, matching rsync. `--dirs` + `--files-from` missing entries are skipped the same way. Every skipped entry is logged and a per-run warning names the count, so the handling is never a silent no-op. Divergences: an EMPTY `--files-from` file stays a hard error in every mode (no argument was requested at all; rsync likewise reports "no source files specified"); missing-arg skipping only applies to the pre-transfer list validation, so an entry that is present at preflight and vanishes mid-transfer still fails (matching rsync, whose flag "does not affect subsequent vanished-file errors"); `--no-ignore-missing-args` is not a supported negation |
|
||||||
| `--delete-missing-args` | Delete missing source args | ✅ Implemented | Implies `--ignore-missing-args` (order-independent) and additionally removes each missing entry's destination mirror receiver-side. The mirror is computed exactly like a present sibling's wire path: the bare relative entry under `-R`, otherwise the full source-mirror path below the destination root. rsync parity, verified against the man page: it does **not** imply `--delete` generally and is "independent of any other type of delete processing" — unrelated destination extras are untouched unless `--delete` is also present. Composition with `--delete` + timing: the exact-path deletions commit with the manifest, early for `--delete-before`/`--delete-during`, else only after a fully-successful transfer (delete-after/commit). A non-empty directory mirror is removed only when `--force` or `--delete` is in effect (otherwise it is left with a warning and the run continues, like rsync); an absent mirror is a no-op. An explicitly listed missing arg is a user request, not an excluded file: its deletion is never blocked by the filter-exclusion protection of excluded destination mirrors (a mirror sitting inside a filter-excluded directory is still removed). Safety/policy: gated by the server `--allow-delete` policy like `--delete`; the request paths cross the wire only in the delete-manifest frame and are confined by the same receiver validation as the keep-set (non-empty, relative, traversal-free, bounded by the per-section/per-frame manifest caps); the `--delay-updates` staging directory and basis snapshots are protected exactly as in the extras walker. Divergence: the missing-args deletions are not counted toward `--max-delete` (they are explicit per-path requests, not discovered extras). See the Phase-3 wire note below for the `PROTOCOL_VERSION` bump |
|
| `--delete-missing-args` | Delete missing source args | ✅ Implemented | Implies `--ignore-missing-args` (order-independent) and additionally removes each missing entry's destination mirror receiver-side. The mirror is computed exactly like a present sibling's wire path: the bare relative entry under `-R`, otherwise the full source-mirror path below the destination root. rsync parity, verified against the man page: it does **not** imply `--delete` generally and is "independent of any other type of delete processing" — unrelated destination extras are untouched unless `--delete` is also present. Composition with `--delete` + timing: the exact-path deletions commit with the manifest, early for `--delete-before`/`--delete-during`, else only after a fully-successful transfer (delete-after/commit). A non-empty directory mirror is removed only when `--force` or `--delete` is in effect (otherwise it is left with a warning and the run continues, like rsync); an absent mirror is a no-op. `--force` is deletion authority and is therefore gated by the server `--allow-delete` policy exactly like `--delete`/`--delete-missing-args`: without it the receiver clears the flag, so a client cannot use `--force` to recursively replace or remove a destination directory tree. An explicitly listed missing arg is a user request, not an excluded file: its deletion is never blocked by the filter-exclusion protection of excluded destination mirrors (a mirror sitting inside a filter-excluded directory is still removed). Safety/policy: gated by the server `--allow-delete` policy like `--delete`; the request paths cross the wire only in the delete-manifest frame and are confined by the same receiver validation as the keep-set (non-empty, relative, traversal-free, bounded by the per-section/per-frame manifest caps); the `--delay-updates` staging directory and basis snapshots are protected exactly as in the extras walker. Divergence: the missing-args deletions are not counted toward `--max-delete` (they are explicit per-path requests, not discovered extras). See the Phase-3 wire note below for the `PROTOCOL_VERSION` bump |
|
||||||
|
|
||||||
## 16. Batch Operations
|
## 16. Batch Operations
|
||||||
|
|
||||||
@@ -677,7 +680,7 @@ now transmits targets (the prior behavior was broken/partial); its status moved
|
|||||||
| `--stop-after=MINS` | Stop after N minutes | ✅ Implemented | Client-only sender stop deadline (Phase 6): computing `--stop-after=MINS` (a positive minute count; 0/negative/garbage rejected) and `--stop-at=TIME` (`HH:MM`, `HH:MM:SS`, or `now+N[smhd]`; a past time stops immediately). The transfer stops ELEGANTLY at the next chunk boundary: everything already fully sent is kept and applied, the run returns 0, and --delete (late/delete-after timing) does NOT wipe the destination — when the scan is cut short the partial keep-set manifest is suppressed with a warning (the delete walk is skipped rather than acting on an incomplete keep-set, so unscanned source mirrors survive). `--delete-before`/`--delete-during` still run their complete pre-scan (which ignores the deadline). Local client-only fields: never serialized into the wire config frame, so no PROTOCOL_VERSION bump. `--stop-after` uses CLOCK_MONOTONIC; `--stop-at` uses the wall clock. Works single-threaded and under `-j`/`--threads` (multithreaded). Divergence: rsync computes `--stop-after` from the run start; FastSync likewise. When both are given, the earlier of the two deadlines wins (checked per iteration). See the Phase-6 stop notes below |
|
| `--stop-after=MINS` | Stop after N minutes | ✅ Implemented | Client-only sender stop deadline (Phase 6): computing `--stop-after=MINS` (a positive minute count; 0/negative/garbage rejected) and `--stop-at=TIME` (`HH:MM`, `HH:MM:SS`, or `now+N[smhd]`; a past time stops immediately). The transfer stops ELEGANTLY at the next chunk boundary: everything already fully sent is kept and applied, the run returns 0, and --delete (late/delete-after timing) does NOT wipe the destination — when the scan is cut short the partial keep-set manifest is suppressed with a warning (the delete walk is skipped rather than acting on an incomplete keep-set, so unscanned source mirrors survive). `--delete-before`/`--delete-during` still run their complete pre-scan (which ignores the deadline). Local client-only fields: never serialized into the wire config frame, so no PROTOCOL_VERSION bump. `--stop-after` uses CLOCK_MONOTONIC; `--stop-at` uses the wall clock. Works single-threaded and under `-j`/`--threads` (multithreaded). Divergence: rsync computes `--stop-after` from the run start; FastSync likewise. When both are given, the earlier of the two deadlines wins (checked per iteration). See the Phase-6 stop notes below |
|
||||||
| `--stop-at=TIME` | Stop at specified time | ✅ Implemented | Same feature as `--stop-after` (deadline transfer stop), absolute wall-clock form (`HH:MM[:SS]` or `now+N[smhd]`). See the row above and the Phase-6 stop notes |
|
| `--stop-at=TIME` | Stop at specified time | ✅ Implemented | Same feature as `--stop-after` (deadline transfer stop), absolute wall-clock form (`HH:MM[:SS]` or `now+N[smhd]`). See the row above and the Phase-6 stop notes |
|
||||||
| `--fsync` | Fsync every written file before publication | ✅ Implemented | |
|
| `--fsync` | Fsync every written file before publication | ✅ Implemented | |
|
||||||
| `--protocol=NUM` | Force older protocol version | ✅ Implemented | Forces the wire protocol version for this transfer. FastSync has exactly ONE wire format (`PROTOCOL_VERSION`, currently 2.19.0) with no downgrade/backward-compat code paths, so `--protocol=2.19.0` is accepted (it sets the version claim the client sends, which the server already requires to match exactly) and **every other value is rejected up front** with a clear error before any connection — it does not and cannot speak an older or virtual wire format. Divergence from rsync (which negotiates a range and downgrades to an integer 0..31): FastSync's honest contract is force-to-the-one-supported-value; a genuine downgrade would require a per-version compatibility layer that does not exist. Client-only; the server-side exact-match check is unchanged. `--protocol=2.18.0`/`2.18`/`2.17.0`/`2.16.0`/`2.15.0`/`216`/`31`/garbage are all rejected. See the Phase-6 protocol note below |
|
| `--protocol=NUM` | Force older protocol version | ✅ Implemented | Forces the wire protocol version for this transfer. FastSync has exactly ONE wire format (`PROTOCOL_VERSION`, currently 2.21.0) with no downgrade/backward-compat code paths, so `--protocol=2.21.0` is accepted (it sets the version claim the client sends, which the server already requires to match exactly) and **every other value is rejected up front** with a clear error before any connection — it does not and cannot speak an older or virtual wire format. Divergence from rsync (which negotiates a range and downgrades to an integer 0..31): FastSync's honest contract is force-to-the-one-supported-value; a genuine downgrade would require a per-version compatibility layer that does not exist. Client-only; the server-side exact-match check is unchanged. `--protocol=2.20.0`/`2.19.0`/`2.18.0`/`2.18`/`2.17.0`/`2.16.0`/`2.15.0`/`216`/`31`/garbage are all rejected. See the Phase-6 protocol note below |
|
||||||
| `--iconv=CONVERT_SPEC` | Charset conversion | ✅ Implemented | Charset conversion of FILE NAMES (not content) at the protocol boundary via iconv(3): `--iconv=LOCAL[,REMOTE]` — the sender converts each local filename LOCAL→REMOTE before transmitting, and the receiver converts each wire filename REMOTE→LOCAL before creating/writing. The full CONVERT_SPEC is serialized into the config frame as a new trailing string field so the peer knows the wire charset; **PROTOCOL_VERSION bumped 2.15.0 → 2.16.0**. `LOCAL[,REMOTE]` parse: single charset ⇒ LOCAL==REMOTE (identity both ways); garbage rejected up front. Validation probes BOTH directions (a spec that only opens one way is refused, as is a NUL-emitting target charset like utf-16/utf-32/ucs-2, since filenames cannot contain NUL). An unrepresentable name (EILSEQ/EINVAL) fails that path cleanly with a logged `--iconv: cannot convert file name ...` and is never written mangled/truncated. Conversion is applied at EVERY wire-path site (regular/MKDIR/hardlink path+target/symlink path+target/SPECIAL, the delete manifest, the incremental-check path, and the `-s`/`chunk_serialize` embedded blob path), on both client and server (`--iconv` is also a server/daemon option). Zero overhead when unset. See the Phase-6 iconv notes below |
|
| `--iconv=CONVERT_SPEC` | Charset conversion | ✅ Implemented | Charset conversion of FILE NAMES (not content) at the protocol boundary via iconv(3): `--iconv=LOCAL[,REMOTE]` — the sender converts each local filename LOCAL→REMOTE before transmitting, and the receiver converts each wire filename REMOTE→LOCAL before creating/writing. The full CONVERT_SPEC is serialized into the config frame as a new trailing string field so the peer knows the wire charset; **PROTOCOL_VERSION bumped 2.15.0 → 2.16.0**. `LOCAL[,REMOTE]` parse: single charset ⇒ LOCAL==REMOTE (identity both ways); garbage rejected up front. Validation probes BOTH directions (a spec that only opens one way is refused, as is a NUL-emitting target charset like utf-16/utf-32/ucs-2, since filenames cannot contain NUL). An unrepresentable name (EILSEQ/EINVAL) fails that path cleanly with a logged `--iconv: cannot convert file name ...` and is never written mangled/truncated. Conversion is applied at EVERY wire-path site (regular/MKDIR/hardlink path+target/symlink path+target/SPECIAL, the delete manifest, the incremental-check path, and the `-s`/`chunk_serialize` embedded blob path), on both client and server (`--iconv` is also a server/daemon option). Zero overhead when unset. See the Phase-6 iconv notes below |
|
||||||
| `--checksum-seed=NUM` | Set checksum seed | ✅ Implemented | Sets the seed for FastSync's whole-file xxHash64 digest (full 64-bit seed) and for the delta path's per-block xxHash32 strong checksum (low 32 bits of the seed). An explicit seed deterministically changes every computed digest on BOTH endpoints (sender and receiver share the seed via the config frame, protocol 2.10.0), so identical runs with the same seed skip the same files and a changed seed changes the digests — the explicit-seed path that makes xxHash comparisons deterministic. `--checksum-choice=md5` has no seed and ignores it (documented). The value is a strict decimal 0..2⁶⁴-1 (blank, signed, or non-numeric values are rejected). Like rsync, a seed only matters where a digest is actually computed (`--checksum` or a basis-dir run, or a delta transfer); it does not by itself enable `--checksum`/`--delta`. Divergence from rsync: the default is seed 0, and FastSync never randomizes the seed (rsync uses a random per-transfer seed when `--checksum-seed` is unset); FastSync's unset default therefore reproduces its historical byte-for-byte behavior |
|
| `--checksum-seed=NUM` | Set checksum seed | ✅ Implemented | Sets the seed for FastSync's whole-file xxHash64 digest (full 64-bit seed) and for the delta path's per-block xxHash32 strong checksum (low 32 bits of the seed). An explicit seed deterministically changes every computed digest on BOTH endpoints (sender and receiver share the seed via the config frame, protocol 2.10.0), so identical runs with the same seed skip the same files and a changed seed changes the digests — the explicit-seed path that makes xxHash comparisons deterministic. `--checksum-choice=md5` has no seed and ignores it (documented). The value is a strict decimal 0..2⁶⁴-1 (blank, signed, or non-numeric values are rejected). Like rsync, a seed only matters where a digest is actually computed (`--checksum` or a basis-dir run, or a delta transfer); it does not by itself enable `--checksum`/`--delta`. Divergence from rsync: the default is seed 0, and FastSync never randomizes the seed (rsync uses a random per-transfer seed when `--checksum-seed` is unset); FastSync's unset default therefore reproduces its historical byte-for-byte behavior |
|
||||||
| `--secluded-args`, `-s` | Use protocol to send args | ⛔ Impossible/Divergence | Accepted for CLI compatibility (including the rsync short `-s`, Phase 7 Wave A) but a documented **no-op / divergence**. rsync's `-s` protects arguments from shell expansion by shipping them over the protocol; FastSync never passes remote arguments through a shell expansion boundary in the first place — its SSH transport builds the remote argv as **single-quote-escaped shell words** (`ssh_build_remote_command`), so the injection/leak that `-s` guards against does not exist and there is nothing to "seclude". Implementing a true arg-send protocol would mean replacing the argv-based SSH launch with an in-band argument channel, a large redesign of the transport that buys no security here. Chunk serialization remains the long-only `--chunk-serialization`. |
|
| `--secluded-args`, `-s` | Use protocol to send args | ⛔ Impossible/Divergence | Accepted for CLI compatibility (including the rsync short `-s`, Phase 7 Wave A) but a documented **no-op / divergence**. rsync's `-s` protects arguments from shell expansion by shipping them over the protocol; FastSync never passes remote arguments through a shell expansion boundary in the first place — its SSH transport builds the remote argv as **single-quote-escaped shell words** (`ssh_build_remote_command`), so the injection/leak that `-s` guards against does not exist and there is nothing to "seclude". Implementing a true arg-send protocol would mean replacing the argv-based SSH launch with an in-band argument channel, a large redesign of the transport that buys no security here. Chunk serialization remains the long-only `--chunk-serialization`. |
|
||||||
@@ -793,7 +796,7 @@ These are the hardest compatibility items because they require durable formats o
|
|||||||
|
|
||||||
**Phase 6, Wave B (iconv) shipping note (PROTOCOL 2.15.0 → 2.16.0):** `--iconv=LOCAL[,REMOTE]` converts file NAMES at the wire boundary (never content). The full CONVERT_SPEC is serialized into the config frame as a new trailing string field (empty→NULL canonicalized), so both ends share the same wire charset interpretation; this required the PROTOCOL bump because the frame is a strict ordered sequence and a peer that does not parse the new trailing field would desynchronize. Each end derives LOCAL (its own charset) and REMOTE (the wire charset): the sender opens LOCAL→REMOTE and converts every transmitted filename; the receiver opens REMOTE→LOCAL and converts every received filename before creating/writing. Conversion is applied at every wire-path site (regular/MKDIR/hardlink path+target/symlink path+target/SPECIAL, the delete manifest keep/protected/missing entries, the incremental-check path, and the embedded `-s`/chunk-blob path). A name it cannot convert (EILSEQ/EINVAL) is failed cleanly with a logged `--iconv: cannot convert file name ...` and is never written truncated/mangled. Validation probes both directions up front (both the sender local→remote and the receiver remote→local, and, for a server/daemon with its own `--iconv`, the client-REMOTE→server-LOCAL pair) so an unusable spec is rejected before the connection rather than mid-transfer, and NUL-emitting target charsets (utf-16/utf-32/ucs-2) are refused because filenames cannot contain NUL. Divergence documented upstream: the receiver does NOT half-swap; the wire charset always comes from the sender's REMOTE half, so a server whose local charset differs from the client's LOCAL must declare it with its own `--iconv`. Conversion is process-global and runs on a single thread per process (sender thread / receiver-loop thread), initialized before worker threads start and freed after they join.
|
**Phase 6, Wave B (iconv) shipping note (PROTOCOL 2.15.0 → 2.16.0):** `--iconv=LOCAL[,REMOTE]` converts file NAMES at the wire boundary (never content). The full CONVERT_SPEC is serialized into the config frame as a new trailing string field (empty→NULL canonicalized), so both ends share the same wire charset interpretation; this required the PROTOCOL bump because the frame is a strict ordered sequence and a peer that does not parse the new trailing field would desynchronize. Each end derives LOCAL (its own charset) and REMOTE (the wire charset): the sender opens LOCAL→REMOTE and converts every transmitted filename; the receiver opens REMOTE→LOCAL and converts every received filename before creating/writing. Conversion is applied at every wire-path site (regular/MKDIR/hardlink path+target/symlink path+target/SPECIAL, the delete manifest keep/protected/missing entries, the incremental-check path, and the embedded `-s`/chunk-blob path). A name it cannot convert (EILSEQ/EINVAL) is failed cleanly with a logged `--iconv: cannot convert file name ...` and is never written truncated/mangled. Validation probes both directions up front (both the sender local→remote and the receiver remote→local, and, for a server/daemon with its own `--iconv`, the client-REMOTE→server-LOCAL pair) so an unusable spec is rejected before the connection rather than mid-transfer, and NUL-emitting target charsets (utf-16/utf-32/ucs-2) are refused because filenames cannot contain NUL. Divergence documented upstream: the receiver does NOT half-swap; the wire charset always comes from the sender's REMOTE half, so a server whose local charset differs from the client's LOCAL must declare it with its own `--iconv`. Conversion is process-global and runs on a single thread per process (sender thread / receiver-loop thread), initialized before worker threads start and freed after they join.
|
||||||
|
|
||||||
**Phase 6, Wave C (protocol-version) shipping note (no PROTOCOL_VERSION change):** `--protocol=NUM` lets the client force the wire protocol version for a transfer. FastSync's protocol is a single lockstep format: the config frame is a strict ordered sequence and the server requires the client's version string to equal `PROTOCOL_VERSION` exactly (`config_receive_with_validate`, src/shared/config.c) — there are no older-format code paths and no downgrade/negotiation machinery, so a lower/higher/virtual version can never be spoken. The honest contract is therefore: `--protocol=2.19.0` (the current `PROTOCOL_VERSION`, as of the A7 auth redesign) is accepted and stored into the client's `version` claim (which `config_send` already transmits), and every other value — `2.18.0`, `2.18`, `2.17.0`, `2.16.0`, `2.15.0`, `3.0.0`, rsync-integer spellings like `216`/`31`, garbage, empty — is rejected up front in `validate_config()` before any connection, with a clear error that FastSync supports only its current wire protocol and cannot speak an older or virtual one. Implementation is client-only: a server-side `--protocol` is intentionally not added because the server has no negotiation (it only enforces exact match), and it could only ever be the current version. This preserves (and slightly tightens) existing validation: the client now also refuses to launch with a version it cannot actually speak, rather than only the server rejecting it later. A genuine downgrade would require a per-version compatibility layer for every frame/feature added since (append 2.10, preallocate 2.11, hardlinks 2.12, devices/specials/symlink-trust/xattr 2.13, remote-option 2.14, daemon module/auth 2.15, iconv 2.16, dir/symlink times 2.17, privilege flags --super/--copy-as 2.18, SCRAM daemon auth 2.19) and is intentionally out of scope — documented divergences from rsync's integer-negotiated downgrade remain.
|
**Phase 6, Wave C (protocol-version) shipping note (no PROTOCOL_VERSION change):** `--protocol=NUM` lets the client force the wire protocol version for a transfer. FastSync's protocol is a single lockstep format: the config frame is a strict ordered sequence and the server requires the client's version string to equal `PROTOCOL_VERSION` exactly (`config_receive_with_validate`, src/shared/config.c) — there are no older-format code paths and no downgrade/negotiation machinery, so a lower/higher/virtual version can never be spoken. The honest contract is therefore: `--protocol=2.21.0` (the current `PROTOCOL_VERSION`, as of the combined error-detail + server-contacting dry-run wave) is accepted and stored into the client's `version` claim (which `config_send` already transmits), and every other value — `2.20.0`, `2.19.0`, `2.18.0`, `2.18`, `2.17.0`, `2.16.0`, `2.15.0`, `3.0.0`, rsync-integer spellings like `216`/`31`, garbage, empty — is rejected up front in `validate_config()` before any connection, with a clear error that FastSync supports only its current wire protocol and cannot speak an older or virtual one. Implementation is client-only: a server-side `--protocol` is intentionally not added because the server has no negotiation (it only enforces exact match), and it could only ever be the current version. This preserves (and slightly tightens) existing validation: the client now also refuses to launch with a version it cannot actually speak, rather than only the server rejecting it later. A genuine downgrade would require a per-version compatibility layer for every frame/feature added since (append 2.10, preallocate 2.11, hardlinks 2.12, devices/specials/symlink-trust/xattr 2.13, remote-option 2.14, daemon module/auth 2.15, iconv 2.16, dir/symlink times 2.17, privilege flags --super/--copy-as 2.18, SCRAM daemon auth 2.19, packed metadata 2.20) and is intentionally out of scope — documented divergences from rsync's integer-negotiated downgrade remain.
|
||||||
|
|
||||||
**Phase-1/2 selection-and-update status correction (docs):** `-I/--ignore-times`, `--size-only`, `-@/--modify-window`, `--existing`, `--ignore-existing`, `-u/--update`, `-W/--whole-file`, and `--compress-threads` were previously listed as not-implemented in this document but are in fact fully implemented and tested on `dev`. This pass corrects the matrix to match the code. The realistic model of these is that FastSync is a *sender-driven* whole-tree copy, so the size+mtime quick-check and all three receiver-policy skips (`--existing`, `--ignore-existing`, `-u`) are evaluated against the **destination** on the receiver side, and their booleans cross the wire in the config frame. `-I`/`--size-only`/`--modify-window` modify the `--incremental` per-file `STATUS_CHECK` handshake's match predicate (`-I` disables the mtime leg and forces transfer; `--size-only` drops only the mtime leg; `--modify-window` adds tolerance to `metadata_mtime_matches`); they require `--incremental` (or a basis dir) to have a handshake to affect, mirroring how they only matter where a quick-check exists in rsync. `--existing`/`--ignore-existing`/`-u` are receiver write-time policies (skipping the write / newer-destination guard) applied across the regular-file, `--delay-updates`-staged, hardlink-sibling, and special/device paths; `-u` implies `-M` metadata and uses a second-then-nanosecond strict `>` newer check; both correctly influence `--remove-source-files` (a skipped source is not removed). `-W/--whole-file` disables block-level delta (opt-in via `--delta`), folded into the wire `use_delta` so no protocol bump was needed, and makes `--fuzzy` inert; `--append`/`--append-verify` are rejected with `-W`. `--compress-threads=NUM` (1..64, client-only, never crosses the wire) sizes the zstd compression worker pool. No code was changed by this correction; the implementation had landed in earlier merge waves (feat/ignore-times, feat/ignore-existing via the newer `file_to_disk_secure_no_replace`/`linkat EEXIST` path, feat/size-only, feat/modify-window, feat/whole-file, feat/update, compression-threads).
|
**Phase-1/2 selection-and-update status correction (docs):** `-I/--ignore-times`, `--size-only`, `-@/--modify-window`, `--existing`, `--ignore-existing`, `-u/--update`, `-W/--whole-file`, and `--compress-threads` were previously listed as not-implemented in this document but are in fact fully implemented and tested on `dev`. This pass corrects the matrix to match the code. The realistic model of these is that FastSync is a *sender-driven* whole-tree copy, so the size+mtime quick-check and all three receiver-policy skips (`--existing`, `--ignore-existing`, `-u`) are evaluated against the **destination** on the receiver side, and their booleans cross the wire in the config frame. `-I`/`--size-only`/`--modify-window` modify the `--incremental` per-file `STATUS_CHECK` handshake's match predicate (`-I` disables the mtime leg and forces transfer; `--size-only` drops only the mtime leg; `--modify-window` adds tolerance to `metadata_mtime_matches`); they require `--incremental` (or a basis dir) to have a handshake to affect, mirroring how they only matter where a quick-check exists in rsync. `--existing`/`--ignore-existing`/`-u` are receiver write-time policies (skipping the write / newer-destination guard) applied across the regular-file, `--delay-updates`-staged, hardlink-sibling, and special/device paths; `-u` implies `-M` metadata and uses a second-then-nanosecond strict `>` newer check; both correctly influence `--remove-source-files` (a skipped source is not removed). `-W/--whole-file` disables block-level delta (opt-in via `--delta`), folded into the wire `use_delta` so no protocol bump was needed, and makes `--fuzzy` inert; `--append`/`--append-verify` are rejected with `-W`. `--compress-threads=NUM` (1..64, client-only, never crosses the wire) sizes the zstd compression worker pool. No code was changed by this correction; the implementation had landed in earlier merge waves (feat/ignore-times, feat/ignore-existing via the newer `file_to_disk_secure_no_replace`/`linkat EEXIST` path, feat/size-only, feat/modify-window, feat/whole-file, feat/update, compression-threads).
|
||||||
|
|
||||||
@@ -830,14 +833,31 @@ These are the last compatibility items and the closing phase toward rsync flag p
|
|||||||
|
|
||||||
**Wave E (LAST) — Privilege: `--super`/`--no-super` and `--copy-as=USER[:GROUP]` (✅ implemented).** FastSync adopts a **safe-subset + clear-refusal** privilege model: it never blind-elevates and never calls `setuid`/`seteuid`/`setgid`. All privileged operations remain fd-relative and confined below the authorized receive root.
|
**Wave E (LAST) — Privilege: `--super`/`--no-super` and `--copy-as=USER[:GROUP]` (✅ implemented).** FastSync adopts a **safe-subset + clear-refusal** privilege model: it never blind-elevates and never calls `setuid`/`seteuid`/`setgid`. All privileged operations remain fd-relative and confined below the authorized receive root.
|
||||||
|
|
||||||
`--super`/`--no-super` set a receiver-side tri-state `Config->super_mode` (`SUPER_MODE_AUTO`/`ON`/`OFF`). `privilege_super_permitted()` / `privilege_super_mode_permitted()` (src/shared/identity.c) return true for `ON` and `AUTO` (AUTO preserves FastSync's historical best-effort attempt, where the kernel refuses an unprivileged call and the caller skips it) and false only for `OFF`. The gate covers every super-user activity FastSync performs: ownership application (`identity_apply_ownership`/`_link`), char/block device-node creation (`file_save_special_to_disk`), writes into an existing device (`--write-devices`), and the `--fake-super` owner replay. Unprivileged FIFO creation is deliberately unaffected. `--super` does **not** imply `--numeric-ids`: ownership is applied only when an explicit identity policy (`--usermap`/`--groupmap`/`--chown`/`--numeric-ids`/`--copy-as`) is also given. `--no-super` suppresses those activities even for a root receiver. A non-root receiver given `--super` logs one warning at activation (`identity_set_active`); each confined attempt is then refused by the kernel and skipped, never aborting. The confinement floor is unchanged (`file_open_secure_parent`, `O_NOFOLLOW`, root/path checks). Operator control: the server CLI accepts `--no-super`, a veto that forces `OFF` for every connection, refuses any client `--copy-as`, and neutralizes an explicit `--super` (the connection is accepted but no super-user activity is attempted). On a daemon, a module that has not opted in with `client owner = yes` additionally has super-user device activity forced off (see the Daemon Mode notes).
|
`--super`/`--no-super` set a receiver-side tri-state `Config->super_mode` (`SUPER_MODE_AUTO`/`ON`/`OFF`). `privilege_super_permitted()` / `privilege_super_mode_permitted()` (src/shared/identity.c) return true for `ON` and `AUTO` (AUTO preserves FastSync's historical best-effort attempt, where the kernel refuses an unprivileged call and the caller skips it) and false only for `OFF`. The gate covers every super-user activity FastSync performs: ownership application (`identity_apply_ownership`/`_link`), char/block device-node creation (`file_save_special_to_disk`), writes into an existing device (`--write-devices`), and the `--fake-super` owner replay. Unprivileged FIFO creation is deliberately unaffected. `--super` does **not** imply `--numeric-ids`: ownership is applied only when an explicit identity policy (`--usermap`/`--groupmap`/`--chown`/`--numeric-ids`/`--copy-as`) is also given. `--no-super` suppresses those activities even for a root receiver. A non-root receiver given `--super` logs one warning at activation (`identity_set_active`); each confined attempt is then refused by the kernel and skipped, never aborting. The confinement floor is unchanged (`file_open_secure_parent`, `O_NOFOLLOW`, root/path checks). Operator control: the server CLI accepts `--no-super`, a veto that forces `OFF` for every connection, refuses any client `--copy-as`, and neutralizes an explicit `--super` (the connection is accepted but no super-user activity is attempted). A privileged (root) standalone TCP listener instead defaults to `OFF` and requires the server-only `--allow-super` opt-in to attempt any super-user activity (the flag is rejected with `--stdio`, whose client-composed remote argv must never defeat the default; use a forced command if the default must hold); a non-root server is unchanged. On a daemon, a module that has not opted in with `client owner = yes` additionally has super-user device activity forced off (see the Daemon Mode notes).
|
||||||
|
|
||||||
`--copy-as=USER[:GROUP]` is the safe subset. FastSync's receiver is multithreaded, so a real credential switch is unsafe; instead the receiver forces the ownership of **every entry it writes** — regular files, symlinks, directories (including implicitly-created parents), and special nodes — to the resolved target ids through the confined fd-relative identity path. USER is resolved on the client (name, `@N`/bare N, or `*` = client euid); when `:GROUP` is omitted the user's primary gid is used (falling back to `gid == uid` for a numeric id with no local passwd entry). It requires a privileged (root) receiver: an unprivileged receiver refuses the whole transfer at the config handshake, before `STATUS_OK`, so no data is ever written with the wrong ownership. A `--copy-as` chown failure on a capability-restricted root is logged at ERROR (never silently downgraded). `--copy-as` implies metadata (`--no-preserve` is rejected) and `--fake-super` cannot override it. Daemon policy: a `--daemon` receiver refuses **every** client-chosen-ownership / super-user request — `--numeric-ids`, `--chown`, `--usermap`/`--groupmap`, `--fake-super`, `--copy-as`, and explicit `--super` — unless the selected module opts in with `client owner = yes`; without that per-module opt-in any client could force arbitrary ownership inside the module root (the standalone listener and the SSH-launched `--stdio` server, which each serve one operator-authorized root, honor these requests). A `--copy-as` chown failure on a capability-restricted root marks the entry as failed rather than reporting success with the wrong owner.
|
`--copy-as=USER[:GROUP]` is the safe subset. FastSync's receiver is multithreaded, so a real credential switch is unsafe; instead the receiver forces the ownership of **every entry it writes** — regular files, symlinks, directories (including implicitly-created parents), and special nodes — to the resolved target ids through the confined fd-relative identity path. USER is resolved on the client (name, `@N`/bare N, or `*` = client euid); when `:GROUP` is omitted the user's primary gid is used (falling back to `gid == uid` for a numeric id with no local passwd entry). It requires a privileged (root) receiver: an unprivileged receiver refuses the whole transfer at the config handshake, before `STATUS_OK`, so no data is ever written with the wrong ownership. A `--copy-as` chown failure on a capability-restricted root is logged at ERROR (never silently downgraded). `--copy-as` implies metadata (`--no-preserve` is rejected) and `--fake-super` cannot override it. Daemon policy: a `--daemon` receiver refuses **every** client-chosen-ownership / super-user request — `--numeric-ids`, `--chown`, `--usermap`/`--groupmap`, `--fake-super`, `--copy-as`, and explicit `--super` — unless the selected module opts in with `client owner = yes`; without that per-module opt-in any client could force arbitrary ownership inside the module root (a root standalone TCP listener, which serves one operator-authorized root, honors these requests only when started with `--allow-super`; the flag is rejected with `--stdio`). A `--copy-as` chown failure on a capability-restricted root marks the entry as failed rather than reporting success with the wrong owner.
|
||||||
|
|
||||||
**Wire:** two trailing config-frame blocks after the `--iconv` spec, in fixed order — `send_privilege_options`/`receive_privilege_options` (one `super_mode` int, validated `0..2`), then `send_copy_as_options`/`receive_copy_as_options` (presence int + two int32 ids, validated `>= 0`, with `copy_as_set ⇒ use_metadata`). `PROTOCOL_VERSION` bumped **2.17.0 → 2.18.0**. **Divergences from rsync:** rsync's `--super` elevates the receiver and `--copy-as` actually switches its credentials; FastSync never elevates and only permits/forwards confined attempts, and `--copy-as` forces ownership rather than switching identity.
|
**Wire:** two trailing config-frame blocks after the `--iconv` spec, in fixed order — `send_privilege_options`/`receive_privilege_options` (one `super_mode` int, validated `0..2`), then `send_copy_as_options`/`receive_copy_as_options` (presence int + two int32 ids, validated `>= 0`, with `copy_as_set ⇒ use_metadata`). `PROTOCOL_VERSION` bumped **2.17.0 → 2.18.0**. **Divergences from rsync:** rsync's `--super` elevates the receiver and `--copy-as` actually switches its credentials; FastSync never elevates and only permits/forwards confined attempts, and `--copy-as` forces ownership rather than switching identity.
|
||||||
|
|
||||||
**Post-Phase-7 Summary (after Waves A–E).** ✅143 / 🔀0 / ⛔4 / ⚠️0 / 🔄0 / ❌0 = 147. The 3 `🔀 Alt Arg` rows (`-a`, `-p`, `-z`) are ✅ (Wave A). All 10 prior `⚠️ Partial` rows are resolved to ✅ (`-S`, `-P`, `--block-size`, `--fake-super`, `--devices`, `--copy-devices`, `--write-devices`) or ⛔ (`--stderr=client`, `-N/--crtimes`, `--specials` for the impossible socket case). The 3 `🔄 Compatibility No-op` rows are resolved: `-O`/`-J` are now real ✅ (Wave D), `--secluded-args` is ⛔. The **Impossible/Divergence** bucket holds the 4 physically-impossible/divergent flags: `--stderr=client`, `-N/--crtimes`, `--specials` (sockets), `--secluded-args`. The last two `❌ Not Implemented` rows — `--super` and `--copy-as=USER[:GROUP]` — are now ✅ (Wave E). **No `❌ Not Implemented` rows remain.**
|
**Post-Phase-7 Summary (after Waves A–E).** ✅143 / 🔀0 / ⛔4 / ⚠️0 / 🔄0 / ❌0 = 147. The 3 `🔀 Alt Arg` rows (`-a`, `-p`, `-z`) are ✅ (Wave A). All 10 prior `⚠️ Partial` rows are resolved to ✅ (`-S`, `-P`, `--block-size`, `--fake-super`, `--devices`, `--copy-devices`, `--write-devices`) or ⛔ (`--stderr=client`, `-N/--crtimes`, `--specials` for the impossible socket case). The 3 `🔄 Compatibility No-op` rows are resolved: `-O`/`-J` are now real ✅ (Wave D), `--secluded-args` is ⛔. The **Impossible/Divergence** bucket holds the 4 physically-impossible/divergent flags: `--stderr=client`, `-N/--crtimes`, `--specials` (sockets), `--secluded-args`. The last two `❌ Not Implemented` rows — `--super` and `--copy-as=USER[:GROUP]` — are now ✅ (Wave E). **No `❌ Not Implemented` rows remain.**
|
||||||
|
|
||||||
|
## Packed Metadata Frame (protocol 2.20.0)
|
||||||
|
|
||||||
|
A file's metadata used to cross the wire as up to 12 separate per-field framed
|
||||||
|
messages (a present flag followed by mode/uid/gid/mtime/atime/crtime writes),
|
||||||
|
which cost ~11 extra protocol frames per file on many-small-file trees. FastSync
|
||||||
|
now sends the metadata as ONE packed frame: a single `int32` present flag
|
||||||
|
(`0` = absent) followed, when present, by the fixed
|
||||||
|
`FILE_METADATA_WIRE_SIZE`-byte (68-byte) field record already emitted by the
|
||||||
|
shared `metadata_to_buf()`/`metadata_from_buf()` chunk codec. Absent metadata is
|
||||||
|
a lone `int32` zero. The encoded field layout is unchanged (only the framing
|
||||||
|
collapses), so chunk-serialized blobs remain byte-identical. Protocol data is an
|
||||||
|
unframed byte stream, so the packed encoding is byte-for-byte identical to the
|
||||||
|
old field-by-field writes; `PROTOCOL_VERSION` was bumped `2.19.0 → 2.20.0` as a
|
||||||
|
deliberate lockstep-release marker rather than because of a
|
||||||
|
desynchronization. The strict same-version handshake rejects any mismatch before
|
||||||
|
a byte of the frame is parsed.
|
||||||
|
|
||||||
### Recommended Delivery Order
|
### Recommended Delivery Order
|
||||||
|
|
||||||
1. Resolve short-option conflicts (`-m`, `-M`, `-T`, `-f`, `-s`) and define the compatibility contract.
|
1. Resolve short-option conflicts (`-m`, `-M`, `-T`, `-f`, `-s`) and define the compatibility contract.
|
||||||
@@ -874,7 +894,7 @@ Ranked by user demand, implementation complexity, and interoperability impact (_
|
|||||||
|
|
||||||
| Feature | Description |
|
| Feature | Description |
|
||||||
|---------|-------------|
|
|---------|-------------|
|
||||||
| `-j` / `--threads` | Multithreaded pipeline (scanner/loader/sender) (renamed from `-m` in Phase 7 Wave A; `-m` is now rsync `--prune-empty-dirs`) |
|
| `-j` / `--threads[=N]` | Multithreaded pipeline (scanner/loader/sender); `N` (1–256) sizes the parallel scanner worker pool, bare `-j`/`--threads` uses the built-in default (renamed from `-m` in Phase 7 Wave A; `-m` is now rsync `--prune-empty-dirs`) |
|
||||||
| `--chunk-serialization` | Chunk serialization mode (long form only; `-s` is now rsync `--secluded-args`) |
|
| `--chunk-serialization` | Chunk serialization mode (long form only; `-s` is now rsync `--secluded-args`) |
|
||||||
| `--sendfile` | Zero-copy sendfile() syscall (TCP only) (long form only; `-f` is now rsync `--filter`) |
|
| `--sendfile` | Zero-copy sendfile() syscall (TCP only) (long form only; `-f` is now rsync `--filter`) |
|
||||||
| `-z [level]` / `--compress` | zstd compression level (1-22) (`-c` is now rsync `--checksum`) |
|
| `-z [level]` / `--compress` | zstd compression level (1-22) (`-c` is now rsync `--checksum`) |
|
||||||
|
|||||||
+415
-70
@@ -3,19 +3,24 @@
|
|||||||
|
|
||||||
Compares FastSync configs against rsync (no compression) and rsync+zstd.
|
Compares FastSync configs against rsync (no compression) and rsync+zstd.
|
||||||
Data is ~75% random/incompressible and ~25% structured/compressible by default,
|
Data is ~75% random/incompressible and ~25% structured/compressible by default,
|
||||||
controllable via --random-ratio.
|
controllable via --random-ratio. Transfers are verified by default (source and
|
||||||
|
destination must match) so a fast-but-broken copy is never counted.
|
||||||
|
|
||||||
Usage:
|
Usage:
|
||||||
python3 benchmark/bench.py
|
python3 benchmark/bench.py
|
||||||
python3 benchmark/bench.py --runs 5 --profiles lan wan
|
python3 benchmark/bench.py --runs 5 --profiles lan wan
|
||||||
python3 benchmark/bench.py --random-ratio 0.5 --size-mb 50
|
python3 benchmark/bench.py --random-ratio 0.5 --size-mb 50
|
||||||
python3 benchmark/bench.py --delay 50ms --jitter 10ms --throughput 100mbit
|
python3 benchmark/bench.py --delay 50ms --jitter 10ms --throughput 100mbit
|
||||||
|
python3 benchmark/bench.py --warm --runs 3
|
||||||
python3 benchmark/bench.py --output json
|
python3 benchmark/bench.py --output json
|
||||||
"""
|
"""
|
||||||
import argparse
|
import argparse
|
||||||
|
import filecmp
|
||||||
import json
|
import json
|
||||||
|
import math
|
||||||
import os
|
import os
|
||||||
import random
|
import random
|
||||||
|
import shlex
|
||||||
import shutil
|
import shutil
|
||||||
import socket
|
import socket
|
||||||
import statistics
|
import statistics
|
||||||
@@ -25,8 +30,11 @@ import tempfile
|
|||||||
import time
|
import time
|
||||||
|
|
||||||
PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
|
PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
|
||||||
BUILD_DIR = os.path.join(PROJECT_ROOT, "build")
|
DEFAULT_BUILD_DIR = "build-bench"
|
||||||
SERVER_CMD = [os.path.join(BUILD_DIR, "server")]
|
# Populated by configure_build_dirs(); default to the dedicated bench dir so
|
||||||
|
# importing this module never depends on the user's existing build/ tree.
|
||||||
|
BUILD_DIR = os.path.join(PROJECT_ROOT, DEFAULT_BUILD_DIR)
|
||||||
|
SERVER_CMD = [os.path.join(BUILD_DIR, "server"), "--allow-unauthenticated"]
|
||||||
CLIENT_CMD = [os.path.join(BUILD_DIR, "client")]
|
CLIENT_CMD = [os.path.join(BUILD_DIR, "client")]
|
||||||
BENCH_DIR = os.path.join(PROJECT_ROOT, "bench_data")
|
BENCH_DIR = os.path.join(PROJECT_ROOT, "bench_data")
|
||||||
|
|
||||||
@@ -43,11 +51,12 @@ NETWORK_PROFILES = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
FASTSYNC_CONFIGS = [
|
FASTSYNC_CONFIGS = [
|
||||||
{"name": "fastsync", "flags": [], "tool": "fastsync"},
|
{"name": "fastsync", "flags": [], "tool": "fastsync"},
|
||||||
{"name": "fastsync -c", "flags": ["-c"], "tool": "fastsync"},
|
{"name": "fastsync -z", "flags": ["-z"], "tool": "fastsync"},
|
||||||
{"name": "fastsync -m", "flags": ["-m"], "tool": "fastsync"},
|
{"name": "fastsync -j", "flags": ["-j"], "tool": "fastsync"},
|
||||||
{"name": "fastsync -m -c", "flags": ["-m", "-c"], "tool": "fastsync"},
|
{"name": "fastsync -j -z", "flags": ["-j", "-z"], "tool": "fastsync"},
|
||||||
{"name": "fastsync -m -c -s", "flags": ["-m", "-c", "-s"], "tool": "fastsync"},
|
{"name": "fastsync -j -z --chunk-serialization", "flags": ["-j", "-z", "--chunk-serialization"], "tool": "fastsync"},
|
||||||
|
{"name": "fastsync --sendfile", "flags": ["--sendfile"], "tool": "fastsync"},
|
||||||
]
|
]
|
||||||
|
|
||||||
RSYNC_CONFIGS = [
|
RSYNC_CONFIGS = [
|
||||||
@@ -56,6 +65,7 @@ RSYNC_CONFIGS = [
|
|||||||
{"name": "rsync -z --zstd", "flags": ["-z", "--zc", "zstd"],"tool": "rsync"},
|
{"name": "rsync -z --zstd", "flags": ["-z", "--zc", "zstd"],"tool": "rsync"},
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
class RsyncDaemon:
|
class RsyncDaemon:
|
||||||
"""Manages an rsync daemon for network-fair benchmarking."""
|
"""Manages an rsync daemon for network-fair benchmarking."""
|
||||||
|
|
||||||
@@ -117,6 +127,12 @@ STRUCTURED_FILES = {
|
|||||||
"nested/another.txt": b"another nested file\n" * 50,
|
"nested/another.txt": b"another nested file\n" * 50,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Repeated text used to synthesize genuinely compressible filler of any size.
|
||||||
|
COMPRESSIBLE_TEXT = (
|
||||||
|
b"FastSync benchmark payload: the quick brown fox jumps over the lazy dog. "
|
||||||
|
b"0123456789 ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz\n"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class Progress:
|
class Progress:
|
||||||
"""Simple progress bar with ETA."""
|
"""Simple progress bar with ETA."""
|
||||||
@@ -151,35 +167,127 @@ class Progress:
|
|||||||
sys.stderr.flush()
|
sys.stderr.flush()
|
||||||
|
|
||||||
|
|
||||||
|
def write_compressible(path, nbytes):
|
||||||
|
"""Write exactly nbytes of highly compressible, repeated text content."""
|
||||||
|
if nbytes <= 0:
|
||||||
|
return
|
||||||
|
block = COMPRESSIBLE_TEXT * (max(1, 8192 // len(COMPRESSIBLE_TEXT)) + 1)
|
||||||
|
remaining = nbytes
|
||||||
|
with open(path, "wb") as f:
|
||||||
|
while remaining > 0:
|
||||||
|
piece = block if remaining >= len(block) else block[:remaining]
|
||||||
|
f.write(piece)
|
||||||
|
remaining -= len(piece)
|
||||||
|
|
||||||
|
|
||||||
def generate_bench_data(source_dir, size_mb=25, random_ratio=0.75):
|
def generate_bench_data(source_dir, size_mb=25, random_ratio=0.75):
|
||||||
"""Generate test data. ~random_ratio is incompressible, rest is structured."""
|
"""Generate test data honouring the requested random/compressible split.
|
||||||
|
|
||||||
|
Exactly ``random_ratio * target`` bytes are incompressible random data and
|
||||||
|
the remainder is genuinely compressible structured/repeated content. The
|
||||||
|
measured byte counts are returned so callers can report the real mix.
|
||||||
|
"""
|
||||||
if os.path.exists(source_dir):
|
if os.path.exists(source_dir):
|
||||||
shutil.rmtree(source_dir)
|
shutil.rmtree(source_dir)
|
||||||
os.makedirs(source_dir)
|
os.makedirs(source_dir)
|
||||||
|
|
||||||
target = size_mb * 1024 * 1024
|
target = size_mb * 1024 * 1024
|
||||||
structured_budget = int(target * (1 - random_ratio))
|
random_budget = int(target * random_ratio)
|
||||||
written = 0
|
compressible_budget = target - random_budget
|
||||||
|
compressible_written = 0
|
||||||
|
random_written = 0
|
||||||
|
files = 0
|
||||||
|
|
||||||
|
# A handful of fixed, human-meaningful files (directories, small files, a
|
||||||
|
# binary blob) as long as they fit inside the compressible budget.
|
||||||
for rel_path, content in STRUCTURED_FILES.items():
|
for rel_path, content in STRUCTURED_FILES.items():
|
||||||
if written >= structured_budget:
|
if compressible_written + len(content) > compressible_budget:
|
||||||
break
|
break
|
||||||
full_path = os.path.join(source_dir, rel_path)
|
full_path = os.path.join(source_dir, rel_path)
|
||||||
os.makedirs(os.path.dirname(full_path), exist_ok=True)
|
os.makedirs(os.path.dirname(full_path), exist_ok=True)
|
||||||
with open(full_path, "wb") as f:
|
with open(full_path, "wb") as f:
|
||||||
f.write(content)
|
f.write(content)
|
||||||
written += len(content)
|
compressible_written += len(content)
|
||||||
|
files += 1
|
||||||
|
|
||||||
os.makedirs(os.path.join(source_dir, "bulk"), exist_ok=True)
|
# Fill the rest of the compressible share with generated repeated content.
|
||||||
i = 0
|
if compressible_written < compressible_budget:
|
||||||
while written < target:
|
os.makedirs(os.path.join(source_dir, "compressible"), exist_ok=True)
|
||||||
chunk_size = min(5 * 1024 * 1024, target - written)
|
i = 0
|
||||||
with open(os.path.join(source_dir, f"bulk/file_{i}.dat"), "wb") as f:
|
while compressible_written < compressible_budget:
|
||||||
f.write(random.randbytes(chunk_size))
|
chunk = min(1024 * 1024, compressible_budget - compressible_written)
|
||||||
written += chunk_size
|
write_compressible(os.path.join(source_dir, "compressible", f"text_{i}.dat"), chunk)
|
||||||
i += 1
|
compressible_written += chunk
|
||||||
|
files += 1
|
||||||
|
i += 1
|
||||||
|
|
||||||
return written
|
# Incompressible share.
|
||||||
|
if random_written < random_budget:
|
||||||
|
os.makedirs(os.path.join(source_dir, "bulk"), exist_ok=True)
|
||||||
|
i = 0
|
||||||
|
while random_written < random_budget:
|
||||||
|
chunk = min(5 * 1024 * 1024, random_budget - random_written)
|
||||||
|
with open(os.path.join(source_dir, "bulk", f"file_{i}.dat"), "wb") as f:
|
||||||
|
f.write(random.randbytes(chunk))
|
||||||
|
random_written += chunk
|
||||||
|
files += 1
|
||||||
|
i += 1
|
||||||
|
|
||||||
|
return {
|
||||||
|
"total_bytes": compressible_written + random_written,
|
||||||
|
"compressible_bytes": compressible_written,
|
||||||
|
"random_bytes": random_written,
|
||||||
|
"files": files,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def list_relative_files(root):
|
||||||
|
"""Return the set of file paths (relative to root) under a directory."""
|
||||||
|
found = set()
|
||||||
|
for dirpath, _dirnames, filenames in os.walk(root):
|
||||||
|
for name in filenames:
|
||||||
|
full = os.path.join(dirpath, name)
|
||||||
|
found.add(os.path.relpath(full, root))
|
||||||
|
return found
|
||||||
|
|
||||||
|
|
||||||
|
def verify_transfer(source_dir, dest_dir):
|
||||||
|
"""Recursively check dest matches source (paths, sizes, content).
|
||||||
|
|
||||||
|
Returns (ok, detail). Content is compared byte-for-byte, never hashed, so
|
||||||
|
collisions are impossible. This is intentionally not part of the timing.
|
||||||
|
"""
|
||||||
|
if not os.path.isdir(dest_dir):
|
||||||
|
return False, "destination directory missing"
|
||||||
|
src_files = list_relative_files(source_dir)
|
||||||
|
dst_files = list_relative_files(dest_dir)
|
||||||
|
if src_files != dst_files:
|
||||||
|
missing = src_files - dst_files
|
||||||
|
extra = dst_files - src_files
|
||||||
|
return False, f"path set mismatch (missing {len(missing)}, extra {len(extra)})"
|
||||||
|
for rel in sorted(src_files):
|
||||||
|
src = os.path.join(source_dir, rel)
|
||||||
|
dst = os.path.join(dest_dir, rel)
|
||||||
|
if os.path.getsize(src) != os.path.getsize(dst):
|
||||||
|
return False, f"size mismatch: {rel}"
|
||||||
|
if not filecmp.cmp(src, dst, shallow=False):
|
||||||
|
return False, f"content mismatch: {rel}"
|
||||||
|
return True, ""
|
||||||
|
|
||||||
|
|
||||||
|
def percentile(values, pct):
|
||||||
|
"""Linear-interpolation percentile (matches numpy's default method)."""
|
||||||
|
if not values:
|
||||||
|
return None
|
||||||
|
ordered = sorted(values)
|
||||||
|
if len(ordered) == 1:
|
||||||
|
return ordered[0]
|
||||||
|
rank = (len(ordered) - 1) * (pct / 100.0)
|
||||||
|
low = math.floor(rank)
|
||||||
|
high = math.ceil(rank)
|
||||||
|
if low == high:
|
||||||
|
return ordered[int(rank)]
|
||||||
|
return ordered[low] + (ordered[high] - ordered[low]) * (rank - low)
|
||||||
|
|
||||||
|
|
||||||
def find_free_port():
|
def find_free_port():
|
||||||
@@ -207,18 +315,45 @@ def wait_proc(proc, timeout=5):
|
|||||||
proc.wait()
|
proc.wait()
|
||||||
|
|
||||||
|
|
||||||
|
def _tc_base_cmd():
|
||||||
|
"""Return the command prefix for tc, honouring root vs sudo."""
|
||||||
|
tc = shutil.which("tc")
|
||||||
|
if not tc:
|
||||||
|
raise RuntimeError(
|
||||||
|
"tc (iproute2) not found in PATH; install iproute2 to use network profiles")
|
||||||
|
if os.geteuid() == 0:
|
||||||
|
return [tc]
|
||||||
|
sudo = shutil.which("sudo")
|
||||||
|
if sudo:
|
||||||
|
return [sudo, tc]
|
||||||
|
raise RuntimeError(
|
||||||
|
"applying network limits requires root or sudo; "
|
||||||
|
"re-run as root or install sudo")
|
||||||
|
|
||||||
|
|
||||||
|
def _run_tc(args, check=True):
|
||||||
|
return subprocess.run(_tc_base_cmd() + args, check=check, capture_output=True)
|
||||||
|
|
||||||
|
|
||||||
def netem_apply(delay=None, jitter=None, throughput=None, loss=None):
|
def netem_apply(delay=None, jitter=None, throughput=None, loss=None):
|
||||||
"""Apply tc/netem rules to loopback. Pass None to skip a parameter."""
|
"""Apply tc/netem rules to loopback. Pass None to skip a parameter."""
|
||||||
netem_reset()
|
netem_reset()
|
||||||
cmd = ["sudo", "tc", "qdisc", "add", "dev", "lo", "root", "netem"]
|
params = []
|
||||||
if throughput:
|
if throughput:
|
||||||
cmd += ["rate", throughput]
|
params += ["rate", throughput]
|
||||||
if delay:
|
if delay:
|
||||||
cmd += ["delay", delay, jitter or "0ms"]
|
params += ["delay", delay, jitter or "0ms"]
|
||||||
if loss:
|
if loss:
|
||||||
cmd += ["loss", loss]
|
params += ["loss", loss]
|
||||||
if len(cmd) > 6:
|
if not params:
|
||||||
subprocess.run(cmd, check=True, capture_output=True)
|
return
|
||||||
|
try:
|
||||||
|
_run_tc(["qdisc", "add", "dev", "lo", "root", "netem"] + params)
|
||||||
|
except subprocess.CalledProcessError as exc:
|
||||||
|
detail = exc.stderr.decode(errors="replace").strip() if exc.stderr else str(exc)
|
||||||
|
raise RuntimeError(f"failed to apply network profile via tc/netem: {detail}") from exc
|
||||||
|
except RuntimeError:
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
def netem_apply_profile(profile_name):
|
def netem_apply_profile(profile_name):
|
||||||
@@ -235,7 +370,11 @@ def netem_apply_profile(profile_name):
|
|||||||
|
|
||||||
|
|
||||||
def netem_reset():
|
def netem_reset():
|
||||||
subprocess.run("sudo tc qdisc del dev lo root".split(), capture_output=True)
|
"""Best-effort removal of any loopback qdisc. Always safe to call."""
|
||||||
|
try:
|
||||||
|
_run_tc(["qdisc", "del", "dev", "lo", "root"], check=False)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
def run_fastsync(source_dir, dest_dir, flags, port):
|
def run_fastsync(source_dir, dest_dir, flags, port):
|
||||||
@@ -252,8 +391,10 @@ def run_fastsync(source_dir, dest_dir, flags, port):
|
|||||||
duration = time.monotonic() - start
|
duration = time.monotonic() - start
|
||||||
if result.returncode == 0:
|
if result.returncode == 0:
|
||||||
return duration
|
return duration
|
||||||
|
sys.stderr.write(f" fastsync failed (exit {result.returncode}): "
|
||||||
|
f"{result.stderr.strip()[:500]}\n")
|
||||||
except subprocess.TimeoutExpired:
|
except subprocess.TimeoutExpired:
|
||||||
pass
|
sys.stderr.write(" fastsync timed out after 120s\n")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
@@ -269,8 +410,10 @@ def run_rsync(source_dir, dest_dir, flags, rsync_daemon=None):
|
|||||||
duration = time.monotonic() - start
|
duration = time.monotonic() - start
|
||||||
if result.returncode == 0:
|
if result.returncode == 0:
|
||||||
return duration
|
return duration
|
||||||
|
sys.stderr.write(f" rsync failed (exit {result.returncode}): "
|
||||||
|
f"{result.stderr.strip()[:500]}\n")
|
||||||
except subprocess.TimeoutExpired:
|
except subprocess.TimeoutExpired:
|
||||||
pass
|
sys.stderr.write(" rsync timed out after 120s\n")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
@@ -282,7 +425,81 @@ def run_transfer(config, source_dir, dest_dir, port=None, rsync_daemon=None):
|
|||||||
return run_fastsync(source_dir, dest_dir, config["flags"], port)
|
return run_fastsync(source_dir, dest_dir, config["flags"], port)
|
||||||
|
|
||||||
|
|
||||||
def run_benchmark(source_dir, dest_dir, configs, runs, profile_name, progress=None):
|
def apply_incremental_changes(source_dir, target_bytes):
|
||||||
|
"""Add and modify a few files so a warm transfer has real work to do.
|
||||||
|
|
||||||
|
Returns a mutation record (changed byte count plus enough data to revert
|
||||||
|
and re-apply it) so every warm run can start from a pristine source.
|
||||||
|
"""
|
||||||
|
modified_n = 3
|
||||||
|
added_n = 2
|
||||||
|
per_file = max(4096, target_bytes // (modified_n + added_n))
|
||||||
|
modified = {}
|
||||||
|
added = {}
|
||||||
|
changed = 0
|
||||||
|
|
||||||
|
existing = sorted(list_relative_files(source_dir))
|
||||||
|
if existing:
|
||||||
|
step = max(1, len(existing) // modified_n)
|
||||||
|
for rel in existing[::step][:modified_n]:
|
||||||
|
path = os.path.join(source_dir, rel)
|
||||||
|
original_size = os.path.getsize(path)
|
||||||
|
with open(path, "ab") as f:
|
||||||
|
f.write(random.randbytes(per_file))
|
||||||
|
modified[rel] = (original_size, per_file)
|
||||||
|
changed += per_file
|
||||||
|
|
||||||
|
for i in range(added_n):
|
||||||
|
os.makedirs(os.path.join(source_dir, "incremental"), exist_ok=True)
|
||||||
|
rel = os.path.join("incremental", f"new_{i}.dat")
|
||||||
|
write_compressible(os.path.join(source_dir, rel), per_file)
|
||||||
|
added[rel] = per_file
|
||||||
|
changed += per_file
|
||||||
|
|
||||||
|
return {"changed": changed, "modified": modified, "added": added}
|
||||||
|
|
||||||
|
|
||||||
|
def revert_incremental_changes(source_dir, mutation):
|
||||||
|
"""Undo apply_incremental_changes so the source is pristine again."""
|
||||||
|
if not mutation:
|
||||||
|
return
|
||||||
|
for rel, (original_size, _appended) in mutation["modified"].items():
|
||||||
|
path = os.path.join(source_dir, rel)
|
||||||
|
if os.path.exists(path):
|
||||||
|
with open(path, "r+b") as f:
|
||||||
|
f.truncate(original_size)
|
||||||
|
for rel in mutation["added"]:
|
||||||
|
path = os.path.join(source_dir, rel)
|
||||||
|
if os.path.exists(path):
|
||||||
|
os.remove(path)
|
||||||
|
|
||||||
|
|
||||||
|
def reapply_incremental_changes(source_dir, mutation):
|
||||||
|
"""Re-apply a mutation after an untimed pristine seed transfer."""
|
||||||
|
if not mutation:
|
||||||
|
return
|
||||||
|
for rel, (_original_size, appended) in mutation["modified"].items():
|
||||||
|
with open(os.path.join(source_dir, rel), "ab") as f:
|
||||||
|
f.write(random.randbytes(appended))
|
||||||
|
for rel, size in mutation["added"].items():
|
||||||
|
write_compressible(os.path.join(source_dir, rel), size)
|
||||||
|
|
||||||
|
|
||||||
|
def expected_received_root(dest_dir, source_dir, tool):
|
||||||
|
"""Where a tool places transferred files inside dest_dir.
|
||||||
|
|
||||||
|
FastSync mirrors the absolute source path under dest_dir (see the
|
||||||
|
integration suite's get_dest_received_dir); rsync copies the source tree
|
||||||
|
contents directly into dest_dir.
|
||||||
|
"""
|
||||||
|
if tool == "rsync":
|
||||||
|
return dest_dir
|
||||||
|
return os.path.join(dest_dir, os.path.abspath(source_dir).lstrip(os.sep))
|
||||||
|
|
||||||
|
|
||||||
|
def run_benchmark(source_dir, dest_dir, configs, runs, profile_name,
|
||||||
|
measure_bytes, verify=True, warm=False, mutation=None,
|
||||||
|
progress=None):
|
||||||
"""Run benchmark for all configs, returns list of results."""
|
"""Run benchmark for all configs, returns list of results."""
|
||||||
is_limited = profile_name != "unlimited"
|
is_limited = profile_name != "unlimited"
|
||||||
has_rsync = any(c["tool"] == "rsync" for c in configs)
|
has_rsync = any(c["tool"] == "rsync" for c in configs)
|
||||||
@@ -298,7 +515,10 @@ def run_benchmark(source_dir, dest_dir, configs, runs, profile_name, progress=No
|
|||||||
results = []
|
results = []
|
||||||
for config in configs:
|
for config in configs:
|
||||||
times = []
|
times = []
|
||||||
|
invalid = 0
|
||||||
for run_idx in range(runs):
|
for run_idx in range(runs):
|
||||||
|
if warm:
|
||||||
|
revert_incremental_changes(source_dir, mutation)
|
||||||
if os.path.exists(dest_dir):
|
if os.path.exists(dest_dir):
|
||||||
shutil.rmtree(dest_dir)
|
shutil.rmtree(dest_dir)
|
||||||
os.makedirs(dest_dir, exist_ok=True)
|
os.makedirs(dest_dir, exist_ok=True)
|
||||||
@@ -306,15 +526,33 @@ def run_benchmark(source_dir, dest_dir, configs, runs, profile_name, progress=No
|
|||||||
port = find_free_port()
|
port = find_free_port()
|
||||||
server = None
|
server = None
|
||||||
try:
|
try:
|
||||||
if config["tool"] == "fastsync":
|
if config["tool"] == "fastsync" or warm:
|
||||||
server = subprocess.Popen(
|
server = subprocess.Popen(
|
||||||
SERVER_CMD + ["-p", str(port)],
|
SERVER_CMD + ["-p", str(port)],
|
||||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||||||
)
|
)
|
||||||
wait_for_port(port)
|
wait_for_port(port)
|
||||||
|
|
||||||
|
if warm:
|
||||||
|
seed = run_transfer(config, source_dir, dest_dir, port, rsync_daemon)
|
||||||
|
if seed is None:
|
||||||
|
invalid += 1
|
||||||
|
sys.stderr.write(" warm-mode seeding failed; run not counted\n")
|
||||||
|
continue
|
||||||
|
reapply_incremental_changes(source_dir, mutation)
|
||||||
|
|
||||||
t = run_transfer(config, source_dir, dest_dir, port, rsync_daemon)
|
t = run_transfer(config, source_dir, dest_dir, port, rsync_daemon)
|
||||||
if t is not None:
|
if t is None:
|
||||||
|
invalid += 1
|
||||||
|
elif verify:
|
||||||
|
root = expected_received_root(dest_dir, source_dir, config["tool"])
|
||||||
|
ok, detail = verify_transfer(source_dir, root)
|
||||||
|
if ok:
|
||||||
|
times.append(t)
|
||||||
|
else:
|
||||||
|
invalid += 1
|
||||||
|
sys.stderr.write(f" verification FAILED ({detail}); run not counted\n")
|
||||||
|
else:
|
||||||
times.append(t)
|
times.append(t)
|
||||||
finally:
|
finally:
|
||||||
if server:
|
if server:
|
||||||
@@ -327,15 +565,22 @@ def run_benchmark(source_dir, dest_dir, configs, runs, profile_name, progress=No
|
|||||||
"config": config["name"],
|
"config": config["name"],
|
||||||
"tool": config["tool"],
|
"tool": config["tool"],
|
||||||
"profile": profile_name,
|
"profile": profile_name,
|
||||||
|
"warm": warm,
|
||||||
"runs": len(times),
|
"runs": len(times),
|
||||||
|
"invalid": invalid,
|
||||||
"times": [round(t, 4) for t in times],
|
"times": [round(t, 4) for t in times],
|
||||||
}
|
}
|
||||||
if times:
|
if times:
|
||||||
entry["p50"] = round(statistics.median(times), 4)
|
p50 = percentile(times, 50)
|
||||||
entry["p95"] = round(sorted(times)[int(len(times) * 0.95)], 4) if len(times) > 1 else entry["p50"]
|
p95 = percentile(times, 95)
|
||||||
|
entry["p50"] = round(p50, 4)
|
||||||
|
entry["p95"] = round(p95, 4)
|
||||||
entry["min"] = round(min(times), 4)
|
entry["min"] = round(min(times), 4)
|
||||||
entry["max"] = round(max(times), 4)
|
entry["max"] = round(max(times), 4)
|
||||||
entry["stdev"] = round(statistics.stdev(times), 4) if len(times) > 1 else 0.0
|
entry["stdev"] = round(statistics.stdev(times), 4) if len(times) > 1 else 0.0
|
||||||
|
if measure_bytes:
|
||||||
|
entry["throughput_mbps"] = round(
|
||||||
|
(measure_bytes / (1024 * 1024)) / p50, 3)
|
||||||
results.append(entry)
|
results.append(entry)
|
||||||
return results
|
return results
|
||||||
finally:
|
finally:
|
||||||
@@ -345,44 +590,59 @@ def run_benchmark(source_dir, dest_dir, configs, runs, profile_name, progress=No
|
|||||||
netem_reset()
|
netem_reset()
|
||||||
|
|
||||||
|
|
||||||
def print_table(results, total_bytes, random_ratio):
|
def print_table(results, measure_bytes, stats, warm):
|
||||||
"""Print results as a human-readable table grouped by profile."""
|
"""Print results as a human-readable table grouped by profile."""
|
||||||
profiles = {}
|
profiles = {}
|
||||||
for r in results:
|
for r in results:
|
||||||
profiles.setdefault(r["profile"], []).append(r)
|
profiles.setdefault(r["profile"], []).append(r)
|
||||||
|
|
||||||
|
total = stats["total_bytes"]
|
||||||
|
comp_pct = stats["compressible_bytes"] / total * 100 if total else 0
|
||||||
|
rand_pct = stats["random_bytes"] / total * 100 if total else 0
|
||||||
|
|
||||||
for profile, entries in profiles.items():
|
for profile, entries in profiles.items():
|
||||||
params = NETWORK_PROFILES.get(profile, {})
|
params = NETWORK_PROFILES.get(profile, {})
|
||||||
print(f"\n{'=' * 85}")
|
print(f"\n{'=' * 95}")
|
||||||
print(f" Profile: {profile.upper()}")
|
print(f" Profile: {profile.upper()}")
|
||||||
if params.get("rate"):
|
if params.get("rate"):
|
||||||
print(f" Network: {params['rate']}, {params['delay']} +/- {params['jitter']}, loss {params['loss']}")
|
print(f" Network: {params['rate']}, {params['delay']} +/- {params['jitter']}, loss {params['loss']}")
|
||||||
else:
|
else:
|
||||||
print(f" Network: unlimited")
|
print(f" Network: unlimited")
|
||||||
print(f" Data: {total_bytes / (1024*1024):.1f} MB ({random_ratio*100:.0f}% random, {(1-random_ratio)*100:.0f}% compressible)")
|
print(f" Data: {total / (1024*1024):.1f} MB "
|
||||||
print(f"{'=' * 85}")
|
f"({rand_pct:.0f}% random, {comp_pct:.0f}% compressible actual)")
|
||||||
|
if warm:
|
||||||
|
print(f" Mode: warm (incremental) — measured {measure_bytes / (1024*1024):.2f} MB "
|
||||||
|
f"changed after an untimed full seed")
|
||||||
|
else:
|
||||||
|
print(" Mode: cold (full copy)")
|
||||||
|
print(f"{'=' * 95}")
|
||||||
|
|
||||||
fs_entries = [e for e in entries if e.get("tool") == "fastsync"]
|
fs_entries = [e for e in entries if e.get("tool") == "fastsync"]
|
||||||
rsync_entries = [e for e in entries if e.get("tool") == "rsync"]
|
rsync_entries = [e for e in entries if e.get("tool") == "rsync"]
|
||||||
|
|
||||||
|
header = (f" {'Config':<38} {'p50':>8} {'p95':>8} {'min':>8} {'max':>8} "
|
||||||
|
f"{'stdev':>8} {'MB/s':>9} {'runs':>5} {'bad':>4}")
|
||||||
|
rule = (f" {'-' * 38} {'-' * 8} {'-' * 8} {'-' * 8} {'-' * 8} "
|
||||||
|
f"{'-' * 8} {'-' * 9} {'-' * 5} {'-' * 4}")
|
||||||
|
|
||||||
if fs_entries:
|
if fs_entries:
|
||||||
print(f"\n FastSync:")
|
print(f"\n FastSync:")
|
||||||
print(f" {'Config':<25} {'p50':>8} {'p95':>8} {'min':>8} {'max':>8} {'stdev':>8} {'runs':>5}")
|
print(header)
|
||||||
print(f" {'-' * 25} {'-' * 8} {'-' * 8} {'-' * 8} {'-' * 8} {'-' * 8} {'-' * 5}")
|
print(rule)
|
||||||
for e in sorted(fs_entries, key=lambda x: x.get("p50", 999)):
|
for e in sorted(fs_entries, key=lambda x: x.get("p50", 999)):
|
||||||
_print_entry(e)
|
_print_entry(e)
|
||||||
|
|
||||||
if rsync_entries:
|
if rsync_entries:
|
||||||
print(f"\n rsync:")
|
print(f"\n rsync:")
|
||||||
print(f" {'Config':<25} {'p50':>8} {'p95':>8} {'min':>8} {'max':>8} {'stdev':>8} {'runs':>5}")
|
print(header)
|
||||||
print(f" {'-' * 25} {'-' * 8} {'-' * 8} {'-' * 8} {'-' * 8} {'-' * 8} {'-' * 5}")
|
print(rule)
|
||||||
for e in sorted(rsync_entries, key=lambda x: x.get("p50", 999)):
|
for e in sorted(rsync_entries, key=lambda x: x.get("p50", 999)):
|
||||||
_print_entry(e)
|
_print_entry(e)
|
||||||
|
|
||||||
if params.get("rate_bps") and fs_entries and rsync_entries:
|
if params.get("rate_bps") and fs_entries and rsync_entries:
|
||||||
fs_best = min((e["p50"] for e in fs_entries if "p50" in e), default=None)
|
fs_best = min((e["p50"] for e in fs_entries if "p50" in e), default=None)
|
||||||
rsync_best = min((e["p50"] for e in rsync_entries if "p50" in e), default=None)
|
rsync_best = min((e["p50"] for e in rsync_entries if "p50" in e), default=None)
|
||||||
theoretical = total_bytes / params["rate_bps"]
|
theoretical = measure_bytes / params["rate_bps"]
|
||||||
if fs_best and rsync_best:
|
if fs_best and rsync_best:
|
||||||
print(f"\n Theoretical max (line rate): {theoretical:.4f}s")
|
print(f"\n Theoretical max (line rate): {theoretical:.4f}s")
|
||||||
print(f" FastSync best: {fs_best:.4f}s ({theoretical/fs_best:.2f}x vs line rate)")
|
print(f" FastSync best: {fs_best:.4f}s ({theoretical/fs_best:.2f}x vs line rate)")
|
||||||
@@ -392,10 +652,43 @@ def print_table(results, total_bytes, random_ratio):
|
|||||||
|
|
||||||
def _print_entry(e):
|
def _print_entry(e):
|
||||||
if "p50" in e:
|
if "p50" in e:
|
||||||
print(f" {e['config']:<25} {e['p50']:>7.4f}s {e['p95']:>7.4f}s "
|
tp = f"{e['throughput_mbps']:.2f}" if "throughput_mbps" in e else "N/A"
|
||||||
f"{e['min']:>7.4f}s {e['max']:>7.4f}s {e['stdev']:>7.4f} {e['runs']:>5}")
|
print(f" {e['config']:<38} {e['p50']:>7.4f}s {e['p95']:>7.4f}s "
|
||||||
|
f"{e['min']:>7.4f}s {e['max']:>7.4f}s {e['stdev']:>7.4f} "
|
||||||
|
f"{tp:>9} {e['runs']:>5} {e.get('invalid', 0):>4}")
|
||||||
else:
|
else:
|
||||||
print(f" {e['config']:<25} {'N/A':>8} {'N/A':>8} {'N/A':>8} {'N/A':>8} {'N/A':>8} {e['runs']:>5}")
|
print(f" {e['config']:<38} {'N/A':>8} {'N/A':>8} {'N/A':>8} {'N/A':>8} "
|
||||||
|
f"{'N/A':>8} {'N/A':>9} {e['runs']:>5} {e.get('invalid', 0):>4}")
|
||||||
|
|
||||||
|
|
||||||
|
def configure_build_dirs(build_dir):
|
||||||
|
"""Install the selected build directory and derived binary paths."""
|
||||||
|
global BUILD_DIR, SERVER_CMD, CLIENT_CMD
|
||||||
|
if not os.path.isabs(build_dir):
|
||||||
|
build_dir = os.path.join(PROJECT_ROOT, build_dir)
|
||||||
|
BUILD_DIR = os.path.abspath(build_dir)
|
||||||
|
SERVER_CMD = [os.path.join(BUILD_DIR, "server"), "--allow-unauthenticated"]
|
||||||
|
CLIENT_CMD = [os.path.join(BUILD_DIR, "client")]
|
||||||
|
|
||||||
|
|
||||||
|
def build_project():
|
||||||
|
"""Configure (Release) and build into the dedicated bench build dir."""
|
||||||
|
if shutil.which("cmake") is None:
|
||||||
|
sys.stderr.write("cmake not found in PATH; cannot build\n")
|
||||||
|
sys.exit(1)
|
||||||
|
os.makedirs(BUILD_DIR, exist_ok=True)
|
||||||
|
configure = ["cmake", "-B", BUILD_DIR, "-S", PROJECT_ROOT,
|
||||||
|
"-DCMAKE_BUILD_TYPE=Release"]
|
||||||
|
result = subprocess.run(configure, capture_output=True, text=True)
|
||||||
|
if result.returncode != 0:
|
||||||
|
sys.stderr.write("CMake configure failed:\n" + result.stdout + result.stderr + "\n")
|
||||||
|
sys.exit(1)
|
||||||
|
jobs = str(os.cpu_count() or 1)
|
||||||
|
result = subprocess.run(["cmake", "--build", BUILD_DIR, "-j", jobs],
|
||||||
|
capture_output=True, text=True)
|
||||||
|
if result.returncode != 0:
|
||||||
|
sys.stderr.write("Build failed:\n" + result.stdout + result.stderr + "\n")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
@@ -412,12 +705,20 @@ Custom network limits (--delay/--jitter/--throughput) override profiles.
|
|||||||
|
|
||||||
Data mix:
|
Data mix:
|
||||||
Default is ~75%% random/incompressible + ~25%% structured/compressible,
|
Default is ~75%% random/incompressible + ~25%% structured/compressible,
|
||||||
reflecting typical real-world file sets.
|
reflecting typical real-world file sets. The actual mix is measured and
|
||||||
|
reported. Transfers are verified (destination must match source) unless
|
||||||
|
--no-verify is given.
|
||||||
|
|
||||||
|
Warm mode:
|
||||||
|
--warm seeds the destination with an untimed full copy of a pristine base,
|
||||||
|
then measures only the incremental transfer after modifying a few files.
|
||||||
|
|
||||||
Examples:
|
Examples:
|
||||||
%(prog)s --profiles wan --runs 5
|
%(prog)s --profiles wan --runs 5
|
||||||
%(prog)s --throughput 50mbit --delay 30ms --jitter 5ms
|
%(prog)s --throughput 50mbit --delay 30ms --jitter 5ms
|
||||||
%(prog)s --random-ratio 0.5 --size-mb 100
|
%(prog)s --random-ratio 0.5 --size-mb 100
|
||||||
|
%(prog)s --warm --runs 3 --no-rsync
|
||||||
|
%(prog)s --dry-run --size-mb 4 --random-ratio 0.25
|
||||||
""")
|
""")
|
||||||
parser.add_argument("--runs", type=int, default=3,
|
parser.add_argument("--runs", type=int, default=3,
|
||||||
help="Number of runs per config (default: 3)")
|
help="Number of runs per config (default: 3)")
|
||||||
@@ -425,7 +726,8 @@ Examples:
|
|||||||
choices=list(NETWORK_PROFILES.keys()),
|
choices=list(NETWORK_PROFILES.keys()),
|
||||||
help="Predefined network profiles (default: unlimited)")
|
help="Predefined network profiles (default: unlimited)")
|
||||||
parser.add_argument("--configs", nargs="+", default=None,
|
parser.add_argument("--configs", nargs="+", default=None,
|
||||||
help="Custom FastSync config flags")
|
help="Custom FastSync config flags (shell-quoted, e.g. "
|
||||||
|
"\"-j -z --chunk-serialization\")")
|
||||||
parser.add_argument("--size-mb", type=int, default=25,
|
parser.add_argument("--size-mb", type=int, default=25,
|
||||||
help="Test data size in MB (default: 25)")
|
help="Test data size in MB (default: 25)")
|
||||||
parser.add_argument("--random-ratio", type=float, default=0.75,
|
parser.add_argument("--random-ratio", type=float, default=0.75,
|
||||||
@@ -440,6 +742,14 @@ Examples:
|
|||||||
help="Custom packet loss (e.g. 1%%)")
|
help="Custom packet loss (e.g. 1%%)")
|
||||||
parser.add_argument("--no-rsync", action="store_true",
|
parser.add_argument("--no-rsync", action="store_true",
|
||||||
help="Skip rsync comparison")
|
help="Skip rsync comparison")
|
||||||
|
parser.add_argument("--no-verify", action="store_true",
|
||||||
|
help="Skip source/destination verification after each run")
|
||||||
|
parser.add_argument("--warm", action="store_true",
|
||||||
|
help="Incremental mode: seed dest first, measure only changes")
|
||||||
|
parser.add_argument("--build-dir", default=DEFAULT_BUILD_DIR,
|
||||||
|
help=f"Build directory (default: {DEFAULT_BUILD_DIR})")
|
||||||
|
parser.add_argument("--dry-run", action="store_true",
|
||||||
|
help="Only generate data and report its composition, then exit")
|
||||||
parser.add_argument("--progress", action="store_true",
|
parser.add_argument("--progress", action="store_true",
|
||||||
help="Show progress bar with ETA")
|
help="Show progress bar with ETA")
|
||||||
parser.add_argument("--output", choices=["table", "json"], default="table",
|
parser.add_argument("--output", choices=["table", "json"], default="table",
|
||||||
@@ -448,12 +758,48 @@ Examples:
|
|||||||
help="Don't clean up test data")
|
help="Don't clean up test data")
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
# Build
|
if not 0.0 <= args.random_ratio <= 1.0:
|
||||||
print("Building...")
|
parser.error("--random-ratio must be between 0.0 and 1.0")
|
||||||
if os.system(f"cmake -B {BUILD_DIR} -S {PROJECT_ROOT} > /dev/null 2>&1") != 0:
|
if args.size_mb <= 0:
|
||||||
print("CMake configure failed"); sys.exit(1)
|
parser.error("--size-mb must be positive")
|
||||||
if os.system(f"cmake --build {BUILD_DIR} -j$(nproc) > /dev/null 2>&1") != 0:
|
|
||||||
print("Build failed"); sys.exit(1)
|
configure_build_dirs(args.build_dir)
|
||||||
|
|
||||||
|
# Generate data
|
||||||
|
source_dir = os.path.join(BENCH_DIR, "source")
|
||||||
|
dest_dir = os.path.join(BENCH_DIR, "dest")
|
||||||
|
stats = generate_bench_data(source_dir, args.size_mb, args.random_ratio)
|
||||||
|
total_bytes = stats["total_bytes"]
|
||||||
|
comp_pct = stats["compressible_bytes"] / total_bytes * 100 if total_bytes else 0
|
||||||
|
rand_pct = stats["random_bytes"] / total_bytes * 100 if total_bytes else 0
|
||||||
|
print(f"Generated {total_bytes / (1024*1024):.1f} MB in {stats['files']} files "
|
||||||
|
f"({rand_pct:.0f}% random, {comp_pct:.0f}% compressible actual)",
|
||||||
|
file=sys.stderr)
|
||||||
|
|
||||||
|
if args.dry_run:
|
||||||
|
print(f"size_mb={args.size_mb} random_ratio={args.random_ratio:.4f} "
|
||||||
|
f"total_bytes={stats['total_bytes']} "
|
||||||
|
f"compressible_bytes={stats['compressible_bytes']} "
|
||||||
|
f"random_bytes={stats['random_bytes']} files={stats['files']}")
|
||||||
|
if not args.keep_data:
|
||||||
|
shutil.rmtree(BENCH_DIR, ignore_errors=True)
|
||||||
|
return
|
||||||
|
|
||||||
|
# Warm mode: keep a pristine base copy, then mutate the live source.
|
||||||
|
base_dir = None
|
||||||
|
measure_bytes = total_bytes
|
||||||
|
mutation = None
|
||||||
|
if args.warm:
|
||||||
|
change_target = max(64 * 1024, min(int(total_bytes * 0.01), 4 * 1024 * 1024))
|
||||||
|
mutation = apply_incremental_changes(source_dir, change_target)
|
||||||
|
measure_bytes = mutation["changed"]
|
||||||
|
revert_incremental_changes(source_dir, mutation)
|
||||||
|
print(f"Warm mode: each run seeds a full copy, then measures "
|
||||||
|
f"{measure_bytes / (1024*1024):.3f} MB of add/change deltas", file=sys.stderr)
|
||||||
|
|
||||||
|
# Build (Release: benchmarking a debug build is meaningless)
|
||||||
|
print(f"Building (Release) into {BUILD_DIR}...", file=sys.stderr)
|
||||||
|
build_project()
|
||||||
|
|
||||||
# Determine active profile for display
|
# Determine active profile for display
|
||||||
has_custom_net = args.delay or args.jitter or args.throughput or args.loss
|
has_custom_net = args.delay or args.jitter or args.throughput or args.loss
|
||||||
@@ -473,18 +819,10 @@ Examples:
|
|||||||
else:
|
else:
|
||||||
profiles_to_run = args.profiles or ["unlimited"]
|
profiles_to_run = args.profiles or ["unlimited"]
|
||||||
|
|
||||||
# Generate data
|
# Build config list (shlex so quoted/space-separated flags survive)
|
||||||
source_dir = os.path.join(BENCH_DIR, "source")
|
|
||||||
dest_dir = os.path.join(BENCH_DIR, "dest")
|
|
||||||
total_bytes = generate_bench_data(source_dir, args.size_mb, args.random_ratio)
|
|
||||||
compressible_pct = (1 - args.random_ratio) * 100
|
|
||||||
random_pct = args.random_ratio * 100
|
|
||||||
print(f"Generated {total_bytes / (1024*1024):.1f} MB "
|
|
||||||
f"({random_pct:.0f}% random, {compressible_pct:.0f}% compressible)")
|
|
||||||
|
|
||||||
# Build config list
|
|
||||||
if args.configs:
|
if args.configs:
|
||||||
fastsync_configs = [{"name": c, "flags": c.split(), "tool": "fastsync"} for c in args.configs]
|
fastsync_configs = [{"name": c, "flags": shlex.split(c), "tool": "fastsync"}
|
||||||
|
for c in args.configs]
|
||||||
else:
|
else:
|
||||||
fastsync_configs = list(FASTSYNC_CONFIGS)
|
fastsync_configs = list(FASTSYNC_CONFIGS)
|
||||||
|
|
||||||
@@ -496,14 +834,21 @@ Examples:
|
|||||||
total_runs = len(configs) * args.runs * len(profiles_to_run)
|
total_runs = len(configs) * args.runs * len(profiles_to_run)
|
||||||
progress = Progress(total_runs, "Benchmarking") if args.progress else None
|
progress = Progress(total_runs, "Benchmarking") if args.progress else None
|
||||||
if progress:
|
if progress:
|
||||||
print(f"Running {total_runs} transfers...")
|
print(f"Running {total_runs} transfers...", file=sys.stderr)
|
||||||
|
|
||||||
all_results = []
|
all_results = []
|
||||||
try:
|
try:
|
||||||
for profile in profiles_to_run:
|
for profile in profiles_to_run:
|
||||||
results = run_benchmark(source_dir, dest_dir, configs, args.runs, profile, progress)
|
results = run_benchmark(source_dir, dest_dir, configs, args.runs, profile,
|
||||||
|
measure_bytes, verify=not args.no_verify,
|
||||||
|
warm=args.warm, mutation=mutation,
|
||||||
|
progress=progress)
|
||||||
all_results.extend(results)
|
all_results.extend(results)
|
||||||
|
except RuntimeError as exc:
|
||||||
|
sys.stderr.write(f"error: {exc}\n")
|
||||||
|
sys.exit(1)
|
||||||
finally:
|
finally:
|
||||||
|
netem_reset()
|
||||||
if not args.keep_data:
|
if not args.keep_data:
|
||||||
shutil.rmtree(BENCH_DIR, ignore_errors=True)
|
shutil.rmtree(BENCH_DIR, ignore_errors=True)
|
||||||
|
|
||||||
@@ -511,7 +856,7 @@ Examples:
|
|||||||
if args.output == "json":
|
if args.output == "json":
|
||||||
print(json.dumps(all_results, indent=2))
|
print(json.dumps(all_results, indent=2))
|
||||||
else:
|
else:
|
||||||
print_table(all_results, total_bytes, args.random_ratio)
|
print_table(all_results, measure_bytes, stats, args.warm)
|
||||||
print()
|
print()
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -3,11 +3,35 @@
|
|||||||
}:
|
}:
|
||||||
|
|
||||||
pkgs.mkShell {
|
pkgs.mkShell {
|
||||||
|
# Development shell for FastSync. Provides the host-side toolchain needed to
|
||||||
|
# build, lint, unit-test, integration-test and benchmark the project.
|
||||||
|
# It deliberately does NOT build on entry: run the CMake commands in README.md
|
||||||
|
# (or use the CI Docker image for exact CI parity).
|
||||||
nativeBuildInputs = with pkgs; [
|
nativeBuildInputs = with pkgs; [
|
||||||
|
# build
|
||||||
gcc
|
gcc
|
||||||
cmake
|
cmake
|
||||||
gnumake
|
gnumake
|
||||||
pkg-config
|
pkg-config
|
||||||
|
# lint / static analysis (matches CI)
|
||||||
|
clang-tools # clang-format
|
||||||
|
cppcheck
|
||||||
|
# tests
|
||||||
|
(python3.withPackages (ps: with ps; [ pytest pytest-xdist psutil ]))
|
||||||
|
openssh # SSH transport integration tests
|
||||||
|
# debugging
|
||||||
|
gdb
|
||||||
|
valgrind
|
||||||
|
# coverage
|
||||||
|
lcov
|
||||||
|
# benchmark tooling
|
||||||
|
rsync
|
||||||
|
iproute2 # tc/netem for network shaping
|
||||||
|
# misc
|
||||||
|
git
|
||||||
|
curl
|
||||||
|
nodejs
|
||||||
|
nixpkgs-fmt
|
||||||
docker
|
docker
|
||||||
tea
|
tea
|
||||||
];
|
];
|
||||||
@@ -15,14 +39,21 @@ pkgs.mkShell {
|
|||||||
buildInputs = with pkgs; [
|
buildInputs = with pkgs; [
|
||||||
zstd
|
zstd
|
||||||
openssl
|
openssl
|
||||||
(python3.withPackages (ps: with ps; [ pytest ]))
|
|
||||||
];
|
];
|
||||||
|
|
||||||
|
# The CMake configure step fetches xxHash via FetchContent, which needs
|
||||||
|
# network access; NIX_ENFORCE_PURITY must be off so the sandbox does not block.
|
||||||
NIX_ENFORCE_PURITY = 0;
|
NIX_ENFORCE_PURITY = 0;
|
||||||
|
|
||||||
shellHook = ''
|
shellHook = ''
|
||||||
export NIX_ENFORCE_PURITY=0
|
export NIX_ENFORCE_PURITY=0
|
||||||
cmake -B build
|
# Make an existing build tree available on PATH, but never build here.
|
||||||
export PATH="$PWD/build:$PATH"
|
if [ -d "$PWD/build" ]; then
|
||||||
|
export PATH="$PWD/build:$PATH"
|
||||||
|
fi
|
||||||
|
echo "FastSync dev shell ready."
|
||||||
|
echo " Build: cmake -B build -S . && cmake --build build -j\$(nproc)"
|
||||||
|
echo " Unit: ./build/tests"
|
||||||
|
echo " CI parity: docker run --rm --user \"\$(id -u):\$(id -g)\" -v \"\$PWD:/workspace\" -w /workspace gitea.tap-tap.win/taptap/fastsync-ci:v10 ..."
|
||||||
'';
|
'';
|
||||||
}
|
}
|
||||||
|
|||||||
+1159
-635
File diff suppressed because it is too large
Load Diff
+409
-56
@@ -37,6 +37,33 @@
|
|||||||
|
|
||||||
#define STREAM_THRESHOLD (64ULL * 1024 * 1024)
|
#define STREAM_THRESHOLD (64ULL * 1024 * 1024)
|
||||||
|
|
||||||
|
/* Aggregate loaded payload bytes the sender may buffer across the loader queue
|
||||||
|
and the chunk in flight. Sending one chunk adds up to ~2 * MAX_CHUNK_SIZE of
|
||||||
|
transient serialize/compress buffers on top of the queued payloads, so this
|
||||||
|
ceiling keeps total pipeline memory within MAX_CONNECTION_MEMORY (mirrors the
|
||||||
|
receiver's RECEIVER_QUEUE_MAX_BYTES). */
|
||||||
|
#define SENDER_QUEUE_MAX_BYTES (MAX_CONNECTION_MEMORY - 2 * MAX_CHUNK_SIZE)
|
||||||
|
|
||||||
|
/* One mebibyte in bytes; the unit used by the --stats/--progress lines.
|
||||||
|
Always cast to double when dividing so the output stays fractional. */
|
||||||
|
#define BYTES_PER_MIB (1024ULL * 1024ULL)
|
||||||
|
|
||||||
|
/* Surface a server rejection to the user. When the last status exchange
|
||||||
|
carried a STATUS_ERROR_DETAIL reason (protocol 2.21.0) it is appended to the
|
||||||
|
client-side context; a bare STATUS_ERROR still logs the context alone. */
|
||||||
|
static void log_server_rejection(const char* context) {
|
||||||
|
const char* detail = protocol_last_error();
|
||||||
|
if (detail && detail[0] != '\0') {
|
||||||
|
/* The detail is peer-controlled: escape it so terminal/log-format
|
||||||
|
* metacharacters cannot be injected into the client's output. */
|
||||||
|
char* escaped = output_escape(detail, log_get_8_bit_output());
|
||||||
|
log_message(LOG_LEVEL_ERROR, "%s: %s", context, escaped ? escaped : "<allocation failed>");
|
||||||
|
free(escaped);
|
||||||
|
} else {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "%s", context);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/* Forward declaration for progress-reporting thread used in multithreaded send. */
|
/* Forward declaration for progress-reporting thread used in multithreaded send. */
|
||||||
static int progress_thread_fn(void* arg);
|
static int progress_thread_fn(void* arg);
|
||||||
|
|
||||||
@@ -44,10 +71,32 @@ static const char* display_bytes(unsigned long long bytes, bool human_readable,
|
|||||||
size_t buffer_size) {
|
size_t buffer_size) {
|
||||||
if (human_readable && format_human_bytes(bytes, buffer, buffer_size))
|
if (human_readable && format_human_bytes(bytes, buffer, buffer_size))
|
||||||
return buffer;
|
return buffer;
|
||||||
snprintf(buffer, buffer_size, "%.1f MB", bytes / 1048576.0);
|
snprintf(buffer, buffer_size, "%.1f MB", (double)bytes / (double)BYTES_PER_MIB);
|
||||||
return buffer;
|
return buffer;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Print the canonical `--stats` line. Shared by the single-threaded and
|
||||||
|
multithreaded send paths so both honor --stats, --human-readable and --quiet
|
||||||
|
identically; `start` marks the beginning of the transfer for the rate. */
|
||||||
|
static void report_transfer_stats(const Config* config, int total_files,
|
||||||
|
unsigned long long total_bytes, time_t start) {
|
||||||
|
if (!config->stats || config->quiet)
|
||||||
|
return;
|
||||||
|
double elapsed = difftime(time(NULL), start);
|
||||||
|
double rate = elapsed > 0.0 ? (double)total_bytes / ((double)BYTES_PER_MIB * elapsed) : 0.0;
|
||||||
|
if (config->human_readable) {
|
||||||
|
char total_buffer[32];
|
||||||
|
char rate_buffer[32];
|
||||||
|
fprintf(stderr, "Stats: %d files, %s, %s/s\n", total_files,
|
||||||
|
display_bytes(total_bytes, true, total_buffer, sizeof(total_buffer)),
|
||||||
|
display_bytes((unsigned long long)(rate * (double)BYTES_PER_MIB), true, rate_buffer,
|
||||||
|
sizeof(rate_buffer)));
|
||||||
|
} else {
|
||||||
|
fprintf(stderr, "Stats: %d files, %.1f MB, %.1f MB/s\n", total_files,
|
||||||
|
(double)total_bytes / (double)BYTES_PER_MIB, rate);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/* Compiled scanner inputs that are shared read-only across scanner instances
|
/* Compiled scanner inputs that are shared read-only across scanner instances
|
||||||
* and, in -m mode, across worker threads. `base_filters` owns the compiled
|
* and, in -m mode, across worker threads. `base_filters` owns the compiled
|
||||||
* command-line + -C rules; the FileListSet allow-set lives in the Config.
|
* command-line + -C rules; the FileListSet allow-set lives in the Config.
|
||||||
@@ -261,8 +310,11 @@ static bool files_from_list_check(const Config* config, ArrayList* missing_dest,
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
if (set->count == 0) {
|
if (set->count == 0) {
|
||||||
|
char* escaped_list =
|
||||||
|
output_escape(config->files_from ? config->files_from : "", log_get_8_bit_output());
|
||||||
log_message(LOG_LEVEL_ERROR, "--files-from file '%s' contains no entries; nothing to transfer",
|
log_message(LOG_LEVEL_ERROR, "--files-from file '%s' contains no entries; nothing to transfer",
|
||||||
config->files_from ? config->files_from : "");
|
escaped_list ? escaped_list : "<allocation failed>");
|
||||||
|
free(escaped_list);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
bool ignore = config->ignore_missing_args || config->delete_missing_args;
|
bool ignore = config->ignore_missing_args || config->delete_missing_args;
|
||||||
@@ -280,7 +332,10 @@ static bool files_from_list_check(const Config* config, ArrayList* missing_dest,
|
|||||||
free(full);
|
free(full);
|
||||||
if (ignore) {
|
if (ignore) {
|
||||||
(*skipped_out)++;
|
(*skipped_out)++;
|
||||||
log_info_message(LOG_INFO_MISC, "skipping missing --files-from entry '%s'", entry);
|
char* escaped_entry = output_escape(entry, log_get_8_bit_output());
|
||||||
|
log_info_message(LOG_INFO_MISC, "skipping missing --files-from entry '%s'",
|
||||||
|
escaped_entry ? escaped_entry : "<allocation failed>");
|
||||||
|
free(escaped_entry);
|
||||||
if (config->delete_missing_args && missing_dest) {
|
if (config->delete_missing_args && missing_dest) {
|
||||||
char* mirror = files_from_missing_dest_path(config, entry);
|
char* mirror = files_from_missing_dest_path(config, entry);
|
||||||
if (!mirror || !array_list_add(missing_dest, mirror)) {
|
if (!mirror || !array_list_add(missing_dest, mirror)) {
|
||||||
@@ -291,8 +346,13 @@ static bool files_from_list_check(const Config* config, ArrayList* missing_dest,
|
|||||||
}
|
}
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
log_message(LOG_LEVEL_ERROR, "--files-from entry '%s' not found in source '%s'", entry,
|
char* escaped_entry = output_escape(entry, log_get_8_bit_output());
|
||||||
config->send_directory);
|
char* escaped_src = output_escape(config->send_directory, log_get_8_bit_output());
|
||||||
|
log_message(LOG_LEVEL_ERROR, "--files-from entry '%s' not found in source '%s'",
|
||||||
|
escaped_entry ? escaped_entry : "<allocation failed>",
|
||||||
|
escaped_src ? escaped_src : "<allocation failed>");
|
||||||
|
free(escaped_entry);
|
||||||
|
free(escaped_src);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
free(full);
|
free(full);
|
||||||
@@ -338,9 +398,10 @@ static bool basis_oversize_preflight(const Config* config) {
|
|||||||
return false;
|
return false;
|
||||||
DirectoryScanner* scanner =
|
DirectoryScanner* scanner =
|
||||||
directory_scanner_create_with_options(config->send_directory, &prepared.options);
|
directory_scanner_create_with_options(config->send_directory, &prepared.options);
|
||||||
prepared_scanner_destroy(&prepared);
|
if (!scanner) {
|
||||||
if (!scanner)
|
prepared_scanner_destroy(&prepared);
|
||||||
return false;
|
return false;
|
||||||
|
}
|
||||||
bool ok = true;
|
bool ok = true;
|
||||||
Chunk* chunk;
|
Chunk* chunk;
|
||||||
while ((chunk = directory_scanner_next(scanner)) != NULL) {
|
while ((chunk = directory_scanner_next(scanner)) != NULL) {
|
||||||
@@ -364,7 +425,10 @@ static bool basis_oversize_preflight(const Config* config) {
|
|||||||
}
|
}
|
||||||
if (directory_scanner_failed(scanner) || directory_scanner_had_io_error(scanner))
|
if (directory_scanner_failed(scanner) || directory_scanner_had_io_error(scanner))
|
||||||
ok = false;
|
ok = false;
|
||||||
|
/* The scanner borrows prepared.options' base_filters/hardlinks pointers, so
|
||||||
|
prepared must outlive the scanner. */
|
||||||
directory_scanner_destroy(scanner);
|
directory_scanner_destroy(scanner);
|
||||||
|
prepared_scanner_destroy(&prepared);
|
||||||
return ok;
|
return ok;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -444,6 +508,28 @@ static void disconnect_transfer_client(Client* client) {
|
|||||||
client_delete(client);
|
client_delete(client);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* True when --dry-run should contact a receiver rather than running the
|
||||||
|
* client-side local manifest. Any target a real run would reach over the wire
|
||||||
|
* selects the server-contacting path: a remote (SSH host:path), a daemon
|
||||||
|
* (host::module/path), an explicit --server-host, --server-port/--port, TLS, or
|
||||||
|
* a source-bind --address. A plain local destination (none of these) keeps the
|
||||||
|
* original client-side behavior, which never dials the default 127.0.0.1:8080. */
|
||||||
|
static bool dry_run_targets_server(const Config* config) {
|
||||||
|
if (!config)
|
||||||
|
return false;
|
||||||
|
if (config->transport == TRANSPORT_SSH)
|
||||||
|
return true;
|
||||||
|
if (config->module && config->module[0] != '\0')
|
||||||
|
return true;
|
||||||
|
if (config->server_host_set || config->server_port_set)
|
||||||
|
return true;
|
||||||
|
if (config->use_tls)
|
||||||
|
return true;
|
||||||
|
if (config->address != NULL)
|
||||||
|
return true;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
static bool add_chunk_to_manifest(ArrayList* manifest, const Chunk* chunk) {
|
static bool add_chunk_to_manifest(ArrayList* manifest, const Chunk* chunk) {
|
||||||
if (!manifest)
|
if (!manifest)
|
||||||
return true;
|
return true;
|
||||||
@@ -513,8 +599,12 @@ static void remove_transferred_sources(const Config* config, ArrayList* paths) {
|
|||||||
close(dirfd);
|
close(dirfd);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if (unlinkat(dirfd, leaf, 0) != 0)
|
if (unlinkat(dirfd, leaf, 0) != 0) {
|
||||||
log_message(LOG_LEVEL_WARNING, "Could not remove source file %s", source->path);
|
char* escaped_path = output_escape(source->path, log_get_8_bit_output());
|
||||||
|
log_message(LOG_LEVEL_WARNING, "Could not remove source file %s",
|
||||||
|
escaped_path ? escaped_path : "<allocation failed>");
|
||||||
|
free(escaped_path);
|
||||||
|
}
|
||||||
close(dirfd);
|
close(dirfd);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -571,8 +661,10 @@ static bool finalize_transfer(Client* client, const Config* config, ArrayList* r
|
|||||||
Status per_file;
|
Status per_file;
|
||||||
if (!receive_status(client->file_descriptor, &per_file))
|
if (!receive_status(client->file_descriptor, &per_file))
|
||||||
return false;
|
return false;
|
||||||
if (per_file == STATUS_ERROR)
|
if (per_file == STATUS_ERROR) {
|
||||||
|
log_server_rejection("Receiver reported a per-file error");
|
||||||
return false;
|
return false;
|
||||||
|
}
|
||||||
if (per_file == STATUS_OK) {
|
if (per_file == STATUS_OK) {
|
||||||
((SourceFile*)remove_sources->items[i])->skipped = true;
|
((SourceFile*)remove_sources->items[i])->skipped = true;
|
||||||
} else if (per_file != STATUS_NEXT) {
|
} else if (per_file != STATUS_NEXT) {
|
||||||
@@ -582,7 +674,13 @@ static bool finalize_transfer(Client* client, const Config* config, ArrayList* r
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
Status status;
|
Status status;
|
||||||
return receive_status(client->file_descriptor, &status) && status == STATUS_OK;
|
if (!receive_status(client->file_descriptor, &status))
|
||||||
|
return false;
|
||||||
|
if (status != STATUS_OK) {
|
||||||
|
log_server_rejection("Receiver reported transfer failure");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
static void pipeline_cancel(PipelineContextSender* context) {
|
static void pipeline_cancel(PipelineContextSender* context) {
|
||||||
@@ -677,7 +775,7 @@ static int send_dry_run_manifest(const Config* config) {
|
|||||||
printf("Total: %d files, %s\n", file_count,
|
printf("Total: %d files, %s\n", file_count,
|
||||||
display_bytes(total_bytes, true, size_buffer, sizeof(size_buffer)));
|
display_bytes(total_bytes, true, size_buffer, sizeof(size_buffer)));
|
||||||
else
|
else
|
||||||
printf("Total: %d files, %.1f MB\n", file_count, total_bytes / 1048576.0);
|
printf("Total: %d files, %.1f MB\n", file_count, (double)total_bytes / (double)BYTES_PER_MIB);
|
||||||
}
|
}
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
@@ -848,6 +946,10 @@ static int send_delete_manifest(int fd, ArrayList* manifest, ArrayList* protecte
|
|||||||
unlinks) before replying, so the wait uses a generous explicit deadline
|
unlinks) before replying, so the wait uses a generous explicit deadline
|
||||||
instead of the default 60 s receive window. */
|
instead of the default 60 s receive window. */
|
||||||
#define DELETE_ACK_TIMEOUT_SEC 3600
|
#define DELETE_ACK_TIMEOUT_SEC 3600
|
||||||
|
/* While waiting for the (potentially slow) receiver-side deletion, send a
|
||||||
|
* STATUS_KEEPALIVE at most this often so the connection is demonstrably alive
|
||||||
|
* and neither side's per-message timeout trips. */
|
||||||
|
#define DELETE_ACK_KEEPALIVE_SEC 10
|
||||||
|
|
||||||
static bool send_delete_manifest_early(Client* client, ArrayList* manifest,
|
static bool send_delete_manifest_early(Client* client, ArrayList* manifest,
|
||||||
ArrayList* protected_prefixes, ArrayList* missing_args) {
|
ArrayList* protected_prefixes, ArrayList* missing_args) {
|
||||||
@@ -857,10 +959,23 @@ static bool send_delete_manifest_early(Client* client, ArrayList* manifest,
|
|||||||
0)
|
0)
|
||||||
return false;
|
return false;
|
||||||
Status ack;
|
Status ack;
|
||||||
if (!receive_status_timed(client->file_descriptor, &ack, DELETE_ACK_TIMEOUT_SEC))
|
/* The wait is long (up to an hour) and runs inline on this thread: a helper
|
||||||
|
* thread would race the non-thread-safe protocol send path, so keepalives are
|
||||||
|
* emitted from this wait loop itself. A Ctrl-C/SIGTERM abort flag also ends
|
||||||
|
* the wait; the caller then best-effort sends STATUS_ABORT. */
|
||||||
|
if (!receive_status_keepalive(client->file_descriptor, &ack, DELETE_ACK_TIMEOUT_SEC,
|
||||||
|
DELETE_ACK_KEEPALIVE_SEC, client_abort_pending)) {
|
||||||
|
/* A Ctrl-C/SIGTERM abort ends the wait above; tell the receiver before the
|
||||||
|
caller tears the connection down (best-effort). */
|
||||||
|
if (client_abort_pending()) {
|
||||||
|
log_info_message(LOG_INFO_MISC,
|
||||||
|
"Abort requested while awaiting delete ack; sending STATUS_ABORT");
|
||||||
|
send_status(client->file_descriptor, STATUS_ABORT);
|
||||||
|
}
|
||||||
return false;
|
return false;
|
||||||
|
}
|
||||||
if (ack != STATUS_OK) {
|
if (ack != STATUS_OK) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Server failed to delete files before the transfer");
|
log_server_rejection("Server failed to delete files before the transfer");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
return true;
|
return true;
|
||||||
@@ -937,7 +1052,7 @@ static int incremental_check(Client* client, File* file, const Config* config,
|
|||||||
if (!receive_status(client->file_descriptor, &s))
|
if (!receive_status(client->file_descriptor, &s))
|
||||||
return -1;
|
return -1;
|
||||||
if (s == STATUS_ERROR) {
|
if (s == STATUS_ERROR) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Server reported error for file");
|
log_server_rejection("Server reported error for file");
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
if (s == STATUS_OK)
|
if (s == STATUS_OK)
|
||||||
@@ -970,8 +1085,21 @@ static int incremental_check(Client* client, File* file, const Config* config,
|
|||||||
*resume_offset = offset;
|
*resume_offset = offset;
|
||||||
return 3;
|
return 3;
|
||||||
}
|
}
|
||||||
|
/* Server-contacting --dry-run: the receiver decided the file is not up to
|
||||||
|
date and answered "would transfer" WITHOUT expecting any data. Treat it as
|
||||||
|
the dry-run code ONLY when this session actually requested dry-run. A
|
||||||
|
hostile/buggy peer that emits it outside dry-run is a protocol error: fail
|
||||||
|
closed (and send STATUS_ERROR) rather than fall through to the normal path,
|
||||||
|
which would transmit file data the receiver is not reading and desync. */
|
||||||
|
if (s == STATUS_DRY_RUN_TRANSFER) {
|
||||||
|
if (config->dry_run)
|
||||||
|
return 4;
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Unexpected DRY_RUN_TRANSFER status outside a --dry-run session");
|
||||||
|
send_status(client->file_descriptor, STATUS_ERROR);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
if (s != STATUS_NEXT) {
|
if (s != STATUS_NEXT) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Unexpected server status");
|
log_server_rejection("Unexpected server status");
|
||||||
send_status(client->file_descriptor, STATUS_ERROR);
|
send_status(client->file_descriptor, STATUS_ERROR);
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
@@ -1065,6 +1193,7 @@ static int send_append(const Client* client, File* file, Config* config,
|
|||||||
return rc;
|
return rc;
|
||||||
}
|
}
|
||||||
if (resp != STATUS_APPEND_OK) {
|
if (resp != STATUS_APPEND_OK) {
|
||||||
|
log_server_rejection("Unexpected append-verify response");
|
||||||
send_status(fd, STATUS_ERROR);
|
send_status(fd, STATUS_ERROR);
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
@@ -1103,11 +1232,180 @@ static int send_append(const Client* client, File* file, Config* config,
|
|||||||
tail_view.data = (char*)file->data->data + off;
|
tail_view.data = (char*)file->data->data + off;
|
||||||
tail_view.size = tail_len;
|
tail_view.size = tail_len;
|
||||||
tail_view.protocol_charge = 0;
|
tail_view.protocol_charge = 0;
|
||||||
|
tail_view.owner = NULL;
|
||||||
ok = send_data(fd, &tail_view);
|
ok = send_data(fd, &tail_view);
|
||||||
}
|
}
|
||||||
return ok ? 0 : -1;
|
return ok ? 0 : -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Server-contacting --dry-run. Connects to the configured remote/daemon and
|
||||||
|
* runs the normal per-file incremental decision WITHOUT transmitting any file
|
||||||
|
* data: the receiver (which also sees dry_run=true on the wire) answers
|
||||||
|
* STATUS_OK for an up-to-date file and STATUS_DRY_RUN_TRANSFER for a file it
|
||||||
|
* would otherwise write, mutating nothing on either side. The would-transfer
|
||||||
|
* set and the same trailer as the local dry-run are printed. A
|
||||||
|
* --compare-dest exact basis hit with no destination copy is reported as a
|
||||||
|
* skip by the receiver.
|
||||||
|
*
|
||||||
|
* Only regular files take the receiver-consulted check; directory / symlink /
|
||||||
|
* special / hard-link-sibling entries have no per-file content check, so they
|
||||||
|
* are reported conservatively as would-transfer and their frames are never
|
||||||
|
* sent (which is what keeps the receiver mutation-free). --delete* is
|
||||||
|
* deliberately NOT transmitted in dry-run, so no deletion can occur; the
|
||||||
|
* would-delete manifest report is a documented follow-up.
|
||||||
|
*
|
||||||
|
* Returns 0 on success, 1 on error. */
|
||||||
|
static int send_dry_run_remote(Config* config) {
|
||||||
|
int from_skipped = 0;
|
||||||
|
ArrayList* missing_args = NULL;
|
||||||
|
if (config->delete_missing_args) {
|
||||||
|
missing_args = array_list_create(free);
|
||||||
|
if (!missing_args)
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
if (!files_from_list_check(config, missing_args, &from_skipped)) {
|
||||||
|
if (missing_args)
|
||||||
|
array_list_delete(missing_args);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
if (missing_args)
|
||||||
|
array_list_delete(missing_args);
|
||||||
|
/* Alternate basis dirs force the whole-file per-file check on the real
|
||||||
|
receiver; refuse an oversize source up front exactly as send_files does so
|
||||||
|
dry-run reports the same clear diagnostic instead of aborting mid-stream. */
|
||||||
|
if (config_has_basis(config) && !basis_oversize_preflight(config))
|
||||||
|
return 1;
|
||||||
|
/* Would-delete reporting requires a receiver-side read-only extras walk that
|
||||||
|
is not implemented yet; be explicit that --delete is a no-op in dry-run
|
||||||
|
rather than silently ignoring it. */
|
||||||
|
if ((config->use_delete || config->delete_missing_args) && !config->quiet)
|
||||||
|
log_message(LOG_LEVEL_WARNING,
|
||||||
|
"--dry-run: would-delete reporting is not available in this release; nothing is "
|
||||||
|
"deleted");
|
||||||
|
|
||||||
|
/* A live session may follow, so arm graceful abort handling. */
|
||||||
|
client_set_abort_armed(true);
|
||||||
|
Client* client = connect_transfer_client(config);
|
||||||
|
if (!client) {
|
||||||
|
if (config->transport == TRANSPORT_TCP)
|
||||||
|
log_message(LOG_LEVEL_ERROR, "could not connect to server%s",
|
||||||
|
config->use_tls ? " via TLS" : "");
|
||||||
|
client_set_abort_armed(false);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
ProtocolSession session;
|
||||||
|
protocol_session_init(&session, client->file_descriptor, client->file_descriptor);
|
||||||
|
protocol_session_set_io_timeout(&session, config->timeout);
|
||||||
|
protocol_session_set_ssl(&session, (SSL*)client->ssl);
|
||||||
|
protocol_session_bind(&session);
|
||||||
|
|
||||||
|
int ret = 1;
|
||||||
|
PreparedScanner prepared;
|
||||||
|
memset(&prepared, 0, sizeof(prepared));
|
||||||
|
DirectoryScanner* scanner = NULL;
|
||||||
|
if (!config_send(client->file_descriptor, config))
|
||||||
|
goto dry_fail;
|
||||||
|
receive_daemon_motd(client, config);
|
||||||
|
if (!prepare_scanner(config, 0, &prepared))
|
||||||
|
goto dry_fail;
|
||||||
|
scanner = directory_scanner_create_with_options(config->send_directory, &prepared.options);
|
||||||
|
if (!scanner)
|
||||||
|
goto dry_fail;
|
||||||
|
|
||||||
|
int file_count = 0;
|
||||||
|
unsigned long long total_bytes = 0;
|
||||||
|
char size_buffer[32];
|
||||||
|
if (!config->quiet)
|
||||||
|
printf("Dry run: files to be transferred\n");
|
||||||
|
Chunk* chunk;
|
||||||
|
while ((chunk = directory_scanner_next(scanner)) != NULL) {
|
||||||
|
for (int i = 0; i < chunk->element_count; i++) {
|
||||||
|
File* f = chunk->items[i];
|
||||||
|
if (!f)
|
||||||
|
continue;
|
||||||
|
unsigned long long fsize = f->data ? f->data->size : 0;
|
||||||
|
bool would;
|
||||||
|
if (f->is_dir || f->is_symlink || f->is_special ||
|
||||||
|
(f->link_group != 0 && !f->link_first && f->hardlink_target != NULL)) {
|
||||||
|
/* No receiver-side content check exists for these frame types; a real
|
||||||
|
run would (re)create them, so report would-transfer and send no
|
||||||
|
frame (the receiver must stay mutation-free). */
|
||||||
|
would = true;
|
||||||
|
} else if (fsize > MAX_RECEIVE_WHOLE_FILE_SIZE && !config->use_incremental &&
|
||||||
|
!config_has_basis(config)) {
|
||||||
|
/* A non-incremental run streams a >whole-file-limit source without the
|
||||||
|
STATUS_CHECK handshake, so no read-only receiver decision is possible
|
||||||
|
(and none is needed: a real run would transfer it). */
|
||||||
|
would = true;
|
||||||
|
} else {
|
||||||
|
DeltaSignature* sig = NULL;
|
||||||
|
unsigned long long resume_offset = 0;
|
||||||
|
int rc = incremental_check(client, f, config, &sig, &resume_offset);
|
||||||
|
delta_signature_destroy(sig);
|
||||||
|
if (rc < 0) {
|
||||||
|
chunk_destroy(chunk);
|
||||||
|
goto dry_fail;
|
||||||
|
}
|
||||||
|
if (rc == 1)
|
||||||
|
continue; /* up to date; nothing to report */
|
||||||
|
if (rc != 4) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Unexpected receiver reply during dry-run");
|
||||||
|
chunk_destroy(chunk);
|
||||||
|
goto dry_fail;
|
||||||
|
}
|
||||||
|
would = true;
|
||||||
|
}
|
||||||
|
if (would) {
|
||||||
|
if (!config->quiet) {
|
||||||
|
char* escaped_path = output_escape(file_wire_path(f), config->eight_bit_output);
|
||||||
|
if (!escaped_path) {
|
||||||
|
chunk_destroy(chunk);
|
||||||
|
goto dry_fail;
|
||||||
|
}
|
||||||
|
if (config->human_readable)
|
||||||
|
printf(" %s (%s)\n", escaped_path,
|
||||||
|
display_bytes(fsize, true, size_buffer, sizeof(size_buffer)));
|
||||||
|
else
|
||||||
|
printf(" %s (%llu bytes)\n", escaped_path, fsize);
|
||||||
|
free(escaped_path);
|
||||||
|
}
|
||||||
|
total_bytes += fsize;
|
||||||
|
file_count++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
chunk_destroy(chunk);
|
||||||
|
}
|
||||||
|
bool io_error = directory_scanner_had_io_error(scanner);
|
||||||
|
if (directory_scanner_failed(scanner))
|
||||||
|
goto dry_fail;
|
||||||
|
if (io_error)
|
||||||
|
log_message(LOG_LEVEL_WARNING, "source scan hit an unreadable directory");
|
||||||
|
/* Terminate the stream so the receiver emits its success frame; no data
|
||||||
|
frame and no delete manifest are ever sent in dry-run. */
|
||||||
|
if (!send_status(client->file_descriptor, STATUS_FINISHED))
|
||||||
|
goto dry_fail;
|
||||||
|
Status status;
|
||||||
|
if (!receive_status(client->file_descriptor, &status) || status != STATUS_OK)
|
||||||
|
goto dry_fail;
|
||||||
|
if (!config->quiet) {
|
||||||
|
if (config->human_readable)
|
||||||
|
printf("Total: %d files, %s\n", file_count,
|
||||||
|
display_bytes(total_bytes, true, size_buffer, sizeof(size_buffer)));
|
||||||
|
else
|
||||||
|
printf("Total: %d files, %.1f MB\n", file_count, (double)total_bytes / (double)BYTES_PER_MIB);
|
||||||
|
}
|
||||||
|
ret = io_error ? 1 : 0;
|
||||||
|
|
||||||
|
dry_fail:
|
||||||
|
if (scanner)
|
||||||
|
directory_scanner_destroy(scanner);
|
||||||
|
prepared_scanner_destroy(&prepared);
|
||||||
|
disconnect_transfer_client(client);
|
||||||
|
protocol_session_unbind();
|
||||||
|
client_set_abort_armed(false);
|
||||||
|
return ret;
|
||||||
|
}
|
||||||
|
|
||||||
// Send a single file directly (non-incremental path).
|
// Send a single file directly (non-incremental path).
|
||||||
static bool send_file_direct(File* file, int fd, bool use_metadata, int compression_level,
|
static bool send_file_direct(File* file, int fd, bool use_metadata, int compression_level,
|
||||||
const Config* config) {
|
const Config* config) {
|
||||||
@@ -1234,6 +1532,16 @@ static int send_single_file(Client* client, File* file, Config* config, bool use
|
|||||||
}
|
}
|
||||||
return arc == 0 ? 0 : -1;
|
return arc == 0 ? 0 : -1;
|
||||||
}
|
}
|
||||||
|
// rc == 4: the receiver answered DRY_RUN_TRANSFER, which is only valid in
|
||||||
|
// incremental_check's dedicated dry-run consumer. send_single_file never
|
||||||
|
// runs a dry-run session, so this is a protocol error: abort instead of
|
||||||
|
// falling through and sending data the receiver is not reading.
|
||||||
|
if (rc == 4) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Receiver answered DRY_RUN_TRANSFER in a non-dry-run transfer");
|
||||||
|
delta_signature_destroy(sig);
|
||||||
|
send_status(client->file_descriptor, STATUS_ERROR);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
// rc == 0: unchanged file, skip
|
// rc == 0: unchanged file, skip
|
||||||
// rc == 2: server sent delta signature but sendfile doesn't support delta
|
// rc == 2: server sent delta signature but sendfile doesn't support delta
|
||||||
delta_signature_destroy(sig);
|
delta_signature_destroy(sig);
|
||||||
@@ -1275,6 +1583,14 @@ static int send_single_file(Client* client, File* file, Config* config, bool use
|
|||||||
}
|
}
|
||||||
return arc == 0 ? 0 : -1;
|
return arc == 0 ? 0 : -1;
|
||||||
}
|
}
|
||||||
|
if (rc == 4) {
|
||||||
|
/* See the sendfile branch above: DRY_RUN_TRANSFER is only valid in the
|
||||||
|
dedicated dry-run consumer, never in the normal per-file send path. */
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Receiver answered DRY_RUN_TRANSFER in a non-dry-run transfer");
|
||||||
|
delta_signature_destroy(sig);
|
||||||
|
send_status(client->file_descriptor, STATUS_ERROR);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
if (rc == 2 && config->use_delta && !config->whole_file) {
|
if (rc == 2 && config->use_delta && !config->whole_file) {
|
||||||
int drc = send_delta(client, file, sig, config);
|
int drc = send_delta(client, file, sig, config);
|
||||||
delta_signature_destroy(sig);
|
delta_signature_destroy(sig);
|
||||||
@@ -1414,12 +1730,9 @@ static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config,
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
int send_chunk(Client* client, Chunk* chunk, Config* config) {
|
|
||||||
return send_chunk_with_removal(client, chunk, config, NULL);
|
|
||||||
}
|
|
||||||
|
|
||||||
static int send_chunks_multithreaded(void* pipeline_context) {
|
static int send_chunks_multithreaded(void* pipeline_context) {
|
||||||
PipelineContextSender* context = (PipelineContextSender*)pipeline_context;
|
PipelineContextSender* context = (PipelineContextSender*)pipeline_context;
|
||||||
|
time_t start = time(NULL);
|
||||||
Client* client = connect_transfer_client(context->config);
|
Client* client = connect_transfer_client(context->config);
|
||||||
if (!client) {
|
if (!client) {
|
||||||
if (context->config->transport == TRANSPORT_TCP)
|
if (context->config->transport == TRANSPORT_TCP)
|
||||||
@@ -1431,6 +1744,7 @@ static int send_chunks_multithreaded(void* pipeline_context) {
|
|||||||
}
|
}
|
||||||
ProtocolSession session;
|
ProtocolSession session;
|
||||||
protocol_session_init(&session, client->file_descriptor, client->file_descriptor);
|
protocol_session_init(&session, client->file_descriptor, client->file_descriptor);
|
||||||
|
protocol_session_set_io_timeout(&session, context->config->timeout);
|
||||||
protocol_session_set_ssl(&session, (SSL*)client->ssl);
|
protocol_session_set_ssl(&session, (SSL*)client->ssl);
|
||||||
protocol_session_bind(&session);
|
protocol_session_bind(&session);
|
||||||
if (!config_send(client->file_descriptor, context->config)) {
|
if (!config_send(client->file_descriptor, context->config)) {
|
||||||
@@ -1455,6 +1769,19 @@ static int send_chunks_multithreaded(void* pipeline_context) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
while (true) {
|
while (true) {
|
||||||
|
/* Graceful abort (Ctrl-C/SIGTERM): tell the receiver to clean up instead of
|
||||||
|
dying abruptly. Best-effort: a failed send just means the peer is gone.
|
||||||
|
Only reached while the session is active (config_send already succeeded). */
|
||||||
|
if (client_abort_pending()) {
|
||||||
|
log_info_message(LOG_INFO_MISC,
|
||||||
|
"Abort requested; sending STATUS_ABORT to server and disconnecting");
|
||||||
|
send_status(client->file_descriptor, STATUS_ABORT);
|
||||||
|
pipeline_cancel(context);
|
||||||
|
disconnect_transfer_client(client);
|
||||||
|
mark_sender_done(context);
|
||||||
|
protocol_session_unbind();
|
||||||
|
return thrd_error;
|
||||||
|
}
|
||||||
/* Phase 6: stop-elegantly at the next chunk boundary once the --stop-after
|
/* Phase 6: stop-elegantly at the next chunk boundary once the --stop-after
|
||||||
/ --stop-at deadline has passed. Everything already sent is finalized by
|
/ --stop-at deadline has passed. Everything already sent is finalized by
|
||||||
the completion tail below; the run still returns success. */
|
the completion tail below; the run still returns success. */
|
||||||
@@ -1488,6 +1815,10 @@ static int send_chunks_multithreaded(void* pipeline_context) {
|
|||||||
protocol_session_unbind();
|
protocol_session_unbind();
|
||||||
return thrd_error;
|
return thrd_error;
|
||||||
}
|
}
|
||||||
|
/* Payload bytes this chunk was charged for on the loader's byte budget.
|
||||||
|
Computed before destruction and released after the memory is actually
|
||||||
|
freed, so a loader blocked on the budget wakes only once room exists. */
|
||||||
|
size_t queued_payload = pipeline_context_sender_chunk_bytes(current_chunk);
|
||||||
unsigned long long chunk_bytes = 0;
|
unsigned long long chunk_bytes = 0;
|
||||||
int chunk_files = 0;
|
int chunk_files = 0;
|
||||||
for (int i = 0; i < current_chunk->element_count; i++) {
|
for (int i = 0; i < current_chunk->element_count; i++) {
|
||||||
@@ -1502,6 +1833,7 @@ static int send_chunks_multithreaded(void* pipeline_context) {
|
|||||||
context->progress_bytes = context->total_bytes;
|
context->progress_bytes = context->total_bytes;
|
||||||
mtx_unlock(&context->mutex_progress);
|
mtx_unlock(&context->mutex_progress);
|
||||||
chunk_destroy(current_chunk);
|
chunk_destroy(current_chunk);
|
||||||
|
pipeline_context_sender_note_bytes_released(context, queued_payload);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Completion tail: reached on natural exhaustion or an early stop deadline.
|
/* Completion tail: reached on natural exhaustion or an early stop deadline.
|
||||||
@@ -1561,10 +1893,9 @@ static int send_chunks_multithreaded(void* pipeline_context) {
|
|||||||
int total_files = context->total_files;
|
int total_files = context->total_files;
|
||||||
unsigned long long total_bytes = context->total_bytes;
|
unsigned long long total_bytes = context->total_bytes;
|
||||||
mtx_unlock(&context->mutex_progress);
|
mtx_unlock(&context->mutex_progress);
|
||||||
if (context->config->stats)
|
report_transfer_stats(context->config, total_files, total_bytes, start);
|
||||||
fprintf(stderr, "Stats: %d files, %.1f MB\n", total_files, total_bytes / 1048576.0);
|
|
||||||
log_info_message(LOG_INFO_STATS, "Transfer summary: %d files, %.1f MB", total_files,
|
log_info_message(LOG_INFO_STATS, "Transfer summary: %d files, %.1f MB", total_files,
|
||||||
total_bytes / 1048576.0);
|
(double)total_bytes / (double)BYTES_PER_MIB);
|
||||||
disconnect_transfer_client(client);
|
disconnect_transfer_client(client);
|
||||||
mark_sender_done(context);
|
mark_sender_done(context);
|
||||||
protocol_session_unbind();
|
protocol_session_unbind();
|
||||||
@@ -1585,7 +1916,9 @@ static int scan_directory_multithreaded(void* pipeline_context) {
|
|||||||
PipelineContextSender* context = (PipelineContextSender*)pipeline_context;
|
PipelineContextSender* context = (PipelineContextSender*)pipeline_context;
|
||||||
protocol_session_bind(&context->allocation_session);
|
protocol_session_bind(&context->allocation_session);
|
||||||
PreparedScanner prepared;
|
PreparedScanner prepared;
|
||||||
if (!prepare_scanner(context->config, 4, &prepared)) {
|
/* -j/--threads=N sizes the parallel scanner's worker pool; 0 (bare -j) lets
|
||||||
|
* the scanner apply its built-in default. */
|
||||||
|
if (!prepare_scanner(context->config, context->config->scanner_threads, &prepared)) {
|
||||||
pipeline_cancel(context);
|
pipeline_cancel(context);
|
||||||
protocol_session_unbind();
|
protocol_session_unbind();
|
||||||
return thrd_error;
|
return thrd_error;
|
||||||
@@ -1723,11 +2056,7 @@ static int load_files_multithreaded(void* pipeline_context) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (!queue_enqueue_multithreaded_cancel(context->queue_loader, chunk, &context->mutex_loader,
|
if (!pipeline_context_sender_enqueue_chunk(context, chunk)) {
|
||||||
&context->condition_not_empty_loader,
|
|
||||||
&context->condition_not_full_loader,
|
|
||||||
&context->cancelled)) {
|
|
||||||
chunk_destroy(chunk);
|
|
||||||
pipeline_cancel(context);
|
pipeline_cancel(context);
|
||||||
protocol_session_unbind();
|
protocol_session_unbind();
|
||||||
return thrd_error;
|
return thrd_error;
|
||||||
@@ -1741,17 +2070,18 @@ static int load_files_multithreaded(void* pipeline_context) {
|
|||||||
static void print_transfer_progress(unsigned long long total_bytes, time_t start,
|
static void print_transfer_progress(unsigned long long total_bytes, time_t start,
|
||||||
const char* suffix, bool human_readable) {
|
const char* suffix, bool human_readable) {
|
||||||
double elapsed = difftime(time(NULL), start);
|
double elapsed = difftime(time(NULL), start);
|
||||||
double rate = elapsed > 0.0 ? total_bytes / (1048576.0 * elapsed) : 0.0;
|
double rate = elapsed > 0.0 ? (double)total_bytes / ((double)BYTES_PER_MIB * elapsed) : 0.0;
|
||||||
if (human_readable) {
|
if (human_readable) {
|
||||||
char total_buffer[32];
|
char total_buffer[32];
|
||||||
char rate_buffer[32];
|
char rate_buffer[32];
|
||||||
fprintf(stderr, "\rSent %s (%s/s) %s",
|
fprintf(stderr, "\rSent %s (%s/s) %s",
|
||||||
display_bytes(total_bytes, true, total_buffer, sizeof(total_buffer)),
|
display_bytes(total_bytes, true, total_buffer, sizeof(total_buffer)),
|
||||||
display_bytes((unsigned long long)(rate * 1048576.0), true, rate_buffer,
|
display_bytes((unsigned long long)(rate * (double)BYTES_PER_MIB), true, rate_buffer,
|
||||||
sizeof(rate_buffer)),
|
sizeof(rate_buffer)),
|
||||||
suffix);
|
suffix);
|
||||||
} else {
|
} else {
|
||||||
fprintf(stderr, "\rSent %.1f MB (%.1f MB/s) %s", total_bytes / 1048576.0, rate, suffix);
|
fprintf(stderr, "\rSent %.1f MB (%.1f MB/s) %s", (double)total_bytes / (double)BYTES_PER_MIB,
|
||||||
|
rate, suffix);
|
||||||
}
|
}
|
||||||
fflush(stderr);
|
fflush(stderr);
|
||||||
}
|
}
|
||||||
@@ -1805,7 +2135,7 @@ int write_batch_from_source(const Config* config, const char* batch_path) {
|
|||||||
prepared_scanner_destroy(&prepared);
|
prepared_scanner_destroy(&prepared);
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
int fd = open(batch_path, O_WRONLY | O_CREAT | O_TRUNC, 0644);
|
int fd = open(batch_path, O_WRONLY | O_CREAT | O_TRUNC | O_NOFOLLOW | O_CLOEXEC, 0600);
|
||||||
if (fd < 0) {
|
if (fd < 0) {
|
||||||
log_perror("could not create batch file");
|
log_perror("could not create batch file");
|
||||||
directory_scanner_destroy(scanner);
|
directory_scanner_destroy(scanner);
|
||||||
@@ -1820,8 +2150,10 @@ int write_batch_from_source(const Config* config, const char* batch_path) {
|
|||||||
if (f == NULL || f->data == NULL)
|
if (f == NULL || f->data == NULL)
|
||||||
continue;
|
continue;
|
||||||
if (f->data->size > 0 && f->data->data == NULL && !file_load_data(f)) {
|
if (f->data->size > 0 && f->data->data == NULL && !file_load_data(f)) {
|
||||||
|
char* escaped_path = output_escape(f->path ? f->path : "", log_get_8_bit_output());
|
||||||
log_message(LOG_LEVEL_ERROR, "batch: failed to load data for %s",
|
log_message(LOG_LEVEL_ERROR, "batch: failed to load data for %s",
|
||||||
f->path ? f->path : "<no path>");
|
escaped_path ? escaped_path : "<allocation failed>");
|
||||||
|
free(escaped_path);
|
||||||
ok = false;
|
ok = false;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
@@ -1862,7 +2194,8 @@ int send_files(Config* config) {
|
|||||||
if (config->list_only)
|
if (config->list_only)
|
||||||
return send_list_only(config);
|
return send_list_only(config);
|
||||||
if (config->dry_run)
|
if (config->dry_run)
|
||||||
return send_dry_run_manifest(config);
|
return dry_run_targets_server(config) ? send_dry_run_remote(config)
|
||||||
|
: send_dry_run_manifest(config);
|
||||||
ArrayList* missing_args = NULL;
|
ArrayList* missing_args = NULL;
|
||||||
int skipped = 0;
|
int skipped = 0;
|
||||||
if (config->delete_missing_args) {
|
if (config->delete_missing_args) {
|
||||||
@@ -1881,15 +2214,21 @@ int send_files(Config* config) {
|
|||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* From here on a server session may be live, so Ctrl-C/SIGTERM should set the
|
||||||
|
abort flag (and be forwarded as STATUS_ABORT) instead of terminating. */
|
||||||
|
client_set_abort_armed(true);
|
||||||
Client* client = connect_transfer_client(config);
|
Client* client = connect_transfer_client(config);
|
||||||
if (!client) {
|
if (!client) {
|
||||||
if (config->transport == TRANSPORT_TCP)
|
if (config->transport == TRANSPORT_TCP)
|
||||||
log_message(LOG_LEVEL_ERROR, "could not connect to server%s",
|
log_message(LOG_LEVEL_ERROR, "could not connect to server%s",
|
||||||
config->use_tls ? " via TLS" : "");
|
config->use_tls ? " via TLS" : "");
|
||||||
|
if (missing_args)
|
||||||
|
array_list_delete(missing_args);
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
ProtocolSession session;
|
ProtocolSession session;
|
||||||
protocol_session_init(&session, client->file_descriptor, client->file_descriptor);
|
protocol_session_init(&session, client->file_descriptor, client->file_descriptor);
|
||||||
|
protocol_session_set_io_timeout(&session, config->timeout);
|
||||||
protocol_session_set_ssl(&session, (SSL*)client->ssl);
|
protocol_session_set_ssl(&session, (SSL*)client->ssl);
|
||||||
protocol_session_bind(&session);
|
protocol_session_bind(&session);
|
||||||
int ret = 1;
|
int ret = 1;
|
||||||
@@ -1997,6 +2336,15 @@ int send_files(Config* config) {
|
|||||||
only a prefix of the source. */
|
only a prefix of the source. */
|
||||||
bool scan_stopped_early = false;
|
bool scan_stopped_early = false;
|
||||||
while ((current_chunk = directory_scanner_next(scanner)) != NULL) {
|
while ((current_chunk = directory_scanner_next(scanner)) != NULL) {
|
||||||
|
/* Graceful abort (Ctrl-C/SIGTERM): notify the receiver and clean up. The
|
||||||
|
session is active (config_send already succeeded); a send failure here is
|
||||||
|
fine because the client is exiting anyway. */
|
||||||
|
if (client_abort_pending()) {
|
||||||
|
log_info_message(LOG_INFO_MISC, "Abort requested; sending STATUS_ABORT to server");
|
||||||
|
chunk_destroy(current_chunk);
|
||||||
|
send_status(client->file_descriptor, STATUS_ABORT);
|
||||||
|
goto send_fail;
|
||||||
|
}
|
||||||
/* Phase 6: stop-elegantly at the next chunk boundary once the deadline has
|
/* Phase 6: stop-elegantly at the next chunk boundary once the deadline has
|
||||||
passed. The scanner may also have stopped early itself; either way the
|
passed. The scanner may also have stopped early itself; either way the
|
||||||
completion tail below keeps everything already sent. */
|
completion tail below keeps everything already sent. */
|
||||||
@@ -2060,6 +2408,13 @@ int send_files(Config* config) {
|
|||||||
goto send_fail;
|
goto send_fail;
|
||||||
if (directory_scanner_had_io_error(scanner))
|
if (directory_scanner_had_io_error(scanner))
|
||||||
had_scan_io = true;
|
had_scan_io = true;
|
||||||
|
/* An abort that arrived after the last chunk must still stop the completion
|
||||||
|
tail (manifest/finalize) rather than let it run to success. */
|
||||||
|
if (client_abort_pending()) {
|
||||||
|
log_info_message(LOG_INFO_MISC, "Abort requested; sending STATUS_ABORT to server");
|
||||||
|
send_status(client->file_descriptor, STATUS_ABORT);
|
||||||
|
goto send_fail;
|
||||||
|
}
|
||||||
/* Phase 6: the scanner may have stopped early (returning NULL without a
|
/* Phase 6: the scanner may have stopped early (returning NULL without a
|
||||||
failure) as soon as the deadline passed, so reflect that here too. A
|
failure) as soon as the deadline passed, so reflect that here too. A
|
||||||
deadline that cut the scan short leaves an incomplete keep-set; transmitting
|
deadline that cut the scan short leaves an incomplete keep-set; transmitting
|
||||||
@@ -2116,23 +2471,9 @@ int send_files(Config* config) {
|
|||||||
remove_transferred_sources(config, remove_sources);
|
remove_transferred_sources(config, remove_sources);
|
||||||
if (config->show_progress && !config->quiet)
|
if (config->show_progress && !config->quiet)
|
||||||
print_transfer_progress(total_bytes, start, "Done.\n", config->human_readable);
|
print_transfer_progress(total_bytes, start, "Done.\n", config->human_readable);
|
||||||
if (config->stats && !config->quiet) {
|
report_transfer_stats(config, total_files, total_bytes, start);
|
||||||
double elapsed_total = difftime(time(NULL), start);
|
|
||||||
double rate = elapsed_total > 0 ? total_bytes / (1048576.0 * elapsed_total) : 0;
|
|
||||||
if (config->human_readable) {
|
|
||||||
char total_buffer[32];
|
|
||||||
char rate_buffer[32];
|
|
||||||
fprintf(stderr, "Stats: %d files, %s, %s/s\n", total_files,
|
|
||||||
display_bytes(total_bytes, true, total_buffer, sizeof(total_buffer)),
|
|
||||||
display_bytes((unsigned long long)(rate * 1048576.0), true, rate_buffer,
|
|
||||||
sizeof(rate_buffer)));
|
|
||||||
} else {
|
|
||||||
fprintf(stderr, "Stats: %d files, %.1f MB, %.1f MB/s\n", total_files, total_bytes / 1048576.0,
|
|
||||||
rate);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
log_info_message(LOG_INFO_STATS, "Transfer summary: %d files, %.1f MB", total_files,
|
log_info_message(LOG_INFO_STATS, "Transfer summary: %d files, %.1f MB", total_files,
|
||||||
total_bytes / 1048576.0);
|
(double)total_bytes / (double)BYTES_PER_MIB);
|
||||||
/* --ignore-errors: an unreadable source directory was skipped but the run
|
/* --ignore-errors: an unreadable source directory was skipped but the run
|
||||||
still completed (and deleted); report the run as errored like rsync does. */
|
still completed (and deleted); report the run as errored like rsync does. */
|
||||||
ret = (ok && !had_scan_io) ? 0 : 1;
|
ret = (ok && !had_scan_io) ? 0 : 1;
|
||||||
@@ -2155,6 +2496,7 @@ send_fail:
|
|||||||
prepared_scanner_destroy(&prepared);
|
prepared_scanner_destroy(&prepared);
|
||||||
disconnect_transfer_client(client);
|
disconnect_transfer_client(client);
|
||||||
protocol_session_unbind();
|
protocol_session_unbind();
|
||||||
|
client_set_abort_armed(false);
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2165,7 +2507,8 @@ int send_files_multithreaded(Config** config_ptr) {
|
|||||||
if (config->list_only)
|
if (config->list_only)
|
||||||
return send_list_only(config);
|
return send_list_only(config);
|
||||||
if (config->dry_run)
|
if (config->dry_run)
|
||||||
return send_dry_run_manifest(config);
|
return dry_run_targets_server(config) ? send_dry_run_remote(config)
|
||||||
|
: send_dry_run_manifest(config);
|
||||||
ArrayList* missing_args = NULL;
|
ArrayList* missing_args = NULL;
|
||||||
int skipped = 0;
|
int skipped = 0;
|
||||||
if (config->delete_missing_args) {
|
if (config->delete_missing_args) {
|
||||||
@@ -2184,13 +2527,21 @@ int send_files_multithreaded(Config** config_ptr) {
|
|||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Armed only once a session may go live (see send_files). */
|
||||||
|
client_set_abort_armed(true);
|
||||||
|
|
||||||
long pages = sysconf(_SC_AVPHYS_PAGES);
|
long pages = sysconf(_SC_AVPHYS_PAGES);
|
||||||
long page_size = sysconf(_SC_PAGE_SIZE);
|
long page_size = sysconf(_SC_PAGE_SIZE);
|
||||||
unsigned long long available_memory =
|
unsigned long long available_memory =
|
||||||
pages > 0 && page_size > 0 ? (unsigned long long)pages * (unsigned long long)page_size
|
pages > 0 && page_size > 0 ? (unsigned long long)pages * (unsigned long long)page_size
|
||||||
: 512ULL * 1024 * 1024;
|
: 512ULL * 1024 * 1024;
|
||||||
unsigned long long avg_file_size = 1024 * 1024;
|
/* Size the chunk queues from the actual chunk size rather than a fixed 1 MiB
|
||||||
int qsize = (int)(available_memory / avg_file_size);
|
average: a chunk holds roughly `chunk_size` bytes of file data, so counting
|
||||||
|
chunks at 1 MiB over-estimated the queue capacity by up to 10x. The byte
|
||||||
|
budget below is the authoritative bound; this count keeps the unloaded
|
||||||
|
chunks waiting in the scanner queue bounded too. */
|
||||||
|
unsigned long long chunk_size = config->chunk_size > 0 ? config->chunk_size : DEFAULT_CHUNK_SIZE;
|
||||||
|
int qsize = (int)(available_memory / chunk_size);
|
||||||
if (qsize < 10)
|
if (qsize < 10)
|
||||||
qsize = 10;
|
qsize = 10;
|
||||||
if (qsize > 1000)
|
if (qsize > 1000)
|
||||||
@@ -2215,7 +2566,8 @@ int send_files_multithreaded(Config** config_ptr) {
|
|||||||
}
|
}
|
||||||
context->missing_args = missing_args;
|
context->missing_args = missing_args;
|
||||||
missing_args = NULL; /* owned by the context from here on */
|
missing_args = NULL; /* owned by the context from here on */
|
||||||
*config_ptr = NULL; /* context now owns config through all remaining paths */
|
pipeline_context_sender_set_queue_byte_limit(context, SENDER_QUEUE_MAX_BYTES);
|
||||||
|
/* The context borrows `config`; the caller (main) still owns and frees it. */
|
||||||
struct timespec now_mono;
|
struct timespec now_mono;
|
||||||
if (clock_gettime(CLOCK_MONOTONIC, &now_mono) != 0) {
|
if (clock_gettime(CLOCK_MONOTONIC, &now_mono) != 0) {
|
||||||
now_mono.tv_sec = 0;
|
now_mono.tv_sec = 0;
|
||||||
@@ -2245,7 +2597,7 @@ int send_files_multithreaded(Config** config_ptr) {
|
|||||||
fills the protected excluded prefixes. */
|
fills the protected excluded prefixes. */
|
||||||
PreparedScanner prepared;
|
PreparedScanner prepared;
|
||||||
memset(&prepared, 0, sizeof(prepared));
|
memset(&prepared, 0, sizeof(prepared));
|
||||||
bool prepared_ok = prepare_scanner(config, 4, &prepared);
|
bool prepared_ok = prepare_scanner(config, config->scanner_threads, &prepared);
|
||||||
if (prepared_ok && context->excluded_paths)
|
if (prepared_ok && context->excluded_paths)
|
||||||
prepared.options.excluded_paths = context->excluded_paths;
|
prepared.options.excluded_paths = context->excluded_paths;
|
||||||
bool prebuilt = prepared_ok && scan_paths_only(config, &prepared.options, context->manifest,
|
bool prebuilt = prepared_ok && scan_paths_only(config, &prepared.options, context->manifest,
|
||||||
@@ -2332,5 +2684,6 @@ int send_files_multithreaded(Config** config_ptr) {
|
|||||||
/* --ignore-errors: the run completed (and deleted) past an unreadable source
|
/* --ignore-errors: the run completed (and deleted) past an unreadable source
|
||||||
directory; report it as errored like rsync does. */
|
directory; report it as errored like rsync does. */
|
||||||
pipeline_context_sender_destroy(context);
|
pipeline_context_sender_destroy(context);
|
||||||
|
client_set_abort_armed(false);
|
||||||
return sender_ok && !scan_io ? 0 : 1;
|
return sender_ok && !scan_io ? 0 : 1;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,10 +4,24 @@
|
|||||||
#include "chunk.h"
|
#include "chunk.h"
|
||||||
#include "config.h"
|
#include "config.h"
|
||||||
#include "transport_tcp.h"
|
#include "transport_tcp.h"
|
||||||
|
#include <signal.h>
|
||||||
|
#include <stdbool.h>
|
||||||
|
|
||||||
int send_chunk(Client* client, Chunk* chunk, Config* config);
|
/* Set ONLY by the client's SIGINT/SIGTERM handler (async-signal-safe: the
|
||||||
|
* handler stores 1 and does nothing else). The send loops poll it via
|
||||||
|
* client_abort_pending() and, when set, best-effort send STATUS_ABORT so the
|
||||||
|
* receiver can clean up before the client exits. */
|
||||||
|
extern volatile sig_atomic_t client_abort_requested;
|
||||||
|
bool client_abort_pending(void);
|
||||||
|
/* Arm/disarm abort handling around the network phase. While disarmed, a
|
||||||
|
* SIGINT/SIGTERM takes the default action (immediate termination) so local-only
|
||||||
|
* modes are not left unresponsive. Defined in client_cli.c. */
|
||||||
|
void client_set_abort_armed(bool armed);
|
||||||
|
|
||||||
|
/* Both sender entry points BORROW `config` for the duration of the call; they
|
||||||
|
* never free it, and the caller retains ownership (freeing it with
|
||||||
|
* config_delete() once the call returns). */
|
||||||
int send_files(Config* config);
|
int send_files(Config* config);
|
||||||
/* Takes ownership only when *config is set to NULL on return. */
|
|
||||||
int send_files_multithreaded(Config** config);
|
int send_files_multithreaded(Config** config);
|
||||||
/* Phase 6 residual-batch (client-only). See client_send.c. */
|
/* Phase 6 residual-batch (client-only). See client_send.c. */
|
||||||
int write_batch_from_source(const Config* config, const char* batch_path);
|
int write_batch_from_source(const Config* config, const char* batch_path);
|
||||||
|
|||||||
+19
-107
@@ -1,6 +1,4 @@
|
|||||||
#include "client_validation.h"
|
#include "client_validation.h"
|
||||||
#include "charset.h"
|
|
||||||
#include "delay_updates.h"
|
|
||||||
#include "log.h"
|
#include "log.h"
|
||||||
#include "usage.h"
|
#include "usage.h"
|
||||||
#include "utils.h"
|
#include "utils.h"
|
||||||
@@ -24,6 +22,19 @@ bool validate_config(const Config* config) {
|
|||||||
"--write-batch, --only-write-batch, and --read-batch are mutually exclusive");
|
"--write-batch, --only-write-batch, and --read-batch are mutually exclusive");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
/* A dry-run of a local batch apply is not meaningful: --read-batch bypasses
|
||||||
|
the client-side scan/server decision entirely, so dry-run would have no
|
||||||
|
wire state to report (and must not be used as a mutation escape hatch).
|
||||||
|
--only-write-batch likewise never contacts a receiver. --write-batch DOES
|
||||||
|
run a live transfer but additionally mutates the filesystem by emitting the
|
||||||
|
batch file, so a dry-run must not write it either. Reject all three up
|
||||||
|
front instead of silently ignoring --dry-run. */
|
||||||
|
if (config->dry_run && (read_batch || only_write_batch || write_batch)) {
|
||||||
|
log_message(LOG_LEVEL_ERROR,
|
||||||
|
"--dry-run cannot be combined with --read-batch, --only-write-batch, or "
|
||||||
|
"--write-batch; a dry-run must not mutate anything, including batch files");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
if (read_batch) {
|
if (read_batch) {
|
||||||
if (!config->receive_root_directory) {
|
if (!config->receive_root_directory) {
|
||||||
log_message(LOG_LEVEL_ERROR, "--read-batch requires a destination directory");
|
log_message(LOG_LEVEL_ERROR, "--read-batch requires a destination directory");
|
||||||
@@ -41,17 +52,6 @@ bool validate_config(const Config* config) {
|
|||||||
print_usage();
|
print_usage();
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
if (config_has_basis(config) && config->use_chunk_serialization) {
|
|
||||||
log_message(LOG_LEVEL_ERROR,
|
|
||||||
"--compare-dest/--copy-dest/--link-dest require per-file incremental checks and "
|
|
||||||
"cannot be combined with -s (chunk serialization)");
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if (config->use_sendfile && (config->use_chunk_serialization || config->use_compression)) {
|
|
||||||
log_message(LOG_LEVEL_ERROR, "-f/--sendfile cannot be combined with -c (compression) or -s "
|
|
||||||
"(chunk serialization)");
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if (config->compression_threads > 0 && !config->use_compression) {
|
if (config->compression_threads > 0 && !config->use_compression) {
|
||||||
log_message(LOG_LEVEL_ERROR, "--compress-threads requires compression (-c or -z)");
|
log_message(LOG_LEVEL_ERROR, "--compress-threads requires compression (-c or -z)");
|
||||||
return false;
|
return false;
|
||||||
@@ -60,73 +60,11 @@ bool validate_config(const Config* config) {
|
|||||||
log_message(LOG_LEVEL_ERROR, "-f/--sendfile is not supported with SSH transport");
|
log_message(LOG_LEVEL_ERROR, "-f/--sendfile is not supported with SSH transport");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
if (config->use_incremental && config->use_chunk_serialization) {
|
|
||||||
log_message(LOG_LEVEL_ERROR, "--incremental is not supported with -s (chunk serialization)");
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
/* -4 and -6 are mutually exclusive: a socket address family cannot be both. */
|
/* -4 and -6 are mutually exclusive: a socket address family cannot be both. */
|
||||||
if (config->ipv4 && config->ipv6) {
|
if (config->ipv4 && config->ipv6) {
|
||||||
log_message(LOG_LEVEL_ERROR, "-4/--ipv4 and -6/--ipv6 are mutually exclusive");
|
log_message(LOG_LEVEL_ERROR, "-4/--ipv4 and -6/--ipv6 are mutually exclusive");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
if (config->skip_compress_set && config->use_chunk_serialization) {
|
|
||||||
log_message(LOG_LEVEL_ERROR,
|
|
||||||
"--skip-compress cannot be combined with -s (chunk serialization)");
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if (config->use_delta && !config->whole_file && !config->use_incremental) {
|
|
||||||
log_message(LOG_LEVEL_ERROR, "--delta requires --incremental");
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if (config->use_delta && !config->whole_file && config->use_chunk_serialization) {
|
|
||||||
log_message(LOG_LEVEL_ERROR, "--delta cannot be combined with -s (chunk serialization)");
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if (config->use_delta && !config->whole_file && config->use_sendfile) {
|
|
||||||
log_message(LOG_LEVEL_ERROR, "--delta cannot be combined with -f (sendfile)");
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
/* --append / --append-verify resume a shorter existing destination by
|
|
||||||
transmitting only the tail. The resume needs the per-file STATUS_CHECK
|
|
||||||
handshake (so the dest length is learned), which chunk serialization -s
|
|
||||||
disables; and whole-file is the opposite intent (send everything), so the
|
|
||||||
two would silently make the resume pointless. Both are rejected up front
|
|
||||||
rather than silently degrading to a full transfer. */
|
|
||||||
if ((config->append || config->append_verify) && config->use_chunk_serialization) {
|
|
||||||
log_message(LOG_LEVEL_ERROR,
|
|
||||||
"--append/--append-verify require the per-file incremental check and cannot be "
|
|
||||||
"combined with -s (chunk serialization)");
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if ((config->append || config->append_verify) && config->whole_file) {
|
|
||||||
log_message(LOG_LEVEL_ERROR,
|
|
||||||
"--append/--append-verify are incompatible with --whole-file (which forces a "
|
|
||||||
"full transfer)");
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
/* --hard-links/-H transmits each later group member as a dedicated per-file
|
|
||||||
STATUS_HARDLINK frame, which chunk serialization -s does not support; and a
|
|
||||||
hard-links sibling carries no payload, so the tail-resume of --append is
|
|
||||||
meaningless for it. Both combinations are rejected up front rather than
|
|
||||||
silently degrading. */
|
|
||||||
if (config->preserve_hard_links && config->use_chunk_serialization) {
|
|
||||||
log_message(LOG_LEVEL_ERROR,
|
|
||||||
"--hard-links/-H cannot be combined with -s (chunk serialization)");
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
/* -X/-A ride the per-file metadata frame; the buffer-based chunk-serialization
|
|
||||||
wire format does not carry the xattr block, so the pair is rejected up front
|
|
||||||
(mirroring -H + -s) rather than silently dropping attributes. */
|
|
||||||
if ((config->preserve_xattrs || config->preserve_acls) && config->use_chunk_serialization) {
|
|
||||||
log_message(LOG_LEVEL_ERROR,
|
|
||||||
"--xattrs/-X and --acls/-A cannot be combined with -s (chunk serialization)");
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if (config->preserve_hard_links && (config->append || config->append_verify)) {
|
|
||||||
log_message(LOG_LEVEL_ERROR,
|
|
||||||
"--hard-links/-H cannot be combined with --append/--append-verify");
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if (config->log_file_format && !config->log_file) {
|
if (config->log_file_format && !config->log_file) {
|
||||||
log_message(LOG_LEVEL_ERROR, "--log-file-format requires --log-file");
|
log_message(LOG_LEVEL_ERROR, "--log-file-format requires --log-file");
|
||||||
return false;
|
return false;
|
||||||
@@ -146,28 +84,12 @@ bool validate_config(const Config* config) {
|
|||||||
log_message(LOG_LEVEL_ERROR, "sending daemon credentials to a non-local server requires --tls");
|
log_message(LOG_LEVEL_ERROR, "sending daemon credentials to a non-local server requires --tls");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
if (config->delay_updates && config->inplace) {
|
/* Every cross-field invariant the receiver enforces lives in one shared
|
||||||
log_message(LOG_LEVEL_ERROR, "--delay-updates does not work with --inplace");
|
predicate so the client and the server can never disagree. The client
|
||||||
return false;
|
reports the specific reason here, before any network I/O. */
|
||||||
}
|
const char* invariants_error = config_invariants_error(config);
|
||||||
if (config->delay_updates && delay_updates_staging_name_conflict(config->backup_dir)) {
|
if (invariants_error) {
|
||||||
log_message(LOG_LEVEL_ERROR,
|
log_message(LOG_LEVEL_ERROR, "%s", invariants_error);
|
||||||
"--backup-dir is reserved when --delay-updates is active (used for the internal "
|
|
||||||
"staging directory)");
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if (!config_has_valid_delete_timing(config)) {
|
|
||||||
log_message(LOG_LEVEL_ERROR,
|
|
||||||
"--delete-before/--delete-during/--delete-delay/--delete-after select the delete "
|
|
||||||
"timing; at most one may be given and each implies --delete");
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
/* --iconv: reject a malformed CONVERT_SPEC or an unsupported charset name at
|
|
||||||
startup (a probe iconv_open is attempted), so a typo'd charset never fails
|
|
||||||
the run mid-transfer with per-file errors. */
|
|
||||||
if (!charset_spec_valid(config->iconv_spec)) {
|
|
||||||
log_message(LOG_LEVEL_ERROR,
|
|
||||||
"--iconv requires LOCAL[,REMOTE] charset names supported by iconv");
|
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
/* --protocol: FastSync has exactly one wire format, so the forced version
|
/* --protocol: FastSync has exactly one wire format, so the forced version
|
||||||
@@ -180,15 +102,5 @@ bool validate_config(const Config* config) {
|
|||||||
PROTOCOL_VERSION);
|
PROTOCOL_VERSION);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
/* --copy-as pushes the source ids through the metadata path (it implies
|
|
||||||
--preserve). A later --no-preserve would clear use_metadata, leaving the
|
|
||||||
transfer with nothing to chown while the receiver gate would still pass.
|
|
||||||
Refuse the combination up front rather than silently chowning nothing. */
|
|
||||||
if (config->copy_as_set && !config->use_metadata) {
|
|
||||||
log_message(LOG_LEVEL_ERROR,
|
|
||||||
"--copy-as requires metadata preservation and cannot be combined with "
|
|
||||||
"--no-preserve");
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|||||||
+86
-118
@@ -177,9 +177,9 @@ static bool entry_passes_selection(const FileListSet* file_list, const FilterRul
|
|||||||
/* Best-effort capture of the file's whitelisted xattrs (-X/-A). A failure to
|
/* Best-effort capture of the file's whitelisted xattrs (-X/-A). A failure to
|
||||||
* read xattrs is non-fatal: the file is transferred without them. */
|
* read xattrs is non-fatal: the file is transferred without them. */
|
||||||
static void scanner_capture_xattrs(const DirectoryScanner* scanner, File* file) {
|
static void scanner_capture_xattrs(const DirectoryScanner* scanner, File* file) {
|
||||||
if (!scanner || !file || !(scanner->preserve_xattrs || scanner->preserve_acls))
|
if (!scanner || !file || !(scanner->options.preserve_xattrs || scanner->options.preserve_acls))
|
||||||
return;
|
return;
|
||||||
file->xattrs = xattr_capture_path(file->path);
|
file->xattrs = xattr_capture_path(file->path, scanner->options.preserve_acls);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Apply --hard-links (-H) detection to one regular File. On a sibling (a
|
/* Apply --hard-links (-H) detection to one regular File. On a sibling (a
|
||||||
@@ -263,10 +263,10 @@ static bool excluded_sink_append(ArrayList* list, mtx_t* mtx, const char* rel) {
|
|||||||
how manifest keep entries are stored), so the receiver's walker prefixes
|
how manifest keep entries are stored), so the receiver's walker prefixes
|
||||||
match the destination layout. An allocation failure is a fatal scan error. */
|
match the destination layout. An allocation failure is a fatal scan error. */
|
||||||
static void scanner_record_excluded(DirectoryScanner* scanner, const char* fs_path) {
|
static void scanner_record_excluded(DirectoryScanner* scanner, const char* fs_path) {
|
||||||
if (!scanner->excluded_paths || !fs_path)
|
if (!scanner->options.excluded_paths || !fs_path)
|
||||||
return;
|
return;
|
||||||
const char* rel = *fs_path == '/' ? fs_path + 1 : fs_path;
|
const char* rel = *fs_path == '/' ? fs_path + 1 : fs_path;
|
||||||
if (!excluded_sink_append(scanner->excluded_paths, scanner->excluded_mutex, rel))
|
if (!excluded_sink_append(scanner->options.excluded_paths, scanner->options.excluded_mutex, rel))
|
||||||
scanner->failed = true;
|
scanner->failed = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -274,7 +274,7 @@ static void scanner_record_excluded(DirectoryScanner* scanner, const char* fs_pa
|
|||||||
* context, returning the context used for this directory's entries. On a parse
|
* context, returning the context used for this directory's entries. On a parse
|
||||||
* error the scanner is marked failed. Returns 0 on success, -1 on failure. */
|
* error the scanner is marked failed. Returns 0 on success, -1 on failure. */
|
||||||
static int open_directory_filter_context(DirectoryScanner* scanner, const FilterNode* inherited) {
|
static int open_directory_filter_context(DirectoryScanner* scanner, const FilterNode* inherited) {
|
||||||
if (!scanner->per_dir_filters) {
|
if (!scanner->options.per_dir_filters) {
|
||||||
scanner->current_node = (FilterNode*)inherited;
|
scanner->current_node = (FilterNode*)inherited;
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
@@ -284,7 +284,10 @@ static int open_directory_filter_context(DirectoryScanner* scanner, const Filter
|
|||||||
filter_file_read(scanner->current_path, scanner->current_rel ? scanner->current_rel : "",
|
filter_file_read(scanner->current_path, scanner->current_rel ? scanner->current_rel : "",
|
||||||
&exists, err, sizeof(err));
|
&exists, err, sizeof(err));
|
||||||
if (!own) {
|
if (!own) {
|
||||||
log_message(LOG_LEVEL_ERROR, "invalid .rsync-filter in %s: %s", scanner->current_path, err);
|
char* escaped_path = output_escape(scanner->current_path, log_get_8_bit_output());
|
||||||
|
log_message(LOG_LEVEL_ERROR, "invalid .rsync-filter in %s: %s",
|
||||||
|
escaped_path ? escaped_path : "<allocation failed>", err);
|
||||||
|
free(escaped_path);
|
||||||
scanner->failed = true;
|
scanner->failed = true;
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
@@ -388,9 +391,13 @@ static int scanner_inspect_entry(const ScannerOptions* options, const char* sour
|
|||||||
goto apply_filters;
|
goto apply_filters;
|
||||||
|
|
||||||
regular:
|
regular:
|
||||||
if (stat(entry->path, &entry->stats) != 0)
|
/* Not a symlink: the lstat() above already described this entry, and lstat
|
||||||
goto skip;
|
and stat are identical for every non-symlink, so reuse that result instead
|
||||||
entry->is_directory = S_ISDIR(entry->stats.st_mode);
|
of issuing a redundant stat() on the scanner hot path. stat() is still
|
||||||
|
used on the dereference paths above/below for actual symlinks (copy-links,
|
||||||
|
safe/copy-unsafe links, and -k symlinks-to-directories). */
|
||||||
|
entry->stats = link_stats;
|
||||||
|
entry->is_directory = S_ISDIR(link_stats.st_mode);
|
||||||
if (entry->is_directory)
|
if (entry->is_directory)
|
||||||
return 1;
|
return 1;
|
||||||
|
|
||||||
@@ -432,6 +439,11 @@ DirectoryScanner* directory_scanner_create_with_options(const char* root_directo
|
|||||||
DirectoryScanner* scanner = calloc(1, sizeof(DirectoryScanner));
|
DirectoryScanner* scanner = calloc(1, sizeof(DirectoryScanner));
|
||||||
if (scanner == NULL)
|
if (scanner == NULL)
|
||||||
return NULL;
|
return NULL;
|
||||||
|
/* One copy of the scan inputs; normalize chunk_size as the old field-by-field
|
||||||
|
copy did. */
|
||||||
|
scanner->options = *options;
|
||||||
|
if (scanner->options.chunk_size == 0)
|
||||||
|
scanner->options.chunk_size = DESIRED_CHUNK_SIZE;
|
||||||
scanner->directories = queue_create(100, dir_entry_destroy);
|
scanner->directories = queue_create(100, dir_entry_destroy);
|
||||||
if (!scanner->directories) {
|
if (!scanner->directories) {
|
||||||
free(scanner);
|
free(scanner);
|
||||||
@@ -439,31 +451,7 @@ DirectoryScanner* directory_scanner_create_with_options(const char* root_directo
|
|||||||
}
|
}
|
||||||
scanner->current_dir = NULL;
|
scanner->current_dir = NULL;
|
||||||
scanner->current_path = NULL;
|
scanner->current_path = NULL;
|
||||||
scanner->use_metadata = options->use_metadata;
|
|
||||||
scanner->preserve_atimes = options->preserve_atimes;
|
|
||||||
scanner->preserve_crtimes = options->preserve_crtimes;
|
|
||||||
scanner->preserve_xattrs = options->preserve_xattrs;
|
|
||||||
scanner->preserve_acls = options->preserve_acls;
|
|
||||||
scanner->chunk_size = options->chunk_size > 0 ? options->chunk_size : DESIRED_CHUNK_SIZE;
|
|
||||||
scanner->exclude_patterns = options->exclude_patterns;
|
|
||||||
scanner->exclude_count = options->exclude_count;
|
|
||||||
scanner->include_patterns = options->include_patterns;
|
|
||||||
scanner->include_count = options->include_count;
|
|
||||||
scanner->max_size = options->max_size;
|
|
||||||
scanner->min_size = options->min_size;
|
|
||||||
scanner->max_depth = options->max_depth;
|
|
||||||
scanner->current_depth = 0;
|
scanner->current_depth = 0;
|
||||||
scanner->follow_symlinks = options->follow_symlinks;
|
|
||||||
scanner->copy_links = options->copy_links;
|
|
||||||
scanner->safe_links = options->safe_links;
|
|
||||||
scanner->copy_unsafe_links = options->copy_unsafe_links;
|
|
||||||
scanner->copy_dirlinks = options->copy_dirlinks;
|
|
||||||
scanner->munge_links = options->munge_links;
|
|
||||||
scanner->checksum = options->checksum;
|
|
||||||
scanner->one_file_system = options->one_file_system;
|
|
||||||
scanner->preserve_devices = options->preserve_devices;
|
|
||||||
scanner->preserve_specials = options->preserve_specials;
|
|
||||||
scanner->copy_devices = options->copy_devices;
|
|
||||||
scanner->failed = false;
|
scanner->failed = false;
|
||||||
scanner->root_path = str_dup(root_directory);
|
scanner->root_path = str_dup(root_directory);
|
||||||
if (!scanner->root_path) {
|
if (!scanner->root_path) {
|
||||||
@@ -475,28 +463,14 @@ DirectoryScanner* directory_scanner_create_with_options(const char* root_directo
|
|||||||
scanner->at_seed_dir = true;
|
scanner->at_seed_dir = true;
|
||||||
scanner->seed_node = NULL;
|
scanner->seed_node = NULL;
|
||||||
scanner->current_node = NULL;
|
scanner->current_node = NULL;
|
||||||
scanner->file_list = options->file_list;
|
|
||||||
scanner->base_filters = options->base_filters;
|
|
||||||
scanner->per_dir_filters = options->per_dir_filters;
|
|
||||||
scanner->excluded_paths = options->excluded_paths;
|
|
||||||
scanner->excluded_mutex = options->excluded_mutex;
|
|
||||||
scanner->ignore_io_errors = options->ignore_io_errors;
|
|
||||||
scanner->ignore_missing_args = options->ignore_missing_args;
|
|
||||||
scanner->io_error = false;
|
scanner->io_error = false;
|
||||||
scanner->dirs_mode = options->dirs;
|
|
||||||
scanner->relative_mode = options->relative && options->file_list != NULL;
|
scanner->relative_mode = options->relative && options->file_list != NULL;
|
||||||
scanner->hardlinks = options->hardlinks;
|
|
||||||
scanner->prune_empty_dirs = options->prune_empty_dirs;
|
|
||||||
scanner->stop_condition = options->stop_condition;
|
|
||||||
scanner->capture_dir_times = options->capture_dir_times;
|
|
||||||
scanner->dir_entries = options->dir_entries;
|
|
||||||
scanner->dir_entries_mutex = options->dir_entries_mutex;
|
|
||||||
scanner->dirs_root_emitted = false;
|
scanner->dirs_root_emitted = false;
|
||||||
scanner->list_index = 0;
|
scanner->list_index = 0;
|
||||||
scanner->dirs_batch = NULL;
|
scanner->dirs_batch = NULL;
|
||||||
scanner->dirs_batch_size = 0;
|
scanner->dirs_batch_size = 0;
|
||||||
scanner->filter_nodes = NULL;
|
scanner->filter_nodes = NULL;
|
||||||
if (scanner->base_filters || scanner->per_dir_filters) {
|
if (scanner->options.base_filters || scanner->options.per_dir_filters) {
|
||||||
scanner->filter_nodes = array_list_create(filter_node_destroy);
|
scanner->filter_nodes = array_list_create(filter_node_destroy);
|
||||||
if (!scanner->filter_nodes) {
|
if (!scanner->filter_nodes) {
|
||||||
free(scanner->root_path);
|
free(scanner->root_path);
|
||||||
@@ -505,7 +479,7 @@ DirectoryScanner* directory_scanner_create_with_options(const char* root_directo
|
|||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (scanner->one_file_system) {
|
if (scanner->options.one_file_system) {
|
||||||
struct stat root_stats;
|
struct stat root_stats;
|
||||||
if (stat(root_directory, &root_stats) != 0) {
|
if (stat(root_directory, &root_stats) != 0) {
|
||||||
log_perror("Could not stat source directory");
|
log_perror("Could not stat source directory");
|
||||||
@@ -587,12 +561,16 @@ void directory_scanner_destroy(DirectoryScanner* scanner) {
|
|||||||
|
|
||||||
static Chunk* chunk_data_to_chunk(ArrayList* chunk_data) {
|
static Chunk* chunk_data_to_chunk(ArrayList* chunk_data) {
|
||||||
void** chunk_items = array_list_to_array(chunk_data);
|
void** chunk_items = array_list_to_array(chunk_data);
|
||||||
if (!chunk_items)
|
if (!chunk_items) {
|
||||||
|
array_list_delete(chunk_data);
|
||||||
return NULL;
|
return NULL;
|
||||||
|
}
|
||||||
Chunk* chunk = chunk_create((File**)chunk_items, chunk_data->size);
|
Chunk* chunk = chunk_create((File**)chunk_items, chunk_data->size);
|
||||||
free(chunk_items);
|
free(chunk_items);
|
||||||
if (!chunk)
|
if (!chunk) {
|
||||||
|
array_list_delete(chunk_data);
|
||||||
return NULL;
|
return NULL;
|
||||||
|
}
|
||||||
chunk_data->item_destroyer = NULL;
|
chunk_data->item_destroyer = NULL;
|
||||||
array_list_delete(chunk_data);
|
array_list_delete(chunk_data);
|
||||||
return chunk;
|
return chunk;
|
||||||
@@ -707,7 +685,7 @@ static int open_next_directory(DirectoryScanner* scanner) {
|
|||||||
scanner->current_rel = NULL;
|
scanner->current_rel = NULL;
|
||||||
free(scanner->current_path);
|
free(scanner->current_path);
|
||||||
scanner->current_path = NULL;
|
scanner->current_path = NULL;
|
||||||
if (!scanner->ignore_io_errors || is_root_seed) {
|
if (!scanner->options.ignore_io_errors || is_root_seed) {
|
||||||
scanner->failed = true;
|
scanner->failed = true;
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
@@ -721,10 +699,11 @@ static int open_next_directory(DirectoryScanner* scanner) {
|
|||||||
scanner->current_path = NULL;
|
scanner->current_path = NULL;
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
if (scanner->capture_dir_times &&
|
if (scanner->options.capture_dir_times &&
|
||||||
!scanner_capture_dir_time(scanner->dir_entries, scanner->dir_entries_mutex,
|
!scanner_capture_dir_time(scanner->options.dir_entries, scanner->options.dir_entries_mutex,
|
||||||
scanner->root_path, scanner->current_path, scanner->relative_mode,
|
scanner->root_path, scanner->current_path, scanner->relative_mode,
|
||||||
scanner->preserve_atimes, scanner->preserve_crtimes)) {
|
scanner->options.preserve_atimes,
|
||||||
|
scanner->options.preserve_crtimes)) {
|
||||||
closedir(scanner->current_dir);
|
closedir(scanner->current_dir);
|
||||||
scanner->current_dir = NULL;
|
scanner->current_dir = NULL;
|
||||||
free(scanner->current_path);
|
free(scanner->current_path);
|
||||||
@@ -765,9 +744,9 @@ static File* dirs_root_dir_file(DirectoryScanner* scanner) {
|
|||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
file->is_dir = true;
|
file->is_dir = true;
|
||||||
if (scanner->use_metadata) {
|
if (scanner->options.use_metadata) {
|
||||||
file->metadata = file_metadata_create(scanner->root_path, &st, scanner->preserve_atimes,
|
file->metadata = file_metadata_create(scanner->root_path, &st, scanner->options.preserve_atimes,
|
||||||
scanner->preserve_crtimes);
|
scanner->options.preserve_crtimes);
|
||||||
if (!file->metadata) {
|
if (!file->metadata) {
|
||||||
file_destroy(file);
|
file_destroy(file);
|
||||||
scanner->failed = true;
|
scanner->failed = true;
|
||||||
@@ -800,12 +779,20 @@ static File* dirs_file_for_entry(DirectoryScanner* scanner, const char* entry) {
|
|||||||
missing argument and is skipped here, exactly as the recursive scan skips
|
missing argument and is skipped here, exactly as the recursive scan skips
|
||||||
nothing (missing entries never appear there). Without the flags it stays
|
nothing (missing entries never appear there). Without the flags it stays
|
||||||
a hard pre-transfer error. */
|
a hard pre-transfer error. */
|
||||||
if (scanner->ignore_missing_args) {
|
if (scanner->options.ignore_missing_args) {
|
||||||
log_info_message(LOG_INFO_MISC, "skipping missing --files-from entry '%s'", entry);
|
char* escaped_entry = output_escape(entry, log_get_8_bit_output());
|
||||||
|
log_info_message(LOG_INFO_MISC, "skipping missing --files-from entry '%s'",
|
||||||
|
escaped_entry ? escaped_entry : "<allocation failed>");
|
||||||
|
free(escaped_entry);
|
||||||
free(abs_path);
|
free(abs_path);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
log_message(LOG_LEVEL_ERROR, "--dirs listed entry is not present under the source: %s", entry);
|
{
|
||||||
|
char* escaped_entry = output_escape(entry, log_get_8_bit_output());
|
||||||
|
log_message(LOG_LEVEL_ERROR, "--dirs listed entry is not present under the source: %s",
|
||||||
|
escaped_entry ? escaped_entry : "<allocation failed>");
|
||||||
|
free(escaped_entry);
|
||||||
|
}
|
||||||
free(abs_path);
|
free(abs_path);
|
||||||
scanner->failed = true;
|
scanner->failed = true;
|
||||||
return NULL;
|
return NULL;
|
||||||
@@ -814,8 +801,8 @@ static File* dirs_file_for_entry(DirectoryScanner* scanner, const char* entry) {
|
|||||||
if (S_ISLNK(link_stats.st_mode)) {
|
if (S_ISLNK(link_stats.st_mode)) {
|
||||||
/* A symlink is transferred (following its referent) only when a link
|
/* A symlink is transferred (following its referent) only when a link
|
||||||
resolution option is active, mirroring the regular scanner. */
|
resolution option is active, mirroring the regular scanner. */
|
||||||
bool resolve = scanner->follow_symlinks || scanner->copy_links || scanner->safe_links ||
|
bool resolve = scanner->options.follow_symlinks || scanner->options.copy_links ||
|
||||||
scanner->copy_unsafe_links;
|
scanner->options.safe_links || scanner->options.copy_unsafe_links;
|
||||||
if (!resolve || stat(abs_path, &effective) != 0) {
|
if (!resolve || stat(abs_path, &effective) != 0) {
|
||||||
free(abs_path);
|
free(abs_path);
|
||||||
return NULL;
|
return NULL;
|
||||||
@@ -843,9 +830,9 @@ static File* dirs_file_for_entry(DirectoryScanner* scanner, const char* entry) {
|
|||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (scanner->use_metadata) {
|
if (scanner->options.use_metadata) {
|
||||||
file->metadata = file_metadata_create(file->path, &effective, scanner->preserve_atimes,
|
file->metadata = file_metadata_create(file->path, &effective, scanner->options.preserve_atimes,
|
||||||
scanner->preserve_crtimes);
|
scanner->options.preserve_crtimes);
|
||||||
if (!file->metadata) {
|
if (!file->metadata) {
|
||||||
file_destroy(file);
|
file_destroy(file);
|
||||||
scanner->failed = true;
|
scanner->failed = true;
|
||||||
@@ -876,18 +863,18 @@ static bool dirs_source_dir_is_empty(const char* path) {
|
|||||||
|
|
||||||
/* The next File from the --dirs generator, or NULL when exhausted. */
|
/* The next File from the --dirs generator, or NULL when exhausted. */
|
||||||
static File* dirs_next_file(DirectoryScanner* scanner) {
|
static File* dirs_next_file(DirectoryScanner* scanner) {
|
||||||
if (!scanner->file_list) {
|
if (!scanner->options.file_list) {
|
||||||
if (scanner->dirs_root_emitted)
|
if (scanner->dirs_root_emitted)
|
||||||
return NULL;
|
return NULL;
|
||||||
scanner->dirs_root_emitted = true;
|
scanner->dirs_root_emitted = true;
|
||||||
/* --prune-empty-dirs: a physically empty source directory's explicit entry
|
/* --prune-empty-dirs: a physically empty source directory's explicit entry
|
||||||
would only create an empty destination directory, so it is omitted. */
|
would only create an empty destination directory, so it is omitted. */
|
||||||
if (scanner->prune_empty_dirs && dirs_source_dir_is_empty(scanner->root_path))
|
if (scanner->options.prune_empty_dirs && dirs_source_dir_is_empty(scanner->root_path))
|
||||||
return NULL;
|
return NULL;
|
||||||
return dirs_root_dir_file(scanner);
|
return dirs_root_dir_file(scanner);
|
||||||
}
|
}
|
||||||
while (scanner->list_index < scanner->file_list->count) {
|
while (scanner->list_index < scanner->options.file_list->count) {
|
||||||
const char* entry = scanner->file_list->entries[scanner->list_index++];
|
const char* entry = scanner->options.file_list->entries[scanner->list_index++];
|
||||||
File* file = dirs_file_for_entry(scanner, entry);
|
File* file = dirs_file_for_entry(scanner, entry);
|
||||||
if (scanner->failed)
|
if (scanner->failed)
|
||||||
return NULL;
|
return NULL;
|
||||||
@@ -914,8 +901,9 @@ static Chunk* dirs_flush_batch(DirectoryScanner* scanner) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
static Chunk* directory_scanner_next_dirs(DirectoryScanner* scanner) {
|
static Chunk* directory_scanner_next_dirs(DirectoryScanner* scanner) {
|
||||||
while (scanner->dirs_batch == NULL || scanner->dirs_batch_size <= scanner->chunk_size) {
|
while (scanner->dirs_batch == NULL || scanner->dirs_batch_size <= scanner->options.chunk_size) {
|
||||||
if (scanner->stop_condition && stop_condition_reached(scanner->stop_condition)) {
|
if (scanner->options.stop_condition &&
|
||||||
|
stop_condition_reached(scanner->options.stop_condition)) {
|
||||||
Chunk* leftover = dirs_flush_batch(scanner);
|
Chunk* leftover = dirs_flush_batch(scanner);
|
||||||
if (leftover)
|
if (leftover)
|
||||||
chunk_destroy(leftover);
|
chunk_destroy(leftover);
|
||||||
@@ -954,7 +942,7 @@ static Chunk* directory_scanner_next_dirs(DirectoryScanner* scanner) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
Chunk* directory_scanner_next(DirectoryScanner* scanner) {
|
Chunk* directory_scanner_next(DirectoryScanner* scanner) {
|
||||||
if (scanner && scanner->dirs_mode)
|
if (scanner && scanner->options.dirs)
|
||||||
return directory_scanner_next_dirs(scanner);
|
return directory_scanner_next_dirs(scanner);
|
||||||
ArrayList* chunk_data = array_list_create(file_destroy);
|
ArrayList* chunk_data = array_list_create(file_destroy);
|
||||||
if (!chunk_data) {
|
if (!chunk_data) {
|
||||||
@@ -964,7 +952,8 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
|
|||||||
unsigned long long chunk_data_size = 0;
|
unsigned long long chunk_data_size = 0;
|
||||||
|
|
||||||
while (1) {
|
while (1) {
|
||||||
if (scanner->stop_condition && stop_condition_reached(scanner->stop_condition)) {
|
if (scanner->options.stop_condition &&
|
||||||
|
stop_condition_reached(scanner->options.stop_condition)) {
|
||||||
array_list_delete(chunk_data);
|
array_list_delete(chunk_data);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
@@ -988,34 +977,9 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
|
|||||||
if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
|
if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
|
||||||
continue;
|
continue;
|
||||||
|
|
||||||
ScannerOptions options = {
|
|
||||||
.use_metadata = scanner->use_metadata,
|
|
||||||
.chunk_size = scanner->chunk_size,
|
|
||||||
.exclude_patterns = scanner->exclude_patterns,
|
|
||||||
.exclude_count = scanner->exclude_count,
|
|
||||||
.include_patterns = scanner->include_patterns,
|
|
||||||
.include_count = scanner->include_count,
|
|
||||||
.max_size = scanner->max_size,
|
|
||||||
.min_size = scanner->min_size,
|
|
||||||
.max_depth = scanner->max_depth,
|
|
||||||
.num_threads = 0,
|
|
||||||
.follow_symlinks = scanner->follow_symlinks,
|
|
||||||
.copy_links = scanner->copy_links,
|
|
||||||
.safe_links = scanner->safe_links,
|
|
||||||
.copy_unsafe_links = scanner->copy_unsafe_links,
|
|
||||||
.copy_dirlinks = scanner->copy_dirlinks,
|
|
||||||
.munge_links = scanner->munge_links,
|
|
||||||
.checksum = scanner->checksum,
|
|
||||||
.one_file_system = scanner->one_file_system,
|
|
||||||
.file_list = scanner->file_list,
|
|
||||||
.base_filters = scanner->base_filters,
|
|
||||||
.per_dir_filters = scanner->per_dir_filters,
|
|
||||||
.dirs = false,
|
|
||||||
.relative = false,
|
|
||||||
};
|
|
||||||
ScannerEntry inspected;
|
ScannerEntry inspected;
|
||||||
int inspection = scanner_inspect_entry(&options, scanner->current_path, scanner->current_path,
|
int inspection = scanner_inspect_entry(&scanner->options, scanner->current_path,
|
||||||
entry->d_name, &inspected);
|
scanner->current_path, entry->d_name, &inspected);
|
||||||
if (inspection < 0) {
|
if (inspection < 0) {
|
||||||
scanner->failed = true;
|
scanner->failed = true;
|
||||||
break;
|
break;
|
||||||
@@ -1047,16 +1011,17 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
|
|||||||
scanner->failed = true;
|
scanner->failed = true;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
bool passes_selection =
|
bool passes_selection = entry_passes_selection(
|
||||||
entry_passes_selection(scanner->file_list, scanner->base_filters, scanner->current_node,
|
scanner->options.file_list, scanner->options.base_filters, scanner->current_node, rel,
|
||||||
rel, entry->d_name, is_dir, scanner->per_dir_filters);
|
entry->d_name, is_dir, scanner->options.per_dir_filters);
|
||||||
if (!passes_selection) {
|
if (!passes_selection) {
|
||||||
/* --files-from subset pruning is not a filter exclusion: its delete
|
/* --files-from subset pruning is not a filter exclusion: its delete
|
||||||
semantics stay keep-set-only (an unlisted source path is treated as
|
semantics stay keep-set-only (an unlisted source path is treated as
|
||||||
absent, so its destination mirror is a deletable extra). A rule-based
|
absent, so its destination mirror is a deletable extra). A rule-based
|
||||||
exclusion is recorded as a protected prefix. -R + --files-from bare
|
exclusion is recorded as a protected prefix. -R + --files-from bare
|
||||||
wire paths are never recorded (see ScannerOptions.excluded_paths). */
|
wire paths are never recorded (see ScannerOptions.excluded_paths). */
|
||||||
bool files_from_prune = scanner->file_list && !file_list_affects(scanner->file_list, rel);
|
bool files_from_prune =
|
||||||
|
scanner->options.file_list && !file_list_affects(scanner->options.file_list, rel);
|
||||||
if (!files_from_prune && !scanner->relative_mode)
|
if (!files_from_prune && !scanner->relative_mode)
|
||||||
scanner_record_excluded(scanner, cur_path);
|
scanner_record_excluded(scanner, cur_path);
|
||||||
}
|
}
|
||||||
@@ -1077,12 +1042,13 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
|
|||||||
|
|
||||||
if (is_dir) {
|
if (is_dir) {
|
||||||
free(rel_copy);
|
free(rel_copy);
|
||||||
if (!scanner_same_filesystem(scanner->one_file_system, scanner->root_dev, stats.st_dev)) {
|
if (!scanner_same_filesystem(scanner->options.one_file_system, scanner->root_dev,
|
||||||
|
stats.st_dev)) {
|
||||||
free(cur_path);
|
free(cur_path);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
int next_depth = scanner->current_depth + 1;
|
int next_depth = scanner->current_depth + 1;
|
||||||
if (scanner->max_depth <= 0 || next_depth < scanner->max_depth) {
|
if (scanner->options.max_depth <= 0 || next_depth < scanner->options.max_depth) {
|
||||||
DirEntry* de = dir_entry_create(cur_path, next_depth, scanner->current_node);
|
DirEntry* de = dir_entry_create(cur_path, next_depth, scanner->current_node);
|
||||||
if (!de || !queue_enqueue(scanner->directories, de)) {
|
if (!de || !queue_enqueue(scanner->directories, de)) {
|
||||||
dir_entry_destroy(de);
|
dir_entry_destroy(de);
|
||||||
@@ -1091,7 +1057,8 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
|
|||||||
}
|
}
|
||||||
free(cur_path);
|
free(cur_path);
|
||||||
} else {
|
} else {
|
||||||
if (scanner->max_depth > 0 && scanner->current_depth + 1 > scanner->max_depth) {
|
if (scanner->options.max_depth > 0 &&
|
||||||
|
scanner->current_depth + 1 > scanner->options.max_depth) {
|
||||||
free(rel_copy);
|
free(rel_copy);
|
||||||
free(cur_path);
|
free(cur_path);
|
||||||
continue;
|
continue;
|
||||||
@@ -1118,13 +1085,14 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
|
|||||||
}
|
}
|
||||||
/* --devices/--specials: a device/FIFO/socket entry marked for preservation
|
/* --devices/--specials: a device/FIFO/socket entry marked for preservation
|
||||||
becomes a node to recreate (is_special, no data, rdev captured). */
|
becomes a node to recreate (is_special, no data, rdev captured). */
|
||||||
scanner_prepare_special(scanner->preserve_devices, scanner->preserve_specials, file, &stats);
|
scanner_prepare_special(scanner->options.preserve_devices, scanner->options.preserve_specials,
|
||||||
if (scanner->hardlinks && S_ISREG(stats.st_mode))
|
file, &stats);
|
||||||
scanner_assign_hardlink(scanner, scanner->hardlinks, file, &stats);
|
if (scanner->options.hardlinks && S_ISREG(stats.st_mode))
|
||||||
if (scanner->use_metadata)
|
scanner_assign_hardlink(scanner, scanner->options.hardlinks, file, &stats);
|
||||||
file->metadata = file_metadata_create(file->path, &stats, scanner->preserve_atimes,
|
if (scanner->options.use_metadata)
|
||||||
scanner->preserve_crtimes);
|
file->metadata = file_metadata_create(file->path, &stats, scanner->options.preserve_atimes,
|
||||||
if (scanner->use_metadata && !file->metadata) {
|
scanner->options.preserve_crtimes);
|
||||||
|
if (scanner->options.use_metadata && !file->metadata) {
|
||||||
free(rel_copy);
|
free(rel_copy);
|
||||||
file_destroy(file);
|
file_destroy(file);
|
||||||
scanner->failed = true;
|
scanner->failed = true;
|
||||||
@@ -1139,7 +1107,7 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
|
|||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
chunk_data_size += file->data->size;
|
chunk_data_size += file->data->size;
|
||||||
if (chunk_data_size > scanner->chunk_size) {
|
if (chunk_data_size > scanner->options.chunk_size) {
|
||||||
free(rel_copy);
|
free(rel_copy);
|
||||||
Chunk* result = chunk_data_to_chunk(chunk_data);
|
Chunk* result = chunk_data_to_chunk(chunk_data);
|
||||||
if (!result)
|
if (!result)
|
||||||
@@ -1203,7 +1171,7 @@ static int parallel_worker_thread(void* arg) {
|
|||||||
free(ds->root_path);
|
free(ds->root_path);
|
||||||
ds->root_path = str_dup(wa->root_dir);
|
ds->root_path = str_dup(wa->root_dir);
|
||||||
ds->seed_node = wa->ps->root_filter_node;
|
ds->seed_node = wa->ps->root_filter_node;
|
||||||
ds->excluded_mutex = &wa->ps->result_mutex;
|
ds->options.excluded_mutex = &wa->ps->result_mutex;
|
||||||
Chunk* chunk;
|
Chunk* chunk;
|
||||||
while ((chunk = directory_scanner_next(ds)) != NULL) {
|
while ((chunk = directory_scanner_next(ds)) != NULL) {
|
||||||
if (!queue_enqueue_multithreaded_cancel(wa->ps->result_queue, chunk, &wa->ps->result_mutex,
|
if (!queue_enqueue_multithreaded_cancel(wa->ps->result_queue, chunk, &wa->ps->result_mutex,
|
||||||
@@ -1477,7 +1445,7 @@ static void scan_root_entry(const ScannerOptions* options, const FilterNode* roo
|
|||||||
}
|
}
|
||||||
if ((options->preserve_xattrs || options->preserve_acls) &&
|
if ((options->preserve_xattrs || options->preserve_acls) &&
|
||||||
!(file->link_group != 0 && !file->link_first))
|
!(file->link_group != 0 && !file->link_first))
|
||||||
file->xattrs = xattr_capture_path(file->path);
|
file->xattrs = xattr_capture_path(file->path, options->preserve_acls);
|
||||||
if (!array_list_add(root_files, file)) {
|
if (!array_list_add(root_files, file)) {
|
||||||
free(rel);
|
free(rel);
|
||||||
file_destroy(file);
|
file_destroy(file);
|
||||||
|
|||||||
+9
-49
@@ -14,6 +14,10 @@
|
|||||||
#include <sys/types.h>
|
#include <sys/types.h>
|
||||||
#include <threads.h>
|
#include <threads.h>
|
||||||
|
|
||||||
|
/* Upper bound on the configurable parallel scanner worker count (--threads=N):
|
||||||
|
* keeps one transfer from spawning an unbounded pool on a very large machine. */
|
||||||
|
#define MAX_SCANNER_THREADS 256
|
||||||
|
|
||||||
typedef struct {
|
typedef struct {
|
||||||
bool use_metadata;
|
bool use_metadata;
|
||||||
/* Phase 4 metadata capture: -U/--atimes and -N/--crtimes tell the scanner to
|
/* Phase 4 metadata capture: -U/--atimes and -N/--crtimes tell the scanner to
|
||||||
@@ -117,37 +121,16 @@ typedef struct {
|
|||||||
typedef struct FilterNode FilterNode;
|
typedef struct FilterNode FilterNode;
|
||||||
|
|
||||||
typedef struct {
|
typedef struct {
|
||||||
|
/* Scan inputs, copied once at create time. Everything that is also a
|
||||||
|
ScannerOptions field lives here (with the normalized chunk_size); only
|
||||||
|
scanner-owned bookkeeping stays as direct members below. */
|
||||||
|
ScannerOptions options;
|
||||||
Queue* directories;
|
Queue* directories;
|
||||||
DIR* current_dir;
|
DIR* current_dir;
|
||||||
char* current_path;
|
char* current_path;
|
||||||
bool use_metadata;
|
|
||||||
bool preserve_atimes;
|
|
||||||
bool preserve_crtimes;
|
|
||||||
bool preserve_xattrs;
|
|
||||||
bool preserve_acls;
|
|
||||||
unsigned long long chunk_size;
|
|
||||||
char** exclude_patterns;
|
|
||||||
int exclude_count;
|
|
||||||
char** include_patterns;
|
|
||||||
int include_count;
|
|
||||||
unsigned long long max_size;
|
|
||||||
unsigned long long min_size;
|
|
||||||
int max_depth;
|
|
||||||
int current_depth;
|
int current_depth;
|
||||||
bool follow_symlinks;
|
|
||||||
bool copy_links;
|
|
||||||
bool safe_links;
|
|
||||||
bool copy_unsafe_links;
|
|
||||||
bool copy_dirlinks;
|
|
||||||
bool munge_links;
|
|
||||||
bool checksum;
|
|
||||||
bool one_file_system;
|
|
||||||
dev_t root_dev;
|
dev_t root_dev;
|
||||||
bool failed;
|
bool failed;
|
||||||
/* Phase 4 special/devices (see ScannerOptions). */
|
|
||||||
bool preserve_devices;
|
|
||||||
bool preserve_specials;
|
|
||||||
bool copy_devices;
|
|
||||||
/* Phase 2 (files-from / filter layer). */
|
/* Phase 2 (files-from / filter layer). */
|
||||||
char* root_path; /* transfer root (fs path) for rel computation */
|
char* root_path; /* transfer root (fs path) for rel computation */
|
||||||
char* current_rel; /* rel path of the open directory ("" == root) */
|
char* current_rel; /* rel path of the open directory ("" == root) */
|
||||||
@@ -155,40 +138,17 @@ typedef struct {
|
|||||||
FilterNode* seed_node; /* inherited context of the seed dir, or NULL */
|
FilterNode* seed_node; /* inherited context of the seed dir, or NULL */
|
||||||
FilterNode* current_node; /* filter context of the open directory */
|
FilterNode* current_node; /* filter context of the open directory */
|
||||||
ArrayList* filter_nodes; /* owned FilterNode arena (may be NULL) */
|
ArrayList* filter_nodes; /* owned FilterNode arena (may be NULL) */
|
||||||
const FileListSet* file_list;
|
/* --dirs / -R state for the directory-entry generator (options.dirs replaces
|
||||||
const FilterRuleList* base_filters;
|
|
||||||
bool per_dir_filters;
|
|
||||||
/* --dirs / -R state for the directory-entry generator (dirs_mode replaces
|
|
||||||
the recursive scan). */
|
the recursive scan). */
|
||||||
bool dirs_mode;
|
|
||||||
bool relative_mode; /* file_list && relative: send bare relative wire paths */
|
bool relative_mode; /* file_list && relative: send bare relative wire paths */
|
||||||
bool prune_empty_dirs;
|
|
||||||
bool dirs_root_emitted;
|
bool dirs_root_emitted;
|
||||||
int list_index;
|
int list_index;
|
||||||
ArrayList* dirs_batch; /* owned when non-NULL */
|
ArrayList* dirs_batch; /* owned when non-NULL */
|
||||||
unsigned long long dirs_batch_size;
|
unsigned long long dirs_batch_size;
|
||||||
/* Excluded-path sink (see ScannerOptions). `excluded_mutex` is shared across
|
|
||||||
parallel worker threads. */
|
|
||||||
ArrayList* excluded_paths;
|
|
||||||
mtx_t* excluded_mutex;
|
|
||||||
/* --ignore-errors: continue past unreadable directories (records io_error). */
|
|
||||||
bool ignore_io_errors;
|
|
||||||
/* --ignore-missing-args: --dirs listed-but-missing entries are skipped, not
|
|
||||||
fatal (see ScannerOptions.ignore_missing_args). */
|
|
||||||
bool ignore_missing_args;
|
|
||||||
/* A directory could not be opened (I/O error, e.g. EACCES). With
|
/* A directory could not be opened (I/O error, e.g. EACCES). With
|
||||||
--ignore-errors the scan continues past it and the caller decides what to
|
--ignore-errors the scan continues past it and the caller decides what to
|
||||||
do; `failed` is reserved for fatal errors that always abort the scan. */
|
do; `failed` is reserved for fatal errors that always abort the scan. */
|
||||||
bool io_error;
|
bool io_error;
|
||||||
/* --hard-links (-H): shared link-group detection table (see ScannerOptions).
|
|
||||||
NULL when -H is off. */
|
|
||||||
HardLinkTable* hardlinks;
|
|
||||||
/* Phase 6: sender stop deadline (from ScannerOptions). */
|
|
||||||
const StopCondition* stop_condition;
|
|
||||||
/* P7 Wave D directory-time capture (see ScannerOptions). */
|
|
||||||
bool capture_dir_times;
|
|
||||||
ArrayList* dir_entries;
|
|
||||||
mtx_t* dir_entries_mutex;
|
|
||||||
} DirectoryScanner;
|
} DirectoryScanner;
|
||||||
|
|
||||||
typedef struct {
|
typedef struct {
|
||||||
|
|||||||
+6
-1
@@ -2,6 +2,7 @@
|
|||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
#include <delta.h>
|
#include <delta.h>
|
||||||
#include <chunk.h>
|
#include <chunk.h>
|
||||||
|
#include "scanner.h"
|
||||||
|
|
||||||
void print_usage(void) {
|
void print_usage(void) {
|
||||||
printf("Usage:\n");
|
printf("Usage:\n");
|
||||||
@@ -144,7 +145,10 @@ void print_usage(void) {
|
|||||||
printf(" Delta block size in bytes (default: %d)\n", DELTA_BLOCK_SIZE_DEFAULT);
|
printf(" Delta block size in bytes (default: %d)\n", DELTA_BLOCK_SIZE_DEFAULT);
|
||||||
printf(" --delta-max <n> Max file size for delta transfer (default: %llu)\n",
|
printf(" --delta-max <n> Max file size for delta transfer (default: %llu)\n",
|
||||||
DELTA_MAX_FILE_SIZE);
|
DELTA_MAX_FILE_SIZE);
|
||||||
printf(" -j, --threads Enable multithreading\n");
|
printf(" -j, --threads[=N] Enable the multithreaded scanner/loader/sender\n");
|
||||||
|
printf(" pipeline; N (1-%d) sets the parallel scanner worker\n",
|
||||||
|
MAX_SCANNER_THREADS);
|
||||||
|
printf(" count (bare -j/--threads uses the default)\n");
|
||||||
printf(" --chunk-serialization Enable chunk serialization (long form only)\n");
|
printf(" --chunk-serialization Enable chunk serialization (long form only)\n");
|
||||||
printf(" -s, --secluded-args Protect-args compatibility option (no effect; remote\n");
|
printf(" -s, --secluded-args Protect-args compatibility option (no effect; remote\n");
|
||||||
printf(" SSH argv is already built injection-safe)\n");
|
printf(" SSH argv is already built injection-safe)\n");
|
||||||
@@ -203,6 +207,7 @@ void print_usage(void) {
|
|||||||
printf(" --save-to-disk Write received files to disk\n");
|
printf(" --save-to-disk Write received files to disk\n");
|
||||||
printf(" --server-host <ip> Server IP address (default: 127.0.0.1)\n");
|
printf(" --server-host <ip> Server IP address (default: 127.0.0.1)\n");
|
||||||
printf(" --server-port <n> Server port (default: 8080)\n");
|
printf(" --server-port <n> Server port (default: 8080)\n");
|
||||||
|
printf(" --port <n> Alias for --server-port\n");
|
||||||
printf(" --password-file <f> Authenticate a host::module/path daemon destination.\n");
|
printf(" --password-file <f> Authenticate a host::module/path daemon destination.\n");
|
||||||
printf(" The file's first user:password line supplies the\n");
|
printf(" The file's first user:password line supplies the\n");
|
||||||
printf(" username and password (only a SHA-256 digest of the\n");
|
printf(" username and password (only a SHA-256 digest of the\n");
|
||||||
|
|||||||
+136
-7
@@ -12,6 +12,7 @@
|
|||||||
#include "utils.h"
|
#include "utils.h"
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <sys/stat.h>
|
#include <sys/stat.h>
|
||||||
|
#include <time.h>
|
||||||
|
|
||||||
bool receiver_outcomes_append(ReceiverOutcomes* outcomes, unsigned char code) {
|
bool receiver_outcomes_append(ReceiverOutcomes* outcomes, unsigned char code) {
|
||||||
if (!outcomes)
|
if (!outcomes)
|
||||||
@@ -153,6 +154,93 @@ static bool receiver_process_batch(Config* config, int file_descriptor) {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* ---- Anti-slowloris connection bounds ----
|
||||||
|
* A legitimate transfer either streams data frames continuously or, when it
|
||||||
|
* must pause, sends STATUS_KEEPALIVE so the peer sees the connection is alive.
|
||||||
|
* An attacker can therefore squat on a connection slot indefinitely by sending
|
||||||
|
* only keepalives under the per-message timeout. Two CLOCK_MONOTONIC bounds
|
||||||
|
* defeat that without ever punishing a real transfer:
|
||||||
|
*
|
||||||
|
* MAX_SESSION_IDLE_SEC (1 h): the longest a stream may make no forward
|
||||||
|
* progress. Data/status frames count as progress and refresh the timer;
|
||||||
|
* keepalives do not. One hour is far longer than any real pause between
|
||||||
|
* data frames, yet small enough to reap a slowloris well before the 24 h
|
||||||
|
* session cap.
|
||||||
|
*
|
||||||
|
* MAX_SESSION_WALL_SEC (24 h): an absolute ceiling on one connection's
|
||||||
|
* lifetime as defense-in-depth against a trickle of progress frames that
|
||||||
|
* resets the idle timer just below its limit. Larger than any plausible
|
||||||
|
* single transfer while still bounding resource occupancy.
|
||||||
|
*
|
||||||
|
* Both are wall-clock deltas, so the per-message poll timeout (60 s by default,
|
||||||
|
* or --timeout) can never fool them, and both the single-threaded and the -m
|
||||||
|
* receiver paths (receiver_process_pending) share the same logic. */
|
||||||
|
#define MAX_SESSION_IDLE_SEC 3600u
|
||||||
|
#define MAX_SESSION_WALL_SEC 86400u
|
||||||
|
|
||||||
|
static unsigned int g_max_session_idle_sec = MAX_SESSION_IDLE_SEC;
|
||||||
|
static unsigned int g_max_session_wall_sec = MAX_SESSION_WALL_SEC;
|
||||||
|
|
||||||
|
void receiver_set_time_limits(unsigned int idle_sec, unsigned int wall_sec) {
|
||||||
|
g_max_session_idle_sec = idle_sec;
|
||||||
|
g_max_session_wall_sec = wall_sec;
|
||||||
|
}
|
||||||
|
|
||||||
|
void receiver_reset_time_limits(void) {
|
||||||
|
g_max_session_idle_sec = MAX_SESSION_IDLE_SEC;
|
||||||
|
g_max_session_wall_sec = MAX_SESSION_WALL_SEC;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool receiver_time_limit_exceeded(const struct timespec* session_start,
|
||||||
|
const struct timespec* last_progress,
|
||||||
|
const struct timespec* now) {
|
||||||
|
if (!session_start || !last_progress || !now)
|
||||||
|
return false;
|
||||||
|
if (now->tv_sec - session_start->tv_sec >= (time_t)g_max_session_wall_sec)
|
||||||
|
return true;
|
||||||
|
if (now->tv_sec - last_progress->tv_sec >= (time_t)g_max_session_idle_sec)
|
||||||
|
return true;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* A frame proves forward progress only when it cannot be fabricated for free.
|
||||||
|
* KEEPALIVE/ABORT are pure liveness, and CHECK_BATCH/DIR_TIMES may carry zero
|
||||||
|
* entries, so a peer must not be able to hold a connection slot forever by
|
||||||
|
* merely emitting empty frames. */
|
||||||
|
static bool status_counts_as_progress(Status status) {
|
||||||
|
switch (status) {
|
||||||
|
case STATUS_KEEPALIVE:
|
||||||
|
case STATUS_ABORT:
|
||||||
|
case STATUS_CHECK_BATCH:
|
||||||
|
case STATUS_DIR_TIMES:
|
||||||
|
return false;
|
||||||
|
default:
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Refresh the progress timestamp for a forward-moving frame and enforce the
|
||||||
|
* bounds above. Returns false when the connection must be dropped; the
|
||||||
|
* terminal STATUS_ERROR is sent only when the sink owns error reporting (the
|
||||||
|
* -m sink sets send_error=false so the main thread emits exactly one). */
|
||||||
|
static bool receiver_note_status(const struct timespec* session_start,
|
||||||
|
struct timespec* last_progress, Status status, int file_descriptor,
|
||||||
|
const ReceiverSink* sink) {
|
||||||
|
struct timespec now;
|
||||||
|
if (clock_gettime(CLOCK_MONOTONIC, &now) != 0)
|
||||||
|
now = *last_progress;
|
||||||
|
if (status_counts_as_progress(status))
|
||||||
|
*last_progress = now;
|
||||||
|
if (!receiver_time_limit_exceeded(session_start, last_progress, &now))
|
||||||
|
return true;
|
||||||
|
log_message(LOG_LEVEL_ERROR,
|
||||||
|
"Receive session exceeded its time bound (idle %us / total %us); aborting connection",
|
||||||
|
g_max_session_idle_sec, g_max_session_wall_sec);
|
||||||
|
if (!sink || sink->send_error)
|
||||||
|
send_status(file_descriptor, STATUS_ERROR);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
int receiver_process(Config* config, int file_descriptor, const ReceiverSink* sink) {
|
int receiver_process(Config* config, int file_descriptor, const ReceiverSink* sink) {
|
||||||
return receiver_process_pending(config, file_descriptor, sink, NULL);
|
return receiver_process_pending(config, file_descriptor, sink, NULL);
|
||||||
}
|
}
|
||||||
@@ -172,6 +260,15 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver
|
|||||||
Status status;
|
Status status;
|
||||||
if (!receive_status(file_descriptor, &status))
|
if (!receive_status(file_descriptor, &status))
|
||||||
return -1;
|
return -1;
|
||||||
|
/* Wall-clock (=CLOCK_MONOTONIC) anti-slowloris bookkeeping. session_start is
|
||||||
|
* fixed for the whole connection; last_progress is refreshed by every frame
|
||||||
|
* that is not a keepalive/abort. */
|
||||||
|
struct timespec session_start;
|
||||||
|
struct timespec last_progress;
|
||||||
|
clock_gettime(CLOCK_MONOTONIC, &session_start);
|
||||||
|
last_progress = session_start;
|
||||||
|
if (!receiver_note_status(&session_start, &last_progress, status, file_descriptor, sink))
|
||||||
|
return -1;
|
||||||
bool early_delete = config_delete_timing_early(config);
|
bool early_delete = config_delete_timing_early(config);
|
||||||
/* Parked keep-set for the late/commit timing. Every exit path below frees it
|
/* Parked keep-set for the late/commit timing. Every exit path below frees it
|
||||||
exactly once; the only exception is the successful FINISHED handoff, which
|
exactly once; the only exception is the successful FINISHED handoff, which
|
||||||
@@ -191,10 +288,19 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver
|
|||||||
goto fail;
|
goto fail;
|
||||||
}
|
}
|
||||||
if (status == STATUS_CHECK) {
|
if (status == STATUS_CHECK) {
|
||||||
bool skipped;
|
bool skipped = false;
|
||||||
File* file = receive_incremental_check(file_descriptor, config, &skipped);
|
bool would_transfer = false;
|
||||||
if (!skipped && (!file || !sink->store_file(file, sink->context)))
|
File* file = receive_incremental_check_ex(file_descriptor, config, &skipped, &would_transfer);
|
||||||
|
if (config->dry_run) {
|
||||||
|
/* Server-contacting --dry-run: the reply has already been sent
|
||||||
|
(STATUS_OK = up to date, STATUS_DRY_RUN_TRANSFER = would transfer) and
|
||||||
|
nothing may be stored. Both flags false means a genuine protocol
|
||||||
|
error (STATUS_ERROR already sent or sent by receive_error below). */
|
||||||
|
if (!skipped && !would_transfer)
|
||||||
|
goto receive_error;
|
||||||
|
} else if (!skipped && (!file || !sink->store_file(file, sink->context))) {
|
||||||
goto receive_error;
|
goto receive_error;
|
||||||
|
}
|
||||||
} else if (status == STATUS_CHUNK) {
|
} else if (status == STATUS_CHUNK) {
|
||||||
Chunk* chunk = receive_chunk_data(file_descriptor, config);
|
Chunk* chunk = receive_chunk_data(file_descriptor, config);
|
||||||
if (!chunk || !receiver_process_chunk(chunk, sink))
|
if (!chunk || !receiver_process_chunk(chunk, sink))
|
||||||
@@ -226,6 +332,15 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver
|
|||||||
DeleteManifest* manifest = receive_manifest_entries(file_descriptor);
|
DeleteManifest* manifest = receive_manifest_entries(file_descriptor);
|
||||||
if (!manifest)
|
if (!manifest)
|
||||||
goto fail; /* receive_manifest_entries already sent STATUS_ERROR */
|
goto fail; /* receive_manifest_entries already sent STATUS_ERROR */
|
||||||
|
if (config->dry_run) {
|
||||||
|
/* Server-contacting --dry-run mutates nothing, so a keep-set manifest
|
||||||
|
is consumed and discarded. The early-delete mode still needs its ACK
|
||||||
|
so a sender blocked on the delete handshake is not left hanging. */
|
||||||
|
delete_manifest_free(manifest);
|
||||||
|
if (early_delete && !send_status(file_descriptor, STATUS_OK))
|
||||||
|
goto fail;
|
||||||
|
goto next_status;
|
||||||
|
}
|
||||||
if (early_delete) {
|
if (early_delete) {
|
||||||
/* --delete-before / --delete-during: the manifest is authoritative the
|
/* --delete-before / --delete-during: the manifest is authoritative the
|
||||||
moment it arrives, before any file data. Delete now and acknowledge
|
moment it arrives, before any file data. Delete now and acknowledge
|
||||||
@@ -271,6 +386,8 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver
|
|||||||
next_status:
|
next_status:
|
||||||
if (!receive_status(file_descriptor, &status))
|
if (!receive_status(file_descriptor, &status))
|
||||||
goto receive_error;
|
goto receive_error;
|
||||||
|
if (!receiver_note_status(&session_start, &last_progress, status, file_descriptor, sink))
|
||||||
|
goto fail;
|
||||||
}
|
}
|
||||||
if (status != STATUS_FINISHED) {
|
if (status != STATUS_FINISHED) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Did not receive FINISHED Status");
|
log_message(LOG_LEVEL_ERROR, "Did not receive FINISHED Status");
|
||||||
@@ -342,7 +459,11 @@ typedef struct {
|
|||||||
static bool receiver_save_file(File* file, void* context_pointer) {
|
static bool receiver_save_file(File* file, void* context_pointer) {
|
||||||
ReceiverSaveContext* context = context_pointer;
|
ReceiverSaveContext* context = context_pointer;
|
||||||
FileSaveResult result = FILE_SAVE_ERROR;
|
FileSaveResult result = FILE_SAVE_ERROR;
|
||||||
if (!context->config->save_to_disk) {
|
if (context->config->dry_run) {
|
||||||
|
/* Defense in depth: a dry-run receiver mutates nothing even if a data
|
||||||
|
frame reaches the sink (the sender is not supposed to send one). */
|
||||||
|
result = FILE_SAVE_SKIPPED;
|
||||||
|
} else if (!context->config->save_to_disk) {
|
||||||
/* Nothing is stored; report the file as not-written so a
|
/* Nothing is stored; report the file as not-written so a
|
||||||
--remove-source-files sender keeps its source. */
|
--remove-source-files sender keeps its source. */
|
||||||
result = FILE_SAVE_SKIPPED;
|
result = FILE_SAVE_SKIPPED;
|
||||||
@@ -353,13 +474,17 @@ static bool receiver_save_file(File* file, void* context_pointer) {
|
|||||||
metadata now and apply it at the end. -O/--omit-dir-times is honored by
|
metadata now and apply it at the end. -O/--omit-dir-times is honored by
|
||||||
dir_time_list_apply's caller (see receiver_send_success_frame). */
|
dir_time_list_apply's caller (see receiver_send_success_frame). */
|
||||||
if (result != FILE_SAVE_ERROR && file->is_dir && file->metadata &&
|
if (result != FILE_SAVE_ERROR && file->is_dir && file->metadata &&
|
||||||
context->config->use_metadata && !context->config->omit_dir_times &&
|
dir_times_should_capture(context->config) &&
|
||||||
!dir_time_list_add(&context->dir_times, file->path, file->metadata)) {
|
!dir_time_list_add(&context->dir_times, file->path, file->metadata)) {
|
||||||
file_destroy(file);
|
file_destroy(file);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
if (result != FILE_SAVE_ERROR && context->config->remove_source_files && !file->is_dir &&
|
/* A dry-run receiver mutates nothing AND records no per-file outcomes: a
|
||||||
!file->is_special && !file->skip &&
|
hostile dry-run client that streamed data frames anyway must not be able to
|
||||||
|
grow `outcomes` without bound (receiver_outcomes_append reallocs uncharged)
|
||||||
|
or force a per-frame ack. */
|
||||||
|
if (!context->config->dry_run && result != FILE_SAVE_ERROR &&
|
||||||
|
context->config->remove_source_files && !file->is_dir && !file->is_special && !file->skip &&
|
||||||
!receiver_outcomes_append(&context->outcomes, (unsigned char)result)) {
|
!receiver_outcomes_append(&context->outcomes, (unsigned char)result)) {
|
||||||
file_destroy(file);
|
file_destroy(file);
|
||||||
return false;
|
return false;
|
||||||
@@ -370,6 +495,10 @@ static bool receiver_save_file(File* file, void* context_pointer) {
|
|||||||
|
|
||||||
static bool receiver_send_success_frame(int fd, void* context_pointer) {
|
static bool receiver_send_success_frame(int fd, void* context_pointer) {
|
||||||
ReceiverSaveContext* context = context_pointer;
|
ReceiverSaveContext* context = context_pointer;
|
||||||
|
/* Server-contacting --dry-run: nothing was staged or written, so there is
|
||||||
|
nothing to publish and no directory times to stamp. */
|
||||||
|
if (context->config->dry_run)
|
||||||
|
return receiver_send_final_success(fd, context->config, &context->outcomes);
|
||||||
/* --delay-updates: the whole protocol stream (including manifest/delete
|
/* --delay-updates: the whole protocol stream (including manifest/delete
|
||||||
handling, which ran inside receiver_process) has succeeded and every
|
handling, which ran inside receiver_process) has succeeded and every
|
||||||
staged file was fully written. Publish them atomically now, before the
|
staged file was fully written. Publish them atomically now, before the
|
||||||
|
|||||||
@@ -4,6 +4,8 @@
|
|||||||
#include "config.h"
|
#include "config.h"
|
||||||
#include "file.h"
|
#include "file.h"
|
||||||
#include "file_receive.h"
|
#include "file_receive.h"
|
||||||
|
#include <stdbool.h>
|
||||||
|
#include <time.h>
|
||||||
|
|
||||||
typedef bool (*ReceiverFileSink)(File* file, void* context);
|
typedef bool (*ReceiverFileSink)(File* file, void* context);
|
||||||
|
|
||||||
@@ -45,4 +47,22 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver
|
|||||||
DeleteManifest** pending_manifest);
|
DeleteManifest** pending_manifest);
|
||||||
int receiver_receive_files(Config* config, int file_descriptor);
|
int receiver_receive_files(Config* config, int file_descriptor);
|
||||||
|
|
||||||
|
/* ---- Connection time bounds (anti-slowloris) ----
|
||||||
|
* receiver_process_pending() aborts a connection that makes no forward progress
|
||||||
|
* (only STATUS_KEEPALIVE/STATUS_ABORT frames) beyond a wall-clock idle limit,
|
||||||
|
* and enforces a hard cap on the whole session. Both are CLOCK_MONOTONIC
|
||||||
|
* deltas, independent of the per-message poll deadline, so a 60 s (or
|
||||||
|
* --timeout) receive window can never reset them. Defaults are deliberately
|
||||||
|
* generous (see MAX_SESSION_IDLE_SEC / MAX_SESSION_WALL_SEC in receiver.c). */
|
||||||
|
|
||||||
|
/* Test seam: override the idle/session wall-clock limits (0 = abort on the
|
||||||
|
* next status). Always restore with receiver_reset_time_limits(). */
|
||||||
|
void receiver_set_time_limits(unsigned int idle_sec, unsigned int wall_sec);
|
||||||
|
void receiver_reset_time_limits(void);
|
||||||
|
/* Pure predicate over explicit monotonic timestamps, exposed so the bound is
|
||||||
|
* unit-testable without sleeping. True when either the idle or the overall
|
||||||
|
* session limit has elapsed. */
|
||||||
|
bool receiver_time_limit_exceeded(const struct timespec* session_start,
|
||||||
|
const struct timespec* last_progress, const struct timespec* now);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
@@ -0,0 +1,258 @@
|
|||||||
|
#include "receiver_pipeline.h"
|
||||||
|
|
||||||
|
#include "log.h"
|
||||||
|
#include "protocol.h"
|
||||||
|
#include "queue.h"
|
||||||
|
#include "utils.h"
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
|
#include <threads.h>
|
||||||
|
|
||||||
|
PipelineContextReceiver* pipeline_context_receiver_create(Config* config, Queue* queue,
|
||||||
|
int file_descriptor, SSL* ssl) {
|
||||||
|
PipelineContextReceiver* context = malloc(sizeof(PipelineContextReceiver));
|
||||||
|
if (context == NULL)
|
||||||
|
return NULL;
|
||||||
|
context->config = config;
|
||||||
|
context->queue = queue;
|
||||||
|
context->file_descriptor = file_descriptor;
|
||||||
|
context->ssl = ssl;
|
||||||
|
context->outcomes.entries = NULL;
|
||||||
|
context->outcomes.count = 0;
|
||||||
|
context->outcomes.capacity = 0;
|
||||||
|
dir_time_list_init(&context->dir_times);
|
||||||
|
protocol_session_init(&context->session, file_descriptor, file_descriptor);
|
||||||
|
protocol_session_set_ssl(&context->session, ssl);
|
||||||
|
context->receiver_done = false;
|
||||||
|
context->queued_bytes = 0;
|
||||||
|
context->max_queue_bytes = 0;
|
||||||
|
context->deferred_manifest = NULL;
|
||||||
|
atomic_init(&context->cancelled, false);
|
||||||
|
int init = 0;
|
||||||
|
if (mtx_init(&context->mutex, mtx_plain) != thrd_success)
|
||||||
|
goto fail;
|
||||||
|
init++;
|
||||||
|
if (cnd_init(&context->condition_not_full) != thrd_success)
|
||||||
|
goto fail;
|
||||||
|
init++;
|
||||||
|
if (cnd_init(&context->condition_not_empty) != thrd_success)
|
||||||
|
goto fail;
|
||||||
|
// cppcheck-suppress unreadVariable
|
||||||
|
init++;
|
||||||
|
return context;
|
||||||
|
|
||||||
|
fail:
|
||||||
|
log_perror("Error initializing synchronization objects");
|
||||||
|
if (init >= 3)
|
||||||
|
cnd_destroy(&context->condition_not_empty);
|
||||||
|
if (init >= 2)
|
||||||
|
cnd_destroy(&context->condition_not_full);
|
||||||
|
if (init >= 1)
|
||||||
|
mtx_destroy(&context->mutex);
|
||||||
|
free(context);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
void pipeline_context_receiver_destroy(PipelineContextReceiver* context) {
|
||||||
|
config_delete(context->config);
|
||||||
|
if (context->deferred_manifest)
|
||||||
|
delete_manifest_free(context->deferred_manifest);
|
||||||
|
queue_destroy(context->queue);
|
||||||
|
receiver_outcomes_destroy(&context->outcomes);
|
||||||
|
dir_time_list_free(&context->dir_times);
|
||||||
|
mtx_destroy(&context->mutex);
|
||||||
|
cnd_destroy(&context->condition_not_full);
|
||||||
|
cnd_destroy(&context->condition_not_empty);
|
||||||
|
free(context);
|
||||||
|
}
|
||||||
|
|
||||||
|
void pipeline_context_receiver_set_queue_byte_limit(PipelineContextReceiver* context,
|
||||||
|
size_t max_bytes) {
|
||||||
|
if (context == NULL)
|
||||||
|
return;
|
||||||
|
mtx_lock(&context->mutex);
|
||||||
|
context->max_queue_bytes = max_bytes;
|
||||||
|
context->queued_bytes = 0;
|
||||||
|
cnd_broadcast(&context->condition_not_full);
|
||||||
|
mtx_unlock(&context->mutex);
|
||||||
|
}
|
||||||
|
|
||||||
|
void pipeline_context_receiver_note_bytes_released(PipelineContextReceiver* context,
|
||||||
|
size_t released_bytes) {
|
||||||
|
if (context == NULL || context->max_queue_bytes == 0 || released_bytes == 0)
|
||||||
|
return;
|
||||||
|
mtx_lock(&context->mutex);
|
||||||
|
if (released_bytes >= context->queued_bytes)
|
||||||
|
context->queued_bytes = 0;
|
||||||
|
else
|
||||||
|
context->queued_bytes -= released_bytes;
|
||||||
|
cnd_signal(&context->condition_not_full);
|
||||||
|
mtx_unlock(&context->mutex);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool pipeline_context_receiver_enqueue_file(PipelineContextReceiver* context, File* file) {
|
||||||
|
if (context == NULL || file == NULL)
|
||||||
|
return false;
|
||||||
|
size_t file_bytes = file->data ? file->data->size : 0;
|
||||||
|
mtx_lock(&context->mutex);
|
||||||
|
while (!atomic_load(&context->cancelled)) {
|
||||||
|
bool blocked_by_count = queue_is_full(context->queue);
|
||||||
|
bool blocked_by_budget = false;
|
||||||
|
if (context->max_queue_bytes > 0) {
|
||||||
|
size_t budget = context->max_queue_bytes;
|
||||||
|
size_t used = context->queued_bytes;
|
||||||
|
if (used >= budget) {
|
||||||
|
blocked_by_budget = true;
|
||||||
|
} else if (file_bytes > budget - used) {
|
||||||
|
/* A single payload larger than the whole budget (not possible with
|
||||||
|
the per-file receive cap) is only admitted to an empty pipeline so
|
||||||
|
the wait can never deadlock. */
|
||||||
|
blocked_by_budget = used != 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!blocked_by_count && !blocked_by_budget)
|
||||||
|
break;
|
||||||
|
cnd_wait(&context->condition_not_full, &context->mutex);
|
||||||
|
}
|
||||||
|
if (atomic_load(&context->cancelled)) {
|
||||||
|
mtx_unlock(&context->mutex);
|
||||||
|
file_destroy(file);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (!queue_enqueue(context->queue, file)) {
|
||||||
|
mtx_unlock(&context->mutex);
|
||||||
|
file_destroy(file);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
context->queued_bytes += file_bytes;
|
||||||
|
cnd_signal(&context->condition_not_empty);
|
||||||
|
mtx_unlock(&context->mutex);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool receiver_enqueue_file(File* file, void* context_pointer) {
|
||||||
|
PipelineContextReceiver* context = (PipelineContextReceiver*)context_pointer;
|
||||||
|
return pipeline_context_receiver_enqueue_file(context, file);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void receiver_thread_fail(PipelineContextReceiver* context) {
|
||||||
|
mtx_lock(&context->mutex);
|
||||||
|
atomic_store(&context->cancelled, true);
|
||||||
|
context->receiver_done = true;
|
||||||
|
cnd_broadcast(&context->condition_not_empty);
|
||||||
|
cnd_broadcast(&context->condition_not_full);
|
||||||
|
mtx_unlock(&context->mutex);
|
||||||
|
}
|
||||||
|
|
||||||
|
int receive_thread(void* pipeline_context) {
|
||||||
|
PipelineContextReceiver* context = (PipelineContextReceiver*)pipeline_context;
|
||||||
|
protocol_session_bind(&context->session);
|
||||||
|
mtx_lock(&context->mutex);
|
||||||
|
int file_descriptor = context->file_descriptor;
|
||||||
|
const Config* config = context->config;
|
||||||
|
mtx_unlock(&context->mutex);
|
||||||
|
|
||||||
|
ReceiverSink sink = {receiver_enqueue_file, context, false, false, NULL};
|
||||||
|
if (receiver_process_pending((Config*)config, file_descriptor, &sink,
|
||||||
|
&context->deferred_manifest) != 0) {
|
||||||
|
receiver_thread_fail(context);
|
||||||
|
protocol_session_unbind();
|
||||||
|
return thrd_error;
|
||||||
|
}
|
||||||
|
mtx_lock(&context->mutex);
|
||||||
|
context->receiver_done = true;
|
||||||
|
cnd_signal(&context->condition_not_empty);
|
||||||
|
mtx_unlock(&context->mutex);
|
||||||
|
protocol_session_unbind();
|
||||||
|
return thrd_success;
|
||||||
|
}
|
||||||
|
|
||||||
|
int write_thread(void* pipeline_context) {
|
||||||
|
PipelineContextReceiver* context = (PipelineContextReceiver*)pipeline_context;
|
||||||
|
protocol_session_bind(&context->session);
|
||||||
|
mtx_lock(&context->mutex);
|
||||||
|
bool save_to_disk = context->config->save_to_disk;
|
||||||
|
char* root_directory = str_dup(context->config->receive_root_directory);
|
||||||
|
mtx_unlock(&context->mutex);
|
||||||
|
if (save_to_disk && !root_directory) {
|
||||||
|
mtx_lock(&context->mutex);
|
||||||
|
atomic_store(&context->cancelled, true);
|
||||||
|
context->receiver_done = true;
|
||||||
|
cnd_broadcast(&context->condition_not_full);
|
||||||
|
cnd_broadcast(&context->condition_not_empty);
|
||||||
|
mtx_unlock(&context->mutex);
|
||||||
|
protocol_session_unbind();
|
||||||
|
return thrd_error;
|
||||||
|
}
|
||||||
|
|
||||||
|
while (true) {
|
||||||
|
File* file =
|
||||||
|
queue_dequeue_multithreaded(context->queue, &context->mutex, &context->condition_not_empty,
|
||||||
|
&context->condition_not_full, &context->receiver_done);
|
||||||
|
if (file == NULL) {
|
||||||
|
free(root_directory);
|
||||||
|
protocol_session_unbind();
|
||||||
|
return thrd_success;
|
||||||
|
}
|
||||||
|
size_t file_bytes = file->data ? file->data->size : 0;
|
||||||
|
FileSaveResult result = FILE_SAVE_SKIPPED;
|
||||||
|
/* Server-contacting --dry-run: never write. The receiver thread does not
|
||||||
|
enqueue anything on the dry-run path, but this keeps the writer thread
|
||||||
|
provably mutation-free if a data frame ever reached it. */
|
||||||
|
bool dry_run = context->config->dry_run;
|
||||||
|
if (save_to_disk && !dry_run) {
|
||||||
|
result = file_save_to_disk_full(root_directory, file, context->config);
|
||||||
|
if (result == FILE_SAVE_ERROR) {
|
||||||
|
file_destroy(file);
|
||||||
|
pipeline_context_receiver_note_bytes_released(context, file_bytes);
|
||||||
|
mtx_lock(&context->mutex);
|
||||||
|
atomic_store(&context->cancelled, true);
|
||||||
|
context->receiver_done = true;
|
||||||
|
cnd_broadcast(&context->condition_not_full);
|
||||||
|
cnd_broadcast(&context->condition_not_empty);
|
||||||
|
mtx_unlock(&context->mutex);
|
||||||
|
free(root_directory);
|
||||||
|
protocol_session_unbind();
|
||||||
|
return thrd_error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
/* P7 Wave D: a directory's times are never applied inline (a later child
|
||||||
|
write would clobber them); accumulate the metadata here and let the
|
||||||
|
caller apply it once every writer has drained. */
|
||||||
|
if (!dry_run && result != FILE_SAVE_ERROR && file->is_dir && file->metadata &&
|
||||||
|
dir_times_should_capture(context->config) &&
|
||||||
|
!dir_time_list_add(&context->dir_times, file->path, file->metadata)) {
|
||||||
|
file_destroy(file);
|
||||||
|
pipeline_context_receiver_note_bytes_released(context, file_bytes);
|
||||||
|
mtx_lock(&context->mutex);
|
||||||
|
atomic_store(&context->cancelled, true);
|
||||||
|
context->receiver_done = true;
|
||||||
|
cnd_broadcast(&context->condition_not_full);
|
||||||
|
cnd_broadcast(&context->condition_not_empty);
|
||||||
|
mtx_unlock(&context->mutex);
|
||||||
|
free(root_directory);
|
||||||
|
protocol_session_unbind();
|
||||||
|
return thrd_error;
|
||||||
|
}
|
||||||
|
/* Record the per-file outcome so a --remove-source-files sender learns
|
||||||
|
which sources were actually written versus skipped on the receiver.
|
||||||
|
Explicit directory entries and recreated device/special nodes have no
|
||||||
|
source and are never acknowledged (mirrors receiver.c). */
|
||||||
|
if (!dry_run && context->config->remove_source_files && !file->is_dir && !file->is_special &&
|
||||||
|
!file->skip && !receiver_outcomes_append(&context->outcomes, (unsigned char)result)) {
|
||||||
|
file_destroy(file);
|
||||||
|
pipeline_context_receiver_note_bytes_released(context, file_bytes);
|
||||||
|
mtx_lock(&context->mutex);
|
||||||
|
atomic_store(&context->cancelled, true);
|
||||||
|
context->receiver_done = true;
|
||||||
|
cnd_broadcast(&context->condition_not_full);
|
||||||
|
cnd_broadcast(&context->condition_not_empty);
|
||||||
|
mtx_unlock(&context->mutex);
|
||||||
|
free(root_directory);
|
||||||
|
protocol_session_unbind();
|
||||||
|
return thrd_error;
|
||||||
|
}
|
||||||
|
file_destroy(file);
|
||||||
|
pipeline_context_receiver_note_bytes_released(context, file_bytes);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,68 @@
|
|||||||
|
#ifndef RECEIVER_PIPELINE_H
|
||||||
|
#define RECEIVER_PIPELINE_H
|
||||||
|
|
||||||
|
#include <stdatomic.h>
|
||||||
|
#include <stdbool.h>
|
||||||
|
#include <threads.h>
|
||||||
|
|
||||||
|
#include "config.h"
|
||||||
|
#include "file.h"
|
||||||
|
#include "file_receive.h"
|
||||||
|
#include "protocol.h"
|
||||||
|
#include "queue.h"
|
||||||
|
#include "receiver.h"
|
||||||
|
#include <openssl/ssl.h>
|
||||||
|
|
||||||
|
typedef struct PipelineContextReceiver {
|
||||||
|
Queue* queue;
|
||||||
|
Config* config;
|
||||||
|
int file_descriptor;
|
||||||
|
SSL* ssl;
|
||||||
|
ProtocolSession session;
|
||||||
|
ReceiverOutcomes outcomes;
|
||||||
|
mtx_t mutex;
|
||||||
|
cnd_t condition_not_full;
|
||||||
|
cnd_t condition_not_empty;
|
||||||
|
bool receiver_done;
|
||||||
|
atomic_bool cancelled;
|
||||||
|
/* Aggregate payload bytes that have been received but not yet released by
|
||||||
|
the disk writer (queued or in the writer's hand). Guarded by `mutex`.
|
||||||
|
When `max_queue_bytes` is non-zero the receiver blocks before enqueuing
|
||||||
|
once this total would exceed it, so decompressed/copied file payloads
|
||||||
|
buffered ahead of a slow disk writer respect the per-connection memory
|
||||||
|
budget instead of growing without bound. */
|
||||||
|
size_t queued_bytes;
|
||||||
|
size_t max_queue_bytes;
|
||||||
|
/* Keep-set manifest for the commit-style (late) deletion
|
||||||
|
(--delete/--delete-after/--delete-delay). receive_thread parses the whole
|
||||||
|
protocol stream but hands the manifest here instead of deleting while the
|
||||||
|
disk writer may still be draining; the caller (server.c) commits the
|
||||||
|
deletion after both threads have joined, so no extra is removed unless the
|
||||||
|
transfer truly succeeded. NULL in the early delete modes (which delete at
|
||||||
|
the manifest). */
|
||||||
|
DeleteManifest* deferred_manifest;
|
||||||
|
/* P7 Wave D: directory metadata collected by write_thread from received
|
||||||
|
directory entries. Only write_thread mutates it (before it joins); the
|
||||||
|
caller (server.c) applies it after the delete/delay-updates phase. */
|
||||||
|
DirTimeList dir_times;
|
||||||
|
} PipelineContextReceiver;
|
||||||
|
|
||||||
|
PipelineContextReceiver* pipeline_context_receiver_create(Config* config, Queue* queue_receiver,
|
||||||
|
int file_descriptor, SSL* ssl);
|
||||||
|
void pipeline_context_receiver_destroy(PipelineContextReceiver* context);
|
||||||
|
/* Bound the bytes buffered ahead of the disk writer (see max_queue_bytes). */
|
||||||
|
void pipeline_context_receiver_set_queue_byte_limit(PipelineContextReceiver* context,
|
||||||
|
size_t max_bytes);
|
||||||
|
/* Blocking enqueue used by the receive pipeline sink. Blocks while the queue
|
||||||
|
is full by element count or when adding `file` would push queued_bytes over
|
||||||
|
the configured byte limit; waits until the disk writer releases bytes.
|
||||||
|
Takes ownership of `file` on success and destroys it on failure/cancel. */
|
||||||
|
bool pipeline_context_receiver_enqueue_file(PipelineContextReceiver* context, File* file);
|
||||||
|
/* Account for `released_bytes` of payload memory that has been freed by the
|
||||||
|
disk writer, unblocking a receiver that is waiting on the byte limit. */
|
||||||
|
void pipeline_context_receiver_note_bytes_released(PipelineContextReceiver* context,
|
||||||
|
size_t released_bytes);
|
||||||
|
int receive_thread(void* pipeline_context);
|
||||||
|
int write_thread(void* pipeline_context);
|
||||||
|
|
||||||
|
#endif
|
||||||
+560
-239
File diff suppressed because it is too large
Load Diff
@@ -179,6 +179,8 @@ int server_cli_parse(int argc, char* argv[], ServerCliOptions* opts, char* err,
|
|||||||
opts->trust_sender = true;
|
opts->trust_sender = true;
|
||||||
} else if (arg_is(argv[i], "--no-super")) {
|
} else if (arg_is(argv[i], "--no-super")) {
|
||||||
opts->no_super = true;
|
opts->no_super = true;
|
||||||
|
} else if (arg_is(argv[i], "--allow-super")) {
|
||||||
|
opts->allow_super = true;
|
||||||
} else if (arg_is(argv[i], "--allow-unauthenticated")) {
|
} else if (arg_is(argv[i], "--allow-unauthenticated")) {
|
||||||
opts->allow_unauthenticated = true;
|
opts->allow_unauthenticated = true;
|
||||||
} else if (arg_has_value(argv[i], "--iconv", &inline_value)) {
|
} else if (arg_has_value(argv[i], "--iconv", &inline_value)) {
|
||||||
@@ -259,6 +261,28 @@ int server_cli_parse(int argc, char* argv[], ServerCliOptions* opts, char* err,
|
|||||||
set_error(err, err_size, "--hash-credentials cannot be combined with --daemon or --stdio");
|
set_error(err, err_size, "--hash-credentials cannot be combined with --daemon or --stdio");
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
if (opts->allow_super && opts->no_super) {
|
||||||
|
set_error(err, err_size, "--allow-super and --no-super are mutually exclusive");
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
if (opts->allow_super && opts->daemon_mode) {
|
||||||
|
set_error(err, err_size,
|
||||||
|
"--allow-super is for a locally-launched standalone TCP server; daemon modules opt "
|
||||||
|
"in per module with 'client owner = yes'");
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
/* --stdio is the SSH transport: the remote server argv is composed by the
|
||||||
|
* CLIENT (directly and via --remote-option), so a client could otherwise pass
|
||||||
|
* --allow-super to a root --stdio receiver and defeat the C3 secure default.
|
||||||
|
* Never honor it there; the super mode stays forced OFF. An operator who
|
||||||
|
* must keep the historical permissive behavior over SSH has to launch the
|
||||||
|
* receiver through a forced command, not via client-composed argv. */
|
||||||
|
if (opts->allow_super && opts->stdio_mode) {
|
||||||
|
set_error(err, err_size,
|
||||||
|
"--allow-super is not accepted with --stdio (the remote argv is client-composed; "
|
||||||
|
"use a forced command if the default must hold)");
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
if (opts->hash_iterations_set && opts->hash_credentials_file == NULL) {
|
if (opts->hash_iterations_set && opts->hash_credentials_file == NULL) {
|
||||||
set_error(err, err_size, "--iterations requires --hash-credentials");
|
set_error(err, err_size, "--iterations requires --hash-credentials");
|
||||||
return -1;
|
return -1;
|
||||||
|
|||||||
@@ -45,6 +45,17 @@ typedef struct ServerCliOptions {
|
|||||||
* device-node creation) even when running as root. Applies to --stdio and
|
* device-node creation) even when running as root. Applies to --stdio and
|
||||||
* --daemon alike; also makes the server refuse any client --copy-as. */
|
* --daemon alike; also makes the server refuse any client --copy-as. */
|
||||||
bool no_super; /* --no-super */
|
bool no_super; /* --no-super */
|
||||||
|
/* --allow-super: locally-launched standalone TCP listener opt-in that keeps
|
||||||
|
* the historical permissive behavior for a PRIVILEGED (root) receiver.
|
||||||
|
* Without it a root standalone server forces SUPER_MODE_OFF, so a client
|
||||||
|
* --devices / --write-devices / --super / ownership request cannot make it
|
||||||
|
* create device nodes, write raw devices, or apply client-chosen ownership.
|
||||||
|
* It is rejected for --stdio: that path's remote argv is composed by the
|
||||||
|
* client (directly and via --remote-option), so it must never opt a root
|
||||||
|
* receiver back into super mode. Non-root receivers are unaffected (the
|
||||||
|
* kernel refuses the confined attempts). The daemon path instead uses the
|
||||||
|
* per-module `client owner = yes` opt-in. */
|
||||||
|
bool allow_super; /* --allow-super */
|
||||||
/* --iconv=CONVERT_SPEC: the server's own LOCAL charset declaration. The
|
/* --iconv=CONVERT_SPEC: the server's own LOCAL charset declaration. The
|
||||||
* client's full spec rides the wire config frame anyway; when the server is
|
* client's full spec rides the wire config frame anyway; when the server is
|
||||||
* started with its own --iconv, its LOCAL half overrides the local charset
|
* started with its own --iconv, its LOCAL half overrides the local charset
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
#include "log.h"
|
#include "log.h"
|
||||||
#include "array_list.h"
|
#include "array_list.h"
|
||||||
#include "protocol.h"
|
#include "protocol.h"
|
||||||
|
#include <limits.h>
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
@@ -39,6 +40,8 @@ void array_list_delete(ArrayList* array_list) {
|
|||||||
static bool array_list_extend(ArrayList* array_list) {
|
static bool array_list_extend(ArrayList* array_list) {
|
||||||
if (array_list == NULL)
|
if (array_list == NULL)
|
||||||
return false;
|
return false;
|
||||||
|
if (array_list->capacity > INT_MAX / 2)
|
||||||
|
return false;
|
||||||
int new_capacity = array_list->capacity * 2;
|
int new_capacity = array_list->capacity * 2;
|
||||||
if (new_capacity == 0)
|
if (new_capacity == 0)
|
||||||
new_capacity = INITIAL_ARRAY_SIZE;
|
new_capacity = INITIAL_ARRAY_SIZE;
|
||||||
|
|||||||
+100
-96
@@ -1,6 +1,7 @@
|
|||||||
#include <stddef.h>
|
#include <stddef.h>
|
||||||
#include <stdint.h>
|
#include <stdint.h>
|
||||||
#include <limits.h>
|
#include <limits.h>
|
||||||
|
#include <stdatomic.h>
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
@@ -20,6 +21,33 @@
|
|||||||
#define MAX_FILE_DATA_SIZE (64ULL * 1024 * 1024)
|
#define MAX_FILE_DATA_SIZE (64ULL * 1024 * 1024)
|
||||||
#define MAX_FILES_PER_CHUNK 65536U
|
#define MAX_FILES_PER_CHUNK 65536U
|
||||||
|
|
||||||
|
/* Reserve `charge` against `session`'s connection budget. This mirrors the
|
||||||
|
static protocol_reserve_memory() in protocol.c: the receive-side call sites
|
||||||
|
only have the Data.owner pointer (a ProtocolSession*), and protocol.c is out
|
||||||
|
of scope for this fix, so the same atomic CAS accounting is reproduced here.
|
||||||
|
The matching release always goes through data_destroy()'s Data.owner path. */
|
||||||
|
static bool chunk_session_reserve(ProtocolSession* session, size_t charge) {
|
||||||
|
unsigned long long allocated = atomic_load(&session->total_allocated_bytes);
|
||||||
|
while (true) {
|
||||||
|
if (allocated > MAX_CONNECTION_MEMORY ||
|
||||||
|
(unsigned long long)charge > MAX_CONNECTION_MEMORY - allocated)
|
||||||
|
return false;
|
||||||
|
if (atomic_compare_exchange_weak(&session->total_allocated_bytes, &allocated,
|
||||||
|
allocated + (unsigned long long)charge))
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
bool data_charge_session(Data* data, ProtocolSession* session, size_t charge) {
|
||||||
|
if (!data || charge == 0 || session == NULL)
|
||||||
|
return true;
|
||||||
|
if (!chunk_session_reserve(session, charge))
|
||||||
|
return false;
|
||||||
|
data->owner = session;
|
||||||
|
data->protocol_charge = charge;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
Chunk* chunk_create(File** items, int element_count) {
|
Chunk* chunk_create(File** items, int element_count) {
|
||||||
if (element_count < 0 || (element_count > 0 && items == NULL))
|
if (element_count < 0 || (element_count > 0 && items == NULL))
|
||||||
return NULL;
|
return NULL;
|
||||||
@@ -207,17 +235,20 @@ Chunk* chunk_deserialize(Data* data, bool use_metadata) {
|
|||||||
return NULL;
|
return NULL;
|
||||||
char* data_pointer = data->data;
|
char* data_pointer = data->data;
|
||||||
size_t remaining_size = data->size;
|
size_t remaining_size = data->size;
|
||||||
|
/* The element currently being parsed is owned by `files` only after the
|
||||||
|
* array_list_add() at the end of the iteration; until then the error
|
||||||
|
* epilogue destroys it directly. Keeping this one pointer nulled after the
|
||||||
|
* hand-off makes the single cleanup path correct for every failure. */
|
||||||
|
File* file = NULL;
|
||||||
|
|
||||||
while (remaining_size > 0) {
|
while (remaining_size > 0) {
|
||||||
if ((unsigned int)files->size >= MAX_FILES_PER_CHUNK) {
|
if ((unsigned int)files->size >= MAX_FILES_PER_CHUNK) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Chunk contains too many files");
|
log_message(LOG_LEVEL_ERROR, "Chunk contains too many files");
|
||||||
array_list_delete(files);
|
goto error;
|
||||||
return NULL;
|
|
||||||
}
|
}
|
||||||
if (remaining_size < sizeof(size_t)) {
|
if (remaining_size < sizeof(size_t)) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for path length");
|
log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for path length");
|
||||||
array_list_delete(files);
|
goto error;
|
||||||
return NULL;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t path_len;
|
size_t path_len;
|
||||||
@@ -227,26 +258,19 @@ Chunk* chunk_deserialize(Data* data, bool use_metadata) {
|
|||||||
|
|
||||||
if (path_len > SIZE_MAX - 1 || remaining_size < path_len) {
|
if (path_len > SIZE_MAX - 1 || remaining_size < path_len) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for path");
|
log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for path");
|
||||||
array_list_delete(files);
|
goto error;
|
||||||
return NULL;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (path_len == SIZE_MAX) {
|
|
||||||
array_list_delete(files);
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
char* path = protocol_alloc(path_len + 1);
|
char* path = protocol_alloc(path_len + 1);
|
||||||
if (path == NULL) {
|
if (path == NULL) {
|
||||||
log_perror("Could not allocate memory for file path");
|
log_perror("Could not allocate memory for file path");
|
||||||
array_list_delete(files);
|
goto error;
|
||||||
return NULL;
|
|
||||||
}
|
}
|
||||||
memcpy(path, data_pointer, path_len);
|
memcpy(path, data_pointer, path_len);
|
||||||
path[path_len] = '\0';
|
path[path_len] = '\0';
|
||||||
if (memchr(path, '\0', path_len) != NULL) {
|
if (memchr(path, '\0', path_len) != NULL) {
|
||||||
free(path);
|
free(path);
|
||||||
array_list_delete(files);
|
goto error;
|
||||||
return NULL;
|
|
||||||
}
|
}
|
||||||
data_pointer += path_len;
|
data_pointer += path_len;
|
||||||
remaining_size -= path_len;
|
remaining_size -= path_len;
|
||||||
@@ -260,8 +284,7 @@ Chunk* chunk_deserialize(Data* data, bool use_metadata) {
|
|||||||
if (local_path == NULL) {
|
if (local_path == NULL) {
|
||||||
log_message(LOG_LEVEL_ERROR,
|
log_message(LOG_LEVEL_ERROR,
|
||||||
"--iconv: received chunk file name cannot be converted to the local charset");
|
"--iconv: received chunk file name cannot be converted to the local charset");
|
||||||
array_list_delete(files);
|
goto error;
|
||||||
return NULL;
|
|
||||||
}
|
}
|
||||||
path = local_path;
|
path = local_path;
|
||||||
path_len = strlen(path);
|
path_len = strlen(path);
|
||||||
@@ -269,30 +292,23 @@ Chunk* chunk_deserialize(Data* data, bool use_metadata) {
|
|||||||
|
|
||||||
if (path_len == 0 || has_path_traversal(path)) {
|
if (path_len == 0 || has_path_traversal(path)) {
|
||||||
free(path);
|
free(path);
|
||||||
array_list_delete(files);
|
goto error;
|
||||||
return NULL;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
File* file = file_create(path);
|
file = file_create(path);
|
||||||
free(path);
|
free(path);
|
||||||
if (file == NULL) {
|
if (file == NULL)
|
||||||
array_list_delete(files);
|
goto error;
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (remaining_size < sizeof(int)) {
|
if (remaining_size < sizeof(int)) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for entry type");
|
log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for entry type");
|
||||||
file_destroy(file);
|
goto error;
|
||||||
array_list_delete(files);
|
|
||||||
return NULL;
|
|
||||||
}
|
}
|
||||||
int entry_type;
|
int entry_type;
|
||||||
memcpy(&entry_type, data_pointer, sizeof(int));
|
memcpy(&entry_type, data_pointer, sizeof(int));
|
||||||
if (entry_type != 0 && entry_type != 1 && entry_type != 2 && entry_type != 3) {
|
if (entry_type != 0 && entry_type != 1 && entry_type != 2 && entry_type != 3) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Invalid chunk format: bad entry type");
|
log_message(LOG_LEVEL_ERROR, "Invalid chunk format: bad entry type");
|
||||||
file_destroy(file);
|
goto error;
|
||||||
array_list_delete(files);
|
|
||||||
return NULL;
|
|
||||||
}
|
}
|
||||||
file->is_dir = entry_type == 1;
|
file->is_dir = entry_type == 1;
|
||||||
file->is_symlink = entry_type == 2;
|
file->is_symlink = entry_type == 2;
|
||||||
@@ -303,9 +319,7 @@ Chunk* chunk_deserialize(Data* data, bool use_metadata) {
|
|||||||
if (file->is_special) {
|
if (file->is_special) {
|
||||||
if (remaining_size < 2 * (int32_t)sizeof(int32_t)) {
|
if (remaining_size < 2 * (int32_t)sizeof(int32_t)) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for special rdev");
|
log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for special rdev");
|
||||||
file_destroy(file);
|
goto error;
|
||||||
array_list_delete(files);
|
|
||||||
return NULL;
|
|
||||||
}
|
}
|
||||||
int32_t special_major, special_minor;
|
int32_t special_major, special_minor;
|
||||||
memcpy(&special_major, data_pointer, sizeof(special_major));
|
memcpy(&special_major, data_pointer, sizeof(special_major));
|
||||||
@@ -320,9 +334,7 @@ Chunk* chunk_deserialize(Data* data, bool use_metadata) {
|
|||||||
if (special_major < 0 || special_minor < 0 || special_major > 0xffff ||
|
if (special_major < 0 || special_minor < 0 || special_major > 0xffff ||
|
||||||
special_minor > 0x00ffffff) {
|
special_minor > 0x00ffffff) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Invalid chunk format: out-of-range special rdev");
|
log_message(LOG_LEVEL_ERROR, "Invalid chunk format: out-of-range special rdev");
|
||||||
file_destroy(file);
|
goto error;
|
||||||
array_list_delete(files);
|
|
||||||
return NULL;
|
|
||||||
}
|
}
|
||||||
file->rdev_major = special_major;
|
file->rdev_major = special_major;
|
||||||
file->rdev_minor = special_minor;
|
file->rdev_minor = special_minor;
|
||||||
@@ -331,37 +343,32 @@ Chunk* chunk_deserialize(Data* data, bool use_metadata) {
|
|||||||
if (use_metadata) {
|
if (use_metadata) {
|
||||||
if (remaining_size < sizeof(int)) {
|
if (remaining_size < sizeof(int)) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for metadata");
|
log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for metadata");
|
||||||
file_destroy(file);
|
goto error;
|
||||||
array_list_delete(files);
|
|
||||||
return NULL;
|
|
||||||
}
|
}
|
||||||
// Peek at present flag to determine total size needed before reading
|
/* Peek at the present flag to determine the total record size before
|
||||||
|
decoding. metadata_from_buf() independently bounds-checks every read
|
||||||
|
against remaining_size, so a short body can never over-read. */
|
||||||
int present_flag;
|
int present_flag;
|
||||||
memcpy(&present_flag, data_pointer, sizeof(int));
|
memcpy(&present_flag, data_pointer, sizeof(int));
|
||||||
if ((present_flag != 0 && present_flag != 1) ||
|
if ((present_flag != 0 && present_flag != 1) ||
|
||||||
(present_flag == 1 && remaining_size < sizeof(int) + FILE_METADATA_WIRE_SIZE)) {
|
(present_flag == 1 && remaining_size < sizeof(int) + FILE_METADATA_WIRE_SIZE)) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for metadata body");
|
log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for metadata body");
|
||||||
file_destroy(file);
|
goto error;
|
||||||
array_list_delete(files);
|
|
||||||
return NULL;
|
|
||||||
}
|
}
|
||||||
file->metadata = metadata_from_buf(&data_pointer);
|
file->metadata = metadata_from_buf((const uint8_t*)data_pointer, remaining_size);
|
||||||
remaining_size -= sizeof(int);
|
size_t metadata_consumed = sizeof(int);
|
||||||
if (present_flag == 1) {
|
if (present_flag == 1) {
|
||||||
if (file->metadata == NULL) {
|
if (file->metadata == NULL)
|
||||||
file_destroy(file);
|
goto error;
|
||||||
array_list_delete(files);
|
metadata_consumed += FILE_METADATA_WIRE_SIZE;
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
remaining_size -= FILE_METADATA_WIRE_SIZE;
|
|
||||||
}
|
}
|
||||||
|
data_pointer += metadata_consumed;
|
||||||
|
remaining_size -= metadata_consumed;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (remaining_size < sizeof(size_t)) {
|
if (remaining_size < sizeof(size_t)) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for data size");
|
log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for data size");
|
||||||
file_destroy(file);
|
goto error;
|
||||||
array_list_delete(files);
|
|
||||||
return NULL;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t file_data_size;
|
size_t file_data_size;
|
||||||
@@ -371,34 +378,34 @@ Chunk* chunk_deserialize(Data* data, bool use_metadata) {
|
|||||||
|
|
||||||
if (remaining_size < file_data_size) {
|
if (remaining_size < file_data_size) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for file content");
|
log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for file content");
|
||||||
file_destroy(file);
|
goto error;
|
||||||
array_list_delete(files);
|
|
||||||
return NULL;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Reject individual file data larger than the maximum allowed size.
|
// Reject individual file data larger than the maximum allowed size.
|
||||||
if (file_data_size > MAX_FILE_DATA_SIZE) {
|
if (file_data_size > MAX_FILE_DATA_SIZE) {
|
||||||
log_message(LOG_LEVEL_ERROR, "File data size %zu exceeds maximum %llu", file_data_size,
|
log_message(LOG_LEVEL_ERROR, "File data size %zu exceeds maximum %llu", file_data_size,
|
||||||
(unsigned long long)MAX_FILE_DATA_SIZE);
|
(unsigned long long)MAX_FILE_DATA_SIZE);
|
||||||
file_destroy(file);
|
goto error;
|
||||||
array_list_delete(files);
|
|
||||||
return NULL;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t allocation_size = file_data_size > 0 ? file_data_size : 1;
|
size_t allocation_size = file_data_size > 0 ? file_data_size : 1;
|
||||||
void* file_data = protocol_alloc(allocation_size);
|
void* file_data = protocol_alloc(allocation_size);
|
||||||
if (file_data == NULL) {
|
if (file_data == NULL) {
|
||||||
log_perror("Could not allocate memory for file data");
|
log_perror("Could not allocate memory for file data");
|
||||||
file_destroy(file);
|
goto error;
|
||||||
array_list_delete(files);
|
|
||||||
return NULL;
|
|
||||||
}
|
}
|
||||||
memcpy(file_data, data_pointer, file_data_size);
|
memcpy(file_data, data_pointer, file_data_size);
|
||||||
Data* replacement = data_create(file_data, file_data_size);
|
Data* replacement = data_create(file_data, file_data_size);
|
||||||
if (replacement == NULL) {
|
if (replacement == NULL)
|
||||||
file_destroy(file);
|
goto error;
|
||||||
array_list_delete(files);
|
/* Charge the retained per-file copy to the connection budget (when the
|
||||||
return NULL;
|
inbound chunk carries an owning session) so the queued copies are not
|
||||||
|
held outside MAX_CONNECTION_MEMORY (B6). A NULL owner (e.g. a local
|
||||||
|
batch apply) leaves the copy uncharged. */
|
||||||
|
if (!data_charge_session(replacement, data->owner, allocation_size)) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Per-connection memory limit exceeded for chunk file data");
|
||||||
|
data_destroy(replacement);
|
||||||
|
goto error;
|
||||||
}
|
}
|
||||||
data_destroy(file->data);
|
data_destroy(file->data);
|
||||||
file->data = replacement;
|
file->data = replacement;
|
||||||
@@ -408,9 +415,7 @@ Chunk* chunk_deserialize(Data* data, bool use_metadata) {
|
|||||||
if (file->is_symlink) {
|
if (file->is_symlink) {
|
||||||
if (remaining_size < sizeof(size_t)) {
|
if (remaining_size < sizeof(size_t)) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for symlink target");
|
log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for symlink target");
|
||||||
file_destroy(file);
|
goto error;
|
||||||
array_list_delete(files);
|
|
||||||
return NULL;
|
|
||||||
}
|
}
|
||||||
size_t target_len;
|
size_t target_len;
|
||||||
memcpy(&target_len, data_pointer, sizeof(size_t));
|
memcpy(&target_len, data_pointer, sizeof(size_t));
|
||||||
@@ -418,24 +423,18 @@ Chunk* chunk_deserialize(Data* data, bool use_metadata) {
|
|||||||
remaining_size -= sizeof(size_t);
|
remaining_size -= sizeof(size_t);
|
||||||
if (target_len == 0 || remaining_size < target_len) {
|
if (target_len == 0 || remaining_size < target_len) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Invalid chunk format: bad symlink target");
|
log_message(LOG_LEVEL_ERROR, "Invalid chunk format: bad symlink target");
|
||||||
file_destroy(file);
|
goto error;
|
||||||
array_list_delete(files);
|
|
||||||
return NULL;
|
|
||||||
}
|
}
|
||||||
char* target = protocol_alloc(target_len + 1);
|
char* target = protocol_alloc(target_len + 1);
|
||||||
if (!target) {
|
if (!target) {
|
||||||
log_perror("Could not allocate memory for symlink target");
|
log_perror("Could not allocate memory for symlink target");
|
||||||
file_destroy(file);
|
goto error;
|
||||||
array_list_delete(files);
|
|
||||||
return NULL;
|
|
||||||
}
|
}
|
||||||
memcpy(target, data_pointer, target_len);
|
memcpy(target, data_pointer, target_len);
|
||||||
target[target_len] = '\0';
|
target[target_len] = '\0';
|
||||||
if (memchr(target, '\0', target_len) != NULL) {
|
if (memchr(target, '\0', target_len) != NULL) {
|
||||||
free(target);
|
free(target);
|
||||||
file_destroy(file);
|
goto error;
|
||||||
array_list_delete(files);
|
|
||||||
return NULL;
|
|
||||||
}
|
}
|
||||||
/* The symlink target also rides the wire charset; decode it to the local
|
/* The symlink target also rides the wire charset; decode it to the local
|
||||||
charset like the path (a target is a path). */
|
charset like the path (a target is a path). */
|
||||||
@@ -446,9 +445,7 @@ Chunk* chunk_deserialize(Data* data, bool use_metadata) {
|
|||||||
log_message(LOG_LEVEL_ERROR,
|
log_message(LOG_LEVEL_ERROR,
|
||||||
"--iconv: received chunk symlink target cannot be converted to the local "
|
"--iconv: received chunk symlink target cannot be converted to the local "
|
||||||
"charset");
|
"charset");
|
||||||
file_destroy(file);
|
goto error;
|
||||||
array_list_delete(files);
|
|
||||||
return NULL;
|
|
||||||
}
|
}
|
||||||
target = local_target;
|
target = local_target;
|
||||||
}
|
}
|
||||||
@@ -457,29 +454,27 @@ Chunk* chunk_deserialize(Data* data, bool use_metadata) {
|
|||||||
remaining_size -= target_len;
|
remaining_size -= target_len;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!array_list_add(files, file)) {
|
if (!array_list_add(files, file))
|
||||||
file_destroy(file);
|
goto error;
|
||||||
array_list_delete(files);
|
file = NULL;
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
File** file_array = (File**)array_list_to_array(files);
|
File** file_array = (File**)array_list_to_array(files);
|
||||||
if (files->size > 0 && file_array == NULL) {
|
if (files->size > 0 && file_array == NULL)
|
||||||
array_list_delete(files);
|
goto error;
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
Chunk* chunk = chunk_create(file_array, files->size);
|
Chunk* chunk = chunk_create(file_array, files->size);
|
||||||
|
|
||||||
free(file_array);
|
free(file_array);
|
||||||
if (chunk == NULL) {
|
if (chunk == NULL)
|
||||||
array_list_delete(files);
|
goto error;
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
files->item_destroyer = NULL;
|
files->item_destroyer = NULL;
|
||||||
array_list_delete(files);
|
array_list_delete(files);
|
||||||
|
|
||||||
return chunk;
|
return chunk;
|
||||||
|
|
||||||
|
error:
|
||||||
|
if (file)
|
||||||
|
file_destroy(file);
|
||||||
|
array_list_delete(files);
|
||||||
|
return NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
Data* chunk_compress(Chunk* chunk, int compression_level, bool use_metadata) {
|
Data* chunk_compress(Chunk* chunk, int compression_level, bool use_metadata) {
|
||||||
@@ -508,12 +503,21 @@ Chunk* receive_chunk_data(int fd, const Config* config) {
|
|||||||
}
|
}
|
||||||
Data* data_to_process = chunk_data;
|
Data* data_to_process = chunk_data;
|
||||||
if (config->use_compression) {
|
if (config->use_compression) {
|
||||||
|
/* Preserve the inbound session across decompression so the (larger)
|
||||||
|
decompressed chunk is charged to the same connection budget; the
|
||||||
|
compressed buffer's own charge is released by data_destroy below. */
|
||||||
|
ProtocolSession* owner = chunk_data->owner;
|
||||||
data_to_process = data_decompress_limited(chunk_data, MAX_CHUNK_SIZE);
|
data_to_process = data_decompress_limited(chunk_data, MAX_CHUNK_SIZE);
|
||||||
data_destroy(chunk_data);
|
data_destroy(chunk_data);
|
||||||
if (data_to_process == NULL) {
|
if (data_to_process == NULL) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Failed to decompress chunk");
|
log_message(LOG_LEVEL_ERROR, "Failed to decompress chunk");
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
if (!data_charge_session(data_to_process, owner, data_to_process->size)) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Per-connection memory limit exceeded for decompressed chunk");
|
||||||
|
data_destroy(data_to_process);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Reject chunks larger than the maximum allowed size to prevent OOM.
|
// Reject chunks larger than the maximum allowed size to prevent OOM.
|
||||||
|
|||||||
@@ -23,4 +23,15 @@ Data* chunk_compress_with_threads(Chunk* chunk, int compression_level, bool use_
|
|||||||
int compression_threads);
|
int compression_threads);
|
||||||
Chunk* receive_chunk_data(int fd, const Config* config);
|
Chunk* receive_chunk_data(int fd, const Config* config);
|
||||||
|
|
||||||
|
/* Charge `charge` retained bytes of `data` against `session`'s per-connection
|
||||||
|
* budget (MAX_CONNECTION_MEMORY), mirroring the protocol layer's accounting, and
|
||||||
|
* record them on `data` so data_destroy() returns the charge through the
|
||||||
|
* Data.owner path. Returns false (leaving `data` uncharged) when the ceiling
|
||||||
|
* would be exceeded. A NULL/zero-size charge or a NULL session is a no-op
|
||||||
|
* success. The receive-side decompression and chunk-copy paths know the owning
|
||||||
|
* session only through the Data.owner of the buffer they are processing, so
|
||||||
|
* this is the entry point that lets them participate in the connection budget
|
||||||
|
* without a session handle (B6). */
|
||||||
|
bool data_charge_session(Data* data, ProtocolSession* session, size_t charge);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
+203
-52
@@ -2,11 +2,12 @@
|
|||||||
#include "data.h"
|
#include "data.h"
|
||||||
#include "log.h"
|
#include "log.h"
|
||||||
#include "protocol.h"
|
#include "protocol.h"
|
||||||
#include <stdlib.h>
|
|
||||||
#include <limits.h>
|
#include <limits.h>
|
||||||
#include <stdint.h>
|
#include <stdint.h>
|
||||||
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
#include <strings.h>
|
#include <strings.h>
|
||||||
|
#include <threads.h>
|
||||||
#include <unistd.h>
|
#include <unistd.h>
|
||||||
#include <zstd.h>
|
#include <zstd.h>
|
||||||
|
|
||||||
@@ -16,10 +17,6 @@
|
|||||||
static char* SKIP_COMPRESSION_EXTENSIONS[] = {".jpg", ".jpeg", ".png", ".gif", ".mp4", ".mkv",
|
static char* SKIP_COMPRESSION_EXTENSIONS[] = {".jpg", ".jpeg", ".png", ".gif", ".mp4", ".mkv",
|
||||||
".zip", ".gz", ".xz", ".zst", NULL};
|
".zip", ".gz", ".xz", ".zst", NULL};
|
||||||
|
|
||||||
bool compression_should_skip(const char* path) {
|
|
||||||
return compression_should_skip_with_suffixes(path, NULL, -1);
|
|
||||||
}
|
|
||||||
|
|
||||||
bool compression_should_skip_with_suffixes(const char* path, char* const* suffixes, int count) {
|
bool compression_should_skip_with_suffixes(const char* path, char* const* suffixes, int count) {
|
||||||
if (!path)
|
if (!path)
|
||||||
return false;
|
return false;
|
||||||
@@ -39,6 +36,96 @@ bool compression_should_skip_with_suffixes(const char* path, char* const* suffix
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Per-thread cache of zstd contexts plus the grow-only compression scratch
|
||||||
|
* buffer. zstd contexts are stateful and not safe to share between threads,
|
||||||
|
* so each thread keeps its own (see compression_get_thread_ctx). The cache is
|
||||||
|
* stored in a C11 thread-specific storage slot whose destructor releases the
|
||||||
|
* contexts when the thread exits; this keeps LeakSanitizer clean for the
|
||||||
|
* short-lived sender/receiver/scanner worker threads without every worker
|
||||||
|
* entry point having to remember to call compression_free_thread_contexts().
|
||||||
|
* The main thread's slot is not torn down by tss at process exit, so an atexit
|
||||||
|
* hook releases it (and compression_free_thread_contexts allows eager
|
||||||
|
* release). */
|
||||||
|
typedef struct {
|
||||||
|
ZSTD_CCtx* cctx;
|
||||||
|
ZSTD_DCtx* dctx;
|
||||||
|
void* out_buf; /* reusable ZSTD_compressBound-sized output scratch */
|
||||||
|
size_t out_cap; /* bytes currently allocated for out_buf */
|
||||||
|
int level; /* compression level currently applied to cctx */
|
||||||
|
int workers; /* nbWorkers currently applied to cctx */
|
||||||
|
bool params_set;
|
||||||
|
bool cached; /* false when the TSS slot could not be used: caller owns */
|
||||||
|
} CompressionThreadCtx;
|
||||||
|
|
||||||
|
static once_flag compression_tls_once = ONCE_FLAG_INIT;
|
||||||
|
static tss_t compression_tls_key;
|
||||||
|
static bool compression_tls_ready;
|
||||||
|
|
||||||
|
static void compression_tls_make_key(void);
|
||||||
|
|
||||||
|
static void compression_ctx_free(CompressionThreadCtx* ctx) {
|
||||||
|
if (!ctx)
|
||||||
|
return;
|
||||||
|
if (ctx->cctx)
|
||||||
|
ZSTD_freeCCtx(ctx->cctx);
|
||||||
|
if (ctx->dctx)
|
||||||
|
ZSTD_freeDCtx(ctx->dctx);
|
||||||
|
free(ctx->out_buf);
|
||||||
|
free(ctx);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void compression_tls_destructor(void* value) {
|
||||||
|
compression_ctx_free((CompressionThreadCtx*)value);
|
||||||
|
}
|
||||||
|
|
||||||
|
void compression_free_thread_contexts(void) {
|
||||||
|
call_once(&compression_tls_once, compression_tls_make_key);
|
||||||
|
if (!compression_tls_ready)
|
||||||
|
return;
|
||||||
|
CompressionThreadCtx* ctx = (CompressionThreadCtx*)tss_get(compression_tls_key);
|
||||||
|
if (!ctx)
|
||||||
|
return;
|
||||||
|
/* Clear the slot first so the thread-exit destructor cannot free it twice. */
|
||||||
|
tss_set(compression_tls_key, NULL);
|
||||||
|
compression_ctx_free(ctx);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void compression_atexit_cleanup(void) {
|
||||||
|
compression_free_thread_contexts();
|
||||||
|
}
|
||||||
|
|
||||||
|
static void compression_tls_make_key(void) {
|
||||||
|
if (tss_create(&compression_tls_key, compression_tls_destructor) == thrd_success) {
|
||||||
|
compression_tls_ready = true;
|
||||||
|
atexit(compression_atexit_cleanup);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static CompressionThreadCtx* compression_get_thread_ctx(void) {
|
||||||
|
call_once(&compression_tls_once, compression_tls_make_key);
|
||||||
|
if (!compression_tls_ready) {
|
||||||
|
/* Extremely unlikely: fall back to an uncached context the caller frees. */
|
||||||
|
return (CompressionThreadCtx*)calloc(1, sizeof(CompressionThreadCtx));
|
||||||
|
}
|
||||||
|
CompressionThreadCtx* ctx = (CompressionThreadCtx*)tss_get(compression_tls_key);
|
||||||
|
if (ctx)
|
||||||
|
return ctx;
|
||||||
|
ctx = (CompressionThreadCtx*)calloc(1, sizeof(CompressionThreadCtx));
|
||||||
|
if (!ctx)
|
||||||
|
return NULL;
|
||||||
|
ctx->cached = true;
|
||||||
|
if (tss_set(compression_tls_key, ctx) != thrd_success)
|
||||||
|
ctx->cached = false;
|
||||||
|
return ctx;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Release an uncached context immediately; cached contexts are owned by the
|
||||||
|
* thread's TSS slot and freed on thread exit / compression_free_thread_contexts. */
|
||||||
|
static void compression_ctx_put(CompressionThreadCtx* ctx) {
|
||||||
|
if (ctx && !ctx->cached)
|
||||||
|
compression_ctx_free(ctx);
|
||||||
|
}
|
||||||
|
|
||||||
Data* data_compress(Data* data_to_compress, int compression_level) {
|
Data* data_compress(Data* data_to_compress, int compression_level) {
|
||||||
return data_compress_with_threads(data_to_compress, compression_level, 0);
|
return data_compress_with_threads(data_to_compress, compression_level, 0);
|
||||||
}
|
}
|
||||||
@@ -50,68 +137,102 @@ Data* data_compress_with_threads(Data* data_to_compress, int compression_level,
|
|||||||
return NULL;
|
return NULL;
|
||||||
log_message(LOG_LEVEL_DEBUG, "Starting to compress data");
|
log_message(LOG_LEVEL_DEBUG, "Starting to compress data");
|
||||||
size_t dst_size = ZSTD_compressBound(data_to_compress->size);
|
size_t dst_size = ZSTD_compressBound(data_to_compress->size);
|
||||||
Data* compressed_data = data_create_empty(dst_size);
|
|
||||||
if (compressed_data == NULL)
|
|
||||||
return NULL;
|
|
||||||
|
|
||||||
ZSTD_CCtx* cctx = ZSTD_createCCtx();
|
CompressionThreadCtx* ctx = compression_get_thread_ctx();
|
||||||
if (!cctx) {
|
if (ctx == NULL) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Failed to create ZSTD compression context");
|
log_message(LOG_LEVEL_ERROR, "Failed to allocate ZSTD compression context");
|
||||||
data_destroy(compressed_data);
|
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
Data* compressed_data = NULL;
|
||||||
|
|
||||||
size_t zret = ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compression_level);
|
if (!ctx->cctx) {
|
||||||
if (ZSTD_isError(zret)) {
|
ctx->cctx = ZSTD_createCCtx();
|
||||||
log_message(LOG_LEVEL_ERROR, "Failed to set compression level: %s", ZSTD_getErrorName(zret));
|
if (!ctx->cctx) {
|
||||||
ZSTD_freeCCtx(cctx);
|
log_message(LOG_LEVEL_ERROR, "Failed to create ZSTD compression context");
|
||||||
data_destroy(compressed_data);
|
goto cleanup;
|
||||||
return NULL;
|
}
|
||||||
|
ctx->params_set = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Reset only the session: parameters (and any already-allocated zstd worker
|
||||||
|
* pool) stay attached to the context, so compressing the next file does not
|
||||||
|
* rebuild the pool. */
|
||||||
|
ZSTD_CCtx_reset(ctx->cctx, ZSTD_reset_session_only);
|
||||||
|
|
||||||
|
if (!ctx->params_set || ctx->level != compression_level) {
|
||||||
|
size_t zret = ZSTD_CCtx_setParameter(ctx->cctx, ZSTD_c_compressionLevel, compression_level);
|
||||||
|
if (ZSTD_isError(zret)) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Failed to set compression level: %s", ZSTD_getErrorName(zret));
|
||||||
|
goto cleanup;
|
||||||
|
}
|
||||||
|
ctx->level = compression_level;
|
||||||
|
}
|
||||||
|
|
||||||
|
int available_threads = 0;
|
||||||
if (compression_threads > 0) {
|
if (compression_threads > 0) {
|
||||||
long online_cpus = sysconf(_SC_NPROCESSORS_ONLN);
|
long online_cpus = sysconf(_SC_NPROCESSORS_ONLN);
|
||||||
int available_threads = online_cpus > 0 && online_cpus < compression_threads
|
available_threads = online_cpus > 0 && online_cpus < compression_threads ? (int)online_cpus
|
||||||
? (int)online_cpus
|
: compression_threads;
|
||||||
: compression_threads;
|
}
|
||||||
zret = ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, available_threads);
|
if (!ctx->params_set || ctx->workers != available_threads) {
|
||||||
|
size_t zret = ZSTD_CCtx_setParameter(ctx->cctx, ZSTD_c_nbWorkers, available_threads);
|
||||||
if (ZSTD_isError(zret)) {
|
if (ZSTD_isError(zret)) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Failed to set compression threads: %s",
|
log_message(LOG_LEVEL_ERROR, "Failed to set compression threads: %s",
|
||||||
ZSTD_getErrorName(zret));
|
ZSTD_getErrorName(zret));
|
||||||
ZSTD_freeCCtx(cctx);
|
goto cleanup;
|
||||||
data_destroy(compressed_data);
|
|
||||||
return NULL;
|
|
||||||
}
|
}
|
||||||
|
ctx->workers = available_threads;
|
||||||
|
}
|
||||||
|
ctx->params_set = true;
|
||||||
|
|
||||||
|
if (available_threads > 0) {
|
||||||
/* Streaming compression needs the source size before threaded mode can end a frame. */
|
/* Streaming compression needs the source size before threaded mode can end a frame. */
|
||||||
zret = ZSTD_CCtx_setPledgedSrcSize(cctx, data_to_compress->size);
|
size_t zret = ZSTD_CCtx_setPledgedSrcSize(ctx->cctx, data_to_compress->size);
|
||||||
if (ZSTD_isError(zret)) {
|
if (ZSTD_isError(zret)) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Failed to set compression source size: %s",
|
log_message(LOG_LEVEL_ERROR, "Failed to set compression source size: %s",
|
||||||
ZSTD_getErrorName(zret));
|
ZSTD_getErrorName(zret));
|
||||||
ZSTD_freeCCtx(cctx);
|
goto cleanup;
|
||||||
data_destroy(compressed_data);
|
|
||||||
return NULL;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (ctx->out_cap < dst_size) {
|
||||||
|
void* grown = protocol_realloc(ctx->out_buf, dst_size);
|
||||||
|
if (grown == NULL) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Failed to allocate compression buffer");
|
||||||
|
goto cleanup;
|
||||||
|
}
|
||||||
|
ctx->out_buf = grown;
|
||||||
|
ctx->out_cap = dst_size;
|
||||||
|
}
|
||||||
|
|
||||||
ZSTD_inBuffer input = {data_to_compress->data, data_to_compress->size, 0};
|
ZSTD_inBuffer input = {data_to_compress->data, data_to_compress->size, 0};
|
||||||
ZSTD_outBuffer output = {compressed_data->data, dst_size, 0};
|
ZSTD_outBuffer output = {ctx->out_buf, dst_size, 0};
|
||||||
|
|
||||||
size_t ret;
|
size_t ret;
|
||||||
do {
|
do {
|
||||||
ret = ZSTD_compressStream2(cctx, &output, &input, ZSTD_e_end);
|
ret = ZSTD_compressStream2(ctx->cctx, &output, &input, ZSTD_e_end);
|
||||||
if (ZSTD_isError(ret)) {
|
if (ZSTD_isError(ret)) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Compression failed: %s", ZSTD_getErrorName(ret));
|
log_message(LOG_LEVEL_ERROR, "Compression failed: %s", ZSTD_getErrorName(ret));
|
||||||
ZSTD_freeCCtx(cctx);
|
goto cleanup;
|
||||||
data_destroy(compressed_data);
|
|
||||||
return NULL;
|
|
||||||
}
|
}
|
||||||
} while (ret > 0);
|
} while (ret > 0);
|
||||||
|
|
||||||
|
/* Hand off an exactly-sized copy; the scratch buffer stays cached so the next
|
||||||
|
* call does not reallocate a ZSTD_compressBound-sized block. */
|
||||||
|
compressed_data = data_create_empty(output.pos);
|
||||||
|
if (compressed_data == NULL) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Failed to allocate compressed data");
|
||||||
|
goto cleanup;
|
||||||
|
}
|
||||||
|
if (output.pos > 0)
|
||||||
|
memcpy(compressed_data->data, ctx->out_buf, output.pos);
|
||||||
compressed_data->size = output.pos;
|
compressed_data->size = output.pos;
|
||||||
ZSTD_freeCCtx(cctx);
|
|
||||||
|
|
||||||
log_debug_message(LOG_DEBUG_UTIL, "Data succesfully compressed from %zu to %zu",
|
log_debug_message(LOG_DEBUG_UTIL, "Data succesfully compressed from %zu to %zu",
|
||||||
data_to_compress->size, compressed_data->size);
|
data_to_compress->size, compressed_data->size);
|
||||||
|
|
||||||
|
cleanup:
|
||||||
|
compression_ctx_put(ctx);
|
||||||
return compressed_data;
|
return compressed_data;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -122,9 +243,13 @@ Data* data_decompress_limited(Data* compressed_data, size_t maximum_size) {
|
|||||||
log_debug_message(LOG_DEBUG_UTIL, "Start to decompress data");
|
log_debug_message(LOG_DEBUG_UTIL, "Start to decompress data");
|
||||||
unsigned long long dst_size =
|
unsigned long long dst_size =
|
||||||
ZSTD_getFrameContentSize(compressed_data->data, compressed_data->size);
|
ZSTD_getFrameContentSize(compressed_data->data, compressed_data->size);
|
||||||
if (ZSTD_isError(dst_size)) {
|
/* ZSTD_isError() is also true for ZSTD_CONTENTSIZE_ERROR and
|
||||||
log_message(LOG_LEVEL_ERROR, "Failed to get decompressed size: %s",
|
* ZSTD_CONTENTSIZE_UNKNOWN (both are encoded near (size_t)-1), so test the
|
||||||
ZSTD_getErrorName(dst_size));
|
* sentinels explicitly instead of blanket-rejecting every error-ish value:
|
||||||
|
* only CONTENTSIZE_ERROR means an unreadable header, while CONTENTSIZE_UNKNOWN
|
||||||
|
* must reach the estimate fallback below. */
|
||||||
|
if (dst_size == ZSTD_CONTENTSIZE_ERROR) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Failed to get decompressed size: invalid zstd frame");
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -144,20 +269,30 @@ Data* data_decompress_limited(Data* compressed_data, size_t maximum_size) {
|
|||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
ZSTD_DCtx* dctx = ZSTD_createDCtx();
|
CompressionThreadCtx* ctx = compression_get_thread_ctx();
|
||||||
if (!dctx) {
|
if (ctx == NULL) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Failed to create ZSTD decompression context");
|
log_message(LOG_LEVEL_ERROR, "Failed to allocate ZSTD decompression context");
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
Data* uncompressed_data = NULL;
|
||||||
|
|
||||||
|
if (!ctx->dctx) {
|
||||||
|
ctx->dctx = ZSTD_createDCtx();
|
||||||
|
if (!ctx->dctx) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Failed to create ZSTD decompression context");
|
||||||
|
goto cleanup;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
/* Reset only the session; decompression parameters are sticky. */
|
||||||
|
ZSTD_DCtx_reset(ctx->dctx, ZSTD_reset_session_only);
|
||||||
|
|
||||||
size_t buf_size = (dst_size > 0) ? (size_t)dst_size : INITIAL_DECOMPRESS_BUF_SIZE;
|
size_t buf_size = (dst_size > 0) ? (size_t)dst_size : INITIAL_DECOMPRESS_BUF_SIZE;
|
||||||
if (buf_size > maximum_size)
|
if (buf_size > maximum_size)
|
||||||
buf_size = maximum_size;
|
buf_size = maximum_size;
|
||||||
Data* uncompressed_data = data_create_empty(buf_size);
|
uncompressed_data = data_create_empty(buf_size);
|
||||||
if (!uncompressed_data) {
|
if (!uncompressed_data) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Failed to allocate decompression buffer");
|
log_message(LOG_LEVEL_ERROR, "Failed to allocate decompression buffer");
|
||||||
ZSTD_freeDCtx(dctx);
|
goto cleanup;
|
||||||
return NULL;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
ZSTD_inBuffer input = {compressed_data->data, compressed_data->size, 0};
|
ZSTD_inBuffer input = {compressed_data->data, compressed_data->size, 0};
|
||||||
@@ -165,20 +300,20 @@ Data* data_decompress_limited(Data* compressed_data, size_t maximum_size) {
|
|||||||
|
|
||||||
size_t ret;
|
size_t ret;
|
||||||
do {
|
do {
|
||||||
ret = ZSTD_decompressStream(dctx, &output, &input);
|
ret = ZSTD_decompressStream(ctx->dctx, &output, &input);
|
||||||
if (ZSTD_isError(ret)) {
|
if (ZSTD_isError(ret)) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Decompression failed: %s", ZSTD_getErrorName(ret));
|
log_message(LOG_LEVEL_ERROR, "Decompression failed: %s", ZSTD_getErrorName(ret));
|
||||||
ZSTD_freeDCtx(dctx);
|
|
||||||
data_destroy(uncompressed_data);
|
data_destroy(uncompressed_data);
|
||||||
return NULL;
|
uncompressed_data = NULL;
|
||||||
|
goto cleanup;
|
||||||
}
|
}
|
||||||
if (ret > 0 && output.pos == output.size) {
|
if (ret > 0 && output.pos == output.size) {
|
||||||
if (buf_size >= hard_limit || buf_size > SIZE_MAX / 2) {
|
if (buf_size >= hard_limit || buf_size > SIZE_MAX / 2) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Decompressed data exceeds %llu bytes",
|
log_message(LOG_LEVEL_ERROR, "Decompressed data exceeds %llu bytes",
|
||||||
(unsigned long long)MAX_DECOMPRESSED_SIZE);
|
(unsigned long long)MAX_DECOMPRESSED_SIZE);
|
||||||
ZSTD_freeDCtx(dctx);
|
|
||||||
data_destroy(uncompressed_data);
|
data_destroy(uncompressed_data);
|
||||||
return NULL;
|
uncompressed_data = NULL;
|
||||||
|
goto cleanup;
|
||||||
}
|
}
|
||||||
buf_size *= 2;
|
buf_size *= 2;
|
||||||
if (buf_size > hard_limit)
|
if (buf_size > hard_limit)
|
||||||
@@ -186,20 +321,36 @@ Data* data_decompress_limited(Data* compressed_data, size_t maximum_size) {
|
|||||||
void* new_data = protocol_realloc(uncompressed_data->data, buf_size);
|
void* new_data = protocol_realloc(uncompressed_data->data, buf_size);
|
||||||
if (!new_data) {
|
if (!new_data) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Failed to grow decompression buffer");
|
log_message(LOG_LEVEL_ERROR, "Failed to grow decompression buffer");
|
||||||
ZSTD_freeDCtx(dctx);
|
|
||||||
data_destroy(uncompressed_data);
|
data_destroy(uncompressed_data);
|
||||||
return NULL;
|
uncompressed_data = NULL;
|
||||||
|
goto cleanup;
|
||||||
}
|
}
|
||||||
uncompressed_data->data = new_data;
|
uncompressed_data->data = new_data;
|
||||||
output.dst = new_data;
|
output.dst = new_data;
|
||||||
output.size = buf_size;
|
output.size = buf_size;
|
||||||
|
/* Re-attempt with the larger output buffer; the truncated-frame check
|
||||||
|
* below must not reject a complete frame that merely filled the previous
|
||||||
|
* buffer exactly. */
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
/* A positive hint with all input consumed means the frame is incomplete: a
|
||||||
|
* truncated stream would otherwise spin here forever (ZSTD_decompressStream
|
||||||
|
* keeps returning the same hint). Fail instead of burning CPU. */
|
||||||
|
if (ret != 0 && input.pos == input.size) {
|
||||||
|
log_message(LOG_LEVEL_ERROR,
|
||||||
|
"Truncated zstd frame: input exhausted with %zu bytes still expected", ret);
|
||||||
|
data_destroy(uncompressed_data);
|
||||||
|
uncompressed_data = NULL;
|
||||||
|
goto cleanup;
|
||||||
}
|
}
|
||||||
} while (ret > 0);
|
} while (ret > 0);
|
||||||
|
|
||||||
uncompressed_data->size = output.pos;
|
uncompressed_data->size = output.pos;
|
||||||
ZSTD_freeDCtx(dctx);
|
|
||||||
|
|
||||||
log_debug_message(LOG_DEBUG_UTIL, "Decompressed data successfully");
|
log_debug_message(LOG_DEBUG_UTIL, "Decompressed data successfully");
|
||||||
|
|
||||||
|
cleanup:
|
||||||
|
compression_ctx_put(ctx);
|
||||||
return uncompressed_data;
|
return uncompressed_data;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -11,7 +11,14 @@ Data* data_compress_with_threads(Data* data_to_compress, int compression_level,
|
|||||||
int compression_threads);
|
int compression_threads);
|
||||||
Data* data_decompress(Data* compressed_data);
|
Data* data_decompress(Data* compressed_data);
|
||||||
Data* data_decompress_limited(Data* compressed_data, size_t maximum_size);
|
Data* data_decompress_limited(Data* compressed_data, size_t maximum_size);
|
||||||
bool compression_should_skip(const char* path);
|
|
||||||
bool compression_should_skip_with_suffixes(const char* path, char* const* suffixes, int count);
|
bool compression_should_skip_with_suffixes(const char* path, char* const* suffixes, int count);
|
||||||
|
|
||||||
|
/* Release the calling thread's cached zstd contexts (compressor, decompressor
|
||||||
|
* and scratch buffer). The cache is thread-local and is also released
|
||||||
|
* automatically when a worker thread exits (via a C11 tss destructor) and for
|
||||||
|
* the main thread at process exit; this explicit entry point exists so tests
|
||||||
|
* and long-lived callers can drop the cache deterministically. Safe to call
|
||||||
|
* when no context has been created, and idempotent. */
|
||||||
|
void compression_free_thread_contexts(void);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
+574
-667
File diff suppressed because it is too large
Load Diff
+488
-288
@@ -68,104 +68,251 @@ typedef struct {
|
|||||||
int value; /* 0/1 for booleans, byte count for SO_RCVBUF/SO_SNDBUF */
|
int value; /* 0/1 for booleans, byte count for SO_RCVBUF/SO_SNDBUF */
|
||||||
} SockOptEntry;
|
} SockOptEntry;
|
||||||
|
|
||||||
|
/* --super / --no-super tri-state (Config->super_mode). AUTO (default) and ON
|
||||||
|
* both permit a confined super-user attempt (AUTO preserves FastSync's
|
||||||
|
* historical best-effort behavior; an unprivileged attempt is refused by the
|
||||||
|
* kernel and skipped per entry); OFF forbids the attempt even for root. See
|
||||||
|
* privilege_super_mode_permitted() in identity.h. */
|
||||||
|
typedef enum SuperMode { SUPER_MODE_AUTO = 0, SUPER_MODE_ON = 1, SUPER_MODE_OFF = 2 } SuperMode;
|
||||||
|
|
||||||
|
/* ===========================================================================
|
||||||
|
* Config wire-field table (single source of truth for protocol 2.21.0).
|
||||||
|
*
|
||||||
|
* Every field below crosses the wire. The table is the ONLY place a
|
||||||
|
* serialized field is named: config.h expands CONFIG_WIRE_FIELDS() to declare
|
||||||
|
* the struct member, config_set_defaults() expands it to assign the default,
|
||||||
|
* and config_send_wire_block()/config_receive_with_validate() expand the
|
||||||
|
* per-segment lists to emit/consume the frame in exactly this order. Do NOT
|
||||||
|
* reorder entries and do NOT change a field's segment/KIND without a
|
||||||
|
* PROTOCOL_VERSION bump: the resulting byte stream is pinned by
|
||||||
|
* test_config_wire_golden().
|
||||||
|
*
|
||||||
|
* Entry layout: X(MEMBER, CTYPE, DEFAULT, KIND)
|
||||||
|
* MEMBER struct member name (public; never rename)
|
||||||
|
* CTYPE C type of the member
|
||||||
|
* DEFAULT default-value expression used by config_set_defaults()
|
||||||
|
* KIND wire codec, dispatched to CONFIG_SEND_<KIND>/CONFIG_RECV_<KIND>
|
||||||
|
* in config.c (strings receive through a ConfigStringBudget).
|
||||||
|
*
|
||||||
|
* Fields with genuinely custom logic keep dedicated helpers but are still
|
||||||
|
* declared here exactly once: the protocol-version handshake (HEADER), the
|
||||||
|
* daemon SCRAM auth username (STR_REDACTED_AUTH), the daemon module name
|
||||||
|
* (STR_MODULE), repeated count+array blocks (BLOCK_*), --copy-as presence
|
||||||
|
* (COPY_AS_*), and the derived --delta / use_xattrs bits (DERIVED_DELTA,
|
||||||
|
* BOOL_XATTR_DERIVE).
|
||||||
|
*
|
||||||
|
* SCOPE: this table covers ONLY the serialized wire frame. The client CLI
|
||||||
|
* option tables in client_cli.c (OPTION_TABLE / NEGATABLE_OPTIONS) are still
|
||||||
|
* hand-maintained and are deliberately NOT generated from this table: the CLI
|
||||||
|
* surface carries client-only fields and flag/alias/negation semantics that
|
||||||
|
* have no wire representation. Do not assume the two are folded together.
|
||||||
|
* =========================================================================== */
|
||||||
|
#define CONFIG_WIRE_HEADER_FIELDS(X) X(version, char*, str_dup(PROTOCOL_VERSION), STR)
|
||||||
|
|
||||||
|
/* dry_run (--dry-run) is CLIENT-INTENT that now CROSSES the wire (protocol
|
||||||
|
* 2.21.0): the receiver needs it to answer what WOULD transfer/skip without
|
||||||
|
* touching disk. The client-only launch behavior (no server contact for a
|
||||||
|
* local destination) is decided separately in client_send.c before the frame
|
||||||
|
* is ever sent. */
|
||||||
|
#define CONFIG_WIRE_CORE_FIELDS(X) \
|
||||||
|
X(eight_bit_output, bool, false, BOOL_8BIT) \
|
||||||
|
X(max_alloc, unsigned long long, DEFAULT_MAX_ALLOC, RAW_MAXALLOC) \
|
||||||
|
X(send_directory, char*, NULL, STR) \
|
||||||
|
X(receive_root_directory, char*, NULL, STR) \
|
||||||
|
X(save_to_disk, bool, false, BOOL) \
|
||||||
|
X(use_multithreading, bool, false, BOOL) \
|
||||||
|
X(use_chunk_serialization, bool, false, BOOL) \
|
||||||
|
X(use_compression, bool, false, BOOL) \
|
||||||
|
X(use_metadata, bool, false, BOOL) \
|
||||||
|
X(use_executability, bool, false, BOOL) \
|
||||||
|
X(compression_level, int, 5, INT) \
|
||||||
|
X(chunk_size, unsigned long long, DEFAULT_CHUNK_SIZE, RAW) \
|
||||||
|
X(use_sendfile, bool, false, BOOL) \
|
||||||
|
X(dry_run, bool, false, BOOL)
|
||||||
|
|
||||||
|
#define CONFIG_WIRE_DELTA_FIELDS(X) \
|
||||||
|
X(use_delete, bool, false, BOOL) \
|
||||||
|
X(use_incremental, bool, false, BOOL) \
|
||||||
|
X(size_only, bool, false, BOOL) \
|
||||||
|
X(ignore_times, bool, false, BOOL) \
|
||||||
|
X(use_delta, bool, false, DERIVED_DELTA) \
|
||||||
|
X(delta_block_size, uint32_t, DELTA_BLOCK_SIZE_DEFAULT, RAW) \
|
||||||
|
X(delta_max_file_size, unsigned long long, DELTA_MAX_FILE_SIZE, RAW)
|
||||||
|
|
||||||
|
#define CONFIG_WIRE_FILE_OPTIONS_FIELDS(X) \
|
||||||
|
X(backup, bool, false, BOOL) \
|
||||||
|
X(backup_dir, char*, NULL, STR_OPT) \
|
||||||
|
X(remove_source_files, bool, false, BOOL) \
|
||||||
|
X(follow_symlinks, bool, false, BOOL) \
|
||||||
|
X(copy_links, bool, false, BOOL) \
|
||||||
|
X(safe_links, bool, false, BOOL) \
|
||||||
|
X(copy_unsafe_links, bool, false, BOOL) \
|
||||||
|
X(preserve_hard_links, bool, false, BOOL) \
|
||||||
|
X(preserve_acls, bool, false, BOOL) \
|
||||||
|
X(preserve_xattrs, bool, false, BOOL) \
|
||||||
|
X(preserve_devices, bool, false, BOOL) \
|
||||||
|
X(preserve_sparse, bool, false, BOOL) \
|
||||||
|
X(preserve_specials, bool, false, BOOL) \
|
||||||
|
X(copy_devices, bool, false, BOOL) \
|
||||||
|
X(write_devices, bool, false, BOOL)
|
||||||
|
|
||||||
|
#define CONFIG_WIRE_SELECTION_FIELDS(X) \
|
||||||
|
X(ignore_existing, bool, false, BOOL) \
|
||||||
|
X(existing, bool, false, BOOL) \
|
||||||
|
X(update, bool, false, BOOL) \
|
||||||
|
X(inplace, bool, false, BOOL) \
|
||||||
|
X(delay_updates, bool, false, BOOL) \
|
||||||
|
X(append, bool, false, BOOL) \
|
||||||
|
X(use_fsync, bool, false, BOOL) \
|
||||||
|
X(append_verify, bool, false, BOOL) \
|
||||||
|
X(delete_excluded, bool, false, BOOL) \
|
||||||
|
X(force_delete, bool, false, BOOL) \
|
||||||
|
X(delete_missing_args, bool, false, BOOL) \
|
||||||
|
X(delete_after, bool, false, BOOL) \
|
||||||
|
X(preallocate, bool, false, BOOL) \
|
||||||
|
X(max_delete, int, -1, RAW) \
|
||||||
|
X(relative, bool, false, BOOL) \
|
||||||
|
X(prune_empty_dirs, bool, false, BOOL) \
|
||||||
|
X(mkpath, bool, false, BOOL) \
|
||||||
|
X(delete_during, bool, false, BOOL) \
|
||||||
|
X(delete_delay, bool, false, BOOL)
|
||||||
|
|
||||||
|
#define CONFIG_WIRE_RESUME_FIELDS(X) \
|
||||||
|
X(temp_dir, char*, NULL, STR_OPT) \
|
||||||
|
X(partial, bool, false, BOOL) \
|
||||||
|
X(partial_dir, char*, NULL, STR_OPT) \
|
||||||
|
X(suffix, char*, NULL, STR_OPT) \
|
||||||
|
X(delete_before, bool, false, BOOL) \
|
||||||
|
X(checksum, bool, false, BOOL) \
|
||||||
|
X(modify_window, int, 0, RAW) \
|
||||||
|
X(compress_choice, char*, NULL, STR_KEEP) \
|
||||||
|
X(chmod_spec, char*, NULL, STR_KEEP) \
|
||||||
|
X(skip_compress_set, bool, false, BOOL) \
|
||||||
|
X(skip_compress_count, int, 0, INT_SKIPCOUNT) \
|
||||||
|
X(skip_compress_suffixes, char**, NULL, BLOCK_SKIP_SUFFIXES)
|
||||||
|
|
||||||
|
#define CONFIG_WIRE_BASIS_FIELDS(X) \
|
||||||
|
X(basis_count, int, 0, INT_BASISCOUNT) \
|
||||||
|
X(basis_dirs, BasisDest*, NULL, BLOCK_BASIS)
|
||||||
|
|
||||||
|
#define CONFIG_WIRE_FUZZY_FIELDS(X) X(fuzzy, bool, false, BOOL)
|
||||||
|
|
||||||
|
#define CONFIG_WIRE_CHECKSUM_FIELDS(X) \
|
||||||
|
X(checksum_algo, int, CHECKSUM_ALGO_XXH64, INT_CHECKSUM_ALGO) \
|
||||||
|
X(checksum_seed, uint64_t, 0, RAW)
|
||||||
|
|
||||||
|
#define CONFIG_WIRE_IDENTITY_FIELDS(X) \
|
||||||
|
X(numeric_ids, bool, false, BOOL) \
|
||||||
|
X(chown_uid_set, bool, false, BOOL) \
|
||||||
|
X(chown_uid, int32_t, 0, INT_IDENTITY) \
|
||||||
|
X(chown_gid_set, bool, false, BOOL) \
|
||||||
|
X(chown_gid, int32_t, 0, INT_IDENTITY) \
|
||||||
|
X(usermap_count, int, 0, INT_IDMAPCOUNT) \
|
||||||
|
X(usermap, IdentityMap*, NULL, BLOCK_IDMAP) \
|
||||||
|
X(groupmap_count, int, 0, INT_IDMAPCOUNT) \
|
||||||
|
X(groupmap, IdentityMap*, NULL, BLOCK_IDMAP)
|
||||||
|
|
||||||
|
#define CONFIG_WIRE_METADATA_TIMES_FIELDS(X) \
|
||||||
|
X(preserve_atimes, bool, false, BOOL) \
|
||||||
|
X(preserve_crtimes, bool, false, BOOL) \
|
||||||
|
X(omit_dir_times, bool, false, BOOL) \
|
||||||
|
X(omit_link_times, bool, false, BOOL)
|
||||||
|
|
||||||
|
#define CONFIG_WIRE_SYMLINK_TRUST_FIELDS(X) \
|
||||||
|
X(munge_links, bool, false, BOOL) \
|
||||||
|
X(keep_dirlinks, bool, false, BOOL)
|
||||||
|
|
||||||
|
#define CONFIG_WIRE_XATTR_FIELDS(X) X(fake_super, bool, false, BOOL_XATTR_DERIVE)
|
||||||
|
|
||||||
|
#define CONFIG_WIRE_MODULE_FIELDS(X) X(module, char*, NULL, STR_MODULE)
|
||||||
|
|
||||||
|
#define CONFIG_WIRE_DAEMON_AUTH_FIELDS(X) X(auth_user, char*, NULL, STR_REDACTED_AUTH)
|
||||||
|
|
||||||
|
#define CONFIG_WIRE_ICONV_FIELDS(X) X(iconv_spec, char*, NULL, STR_OPT)
|
||||||
|
|
||||||
|
#define CONFIG_WIRE_PRIVILEGE_FIELDS(X) X(super_mode, SuperMode, SUPER_MODE_AUTO, SUPERMODE)
|
||||||
|
|
||||||
|
#define CONFIG_WIRE_COPY_AS_FIELDS(X) \
|
||||||
|
X(copy_as_set, bool, false, COPY_AS_PRESENCE) \
|
||||||
|
X(copy_as_uid, int32_t, 0, COPY_AS_ID) \
|
||||||
|
X(copy_as_gid, int32_t, 0, COPY_AS_ID)
|
||||||
|
|
||||||
|
/* All serialized fields, in exact wire order. Concatenating the per-segment
|
||||||
|
* lists here is what keeps the declaration order = the wire order. */
|
||||||
|
#define CONFIG_WIRE_FIELDS(X) \
|
||||||
|
CONFIG_WIRE_HEADER_FIELDS(X) \
|
||||||
|
CONFIG_WIRE_CORE_FIELDS(X) \
|
||||||
|
CONFIG_WIRE_DELTA_FIELDS(X) \
|
||||||
|
CONFIG_WIRE_FILE_OPTIONS_FIELDS(X) \
|
||||||
|
CONFIG_WIRE_SELECTION_FIELDS(X) \
|
||||||
|
CONFIG_WIRE_RESUME_FIELDS(X) \
|
||||||
|
CONFIG_WIRE_BASIS_FIELDS(X) \
|
||||||
|
CONFIG_WIRE_FUZZY_FIELDS(X) \
|
||||||
|
CONFIG_WIRE_CHECKSUM_FIELDS(X) \
|
||||||
|
CONFIG_WIRE_IDENTITY_FIELDS(X) \
|
||||||
|
CONFIG_WIRE_METADATA_TIMES_FIELDS(X) \
|
||||||
|
CONFIG_WIRE_SYMLINK_TRUST_FIELDS(X) \
|
||||||
|
CONFIG_WIRE_XATTR_FIELDS(X) \
|
||||||
|
CONFIG_WIRE_MODULE_FIELDS(X) \
|
||||||
|
CONFIG_WIRE_DAEMON_AUTH_FIELDS(X) \
|
||||||
|
CONFIG_WIRE_ICONV_FIELDS(X) \
|
||||||
|
CONFIG_WIRE_PRIVILEGE_FIELDS(X) \
|
||||||
|
CONFIG_WIRE_COPY_AS_FIELDS(X)
|
||||||
|
|
||||||
typedef struct Config {
|
typedef struct Config {
|
||||||
char* version;
|
/* -j/--threads=N: number of parallel scanner worker threads for the -m
|
||||||
char* send_directory;
|
* pipeline. 0 (the default, also set by bare -j/--threads) means "use the
|
||||||
char* receive_root_directory;
|
* scanner's built-in default" (4). CLIENT-ONLY: it is a local scheduling
|
||||||
bool save_to_disk;
|
* concern and is NEVER serialized into the wire config frame. */
|
||||||
bool use_multithreading;
|
int scanner_threads;
|
||||||
bool use_chunk_serialization;
|
|
||||||
bool use_compression;
|
|
||||||
bool use_sendfile;
|
|
||||||
bool use_metadata;
|
|
||||||
bool use_executability;
|
|
||||||
bool metadata_explicitly_disabled;
|
bool metadata_explicitly_disabled;
|
||||||
bool show_progress;
|
bool show_progress;
|
||||||
bool dry_run;
|
|
||||||
bool remove_source_files;
|
|
||||||
bool use_delete;
|
|
||||||
int compression_level;
|
|
||||||
int compression_threads;
|
int compression_threads;
|
||||||
unsigned long long chunk_size;
|
|
||||||
int ssh_port;
|
int ssh_port;
|
||||||
TransportType transport;
|
TransportType transport;
|
||||||
char* ssh_destination;
|
char* ssh_destination;
|
||||||
/* Daemon module selection (Wave A, protocol 2.15.0). Client-composed from a
|
|
||||||
* host::module/path destination; NULL or "" means "no module" (the ordinary
|
|
||||||
* standalone-server path). Crosses the wire as a trailing config-frame
|
|
||||||
* string so the daemon can look the module up in its own config and confine
|
|
||||||
* the connection to the module's root (never a client-chosen root). */
|
|
||||||
char* module;
|
|
||||||
/* Daemon password authentication (A7 remediation, protocol 2.19.0).
|
|
||||||
* Client-composed from a --password-file whose first meaningful line is
|
|
||||||
* `user:password`: the client sends ONLY the username in the config frame
|
|
||||||
* (auth_user); the literal password is kept in auth_password CLIENT-SIDE for
|
|
||||||
* the duration of the SCRAM challenge/response and is NEVER serialized. Both
|
|
||||||
* are NULL when the client has no credentials to present; a module WITHOUT
|
|
||||||
* `auth users` stays open and the server ignores any credentials that do
|
|
||||||
* arrive (the client sends them opportunistically and the server decides). */
|
|
||||||
char* auth_user;
|
|
||||||
char* auth_password;
|
char* auth_password;
|
||||||
/* Client-only path of --password-file (never crosses the wire; it is read to
|
/* Client-only path of --password-file (never crosses the wire; it is read to
|
||||||
* populate auth_user/auth_password before connecting). */
|
* populate auth_user/auth_password before connecting). */
|
||||||
char* password_file;
|
char* password_file;
|
||||||
char* fastsync_server_path;
|
char* fastsync_server_path;
|
||||||
/* --iconv=CONVERT_SPEC (protocol 2.16.0, rsync compatibility): convert the
|
|
||||||
* charset of FILE NAMES at the wire boundary. CONVERT_SPEC is
|
|
||||||
* "LOCAL[,REMOTE]": LOCAL is the charset of our own file names, REMOTE is
|
|
||||||
* the remote side's charset and defaults to LOCAL. The sender converts
|
|
||||||
* every path LOCAL->REMOTE before transmitting it; the receiver converts
|
|
||||||
* every received path back REMOTE->LOCAL before creating/writing it. The
|
|
||||||
* FULL SPEC crosses the wire as a trailing config-frame string so each end
|
|
||||||
* derives its own LOCAL and the wire (REMOTE) charset symmetrically. NULL
|
|
||||||
* (or "") means no conversion: identity with zero overhead. See charset.c
|
|
||||||
* and the PROTOCOL_VERSION note below. */
|
|
||||||
char* iconv_spec;
|
|
||||||
char** exclude_patterns;
|
char** exclude_patterns;
|
||||||
int exclude_count;
|
int exclude_count;
|
||||||
char** include_patterns;
|
char** include_patterns;
|
||||||
int include_count;
|
int include_count;
|
||||||
unsigned long long max_size;
|
unsigned long long max_size;
|
||||||
unsigned long long min_size;
|
unsigned long long min_size;
|
||||||
unsigned long long max_alloc;
|
|
||||||
bool use_incremental;
|
|
||||||
bool ignore_times;
|
|
||||||
bool size_only;
|
|
||||||
bool use_delta;
|
|
||||||
bool whole_file;
|
bool whole_file;
|
||||||
/* -y/--fuzzy: when a file must be transferred and the destination holds no
|
|
||||||
* usable file at the exact path, the receiver may reuse a SIMILAR-named
|
|
||||||
* existing regular file in the same destination directory as the delta
|
|
||||||
* basis so the sender transmits only the differences. Crosses the wire
|
|
||||||
* (the receiver performs the candidate search); the CLI implies
|
|
||||||
* --incremental + --delta because the similar-basis only matters on the
|
|
||||||
* receiver-driven delta path. Off by default. */
|
|
||||||
bool fuzzy;
|
|
||||||
int modify_window;
|
|
||||||
uint32_t delta_block_size;
|
|
||||||
unsigned long long delta_max_file_size;
|
|
||||||
bool use_tls;
|
bool use_tls;
|
||||||
char* server_host;
|
char* server_host;
|
||||||
int server_port;
|
int server_port;
|
||||||
|
/* True when --server-port/--port was explicitly given. CLIENT-ONLY (never
|
||||||
|
* serialized): --dry-run uses it to decide whether a real server handshake
|
||||||
|
* was requested, so a plain local destination (no explicit port) keeps the
|
||||||
|
* existing client-side dry-run behavior instead of dialing the default
|
||||||
|
* 127.0.0.1:8080. */
|
||||||
|
bool server_port_set;
|
||||||
|
/* True when --server-host was explicitly given. CLIENT-ONLY (never
|
||||||
|
* serialized), and distinct from the "127.0.0.1" default: --dry-run uses it
|
||||||
|
* to route an explicit remote target to the server so it reports receiver
|
||||||
|
* state exactly like a real run, instead of silently running the client-side
|
||||||
|
* manifest. */
|
||||||
|
bool server_host_set;
|
||||||
char* tls_cert;
|
char* tls_cert;
|
||||||
char* tls_key;
|
char* tls_key;
|
||||||
char* tls_ca;
|
char* tls_ca;
|
||||||
|
/* --timeout: per-message I/O deadline in seconds. 0 (the default/unset
|
||||||
|
* sentinel) leaves the transport's built-in 30 s socket timeout and the
|
||||||
|
* protocol's built-in 60 s per-message deadline in place; a positive value
|
||||||
|
* overrides both. See protocol_session_set_io_timeout. */
|
||||||
int timeout;
|
int timeout;
|
||||||
|
/* --contimeout: connect()/accept timeout, transport layer only. */
|
||||||
int contimeout;
|
int contimeout;
|
||||||
bool quiet;
|
bool quiet;
|
||||||
bool backup;
|
|
||||||
char* backup_dir;
|
|
||||||
bool stats;
|
bool stats;
|
||||||
int max_depth;
|
int max_depth;
|
||||||
FILE* log_file;
|
FILE* log_file;
|
||||||
int queue_size;
|
|
||||||
bool follow_symlinks;
|
|
||||||
bool partial;
|
|
||||||
|
|
||||||
// Issue #120: Symlink handling
|
|
||||||
bool copy_links;
|
|
||||||
bool safe_links;
|
|
||||||
bool copy_unsafe_links;
|
|
||||||
/* Phase 4 symlink-trust. -k/--copy-dirlinks and --munge-links are
|
/* Phase 4 symlink-trust. -k/--copy-dirlinks and --munge-links are
|
||||||
* CLIENT/sender-side only (they decide how the SENDER scans and rewrites
|
* CLIENT/sender-side only (they decide how the SENDER scans and rewrites
|
||||||
* symlinks; the receiver never reads them), so they never cross the wire.
|
* symlinks; the receiver never reads them), so they never cross the wire.
|
||||||
@@ -173,31 +320,6 @@ typedef struct Config {
|
|||||||
* symlink-to-directory as a directory) and CROSSES the wire along with
|
* symlink-to-directory as a directory) and CROSSES the wire along with
|
||||||
* --munge-links (so the receiver knows to unmunge). */
|
* --munge-links (so the receiver knows to unmunge). */
|
||||||
bool copy_dirlinks; /* client-only, sender-side (-k) */
|
bool copy_dirlinks; /* client-only, sender-side (-k) */
|
||||||
bool munge_links; /* crosses the wire */
|
|
||||||
bool keep_dirlinks; /* crosses the wire (-K) */
|
|
||||||
|
|
||||||
// Issue #121: Extended metadata preservation
|
|
||||||
bool preserve_hard_links;
|
|
||||||
bool preserve_acls;
|
|
||||||
bool preserve_xattrs;
|
|
||||||
bool preserve_devices;
|
|
||||||
bool preserve_sparse;
|
|
||||||
/* Phase 4 special/devices: preserve special files (FIFOs, sockets) and device
|
|
||||||
* nodes on the destination by recreating them (mknod/mkfifo) instead of
|
|
||||||
* transferring content. preserve_specials mirrors rsync --specials (the
|
|
||||||
* special-file half of -D); preserve_devices mirrors --devices (the device
|
|
||||||
* half of -D); both CROSS the wire so the receiver knows a special/device
|
|
||||||
* entry must be recreated rather than written as a regular file. */
|
|
||||||
bool preserve_specials;
|
|
||||||
/* --copy-devices: copy the CONTENT of a source device as an ordinary regular
|
|
||||||
* file on the destination (rsync's non-privileged safe mode), instead of
|
|
||||||
* recreating the device node. CROSSES the wire (receiver treats the entry as
|
|
||||||
* a regular file, which is the default, so this is belt-and-braces). */
|
|
||||||
bool copy_devices;
|
|
||||||
/* --write-devices: write the received data directly INTO an existing device
|
|
||||||
* node on the destination instead of creating a regular file. Dangeroud;
|
|
||||||
* see RSYNC_COMPAT.md for the tight gating. CROSSES the wire. */
|
|
||||||
bool write_devices;
|
|
||||||
|
|
||||||
// Issue #122: Output/logging options
|
// Issue #122: Output/logging options
|
||||||
bool itemize_changes;
|
bool itemize_changes;
|
||||||
@@ -207,57 +329,17 @@ typedef struct Config {
|
|||||||
int debug_level;
|
int debug_level;
|
||||||
bool list_only;
|
bool list_only;
|
||||||
bool human_readable;
|
bool human_readable;
|
||||||
bool eight_bit_output;
|
|
||||||
|
|
||||||
// Issue #127: Transfer modes
|
|
||||||
bool existing;
|
|
||||||
bool ignore_existing;
|
|
||||||
bool update;
|
|
||||||
bool inplace;
|
|
||||||
bool delay_updates;
|
|
||||||
bool use_fsync;
|
|
||||||
bool append;
|
|
||||||
bool append_verify;
|
|
||||||
/* --preallocate: allocates the destination file's full expected space up
|
|
||||||
* front (before any data is written) so a transfer that would overflow disk
|
|
||||||
* fails fast at allocation time and the file is laid out contiguously,
|
|
||||||
* avoiding fragmentation. Receiver-side, crosses the wire. */
|
|
||||||
bool preallocate;
|
|
||||||
|
|
||||||
// Issue #128: Extended delete options
|
|
||||||
/* --delete-excluded: also delete destination entries that were excluded on
|
|
||||||
* the source. Default (off) matches rsync: excluded paths are protected from
|
|
||||||
* deletion. Crosses the wire (the sender encodes the choice by whether it
|
|
||||||
* transmits a protected-prefix list with the keep-set manifest). */
|
|
||||||
bool delete_excluded;
|
|
||||||
bool delete_after;
|
|
||||||
/* --max-delete=NUM: the receiver refuses to delete more than NUM entries per
|
|
||||||
* run (all-or-nothing: when the extras would exceed NUM nothing is removed and
|
|
||||||
* the transfer fails with a distinct error). -1 == no client limit (the
|
|
||||||
* server hard bound MAX_SERVER_DELETE_COUNT still applies). */
|
|
||||||
int max_delete;
|
|
||||||
/* --ignore-errors (client-only, never serialized): a sender-side source I/O
|
/* --ignore-errors (client-only, never serialized): a sender-side source I/O
|
||||||
* error (an unreadable directory during the scan) normally aborts the run so
|
* error (an unreadable directory during the scan) normally aborts the run so
|
||||||
* no deletion happens; with --ignore-errors the scan continues and the
|
* no deletion happens; with --ignore-errors the scan continues and the
|
||||||
* (partial) keep-set is still transmitted so the deletion runs. */
|
* (partial) keep-set is still transmitted so the deletion runs. */
|
||||||
bool ignore_errors;
|
bool ignore_errors;
|
||||||
/* --force (receiver-side): a regular file may replace a destination
|
|
||||||
* directory by removing that (possibly non-empty, symlink-safe) directory
|
|
||||||
* tree first, instead of failing the write. Crosses the wire. */
|
|
||||||
bool force_delete;
|
|
||||||
/* --ignore-missing-args (client-only, never serialized): a --files-from
|
/* --ignore-missing-args (client-only, never serialized): a --files-from
|
||||||
* entry that does not exist under the source is silently skipped instead of
|
* entry that does not exist under the source is silently skipped instead of
|
||||||
* failing the run. Sender-side only: nothing is sent for it and it never
|
* failing the run. Sender-side only: nothing is sent for it and it never
|
||||||
* enters the keep-set. Implied by --delete-missing-args. */
|
* enters the keep-set. Implied by --delete-missing-args. */
|
||||||
bool ignore_missing_args;
|
bool ignore_missing_args;
|
||||||
/* --delete-missing-args: implies --ignore-missing-args; additionally each
|
|
||||||
* missing entry's destination mirror (computed like a present entry's wire
|
|
||||||
* path) is deleted receiver-side. Crosses the wire and is gated by the
|
|
||||||
* server's --allow-delete policy like --delete. rsync-parity: independent
|
|
||||||
* of ordinary --delete processing (it does not imply --delete); a non-empty
|
|
||||||
* directory mirror is only removed with --force or --delete in effect, and
|
|
||||||
* the missing-args deletions are not counted toward --max-delete. */
|
|
||||||
bool delete_missing_args;
|
|
||||||
|
|
||||||
// Issue #129: Advanced file selection. These fields are CLIENT-ONLY: they are
|
// Issue #129: Advanced file selection. These fields are CLIENT-ONLY: they are
|
||||||
// never serialized to the wire (the receiver must not learn them).
|
// never serialized to the wire (the receiver must not learn them).
|
||||||
@@ -267,23 +349,14 @@ typedef struct Config {
|
|||||||
bool from0; /* -0/--from0: NUL-delimited *-from files */
|
bool from0; /* -0/--from0: NUL-delimited *-from files */
|
||||||
bool cvs_exclude; /* -C/--cvs-exclude: standard CVS ignore set */
|
bool cvs_exclude; /* -C/--cvs-exclude: standard CVS ignore set */
|
||||||
bool per_dir_filter; /* -F: apply per-directory .rsync-filter files */
|
bool per_dir_filter; /* -F: apply per-directory .rsync-filter files */
|
||||||
bool prune_empty_dirs;
|
|
||||||
bool one_file_system; /* -x/--one-file-system: do not cross filesystem boundaries */
|
bool one_file_system; /* -x/--one-file-system: do not cross filesystem boundaries */
|
||||||
/* -R/--relative: crosses the wire; with --files-from listed entries keep
|
|
||||||
* their bare relative destination path (no source-root mirror prefix). */
|
|
||||||
bool relative;
|
|
||||||
/* --no-implied-dirs: client-only. With -R + --files-from, refuse to place a
|
/* --no-implied-dirs: client-only. With -R + --files-from, refuse to place a
|
||||||
* listed file whose ancestor directory is not itself explicitly listed. */
|
* listed file whose ancestor directory is not itself explicitly listed. */
|
||||||
bool no_implied_dirs;
|
bool no_implied_dirs;
|
||||||
/* -d/--dirs: client-only. Transfer the directory entries named by the
|
/* -d/--dirs: client-only. Transfer the directory entries named by the
|
||||||
* source argument / --files-from list without recursing into contents. */
|
* source argument / --files-from list without recursing into contents. */
|
||||||
bool dirs;
|
bool dirs;
|
||||||
/* --mkpath: crosses the wire. Tells the server to create the destination
|
|
||||||
* root directory (and missing leading components below its authorized root)
|
|
||||||
* at connection start instead of requiring it to already exist. */
|
|
||||||
bool mkpath;
|
|
||||||
|
|
||||||
// Issue #130: Remote shell/connection options
|
|
||||||
/* -e/--rsh: the remote-shell program used to establish the SSH transport.
|
/* -e/--rsh: the remote-shell program used to establish the SSH transport.
|
||||||
* NULL means the default "ssh". Client-only launch concern: NEVER crosses
|
* NULL means the default "ssh". Client-only launch concern: NEVER crosses
|
||||||
* the wire (it is not meaningful to the daemon/server handshake). */
|
* the wire (it is not meaningful to the daemon/server handshake). */
|
||||||
@@ -296,7 +369,6 @@ typedef struct Config {
|
|||||||
* concern: NEVER crosses the wire. */
|
* concern: NEVER crosses the wire. */
|
||||||
int outbuf;
|
int outbuf;
|
||||||
bool old_args;
|
bool old_args;
|
||||||
char* temp_dir;
|
|
||||||
/* --remote-option=OPT (Phase 5, long form only): one or more extra command-line
|
/* --remote-option=OPT (Phase 5, long form only): one or more extra command-line
|
||||||
* options to append to the REMOTE server invocation over SSH. CLIENT-ONLY:
|
* options to append to the REMOTE server invocation over SSH. CLIENT-ONLY:
|
||||||
* they are composed into the remote command line by ssh_build_remote_command()
|
* they are composed into the remote command line by ssh_build_remote_command()
|
||||||
@@ -305,52 +377,20 @@ typedef struct Config {
|
|||||||
* do NOT cross the wire and are never parsed on the receiver process. */
|
* do NOT cross the wire and are never parsed on the receiver process. */
|
||||||
char** remote_options;
|
char** remote_options;
|
||||||
int remote_option_count;
|
int remote_option_count;
|
||||||
/* Alternate basis directories, ordered by command-line appearance. Each
|
|
||||||
* entry's type selects compare/copy/link behavior on an exact match. These
|
|
||||||
* cross the wire so the receiver can consult them; they are interpreted
|
|
||||||
* relative to the destination root and confined there. */
|
|
||||||
BasisDest* basis_dirs;
|
|
||||||
int basis_count;
|
|
||||||
|
|
||||||
// PR #174: Partial transfer resumption
|
|
||||||
char* partial_dir;
|
|
||||||
|
|
||||||
// PR #178: Backup versioning
|
|
||||||
char* suffix;
|
|
||||||
|
|
||||||
// PR #179: Delete policies
|
|
||||||
bool delete_before;
|
|
||||||
|
|
||||||
/* rsync deletion-timing family (real from Phase 3). At most one of
|
|
||||||
delete_before / delete_during / delete_delay / delete_after may be set, and
|
|
||||||
only together with use_delete (the CLI implies --delete for each of them).
|
|
||||||
delete_before and delete_during select the EARLY engine mode: the keep-set
|
|
||||||
manifest is transmitted before any file data and extras are removed then,
|
|
||||||
acknowledged, before the first data byte. delete_delay and delete_after
|
|
||||||
select the LATE commit mode: extras are removed only after the whole
|
|
||||||
transfer has succeeded (plain --delete keeps this mode). The exact
|
|
||||||
semantics and the divergences from rsync are documented in RSYNC_COMPAT.md
|
|
||||||
and in config_delete_timing_early() below. */
|
|
||||||
bool delete_during;
|
|
||||||
bool delete_delay;
|
|
||||||
|
|
||||||
// PR #181: IPv6 and bind address
|
// PR #181: IPv6 and bind address
|
||||||
char* address;
|
char* address;
|
||||||
char* bind_address;
|
|
||||||
bool ipv6;
|
bool ipv6;
|
||||||
bool ipv4;
|
bool ipv4;
|
||||||
/* --sockopts=OPTIONS (Phase 5, Wave B): strict allowlist of TCP/socket
|
/* --sockopts=OPTIONS (Phase 5, Wave B): strict allowlist of TCP/socket
|
||||||
* options applied via setsockopt after socket() and before connect()/bind().
|
* options applied via setsockopt after socket() and before connect()/bind().
|
||||||
* These are LOCAL socket concerns: they never cross the wire config frame.
|
* These are LOCAL socket concerns: they never cross the wire config frame.
|
||||||
* .address is the outgoing/source bind address (--address); .bind_address is
|
* .address is the outgoing/source bind address (--address). */
|
||||||
* reserved for daemon-side binding and is not wired yet. */
|
|
||||||
SockOptEntry* sockopts;
|
SockOptEntry* sockopts;
|
||||||
int sockopt_count;
|
int sockopt_count;
|
||||||
|
|
||||||
// PR #182: Daemon/server mode
|
// PR #182: Daemon/server mode
|
||||||
bool daemon;
|
bool daemon;
|
||||||
char* daemon_config;
|
|
||||||
bool server_mode;
|
|
||||||
/* --no-motd (Wave C): CLIENT-ONLY, never crosses the wire. Suppresses
|
/* --no-motd (Wave C): CLIENT-ONLY, never crosses the wire. Suppresses
|
||||||
* DISPLAY of the daemon's MOTD; the daemon still sends the MOTD frame, so
|
* DISPLAY of the daemon's MOTD; the daemon still sends the MOTD frame, so
|
||||||
* the client reads and discards it to keep the stream in sync. rsync's
|
* the client reads and discards it to keep the stream in sync. rsync's
|
||||||
@@ -358,119 +398,19 @@ typedef struct Config {
|
|||||||
* MOTD is shown when a daemon offers one). */
|
* MOTD is shown when a daemon offers one). */
|
||||||
bool no_motd;
|
bool no_motd;
|
||||||
|
|
||||||
// PR #183: Checksum comparison
|
|
||||||
bool checksum;
|
|
||||||
|
|
||||||
// PR #184: Compression algorithm negotiation
|
|
||||||
char* compress_choice;
|
|
||||||
char* chmod_spec;
|
|
||||||
|
|
||||||
/* --checksum-choice / --cc and --checksum-seed. checksum_algo is the id of
|
|
||||||
* the whole-file content-digest algorithm used by the per-file --incremental
|
|
||||||
* handshake (sender computes it, receiver compares it to skip unchanged
|
|
||||||
* files) and by the basis-dir content verification. checksum_seed is passed
|
|
||||||
* to xxHash64 (and to the delta block strong hash, low 32 bits); md5 has no
|
|
||||||
* seed so it is ignored there. Both cross the wire: the receiver MUST hash
|
|
||||||
* the on-disk old file with the same algorithm and seed to reach a matching
|
|
||||||
* digest. Defaults (XXH64 / seed 0) reproduce the pre-existing behavior
|
|
||||||
* byte-for-byte. */
|
|
||||||
int checksum_algo; /* ChecksumAlgo, default CHECKSUM_ALGO_XXH64 */
|
|
||||||
uint64_t checksum_seed; /* default 0 */
|
|
||||||
|
|
||||||
char** skip_compress_suffixes;
|
|
||||||
int skip_compress_count;
|
|
||||||
bool skip_compress_set;
|
|
||||||
|
|
||||||
// Issue #131: Identity mapping. These configure whether and how the receiver
|
|
||||||
// applies ownership when it is actually preserved/applied. ALL of them cross
|
|
||||||
// the wire (protocol 2.11.0) so the receiver resolves and applies ownership
|
|
||||||
// with the exact policy the client requested. Plain -M/--preserve still does
|
|
||||||
// NOT apply ownership (FastSync's deliberate conservative default); it is
|
|
||||||
// only attempted when at least one of these is set (see identity.h).
|
|
||||||
/* --numeric-ids: no name lookup, use the transmitted numeric ids raw. */
|
|
||||||
bool numeric_ids;
|
|
||||||
/* --chown USER (owner) override; IDENTITY_CURRENT = the receiver's euid. */
|
|
||||||
bool chown_uid_set;
|
|
||||||
int32_t chown_uid;
|
|
||||||
/* --chown :GROUP (group) override; IDENTITY_CURRENT = the receiver's egid. */
|
|
||||||
bool chown_gid_set;
|
|
||||||
int32_t chown_gid;
|
|
||||||
/* --usermap / --groupmap entries, in order (first match wins). */
|
|
||||||
IdentityMap* usermap;
|
|
||||||
int usermap_count;
|
|
||||||
IdentityMap* groupmap;
|
|
||||||
int groupmap_count;
|
|
||||||
|
|
||||||
/* --super / --no-super (P7 Wave E, protocol 2.18.0): receiver-side privilege
|
|
||||||
* policy for super-user activities confined below the authorized receive
|
|
||||||
* root. SUPER_MODE_AUTO (default) preserves the pre-existing best-effort
|
|
||||||
* behavior: the confined super-user operation is ALWAYS attempted and an
|
|
||||||
* unprivileged attempt is refused by the kernel and skipped per entry.
|
|
||||||
* SUPER_MODE_ON (--super) explicitly REQUESTS those activities (char/block
|
|
||||||
* device-node creation, --write-devices); it does NOT imply --numeric-ids and
|
|
||||||
* never enables ownership application on its own. SUPER_MODE_OFF
|
|
||||||
* (--no-super) FORBIDS them even when running as root. FastSync NEVER
|
|
||||||
* elevates privileges (no setuid/seteuid/setgid) and never bypasses the
|
|
||||||
* fd-relative confinement (file_open_secure_parent, O_NOFOLLOW, root checks);
|
|
||||||
* --super only permits an attempt that is already confined. Crosses the wire
|
|
||||||
* as a trailing int so the receiver can enforce the policy. See
|
|
||||||
* privilege_super_permitted() and identity_ownership_requested() in
|
|
||||||
* identity.h. */
|
|
||||||
int super_mode;
|
|
||||||
|
|
||||||
// Receiver-side runtime staging registry for --delay-updates. Never sent
|
// Receiver-side runtime staging registry for --delay-updates. Never sent
|
||||||
// over the wire and never set on the sender side.
|
// over the wire and never set on the sender side.
|
||||||
DelayUpdatesContext* delay_context;
|
DelayUpdatesContext* delay_context;
|
||||||
|
|
||||||
// Phase 4: metadata time preservation. -U/--atimes and -N/--crtimes capture
|
|
||||||
// and transmit the source access / birth time (both sender and receiver
|
|
||||||
// effect, so they CROSS the wire). --omit-dir-times/-O and
|
|
||||||
// --omit-link-times/-J are receiver-side prefs (CROSS the wire). Their
|
|
||||||
// exact capture/transmit/apply semantics are documented in RSYNC_COMPAT.md.
|
|
||||||
/* -U/--atimes: preserve source access times on the destination. */
|
|
||||||
bool preserve_atimes;
|
|
||||||
/* -N/--crtimes: capture+transmit source birth time; see RSYNC_COMPAT for the
|
|
||||||
* receiver not-applied divergence. */
|
|
||||||
bool preserve_crtimes;
|
|
||||||
/* -O/--omit-dir-times: do not apply mtimes to directories. */
|
|
||||||
bool omit_dir_times;
|
|
||||||
/* -J/--omit-link-times: do not apply times to symlinks. */
|
|
||||||
bool omit_link_times;
|
|
||||||
/* --open-noatime: CLIENT-ONLY (never crosses the wire). The sender opens
|
/* --open-noatime: CLIENT-ONLY (never crosses the wire). The sender opens
|
||||||
* source files with O_NOATIME so reading for transfer does not bump the
|
* source files with O_NOATIME so reading for transfer does not bump the
|
||||||
* source access time. */
|
* source access time. */
|
||||||
bool open_noatime;
|
bool open_noatime;
|
||||||
|
|
||||||
// Phase 4: xattr / ACL / fake-super preservation.
|
|
||||||
/* -X/--xattrs and -A/--acls toggle the sender's capture and the receiver's
|
|
||||||
* application of per-file extended attributes (xattrs). Both cross the wire:
|
|
||||||
* the sender only transmits the bounded, whitelisted attribute set it
|
|
||||||
* captures and the receiver re-validates namespaces/sizes before applying
|
|
||||||
* fd-relative. With neither set (the default) no xattr block is sent, so the
|
|
||||||
* wire is byte-identical to prior protocol versions for unaffected runs. */
|
|
||||||
/* true when preserve_xattrs || preserve_acls; the sender/receiver gate the
|
/* true when preserve_xattrs || preserve_acls; the sender/receiver gate the
|
||||||
* xattr wire block on this single flag. */
|
* xattr wire block on this single flag. */
|
||||||
bool use_xattrs;
|
bool use_xattrs;
|
||||||
/* --fake-super: receiver-only. When set, each written file additionally gets
|
|
||||||
* a reserved user.fastsync.stat xattr recording the source uid/gid/mode/mtime
|
|
||||||
* so a later privileged restore could re-apply them. Crosses the wire. */
|
|
||||||
bool fake_super;
|
|
||||||
/* --copy-as=USER[:GROUP] (P7 Wave E, protocol 2.18.0). Safe-subset
|
|
||||||
* implementation, a documented divergence from rsync's real identity switch:
|
|
||||||
* the receiver does NOT change its process credentials (FastSync's receiver
|
|
||||||
* is multithreaded, so a setuid/seteuid drop would be unsafe). Instead the
|
|
||||||
* receiver FORCES the ownership of every entry it writes to copy_as_uid /
|
|
||||||
* copy_as_gid through the existing confined, fd-relative identity path
|
|
||||||
* (fchown/fchownat), which REQUIRES receiver privilege (root); an
|
|
||||||
* unprivileged receiver REFUSES the whole transfer up front at the config
|
|
||||||
* handshake (never a silent wrong-ownership result). All three fields CROSS
|
|
||||||
* the wire as a trailing config-frame block so the receiver learns the
|
|
||||||
* requested ids; see the PROTOCOL_VERSION note below. */
|
|
||||||
bool copy_as_set;
|
|
||||||
int32_t copy_as_uid;
|
|
||||||
int32_t copy_as_gid;
|
|
||||||
|
|
||||||
// Phase 5: --trust-sender
|
|
||||||
/* Long-form-only, receiver-local policy. rsync's --trust-sender tells the
|
/* Long-form-only, receiver-local policy. rsync's --trust-sender tells the
|
||||||
* receiving side to trust that the sender already produced a sane file list,
|
* receiving side to trust that the sender already produced a sane file list,
|
||||||
* relaxing the receiver's own up-front re-validation of every incoming path.
|
* relaxing the receiver's own up-front re-validation of every incoming path.
|
||||||
@@ -489,7 +429,6 @@ typedef struct Config {
|
|||||||
* default; only relaxes validation when explicitly requested. */
|
* default; only relaxes validation when explicitly requested. */
|
||||||
bool trust_sender;
|
bool trust_sender;
|
||||||
|
|
||||||
// Phase 6: --stop-after / --stop-at
|
|
||||||
/* Client-only sender-side transfer stop deadlines. --stop-after=MINS stops
|
/* Client-only sender-side transfer stop deadlines. --stop-after=MINS stops
|
||||||
* the transfer after a number of elapsed minutes (checked against
|
* the transfer after a number of elapsed minutes (checked against
|
||||||
* CLOCK_MONOTONIC so clock changes do not skew it); --stop-at=TIME stops at
|
* CLOCK_MONOTONIC so clock changes do not skew it); --stop-at=TIME stops at
|
||||||
@@ -501,7 +440,6 @@ typedef struct Config {
|
|||||||
time_t stop_at; /* --stop-at=... absolute wall-clock deadline */
|
time_t stop_at; /* --stop-at=... absolute wall-clock deadline */
|
||||||
bool stop_at_set; /* true when --stop-at was given */
|
bool stop_at_set; /* true when --stop-at was given */
|
||||||
|
|
||||||
// Phase 6: --write-batch / --only-write-batch / --read-batch
|
|
||||||
/* Client-only residual-batch paths. A residual batch is a self-contained
|
/* Client-only residual-batch paths. A residual batch is a self-contained
|
||||||
* single-file record of the whole source tree (full file images using the
|
* single-file record of the whole source tree (full file images using the
|
||||||
* chunk codec), independent of any live server. --write-batch=FILE runs the
|
* chunk codec), independent of any live server. --write-batch=FILE runs the
|
||||||
@@ -513,6 +451,196 @@ typedef struct Config {
|
|||||||
char* write_batch; /* --write-batch=FILE path, or NULL */
|
char* write_batch; /* --write-batch=FILE path, or NULL */
|
||||||
char* only_write_batch; /* --only-write-batch=FILE path, or NULL */
|
char* only_write_batch; /* --only-write-batch=FILE path, or NULL */
|
||||||
char* read_batch; /* --read-batch=FILE path, or NULL */
|
char* read_batch; /* --read-batch=FILE path, or NULL */
|
||||||
|
|
||||||
|
/* ===================================================================
|
||||||
|
* Serialized wire fields. Their members, defaults and send/receive
|
||||||
|
* sequence are generated from the CONFIG_WIRE_*_FIELDS table above (the
|
||||||
|
* single source of truth); they are declared here in exact wire order.
|
||||||
|
* The per-field notes were moved here from their original positions and
|
||||||
|
* are listed in wire order.
|
||||||
|
* =================================================================== */
|
||||||
|
/* copy_links */
|
||||||
|
// Issue #120: Symlink handling
|
||||||
|
/* preserve_hard_links */
|
||||||
|
// Issue #121: Extended metadata preservation
|
||||||
|
/* preserve_specials */
|
||||||
|
/* Phase 4 special/devices: preserve special files (FIFOs, sockets) and device
|
||||||
|
* nodes on the destination by recreating them (mknod/mkfifo) instead of
|
||||||
|
* transferring content. preserve_specials mirrors rsync --specials (the
|
||||||
|
* special-file half of -D); preserve_devices mirrors --devices (the device
|
||||||
|
* half of -D); both CROSS the wire so the receiver knows a special/device
|
||||||
|
* entry must be recreated rather than written as a regular file. */
|
||||||
|
/* copy_devices */
|
||||||
|
/* --copy-devices: copy the CONTENT of a source device as an ordinary regular
|
||||||
|
* file on the destination (rsync's non-privileged safe mode), instead of
|
||||||
|
* recreating the device node. CROSSES the wire (receiver treats the entry as
|
||||||
|
* a regular file, which is the default, so this is belt-and-braces). */
|
||||||
|
/* write_devices */
|
||||||
|
/* --write-devices: write the received data directly INTO an existing device
|
||||||
|
* node on the destination instead of creating a regular file. Dangeroud;
|
||||||
|
* see RSYNC_COMPAT.md for the tight gating. CROSSES the wire. */
|
||||||
|
/* existing */
|
||||||
|
// Issue #127: Transfer modes
|
||||||
|
/* delete_excluded */
|
||||||
|
/* --delete-excluded: also delete destination entries that were excluded on
|
||||||
|
* the source. Default (off) matches rsync: excluded paths are protected from
|
||||||
|
* deletion. Crosses the wire (the sender encodes the choice by whether it
|
||||||
|
* transmits a protected-prefix list with the keep-set manifest). */
|
||||||
|
/* force_delete */
|
||||||
|
/* --force (receiver-side): a regular file may replace a destination
|
||||||
|
* directory by removing that (possibly non-empty, symlink-safe) directory
|
||||||
|
* tree first, instead of failing the write. Crosses the wire. */
|
||||||
|
/* delete_missing_args */
|
||||||
|
/* --delete-missing-args: implies --ignore-missing-args; additionally each
|
||||||
|
* missing entry's destination mirror (computed like a present entry's wire
|
||||||
|
* path) is deleted receiver-side. Crosses the wire and is gated by the
|
||||||
|
* server's --allow-delete policy like --delete. rsync-parity: independent
|
||||||
|
* of ordinary --delete processing (it does not imply --delete); a non-empty
|
||||||
|
* directory mirror is only removed with --force or --delete in effect, and
|
||||||
|
* the missing-args deletions are not counted toward --max-delete. */
|
||||||
|
/* preallocate */
|
||||||
|
/* --preallocate: allocates the destination file's full expected space up
|
||||||
|
* front (before any data is written) so a transfer that would overflow disk
|
||||||
|
* fails fast at allocation time and the file is laid out contiguously,
|
||||||
|
* avoiding fragmentation. Receiver-side, crosses the wire. */
|
||||||
|
/* max_delete */
|
||||||
|
/* --max-delete=NUM: the receiver refuses to delete more than NUM entries per
|
||||||
|
* run (all-or-nothing: when the extras would exceed NUM nothing is removed and
|
||||||
|
* the transfer fails with a distinct error). -1 == no client limit (the
|
||||||
|
* server hard bound MAX_SERVER_DELETE_COUNT still applies). */
|
||||||
|
/* relative */
|
||||||
|
/* -R/--relative: crosses the wire; with --files-from listed entries keep
|
||||||
|
* their bare relative destination path (no source-root mirror prefix). */
|
||||||
|
/* mkpath */
|
||||||
|
/* --mkpath: crosses the wire. Tells the server to create the destination
|
||||||
|
* root directory (and missing leading components below its authorized root)
|
||||||
|
* at connection start instead of requiring it to already exist. */
|
||||||
|
/* delete_during */
|
||||||
|
/* rsync deletion-timing family (real from Phase 3). At most one of
|
||||||
|
delete_before / delete_during / delete_delay / delete_after may be set, and
|
||||||
|
only together with use_delete (the CLI implies --delete for each of them).
|
||||||
|
delete_before and delete_during select the EARLY engine mode: the keep-set
|
||||||
|
manifest is transmitted before any file data and extras are removed then,
|
||||||
|
acknowledged, before the first data byte. delete_delay and delete_after
|
||||||
|
select the LATE commit mode: extras are removed only after the whole
|
||||||
|
transfer has succeeded (plain --delete keeps this mode). The exact
|
||||||
|
semantics and the divergences from rsync are documented in RSYNC_COMPAT.md
|
||||||
|
and in config_delete_timing_early() below. */
|
||||||
|
/* partial_dir */
|
||||||
|
// PR #174: Partial transfer resumption
|
||||||
|
/* suffix */
|
||||||
|
// PR #178: Backup versioning
|
||||||
|
/* delete_before */
|
||||||
|
// PR #179: Delete policies
|
||||||
|
/* checksum */
|
||||||
|
// PR #183: Checksum comparison
|
||||||
|
/* compress_choice */
|
||||||
|
// PR #184: Compression algorithm negotiation
|
||||||
|
/* basis_dirs */
|
||||||
|
/* Alternate basis directories, ordered by command-line appearance. Each
|
||||||
|
* entry's type selects compare/copy/link behavior on an exact match. These
|
||||||
|
* cross the wire so the receiver can consult them; they are interpreted
|
||||||
|
* relative to the destination root and confined there. */
|
||||||
|
/* fuzzy */
|
||||||
|
/* -y/--fuzzy: when a file must be transferred and the destination holds no
|
||||||
|
* usable file at the exact path, the receiver may reuse a SIMILAR-named
|
||||||
|
* existing regular file in the same destination directory as the delta
|
||||||
|
* basis so the sender transmits only the differences. Crosses the wire
|
||||||
|
* (the receiver performs the candidate search); the CLI implies
|
||||||
|
* --incremental + --delta because the similar-basis only matters on the
|
||||||
|
* receiver-driven delta path. Off by default. */
|
||||||
|
/* checksum_algo / checksum_seed */
|
||||||
|
/* --checksum-choice / --cc and --checksum-seed. checksum_algo is the id of
|
||||||
|
* the whole-file content-digest algorithm used by the per-file --incremental
|
||||||
|
* handshake (sender computes it, receiver compares it to skip unchanged
|
||||||
|
* files) and by the basis-dir content verification. checksum_seed is passed
|
||||||
|
* to xxHash64 (and to the delta block strong hash, low 32 bits); md5 has no
|
||||||
|
* seed so it is ignored there. Both cross the wire: the receiver MUST hash
|
||||||
|
* the on-disk old file with the same algorithm and seed to reach a matching
|
||||||
|
* digest. */
|
||||||
|
/* munge_links / keep_dirlinks */
|
||||||
|
/* Phase 4 symlink-trust: both cross the wire (the receiver unmunges symlink
|
||||||
|
* targets and, with -K, follows an in-root destination symlink-to-directory);
|
||||||
|
* -k/--copy-dirlinks is sender-only and is never serialized. */
|
||||||
|
/* numeric_ids */
|
||||||
|
/* --numeric-ids: no name lookup, use the transmitted numeric ids raw. */
|
||||||
|
/* chown_uid_set */
|
||||||
|
/* --chown USER (owner) override; IDENTITY_CURRENT = the receiver's euid. */
|
||||||
|
/* chown_gid_set */
|
||||||
|
/* --chown :GROUP (group) override; IDENTITY_CURRENT = the receiver's egid. */
|
||||||
|
/* usermap */
|
||||||
|
/* --usermap / --groupmap entries, in order (first match wins). */
|
||||||
|
/* preserve_atimes */
|
||||||
|
/* -U/--atimes: preserve source access times on the destination. */
|
||||||
|
/* preserve_crtimes */
|
||||||
|
/* -N/--crtimes: capture+transmit source birth time; see RSYNC_COMPAT for the
|
||||||
|
* receiver not-applied divergence. */
|
||||||
|
/* omit_dir_times */
|
||||||
|
/* -O/--omit-dir-times: do not apply mtimes to directories. */
|
||||||
|
/* omit_link_times */
|
||||||
|
/* -J/--omit-link-times: do not apply times to symlinks. */
|
||||||
|
/* fake_super */
|
||||||
|
/* --fake-super: receiver-only. When set, each written file additionally gets
|
||||||
|
* a reserved user.fastsync.stat xattr recording the source uid/gid/mode/mtime
|
||||||
|
* so a later privileged restore could re-apply them. Crosses the wire. */
|
||||||
|
/* module */
|
||||||
|
/* Daemon module selection (Wave A, protocol 2.15.0). Client-composed from a
|
||||||
|
* host::module/path destination; NULL or "" means "no module" (the ordinary
|
||||||
|
* standalone-server path). Crosses the wire as a trailing config-frame
|
||||||
|
* string so the daemon can look the module up in its own config and confine
|
||||||
|
* the connection to the module's root (never a client-chosen root). */
|
||||||
|
/* auth_user */
|
||||||
|
/* Daemon password authentication (A7 remediation, protocol 2.19.0).
|
||||||
|
* Client-composed from a --password-file whose first meaningful line is
|
||||||
|
* `user:password`: the client sends ONLY the username in the config frame
|
||||||
|
* (auth_user); the literal password is kept in auth_password CLIENT-SIDE for
|
||||||
|
* the duration of the SCRAM challenge/response and is NEVER serialized. Both
|
||||||
|
* are NULL when the client has no credentials to present; a module WITHOUT
|
||||||
|
* `auth users` stays open and the server ignores any credentials that do
|
||||||
|
* arrive (the client sends them opportunistically and the server decides). */
|
||||||
|
/* iconv_spec */
|
||||||
|
/* --iconv=CONVERT_SPEC (protocol 2.16.0, rsync compatibility): convert the
|
||||||
|
* charset of FILE NAMES at the wire boundary. CONVERT_SPEC is
|
||||||
|
* "LOCAL[,REMOTE]": LOCAL is the charset of our own file names, REMOTE is
|
||||||
|
* the remote side's charset and defaults to LOCAL. The sender converts
|
||||||
|
* every path LOCAL->REMOTE before transmitting it; the receiver converts
|
||||||
|
* every received path back REMOTE->LOCAL before creating/writing it. The
|
||||||
|
* FULL SPEC crosses the wire as a trailing config-frame string so each end
|
||||||
|
* derives its own LOCAL and the wire (REMOTE) charset symmetrically. NULL
|
||||||
|
* (or "") means no conversion: identity with zero overhead. See charset.c
|
||||||
|
* and the PROTOCOL_VERSION note below. */
|
||||||
|
/* super_mode */
|
||||||
|
/* --super / --no-super (P7 Wave E, protocol 2.18.0): receiver-side privilege
|
||||||
|
* policy for super-user activities confined below the authorized receive
|
||||||
|
* root. SUPER_MODE_AUTO (default) preserves the pre-existing best-effort
|
||||||
|
* behavior: the confined super-user operation is ALWAYS attempted and an
|
||||||
|
* unprivileged attempt is refused by the kernel and skipped per entry.
|
||||||
|
* SUPER_MODE_ON (--super) explicitly REQUESTS those activities (char/block
|
||||||
|
* device-node creation, --write-devices); it does NOT imply --numeric-ids and
|
||||||
|
* never enables ownership application on its own. SUPER_MODE_OFF
|
||||||
|
* (--no-super) FORBIDS them even when running as root. FastSync NEVER
|
||||||
|
* elevates privileges (no setuid/seteuid/setgid) and never bypasses the
|
||||||
|
* fd-relative confinement (file_open_secure_parent, O_NOFOLLOW, root checks);
|
||||||
|
* --super only permits an attempt that is already confined. Crosses the wire
|
||||||
|
* as a trailing int so the receiver can enforce the policy. See
|
||||||
|
* privilege_super_permitted() and identity_ownership_requested() in
|
||||||
|
* identity.h. */
|
||||||
|
/* copy_as_set */
|
||||||
|
/* --copy-as=USER[:GROUP] (P7 Wave E, protocol 2.18.0). Safe-subset
|
||||||
|
* implementation, a documented divergence from rsync's real identity switch:
|
||||||
|
* the receiver does NOT change its process credentials (FastSync's receiver
|
||||||
|
* is multithreaded, so a setuid/seteuid drop would be unsafe). Instead the
|
||||||
|
* receiver FORCES the ownership of every entry it writes to copy_as_uid /
|
||||||
|
* copy_as_gid through the existing confined, fd-relative identity path
|
||||||
|
* (fchown/fchownat), which REQUIRES receiver privilege (root); an
|
||||||
|
* unprivileged receiver REFUSES the whole transfer up front at the config
|
||||||
|
* handshake (never a silent wrong-ownership result). All three fields CROSS
|
||||||
|
* the wire as a trailing config-frame block so the receiver learns the
|
||||||
|
* requested ids; see the PROTOCOL_VERSION note below. */
|
||||||
|
|
||||||
|
#define CONFIG_STRUCT_MEMBER(name, ctype, def, kind) ctype name;
|
||||||
|
CONFIG_WIRE_FIELDS(CONFIG_STRUCT_MEMBER)
|
||||||
|
#undef CONFIG_STRUCT_MEMBER
|
||||||
} Config;
|
} Config;
|
||||||
|
|
||||||
/* Phase 5 (remote-option wave): 2.13.0 -> 2.14.0.
|
/* Phase 5 (remote-option wave): 2.13.0 -> 2.14.0.
|
||||||
@@ -630,12 +758,75 @@ typedef struct Config {
|
|||||||
* anything else) is what keeps a 2.19 client and a 2.18 server from ever
|
* anything else) is what keeps a 2.19 client and a 2.18 server from ever
|
||||||
* reaching that state. SECURITY: a 2.19 store holds a salted PBKDF2 verifier
|
* reaching that state. SECURITY: a 2.19 store holds a salted PBKDF2 verifier
|
||||||
* and cannot verify (and refuses to load) a legacy unsalted-SHA-256 store line,
|
* and cannot verify (and refuses to load) a legacy unsalted-SHA-256 store line,
|
||||||
* so an old bearer digest can never be replayed against a 2.19 daemon. */
|
* so an old bearer digest can never be replayed against a 2.19 daemon.
|
||||||
#define PROTOCOL_VERSION "2.19.0"
|
*
|
||||||
|
* Packed Metadata Wave: 2.19.0 -> 2.20.0.
|
||||||
|
*
|
||||||
|
* WHY the bump: metadata_send()/metadata_receive() no longer emit/consume the
|
||||||
|
* metadata as up to 12 separate per-field writes. A file's metadata now
|
||||||
|
* crosses the wire as ONE packed frame: a single int32 present flag (0 =
|
||||||
|
* absent, 1 = present) followed, when present, by the fixed
|
||||||
|
* FILE_METADATA_WIRE_SIZE-byte (68-byte) field record produced by
|
||||||
|
* metadata_to_buf(). Protocol data is an unframed byte stream, so the packed
|
||||||
|
* encoding is byte-for-byte identical to the old field-by-field writes (same
|
||||||
|
* fields, same order, same widths); the change only removes per-field syscalls.
|
||||||
|
* The bump is therefore a deliberate lockstep-release marker, not a
|
||||||
|
* desynchronization fix — the strict same-version handshake still rejects a
|
||||||
|
* mixed 2.19/2.20 deployment. The chunk codec, which already used the packed
|
||||||
|
* metadata_to_buf()/metadata_from_buf() form, is unchanged.
|
||||||
|
*
|
||||||
|
* Error-Detail + Server-contacting Dry-run Wave: 2.20.0 -> 2.21.0.
|
||||||
|
*
|
||||||
|
* WHY the bump, grounded in the wire: this release combines two changes on the
|
||||||
|
* same lockstep version.
|
||||||
|
*
|
||||||
|
* (1) Error detail: a server may now answer a rejected operation with
|
||||||
|
* STATUS_ERROR_DETAIL followed by a bounded (<= MAX_ERROR_DETAIL_BYTES)
|
||||||
|
* length-prefixed string instead of a bare STATUS_ERROR (see protocol.h). The
|
||||||
|
* config-frame LAYOUT is unchanged, but the FRAME STREAM gains a new framed
|
||||||
|
* body after a status, so a 2.20 peer that does not consume it would
|
||||||
|
* desynchronize on the following exchange. receive_status() transparently maps
|
||||||
|
* STATUS_ERROR_DETAIL back to STATUS_ERROR for every existing call site and
|
||||||
|
* captures the reason into a thread-local buffer consulted via
|
||||||
|
* protocol_last_error().
|
||||||
|
*
|
||||||
|
* (2) --dry-run: --dry-run now contacts the receiver and reports exactly what
|
||||||
|
* WOULD change. The binary config frame gains one serialized bool
|
||||||
|
* (Config->dry_run) appended to CONFIG_WIRE_CORE_FIELDS after use_sendfile, and
|
||||||
|
* the frame stream gains one terminal status (STATUS_DRY_RUN_TRANSFER) sent in
|
||||||
|
* reply to a per-file STATUS_CHECK when the file is not already up to date.
|
||||||
|
* The receiver performs the normal read-only incremental decision but no
|
||||||
|
* mutation; the sender then skips the data.
|
||||||
|
*
|
||||||
|
* Any config-frame layout or frame-sequence change must bump the protocol
|
||||||
|
* version: a 2.20 peer would desynchronize on the extra trailing byte, the
|
||||||
|
* unknown status, or the unconsumed detail body, and the strict same-version
|
||||||
|
* handshake (config_receive rejects a mismatched version before parsing
|
||||||
|
* anything else) is what keeps a 2.21 client and a 2.20 server from ever
|
||||||
|
* reaching that state. */
|
||||||
|
#define PROTOCOL_VERSION "2.21.0"
|
||||||
#define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024)
|
#define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024)
|
||||||
/* Upper bound on total basis-dir entries (rsync caps --link-dest at 20). */
|
/* Upper bound on total basis-dir entries (rsync caps --link-dest at 20). */
|
||||||
#define MAX_BASIS_DIRS 64
|
#define MAX_BASIS_DIRS 64
|
||||||
|
|
||||||
|
/* Upper bound on the number of --skip-compress suffixes accepted from the wire.
|
||||||
|
* Each suffix is an independent wire string (up to MAX_STRING_SIZE = 64 KiB), so
|
||||||
|
* without this a hostile pre-auth client could otherwise retain
|
||||||
|
* skip_count * MAX_STRING_SIZE bytes on the server before authentication; 256
|
||||||
|
* covers any realistic suffix list while keeping the worst case small. */
|
||||||
|
#define MAX_SKIP_COMPRESS_SUFFIXES 256
|
||||||
|
|
||||||
|
/* Aggregate ceiling on the bytes retained by ALL strings in one received config
|
||||||
|
* frame (version, send/receive roots, backup/temp/partial/suffix, compression
|
||||||
|
* choice, chmod spec, skip-compress suffixes, basis paths, module, auth user,
|
||||||
|
* iconv spec, ...). The config frame is parsed BEFORE authentication and every
|
||||||
|
* one of these strings lives for the whole connection, so this cumulative
|
||||||
|
* (never released) budget bounds the pre-auth memory a single connection can
|
||||||
|
* pin. MAX_SKIP_COMPRESS_SUFFIXES / MAX_BASIS_DIRS bound the individual
|
||||||
|
* repeatable counts; this budget bounds their product and any single oversized
|
||||||
|
* field. */
|
||||||
|
#define MAX_CONFIG_STRING_BYTES (1ULL * 1024 * 1024)
|
||||||
|
|
||||||
/* Identity-mapping sentinels and bounds (see identity.h for semantics).
|
/* Identity-mapping sentinels and bounds (see identity.h for semantics).
|
||||||
* IDENTITY_MATCH_ANY is a usermap/groupmap FROM '*' (matches any id);
|
* IDENTITY_MATCH_ANY is a usermap/groupmap FROM '*' (matches any id);
|
||||||
* IDENTITY_CURRENT is a chown / map TO '*' (resolve to the receiver's current
|
* IDENTITY_CURRENT is a chown / map TO '*' (resolve to the receiver's current
|
||||||
@@ -644,15 +835,6 @@ typedef struct Config {
|
|||||||
#define IDENTITY_CURRENT (-1)
|
#define IDENTITY_CURRENT (-1)
|
||||||
#define MAX_IDENTITY_MAP 128
|
#define MAX_IDENTITY_MAP 128
|
||||||
|
|
||||||
/* --super / --no-super tri-state (Config->super_mode). AUTO (default) and ON
|
|
||||||
* both permit a confined super-user attempt (AUTO preserves FastSync's
|
|
||||||
* historical best-effort behavior; an unprivileged attempt is refused by the
|
|
||||||
* kernel and skipped per entry); OFF forbids the attempt even for root. See
|
|
||||||
* privilege_super_mode_permitted() in identity.h. */
|
|
||||||
#define SUPER_MODE_AUTO 0
|
|
||||||
#define SUPER_MODE_ON 1
|
|
||||||
#define SUPER_MODE_OFF 2
|
|
||||||
|
|
||||||
Config* config_create(void);
|
Config* config_create(void);
|
||||||
void config_delete(Config* config);
|
void config_delete(Config* config);
|
||||||
|
|
||||||
@@ -663,9 +845,16 @@ void config_delete(Config* config);
|
|||||||
void config_burn_auth(Config* config);
|
void config_burn_auth(Config* config);
|
||||||
|
|
||||||
bool config_send(int file_descriptor, const Config* config);
|
bool config_send(int file_descriptor, const Config* config);
|
||||||
|
/* Emit the config frame BODY (every serialized field, in wire order) without
|
||||||
|
* the trailing STATUS_OK handshake. config_send() is this plus the handshake;
|
||||||
|
* the wire-compatibility golden test uses it to hash the exact byte stream. */
|
||||||
|
bool config_send_wire_block(int file_descriptor, const Config* config);
|
||||||
Config* config_receive(int file_descriptor);
|
Config* config_receive(int file_descriptor);
|
||||||
bool config_is_remote_dest(const char* s);
|
bool config_is_remote_dest(const char* s);
|
||||||
void config_parse_ssh_dest(Config* config);
|
/* Parse a single-colon host:path SSH destination (0 = not an SSH destination or
|
||||||
|
* parsed successfully, -1 = rejected, e.g. a user@host beginning with '-'; the
|
||||||
|
* reason is logged). */
|
||||||
|
int config_parse_ssh_dest(Config* config);
|
||||||
|
|
||||||
/* A ConfigValidateFunc may return this sentinel to tell
|
/* A ConfigValidateFunc may return this sentinel to tell
|
||||||
* config_receive_with_validate that the callback ALREADY sent a terminal status
|
* config_receive_with_validate that the callback ALREADY sent a terminal status
|
||||||
@@ -718,6 +907,17 @@ bool config_delete_timing_early(const Config* config);
|
|||||||
* set (none = the default delete-after commit timing); without deletion no
|
* set (none = the default delete-after commit timing); without deletion no
|
||||||
* timing flag may be set (each timing flag implies --delete). */
|
* timing flag may be set (each timing flag implies --delete). */
|
||||||
bool config_has_valid_delete_timing(const Config* config);
|
bool config_has_valid_delete_timing(const Config* config);
|
||||||
|
|
||||||
|
/* Single source of truth for the cross-field ("combination") invariants a
|
||||||
|
* Config must satisfy. Returns NULL when `config` is consistent, or a static,
|
||||||
|
* human-readable error string (no trailing period) describing the FIRST
|
||||||
|
* violation found. No I/O, no logging and no printing, so it is safe to call
|
||||||
|
* from every trust boundary; the iconv rule does invoke charset_spec_valid
|
||||||
|
* (which parses via str_dup/iconv_open), so it is not allocation-free. The client calls
|
||||||
|
* it from validate_config() for up-front UX and the server calls it from
|
||||||
|
* validate_received_config() so the receiver enforces exactly the same
|
||||||
|
* invariants it relies on (the server is the trust boundary). */
|
||||||
|
const char* config_invariants_error(const Config* config);
|
||||||
/* True when at least one --compare-dest/--copy-dest/--link-dest was set. */
|
/* True when at least one --compare-dest/--copy-dest/--link-dest was set. */
|
||||||
bool config_has_basis(const Config* config);
|
bool config_has_basis(const Config* config);
|
||||||
/* Append one basis-dir entry. Returns 0 on success, -1 on allocation failure. */
|
/* Append one basis-dir entry. Returns 0 on success, -1 on allocation failure. */
|
||||||
|
|||||||
@@ -158,13 +158,13 @@ static int hex_value(char c) {
|
|||||||
* digits. Such a line is refused loudly (and never accepted) so an operator
|
* digits. Such a line is refused loudly (and never accepted) so an operator
|
||||||
* cannot keep a replayable bearer digest in place after the protocol bump. */
|
* cannot keep a replayable bearer digest in place after the protocol bump. */
|
||||||
static bool secret_is_legacy_hex(const char* s) {
|
static bool secret_is_legacy_hex(const char* s) {
|
||||||
if (!s)
|
if (!s || strlen(s) != 64)
|
||||||
return false;
|
return false;
|
||||||
for (int i = 0; i < 64; i++) {
|
for (int i = 0; i < 64; i++) {
|
||||||
if (hex_value(s[i]) < 0)
|
if (hex_value(s[i]) < 0)
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
return s[64] == '\0';
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool credentials_b64_encode(const uint8_t* in, size_t n, char* out, size_t out_sz) {
|
bool credentials_b64_encode(const uint8_t* in, size_t n, char* out, size_t out_sz) {
|
||||||
|
|||||||
+344
-4
@@ -1,8 +1,13 @@
|
|||||||
#include "daemon_conf.h"
|
#include "daemon_conf.h"
|
||||||
|
#include "credentials.h"
|
||||||
#include "utils.h"
|
#include "utils.h"
|
||||||
|
#include <arpa/inet.h>
|
||||||
#include <ctype.h>
|
#include <ctype.h>
|
||||||
#include <errno.h>
|
#include <errno.h>
|
||||||
|
#include <limits.h>
|
||||||
|
#include <netinet/in.h>
|
||||||
#include <stdarg.h>
|
#include <stdarg.h>
|
||||||
|
#include <stdint.h>
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
@@ -48,6 +53,191 @@ static bool parse_bool_value(const char* value, bool* out) {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Parse an IPv4/IPv6 CIDR "addr/prefix" into `bytes`/`*family`. Returns false
|
||||||
|
* for a malformed address, a missing/oversized prefix, or a prefix that does
|
||||||
|
* not fit the address family. */
|
||||||
|
static bool parse_cidr(const char* cidr, int* prefix_out, uint8_t* bytes, int* family_out) {
|
||||||
|
const char* slash = strchr(cidr, '/');
|
||||||
|
if (!slash)
|
||||||
|
return false;
|
||||||
|
size_t addr_len = (size_t)(slash - cidr);
|
||||||
|
if (addr_len == 0 || addr_len >= INET6_ADDRSTRLEN)
|
||||||
|
return false;
|
||||||
|
char addr[INET6_ADDRSTRLEN];
|
||||||
|
memcpy(addr, cidr, addr_len);
|
||||||
|
addr[addr_len] = '\0';
|
||||||
|
char* end = NULL;
|
||||||
|
long prefix = strtol(slash + 1, &end, 10);
|
||||||
|
if (end == slash + 1 || *end != '\0')
|
||||||
|
return false;
|
||||||
|
struct in_addr v4;
|
||||||
|
struct in6_addr v6;
|
||||||
|
if (inet_pton(AF_INET, addr, &v4) == 1) {
|
||||||
|
if (prefix < 0 || prefix > 32)
|
||||||
|
return false;
|
||||||
|
memcpy(bytes, &v4, sizeof(v4));
|
||||||
|
*prefix_out = (int)prefix;
|
||||||
|
*family_out = AF_INET;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (inet_pton(AF_INET6, addr, &v6) == 1) {
|
||||||
|
if (prefix < 0 || prefix > 128)
|
||||||
|
return false;
|
||||||
|
memcpy(bytes, &v6, sizeof(v6));
|
||||||
|
*prefix_out = (int)prefix;
|
||||||
|
*family_out = AF_INET6;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* A host pattern is valid when it is `*`, a valid IPv4/IPv6 literal, or a valid
|
||||||
|
* CIDR. Peer addresses reaching the matcher are always numeric, so hostname
|
||||||
|
* globs are rejected at parse time: accepting one would create a deny rule that
|
||||||
|
* silently never matches (fail-open). */
|
||||||
|
static bool host_pattern_valid(const char* pattern) {
|
||||||
|
if (!pattern || *pattern == '\0')
|
||||||
|
return false;
|
||||||
|
if (strcmp(pattern, "*") == 0)
|
||||||
|
return true;
|
||||||
|
if (strchr(pattern, '/')) {
|
||||||
|
uint8_t bytes[16];
|
||||||
|
int prefix;
|
||||||
|
int family;
|
||||||
|
return parse_cidr(pattern, &prefix, bytes, &family);
|
||||||
|
}
|
||||||
|
struct in_addr v4;
|
||||||
|
struct in6_addr v6;
|
||||||
|
return inet_pton(AF_INET, pattern, &v4) == 1 || inet_pton(AF_INET6, pattern, &v6) == 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Append every comma- and/or whitespace-separated host pattern in `value` to
|
||||||
|
* the heap-owned list (or replace the list when `replace` is set, which --dparam
|
||||||
|
* uses so an override can narrow access rather than only widen it). Returns
|
||||||
|
* false (err filled) on an invalid pattern or an allocation failure. */
|
||||||
|
static bool store_host_list(char*** list, int* count, const char* value, const char* key,
|
||||||
|
const char* module_name, bool replace, char* err, size_t err_size) {
|
||||||
|
if (replace) {
|
||||||
|
for (int i = 0; i < *count; i++)
|
||||||
|
free((*list)[i]);
|
||||||
|
free(*list);
|
||||||
|
*list = NULL;
|
||||||
|
*count = 0;
|
||||||
|
}
|
||||||
|
char* copy = str_dup(value);
|
||||||
|
if (!copy) {
|
||||||
|
if (module_name)
|
||||||
|
set_error(err, err_size, "out of memory parsing '%s' for module '%s'", key, module_name);
|
||||||
|
else
|
||||||
|
set_error(err, err_size, "out of memory parsing '%s'", key);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
char* save = NULL;
|
||||||
|
int added = 0;
|
||||||
|
for (char* token = strtok_r(copy, ", \t", &save); token; token = strtok_r(NULL, ", \t", &save)) {
|
||||||
|
if (!host_pattern_valid(token)) {
|
||||||
|
if (module_name)
|
||||||
|
set_error(err, err_size, "module '%s': invalid host pattern '%s' in '%s'", module_name,
|
||||||
|
token, key);
|
||||||
|
else
|
||||||
|
set_error(err, err_size, "invalid host pattern '%s' in '%s'", token, key);
|
||||||
|
free(copy);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
char** grown = realloc(*list, (size_t)(*count + 1) * sizeof(char*));
|
||||||
|
if (!grown) {
|
||||||
|
if (module_name)
|
||||||
|
set_error(err, err_size, "out of memory parsing '%s' for module '%s'", key, module_name);
|
||||||
|
else
|
||||||
|
set_error(err, err_size, "out of memory parsing '%s'", key);
|
||||||
|
free(copy);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
*list = grown;
|
||||||
|
char* dup = str_dup(token);
|
||||||
|
if (!dup) {
|
||||||
|
if (module_name)
|
||||||
|
set_error(err, err_size, "out of memory parsing '%s' for module '%s'", key, module_name);
|
||||||
|
else
|
||||||
|
set_error(err, err_size, "out of memory parsing '%s'", key);
|
||||||
|
free(copy);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
(*list)[(*count)++] = dup;
|
||||||
|
added++;
|
||||||
|
}
|
||||||
|
free(copy);
|
||||||
|
/* A present key with an empty (or separator-only) value would otherwise
|
||||||
|
* install a zero-length list, i.e. no ACL at all: a strict-parse config must
|
||||||
|
* never silently turn a restrictive directive into "allow everyone". */
|
||||||
|
if (added == 0) {
|
||||||
|
if (module_name)
|
||||||
|
set_error(err, err_size, "module '%s': '%s' must list at least one host pattern", module_name,
|
||||||
|
key);
|
||||||
|
else
|
||||||
|
set_error(err, err_size, "'%s' must list at least one host pattern", key);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Parse a `max connections` value: a positive integer (0/negative/garbage are
|
||||||
|
* rejected because they would silently disable the cap or admit nothing). */
|
||||||
|
static bool store_max_connections(int* slot, const char* value, const char* module_name, char* err,
|
||||||
|
size_t err_size) {
|
||||||
|
char* end = NULL;
|
||||||
|
errno = 0;
|
||||||
|
long n = strtol(value, &end, 10);
|
||||||
|
if (*value == '\0' || errno != 0 || *end != '\0' || n <= 0 || n > INT_MAX) {
|
||||||
|
if (module_name)
|
||||||
|
set_error(err, err_size,
|
||||||
|
"module '%s': invalid 'max connections' '%s' (must be a positive "
|
||||||
|
"integer)",
|
||||||
|
module_name, value);
|
||||||
|
else
|
||||||
|
set_error(err, err_size, "invalid 'max connections' '%s' (must be a positive integer)",
|
||||||
|
value);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
*slot = (int)n;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Parse a non-negative concurrency cap where 0 means unlimited/disabled
|
||||||
|
* (per-module `max connections`, `max connections per host`,
|
||||||
|
* `auth lockout threshold`). Negative/garbage/oversized values are rejected. */
|
||||||
|
static bool store_optional_cap(int* slot, const char* value, int max_value, const char* key,
|
||||||
|
const char* module_name, char* err, size_t err_size) {
|
||||||
|
char* end = NULL;
|
||||||
|
errno = 0;
|
||||||
|
long n = strtol(value, &end, 10);
|
||||||
|
if (*value == '\0' || errno != 0 || *end != '\0' || n < 0 || n > max_value) {
|
||||||
|
if (module_name)
|
||||||
|
set_error(err, err_size, "module '%s': invalid '%s' '%s' (must be 0-%d)", module_name, key,
|
||||||
|
value, max_value);
|
||||||
|
else
|
||||||
|
set_error(err, err_size, "invalid '%s' '%s' (must be 0-%d)", key, value, max_value);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
*slot = (int)n;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Parse an `auth failure delay` value: 0 (disabled) through the configured cap. */
|
||||||
|
static bool store_auth_failure_delay(int* slot, const char* value, char* err, size_t err_size) {
|
||||||
|
char* end = NULL;
|
||||||
|
errno = 0;
|
||||||
|
long n = strtol(value, &end, 10);
|
||||||
|
if (*value == '\0' || errno != 0 || *end != '\0' || n < 0 ||
|
||||||
|
n > DAEMON_CONF_MAX_AUTH_FAILURE_DELAY_MS) {
|
||||||
|
set_error(err, err_size, "invalid 'auth failure delay' '%s' (must be 0-%d milliseconds)", value,
|
||||||
|
DAEMON_CONF_MAX_AUTH_FAILURE_DELAY_MS);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
*slot = (int)n;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
bool daemon_module_name_valid(const char* name) {
|
bool daemon_module_name_valid(const char* name) {
|
||||||
if (!name || *name == '\0')
|
if (!name || *name == '\0')
|
||||||
return false;
|
return false;
|
||||||
@@ -68,14 +258,28 @@ DaemonConf* daemon_conf_create(void) {
|
|||||||
if (!conf)
|
if (!conf)
|
||||||
return NULL;
|
return NULL;
|
||||||
conf->global.port = DAEMON_CONF_DEFAULT_PORT;
|
conf->global.port = DAEMON_CONF_DEFAULT_PORT;
|
||||||
|
conf->global.max_connections = DAEMON_CONF_DEFAULT_MAX_CONNECTIONS;
|
||||||
|
conf->global.auth_failure_delay_ms = DAEMON_CONF_DEFAULT_AUTH_FAILURE_DELAY_MS;
|
||||||
|
conf->global.max_connections_per_host = DAEMON_CONF_DEFAULT_MAX_CONNECTIONS_PER_HOST;
|
||||||
|
conf->global.auth_lockout_threshold = DAEMON_CONF_DEFAULT_AUTH_LOCKOUT_THRESHOLD;
|
||||||
|
conf->global.auth_lockout_duration_sec = DAEMON_CONF_DEFAULT_AUTH_LOCKOUT_DURATION_SEC;
|
||||||
return conf;
|
return conf;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Free a heap-owned pattern list of `count` entries. */
|
||||||
|
static void free_string_list(char** list, int count) {
|
||||||
|
for (int i = 0; i < count; i++)
|
||||||
|
free(list[i]);
|
||||||
|
free(list);
|
||||||
|
}
|
||||||
|
|
||||||
void daemon_conf_free(DaemonConf* conf) {
|
void daemon_conf_free(DaemonConf* conf) {
|
||||||
if (!conf)
|
if (!conf)
|
||||||
return;
|
return;
|
||||||
free(conf->global.motd_file);
|
free(conf->global.motd_file);
|
||||||
free(conf->global.address);
|
free(conf->global.address);
|
||||||
|
free_string_list(conf->global.hosts_allow, conf->global.hosts_allow_count);
|
||||||
|
free_string_list(conf->global.hosts_deny, conf->global.hosts_deny_count);
|
||||||
for (int i = 0; i < conf->module_count; i++) {
|
for (int i = 0; i < conf->module_count; i++) {
|
||||||
DaemonModule* m = &conf->modules[i];
|
DaemonModule* m = &conf->modules[i];
|
||||||
free(m->name);
|
free(m->name);
|
||||||
@@ -83,6 +287,8 @@ void daemon_conf_free(DaemonConf* conf) {
|
|||||||
for (int j = 0; j < m->auth_user_count; j++)
|
for (int j = 0; j < m->auth_user_count; j++)
|
||||||
free(m->auth_users[j]);
|
free(m->auth_users[j]);
|
||||||
free(m->auth_users);
|
free(m->auth_users);
|
||||||
|
free_string_list(m->hosts_allow, m->hosts_allow_count);
|
||||||
|
free_string_list(m->hosts_deny, m->hosts_deny_count);
|
||||||
}
|
}
|
||||||
free(conf->modules);
|
free(conf->modules);
|
||||||
free(conf);
|
free(conf);
|
||||||
@@ -122,8 +328,8 @@ static bool store_port(int* slot, const char* value, char* err, size_t err_size)
|
|||||||
|
|
||||||
/* Apply a global scalar key/value. Keys are case-insensitive. Returns false
|
/* Apply a global scalar key/value. Keys are case-insensitive. Returns false
|
||||||
* (err filled) on an unknown key or an invalid value. */
|
* (err filled) on an unknown key or an invalid value. */
|
||||||
static bool apply_global_key(DaemonConf* conf, char* key, const char* value, char* err,
|
static bool apply_global_key(DaemonConf* conf, char* key, const char* value, bool replace_hosts,
|
||||||
size_t err_size) {
|
char* err, size_t err_size) {
|
||||||
if (key_equals(key, "port"))
|
if (key_equals(key, "port"))
|
||||||
return store_port(&conf->global.port, value, err, err_size);
|
return store_port(&conf->global.port, value, err, err_size);
|
||||||
if (key_equals(key, "motd file")) {
|
if (key_equals(key, "motd file")) {
|
||||||
@@ -140,6 +346,28 @@ static bool apply_global_key(DaemonConf* conf, char* key, const char* value, cha
|
|||||||
}
|
}
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
if (key_equals(key, "max connections"))
|
||||||
|
return store_max_connections(&conf->global.max_connections, value, NULL, err, err_size);
|
||||||
|
if (key_equals(key, "max connections per host"))
|
||||||
|
return store_optional_cap(&conf->global.max_connections_per_host, value,
|
||||||
|
DAEMON_CONF_MAX_CONCURRENCY_LIMIT, "max connections per host", NULL,
|
||||||
|
err, err_size);
|
||||||
|
if (key_equals(key, "auth failure delay"))
|
||||||
|
return store_auth_failure_delay(&conf->global.auth_failure_delay_ms, value, err, err_size);
|
||||||
|
if (key_equals(key, "auth lockout threshold"))
|
||||||
|
return store_optional_cap(&conf->global.auth_lockout_threshold, value,
|
||||||
|
DAEMON_CONF_MAX_CONCURRENCY_LIMIT, "auth lockout threshold", NULL,
|
||||||
|
err, err_size);
|
||||||
|
if (key_equals(key, "auth lockout duration"))
|
||||||
|
return store_optional_cap(&conf->global.auth_lockout_duration_sec, value,
|
||||||
|
DAEMON_CONF_MAX_AUTH_LOCKOUT_DURATION_SEC, "auth lockout duration",
|
||||||
|
NULL, err, err_size);
|
||||||
|
if (key_equals(key, "hosts allow"))
|
||||||
|
return store_host_list(&conf->global.hosts_allow, &conf->global.hosts_allow_count, value,
|
||||||
|
"hosts allow", NULL, replace_hosts, err, err_size);
|
||||||
|
if (key_equals(key, "hosts deny"))
|
||||||
|
return store_host_list(&conf->global.hosts_deny, &conf->global.hosts_deny_count, value,
|
||||||
|
"hosts deny", NULL, replace_hosts, err, err_size);
|
||||||
set_error(err, err_size, "unknown global key '%s'", key);
|
set_error(err, err_size, "unknown global key '%s'", key);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
@@ -188,10 +416,17 @@ static bool apply_module_key(DaemonModule* module, char* key, char* value, char*
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
char* save = NULL;
|
char* save = NULL;
|
||||||
|
int added = 0;
|
||||||
for (char* token = strtok_r(list, ",", &save); token; token = strtok_r(NULL, ",", &save)) {
|
for (char* token = strtok_r(list, ",", &save); token; token = strtok_r(NULL, ",", &save)) {
|
||||||
const char* user = trim_ws(token);
|
const char* user = trim_ws(token);
|
||||||
if (*user == '\0')
|
if (*user == '\0')
|
||||||
continue;
|
continue;
|
||||||
|
if (!credentials_username_valid(user)) {
|
||||||
|
set_error(err, err_size, "module '%s': invalid 'auth users' entry '%s'", module->name,
|
||||||
|
user);
|
||||||
|
free(list);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
char** grown =
|
char** grown =
|
||||||
realloc(module->auth_users, (size_t)(module->auth_user_count + 1) * sizeof(char*));
|
realloc(module->auth_users, (size_t)(module->auth_user_count + 1) * sizeof(char*));
|
||||||
if (!grown) {
|
if (!grown) {
|
||||||
@@ -209,10 +444,27 @@ static bool apply_module_key(DaemonModule* module, char* key, char* value, char*
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
module->auth_users[module->auth_user_count++] = dup;
|
module->auth_users[module->auth_user_count++] = dup;
|
||||||
|
added++;
|
||||||
}
|
}
|
||||||
free(list);
|
free(list);
|
||||||
|
/* An empty/separator-only value must not silently disable authentication:
|
||||||
|
* the key's presence is an explicit request for an allow-list. */
|
||||||
|
if (added == 0) {
|
||||||
|
set_error(err, err_size, "module '%s': 'auth users' must list at least one user",
|
||||||
|
module->name);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
if (key_equals(key, "max connections"))
|
||||||
|
return store_optional_cap(&module->max_connections, value, DAEMON_CONF_MAX_CONCURRENCY_LIMIT,
|
||||||
|
"max connections", module->name, err, err_size);
|
||||||
|
if (key_equals(key, "hosts allow"))
|
||||||
|
return store_host_list(&module->hosts_allow, &module->hosts_allow_count, value, "hosts allow",
|
||||||
|
module->name, false, err, err_size);
|
||||||
|
if (key_equals(key, "hosts deny"))
|
||||||
|
return store_host_list(&module->hosts_deny, &module->hosts_deny_count, value, "hosts deny",
|
||||||
|
module->name, false, err, err_size);
|
||||||
set_error(err, err_size, "unknown key '%s' in module '%s'", key, module->name);
|
set_error(err, err_size, "unknown key '%s' in module '%s'", key, module->name);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
@@ -250,6 +502,11 @@ static int open_module(DaemonConf* conf, int* current_module, const char* name,
|
|||||||
set_error(err, err_size, "duplicate module '%s'", name);
|
set_error(err, err_size, "duplicate module '%s'", name);
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
if (conf->module_count >= DAEMON_CONF_MAX_MODULES) {
|
||||||
|
set_error(err, err_size, "too many modules (limit %d); module '%s' rejected",
|
||||||
|
DAEMON_CONF_MAX_MODULES, name);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
DaemonModule* grown =
|
DaemonModule* grown =
|
||||||
realloc(conf->modules, (size_t)(conf->module_count + 1) * sizeof(DaemonModule));
|
realloc(conf->modules, (size_t)(conf->module_count + 1) * sizeof(DaemonModule));
|
||||||
if (!grown) {
|
if (!grown) {
|
||||||
@@ -393,7 +650,7 @@ DaemonConf* daemon_conf_load(const char* path, char* err, size_t err_size) {
|
|||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
if (!apply_global_key(conf, key, value, err, err_size)) {
|
if (!apply_global_key(conf, key, value, false, err, err_size)) {
|
||||||
ok = false;
|
ok = false;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
@@ -448,7 +705,90 @@ int daemon_conf_apply_dparam(DaemonConf* conf, const char* assignment, char* err
|
|||||||
set_error(err, err_size, "--dparam '%s' has an empty value", assignment);
|
set_error(err, err_size, "--dparam '%s' has an empty value", assignment);
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
bool ok = apply_global_key(conf, key, value, err, err_size);
|
bool ok = apply_global_key(conf, key, value, true, err, err_size);
|
||||||
free(copy);
|
free(copy);
|
||||||
return ok ? 0 : -1;
|
return ok ? 0 : -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Compare the first `prefix` bits of two 16-byte address buffers. */
|
||||||
|
static bool bit_prefix_match(const uint8_t* a, const uint8_t* b, int prefix) {
|
||||||
|
int whole = prefix / 8;
|
||||||
|
if (whole > 0 && memcmp(a, b, (size_t)whole) != 0)
|
||||||
|
return false;
|
||||||
|
int remainder = prefix % 8;
|
||||||
|
if (remainder == 0)
|
||||||
|
return true;
|
||||||
|
uint8_t mask = (uint8_t)(0xffu << (8 - remainder));
|
||||||
|
return (a[whole] & mask) == (b[whole] & mask);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Case-insensitive glob match used for hostname patterns. Falls back to the
|
||||||
|
* shared case-sensitive matcher when an operand is too long for the stack
|
||||||
|
* buffers. */
|
||||||
|
static bool host_glob_match(const char* pattern, const char* str) {
|
||||||
|
char pbuf[256];
|
||||||
|
char sbuf[256];
|
||||||
|
size_t plen = strlen(pattern);
|
||||||
|
size_t slen = strlen(str);
|
||||||
|
if (plen >= sizeof(pbuf) || slen >= sizeof(sbuf))
|
||||||
|
return glob_match(pattern, str);
|
||||||
|
for (size_t i = 0; i <= plen; i++)
|
||||||
|
pbuf[i] = (char)tolower((unsigned char)pattern[i]);
|
||||||
|
for (size_t i = 0; i <= slen; i++)
|
||||||
|
sbuf[i] = (char)tolower((unsigned char)str[i]);
|
||||||
|
return glob_match(pbuf, sbuf);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool daemon_host_pattern_match(const char* pattern, const char* peer_ip) {
|
||||||
|
if (!pattern || *pattern == '\0' || !peer_ip || *peer_ip == '\0')
|
||||||
|
return false;
|
||||||
|
if (strcmp(pattern, "*") == 0)
|
||||||
|
return true;
|
||||||
|
if (strchr(pattern, '/')) {
|
||||||
|
uint8_t pattern_bytes[16];
|
||||||
|
uint8_t peer_bytes[16];
|
||||||
|
int prefix = 0;
|
||||||
|
int family = AF_UNSPEC;
|
||||||
|
if (!parse_cidr(pattern, &prefix, pattern_bytes, &family))
|
||||||
|
return false;
|
||||||
|
if (inet_pton(family, peer_ip, peer_bytes) != 1)
|
||||||
|
return false;
|
||||||
|
return bit_prefix_match(pattern_bytes, peer_bytes, prefix);
|
||||||
|
}
|
||||||
|
struct in_addr pattern_v4;
|
||||||
|
struct in_addr peer_v4;
|
||||||
|
if (inet_pton(AF_INET, pattern, &pattern_v4) == 1)
|
||||||
|
return inet_pton(AF_INET, peer_ip, &peer_v4) == 1 && pattern_v4.s_addr == peer_v4.s_addr;
|
||||||
|
struct in6_addr pattern_v6;
|
||||||
|
struct in6_addr peer_v6;
|
||||||
|
if (inet_pton(AF_INET6, pattern, &pattern_v6) == 1)
|
||||||
|
return inet_pton(AF_INET6, peer_ip, &peer_v6) == 1 &&
|
||||||
|
memcmp(&pattern_v6, &peer_v6, sizeof(pattern_v6)) == 0;
|
||||||
|
/* Not a literal: a hostname/glob pattern. */
|
||||||
|
return host_glob_match(pattern, peer_ip);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool daemon_hosts_allowed(const char* peer_ip, char* const* allow, int allow_count,
|
||||||
|
char* const* deny, int deny_count) {
|
||||||
|
if (!peer_ip)
|
||||||
|
return false;
|
||||||
|
for (int i = 0; i < deny_count; i++) {
|
||||||
|
if (daemon_host_pattern_match(deny[i], peer_ip))
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (allow_count > 0) {
|
||||||
|
for (int i = 0; i < allow_count; i++) {
|
||||||
|
if (daemon_host_pattern_match(allow[i], peer_ip))
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool daemon_hosts_restricted(char* const* allow, int allow_count, char* const* deny,
|
||||||
|
int deny_count) {
|
||||||
|
(void)allow;
|
||||||
|
(void)deny;
|
||||||
|
return allow_count > 0 || deny_count > 0;
|
||||||
|
}
|
||||||
|
|||||||
@@ -52,14 +52,42 @@ typedef struct DaemonModule {
|
|||||||
activities. Without it the daemon refuses all of them. */
|
activities. Without it the daemon refuses all of them. */
|
||||||
char** auth_users; /* `auth users = a,b`; Wave B credential list */
|
char** auth_users; /* `auth users = a,b`; Wave B credential list */
|
||||||
int auth_user_count;
|
int auth_user_count;
|
||||||
|
/* `max connections = N` (optional per-module cap). 0 means unlimited. The
|
||||||
|
* per-connection child records the selected module in the shared registry
|
||||||
|
* (daemon_limits.c) once the config frame names it, so the cap is enforced
|
||||||
|
* across all forked children; the parent reclaims the slot on SIGCHLD. */
|
||||||
|
int max_connections;
|
||||||
|
char** hosts_allow; /* `hosts allow = a,b`; host access allow patterns */
|
||||||
|
int hosts_allow_count;
|
||||||
|
char** hosts_deny; /* `hosts deny = a,b`; host access deny patterns */
|
||||||
|
int hosts_deny_count;
|
||||||
} DaemonModule;
|
} DaemonModule;
|
||||||
|
|
||||||
/* Global (pre-module) scalar keys. `motd file` is parsed and stored but has
|
/* Global (pre-module) scalar keys. `motd file` is parsed and stored but has
|
||||||
* no wire effect yet (MOTD display is Wave C). */
|
* no wire effect yet (MOTD display is Wave C). */
|
||||||
typedef struct DaemonConfGlobals {
|
typedef struct DaemonConfGlobals {
|
||||||
int port; /* `port`, default DAEMON_CONF_DEFAULT_PORT (873) */
|
int port; /* `port`, default DAEMON_CONF_DEFAULT_PORT (873) */
|
||||||
char* motd_file; /* `motd file`, may be NULL */
|
char* motd_file; /* `motd file`, may be NULL */
|
||||||
char* address; /* `address` (optional bind address), may be NULL */
|
char* address; /* `address` (optional bind address), may be NULL */
|
||||||
|
int max_connections; /* `max connections`, default
|
||||||
|
DAEMON_CONF_DEFAULT_MAX_CONNECTIONS (100) */
|
||||||
|
int auth_failure_delay_ms; /* `auth failure delay`, milliseconds; default
|
||||||
|
DAEMON_CONF_DEFAULT_AUTH_FAILURE_DELAY_MS */
|
||||||
|
int max_connections_per_host; /* `max connections per host`, concurrent cap per
|
||||||
|
source IP; default
|
||||||
|
DAEMON_CONF_DEFAULT_MAX_CONNECTIONS_PER_HOST (0 =
|
||||||
|
unlimited) */
|
||||||
|
int auth_lockout_threshold; /* `auth lockout threshold`, failed attempts from
|
||||||
|
one source before lockout; default
|
||||||
|
DAEMON_CONF_DEFAULT_AUTH_LOCKOUT_THRESHOLD (0
|
||||||
|
disables) */
|
||||||
|
int auth_lockout_duration_sec; /* `auth lockout duration`, seconds; default
|
||||||
|
DAEMON_CONF_DEFAULT_AUTH_LOCKOUT_DURATION_SEC
|
||||||
|
(0 disables) */
|
||||||
|
char** hosts_allow; /* `hosts allow`; global host access allow patterns */
|
||||||
|
int hosts_allow_count;
|
||||||
|
char** hosts_deny; /* `hosts deny`; global host access deny patterns */
|
||||||
|
int hosts_deny_count;
|
||||||
} DaemonConfGlobals;
|
} DaemonConfGlobals;
|
||||||
|
|
||||||
typedef struct DaemonConf {
|
typedef struct DaemonConf {
|
||||||
@@ -69,6 +97,31 @@ typedef struct DaemonConf {
|
|||||||
} DaemonConf;
|
} DaemonConf;
|
||||||
|
|
||||||
#define DAEMON_CONF_DEFAULT_PORT 873
|
#define DAEMON_CONF_DEFAULT_PORT 873
|
||||||
|
/* Default global connection cap when `max connections` is absent. Matches the
|
||||||
|
* historical hardcoded listener value. */
|
||||||
|
#define DAEMON_CONF_DEFAULT_MAX_CONNECTIONS 100
|
||||||
|
/* Default `auth failure delay` in milliseconds (0 disables the throttle). */
|
||||||
|
#define DAEMON_CONF_DEFAULT_AUTH_FAILURE_DELAY_MS 500
|
||||||
|
/* Default `max connections per host` (0 = unlimited). */
|
||||||
|
#define DAEMON_CONF_DEFAULT_MAX_CONNECTIONS_PER_HOST 0
|
||||||
|
/* Default cross-process auth lockout: 10 failed attempts from one source lock
|
||||||
|
* it out for 300 s (0 disables either knob). */
|
||||||
|
#define DAEMON_CONF_DEFAULT_AUTH_LOCKOUT_THRESHOLD 10
|
||||||
|
#define DAEMON_CONF_DEFAULT_AUTH_LOCKOUT_DURATION_SEC 300
|
||||||
|
/* Upper bound on a `max connections per host` or `auth lockout threshold`
|
||||||
|
* value, so a typo cannot size the shared registry absurdly. */
|
||||||
|
#define DAEMON_CONF_MAX_CONCURRENCY_LIMIT 1000000
|
||||||
|
/* Upper bound on `auth lockout duration` (7 days). */
|
||||||
|
#define DAEMON_CONF_MAX_AUTH_LOCKOUT_DURATION_SEC 604800
|
||||||
|
/* Largest accepted `auth failure delay`, so a typo cannot pin a connection
|
||||||
|
* child in nanosleep for an absurd time. */
|
||||||
|
/* Bounded well below the socket I/O timeout so a failed-auth child cannot hold
|
||||||
|
* a connection slot for long enough to amplify connection-cap exhaustion. */
|
||||||
|
#define DAEMON_CONF_MAX_AUTH_FAILURE_DELAY_MS 5000
|
||||||
|
/* Upper bound on the number of [module] sections, so the shared registry's
|
||||||
|
* per-module counter array stays fixed-size. The parser rejects the next
|
||||||
|
* section past this bound. */
|
||||||
|
#define DAEMON_CONF_MAX_MODULES 256
|
||||||
/* Longest accepted config line (excluding the trailing newline). Longer lines
|
/* Longest accepted config line (excluding the trailing newline). Longer lines
|
||||||
* are rejected rather than buffered unboundedly. */
|
* are rejected rather than buffered unboundedly. */
|
||||||
#define DAEMON_CONF_MAX_LINE 4096
|
#define DAEMON_CONF_MAX_LINE 4096
|
||||||
@@ -99,9 +152,31 @@ const DaemonModule* daemon_conf_find_module(const DaemonConf* conf, const char*
|
|||||||
bool daemon_module_name_valid(const char* name);
|
bool daemon_module_name_valid(const char* name);
|
||||||
|
|
||||||
/* Parse one --dparam=KEY=VALUE (or "--dparam KEY=VALUE") override string and
|
/* Parse one --dparam=KEY=VALUE (or "--dparam KEY=VALUE") override string and
|
||||||
* apply it to the global scalars only. Keys are case-insensitive and limited
|
* apply it to the global keys only. Keys are case-insensitive and limited to
|
||||||
* to the global scalar keys defined by the grammar (port, motd file, address).
|
* the global keys defined by the grammar (port, motd file, address,
|
||||||
|
* max connections, max connections per host, auth failure delay,
|
||||||
|
* auth lockout threshold, auth lockout duration, hosts allow, hosts deny).
|
||||||
* Returns 0 on success, -1 on error (err filled). */
|
* Returns 0 on success, -1 on error (err filled). */
|
||||||
int daemon_conf_apply_dparam(DaemonConf* conf, const char* assignment, char* err, size_t err_size);
|
int daemon_conf_apply_dparam(DaemonConf* conf, const char* assignment, char* err, size_t err_size);
|
||||||
|
|
||||||
|
/* Host access-control matching (pure; no I/O). `daemon_host_pattern_match`
|
||||||
|
* matches one configured pattern against a numeric peer IP string. Supported
|
||||||
|
* patterns: `*` (match anything), an IPv4/IPv6 literal, an IPv4/IPv6 CIDR
|
||||||
|
* (`10.0.0.0/8`, `2001:db8::/32`), or a glob (`*.example.com`) evaluated with
|
||||||
|
* the same matcher as file globs; a glob only matches a peer string of the
|
||||||
|
* same shape, so a numeric peer never matches a hostname glob. */
|
||||||
|
bool daemon_host_pattern_match(const char* pattern, const char* peer_ip);
|
||||||
|
|
||||||
|
/* rsync-like combined decision over a deny list and an allow list: a matching
|
||||||
|
* deny rejects (deny takes precedence); otherwise, when any allow entries
|
||||||
|
* exist, a peer that matches none is rejected; with no allow entries every
|
||||||
|
* peer not denied is accepted. An empty/unset pair returns true. */
|
||||||
|
bool daemon_hosts_allowed(const char* peer_ip, char* const* allow, int allow_count,
|
||||||
|
char* const* deny, int deny_count);
|
||||||
|
|
||||||
|
/* True when at least one allow or deny pattern is configured (i.e. an
|
||||||
|
* unprovable peer must fail closed rather than being treated as unrestricted). */
|
||||||
|
bool daemon_hosts_restricted(char* const* allow, int allow_count, char* const* deny,
|
||||||
|
int deny_count);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
@@ -0,0 +1,494 @@
|
|||||||
|
#include "daemon_limits.h"
|
||||||
|
#include "daemon_conf.h"
|
||||||
|
#include "log.h"
|
||||||
|
#include <arpa/inet.h>
|
||||||
|
#include <netinet/in.h>
|
||||||
|
#include <stdatomic.h>
|
||||||
|
#include <stdint.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
|
#include <sys/mman.h>
|
||||||
|
#include <time.h>
|
||||||
|
|
||||||
|
/* The two module-count bounds must agree: the daemon config parser never
|
||||||
|
* produces more than DAEMON_CONF_MAX_MODULES modules, so the shared registry's
|
||||||
|
* per-module counter array is sized from the same bound. */
|
||||||
|
_Static_assert(DAEMON_LIMITS_MAX_MODULES == DAEMON_CONF_MAX_MODULES,
|
||||||
|
"daemon_limits module bound must match daemon_conf");
|
||||||
|
|
||||||
|
/* Slot lifecycle states (stored in slot_state). */
|
||||||
|
enum {
|
||||||
|
SLOT_FREE = 0,
|
||||||
|
SLOT_CLAIMED = 1,
|
||||||
|
SLOT_REGISTERED = 2,
|
||||||
|
};
|
||||||
|
|
||||||
|
/* The registry header lives at the base of the shared mapping; the pointer
|
||||||
|
* fields point at the arrays carved out of the same mapping. Absolute pointers
|
||||||
|
* remain valid in a forked child because fork() clones the address space and
|
||||||
|
* mapping, so parent and child observe the same virtual addresses. */
|
||||||
|
struct DaemonLimitRegistry {
|
||||||
|
int max_slots;
|
||||||
|
int module_count;
|
||||||
|
int host_slots; /* power of two; 1 when no per-source tracking is needed */
|
||||||
|
int per_host_cap;
|
||||||
|
int lockout_threshold;
|
||||||
|
int lockout_duration_sec;
|
||||||
|
size_t map_size;
|
||||||
|
_Atomic long long host_full_warn; /* last "table full" warning epoch */
|
||||||
|
_Atomic int* slot_state;
|
||||||
|
_Atomic int* slot_pid;
|
||||||
|
_Atomic int* slot_module;
|
||||||
|
_Atomic int* slot_host; /* per-source table bucket, or -1 */
|
||||||
|
_Atomic int* module_active;
|
||||||
|
_Atomic uint64_t* host_key; /* 0 == empty bucket */
|
||||||
|
_Atomic int* host_active;
|
||||||
|
_Atomic int* host_fail;
|
||||||
|
_Atomic long long* host_until; /* epoch seconds the lockout expires */
|
||||||
|
_Atomic long long* host_last_use; /* epoch seconds the bucket was last touched */
|
||||||
|
};
|
||||||
|
|
||||||
|
static size_t round_up(size_t n, size_t align) {
|
||||||
|
return (n + align - 1) & ~(align - 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
static size_t next_pow2(size_t n) {
|
||||||
|
size_t p = 1;
|
||||||
|
while (p < n)
|
||||||
|
p <<= 1;
|
||||||
|
return p;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Parse a numeric IPv4/IPv6 peer string into family + raw bytes. */
|
||||||
|
static bool parse_peer_ip(const char* peer_ip, int* family, unsigned char* bytes) {
|
||||||
|
if (!peer_ip || *peer_ip == '\0')
|
||||||
|
return false;
|
||||||
|
struct in_addr v4;
|
||||||
|
if (inet_pton(AF_INET, peer_ip, &v4) == 1) {
|
||||||
|
memcpy(bytes, &v4, sizeof(v4));
|
||||||
|
*family = AF_INET;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
struct in6_addr v6;
|
||||||
|
if (inet_pton(AF_INET6, peer_ip, &v6) == 1) {
|
||||||
|
memcpy(bytes, &v6, sizeof(v6));
|
||||||
|
*family = AF_INET6;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
uint64_t daemon_limits_host_hash(const char* peer_ip, bool* ok) {
|
||||||
|
if (ok)
|
||||||
|
*ok = false;
|
||||||
|
unsigned char bytes[16];
|
||||||
|
int family = AF_UNSPEC;
|
||||||
|
if (!parse_peer_ip(peer_ip, &family, bytes))
|
||||||
|
return 0;
|
||||||
|
uint64_t hash = 14695981039346656037ULL ^ (uint64_t)(uint32_t)family;
|
||||||
|
size_t length = family == AF_INET ? 4 : 16;
|
||||||
|
for (size_t i = 0; i < length; i++) {
|
||||||
|
hash ^= bytes[i];
|
||||||
|
hash *= 1099511628211ULL;
|
||||||
|
}
|
||||||
|
if (hash == 0)
|
||||||
|
hash = 0x9e3779b97f4a7c15ULL;
|
||||||
|
if (ok)
|
||||||
|
*ok = true;
|
||||||
|
return hash;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* True when the registry must maintain per-source buckets: either the per-host
|
||||||
|
* cap is configured, or the auth lockout is (threshold AND duration > 0). A
|
||||||
|
* lockout threshold without a duration is a no-op, so it must not size or intern
|
||||||
|
* the table. create(), register() and the lockout paths all agree on this. */
|
||||||
|
static bool registry_tracks_hosts(const DaemonLimitRegistry* registry) {
|
||||||
|
return registry->per_host_cap > 0 ||
|
||||||
|
(registry->lockout_threshold > 0 && registry->lockout_duration_sec > 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Find the bucket holding `peer_ip`, or -1 when it has no entry. Finding a
|
||||||
|
* bucket refreshes its last-use time so the eviction policy sees it as live. */
|
||||||
|
static int host_lookup(DaemonLimitRegistry* registry, const char* peer_ip) {
|
||||||
|
bool ok = false;
|
||||||
|
uint64_t key = daemon_limits_host_hash(peer_ip, &ok);
|
||||||
|
if (!ok)
|
||||||
|
return -1;
|
||||||
|
size_t mask = (size_t)registry->host_slots - 1;
|
||||||
|
size_t start = (size_t)(key & mask);
|
||||||
|
for (size_t i = 0; i < (size_t)registry->host_slots; i++) {
|
||||||
|
size_t idx = (start + i) & mask;
|
||||||
|
uint64_t current = atomic_load_explicit(®istry->host_key[idx], memory_order_acquire);
|
||||||
|
if (current == key) {
|
||||||
|
atomic_store_explicit(®istry->host_last_use[idx], (long long)time(NULL),
|
||||||
|
memory_order_relaxed);
|
||||||
|
return (int)idx;
|
||||||
|
}
|
||||||
|
if (current == 0)
|
||||||
|
return -1; /* no tombstones: an empty bucket ends the probe chain */
|
||||||
|
}
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* A bucket with no live connection may be repurposed: immediately when its
|
||||||
|
* lockout deadline has already passed (the review's "expired" case), or after an
|
||||||
|
* idle window when it holds no pending lockout. A bucket with a future lockout
|
||||||
|
* deadline is retained so the lockout actually lasts its configured duration. */
|
||||||
|
static bool host_bucket_reclaimable(DaemonLimitRegistry* registry, size_t idx, long long now) {
|
||||||
|
if (atomic_load_explicit(®istry->host_active[idx], memory_order_relaxed) != 0)
|
||||||
|
return false;
|
||||||
|
long long until = atomic_load_explicit(®istry->host_until[idx], memory_order_relaxed);
|
||||||
|
if (until != 0)
|
||||||
|
return until <= now;
|
||||||
|
long long last_use = atomic_load_explicit(®istry->host_last_use[idx], memory_order_relaxed);
|
||||||
|
/* A bucket whose key is published but whose last_use has not yet been stamped
|
||||||
|
* (last_use == 0) must be treated as live: reclaiming it here would steal a
|
||||||
|
* bucket a racing child just claimed. The claim path also stamps last_use
|
||||||
|
* before publishing the key, so this window cannot persist. */
|
||||||
|
return last_use != 0 && now - last_use >= DAEMON_LIMITS_HOST_EVICT_IDLE_SEC;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Emit at most one "per-source table full" warning per
|
||||||
|
* DAEMON_LIMITS_HOST_FULL_WARN_SEC across all forked children. Called from a
|
||||||
|
* normal (non-signal) child path, so logging is safe here. */
|
||||||
|
static void host_warn_table_full(DaemonLimitRegistry* registry, long long now) {
|
||||||
|
long long last = atomic_load_explicit(®istry->host_full_warn, memory_order_relaxed);
|
||||||
|
if (last != 0 && now - last < DAEMON_LIMITS_HOST_FULL_WARN_SEC)
|
||||||
|
return;
|
||||||
|
if (atomic_compare_exchange_strong_explicit(®istry->host_full_warn, &last, now,
|
||||||
|
memory_order_relaxed, memory_order_relaxed)) {
|
||||||
|
log_message(LOG_LEVEL_WARNING,
|
||||||
|
"daemon: per-source registry is full (%d slots) and no bucket can be reclaimed; "
|
||||||
|
"'max connections per host' and the auth lockout are temporarily not enforced for "
|
||||||
|
"new sources (the per-module cap and host ACLs still apply)",
|
||||||
|
registry->host_slots);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Find or insert the bucket for `peer_ip`. Insertion is a lock-free CAS so two
|
||||||
|
* forked children racing on the same source converge on one bucket.
|
||||||
|
*
|
||||||
|
* When the probe finds no empty bucket it reclaims, via a key CAS, the first
|
||||||
|
* bucket that is reclaimable (expired lockout or idle, and no active
|
||||||
|
* connection) and resets its counters. This bounds the table's lifetime so it
|
||||||
|
* cannot fill permanently and stay fail-open. Returns -1 only when the address
|
||||||
|
* is unparseable or the table is genuinely full of live/locked buckets
|
||||||
|
* (callers fail open: the global/module caps and ACLs still apply). */
|
||||||
|
static int host_intern(DaemonLimitRegistry* registry, const char* peer_ip) {
|
||||||
|
bool ok = false;
|
||||||
|
uint64_t key = daemon_limits_host_hash(peer_ip, &ok);
|
||||||
|
if (!ok)
|
||||||
|
return -1;
|
||||||
|
long long now = (long long)time(NULL);
|
||||||
|
size_t mask = (size_t)registry->host_slots - 1;
|
||||||
|
size_t start = (size_t)(key & mask);
|
||||||
|
/* A couple of passes bound the work: the first normally claims/seeds a bucket;
|
||||||
|
* a lost eviction CAS retries once against the freshly observed table. */
|
||||||
|
for (int pass = 0; pass < 2; pass++) {
|
||||||
|
int evict = -1;
|
||||||
|
uint64_t evict_key = 0;
|
||||||
|
for (size_t i = 0; i < (size_t)registry->host_slots; i++) {
|
||||||
|
size_t idx = (start + i) & mask;
|
||||||
|
uint64_t current = atomic_load_explicit(®istry->host_key[idx], memory_order_acquire);
|
||||||
|
if (current == key) {
|
||||||
|
atomic_store_explicit(®istry->host_last_use[idx], now, memory_order_relaxed);
|
||||||
|
return (int)idx;
|
||||||
|
}
|
||||||
|
if (current == 0) {
|
||||||
|
/* Stamp last_use *before* publishing the key so a reclaimer racing the
|
||||||
|
* claim can never observe a claimed bucket with last_use == 0 and
|
||||||
|
* evict it. A pre-stamp is harmless if the CAS loses: the bucket is
|
||||||
|
* either still empty (never inspected for reclaim) or has just been
|
||||||
|
* taken by another source that wants a fresh timestamp anyway. */
|
||||||
|
atomic_store_explicit(®istry->host_last_use[idx], now, memory_order_relaxed);
|
||||||
|
uint64_t expected = 0;
|
||||||
|
if (atomic_compare_exchange_strong_explicit(®istry->host_key[idx], &expected, key,
|
||||||
|
memory_order_acq_rel, memory_order_acquire)) {
|
||||||
|
return (int)idx;
|
||||||
|
}
|
||||||
|
if (atomic_load_explicit(®istry->host_key[idx], memory_order_acquire) == key) {
|
||||||
|
return (int)idx;
|
||||||
|
}
|
||||||
|
continue; /* another child won this empty bucket; keep probing */
|
||||||
|
}
|
||||||
|
if (evict < 0 && host_bucket_reclaimable(registry, idx, now)) {
|
||||||
|
evict = (int)idx;
|
||||||
|
evict_key = current;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (evict >= 0) {
|
||||||
|
/* Refresh the timestamp before the key changes hands so the reused bucket
|
||||||
|
* is not seen as immediately idle by a racing reclaimer. */
|
||||||
|
atomic_store_explicit(®istry->host_last_use[evict], now, memory_order_relaxed);
|
||||||
|
uint64_t expected = evict_key;
|
||||||
|
if (atomic_compare_exchange_strong_explicit(®istry->host_key[evict], &expected, key,
|
||||||
|
memory_order_acq_rel, memory_order_acquire)) {
|
||||||
|
/* The bucket now belongs to the new source; clear the evicted source's
|
||||||
|
* stale lockout/failure state. */
|
||||||
|
atomic_store_explicit(®istry->host_active[evict], 0, memory_order_relaxed);
|
||||||
|
atomic_store_explicit(®istry->host_fail[evict], 0, memory_order_relaxed);
|
||||||
|
atomic_store_explicit(®istry->host_until[evict], 0, memory_order_relaxed);
|
||||||
|
/* Two children can race to intern the same brand-new key into different
|
||||||
|
* eviction targets, leaving the table with duplicate buckets for `key`.
|
||||||
|
* Re-scan for the first (canonical) bucket holding `key`; when it
|
||||||
|
* precedes `evict`, drop our duplicate's occupancy and hand back the
|
||||||
|
* canonical bucket so per-source counts are not orphaned on the
|
||||||
|
* duplicate. The duplicate keeps its key, so no tombstone hole is
|
||||||
|
* created and probe chains stay intact; it ages out normally. */
|
||||||
|
for (size_t i = 0; i < (size_t)registry->host_slots; i++) {
|
||||||
|
size_t candidate = (start + i) & mask;
|
||||||
|
uint64_t found =
|
||||||
|
atomic_load_explicit(®istry->host_key[candidate], memory_order_acquire);
|
||||||
|
if (found == key) {
|
||||||
|
if (candidate != (size_t)evict) {
|
||||||
|
atomic_store_explicit(®istry->host_active[evict], 0, memory_order_relaxed);
|
||||||
|
return (int)candidate;
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if (found == 0)
|
||||||
|
break; /* the key is present at `evict`, so this cannot happen first */
|
||||||
|
}
|
||||||
|
return evict;
|
||||||
|
}
|
||||||
|
continue; /* lost the race; re-probe with fresh observations */
|
||||||
|
}
|
||||||
|
break; /* no free and no reclaimable bucket: genuinely full */
|
||||||
|
}
|
||||||
|
host_warn_table_full(registry, now);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
DaemonLimitRegistry* daemon_limits_create(int max_slots, int module_count, int per_host_cap,
|
||||||
|
int lockout_threshold, int lockout_duration_sec) {
|
||||||
|
if (max_slots < DAEMON_LIMITS_MIN_SLOTS)
|
||||||
|
max_slots = DAEMON_LIMITS_MIN_SLOTS;
|
||||||
|
if (max_slots > DAEMON_LIMITS_MAX_SLOTS)
|
||||||
|
max_slots = DAEMON_LIMITS_MAX_SLOTS;
|
||||||
|
if (module_count < 1)
|
||||||
|
module_count = 1;
|
||||||
|
if (module_count > DAEMON_LIMITS_MAX_MODULES)
|
||||||
|
module_count = DAEMON_LIMITS_MAX_MODULES;
|
||||||
|
if (per_host_cap < 0)
|
||||||
|
per_host_cap = 0;
|
||||||
|
if (lockout_threshold < 0)
|
||||||
|
lockout_threshold = 0;
|
||||||
|
if (lockout_duration_sec < 0)
|
||||||
|
lockout_duration_sec = 0;
|
||||||
|
|
||||||
|
bool need_hosts = per_host_cap > 0 || (lockout_threshold > 0 && lockout_duration_sec > 0);
|
||||||
|
int host_slots = 1;
|
||||||
|
if (need_hosts) {
|
||||||
|
size_t want = (size_t)max_slots * 4;
|
||||||
|
if (want < 64)
|
||||||
|
want = 64;
|
||||||
|
if (want > DAEMON_LIMITS_MAX_HOST_SLOTS)
|
||||||
|
want = DAEMON_LIMITS_MAX_HOST_SLOTS;
|
||||||
|
host_slots = (int)next_pow2(want);
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t header = round_up(sizeof(DaemonLimitRegistry), 16);
|
||||||
|
size_t slot_bytes =
|
||||||
|
round_up((size_t)max_slots * sizeof(_Atomic int), 16) * 4; /* state,pid,module,host */
|
||||||
|
size_t module_bytes = round_up((size_t)module_count * sizeof(_Atomic int), 16);
|
||||||
|
size_t host_key_bytes = round_up((size_t)host_slots * sizeof(_Atomic uint64_t), 16);
|
||||||
|
size_t host_int_bytes = round_up((size_t)host_slots * sizeof(_Atomic int), 16) * 2;
|
||||||
|
size_t host_until_bytes = round_up((size_t)host_slots * sizeof(_Atomic long long), 16) * 2;
|
||||||
|
size_t total =
|
||||||
|
header + slot_bytes + module_bytes + host_key_bytes + host_int_bytes + host_until_bytes + 16;
|
||||||
|
|
||||||
|
void* map = mmap(NULL, total, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0);
|
||||||
|
if (map == MAP_FAILED)
|
||||||
|
return NULL;
|
||||||
|
memset(map, 0, total);
|
||||||
|
|
||||||
|
DaemonLimitRegistry* registry = (DaemonLimitRegistry*)map;
|
||||||
|
registry->max_slots = max_slots;
|
||||||
|
registry->module_count = module_count;
|
||||||
|
registry->host_slots = host_slots;
|
||||||
|
registry->per_host_cap = per_host_cap;
|
||||||
|
registry->lockout_threshold = lockout_threshold;
|
||||||
|
registry->lockout_duration_sec = lockout_duration_sec;
|
||||||
|
registry->map_size = total;
|
||||||
|
|
||||||
|
unsigned char* cursor = (unsigned char*)map + header;
|
||||||
|
registry->slot_state = (atomic_int*)cursor;
|
||||||
|
cursor += (size_t)max_slots * sizeof(_Atomic int);
|
||||||
|
registry->slot_pid = (atomic_int*)cursor;
|
||||||
|
cursor += (size_t)max_slots * sizeof(_Atomic int);
|
||||||
|
registry->slot_module = (atomic_int*)cursor;
|
||||||
|
cursor += (size_t)max_slots * sizeof(_Atomic int);
|
||||||
|
registry->slot_host = (atomic_int*)cursor;
|
||||||
|
cursor += (size_t)max_slots * sizeof(_Atomic int);
|
||||||
|
registry->module_active = (atomic_int*)cursor;
|
||||||
|
cursor += (size_t)module_count * sizeof(_Atomic int);
|
||||||
|
cursor = (unsigned char*)round_up((size_t)(uintptr_t)cursor, 16);
|
||||||
|
registry->host_key = (_Atomic uint64_t*)cursor;
|
||||||
|
cursor += (size_t)host_slots * sizeof(_Atomic uint64_t);
|
||||||
|
registry->host_active = (atomic_int*)cursor;
|
||||||
|
cursor += (size_t)host_slots * sizeof(_Atomic int);
|
||||||
|
registry->host_fail = (atomic_int*)cursor;
|
||||||
|
cursor += (size_t)host_slots * sizeof(_Atomic int);
|
||||||
|
cursor = (unsigned char*)round_up((size_t)(uintptr_t)cursor, 16);
|
||||||
|
registry->host_until = (atomic_llong*)cursor;
|
||||||
|
cursor += (size_t)host_slots * sizeof(_Atomic long long);
|
||||||
|
registry->host_last_use = (atomic_llong*)cursor;
|
||||||
|
|
||||||
|
for (int i = 0; i < max_slots; i++) {
|
||||||
|
atomic_store(®istry->slot_module[i], -1);
|
||||||
|
atomic_store(®istry->slot_host[i], -1);
|
||||||
|
}
|
||||||
|
return registry;
|
||||||
|
}
|
||||||
|
|
||||||
|
void daemon_limits_destroy(DaemonLimitRegistry* registry) {
|
||||||
|
if (!registry)
|
||||||
|
return;
|
||||||
|
munmap(registry, registry->map_size);
|
||||||
|
}
|
||||||
|
|
||||||
|
int daemon_limits_claim_slot(DaemonLimitRegistry* registry) {
|
||||||
|
if (!registry)
|
||||||
|
return DAEMON_LIMITS_NO_SLOT;
|
||||||
|
for (int i = 0; i < registry->max_slots; i++) {
|
||||||
|
int expected = SLOT_FREE;
|
||||||
|
if (atomic_compare_exchange_strong(®istry->slot_state[i], &expected, SLOT_CLAIMED)) {
|
||||||
|
atomic_store(®istry->slot_pid[i], 0);
|
||||||
|
atomic_store(®istry->slot_module[i], -1);
|
||||||
|
atomic_store(®istry->slot_host[i], -1);
|
||||||
|
return i;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return DAEMON_LIMITS_NO_SLOT;
|
||||||
|
}
|
||||||
|
|
||||||
|
void daemon_limits_set_slot_pid(DaemonLimitRegistry* registry, int slot, long pid) {
|
||||||
|
if (!registry || slot < 0 || slot >= registry->max_slots)
|
||||||
|
return;
|
||||||
|
atomic_store(®istry->slot_pid[slot], (int)pid);
|
||||||
|
}
|
||||||
|
|
||||||
|
void daemon_limits_reclaim_slot(DaemonLimitRegistry* registry, int slot) {
|
||||||
|
if (!registry || slot < 0 || slot >= registry->max_slots)
|
||||||
|
return;
|
||||||
|
atomic_exchange_explicit(®istry->slot_state[slot], SLOT_FREE, memory_order_acq_rel);
|
||||||
|
atomic_store_explicit(®istry->slot_pid[slot], 0, memory_order_relaxed);
|
||||||
|
/* The module/host occupancy arrays are derived from the slot table; do not
|
||||||
|
* decrement here or a SIGKILL between a child's increment and its REGISTERED
|
||||||
|
* publish would leak a count. Callers that need the derived counts call
|
||||||
|
* daemon_limits_recompute. */
|
||||||
|
}
|
||||||
|
|
||||||
|
void daemon_limits_reclaim_pid(DaemonLimitRegistry* registry, long pid) {
|
||||||
|
if (!registry || pid <= 0)
|
||||||
|
return;
|
||||||
|
for (int i = 0; i < registry->max_slots; i++) {
|
||||||
|
if (atomic_load(®istry->slot_state[i]) == SLOT_FREE)
|
||||||
|
continue;
|
||||||
|
if (atomic_load(®istry->slot_pid[i]) == (int)pid) {
|
||||||
|
daemon_limits_reclaim_slot(registry, i);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void daemon_limits_recompute(DaemonLimitRegistry* registry) {
|
||||||
|
if (!registry)
|
||||||
|
return;
|
||||||
|
/* Zero the derived arrays, then re-derive solely from the REGISTERED slots.
|
||||||
|
* A child that was SIGKILLed after incrementing a counter but before
|
||||||
|
* publishing REGISTERED is not counted, and its leaked increment is erased by
|
||||||
|
* the zeroing, so the leak cannot persist. */
|
||||||
|
for (int m = 0; m < registry->module_count; m++)
|
||||||
|
atomic_store_explicit(®istry->module_active[m], 0, memory_order_relaxed);
|
||||||
|
for (int h = 0; h < registry->host_slots; h++)
|
||||||
|
atomic_store_explicit(®istry->host_active[h], 0, memory_order_relaxed);
|
||||||
|
for (int i = 0; i < registry->max_slots; i++) {
|
||||||
|
if (atomic_load_explicit(®istry->slot_state[i], memory_order_acquire) != SLOT_REGISTERED)
|
||||||
|
continue;
|
||||||
|
int module = atomic_load_explicit(®istry->slot_module[i], memory_order_relaxed);
|
||||||
|
if (module >= 0 && module < registry->module_count)
|
||||||
|
atomic_fetch_add_explicit(®istry->module_active[module], 1, memory_order_relaxed);
|
||||||
|
int host = atomic_load_explicit(®istry->slot_host[i], memory_order_relaxed);
|
||||||
|
if (host >= 0 && host < registry->host_slots)
|
||||||
|
atomic_fetch_add_explicit(®istry->host_active[host], 1, memory_order_relaxed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
DaemonLimitResult daemon_limits_register(DaemonLimitRegistry* registry, int slot, int module_index,
|
||||||
|
const char* peer_ip, int module_cap) {
|
||||||
|
if (!registry || slot < 0 || slot >= registry->max_slots)
|
||||||
|
return DAEMON_LIMIT_UNAVAILABLE;
|
||||||
|
if (module_index < 0 || module_index >= registry->module_count)
|
||||||
|
return DAEMON_LIMIT_UNAVAILABLE;
|
||||||
|
if (atomic_load_explicit(®istry->slot_state[slot], memory_order_acquire) != SLOT_CLAIMED)
|
||||||
|
return DAEMON_LIMIT_UNAVAILABLE;
|
||||||
|
|
||||||
|
int host = -1;
|
||||||
|
if (registry_tracks_hosts(registry))
|
||||||
|
host = host_intern(registry, peer_ip);
|
||||||
|
|
||||||
|
int module_count = atomic_fetch_add(®istry->module_active[module_index], 1) + 1;
|
||||||
|
if (module_cap > 0 && module_count > module_cap) {
|
||||||
|
atomic_fetch_sub(®istry->module_active[module_index], 1);
|
||||||
|
return DAEMON_LIMIT_MODULE_FULL;
|
||||||
|
}
|
||||||
|
if (host >= 0) {
|
||||||
|
int host_count = atomic_fetch_add(®istry->host_active[host], 1) + 1;
|
||||||
|
if (registry->per_host_cap > 0 && host_count > registry->per_host_cap) {
|
||||||
|
atomic_fetch_sub(®istry->host_active[host], 1);
|
||||||
|
atomic_fetch_sub(®istry->module_active[module_index], 1);
|
||||||
|
return DAEMON_LIMIT_HOST_FULL;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
atomic_store(®istry->slot_module[slot], module_index);
|
||||||
|
atomic_store(®istry->slot_host[slot], host);
|
||||||
|
atomic_store_explicit(®istry->slot_state[slot], SLOT_REGISTERED, memory_order_release);
|
||||||
|
return DAEMON_LIMIT_OK;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool daemon_limits_auth_locked(DaemonLimitRegistry* registry, const char* peer_ip,
|
||||||
|
int* seconds_remaining) {
|
||||||
|
if (!registry || registry->lockout_threshold <= 0 || registry->lockout_duration_sec <= 0)
|
||||||
|
return false;
|
||||||
|
int bucket = host_lookup(registry, peer_ip);
|
||||||
|
if (bucket < 0)
|
||||||
|
return false;
|
||||||
|
long long until = atomic_load(®istry->host_until[bucket]);
|
||||||
|
long long now = (long long)time(NULL);
|
||||||
|
if (until > now) {
|
||||||
|
if (seconds_remaining)
|
||||||
|
*seconds_remaining = (int)(until - now);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (until != 0) {
|
||||||
|
/* The previous lockout has expired: clear the stale counter so the source
|
||||||
|
* gets a fresh allowance. */
|
||||||
|
atomic_store(®istry->host_fail[bucket], 0);
|
||||||
|
atomic_store(®istry->host_until[bucket], 0);
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
void daemon_limits_auth_record_failure(DaemonLimitRegistry* registry, const char* peer_ip) {
|
||||||
|
if (!registry || registry->lockout_threshold <= 0 || registry->lockout_duration_sec <= 0)
|
||||||
|
return;
|
||||||
|
int bucket = host_intern(registry, peer_ip);
|
||||||
|
if (bucket < 0)
|
||||||
|
return;
|
||||||
|
int failures = atomic_fetch_add(®istry->host_fail[bucket], 1) + 1;
|
||||||
|
if (failures >= registry->lockout_threshold) {
|
||||||
|
long long now = (long long)time(NULL);
|
||||||
|
atomic_store(®istry->host_until[bucket], now + (long long)registry->lockout_duration_sec);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void daemon_limits_auth_record_success(DaemonLimitRegistry* registry, const char* peer_ip) {
|
||||||
|
if (!registry)
|
||||||
|
return;
|
||||||
|
int bucket = host_lookup(registry, peer_ip);
|
||||||
|
if (bucket < 0)
|
||||||
|
return;
|
||||||
|
atomic_store(®istry->host_fail[bucket], 0);
|
||||||
|
atomic_store(®istry->host_until[bucket], 0);
|
||||||
|
}
|
||||||
@@ -0,0 +1,147 @@
|
|||||||
|
#ifndef DAEMON_LIMITS_H
|
||||||
|
#define DAEMON_LIMITS_H
|
||||||
|
|
||||||
|
#include <stdbool.h>
|
||||||
|
#include <stddef.h>
|
||||||
|
#include <stdint.h>
|
||||||
|
|
||||||
|
/* Cross-process daemon connection registry.
|
||||||
|
*
|
||||||
|
* The daemon listener forks ONE child per accepted connection, so any
|
||||||
|
* per-module / per-source accounting must live in state shared across the
|
||||||
|
* forked children. This module owns a fixed-size registry carved out of an
|
||||||
|
* anonymous shared mapping (mmap(MAP_SHARED | MAP_ANONYMOUS)) created by the
|
||||||
|
* accept-loop PARENT before it forks; every child inherits the mapping (and the
|
||||||
|
* pointer to it) across fork().
|
||||||
|
*
|
||||||
|
* Rules:
|
||||||
|
* - ONLY C11 atomics (atomic_*); never mtx_t/pthread locks, which can deadlock
|
||||||
|
* in a forked child if another thread held them at fork time.
|
||||||
|
* - No heap allocation after fork: the mapping is fixed-size and all access is
|
||||||
|
* atomic load/store/CAS over preallocated arrays.
|
||||||
|
*
|
||||||
|
* Slot lifecycle (the parent reclaims even when a child is SIGKILLed):
|
||||||
|
* FREE --(parent claim_slot)--> CLAIMED
|
||||||
|
* CLAIMED --(child register)--> REGISTERED
|
||||||
|
* any --(parent reclaim)--> FREE
|
||||||
|
* The child records its module index and per-source bucket into the slot before
|
||||||
|
* publishing REGISTERED; the parent's SIGCHLD handler matches the reaped pid to
|
||||||
|
* the slot and, when REGISTERED, decrements the module/per-source counters.
|
||||||
|
* A child killed before registering holds no counts, so reclaiming a CLAIMED
|
||||||
|
* slot only frees the slot.
|
||||||
|
*
|
||||||
|
* Per-source identity is the normalized numeric peer IP (IPv4-mapped IPv6 is
|
||||||
|
* already collapsed to IPv4 by utils_fd_peer_ip); it is interned into an
|
||||||
|
* open-addressed, linear-probing table keyed by a 64-bit hash. The same table
|
||||||
|
* also carries the cross-process auth-failure counter and lockout deadline.
|
||||||
|
*
|
||||||
|
* Per-source table lifetime: a bucket's key is never cleared back to empty (that
|
||||||
|
* would break every later probe chain that passed through it). Instead the
|
||||||
|
* table has a bounded-lifetime eviction policy: when no empty bucket exists, the
|
||||||
|
* first bucket that is reclaimable -- no active connection AND (its lockout
|
||||||
|
* deadline has passed OR it has been idle for
|
||||||
|
* DAEMON_LIMITS_HOST_EVICT_IDLE_SEC) -- is atomically repurposed for the new
|
||||||
|
* source via a CAS of its key, and its counters are reset. The table therefore
|
||||||
|
* cannot fill permanently, and a full table degrades to fail-open for the
|
||||||
|
* per-source cap/lockout of new sources (the per-module cap and host ACLs still
|
||||||
|
* apply) instead of staying fail-open forever. A rate-limited warning is logged
|
||||||
|
* on the fail-open path. The eviction race with a concurrent
|
||||||
|
* registration/reclaim on the same bucket is benign: it can at worst lose one
|
||||||
|
* source's counter (fail-open), never corrupt memory or the module caps.
|
||||||
|
*/
|
||||||
|
|
||||||
|
typedef struct DaemonLimitRegistry DaemonLimitRegistry;
|
||||||
|
|
||||||
|
/* Result of a per-connection admission check. */
|
||||||
|
typedef enum {
|
||||||
|
DAEMON_LIMIT_OK = 0, /* admitted; slot is now REGISTERED */
|
||||||
|
DAEMON_LIMIT_MODULE_FULL, /* module's `max connections` cap reached */
|
||||||
|
DAEMON_LIMIT_HOST_FULL, /* global `max connections per host` cap reached */
|
||||||
|
DAEMON_LIMIT_UNAVAILABLE, /* registry/slot unusable (caller fails open) */
|
||||||
|
} DaemonLimitResult;
|
||||||
|
|
||||||
|
/* Bounds for registry sizing. A slot is one concurrently live child. */
|
||||||
|
#define DAEMON_LIMITS_MIN_SLOTS 16
|
||||||
|
#define DAEMON_LIMITS_MAX_SLOTS 65536
|
||||||
|
#define DAEMON_LIMITS_MAX_HOST_SLOTS 65536
|
||||||
|
#define DAEMON_LIMITS_NO_SLOT (-1)
|
||||||
|
/* Upper bound on `module_count`, matching daemon_conf.h's DAEMON_CONF_MAX_MODULES
|
||||||
|
* (asserted in daemon_limits.c) so a caller can never size the per-module counter
|
||||||
|
* array larger than the config parser can produce. */
|
||||||
|
#define DAEMON_LIMITS_MAX_MODULES 256
|
||||||
|
|
||||||
|
/* Per-source table lifetime: a bucket with no active connection and no pending
|
||||||
|
* lockout is reclaimable once it has been idle this long, so a flood of distinct
|
||||||
|
* sources cannot pin the table full forever. A bucket whose lockout deadline
|
||||||
|
* has passed is reclaimable immediately (independent of this idle window). */
|
||||||
|
#define DAEMON_LIMITS_HOST_EVICT_IDLE_SEC 300
|
||||||
|
/* Minimum spacing between "per-source table is full" warnings, so a table-full
|
||||||
|
* attack cannot flood the log. */
|
||||||
|
#define DAEMON_LIMITS_HOST_FULL_WARN_SEC 60
|
||||||
|
|
||||||
|
/* Create the shared registry in the calling (parent) process. `max_slots` is
|
||||||
|
* the number of concurrently live children to track (clamped to
|
||||||
|
* [DAEMON_LIMITS_MIN_SLOTS, DAEMON_LIMITS_MAX_SLOTS]); `module_count` is the
|
||||||
|
* number of daemon modules (clamped to
|
||||||
|
* [1, DAEMON_LIMITS_MAX_MODULES]); `per_host_cap` and the lockout pair come
|
||||||
|
* from the daemon config (0 disables). Returns NULL on failure (e.g. mmap
|
||||||
|
* allocation); callers must degrade gracefully (global cap + ACLs still
|
||||||
|
* apply). */
|
||||||
|
DaemonLimitRegistry* daemon_limits_create(int max_slots, int module_count, int per_host_cap,
|
||||||
|
int lockout_threshold, int lockout_duration_sec);
|
||||||
|
|
||||||
|
/* Unmap the registry. Only the creating process may call this. */
|
||||||
|
void daemon_limits_destroy(DaemonLimitRegistry* registry);
|
||||||
|
|
||||||
|
/* Parent side: reserve a slot for the next fork. Returns the slot index or
|
||||||
|
* DAEMON_LIMITS_NO_SLOT when every slot is in use. */
|
||||||
|
int daemon_limits_claim_slot(DaemonLimitRegistry* registry);
|
||||||
|
/* Parent side: record the forked child's pid in a claimed slot. */
|
||||||
|
void daemon_limits_set_slot_pid(DaemonLimitRegistry* registry, int slot, long pid);
|
||||||
|
/* Parent side: release a slot. The slot becomes FREE; the module/per-source
|
||||||
|
* occupancy arrays are DERIVED state and are only refreshed by
|
||||||
|
* daemon_limits_recompute, which callers must invoke afterwards when they rely
|
||||||
|
* on the derived counts (the SIGCHLD handler batches one recompute for the whole
|
||||||
|
* reap). Idempotent. */
|
||||||
|
void daemon_limits_reclaim_slot(DaemonLimitRegistry* registry, int slot);
|
||||||
|
/* Parent SIGCHLD side: release the slot owned by `pid` (no-op when not found).
|
||||||
|
* Like reclaim_slot this does not touch the derived occupancy arrays; call
|
||||||
|
* daemon_limits_recompute after a batch of releases. */
|
||||||
|
void daemon_limits_reclaim_pid(DaemonLimitRegistry* registry, long pid);
|
||||||
|
|
||||||
|
/* Parent side (async-signal-safe; atomics only, no malloc/log): rebuild
|
||||||
|
* module_active[] / host_active[] from scratch by scanning the REGISTERED slots.
|
||||||
|
* The slot table is the single source of truth, so this self-heals any
|
||||||
|
* count leaked by a child that was SIGKILLed mid-registration (it zeroes the
|
||||||
|
* arrays and re-derives them). Bounded by max_slots + host_slots. A
|
||||||
|
* registration racing this call can be transiently undercounted until the next
|
||||||
|
* recompute, which can only relax a cap briefly -- never corrupt memory. */
|
||||||
|
void daemon_limits_recompute(DaemonLimitRegistry* registry);
|
||||||
|
|
||||||
|
/* Child side: admit the connection for `module_index` from `peer_ip`. Always
|
||||||
|
* tracks the module/per-source occupancy (so the parent's reclaim is
|
||||||
|
* symmetric); when `module_cap` > 0 it additionally enforces the per-module
|
||||||
|
* cap. A NULL/empty or non-numeric `peer_ip` skips the per-source track (the
|
||||||
|
* callers use that to exempt a trusted loopback peer from the per-host cap; the
|
||||||
|
* per-module cap still applies). Returns DAEMON_LIMIT_OK and publishes the
|
||||||
|
* slot, or a refusal reason. */
|
||||||
|
DaemonLimitResult daemon_limits_register(DaemonLimitRegistry* registry, int slot, int module_index,
|
||||||
|
const char* peer_ip, int module_cap);
|
||||||
|
|
||||||
|
/* Child side: true when `peer_ip` is currently locked out after too many failed
|
||||||
|
* authentications. `seconds_remaining` may be NULL. */
|
||||||
|
bool daemon_limits_auth_locked(DaemonLimitRegistry* registry, const char* peer_ip,
|
||||||
|
int* seconds_remaining);
|
||||||
|
/* Child side: count one failed authentication for `peer_ip`; once the threshold
|
||||||
|
* is reached the source is locked out for the configured duration. */
|
||||||
|
void daemon_limits_auth_record_failure(DaemonLimitRegistry* registry, const char* peer_ip);
|
||||||
|
/* Child side: clear the failure counter/lockout for a source that authenticated
|
||||||
|
* successfully (no-op when the source has no table entry). */
|
||||||
|
void daemon_limits_auth_record_success(DaemonLimitRegistry* registry, const char* peer_ip);
|
||||||
|
|
||||||
|
/* Pure helper: 64-bit FNV-1a hash of a numeric peer IP plus its family, used to
|
||||||
|
* index the per-source table. *ok is set false (and 0 returned) for a NULL or
|
||||||
|
* non-numeric address. Exposed for unit testing. */
|
||||||
|
uint64_t daemon_limits_host_hash(const char* peer_ip, bool* ok);
|
||||||
|
|
||||||
|
#endif
|
||||||
+8
-2
@@ -23,6 +23,7 @@ Data* data_create_reserve(size_t size) {
|
|||||||
d->data = NULL;
|
d->data = NULL;
|
||||||
d->size = size;
|
d->size = size;
|
||||||
d->protocol_charge = 0;
|
d->protocol_charge = 0;
|
||||||
|
d->owner = NULL;
|
||||||
return d;
|
return d;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -36,14 +37,19 @@ Data* data_create(void* data, size_t data_size) {
|
|||||||
new_data->data = data;
|
new_data->data = data;
|
||||||
new_data->size = data_size;
|
new_data->size = data_size;
|
||||||
new_data->protocol_charge = 0;
|
new_data->protocol_charge = 0;
|
||||||
|
new_data->owner = NULL;
|
||||||
return new_data;
|
return new_data;
|
||||||
}
|
}
|
||||||
|
|
||||||
void data_destroy(Data* data) {
|
void data_destroy(Data* data) {
|
||||||
if (data == NULL)
|
if (data == NULL)
|
||||||
return;
|
return;
|
||||||
if (data->protocol_charge != 0)
|
if (data->protocol_charge != 0) {
|
||||||
protocol_release_memory(data->protocol_charge);
|
if (data->owner != NULL)
|
||||||
|
protocol_release_memory_for_session(data->owner, data->protocol_charge);
|
||||||
|
else
|
||||||
|
protocol_release_memory(data->protocol_charge);
|
||||||
|
}
|
||||||
free(data->data);
|
free(data->data);
|
||||||
free(data);
|
free(data);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,11 +3,25 @@
|
|||||||
|
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
|
|
||||||
|
/* Forward declaration for the connection budget a received Data is charged
|
||||||
|
* against; defined in protocol.h (which includes this header). */
|
||||||
|
typedef struct ProtocolSession ProtocolSession;
|
||||||
|
|
||||||
typedef struct {
|
typedef struct {
|
||||||
void* data;
|
void* data;
|
||||||
size_t size;
|
size_t size;
|
||||||
/* Non-zero only for a buffer charged to the protocol connection budget. */
|
/* Non-zero only for a buffer charged to the protocol connection budget. */
|
||||||
size_t protocol_charge;
|
size_t protocol_charge;
|
||||||
|
/* Session whose budget `protocol_charge` was reserved from. When non-NULL,
|
||||||
|
* the charge is returned to this session directly, regardless of which
|
||||||
|
* session (if any) is bound to the destroying thread. owner is not
|
||||||
|
* guaranteed to be set whenever protocol_charge is non-zero: it is NULL for
|
||||||
|
* uncharged Data and for Data that has no recorded owner, in which case any
|
||||||
|
* charge falls back to the session bound at destroy time.
|
||||||
|
*
|
||||||
|
* Lifetime contract: a Data with a non-NULL owner must not outlive that
|
||||||
|
* ProtocolSession -- data_destroy dereferences owner to return the charge. */
|
||||||
|
ProtocolSession* owner;
|
||||||
} Data;
|
} Data;
|
||||||
|
|
||||||
Data* data_create_empty(size_t data_size);
|
Data* data_create_empty(size_t data_size);
|
||||||
@@ -15,5 +29,9 @@ Data* data_create_reserve(size_t size);
|
|||||||
Data* data_create(void* data, size_t data_size);
|
Data* data_create(void* data, size_t data_size);
|
||||||
void data_destroy(Data* data);
|
void data_destroy(Data* data);
|
||||||
void protocol_release_memory(size_t charge);
|
void protocol_release_memory(size_t charge);
|
||||||
|
/* Release `charge` against `session` directly instead of the thread-local bound
|
||||||
|
* session. Used by data_destroy to honor Data.owner; `session` must outlive
|
||||||
|
* the Data whose charge is being returned. A NULL session is a no-op. */
|
||||||
|
void protocol_release_memory_for_session(ProtocolSession* session, size_t charge);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
@@ -18,8 +18,9 @@ typedef struct {
|
|||||||
/* Receiver-side --delay-updates staging registry. All successfully written
|
/* Receiver-side --delay-updates staging registry. All successfully written
|
||||||
files land under a private staging directory inside the receive root and are
|
files land under a private staging directory inside the receive root and are
|
||||||
atomically renamed into their final destination only at the very end of the
|
atomically renamed into their final destination only at the very end of the
|
||||||
transfer. A single PipelineContextReceiver has exactly one writer thread,
|
transfer. A single receiver pipeline (see src/server/receiver_pipeline.h)
|
||||||
but the registry is still mutex-protected so the same object can be safely
|
has exactly one writer thread, but the registry is still mutex-protected so
|
||||||
|
the same object can be safely
|
||||||
shared with the publish/cleanup phase that runs after the threads join. */
|
shared with the publish/cleanup phase that runs after the threads join. */
|
||||||
typedef struct DelayUpdatesContext {
|
typedef struct DelayUpdatesContext {
|
||||||
char* root_directory; /* receive root the staging dir lives under */
|
char* root_directory; /* receive root the staging dir lives under */
|
||||||
|
|||||||
+55
-46
@@ -284,28 +284,6 @@ size_t file_content_to_buffer(File* file) {
|
|||||||
|
|
||||||
/* ---- Secure filesystem primitives ---- */
|
/* ---- Secure filesystem primitives ---- */
|
||||||
|
|
||||||
static int authorized_root_fd = -1;
|
|
||||||
static char* authorized_root_path;
|
|
||||||
|
|
||||||
static bool path_is_within_root(const char* root, const char* path) {
|
|
||||||
size_t root_len = strlen(root);
|
|
||||||
return strncmp(root, path, root_len) == 0 && (path[root_len] == '\0' || path[root_len] == '/');
|
|
||||||
}
|
|
||||||
|
|
||||||
bool file_set_authorized_root(int fd, const char* canonical_path) {
|
|
||||||
char* path_copy = canonical_path ? str_dup(canonical_path) : NULL;
|
|
||||||
if (canonical_path && !path_copy) {
|
|
||||||
authorized_root_fd = -1;
|
|
||||||
free(authorized_root_path);
|
|
||||||
authorized_root_path = NULL;
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
authorized_root_fd = fd;
|
|
||||||
free(authorized_root_path);
|
|
||||||
authorized_root_path = path_copy;
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
bool file_path_exists_secure(const char* path) {
|
bool file_path_exists_secure(const char* path) {
|
||||||
if (!path)
|
if (!path)
|
||||||
return false;
|
return false;
|
||||||
@@ -362,10 +340,6 @@ void file_set_keep_dirlinks(bool enable) {
|
|||||||
file_keep_dirlinks = enable;
|
file_keep_dirlinks = enable;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool file_get_keep_dirlinks(void) {
|
|
||||||
return file_keep_dirlinks;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* --trust-sender (Phase 5) receiver process-wide policy: when set, the receiver
|
/* --trust-sender (Phase 5) receiver process-wide policy: when set, the receiver
|
||||||
* trusts the sender's file list and skips its own redundant up-front re-
|
* trusts the sender's file list and skips its own redundant up-front re-
|
||||||
* validation (empty/".." path rejection, escaping-symlink-target containment).
|
* validation (empty/".." path rejection, escaping-symlink-target containment).
|
||||||
@@ -492,7 +466,10 @@ static int open_dir_beneath_root(const char* resolved, const char* root) {
|
|||||||
rel++;
|
rel++;
|
||||||
if (*rel == '\0')
|
if (*rel == '\0')
|
||||||
return -1;
|
return -1;
|
||||||
int fd = dup(authorized_root_fd);
|
int root_fd = utils_get_authorized_root_fd();
|
||||||
|
if (root_fd < 0)
|
||||||
|
return -1;
|
||||||
|
int fd = dup(root_fd);
|
||||||
if (fd < 0)
|
if (fd < 0)
|
||||||
return -1;
|
return -1;
|
||||||
char* copy = str_dup(rel);
|
char* copy = str_dup(rel);
|
||||||
@@ -534,20 +511,21 @@ int file_open_secure_parent(const char* path, char** leaf_out, bool create_dirs)
|
|||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
int fd;
|
int fd;
|
||||||
if (authorized_root_fd >= 0) {
|
int root_fd = utils_get_authorized_root_fd();
|
||||||
if (!authorized_root_path || path[0] != '/' ||
|
const char* root_path = utils_get_authorized_root_path();
|
||||||
!path_is_within_root(authorized_root_path, path)) {
|
if (root_fd >= 0) {
|
||||||
|
if (!root_path || path[0] != '/' || !path_is_within_root(root_path, path)) {
|
||||||
free(copy);
|
free(copy);
|
||||||
free(leaf);
|
free(leaf);
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
fd = dup(authorized_root_fd);
|
fd = dup(root_fd);
|
||||||
if (fd < 0) {
|
if (fd < 0) {
|
||||||
free(copy);
|
free(copy);
|
||||||
free(leaf);
|
free(leaf);
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
size_t root_len = strlen(authorized_root_path);
|
size_t root_len = strlen(root_path);
|
||||||
char* relative = str_dup(path + root_len);
|
char* relative = str_dup(path + root_len);
|
||||||
if (!relative) {
|
if (!relative) {
|
||||||
free(copy);
|
free(copy);
|
||||||
@@ -611,15 +589,14 @@ int file_open_secure_parent(const char* path, char** leaf_out, bool create_dirs)
|
|||||||
O_NOFOLLOW walk. Only honoured when the symlink resolves to a
|
O_NOFOLLOW walk. Only honoured when the symlink resolves to a
|
||||||
directory that stays beneath the authorized root, so a malicious link
|
directory that stays beneath the authorized root, so a malicious link
|
||||||
can never redirect the write outside it. */
|
can never redirect the write outside it. */
|
||||||
if (next < 0 && file_keep_dirlinks && authorized_root_path != NULL &&
|
if (next < 0 && file_keep_dirlinks && root_path != NULL &&
|
||||||
(errno == ELOOP || errno == ENOTDIR || errno == EACCES)) {
|
(errno == ELOOP || errno == ENOTDIR || errno == EACCES)) {
|
||||||
struct stat lst;
|
struct stat lst;
|
||||||
if (fstatat(fd, component, &lst, AT_SYMLINK_NOFOLLOW) == 0 && S_ISLNK(lst.st_mode)) {
|
if (fstatat(fd, component, &lst, AT_SYMLINK_NOFOLLOW) == 0 && S_ISLNK(lst.st_mode)) {
|
||||||
char candidate[PATH_MAX];
|
char candidate[PATH_MAX];
|
||||||
char root[PATH_MAX];
|
char root[PATH_MAX];
|
||||||
if (realpath(authorized_root_path, root) &&
|
if (realpath(root_path, root) && snprintf(candidate, sizeof(candidate), "%s%s/%s", root,
|
||||||
snprintf(candidate, sizeof(candidate), "%s%s/%s", root, rel_buf, component) <
|
rel_buf, component) < (int)sizeof(candidate)) {
|
||||||
(int)sizeof(candidate)) {
|
|
||||||
char resolved[PATH_MAX];
|
char resolved[PATH_MAX];
|
||||||
if (realpath(candidate, resolved) && strcmp(resolved, root) != 0 &&
|
if (realpath(candidate, resolved) && strcmp(resolved, root) != 0 &&
|
||||||
strncmp(root, resolved, strlen(root)) == 0 &&
|
strncmp(root, resolved, strlen(root)) == 0 &&
|
||||||
@@ -694,8 +671,9 @@ bool file_ensure_directory_secure(const char* path) {
|
|||||||
return false;
|
return false;
|
||||||
/* The authorized root is already an open directory, and the filesystem root
|
/* The authorized root is already an open directory, and the filesystem root
|
||||||
is always present: there is no final component left to create for them. */
|
is always present: there is no final component left to create for them. */
|
||||||
|
const char* root_path = utils_get_authorized_root_path();
|
||||||
bool root_is_open =
|
bool root_is_open =
|
||||||
authorized_root_fd >= 0 && authorized_root_path && strcmp(norm, authorized_root_path) == 0;
|
utils_get_authorized_root_fd() >= 0 && root_path && strcmp(norm, root_path) == 0;
|
||||||
if (root_is_open || strcmp(norm, "/") == 0) {
|
if (root_is_open || strcmp(norm, "/") == 0) {
|
||||||
free(norm);
|
free(norm);
|
||||||
return true;
|
return true;
|
||||||
@@ -742,8 +720,9 @@ bool file_directory_exists_secure(const char* path) {
|
|||||||
char* norm = normalize_directory_path(path);
|
char* norm = normalize_directory_path(path);
|
||||||
if (!norm)
|
if (!norm)
|
||||||
return false;
|
return false;
|
||||||
|
const char* root_path = utils_get_authorized_root_path();
|
||||||
bool root_is_open =
|
bool root_is_open =
|
||||||
authorized_root_fd >= 0 && authorized_root_path && strcmp(norm, authorized_root_path) == 0;
|
utils_get_authorized_root_fd() >= 0 && root_path && strcmp(norm, root_path) == 0;
|
||||||
if (root_is_open || strcmp(norm, "/") == 0) {
|
if (root_is_open || strcmp(norm, "/") == 0) {
|
||||||
free(norm);
|
free(norm);
|
||||||
return true;
|
return true;
|
||||||
@@ -911,13 +890,35 @@ static bool file_to_disk_secure_impl(const char* path, const void* data,
|
|||||||
if (inplace) {
|
if (inplace) {
|
||||||
/* --inplace writes directly into the destination; a scratch --temp-dir
|
/* --inplace writes directly into the destination; a scratch --temp-dir
|
||||||
does not apply and must never redirect these writes. */
|
does not apply and must never redirect these writes. */
|
||||||
fd = openat(dirfd, leaf, O_WRONLY | O_CREAT | O_CLOEXEC | O_NOFOLLOW, 0644);
|
/* Type gate BEFORE opening: an existing destination entry that is not a
|
||||||
|
regular file (FIFO, socket, char/block device, directory) must never be
|
||||||
|
opened for writing. Opening a FIFO would block the receive thread
|
||||||
|
forever and writing into a device would bypass the --write-devices /
|
||||||
|
super-mode gate (a client-controlled device write). fstatat with
|
||||||
|
AT_SYMLINK_NOFOLLOW does not follow a symlink and does not block. */
|
||||||
|
struct stat pre_stat;
|
||||||
|
if (fstatat(dirfd, leaf, &pre_stat, AT_SYMLINK_NOFOLLOW) == 0 && !S_ISREG(pre_stat.st_mode)) {
|
||||||
|
close(dirfd);
|
||||||
|
free(leaf);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
/* O_NONBLOCK: a no-op for a regular file, but a raced-in FIFO cannot block
|
||||||
|
the open before the post-open S_ISREG re-check rejects it. */
|
||||||
|
fd = openat(dirfd, leaf, O_WRONLY | O_CREAT | O_CLOEXEC | O_NOFOLLOW | O_NONBLOCK, 0644);
|
||||||
if (fd >= 0) {
|
if (fd >= 0) {
|
||||||
struct stat destination_stat;
|
struct stat destination_stat;
|
||||||
|
/* Re-check the opened descriptor: a concurrent replacement between the
|
||||||
|
fstatat probe and the open (or a device/FIFO raced in) must never be
|
||||||
|
written through. */
|
||||||
|
if (fstat(fd, &destination_stat) != 0 || !S_ISREG(destination_stat.st_mode)) {
|
||||||
|
close(fd);
|
||||||
|
close(dirfd);
|
||||||
|
free(leaf);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
bool newer = false;
|
bool newer = false;
|
||||||
if (update && metadata && fstat(fd, &destination_stat) == 0 &&
|
if (update && metadata && stat_is_newer(&destination_stat, metadata)) {
|
||||||
S_ISREG(destination_stat.st_mode)) {
|
newer = true;
|
||||||
newer = stat_is_newer(&destination_stat, metadata);
|
|
||||||
}
|
}
|
||||||
if (newer) {
|
if (newer) {
|
||||||
ok = true;
|
ok = true;
|
||||||
@@ -1247,11 +1248,19 @@ static bool file_to_disk_secure_link_impl(const char* path, const char* basis_pa
|
|||||||
if (linked) {
|
if (linked) {
|
||||||
int target_dirfd = scratch_dirfd >= 0 ? scratch_dirfd : dirfd;
|
int target_dirfd = scratch_dirfd >= 0 ? scratch_dirfd : dirfd;
|
||||||
if (use_fsync) {
|
if (use_fsync) {
|
||||||
int tfd = openat(target_dirfd, tmp, O_RDONLY | O_NOFOLLOW | O_CLOEXEC);
|
/* O_NONBLOCK: the freshly linked temp is normally the basis's regular
|
||||||
if (tfd < 0 || fsync(tfd) != 0) {
|
file, but a raced-in FIFO at the name must not block this reopen
|
||||||
|
forever. With O_NONBLOCK such an open fails with ENXIO instead of
|
||||||
|
blocking, which is treated as a benign fsync-skip (the link itself
|
||||||
|
is still installed); any other open/fsync failure falls back to the
|
||||||
|
byte-copy path as before. */
|
||||||
|
int tfd = openat(target_dirfd, tmp, O_RDONLY | O_NOFOLLOW | O_CLOEXEC | O_NONBLOCK);
|
||||||
|
if (tfd < 0) {
|
||||||
|
if (errno != ENXIO)
|
||||||
|
linked = false;
|
||||||
|
} else if (fsync(tfd) != 0) {
|
||||||
linked = false;
|
linked = false;
|
||||||
if (tfd >= 0)
|
close(tfd);
|
||||||
close(tfd);
|
|
||||||
} else {
|
} else {
|
||||||
close(tfd);
|
close(tfd);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -52,7 +52,6 @@ bool file_symlink_at_secure(const char* path, const char* target);
|
|||||||
/* --keep-dirlinks (-K) receiver process-wide policy: allow an in-root existing
|
/* --keep-dirlinks (-K) receiver process-wide policy: allow an in-root existing
|
||||||
* symlink-to-directory to be followed as a directory. */
|
* symlink-to-directory to be followed as a directory. */
|
||||||
void file_set_keep_dirlinks(bool enable);
|
void file_set_keep_dirlinks(bool enable);
|
||||||
bool file_get_keep_dirlinks(void);
|
|
||||||
|
|
||||||
/* --trust-sender receiver process-wide policy (Phase 5). When set, the
|
/* --trust-sender receiver process-wide policy (Phase 5). When set, the
|
||||||
* receiver trusts that the sender already produced a clean file list and skips
|
* receiver trusts that the sender already produced a clean file list and skips
|
||||||
@@ -63,9 +62,6 @@ bool file_get_keep_dirlinks(void);
|
|||||||
void file_set_trust_sender(bool enable);
|
void file_set_trust_sender(bool enable);
|
||||||
bool file_get_trust_sender(void);
|
bool file_get_trust_sender(void);
|
||||||
|
|
||||||
/* A configured fd without a canonical identity deliberately rejects paths. */
|
|
||||||
bool file_set_authorized_root(int fd, const char* canonical_path);
|
|
||||||
|
|
||||||
/* Secure path/filesystem primitives (symlink-safe, O_NOFOLLOW, root-confined). */
|
/* Secure path/filesystem primitives (symlink-safe, O_NOFOLLOW, root-confined). */
|
||||||
bool file_path_exists_secure(const char* path);
|
bool file_path_exists_secure(const char* path);
|
||||||
bool file_stat_secure(const char* path, struct stat* st);
|
bool file_stat_secure(const char* path, struct stat* st);
|
||||||
|
|||||||
+75
-24
@@ -2,6 +2,7 @@
|
|||||||
#include "log.h"
|
#include "log.h"
|
||||||
#include "utils.h"
|
#include "utils.h"
|
||||||
#include <errno.h>
|
#include <errno.h>
|
||||||
|
#include <limits.h>
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
@@ -22,6 +23,8 @@ static void string_list_destroy(StringList* list) {
|
|||||||
|
|
||||||
static bool string_list_add(StringList* list, const char* text) {
|
static bool string_list_add(StringList* list, const char* text) {
|
||||||
if (list->count == list->capacity) {
|
if (list->count == list->capacity) {
|
||||||
|
if (list->capacity > INT_MAX / 2)
|
||||||
|
return false;
|
||||||
int new_cap = list->capacity > 0 ? list->capacity * 2 : 16;
|
int new_cap = list->capacity > 0 ? list->capacity * 2 : 16;
|
||||||
char** grown = realloc(list->items, (size_t)new_cap * sizeof(char*));
|
char** grown = realloc(list->items, (size_t)new_cap * sizeof(char*));
|
||||||
if (!grown)
|
if (!grown)
|
||||||
@@ -51,11 +54,18 @@ static int normalize_entry(const char* raw, size_t len, bool strip_line_endings,
|
|||||||
if (len == 0)
|
if (len == 0)
|
||||||
return 0;
|
return 0;
|
||||||
if (raw[0] == '/') {
|
if (raw[0] == '/') {
|
||||||
snprintf(err, err_size, "absolute path entries are not allowed: '%.*s'", (int)len, raw);
|
int print_len = len > (size_t)INT_MAX ? INT_MAX : (int)len;
|
||||||
|
snprintf(err, err_size, "absolute path entries are not allowed: '%.*s'", print_len, raw);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
/* Reject NUL bytes inside a token defensively. In NUL-delimited mode the
|
||||||
|
* delimiter itself is the final byte and is expected; in line mode any NUL is
|
||||||
|
* embedded garbage (strlen-based parsing would otherwise silently truncate). */
|
||||||
|
size_t scan_len = strip_line_endings ? len : len - 1;
|
||||||
|
if (memchr(raw, '\0', scan_len)) {
|
||||||
|
snprintf(err, err_size, "entry contains an embedded NUL byte");
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
/* Reject NUL bytes inside a token defensively (NUL-delimited mode splits on
|
|
||||||
* them, so this only guards against embedded garbage). */
|
|
||||||
char* dup = malloc(len + 1);
|
char* dup = malloc(len + 1);
|
||||||
if (!dup) {
|
if (!dup) {
|
||||||
snprintf(err, err_size, "memory allocation failed");
|
snprintf(err, err_size, "memory allocation failed");
|
||||||
@@ -102,8 +112,27 @@ static int normalize_entry(const char* raw, size_t len, bool strip_line_endings,
|
|||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Build the membership index over the exact entries only. `file_list_affects`
|
||||||
|
combines the exact/descendant lookups with a walk of the query's own ancestor
|
||||||
|
prefixes, so no ancestor prefix is ever materialized as a copy and the index
|
||||||
|
stays O(entry count) memory regardless of path depth. An empty entry (the
|
||||||
|
source root) sets whole_tree and short-circuits every query. */
|
||||||
|
static bool file_list_index_build(FileListSet* set, char* err, size_t err_size) {
|
||||||
|
if (!path_index_build(&set->index, (const char* const*)set->entries, (size_t)set->count)) {
|
||||||
|
snprintf(err, err_size, "memory allocation failed");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
for (int i = 0; i < set->count; i++) {
|
||||||
|
if (set->entries[i][0] == '\0') {
|
||||||
|
set->whole_tree = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
static FileListSet* string_list_to_set(StringList* raw, char* err, size_t err_size) {
|
static FileListSet* string_list_to_set(StringList* raw, char* err, size_t err_size) {
|
||||||
FileListSet* set = malloc(sizeof(FileListSet));
|
FileListSet* set = calloc(1, sizeof(FileListSet));
|
||||||
if (!set) {
|
if (!set) {
|
||||||
snprintf(err, err_size, "memory allocation failed");
|
snprintf(err, err_size, "memory allocation failed");
|
||||||
return NULL;
|
return NULL;
|
||||||
@@ -112,6 +141,10 @@ static FileListSet* string_list_to_set(StringList* raw, char* err, size_t err_si
|
|||||||
set->entries = raw->items;
|
set->entries = raw->items;
|
||||||
raw->items = NULL;
|
raw->items = NULL;
|
||||||
raw->count = 0;
|
raw->count = 0;
|
||||||
|
if (!file_list_index_build(set, err, err_size)) {
|
||||||
|
file_list_destroy(set);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
return set;
|
return set;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -133,10 +166,20 @@ FileListSet* file_list_load(const char* path, bool null_separated, char* err, si
|
|||||||
StringList raw = {0};
|
StringList raw = {0};
|
||||||
char* line = NULL;
|
char* line = NULL;
|
||||||
size_t line_cap = 0;
|
size_t line_cap = 0;
|
||||||
ssize_t n;
|
|
||||||
bool ok = true;
|
bool ok = true;
|
||||||
char delim = null_separated ? '\0' : '\n';
|
char delim = null_separated ? '\0' : '\n';
|
||||||
while (ok && (n = getdelim(&line, &line_cap, delim, fp)) != -1) {
|
while (ok) {
|
||||||
|
ssize_t n = utils_getdelim_bounded(fp, &line, &line_cap, delim, UTILS_MAX_LINE_LEN);
|
||||||
|
if (n < 0) {
|
||||||
|
if (errno == EFBIG)
|
||||||
|
snprintf(err, err_size, "entry in file list exceeds %d bytes", (int)UTILS_MAX_LINE_LEN);
|
||||||
|
else
|
||||||
|
snprintf(err, err_size, "error reading file list: %s", strerror(errno));
|
||||||
|
ok = false;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if (n == 0)
|
||||||
|
break;
|
||||||
int r = normalize_entry(line, (size_t)n, !null_separated, &raw, err, err_size);
|
int r = normalize_entry(line, (size_t)n, !null_separated, &raw, err, err_size);
|
||||||
if (r < 0) {
|
if (r < 0) {
|
||||||
ok = false;
|
ok = false;
|
||||||
@@ -158,34 +201,42 @@ FileListSet* file_list_load(const char* path, bool null_separated, char* err, si
|
|||||||
void file_list_destroy(FileListSet* set) {
|
void file_list_destroy(FileListSet* set) {
|
||||||
if (!set)
|
if (!set)
|
||||||
return;
|
return;
|
||||||
|
path_index_free(&set->index);
|
||||||
for (int i = 0; i < set->count; i++)
|
for (int i = 0; i < set->count; i++)
|
||||||
free(set->entries[i]);
|
free(set->entries[i]);
|
||||||
free(set->entries);
|
free(set->entries);
|
||||||
free(set);
|
free(set);
|
||||||
}
|
}
|
||||||
|
|
||||||
static bool path_has_prefix(const char* path, const char* prefix) {
|
|
||||||
size_t plen = strlen(prefix);
|
|
||||||
if (strncmp(path, prefix, plen) != 0)
|
|
||||||
return false;
|
|
||||||
return path[plen] == '/' || path[plen] == '\0';
|
|
||||||
}
|
|
||||||
|
|
||||||
bool file_list_affects(const FileListSet* set, const char* rel) {
|
bool file_list_affects(const FileListSet* set, const char* rel) {
|
||||||
if (!set)
|
if (!set)
|
||||||
return true;
|
return true;
|
||||||
if (!rel)
|
if (!rel)
|
||||||
return false;
|
return false;
|
||||||
for (int i = 0; i < set->count; i++) {
|
if (set->whole_tree)
|
||||||
const char* entry = set->entries[i];
|
return true; /* whole tree listed */
|
||||||
if (entry[0] == '\0')
|
/* An exact entry match means `rel` itself is listed. */
|
||||||
return true; /* whole tree listed */
|
if (path_index_contains(&set->index, rel))
|
||||||
if (strcmp(rel, entry) == 0)
|
return true;
|
||||||
return true; /* the entry itself is listed */
|
/* Otherwise `rel` is affected when a listed entry is an ancestor directory of
|
||||||
if (path_has_prefix(rel, entry))
|
it; walk rel's own directory prefixes (which preserve path-boundary
|
||||||
return true; /* rel lives under a listed directory */
|
semantics) and test each for an exact entry. No prefixes are stored. */
|
||||||
if (path_has_prefix(entry, rel))
|
size_t len = strlen(rel);
|
||||||
return true; /* rel is an ancestor directory of a listed entry */
|
while (len > 0) {
|
||||||
|
const char* slash = NULL;
|
||||||
|
for (size_t i = len; i-- > 0;) {
|
||||||
|
if (rel[i] == '/') {
|
||||||
|
slash = rel + i;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!slash)
|
||||||
|
break;
|
||||||
|
len = (size_t)(slash - rel);
|
||||||
|
if (path_index_contains_n(&set->index, rel, len))
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
return false;
|
/* Finally `rel` is affected when it is an ancestor directory of a listed
|
||||||
|
entry (binary search for the first entry at or after `rel` + '/'). */
|
||||||
|
return path_index_has_descendant(&set->index, rel);
|
||||||
}
|
}
|
||||||
|
|||||||
+10
-2
@@ -1,6 +1,7 @@
|
|||||||
#ifndef FILE_LIST_H
|
#ifndef FILE_LIST_H
|
||||||
#define FILE_LIST_H
|
#define FILE_LIST_H
|
||||||
|
|
||||||
|
#include "utils.h"
|
||||||
#include <stdbool.h>
|
#include <stdbool.h>
|
||||||
#include <stddef.h>
|
#include <stddef.h>
|
||||||
|
|
||||||
@@ -12,11 +13,18 @@
|
|||||||
* of "." means the whole tree, absolute entries and ".." traversal are
|
* of "." means the whole tree, absolute entries and ".." traversal are
|
||||||
* rejected at parse time. The set is immutable and shared read-only across
|
* rejected at parse time. The set is immutable and shared read-only across
|
||||||
* scanner worker threads.
|
* scanner worker threads.
|
||||||
*/
|
*
|
||||||
|
* Membership is answered from `index`, built once at load time over the exact
|
||||||
|
* entries only: `index.exact` matches a listed path, the sorted view detects an
|
||||||
|
* ancestor directory of a listed entry, and `rel`'s own directory prefixes are
|
||||||
|
* matched against the exact set while descending. No ancestor prefix is stored
|
||||||
|
* as a separate string, so the index is O(entry count) memory however deep the
|
||||||
|
* paths are, and each query is O(path length) comparisons. */
|
||||||
typedef struct {
|
typedef struct {
|
||||||
char** entries; /* normalized rel paths; "" means the whole tree */
|
char** entries; /* normalized rel paths; "" means the whole tree */
|
||||||
int count;
|
int count;
|
||||||
|
PathIndex index;
|
||||||
|
bool whole_tree; /* an entry of "" lists the source root */
|
||||||
} FileListSet;
|
} FileListSet;
|
||||||
|
|
||||||
/* Load and validate a --files-from file. When `null_separated` (-0/--from0)
|
/* Load and validate a --files-from file. When `null_separated` (-0/--from0)
|
||||||
|
|||||||
+648
-454
File diff suppressed because it is too large
Load Diff
@@ -7,6 +7,15 @@
|
|||||||
|
|
||||||
/* Server-side file receive/save path. */
|
/* Server-side file receive/save path. */
|
||||||
|
|
||||||
|
/* Cumulative caps for the deferred directory-time accumulator. The sender may
|
||||||
|
* legitimately split a large tree across repeated STATUS_DIR_TIMES frames, so a
|
||||||
|
* per-frame bound is not enough: the receiver must bound the TOTAL it retains
|
||||||
|
* against a hostile sender. Mirror the delete-manifest limits
|
||||||
|
* (MAX_MANIFEST_ENTRIES / MAX_MANIFEST_BYTES): the entry count bounds the
|
||||||
|
* metadata array and the byte budget bounds the concatenated path strings. */
|
||||||
|
#define MAX_DIR_TIME_ENTRIES (1024 * 1024)
|
||||||
|
#define MAX_DIR_TIME_BYTES (16ULL * 1024 * 1024)
|
||||||
|
|
||||||
File* file_receive(const Config* config, int file_descriptor);
|
File* file_receive(const Config* config, int file_descriptor);
|
||||||
File* file_receive_directory(int file_descriptor, const Config* config);
|
File* file_receive_directory(int file_descriptor, const Config* config);
|
||||||
File* file_receive_dir_time(int file_descriptor, const Config* config);
|
File* file_receive_dir_time(int file_descriptor, const Config* config);
|
||||||
@@ -15,6 +24,13 @@ File* file_receive_symlink(int file_descriptor, const Config* config);
|
|||||||
File* file_receive_special(int file_descriptor);
|
File* file_receive_special(int file_descriptor);
|
||||||
bool file_special_rdev_valid(int32_t major, int32_t minor, mode_t mode);
|
bool file_special_rdev_valid(int32_t major, int32_t minor, mode_t mode);
|
||||||
File* receive_incremental_check(int fd, const Config* config, bool* skipped);
|
File* receive_incremental_check(int fd, const Config* config, bool* skipped);
|
||||||
|
/* Extended variant used by the receiver. `would_transfer` (may be NULL) is set
|
||||||
|
* true only on the server-contacting --dry-run path when the file is not up to
|
||||||
|
* date: the receiver has already sent STATUS_DRY_RUN_TRANSFER and returns NULL
|
||||||
|
* without storing anything. On that path `*skipped` is true for an up-to-date
|
||||||
|
* (STATUS_OK) file and both flags are false for a genuine error. */
|
||||||
|
File* receive_incremental_check_ex(int fd, const Config* config, bool* skipped,
|
||||||
|
bool* would_transfer);
|
||||||
|
|
||||||
/* P7 Wave D directory-time accumulator. The receiver collects the metadata of
|
/* P7 Wave D directory-time accumulator. The receiver collects the metadata of
|
||||||
* every directory it creates/receives (STATUS_MKDIR with metadata and/or the
|
* every directory it creates/receives (STATUS_MKDIR with metadata and/or the
|
||||||
@@ -28,12 +44,20 @@ typedef struct {
|
|||||||
FileMetadata* entries; /* owned, parallel to paths */
|
FileMetadata* entries; /* owned, parallel to paths */
|
||||||
size_t count;
|
size_t count;
|
||||||
size_t capacity;
|
size_t capacity;
|
||||||
|
size_t bytes; /* cumulative strlen of every retained path */
|
||||||
} DirTimeList;
|
} DirTimeList;
|
||||||
|
|
||||||
|
/* Capture gate shared by the sender-side and receiver-side sinks: directory
|
||||||
|
* metadata is accumulated only when --times/--metadata is in effect and
|
||||||
|
* -O/--omit-dir-times does not suppress it. Kept here, next to the accumulator
|
||||||
|
* it guards, so both call sites express the same condition. */
|
||||||
|
bool dir_times_should_capture(const Config* config);
|
||||||
|
|
||||||
void dir_time_list_init(DirTimeList* list);
|
void dir_time_list_init(DirTimeList* list);
|
||||||
void dir_time_list_free(DirTimeList* list);
|
void dir_time_list_free(DirTimeList* list);
|
||||||
/* Deep-copy one directory's path + metadata into the list. Returns false on
|
/* Deep-copy one directory's path + metadata into the list. Returns false on
|
||||||
* allocation failure (the caller fails the transfer). */
|
* allocation failure OR when the cumulative entry/byte caps would be exceeded
|
||||||
|
* (the caller fails the transfer). */
|
||||||
bool dir_time_list_add(DirTimeList* list, const char* wire_path, const FileMetadata* metadata);
|
bool dir_time_list_add(DirTimeList* list, const char* wire_path, const FileMetadata* metadata);
|
||||||
/* Apply every accumulated directory's mtime (and atime when captured) beneath
|
/* Apply every accumulated directory's mtime (and atime when captured) beneath
|
||||||
* `root_directory`, confined fd-relative. Best-effort per entry: an absent
|
* `root_directory`, confined fd-relative. Best-effort per entry: an absent
|
||||||
|
|||||||
@@ -145,7 +145,7 @@ bool file_send_sendfile_with_skip(File* file, int file_descriptor, bool use_meta
|
|||||||
off_t offset = 0;
|
off_t offset = 0;
|
||||||
struct timespec deadline;
|
struct timespec deadline;
|
||||||
clock_gettime(CLOCK_MONOTONIC, &deadline);
|
clock_gettime(CLOCK_MONOTONIC, &deadline);
|
||||||
deadline.tv_sec += 60;
|
deadline.tv_sec += protocol_get_io_timeout_sec();
|
||||||
while ((unsigned long long)offset < file_size) {
|
while ((unsigned long long)offset < file_size) {
|
||||||
struct timespec now;
|
struct timespec now;
|
||||||
clock_gettime(CLOCK_MONOTONIC, &now);
|
clock_gettime(CLOCK_MONOTONIC, &now);
|
||||||
|
|||||||
@@ -1,129 +1,7 @@
|
|||||||
#include <errno.h>
|
#include <errno.h>
|
||||||
#include <fcntl.h>
|
|
||||||
#include <libgen.h>
|
|
||||||
#include <limits.h>
|
|
||||||
#include <stdio.h>
|
|
||||||
#include <stdlib.h>
|
|
||||||
#include <string.h>
|
|
||||||
#include <unistd.h>
|
#include <unistd.h>
|
||||||
|
|
||||||
#include "file_store.h"
|
#include "file_store.h"
|
||||||
#include "metadata.h"
|
|
||||||
#include "utils.h"
|
|
||||||
|
|
||||||
static int authorized_root_fd = -1;
|
|
||||||
static char* authorized_root_path;
|
|
||||||
|
|
||||||
static bool path_is_within_root(const char* root, const char* path) {
|
|
||||||
size_t root_length = strlen(root);
|
|
||||||
return strncmp(root, path, root_length) == 0 &&
|
|
||||||
(path[root_length] == '\0' || path[root_length] == '/');
|
|
||||||
}
|
|
||||||
|
|
||||||
bool file_store_set_authorized_root(int fd, const char* canonical_path) {
|
|
||||||
char* new_path = canonical_path ? str_dup(canonical_path) : NULL;
|
|
||||||
if (canonical_path && !new_path) {
|
|
||||||
authorized_root_fd = -1;
|
|
||||||
free(authorized_root_path);
|
|
||||||
authorized_root_path = NULL;
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
free(authorized_root_path);
|
|
||||||
authorized_root_path = new_path;
|
|
||||||
authorized_root_fd = fd;
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
int file_store_open_secure_parent(const char* path, char** leaf_out) {
|
|
||||||
char* copy = str_dup(path);
|
|
||||||
if (!copy)
|
|
||||||
return -1;
|
|
||||||
char* parent = dirname(copy);
|
|
||||||
const char* slash = strrchr(path, '/');
|
|
||||||
char* leaf = str_dup(slash ? slash + 1 : path);
|
|
||||||
if (!leaf) {
|
|
||||||
free(copy);
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
int fd;
|
|
||||||
if (authorized_root_fd >= 0) {
|
|
||||||
if (!authorized_root_path || path[0] != '/' ||
|
|
||||||
!path_is_within_root(authorized_root_path, path)) {
|
|
||||||
free(copy);
|
|
||||||
free(leaf);
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
fd = dup(authorized_root_fd);
|
|
||||||
if (fd < 0) {
|
|
||||||
free(copy);
|
|
||||||
free(leaf);
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
size_t root_length = strlen(authorized_root_path);
|
|
||||||
char* relative = str_dup(path + root_length);
|
|
||||||
if (!relative) {
|
|
||||||
free(copy);
|
|
||||||
free(leaf);
|
|
||||||
close(fd);
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
free(copy);
|
|
||||||
copy = relative;
|
|
||||||
parent = dirname(copy);
|
|
||||||
} else {
|
|
||||||
fd = (parent[0] == '/') ? open("/", O_RDONLY | O_DIRECTORY | O_CLOEXEC)
|
|
||||||
: open(".", O_RDONLY | O_DIRECTORY | O_CLOEXEC);
|
|
||||||
}
|
|
||||||
if (fd < 0) {
|
|
||||||
free(copy);
|
|
||||||
free(leaf);
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
char* save = NULL;
|
|
||||||
char* component = strtok_r(parent, "/", &save);
|
|
||||||
while (component) {
|
|
||||||
if (strcmp(component, "..") == 0) {
|
|
||||||
close(fd);
|
|
||||||
free(copy);
|
|
||||||
free(leaf);
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
if (strcmp(component, ".") != 0) {
|
|
||||||
int next = openat(fd, component, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
|
||||||
if (next < 0 && errno == ENOENT) {
|
|
||||||
if (mkdirat(fd, component, 0755) == 0 || errno == EEXIST)
|
|
||||||
next = openat(fd, component, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
|
||||||
}
|
|
||||||
if (next < 0) {
|
|
||||||
close(fd);
|
|
||||||
free(copy);
|
|
||||||
free(leaf);
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
close(fd);
|
|
||||||
fd = next;
|
|
||||||
}
|
|
||||||
component = strtok_r(NULL, "/", &save);
|
|
||||||
}
|
|
||||||
free(copy);
|
|
||||||
*leaf_out = leaf;
|
|
||||||
return fd;
|
|
||||||
}
|
|
||||||
|
|
||||||
bool file_store_rename_secure(const char* old_path, const char* new_path) {
|
|
||||||
char *old_leaf = NULL, *new_leaf = NULL;
|
|
||||||
int old_parent = file_store_open_secure_parent(old_path, &old_leaf);
|
|
||||||
int new_parent = file_store_open_secure_parent(new_path, &new_leaf);
|
|
||||||
bool ok = old_parent >= 0 && new_parent >= 0 &&
|
|
||||||
renameat(old_parent, old_leaf, new_parent, new_leaf) == 0;
|
|
||||||
if (old_parent >= 0)
|
|
||||||
close(old_parent);
|
|
||||||
if (new_parent >= 0)
|
|
||||||
close(new_parent);
|
|
||||||
free(old_leaf);
|
|
||||||
free(new_leaf);
|
|
||||||
return ok;
|
|
||||||
}
|
|
||||||
|
|
||||||
static bool write_all(int fd, const void* data, unsigned long long size) {
|
static bool write_all(int fd, const void* data, unsigned long long size) {
|
||||||
const unsigned char* p = data;
|
const unsigned char* p = data;
|
||||||
@@ -176,67 +54,3 @@ bool file_store_write_sparse(int fd, const unsigned char* data, unsigned long lo
|
|||||||
}
|
}
|
||||||
return ftruncate(fd, (off_t)size) == 0;
|
return ftruncate(fd, (off_t)size) == 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool file_store_write_secure(const char* path, const void* data, unsigned long long data_size,
|
|
||||||
bool inplace, bool sparse, const FileMetadata* metadata,
|
|
||||||
bool preserve_executability) {
|
|
||||||
char* leaf = NULL;
|
|
||||||
int dirfd = file_store_open_secure_parent(path, &leaf);
|
|
||||||
if (dirfd < 0)
|
|
||||||
return false;
|
|
||||||
int fd = -1;
|
|
||||||
bool ok = false;
|
|
||||||
if (inplace) {
|
|
||||||
fd = openat(dirfd, leaf, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC | O_NOFOLLOW, 0644);
|
|
||||||
if (fd >= 0) {
|
|
||||||
if (sparse && data_size > 0) {
|
|
||||||
if (ftruncate(fd, (off_t)data_size) == 0)
|
|
||||||
ok = file_store_write_sparse(fd, data, data_size);
|
|
||||||
} else {
|
|
||||||
ok = write_all(fd, data, data_size);
|
|
||||||
}
|
|
||||||
if (ok && metadata)
|
|
||||||
ok = file_restore_metadata_fd(fd, metadata, preserve_executability);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
int tmp_size = snprintf(NULL, 0, ".%s.tmp.%ld.%u", leaf, (long)getpid(), 99U);
|
|
||||||
if (tmp_size < 0) {
|
|
||||||
close(dirfd);
|
|
||||||
free(leaf);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
char* tmp = malloc((size_t)tmp_size + 1);
|
|
||||||
if (!tmp) {
|
|
||||||
close(dirfd);
|
|
||||||
free(leaf);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
for (unsigned int i = 0; i < 100 && !ok; ++i) {
|
|
||||||
snprintf(tmp, (size_t)tmp_size + 1, ".%s.tmp.%ld.%u", leaf, (long)getpid(), i);
|
|
||||||
fd = openat(dirfd, tmp, O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC | O_NOFOLLOW, 0600);
|
|
||||||
if (fd < 0)
|
|
||||||
continue;
|
|
||||||
if (sparse && data_size > 0)
|
|
||||||
ok = ftruncate(fd, (off_t)data_size) == 0;
|
|
||||||
if (ok || (!sparse || data_size == 0))
|
|
||||||
ok = (sparse && data_size > 0)
|
|
||||||
? file_store_write_sparse(fd, (const unsigned char*)data, data_size)
|
|
||||||
: write_all(fd, data, data_size);
|
|
||||||
if (ok && metadata)
|
|
||||||
ok = file_restore_metadata_fd(fd, metadata, preserve_executability);
|
|
||||||
if (close(fd) != 0)
|
|
||||||
ok = false;
|
|
||||||
fd = -1;
|
|
||||||
if (ok && renameat(dirfd, tmp, dirfd, leaf) != 0)
|
|
||||||
ok = false;
|
|
||||||
if (!ok)
|
|
||||||
unlinkat(dirfd, tmp, 0);
|
|
||||||
}
|
|
||||||
free(tmp);
|
|
||||||
}
|
|
||||||
if (fd >= 0)
|
|
||||||
close(fd);
|
|
||||||
close(dirfd);
|
|
||||||
free(leaf);
|
|
||||||
return ok;
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,15 +1,8 @@
|
|||||||
#ifndef FILE_STORE_H
|
#ifndef FILE_STORE_H
|
||||||
#define FILE_STORE_H
|
#define FILE_STORE_H
|
||||||
|
|
||||||
#include "file.h"
|
|
||||||
#include <stdbool.h>
|
#include <stdbool.h>
|
||||||
|
|
||||||
bool file_store_set_authorized_root(int fd, const char* canonical_path);
|
|
||||||
int file_store_open_secure_parent(const char* path, char** leaf_out);
|
|
||||||
bool file_store_rename_secure(const char* old_path, const char* new_path);
|
|
||||||
bool file_store_write_secure(const char* path, const void* data, unsigned long long data_size,
|
|
||||||
bool inplace, bool sparse, const FileMetadata* metadata,
|
|
||||||
bool preserve_executability);
|
|
||||||
/* Sparse-aware write (--sparse/-S): every all-zero run of at least
|
/* Sparse-aware write (--sparse/-S): every all-zero run of at least
|
||||||
* SPARSE_HOLE_MIN bytes is skipped with lseek(SEEK_CUR) so it becomes a real
|
* SPARSE_HOLE_MIN bytes is skipped with lseek(SEEK_CUR) so it becomes a real
|
||||||
* hole; every other byte is written. The caller pre-sizes the file with
|
* hole; every other byte is written. The caller pre-sizes the file with
|
||||||
|
|||||||
+20
-4
@@ -2,6 +2,7 @@
|
|||||||
#include "log.h"
|
#include "log.h"
|
||||||
#include "utils.h"
|
#include "utils.h"
|
||||||
#include <errno.h>
|
#include <errno.h>
|
||||||
|
#include <limits.h>
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
@@ -176,6 +177,8 @@ bool filter_rule_list_add(FilterRuleList* list, FilterRule* rule) {
|
|||||||
if (!list || !rule)
|
if (!list || !rule)
|
||||||
return false;
|
return false;
|
||||||
if (list->count == list->capacity) {
|
if (list->count == list->capacity) {
|
||||||
|
if (list->capacity > INT_MAX / 2)
|
||||||
|
return false;
|
||||||
int new_cap = list->capacity > 0 ? list->capacity * 2 : 8;
|
int new_cap = list->capacity > 0 ? list->capacity * 2 : 8;
|
||||||
FilterRule** grown = realloc(list->items, (size_t)new_cap * sizeof(FilterRule*));
|
FilterRule** grown = realloc(list->items, (size_t)new_cap * sizeof(FilterRule*));
|
||||||
if (!grown)
|
if (!grown)
|
||||||
@@ -322,8 +325,10 @@ FilterRuleList* filter_file_read(const char* dir_path, const char* owner_rel, bo
|
|||||||
if (!fp) {
|
if (!fp) {
|
||||||
if (errno == ENOENT || errno == ENOTDIR)
|
if (errno == ENOENT || errno == ENOTDIR)
|
||||||
return filter_rule_list_create();
|
return filter_rule_list_create();
|
||||||
log_message(LOG_LEVEL_WARNING, "Could not read .rsync-filter in %s: %s", dir_path,
|
char* escaped_dir = output_escape(dir_path, log_get_8_bit_output());
|
||||||
strerror(errno));
|
log_message(LOG_LEVEL_WARNING, "Could not read .rsync-filter in %s: %s",
|
||||||
|
escaped_dir ? escaped_dir : "<allocation failed>", strerror(errno));
|
||||||
|
free(escaped_dir);
|
||||||
return filter_rule_list_create();
|
return filter_rule_list_create();
|
||||||
}
|
}
|
||||||
if (exists)
|
if (exists)
|
||||||
@@ -336,9 +341,20 @@ FilterRuleList* filter_file_read(const char* dir_path, const char* owner_rel, bo
|
|||||||
}
|
}
|
||||||
char* line = NULL;
|
char* line = NULL;
|
||||||
size_t line_cap = 0;
|
size_t line_cap = 0;
|
||||||
ssize_t n;
|
|
||||||
bool ok = true;
|
bool ok = true;
|
||||||
while ((n = getline(&line, &line_cap, fp)) != -1) {
|
while (true) {
|
||||||
|
ssize_t n = utils_getdelim_bounded(fp, &line, &line_cap, '\n', UTILS_MAX_LINE_LEN);
|
||||||
|
if (n < 0) {
|
||||||
|
if (errno == EFBIG) {
|
||||||
|
snprintf(err, err_size, "line in .rsync-filter exceeds %d bytes", (int)UTILS_MAX_LINE_LEN);
|
||||||
|
} else {
|
||||||
|
snprintf(err, err_size, "error reading .rsync-filter: %s", strerror(errno));
|
||||||
|
}
|
||||||
|
ok = false;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if (n == 0)
|
||||||
|
break;
|
||||||
const char* p = line;
|
const char* p = line;
|
||||||
while (*p == ' ' || *p == '\t')
|
while (*p == ' ' || *p == '\t')
|
||||||
p++;
|
p++;
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ typedef struct {
|
|||||||
/* --super / --no-super tri-state (SUPER_MODE_AUTO when unset). Snapshotted
|
/* --super / --no-super tri-state (SUPER_MODE_AUTO when unset). Snapshotted
|
||||||
* per connection so privilege_super_permitted() can gate super-user
|
* per connection so privilege_super_permitted() can gate super-user
|
||||||
* activities without a Config argument. */
|
* activities without a Config argument. */
|
||||||
int super_mode;
|
SuperMode super_mode;
|
||||||
/* --copy-as=USER[:GROUP]: snapshotted so the ownership resolver can force the
|
/* --copy-as=USER[:GROUP]: snapshotted so the ownership resolver can force the
|
||||||
* target ids without a Config argument. */
|
* target ids without a Config argument. */
|
||||||
bool copy_as_set;
|
bool copy_as_set;
|
||||||
@@ -128,7 +128,7 @@ bool privilege_super_permitted(void) {
|
|||||||
return privilege_super_mode_permitted(g_identity.super_mode);
|
return privilege_super_mode_permitted(g_identity.super_mode);
|
||||||
}
|
}
|
||||||
|
|
||||||
bool privilege_super_mode_permitted(int mode) {
|
bool privilege_super_mode_permitted(SuperMode mode) {
|
||||||
/* AUTO and ON both attempt the confined operation; OFF forbids it even for a
|
/* AUTO and ON both attempt the confined operation; OFF forbids it even for a
|
||||||
* root receiver. AUTO is the historical FastSync behavior (always attempt
|
* root receiver. AUTO is the historical FastSync behavior (always attempt
|
||||||
* and let the kernel refuse an unprivileged call, which the caller skips), so
|
* and let the kernel refuse an unprivileged call, which the caller skips), so
|
||||||
|
|||||||
@@ -128,6 +128,6 @@ bool identity_wire_valid(const Config* config);
|
|||||||
* best-effort behavior where an unprivileged attempt is refused by the kernel
|
* best-effort behavior where an unprivileged attempt is refused by the kernel
|
||||||
* and skipped. Neither EVER elevates privileges. */
|
* and skipped. Neither EVER elevates privileges. */
|
||||||
bool privilege_super_permitted(void);
|
bool privilege_super_permitted(void);
|
||||||
bool privilege_super_mode_permitted(int mode);
|
bool privilege_super_mode_permitted(SuperMode mode);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
+73
-27
@@ -3,7 +3,9 @@
|
|||||||
#include <stdbool.h>
|
#include <stdbool.h>
|
||||||
#include <stdarg.h>
|
#include <stdarg.h>
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
|
#include <threads.h>
|
||||||
#include <time.h>
|
#include <time.h>
|
||||||
|
|
||||||
static const char* log_level_strings[] = {"DEBUG", "INFO", "WARN", "ERROR"};
|
static const char* log_level_strings[] = {"DEBUG", "INFO", "WARN", "ERROR"};
|
||||||
@@ -15,6 +17,18 @@ static FILE* log_fp = NULL;
|
|||||||
static _Thread_local bool eight_bit_output;
|
static _Thread_local bool eight_bit_output;
|
||||||
static LogStderrMode stderr_mode = LOG_STDERR_ERRORS;
|
static LogStderrMode stderr_mode = LOG_STDERR_ERRORS;
|
||||||
|
|
||||||
|
/* Serializes access to log_fp and makes each emitted line atomic: the
|
||||||
|
* timestamp prefix, formatted body, and trailing newline are written as one
|
||||||
|
* critical section so concurrent threads cannot interleave partial lines.
|
||||||
|
* Initialized lazily (matching the protocol.c bw_mutex idiom) because logging
|
||||||
|
* can happen before main() installs any synchronization. */
|
||||||
|
static mtx_t log_mutex;
|
||||||
|
static once_flag log_mutex_once = ONCE_FLAG_INIT;
|
||||||
|
|
||||||
|
static void log_mutex_init(void) {
|
||||||
|
mtx_init(&log_mutex, mtx_plain);
|
||||||
|
}
|
||||||
|
|
||||||
void set_log_level(LogLevel level) {
|
void set_log_level(LogLevel level) {
|
||||||
current_log_level = level;
|
current_log_level = level;
|
||||||
}
|
}
|
||||||
@@ -41,7 +55,10 @@ uint32_t get_log_info_flags(void) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
void log_set_file(FILE* fp) {
|
void log_set_file(FILE* fp) {
|
||||||
|
call_once(&log_mutex_once, log_mutex_init);
|
||||||
|
mtx_lock(&log_mutex);
|
||||||
log_fp = fp;
|
log_fp = fp;
|
||||||
|
mtx_unlock(&log_mutex);
|
||||||
}
|
}
|
||||||
|
|
||||||
void log_set_8_bit_output(bool enabled) {
|
void log_set_8_bit_output(bool enabled) {
|
||||||
@@ -60,13 +77,48 @@ LogStderrMode log_get_stderr_mode(void) {
|
|||||||
return stderr_mode;
|
return stderr_mode;
|
||||||
}
|
}
|
||||||
|
|
||||||
static inline void write_message(FILE* dest_io, LogLevel log_level, struct tm t, const char* format,
|
/* Format one complete log line (timestamp prefix + body + newline) into a
|
||||||
va_list args) {
|
* freshly allocated buffer. This is pure CPU/malloc work and must happen
|
||||||
fprintf(dest_io, "%04d-%02d-%02d %02d:%02d:%02d [%s]: ", t.tm_year + 1900, t.tm_mon + 1,
|
* OUTSIDE the log mutex: the mutex only guards the log_fp pointer, so a
|
||||||
t.tm_mday, t.tm_hour, t.tm_min, t.tm_sec, log_level_strings[log_level]);
|
* stalled stderr/stdout pipe cannot block every logging thread. Returns NULL
|
||||||
|
* on allocation/formatting failure. */
|
||||||
|
static char* format_log_line(LogLevel log_level, const struct tm* t, const char* format,
|
||||||
|
va_list args) {
|
||||||
|
char prefix[64];
|
||||||
|
int prefix_len = snprintf(
|
||||||
|
prefix, sizeof(prefix), "%04d-%02d-%02d %02d:%02d:%02d [%s]: ", t->tm_year + 1900,
|
||||||
|
t->tm_mon + 1, t->tm_mday, t->tm_hour, t->tm_min, t->tm_sec, log_level_strings[log_level]);
|
||||||
|
if (prefix_len < 0 || prefix_len >= (int)sizeof(prefix))
|
||||||
|
return NULL;
|
||||||
|
va_list copy;
|
||||||
|
va_copy(copy, args);
|
||||||
|
int body_len = vsnprintf(NULL, 0, format, copy);
|
||||||
|
va_end(copy);
|
||||||
|
if (body_len < 0)
|
||||||
|
return NULL;
|
||||||
|
size_t total = (size_t)prefix_len + (size_t)body_len;
|
||||||
|
char* line = malloc(total + 2); /* body bytes + '\n' + NUL */
|
||||||
|
if (!line)
|
||||||
|
return NULL;
|
||||||
|
memcpy(line, prefix, (size_t)prefix_len);
|
||||||
|
vsnprintf(line + prefix_len, (size_t)body_len + 1, format, args);
|
||||||
|
line[total] = '\n';
|
||||||
|
line[total + 1] = '\0';
|
||||||
|
return line;
|
||||||
|
}
|
||||||
|
|
||||||
vfprintf(dest_io, format, args);
|
/* Write an already-formatted line to the console and, if configured, the log
|
||||||
fprintf(dest_io, "\n");
|
* file. Only the log_fp pointer is read under the mutex (so log_set_file /
|
||||||
|
* config_delete cannot free it while it is in use); the single console fputs
|
||||||
|
* runs unlocked but is internally atomic per stdio stream. */
|
||||||
|
static void emit_log_line(FILE* console, const char* line) {
|
||||||
|
fputs(line, console);
|
||||||
|
call_once(&log_mutex_once, log_mutex_init);
|
||||||
|
mtx_lock(&log_mutex);
|
||||||
|
FILE* file = log_fp;
|
||||||
|
if (file)
|
||||||
|
fputs(line, file);
|
||||||
|
mtx_unlock(&log_mutex);
|
||||||
}
|
}
|
||||||
|
|
||||||
void log_message(LogLevel log_level, const char* format, ...) {
|
void log_message(LogLevel log_level, const char* format, ...) {
|
||||||
@@ -86,14 +138,12 @@ void log_message(LogLevel log_level, const char* format, ...) {
|
|||||||
|
|
||||||
va_list args;
|
va_list args;
|
||||||
va_start(args, format);
|
va_start(args, format);
|
||||||
write_message(dest_io, log_level, t, format, args);
|
char* line = format_log_line(log_level, &t, format, args);
|
||||||
va_end(args);
|
va_end(args);
|
||||||
|
if (!line)
|
||||||
if (log_fp) {
|
return;
|
||||||
va_start(args, format);
|
emit_log_line(dest_io, line);
|
||||||
write_message(log_fp, log_level, t, format, args);
|
free(line);
|
||||||
va_end(args);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void log_debug_message(LogDebugFlag flag, const char* format, ...) {
|
void log_debug_message(LogDebugFlag flag, const char* format, ...) {
|
||||||
@@ -107,14 +157,12 @@ void log_debug_message(LogDebugFlag flag, const char* format, ...) {
|
|||||||
|
|
||||||
va_list args;
|
va_list args;
|
||||||
va_start(args, format);
|
va_start(args, format);
|
||||||
write_message(stdout, LOG_LEVEL_DEBUG, t, format, args);
|
char* line = format_log_line(LOG_LEVEL_DEBUG, &t, format, args);
|
||||||
va_end(args);
|
va_end(args);
|
||||||
|
if (!line)
|
||||||
if (log_fp) {
|
return;
|
||||||
va_start(args, format);
|
emit_log_line(stdout, line);
|
||||||
write_message(log_fp, LOG_LEVEL_DEBUG, t, format, args);
|
free(line);
|
||||||
va_end(args);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void log_info_message(LogInfoFlag flag, const char* format, ...) {
|
void log_info_message(LogInfoFlag flag, const char* format, ...) {
|
||||||
@@ -129,14 +177,12 @@ void log_info_message(LogInfoFlag flag, const char* format, ...) {
|
|||||||
|
|
||||||
va_list args;
|
va_list args;
|
||||||
va_start(args, format);
|
va_start(args, format);
|
||||||
write_message(stdout, LOG_LEVEL_INFO, t, format, args);
|
char* line = format_log_line(LOG_LEVEL_INFO, &t, format, args);
|
||||||
va_end(args);
|
va_end(args);
|
||||||
|
if (!line)
|
||||||
if (log_fp) {
|
return;
|
||||||
va_start(args, format);
|
emit_log_line(stdout, line);
|
||||||
write_message(log_fp, LOG_LEVEL_INFO, t, format, args);
|
free(line);
|
||||||
va_end(args);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void log_perror(const char* context) {
|
void log_perror(const char* context) {
|
||||||
|
|||||||
+51
-153
@@ -90,63 +90,65 @@ void metadata_to_buf(char** buf, const FileMetadata* m) {
|
|||||||
*buf += sizeof(crtime_nsec);
|
*buf += sizeof(crtime_nsec);
|
||||||
}
|
}
|
||||||
|
|
||||||
FileMetadata* metadata_from_buf(char** buf) {
|
FileMetadata* metadata_from_buf(const uint8_t* buf, size_t len) {
|
||||||
|
if (buf == NULL || len < sizeof(int32_t))
|
||||||
|
return NULL;
|
||||||
int32_t present;
|
int32_t present;
|
||||||
memcpy(&present, *buf, sizeof(present));
|
memcpy(&present, buf, sizeof(present));
|
||||||
*buf += sizeof(present);
|
if (present != 1)
|
||||||
if (present != 0 && present != 1)
|
|
||||||
return NULL;
|
return NULL;
|
||||||
if (!present)
|
if (len < sizeof(int32_t) + FILE_METADATA_WIRE_SIZE)
|
||||||
return NULL;
|
return NULL;
|
||||||
|
const uint8_t* cursor = buf + sizeof(int32_t);
|
||||||
FileMetadata* m = protocol_alloc(sizeof(FileMetadata));
|
FileMetadata* m = protocol_alloc(sizeof(FileMetadata));
|
||||||
if (m == NULL)
|
if (m == NULL)
|
||||||
return NULL;
|
return NULL;
|
||||||
int32_t mode;
|
int32_t mode;
|
||||||
memcpy(&mode, *buf, sizeof(mode));
|
memcpy(&mode, cursor, sizeof(mode));
|
||||||
*buf += sizeof(mode);
|
cursor += sizeof(mode);
|
||||||
m->mode = (mode_t)mode;
|
m->mode = (mode_t)mode;
|
||||||
int32_t uid;
|
int32_t uid;
|
||||||
memcpy(&uid, *buf, sizeof(uid));
|
memcpy(&uid, cursor, sizeof(uid));
|
||||||
*buf += sizeof(uid);
|
cursor += sizeof(uid);
|
||||||
m->uid = (uid_t)uid;
|
m->uid = (uid_t)uid;
|
||||||
int32_t gid;
|
int32_t gid;
|
||||||
memcpy(&gid, *buf, sizeof(gid));
|
memcpy(&gid, cursor, sizeof(gid));
|
||||||
*buf += sizeof(gid);
|
cursor += sizeof(gid);
|
||||||
m->gid = (gid_t)gid;
|
m->gid = (gid_t)gid;
|
||||||
int64_t mtime_sec;
|
int64_t mtime_sec;
|
||||||
memcpy(&mtime_sec, *buf, sizeof(mtime_sec));
|
memcpy(&mtime_sec, cursor, sizeof(mtime_sec));
|
||||||
*buf += sizeof(mtime_sec);
|
cursor += sizeof(mtime_sec);
|
||||||
m->mtime_sec = (time_t)mtime_sec;
|
m->mtime_sec = (time_t)mtime_sec;
|
||||||
int64_t mtime_nsec;
|
int64_t mtime_nsec;
|
||||||
memcpy(&mtime_nsec, *buf, sizeof(mtime_nsec));
|
memcpy(&mtime_nsec, cursor, sizeof(mtime_nsec));
|
||||||
*buf += sizeof(mtime_nsec);
|
cursor += sizeof(mtime_nsec);
|
||||||
m->mtime_nsec = (long)mtime_nsec;
|
m->mtime_nsec = (long)mtime_nsec;
|
||||||
int32_t atime_valid;
|
int32_t atime_valid;
|
||||||
memcpy(&atime_valid, *buf, sizeof(atime_valid));
|
memcpy(&atime_valid, cursor, sizeof(atime_valid));
|
||||||
*buf += sizeof(atime_valid);
|
cursor += sizeof(atime_valid);
|
||||||
int64_t atime_sec;
|
int64_t atime_sec;
|
||||||
memcpy(&atime_sec, *buf, sizeof(atime_sec));
|
memcpy(&atime_sec, cursor, sizeof(atime_sec));
|
||||||
*buf += sizeof(atime_sec);
|
cursor += sizeof(atime_sec);
|
||||||
int64_t atime_nsec;
|
int64_t atime_nsec;
|
||||||
memcpy(&atime_nsec, *buf, sizeof(atime_nsec));
|
memcpy(&atime_nsec, cursor, sizeof(atime_nsec));
|
||||||
*buf += sizeof(atime_nsec);
|
cursor += sizeof(atime_nsec);
|
||||||
int32_t crtime_valid;
|
int32_t crtime_valid;
|
||||||
memcpy(&crtime_valid, *buf, sizeof(crtime_valid));
|
memcpy(&crtime_valid, cursor, sizeof(crtime_valid));
|
||||||
*buf += sizeof(crtime_valid);
|
cursor += sizeof(crtime_valid);
|
||||||
int64_t crtime_sec;
|
int64_t crtime_sec;
|
||||||
memcpy(&crtime_sec, *buf, sizeof(crtime_sec));
|
memcpy(&crtime_sec, cursor, sizeof(crtime_sec));
|
||||||
*buf += sizeof(crtime_sec);
|
cursor += sizeof(crtime_sec);
|
||||||
int64_t crtime_nsec;
|
int64_t crtime_nsec;
|
||||||
memcpy(&crtime_nsec, *buf, sizeof(crtime_nsec));
|
memcpy(&crtime_nsec, cursor, sizeof(crtime_nsec));
|
||||||
*buf += sizeof(crtime_nsec);
|
cursor += sizeof(crtime_nsec);
|
||||||
m->atime_valid = atime_valid != 0;
|
m->atime_valid = atime_valid != 0;
|
||||||
m->atime_sec = (time_t)atime_sec;
|
m->atime_sec = (time_t)atime_sec;
|
||||||
m->atime_nsec = (long)atime_nsec;
|
m->atime_nsec = (long)atime_nsec;
|
||||||
m->crtime_valid = crtime_valid != 0;
|
m->crtime_valid = crtime_valid != 0;
|
||||||
m->crtime_sec = (time_t)crtime_sec;
|
m->crtime_sec = (time_t)crtime_sec;
|
||||||
m->crtime_nsec = (long)crtime_nsec;
|
m->crtime_nsec = (long)crtime_nsec;
|
||||||
if (present != 1 || mtime_nsec < 0 || mtime_nsec >= 1000000000LL || mode < 0 || uid < 0 ||
|
if (mtime_nsec < 0 || mtime_nsec >= 1000000000LL || mode < 0 || uid < 0 || gid < 0 ||
|
||||||
gid < 0 || atime_valid < 0 || atime_valid > 1 || crtime_valid < 0 || crtime_valid > 1 ||
|
atime_valid < 0 || atime_valid > 1 || crtime_valid < 0 || crtime_valid > 1 ||
|
||||||
(atime_valid && (atime_nsec < 0 || atime_nsec >= 1000000000LL)) ||
|
(atime_valid && (atime_nsec < 0 || atime_nsec >= 1000000000LL)) ||
|
||||||
(crtime_valid && (crtime_nsec < 0 || crtime_nsec >= 1000000000LL))) {
|
(crtime_valid && (crtime_nsec < 0 || crtime_nsec >= 1000000000LL))) {
|
||||||
free(m);
|
free(m);
|
||||||
@@ -157,33 +159,17 @@ FileMetadata* metadata_from_buf(char** buf) {
|
|||||||
|
|
||||||
bool metadata_send(int file_descriptor, const FileMetadata* m) {
|
bool metadata_send(int file_descriptor, const FileMetadata* m) {
|
||||||
if (m == NULL) {
|
if (m == NULL) {
|
||||||
int32_t zero = 0;
|
int32_t absent = 0;
|
||||||
return send_n_data(file_descriptor, &zero, sizeof(zero));
|
return send_n_data(file_descriptor, &absent, sizeof(absent));
|
||||||
}
|
}
|
||||||
int32_t present = 1;
|
/* One packed frame (protocol 2.20.0): the int32 present flag followed by the
|
||||||
int32_t mode = (int32_t)m->mode;
|
fixed FILE_METADATA_WIRE_SIZE-byte field record. metadata_to_buf() emits
|
||||||
int32_t uid = (int32_t)m->uid;
|
exactly that layout (present + fields), so build it once and write the
|
||||||
int32_t gid = (int32_t)m->gid;
|
whole record in a single call instead of one frame per field. */
|
||||||
int64_t mtime_sec = (int64_t)m->mtime_sec;
|
char packed[sizeof(int32_t) + FILE_METADATA_WIRE_SIZE];
|
||||||
int64_t mtime_nsec = (int64_t)m->mtime_nsec;
|
char* cursor = packed;
|
||||||
int32_t atime_valid = m->atime_valid ? 1 : 0;
|
metadata_to_buf(&cursor, m);
|
||||||
int64_t atime_sec = (int64_t)m->atime_sec;
|
return send_n_data(file_descriptor, packed, sizeof(packed));
|
||||||
int64_t atime_nsec = (int64_t)m->atime_nsec;
|
|
||||||
int32_t crtime_valid = m->crtime_valid ? 1 : 0;
|
|
||||||
int64_t crtime_sec = (int64_t)m->crtime_sec;
|
|
||||||
int64_t crtime_nsec = (int64_t)m->crtime_nsec;
|
|
||||||
return send_n_data(file_descriptor, &present, sizeof(present)) &&
|
|
||||||
send_n_data(file_descriptor, &mode, sizeof(mode)) &&
|
|
||||||
send_n_data(file_descriptor, &uid, sizeof(uid)) &&
|
|
||||||
send_n_data(file_descriptor, &gid, sizeof(gid)) &&
|
|
||||||
send_n_data(file_descriptor, &mtime_sec, sizeof(mtime_sec)) &&
|
|
||||||
send_n_data(file_descriptor, &mtime_nsec, sizeof(mtime_nsec)) &&
|
|
||||||
send_n_data(file_descriptor, &atime_valid, sizeof(atime_valid)) &&
|
|
||||||
send_n_data(file_descriptor, &atime_sec, sizeof(atime_sec)) &&
|
|
||||||
send_n_data(file_descriptor, &atime_nsec, sizeof(atime_nsec)) &&
|
|
||||||
send_n_data(file_descriptor, &crtime_valid, sizeof(crtime_valid)) &&
|
|
||||||
send_n_data(file_descriptor, &crtime_sec, sizeof(crtime_sec)) &&
|
|
||||||
send_n_data(file_descriptor, &crtime_nsec, sizeof(crtime_nsec));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
FileMetadata* metadata_receive(int file_descriptor, int* ok) {
|
FileMetadata* metadata_receive(int file_descriptor, int* ok) {
|
||||||
@@ -203,109 +189,21 @@ FileMetadata* metadata_receive(int file_descriptor, int* ok) {
|
|||||||
*ok = 0;
|
*ok = 0;
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
FileMetadata* m = protocol_alloc(sizeof(FileMetadata));
|
/* Rebuild the packed record metadata_from_buf() expects: the present flag we
|
||||||
|
just read, followed by exactly FILE_METADATA_WIRE_SIZE field bytes. */
|
||||||
|
char packed[sizeof(int32_t) + FILE_METADATA_WIRE_SIZE];
|
||||||
|
memcpy(packed, &present, sizeof(present));
|
||||||
|
if (!receive_n_data(file_descriptor, packed + sizeof(present), FILE_METADATA_WIRE_SIZE)) {
|
||||||
|
if (ok)
|
||||||
|
*ok = 0;
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
FileMetadata* m = metadata_from_buf((const uint8_t*)packed, sizeof(packed));
|
||||||
if (m == NULL) {
|
if (m == NULL) {
|
||||||
if (ok)
|
if (ok)
|
||||||
*ok = 0;
|
*ok = 0;
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
int32_t mode;
|
|
||||||
if (!receive_n_data(file_descriptor, &mode, sizeof(mode))) {
|
|
||||||
free(m);
|
|
||||||
if (ok)
|
|
||||||
*ok = 0;
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
m->mode = (mode_t)mode;
|
|
||||||
int32_t uid;
|
|
||||||
if (!receive_n_data(file_descriptor, &uid, sizeof(uid))) {
|
|
||||||
free(m);
|
|
||||||
if (ok)
|
|
||||||
*ok = 0;
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
m->uid = (uid_t)uid;
|
|
||||||
int32_t gid;
|
|
||||||
if (!receive_n_data(file_descriptor, &gid, sizeof(gid))) {
|
|
||||||
free(m);
|
|
||||||
if (ok)
|
|
||||||
*ok = 0;
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
m->gid = (gid_t)gid;
|
|
||||||
int64_t mtime_sec;
|
|
||||||
if (!receive_n_data(file_descriptor, &mtime_sec, sizeof(mtime_sec))) {
|
|
||||||
free(m);
|
|
||||||
if (ok)
|
|
||||||
*ok = 0;
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
m->mtime_sec = (time_t)mtime_sec;
|
|
||||||
int64_t mtime_nsec;
|
|
||||||
if (!receive_n_data(file_descriptor, &mtime_nsec, sizeof(mtime_nsec))) {
|
|
||||||
free(m);
|
|
||||||
if (ok)
|
|
||||||
*ok = 0;
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
m->mtime_nsec = (long)mtime_nsec;
|
|
||||||
int32_t atime_valid;
|
|
||||||
if (!receive_n_data(file_descriptor, &atime_valid, sizeof(atime_valid))) {
|
|
||||||
free(m);
|
|
||||||
if (ok)
|
|
||||||
*ok = 0;
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
int64_t atime_sec;
|
|
||||||
if (!receive_n_data(file_descriptor, &atime_sec, sizeof(atime_sec))) {
|
|
||||||
free(m);
|
|
||||||
if (ok)
|
|
||||||
*ok = 0;
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
int64_t atime_nsec;
|
|
||||||
if (!receive_n_data(file_descriptor, &atime_nsec, sizeof(atime_nsec))) {
|
|
||||||
free(m);
|
|
||||||
if (ok)
|
|
||||||
*ok = 0;
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
int32_t crtime_valid;
|
|
||||||
if (!receive_n_data(file_descriptor, &crtime_valid, sizeof(crtime_valid))) {
|
|
||||||
free(m);
|
|
||||||
if (ok)
|
|
||||||
*ok = 0;
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
int64_t crtime_sec;
|
|
||||||
if (!receive_n_data(file_descriptor, &crtime_sec, sizeof(crtime_sec))) {
|
|
||||||
free(m);
|
|
||||||
if (ok)
|
|
||||||
*ok = 0;
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
int64_t crtime_nsec;
|
|
||||||
if (!receive_n_data(file_descriptor, &crtime_nsec, sizeof(crtime_nsec))) {
|
|
||||||
free(m);
|
|
||||||
if (ok)
|
|
||||||
*ok = 0;
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
m->atime_valid = atime_valid != 0;
|
|
||||||
m->atime_sec = (time_t)atime_sec;
|
|
||||||
m->atime_nsec = (long)atime_nsec;
|
|
||||||
m->crtime_valid = crtime_valid != 0;
|
|
||||||
m->crtime_sec = (time_t)crtime_sec;
|
|
||||||
m->crtime_nsec = (long)crtime_nsec;
|
|
||||||
if (mtime_nsec < 0 || mtime_nsec >= 1000000000LL || mode < 0 || uid < 0 || gid < 0 ||
|
|
||||||
atime_valid < 0 || atime_valid > 1 || crtime_valid < 0 || crtime_valid > 1 ||
|
|
||||||
(atime_valid && (atime_nsec < 0 || atime_nsec >= 1000000000LL)) ||
|
|
||||||
(crtime_valid && (crtime_nsec < 0 || crtime_nsec >= 1000000000LL))) {
|
|
||||||
free(m);
|
|
||||||
if (ok)
|
|
||||||
*ok = 0;
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
if (ok)
|
if (ok)
|
||||||
*ok = 1;
|
*ok = 1;
|
||||||
return m;
|
return m;
|
||||||
|
|||||||
+15
-2
@@ -3,6 +3,7 @@
|
|||||||
|
|
||||||
#include "file.h"
|
#include "file.h"
|
||||||
#include <stdbool.h>
|
#include <stdbool.h>
|
||||||
|
#include <stddef.h>
|
||||||
#include <stdint.h>
|
#include <stdint.h>
|
||||||
#include <sys/stat.h>
|
#include <sys/stat.h>
|
||||||
#include <time.h>
|
#include <time.h>
|
||||||
@@ -29,11 +30,23 @@
|
|||||||
|
|
||||||
/* Size of metadata fields on wire, excluding the int32_t `present` field that
|
/* Size of metadata fields on wire, excluding the int32_t `present` field that
|
||||||
* is always sent first. The total wire size for present metadata is
|
* is always sent first. The total wire size for present metadata is
|
||||||
* sizeof(int32_t) + FILE_METADATA_WIRE_SIZE (68 bytes on most platforms). */
|
* sizeof(int32_t) + FILE_METADATA_WIRE_SIZE (68 bytes on most platforms).
|
||||||
|
*
|
||||||
|
* metadata_send()/metadata_receive() (protocol 2.20.0) frame the metadata as a
|
||||||
|
* single packed record: one int32 present flag (0 = absent) followed, when
|
||||||
|
* present, by exactly FILE_METADATA_WIRE_SIZE bytes of field data. This is the
|
||||||
|
* same present+fields byte layout metadata_to_buf()/metadata_from_buf() use, so
|
||||||
|
* the wire metadata is now one frame instead of one frame per field. */
|
||||||
#define FILE_METADATA_WIRE_SIZE (sizeof(int32_t) * 5 + sizeof(int64_t) * 6)
|
#define FILE_METADATA_WIRE_SIZE (sizeof(int32_t) * 5 + sizeof(int64_t) * 6)
|
||||||
|
|
||||||
void metadata_to_buf(char** buf, const FileMetadata* m);
|
void metadata_to_buf(char** buf, const FileMetadata* m);
|
||||||
FileMetadata* metadata_from_buf(char** buf);
|
/* Decode one packed metadata record (an int32 present flag followed, when
|
||||||
|
* present, by FILE_METADATA_WIRE_SIZE field bytes) from `buf`, which has `len`
|
||||||
|
* readable bytes. Every read is bounds-checked against `len`, so the function
|
||||||
|
* can never over-read the caller's buffer: a too-short record, an absent
|
||||||
|
* (present == 0) record and a malformed record all return NULL. A successful
|
||||||
|
* decode returns a heap-allocated FileMetadata owned by the caller. */
|
||||||
|
FileMetadata* metadata_from_buf(const uint8_t* buf, size_t len);
|
||||||
bool metadata_send(int file_descriptor, const FileMetadata* m);
|
bool metadata_send(int file_descriptor, const FileMetadata* m);
|
||||||
FileMetadata* metadata_receive(int file_descriptor, int* ok);
|
FileMetadata* metadata_receive(int file_descriptor, int* ok);
|
||||||
void file_restore_metadata(const char* path, const FileMetadata* metadata,
|
void file_restore_metadata(const char* path, const FileMetadata* metadata,
|
||||||
|
|||||||
+85
-247
@@ -1,15 +1,16 @@
|
|||||||
#include "multiprocessing.h"
|
#include "multiprocessing.h"
|
||||||
#include "receiver.h"
|
|
||||||
|
|
||||||
#include "array_list.h"
|
#include "array_list.h"
|
||||||
#include "chunk.h"
|
#include "chunk.h"
|
||||||
#include "config.h"
|
#include "config.h"
|
||||||
#include "data.h"
|
#include "data.h"
|
||||||
#include "file.h"
|
#include "file.h"
|
||||||
|
#include "file_receive.h"
|
||||||
#include "log.h"
|
#include "log.h"
|
||||||
#include "protocol.h"
|
#include "protocol.h"
|
||||||
#include "queue.h"
|
#include "queue.h"
|
||||||
#include "utils.h"
|
#include "utils.h"
|
||||||
|
#include <stdint.h>
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
@@ -25,6 +26,8 @@ PipelineContextSender* pipeline_context_sender_create(Config* config, Queue* que
|
|||||||
context->queue_loader = queue_loader;
|
context->queue_loader = queue_loader;
|
||||||
context->scanner_done = false;
|
context->scanner_done = false;
|
||||||
context->loader_done = false;
|
context->loader_done = false;
|
||||||
|
context->queued_bytes = 0;
|
||||||
|
context->max_queue_bytes = 0;
|
||||||
context->manifest = NULL;
|
context->manifest = NULL;
|
||||||
context->excluded_paths = NULL;
|
context->excluded_paths = NULL;
|
||||||
context->missing_args = NULL;
|
context->missing_args = NULL;
|
||||||
@@ -96,7 +99,88 @@ fail:
|
|||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void pipeline_context_sender_set_queue_byte_limit(PipelineContextSender* context,
|
||||||
|
size_t max_bytes) {
|
||||||
|
if (context == NULL)
|
||||||
|
return;
|
||||||
|
mtx_lock(&context->mutex_loader);
|
||||||
|
context->max_queue_bytes = max_bytes;
|
||||||
|
context->queued_bytes = 0;
|
||||||
|
cnd_broadcast(&context->condition_not_full_loader);
|
||||||
|
mtx_unlock(&context->mutex_loader);
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t pipeline_context_sender_chunk_bytes(const Chunk* chunk) {
|
||||||
|
if (chunk == NULL || chunk->items == NULL)
|
||||||
|
return 0;
|
||||||
|
size_t total = 0;
|
||||||
|
for (int i = 0; i < chunk->element_count; i++) {
|
||||||
|
const File* file = chunk->items[i];
|
||||||
|
if (file == NULL || file->data == NULL || file->data->data == NULL)
|
||||||
|
continue;
|
||||||
|
if (file->data->size > SIZE_MAX - total)
|
||||||
|
return SIZE_MAX;
|
||||||
|
total += file->data->size;
|
||||||
|
}
|
||||||
|
return total;
|
||||||
|
}
|
||||||
|
|
||||||
|
void pipeline_context_sender_note_bytes_released(PipelineContextSender* context,
|
||||||
|
size_t released_bytes) {
|
||||||
|
if (context == NULL || context->max_queue_bytes == 0 || released_bytes == 0)
|
||||||
|
return;
|
||||||
|
mtx_lock(&context->mutex_loader);
|
||||||
|
if (released_bytes >= context->queued_bytes)
|
||||||
|
context->queued_bytes = 0;
|
||||||
|
else
|
||||||
|
context->queued_bytes -= released_bytes;
|
||||||
|
cnd_signal(&context->condition_not_full_loader);
|
||||||
|
mtx_unlock(&context->mutex_loader);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool pipeline_context_sender_enqueue_chunk(PipelineContextSender* context, Chunk* chunk) {
|
||||||
|
if (context == NULL || chunk == NULL)
|
||||||
|
return false;
|
||||||
|
size_t chunk_bytes = pipeline_context_sender_chunk_bytes(chunk);
|
||||||
|
mtx_lock(&context->mutex_loader);
|
||||||
|
while (!atomic_load(&context->cancelled)) {
|
||||||
|
bool blocked_by_count = queue_is_full(context->queue_loader);
|
||||||
|
bool blocked_by_budget = false;
|
||||||
|
if (context->max_queue_bytes > 0) {
|
||||||
|
size_t budget = context->max_queue_bytes;
|
||||||
|
size_t used = context->queued_bytes;
|
||||||
|
if (used >= budget) {
|
||||||
|
blocked_by_budget = true;
|
||||||
|
} else if (chunk_bytes > budget - used) {
|
||||||
|
/* A single payload larger than the whole budget is only admitted to an
|
||||||
|
empty pipeline so the wait can never deadlock. */
|
||||||
|
blocked_by_budget = used != 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!blocked_by_count && !blocked_by_budget)
|
||||||
|
break;
|
||||||
|
cnd_wait(&context->condition_not_full_loader, &context->mutex_loader);
|
||||||
|
}
|
||||||
|
if (atomic_load(&context->cancelled)) {
|
||||||
|
mtx_unlock(&context->mutex_loader);
|
||||||
|
chunk_destroy(chunk);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (!queue_enqueue(context->queue_loader, chunk)) {
|
||||||
|
mtx_unlock(&context->mutex_loader);
|
||||||
|
chunk_destroy(chunk);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
context->queued_bytes += chunk_bytes;
|
||||||
|
cnd_signal(&context->condition_not_empty_loader);
|
||||||
|
mtx_unlock(&context->mutex_loader);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
void pipeline_context_sender_destroy(PipelineContextSender* context) {
|
void pipeline_context_sender_destroy(PipelineContextSender* context) {
|
||||||
|
/* `config` is borrowed: the caller retains ownership and frees it after the
|
||||||
|
pipeline has been destroyed (the worker threads are already joined, so no
|
||||||
|
config access can outlive this call). */
|
||||||
if (context->manifest) {
|
if (context->manifest) {
|
||||||
array_list_delete(context->manifest);
|
array_list_delete(context->manifest);
|
||||||
}
|
}
|
||||||
@@ -110,7 +194,6 @@ void pipeline_context_sender_destroy(PipelineContextSender* context) {
|
|||||||
array_list_delete(context->dir_entries);
|
array_list_delete(context->dir_entries);
|
||||||
if (context->dir_entries_mutex_init)
|
if (context->dir_entries_mutex_init)
|
||||||
mtx_destroy(&context->dir_entries_mutex);
|
mtx_destroy(&context->dir_entries_mutex);
|
||||||
config_delete(context->config);
|
|
||||||
queue_destroy(context->queue_scanner);
|
queue_destroy(context->queue_scanner);
|
||||||
queue_destroy(context->queue_loader);
|
queue_destroy(context->queue_loader);
|
||||||
mtx_destroy(&context->mutex_scanner);
|
mtx_destroy(&context->mutex_scanner);
|
||||||
@@ -122,248 +205,3 @@ void pipeline_context_sender_destroy(PipelineContextSender* context) {
|
|||||||
mtx_destroy(&context->mutex_progress);
|
mtx_destroy(&context->mutex_progress);
|
||||||
free(context);
|
free(context);
|
||||||
}
|
}
|
||||||
|
|
||||||
PipelineContextReceiver* pipeline_context_receiver_create(Config* config, Queue* queue,
|
|
||||||
int file_descriptor, SSL* ssl) {
|
|
||||||
PipelineContextReceiver* context = malloc(sizeof(PipelineContextReceiver));
|
|
||||||
if (context == NULL)
|
|
||||||
return NULL;
|
|
||||||
context->config = config;
|
|
||||||
context->queue = queue;
|
|
||||||
context->file_descriptor = file_descriptor;
|
|
||||||
context->ssl = ssl;
|
|
||||||
context->outcomes.entries = NULL;
|
|
||||||
context->outcomes.count = 0;
|
|
||||||
context->outcomes.capacity = 0;
|
|
||||||
dir_time_list_init(&context->dir_times);
|
|
||||||
protocol_session_init(&context->session, file_descriptor, file_descriptor);
|
|
||||||
protocol_session_set_ssl(&context->session, ssl);
|
|
||||||
context->receiver_done = false;
|
|
||||||
context->queued_bytes = 0;
|
|
||||||
context->max_queue_bytes = 0;
|
|
||||||
context->deferred_manifest = NULL;
|
|
||||||
atomic_init(&context->cancelled, false);
|
|
||||||
int init = 0;
|
|
||||||
if (mtx_init(&context->mutex, mtx_plain) != thrd_success)
|
|
||||||
goto fail;
|
|
||||||
init++;
|
|
||||||
if (cnd_init(&context->condition_not_full) != thrd_success)
|
|
||||||
goto fail;
|
|
||||||
init++;
|
|
||||||
if (cnd_init(&context->condition_not_empty) != thrd_success)
|
|
||||||
goto fail;
|
|
||||||
// cppcheck-suppress unreadVariable
|
|
||||||
init++;
|
|
||||||
return context;
|
|
||||||
|
|
||||||
fail:
|
|
||||||
log_perror("Error initializing synchronization objects");
|
|
||||||
if (init >= 3)
|
|
||||||
cnd_destroy(&context->condition_not_empty);
|
|
||||||
if (init >= 2)
|
|
||||||
cnd_destroy(&context->condition_not_full);
|
|
||||||
if (init >= 1)
|
|
||||||
mtx_destroy(&context->mutex);
|
|
||||||
free(context);
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
|
|
||||||
void pipeline_context_receiver_destroy(PipelineContextReceiver* context) {
|
|
||||||
config_delete(context->config);
|
|
||||||
if (context->deferred_manifest)
|
|
||||||
delete_manifest_free(context->deferred_manifest);
|
|
||||||
queue_destroy(context->queue);
|
|
||||||
receiver_outcomes_destroy(&context->outcomes);
|
|
||||||
dir_time_list_free(&context->dir_times);
|
|
||||||
mtx_destroy(&context->mutex);
|
|
||||||
cnd_destroy(&context->condition_not_full);
|
|
||||||
cnd_destroy(&context->condition_not_empty);
|
|
||||||
free(context);
|
|
||||||
}
|
|
||||||
|
|
||||||
void pipeline_context_receiver_set_queue_byte_limit(PipelineContextReceiver* context,
|
|
||||||
size_t max_bytes) {
|
|
||||||
if (context == NULL)
|
|
||||||
return;
|
|
||||||
mtx_lock(&context->mutex);
|
|
||||||
context->max_queue_bytes = max_bytes;
|
|
||||||
context->queued_bytes = 0;
|
|
||||||
cnd_broadcast(&context->condition_not_full);
|
|
||||||
mtx_unlock(&context->mutex);
|
|
||||||
}
|
|
||||||
|
|
||||||
void pipeline_context_receiver_note_bytes_released(PipelineContextReceiver* context,
|
|
||||||
size_t released_bytes) {
|
|
||||||
if (context == NULL || context->max_queue_bytes == 0 || released_bytes == 0)
|
|
||||||
return;
|
|
||||||
mtx_lock(&context->mutex);
|
|
||||||
if (released_bytes >= context->queued_bytes)
|
|
||||||
context->queued_bytes = 0;
|
|
||||||
else
|
|
||||||
context->queued_bytes -= released_bytes;
|
|
||||||
cnd_signal(&context->condition_not_full);
|
|
||||||
mtx_unlock(&context->mutex);
|
|
||||||
}
|
|
||||||
|
|
||||||
bool pipeline_context_receiver_enqueue_file(PipelineContextReceiver* context, File* file) {
|
|
||||||
if (context == NULL || file == NULL)
|
|
||||||
return false;
|
|
||||||
size_t file_bytes = file->data ? file->data->size : 0;
|
|
||||||
mtx_lock(&context->mutex);
|
|
||||||
while (!atomic_load(&context->cancelled)) {
|
|
||||||
bool blocked_by_count = queue_is_full(context->queue);
|
|
||||||
bool blocked_by_budget = false;
|
|
||||||
if (context->max_queue_bytes > 0) {
|
|
||||||
size_t budget = context->max_queue_bytes;
|
|
||||||
size_t used = context->queued_bytes;
|
|
||||||
if (used >= budget) {
|
|
||||||
blocked_by_budget = true;
|
|
||||||
} else if (file_bytes > budget - used) {
|
|
||||||
/* A single payload larger than the whole budget (not possible with
|
|
||||||
the per-file receive cap) is only admitted to an empty pipeline so
|
|
||||||
the wait can never deadlock. */
|
|
||||||
blocked_by_budget = used != 0;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (!blocked_by_count && !blocked_by_budget)
|
|
||||||
break;
|
|
||||||
cnd_wait(&context->condition_not_full, &context->mutex);
|
|
||||||
}
|
|
||||||
if (atomic_load(&context->cancelled)) {
|
|
||||||
mtx_unlock(&context->mutex);
|
|
||||||
file_destroy(file);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if (!queue_enqueue(context->queue, file)) {
|
|
||||||
mtx_unlock(&context->mutex);
|
|
||||||
file_destroy(file);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
context->queued_bytes += file_bytes;
|
|
||||||
cnd_signal(&context->condition_not_empty);
|
|
||||||
mtx_unlock(&context->mutex);
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
static bool receiver_enqueue_file(File* file, void* context_pointer) {
|
|
||||||
PipelineContextReceiver* context = (PipelineContextReceiver*)context_pointer;
|
|
||||||
return pipeline_context_receiver_enqueue_file(context, file);
|
|
||||||
}
|
|
||||||
|
|
||||||
static void receiver_thread_fail(PipelineContextReceiver* context) {
|
|
||||||
mtx_lock(&context->mutex);
|
|
||||||
atomic_store(&context->cancelled, true);
|
|
||||||
context->receiver_done = true;
|
|
||||||
cnd_broadcast(&context->condition_not_empty);
|
|
||||||
cnd_broadcast(&context->condition_not_full);
|
|
||||||
mtx_unlock(&context->mutex);
|
|
||||||
}
|
|
||||||
|
|
||||||
int receive_thread(void* pipeline_context) {
|
|
||||||
PipelineContextReceiver* context = (PipelineContextReceiver*)pipeline_context;
|
|
||||||
protocol_session_bind(&context->session);
|
|
||||||
mtx_lock(&context->mutex);
|
|
||||||
int file_descriptor = context->file_descriptor;
|
|
||||||
const Config* config = context->config;
|
|
||||||
mtx_unlock(&context->mutex);
|
|
||||||
|
|
||||||
ReceiverSink sink = {receiver_enqueue_file, context, false, false, NULL};
|
|
||||||
if (receiver_process_pending((Config*)config, file_descriptor, &sink,
|
|
||||||
&context->deferred_manifest) != 0) {
|
|
||||||
receiver_thread_fail(context);
|
|
||||||
protocol_session_unbind();
|
|
||||||
return thrd_error;
|
|
||||||
}
|
|
||||||
mtx_lock(&context->mutex);
|
|
||||||
context->receiver_done = true;
|
|
||||||
cnd_signal(&context->condition_not_empty);
|
|
||||||
mtx_unlock(&context->mutex);
|
|
||||||
protocol_session_unbind();
|
|
||||||
return thrd_success;
|
|
||||||
}
|
|
||||||
|
|
||||||
int write_thread(void* pipeline_context) {
|
|
||||||
PipelineContextReceiver* context = (PipelineContextReceiver*)pipeline_context;
|
|
||||||
protocol_session_bind(&context->session);
|
|
||||||
mtx_lock(&context->mutex);
|
|
||||||
bool save_to_disk = context->config->save_to_disk;
|
|
||||||
char* root_directory = str_dup(context->config->receive_root_directory);
|
|
||||||
mtx_unlock(&context->mutex);
|
|
||||||
if (save_to_disk && !root_directory) {
|
|
||||||
mtx_lock(&context->mutex);
|
|
||||||
atomic_store(&context->cancelled, true);
|
|
||||||
context->receiver_done = true;
|
|
||||||
cnd_broadcast(&context->condition_not_full);
|
|
||||||
cnd_broadcast(&context->condition_not_empty);
|
|
||||||
mtx_unlock(&context->mutex);
|
|
||||||
protocol_session_unbind();
|
|
||||||
return thrd_error;
|
|
||||||
}
|
|
||||||
|
|
||||||
while (true) {
|
|
||||||
File* file =
|
|
||||||
queue_dequeue_multithreaded(context->queue, &context->mutex, &context->condition_not_empty,
|
|
||||||
&context->condition_not_full, &context->receiver_done);
|
|
||||||
if (file == NULL) {
|
|
||||||
free(root_directory);
|
|
||||||
protocol_session_unbind();
|
|
||||||
return thrd_success;
|
|
||||||
}
|
|
||||||
size_t file_bytes = file->data ? file->data->size : 0;
|
|
||||||
FileSaveResult result = FILE_SAVE_SKIPPED;
|
|
||||||
if (save_to_disk) {
|
|
||||||
result = file_save_to_disk_full(root_directory, file, context->config);
|
|
||||||
if (result == FILE_SAVE_ERROR) {
|
|
||||||
file_destroy(file);
|
|
||||||
pipeline_context_receiver_note_bytes_released(context, file_bytes);
|
|
||||||
mtx_lock(&context->mutex);
|
|
||||||
atomic_store(&context->cancelled, true);
|
|
||||||
context->receiver_done = true;
|
|
||||||
cnd_broadcast(&context->condition_not_full);
|
|
||||||
cnd_broadcast(&context->condition_not_empty);
|
|
||||||
mtx_unlock(&context->mutex);
|
|
||||||
free(root_directory);
|
|
||||||
protocol_session_unbind();
|
|
||||||
return thrd_error;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
/* P7 Wave D: a directory's times are never applied inline (a later child
|
|
||||||
write would clobber them); accumulate the metadata here and let the
|
|
||||||
caller apply it once every writer has drained. */
|
|
||||||
if (result != FILE_SAVE_ERROR && file->is_dir && file->metadata &&
|
|
||||||
context->config->use_metadata && !context->config->omit_dir_times &&
|
|
||||||
!dir_time_list_add(&context->dir_times, file->path, file->metadata)) {
|
|
||||||
file_destroy(file);
|
|
||||||
pipeline_context_receiver_note_bytes_released(context, file_bytes);
|
|
||||||
mtx_lock(&context->mutex);
|
|
||||||
atomic_store(&context->cancelled, true);
|
|
||||||
context->receiver_done = true;
|
|
||||||
cnd_broadcast(&context->condition_not_full);
|
|
||||||
cnd_broadcast(&context->condition_not_empty);
|
|
||||||
mtx_unlock(&context->mutex);
|
|
||||||
free(root_directory);
|
|
||||||
protocol_session_unbind();
|
|
||||||
return thrd_error;
|
|
||||||
}
|
|
||||||
/* Record the per-file outcome so a --remove-source-files sender learns
|
|
||||||
which sources were actually written versus skipped on the receiver.
|
|
||||||
Explicit directory entries and recreated device/special nodes have no
|
|
||||||
source and are never acknowledged (mirrors receiver.c). */
|
|
||||||
if (context->config->remove_source_files && !file->is_dir && !file->is_special && !file->skip &&
|
|
||||||
!receiver_outcomes_append(&context->outcomes, (unsigned char)result)) {
|
|
||||||
file_destroy(file);
|
|
||||||
pipeline_context_receiver_note_bytes_released(context, file_bytes);
|
|
||||||
mtx_lock(&context->mutex);
|
|
||||||
atomic_store(&context->cancelled, true);
|
|
||||||
context->receiver_done = true;
|
|
||||||
cnd_broadcast(&context->condition_not_full);
|
|
||||||
cnd_broadcast(&context->condition_not_empty);
|
|
||||||
mtx_unlock(&context->mutex);
|
|
||||||
free(root_directory);
|
|
||||||
protocol_session_unbind();
|
|
||||||
return thrd_error;
|
|
||||||
}
|
|
||||||
file_destroy(file);
|
|
||||||
pipeline_context_receiver_note_bytes_released(context, file_bytes);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -5,11 +5,11 @@
|
|||||||
#include <stdatomic.h>
|
#include <stdatomic.h>
|
||||||
|
|
||||||
#include "array_list.h"
|
#include "array_list.h"
|
||||||
|
#include "chunk.h"
|
||||||
#include "config.h"
|
#include "config.h"
|
||||||
#include "file.h"
|
#include "file.h"
|
||||||
#include "protocol.h"
|
#include "protocol.h"
|
||||||
#include "queue.h"
|
#include "queue.h"
|
||||||
#include "receiver.h"
|
|
||||||
#include "stop_condition.h"
|
#include "stop_condition.h"
|
||||||
#include <openssl/ssl.h>
|
#include <openssl/ssl.h>
|
||||||
|
|
||||||
@@ -25,6 +25,15 @@ typedef struct {
|
|||||||
cnd_t condition_not_full_loader;
|
cnd_t condition_not_full_loader;
|
||||||
cnd_t condition_not_empty_loader;
|
cnd_t condition_not_empty_loader;
|
||||||
bool loader_done;
|
bool loader_done;
|
||||||
|
/* Aggregate loaded payload bytes queued on queue_loader but not yet released
|
||||||
|
by the sender. Guarded by `mutex_loader`. When `max_queue_bytes` is
|
||||||
|
non-zero the loader blocks before enqueueing a chunk that would push this
|
||||||
|
total over it, so the sender buffers a bounded number of bytes rather than
|
||||||
|
an unbounded count of chunks that may each be up to chunk_size (or a single
|
||||||
|
file) in size. Files streamed straight from disk by sendfile hold no
|
||||||
|
payload, so only in-memory (`data->data`) payloads are counted. */
|
||||||
|
size_t queued_bytes;
|
||||||
|
size_t max_queue_bytes;
|
||||||
ArrayList* manifest;
|
ArrayList* manifest;
|
||||||
/* Protected prefixes (paths the source scan excluded by user rules) sent
|
/* Protected prefixes (paths the source scan excluded by user rules) sent
|
||||||
with the keep-set manifest so --delete leaves them alone unless
|
with the keep-set manifest so --delete leaves them alone unless
|
||||||
@@ -76,58 +85,25 @@ typedef struct {
|
|||||||
bool dir_entries_mutex_init;
|
bool dir_entries_mutex_init;
|
||||||
} PipelineContextSender;
|
} PipelineContextSender;
|
||||||
|
|
||||||
typedef struct PipelineContextReceiver {
|
/* `config` is borrowed and must outlive the context: destroy does NOT free it,
|
||||||
Queue* queue;
|
so the caller owns it and frees it with config_delete() afterwards. */
|
||||||
Config* config;
|
|
||||||
int file_descriptor;
|
|
||||||
SSL* ssl;
|
|
||||||
ProtocolSession session;
|
|
||||||
ReceiverOutcomes outcomes;
|
|
||||||
mtx_t mutex;
|
|
||||||
cnd_t condition_not_full;
|
|
||||||
cnd_t condition_not_empty;
|
|
||||||
bool receiver_done;
|
|
||||||
atomic_bool cancelled;
|
|
||||||
/* Aggregate payload bytes that have been received but not yet released by
|
|
||||||
the disk writer (queued or in the writer's hand). Guarded by `mutex`.
|
|
||||||
When `max_queue_bytes` is non-zero the receiver blocks before enqueuing
|
|
||||||
once this total would exceed it, so decompressed/copied file payloads
|
|
||||||
buffered ahead of a slow disk writer respect the per-connection memory
|
|
||||||
budget instead of growing without bound. */
|
|
||||||
size_t queued_bytes;
|
|
||||||
size_t max_queue_bytes;
|
|
||||||
/* Keep-set manifest for the commit-style (late) deletion
|
|
||||||
(--delete/--delete-after/--delete-delay). receive_thread parses the whole
|
|
||||||
protocol stream but hands the manifest here instead of deleting while the
|
|
||||||
disk writer may still be draining; the caller (server.c) commits the
|
|
||||||
deletion after both threads have joined, so no extra is removed unless the
|
|
||||||
transfer truly succeeded. NULL in the early delete modes (which delete at
|
|
||||||
the manifest). */
|
|
||||||
DeleteManifest* deferred_manifest;
|
|
||||||
/* P7 Wave D: directory metadata collected by write_thread from received
|
|
||||||
directory entries. Only write_thread mutates it (before it joins); the
|
|
||||||
caller (server.c) applies it after the delete/delay-updates phase. */
|
|
||||||
DirTimeList dir_times;
|
|
||||||
} PipelineContextReceiver;
|
|
||||||
|
|
||||||
PipelineContextSender* pipeline_context_sender_create(Config* config, Queue* queue_scanner,
|
PipelineContextSender* pipeline_context_sender_create(Config* config, Queue* queue_scanner,
|
||||||
Queue* queue_loader);
|
Queue* queue_loader);
|
||||||
void pipeline_context_sender_destroy(PipelineContextSender* context);
|
void pipeline_context_sender_destroy(PipelineContextSender* context);
|
||||||
PipelineContextReceiver* pipeline_context_receiver_create(Config* config, Queue* queue_receiver,
|
/* Bound the loaded payload bytes the sender may buffer ahead of the network
|
||||||
int file_descriptor, SSL* ssl);
|
writer (see max_queue_bytes). */
|
||||||
void pipeline_context_receiver_destroy(PipelineContextReceiver* context);
|
void pipeline_context_sender_set_queue_byte_limit(PipelineContextSender* context, size_t max_bytes);
|
||||||
/* Bound the bytes buffered ahead of the disk writer (see max_queue_bytes). */
|
/* Total payload bytes a chunk currently holds in memory (loaded file data
|
||||||
void pipeline_context_receiver_set_queue_byte_limit(PipelineContextReceiver* context,
|
only; zero for entries with no payload or data streamed from disk). */
|
||||||
size_t max_bytes);
|
size_t pipeline_context_sender_chunk_bytes(const Chunk* chunk);
|
||||||
/* Blocking enqueue used by the receive pipeline sink. Blocks while the queue
|
/* Blocking enqueue used by the sender's loader stage. Blocks while
|
||||||
is full by element count or when adding `file` would push queued_bytes over
|
queue_loader is full by element count or when adding `chunk` would push the
|
||||||
the configured byte limit; waits until the disk writer releases bytes.
|
queued payload bytes over the configured byte limit; waits until the sender
|
||||||
Takes ownership of `file` on success and destroys it on failure/cancel. */
|
releases bytes. Takes ownership of `chunk` on success and destroys it on
|
||||||
bool pipeline_context_receiver_enqueue_file(PipelineContextReceiver* context, File* file);
|
failure/cancel. */
|
||||||
/* Account for `released_bytes` of payload memory that has been freed by the
|
bool pipeline_context_sender_enqueue_chunk(PipelineContextSender* context, Chunk* chunk);
|
||||||
disk writer, unblocking a receiver that is waiting on the byte limit. */
|
/* Account for `released_bytes` of payload memory that the sender freed after
|
||||||
void pipeline_context_receiver_note_bytes_released(PipelineContextReceiver* context,
|
destroying a chunk, unblocking a loader waiting on the byte limit. */
|
||||||
size_t released_bytes);
|
void pipeline_context_sender_note_bytes_released(PipelineContextSender* context,
|
||||||
int receive_thread(void* pipeline_context);
|
size_t released_bytes);
|
||||||
int write_thread(void* pipeline_context);
|
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
+338
-24
@@ -22,6 +22,10 @@ static __thread ProtocolSession* bound_session;
|
|||||||
static __thread ProtocolSession legacy_io_session = {
|
static __thread ProtocolSession legacy_io_session = {
|
||||||
.read_fd = -1, .write_fd = -1, .max_alloc = DEFAULT_MAX_ALLOC};
|
.read_fd = -1, .write_fd = -1, .max_alloc = DEFAULT_MAX_ALLOC};
|
||||||
|
|
||||||
|
/* Last STATUS_ERROR_DETAIL reason received on this thread (protocol 2.21.0).
|
||||||
|
* Empty when the last status read carried no detail. */
|
||||||
|
static __thread char io_error_detail[MAX_ERROR_DETAIL_BYTES + 1];
|
||||||
|
|
||||||
static unsigned long long io_bwlimit = 0;
|
static unsigned long long io_bwlimit = 0;
|
||||||
static mtx_t bw_mutex;
|
static mtx_t bw_mutex;
|
||||||
static once_flag bw_mutex_once = ONCE_FLAG_INIT;
|
static once_flag bw_mutex_once = ONCE_FLAG_INIT;
|
||||||
@@ -40,7 +44,9 @@ static bool protocol_reserve_memory(ProtocolSession* session, size_t charge) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static void protocol_release_memory_for_session(ProtocolSession* session, size_t charge) {
|
void protocol_release_memory_for_session(ProtocolSession* session, size_t charge) {
|
||||||
|
if (!session)
|
||||||
|
return;
|
||||||
unsigned long long allocated = atomic_load(&session->total_allocated_bytes);
|
unsigned long long allocated = atomic_load(&session->total_allocated_bytes);
|
||||||
while (true) {
|
while (true) {
|
||||||
unsigned long long remaining = (unsigned long long)charge >= allocated ? 0 : allocated - charge;
|
unsigned long long remaining = (unsigned long long)charge >= allocated ? 0 : allocated - charge;
|
||||||
@@ -57,6 +63,10 @@ void io_set_fds(int read_fd, int write_fd) {
|
|||||||
bound_session = NULL;
|
bound_session = NULL;
|
||||||
io_read_fd = read_fd;
|
io_read_fd = read_fd;
|
||||||
io_write_fd = write_fd;
|
io_write_fd = write_fd;
|
||||||
|
/* A descriptor switch starts a new connection on this thread: a stale
|
||||||
|
rejection detail captured from the previous transport must not leak into
|
||||||
|
the new one. */
|
||||||
|
io_error_detail[0] = '\0';
|
||||||
/* A descriptor switch starts a new transport; never reuse a TLS object
|
/* A descriptor switch starts a new transport; never reuse a TLS object
|
||||||
belonging to a previous connection or test pipe. */
|
belonging to a previous connection or test pipe. */
|
||||||
io_ssl = NULL;
|
io_ssl = NULL;
|
||||||
@@ -76,10 +86,23 @@ void protocol_session_init(ProtocolSession* session, int read_fd, int write_fd)
|
|||||||
session->read_fd = read_fd;
|
session->read_fd = read_fd;
|
||||||
session->write_fd = write_fd;
|
session->write_fd = write_fd;
|
||||||
session->max_alloc = DEFAULT_MAX_ALLOC;
|
session->max_alloc = DEFAULT_MAX_ALLOC;
|
||||||
|
session->io_timeout_sec = RECEIVE_TIMEOUT_SEC;
|
||||||
atomic_init(&session->total_allocated_bytes, 0);
|
atomic_init(&session->total_allocated_bytes, 0);
|
||||||
protocol_session_set_bwlimit(session, global_bwlimit());
|
protocol_session_set_bwlimit(session, global_bwlimit());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void protocol_session_set_io_timeout(ProtocolSession* session, int sec) {
|
||||||
|
if (!session)
|
||||||
|
return;
|
||||||
|
session->io_timeout_sec = sec;
|
||||||
|
}
|
||||||
|
|
||||||
|
int protocol_get_io_timeout_sec(void) {
|
||||||
|
const ProtocolSession* session = bound_session ? bound_session : &legacy_io_session;
|
||||||
|
int sec = session->io_timeout_sec;
|
||||||
|
return sec > 0 ? sec : RECEIVE_TIMEOUT_SEC;
|
||||||
|
}
|
||||||
|
|
||||||
void protocol_session_set_max_alloc(ProtocolSession* session, unsigned long long max_alloc) {
|
void protocol_session_set_max_alloc(ProtocolSession* session, unsigned long long max_alloc) {
|
||||||
if (!session)
|
if (!session)
|
||||||
session = bound_session ? bound_session : &legacy_io_session;
|
session = bound_session ? bound_session : &legacy_io_session;
|
||||||
@@ -257,10 +280,11 @@ bool protocol_send_n_data(ProtocolSession* session, const void* data, size_t dat
|
|||||||
log_debug_message(LOG_DEBUG_IO, " Sending n Data: %zu", data_size);
|
log_debug_message(LOG_DEBUG_IO, " Sending n Data: %zu", data_size);
|
||||||
if (!session)
|
if (!session)
|
||||||
return false;
|
return false;
|
||||||
|
int timeout_sec = session->io_timeout_sec > 0 ? session->io_timeout_sec : SEND_TIMEOUT_SEC;
|
||||||
int fd = session->write_fd;
|
int fd = session->write_fd;
|
||||||
struct timespec deadline;
|
struct timespec deadline;
|
||||||
clock_gettime(CLOCK_MONOTONIC, &deadline);
|
clock_gettime(CLOCK_MONOTONIC, &deadline);
|
||||||
deadline.tv_sec += SEND_TIMEOUT_SEC;
|
deadline.tv_sec += timeout_sec;
|
||||||
short wait_events = POLLOUT;
|
short wait_events = POLLOUT;
|
||||||
ssize_t total_bytes_send = 0;
|
ssize_t total_bytes_send = 0;
|
||||||
while ((size_t)total_bytes_send < data_size) {
|
while ((size_t)total_bytes_send < data_size) {
|
||||||
@@ -278,10 +302,14 @@ bool protocol_send_n_data(ProtocolSession* session, const void* data, size_t dat
|
|||||||
if (pfd.revents & (POLLERR | POLLNVAL))
|
if (pfd.revents & (POLLERR | POLLNVAL))
|
||||||
return false;
|
return false;
|
||||||
ssize_t bytes_send;
|
ssize_t bytes_send;
|
||||||
if (session->ssl)
|
if (session->ssl) {
|
||||||
bytes_send = SSL_write(session->ssl, (const char*)data + total_bytes_send, chunk);
|
/* SSL_write takes an int length; clamp a >INT_MAX request into chunks so
|
||||||
else
|
* the size_t downcast can never truncate into a negative/partial write. */
|
||||||
|
size_t ssl_chunk = chunk > (size_t)INT_MAX ? (size_t)INT_MAX : chunk;
|
||||||
|
bytes_send = SSL_write(session->ssl, (const char*)data + total_bytes_send, (int)ssl_chunk);
|
||||||
|
} else {
|
||||||
bytes_send = write(fd, (const char*)data + total_bytes_send, chunk);
|
bytes_send = write(fd, (const char*)data + total_bytes_send, chunk);
|
||||||
|
}
|
||||||
if (bytes_send <= 0) {
|
if (bytes_send <= 0) {
|
||||||
if (session->ssl) {
|
if (session->ssl) {
|
||||||
int ssl_err = SSL_get_error(session->ssl, (int)bytes_send);
|
int ssl_err = SSL_get_error(session->ssl, (int)bytes_send);
|
||||||
@@ -289,6 +317,12 @@ bool protocol_send_n_data(ProtocolSession* session, const void* data, size_t dat
|
|||||||
wait_events = ssl_err == SSL_ERROR_WANT_WRITE ? POLLOUT : POLLIN;
|
wait_events = ssl_err == SSL_ERROR_WANT_WRITE ? POLLOUT : POLLIN;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
/* A signal (e.g. Ctrl-C) interrupts the blocking TLS write: retry so
|
||||||
|
the send loop can observe the abort flag at the next checkpoint. */
|
||||||
|
if (ssl_err == SSL_ERROR_SYSCALL && errno == EINTR)
|
||||||
|
continue;
|
||||||
|
} else if (errno == EINTR) {
|
||||||
|
continue;
|
||||||
}
|
}
|
||||||
log_message(LOG_LEVEL_ERROR, "Could not send data");
|
log_message(LOG_LEVEL_ERROR, "Could not send data");
|
||||||
return false;
|
return false;
|
||||||
@@ -306,30 +340,31 @@ bool protocol_receive_n_data_timed(ProtocolSession* session, void* data, size_t
|
|||||||
int timeout_sec);
|
int timeout_sec);
|
||||||
|
|
||||||
bool protocol_receive_n_data(ProtocolSession* session, void* data, size_t data_size) {
|
bool protocol_receive_n_data(ProtocolSession* session, void* data, size_t data_size) {
|
||||||
return protocol_receive_n_data_timed(session, data, data_size, RECEIVE_TIMEOUT_SEC);
|
/* Honor the session's configured deadline; protocol_receive_n_data_timed
|
||||||
|
* re-applies the built-in 60 s default when the value is <= 0. */
|
||||||
|
int timeout_sec = session ? session->io_timeout_sec : 0;
|
||||||
|
return protocol_receive_n_data_timed(session, data, data_size, timeout_sec);
|
||||||
}
|
}
|
||||||
|
|
||||||
bool protocol_receive_n_data_timed(ProtocolSession* session, void* data, size_t data_size,
|
/* Read exactly `data_size` bytes from `session` before `deadline` elapses
|
||||||
int timeout_sec) {
|
* (CLOCK_MONOTONIC). Shared by the ordinary timed primitive and the error-detail
|
||||||
|
* body reader so the latter can clamp itself to whatever deadline its caller
|
||||||
|
* already established instead of always applying the session's 60 s window. */
|
||||||
|
static bool protocol_receive_n_data_until(ProtocolSession* session, void* data, size_t data_size,
|
||||||
|
const struct timespec* deadline) {
|
||||||
log_debug_message(LOG_DEBUG_IO, " Receiving n Data: %zu", data_size);
|
log_debug_message(LOG_DEBUG_IO, " Receiving n Data: %zu", data_size);
|
||||||
if (!session)
|
if (!session || !deadline)
|
||||||
return false;
|
return false;
|
||||||
int fd = session->read_fd;
|
int fd = session->read_fd;
|
||||||
if (timeout_sec <= 0)
|
|
||||||
timeout_sec = RECEIVE_TIMEOUT_SEC;
|
|
||||||
|
|
||||||
struct timespec deadline;
|
|
||||||
clock_gettime(CLOCK_MONOTONIC, &deadline);
|
|
||||||
deadline.tv_sec += timeout_sec;
|
|
||||||
|
|
||||||
size_t total_bytes_received = 0;
|
size_t total_bytes_received = 0;
|
||||||
short wait_events = POLLIN;
|
short wait_events = POLLIN;
|
||||||
while (total_bytes_received < data_size) {
|
while (total_bytes_received < data_size) {
|
||||||
if (!session->ssl || SSL_pending(session->ssl) == 0) {
|
if (!session->ssl || SSL_pending(session->ssl) == 0) {
|
||||||
struct pollfd pfd = {.fd = fd, .events = wait_events};
|
struct pollfd pfd = {.fd = fd, .events = wait_events};
|
||||||
int poll_result = poll(&pfd, 1, deadline_remaining_ms(&deadline));
|
int poll_result = poll(&pfd, 1, deadline_remaining_ms(deadline));
|
||||||
if (poll_result == 0) {
|
if (poll_result == 0) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Receive timeout after %ds", timeout_sec);
|
log_message(LOG_LEVEL_ERROR, "Receive timeout");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
if (poll_result < 0) {
|
if (poll_result < 0) {
|
||||||
@@ -356,6 +391,13 @@ bool protocol_receive_n_data_timed(ProtocolSession* session, void* data, size_t
|
|||||||
wait_events = ssl_err == SSL_ERROR_WANT_WRITE ? POLLOUT : POLLIN;
|
wait_events = ssl_err == SSL_ERROR_WANT_WRITE ? POLLOUT : POLLIN;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
/* A signal interrupts the blocking TLS read: retry (mirrors the send
|
||||||
|
path and protocol_read_status_until) so the loop reaches its next
|
||||||
|
abort/deadline checkpoint instead of failing spuriously. */
|
||||||
|
if (ssl_err == SSL_ERROR_SYSCALL && errno == EINTR)
|
||||||
|
continue;
|
||||||
|
} else if (errno == EINTR) {
|
||||||
|
continue;
|
||||||
}
|
}
|
||||||
if (bytes_received == 0)
|
if (bytes_received == 0)
|
||||||
log_message(LOG_LEVEL_ERROR, "Connection closed while receiving data");
|
log_message(LOG_LEVEL_ERROR, "Connection closed while receiving data");
|
||||||
@@ -371,6 +413,18 @@ bool protocol_receive_n_data_timed(ProtocolSession* session, void* data, size_t
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool protocol_receive_n_data_timed(ProtocolSession* session, void* data, size_t data_size,
|
||||||
|
int timeout_sec) {
|
||||||
|
if (!session)
|
||||||
|
return false;
|
||||||
|
if (timeout_sec <= 0)
|
||||||
|
timeout_sec = RECEIVE_TIMEOUT_SEC;
|
||||||
|
struct timespec deadline;
|
||||||
|
clock_gettime(CLOCK_MONOTONIC, &deadline);
|
||||||
|
deadline.tv_sec += timeout_sec;
|
||||||
|
return protocol_receive_n_data_until(session, data, data_size, &deadline);
|
||||||
|
}
|
||||||
|
|
||||||
static const char* status_to_string(Status status) {
|
static const char* status_to_string(Status status) {
|
||||||
switch (status) {
|
switch (status) {
|
||||||
case STATUS_OK:
|
case STATUS_OK:
|
||||||
@@ -421,6 +475,10 @@ static const char* status_to_string(Status status) {
|
|||||||
return "AUTH_OK";
|
return "AUTH_OK";
|
||||||
case STATUS_AUTH_FAILED:
|
case STATUS_AUTH_FAILED:
|
||||||
return "AUTH_FAILED";
|
return "AUTH_FAILED";
|
||||||
|
case STATUS_ERROR_DETAIL:
|
||||||
|
return "ERROR_DETAIL";
|
||||||
|
case STATUS_DRY_RUN_TRANSFER:
|
||||||
|
return "DRY_RUN_TRANSFER";
|
||||||
default:
|
default:
|
||||||
return "UNKNOWN";
|
return "UNKNOWN";
|
||||||
}
|
}
|
||||||
@@ -550,13 +608,10 @@ Data* protocol_receive_data_limited(ProtocolSession* session, unsigned long long
|
|||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
result->protocol_charge = allocation_size;
|
result->protocol_charge = allocation_size;
|
||||||
|
result->owner = session;
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
Data* protocol_receive_data(ProtocolSession* session) {
|
|
||||||
return protocol_receive_data_limited(session, MAX_DATA_PAYLOAD_SIZE);
|
|
||||||
}
|
|
||||||
|
|
||||||
bool protocol_send_int(ProtocolSession* session, int data) {
|
bool protocol_send_int(ProtocolSession* session, int data) {
|
||||||
if (!protocol_send_n_data(session, &data, sizeof(int)))
|
if (!protocol_send_n_data(session, &data, sizeof(int)))
|
||||||
return false;
|
return false;
|
||||||
@@ -578,8 +633,82 @@ bool protocol_send_status(ProtocolSession* session, Status status) {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Read the bounded, length-prefixed body of a STATUS_ERROR_DETAIL frame within
|
||||||
|
* `deadline` (CLOCK_MONOTONIC), polling `abort_check` (may be NULL) between
|
||||||
|
* drain chunks. The declared length is validated BEFORE any allocation:
|
||||||
|
*
|
||||||
|
* - `size > MAX_STRING_SIZE`: an absurd framing error. Reading/draining that
|
||||||
|
* many bytes could never finish, so it is fatal (the caller tears the
|
||||||
|
* connection down) rather than drained.
|
||||||
|
* - `MAX_ERROR_DETAIL_BYTES < size <= MAX_STRING_SIZE`: drain exactly `size`
|
||||||
|
* bytes through a small fixed scratch buffer so the stream stays in sync,
|
||||||
|
* leaving the captured detail empty. No allocation happens.
|
||||||
|
* - `size <= MAX_ERROR_DETAIL_BYTES`: read straight into the thread-local
|
||||||
|
* `io_error_detail` buffer (size+1 capacity, already reserved), so the
|
||||||
|
* session's --max-alloc / MAX_CONNECTION_MEMORY budgets are never touched.
|
||||||
|
*
|
||||||
|
* Returns false on a fatal framing problem or any I/O failure; the terminal
|
||||||
|
* detail is then empty. The body is consumed on every non-fatal path even when
|
||||||
|
* the caller ignores protocol_last_error(), so the stream never desyncs. */
|
||||||
|
static bool protocol_receive_error_detail_until(ProtocolSession* session,
|
||||||
|
const struct timespec* deadline,
|
||||||
|
ProtocolWaitAbort abort_check) {
|
||||||
|
io_error_detail[0] = '\0';
|
||||||
|
size_t size = 0;
|
||||||
|
if (!protocol_receive_n_data_until(session, &size, sizeof(size), deadline))
|
||||||
|
return false;
|
||||||
|
if (size > MAX_STRING_SIZE) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Error detail length %zu exceeds maximum %llu", size,
|
||||||
|
(unsigned long long)MAX_STRING_SIZE);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (size > MAX_ERROR_DETAIL_BYTES) {
|
||||||
|
char scratch[256];
|
||||||
|
size_t remaining = size;
|
||||||
|
while (remaining > 0) {
|
||||||
|
if (abort_check && abort_check())
|
||||||
|
return false;
|
||||||
|
size_t chunk = remaining < sizeof(scratch) ? remaining : sizeof(scratch);
|
||||||
|
if (!protocol_receive_n_data_until(session, scratch, chunk, deadline))
|
||||||
|
return false;
|
||||||
|
remaining -= chunk;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (!protocol_receive_n_data_until(session, io_error_detail, size, deadline))
|
||||||
|
return false;
|
||||||
|
io_error_detail[size] = '\0';
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Consume the optional detail body of a STATUS_ERROR_DETAIL frame and map the
|
||||||
|
* status back to STATUS_ERROR for existing callers. Invoked for EVERY status
|
||||||
|
* read so a stale detail from an earlier exchange is never reported for a later
|
||||||
|
* one -- except for STATUS_KEEPALIVE, which carries no body and whose drain
|
||||||
|
* (protocol_receive_status_keepalive) must NOT erase the terminal detail that
|
||||||
|
* arrived just before it. Returns false on a fatal framing error. */
|
||||||
|
static bool protocol_capture_error_detail(ProtocolSession* session, Status* status,
|
||||||
|
const struct timespec* deadline,
|
||||||
|
ProtocolWaitAbort abort_check) {
|
||||||
|
if (*status == STATUS_KEEPALIVE)
|
||||||
|
return true;
|
||||||
|
io_error_detail[0] = '\0';
|
||||||
|
if (*status != STATUS_ERROR_DETAIL)
|
||||||
|
return true;
|
||||||
|
*status = STATUS_ERROR;
|
||||||
|
return protocol_receive_error_detail_until(session, deadline, abort_check);
|
||||||
|
}
|
||||||
|
|
||||||
bool protocol_receive_status(ProtocolSession* session, Status* status) {
|
bool protocol_receive_status(ProtocolSession* session, Status* status) {
|
||||||
if (!protocol_receive_n_data(session, status, sizeof(Status)))
|
if (!session || !status)
|
||||||
|
return false;
|
||||||
|
int timeout_sec = session->io_timeout_sec > 0 ? session->io_timeout_sec : RECEIVE_TIMEOUT_SEC;
|
||||||
|
struct timespec deadline;
|
||||||
|
clock_gettime(CLOCK_MONOTONIC, &deadline);
|
||||||
|
deadline.tv_sec += timeout_sec;
|
||||||
|
if (!protocol_receive_n_data_until(session, status, sizeof(Status), &deadline))
|
||||||
|
return false;
|
||||||
|
if (!protocol_capture_error_detail(session, status, &deadline, NULL))
|
||||||
return false;
|
return false;
|
||||||
log_debug_message(LOG_DEBUG_PROTO, "Received Status: %s", status_to_string(*status));
|
log_debug_message(LOG_DEBUG_PROTO, "Received Status: %s", status_to_string(*status));
|
||||||
return true;
|
return true;
|
||||||
@@ -588,10 +717,169 @@ bool protocol_receive_status(ProtocolSession* session, Status* status) {
|
|||||||
/* protocol_receive_status with an explicit per-message deadline (seconds).
|
/* protocol_receive_status with an explicit per-message deadline (seconds).
|
||||||
Used where a single reply may legitimately take far longer than the default
|
Used where a single reply may legitimately take far longer than the default
|
||||||
60 s receive window - e.g. the sender waiting for the early-delete ACK after
|
60 s receive window - e.g. the sender waiting for the early-delete ACK after
|
||||||
the receiver committed a large (up to MAX_SERVER_DELETE_COUNT) deletion. */
|
the receiver committed a large (up to MAX_SERVER_DELETE_COUNT) deletion. The
|
||||||
|
error-detail body shares the same deadline as the status header. */
|
||||||
bool protocol_receive_status_timed(ProtocolSession* session, Status* status, int timeout_sec) {
|
bool protocol_receive_status_timed(ProtocolSession* session, Status* status, int timeout_sec) {
|
||||||
if (!protocol_receive_n_data_timed(session, status, sizeof(Status), timeout_sec))
|
if (!session || !status)
|
||||||
return false;
|
return false;
|
||||||
|
if (timeout_sec <= 0)
|
||||||
|
timeout_sec = RECEIVE_TIMEOUT_SEC;
|
||||||
|
struct timespec deadline;
|
||||||
|
clock_gettime(CLOCK_MONOTONIC, &deadline);
|
||||||
|
deadline.tv_sec += timeout_sec;
|
||||||
|
if (!protocol_receive_n_data_until(session, status, sizeof(Status), &deadline))
|
||||||
|
return false;
|
||||||
|
if (!protocol_capture_error_detail(session, status, &deadline, NULL))
|
||||||
|
return false;
|
||||||
|
log_debug_message(LOG_DEBUG_PROTO, "Received Status: %s", status_to_string(*status));
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Read exactly one Status frame within `deadline` (CLOCK_MONOTONIC). Unlike
|
||||||
|
* protocol_receive_status_keepalive this never emits a keepalive: it is used
|
||||||
|
* to consume the first byte(s) of an already-signalled frame and to drain the
|
||||||
|
* peer's outstanding keepalive replies, where injecting a write could split a
|
||||||
|
* reply across a frame boundary. Returns false on timeout/EOF/error. */
|
||||||
|
static bool protocol_read_status_until(ProtocolSession* session, Status* status,
|
||||||
|
const struct timespec* deadline) {
|
||||||
|
Status received = STATUS_ERROR;
|
||||||
|
size_t got = 0;
|
||||||
|
short wait_events = POLLIN;
|
||||||
|
while (got < sizeof(Status)) {
|
||||||
|
if (!session->ssl || SSL_pending(session->ssl) == 0) {
|
||||||
|
int remaining_ms = deadline_remaining_ms(deadline);
|
||||||
|
if (remaining_ms <= 0) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Receive timeout while reading status");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
struct pollfd pfd = {.fd = session->read_fd, .events = wait_events};
|
||||||
|
int poll_result = poll(&pfd, 1, remaining_ms);
|
||||||
|
if (poll_result == 0) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Receive timeout while reading status");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (poll_result < 0) {
|
||||||
|
if (errno == EINTR)
|
||||||
|
continue;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (pfd.revents & (POLLERR | POLLNVAL))
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
ssize_t bytes_received;
|
||||||
|
if (session->ssl)
|
||||||
|
bytes_received = SSL_read(session->ssl, (char*)&received + got, sizeof(Status) - got);
|
||||||
|
else
|
||||||
|
bytes_received = read(session->read_fd, (char*)&received + got, sizeof(Status) - got);
|
||||||
|
if (bytes_received <= 0) {
|
||||||
|
if (session->ssl) {
|
||||||
|
int ssl_err = SSL_get_error(session->ssl, (int)bytes_received);
|
||||||
|
if (ssl_err == SSL_ERROR_WANT_READ || ssl_err == SSL_ERROR_WANT_WRITE) {
|
||||||
|
wait_events = ssl_err == SSL_ERROR_WANT_WRITE ? POLLOUT : POLLIN;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (bytes_received < 0 && errno == EINTR)
|
||||||
|
continue;
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Connection closed while receiving status");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
got += (size_t)bytes_received;
|
||||||
|
}
|
||||||
|
*status = received;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool protocol_receive_status_keepalive(ProtocolSession* session, Status* status, int timeout_sec,
|
||||||
|
int keepalive_interval_sec, ProtocolWaitAbort abort_check) {
|
||||||
|
if (!session || !status)
|
||||||
|
return false;
|
||||||
|
if (timeout_sec <= 0)
|
||||||
|
timeout_sec = RECEIVE_TIMEOUT_SEC;
|
||||||
|
if (keepalive_interval_sec <= 0)
|
||||||
|
keepalive_interval_sec = timeout_sec;
|
||||||
|
|
||||||
|
struct timespec deadline;
|
||||||
|
clock_gettime(CLOCK_MONOTONIC, &deadline);
|
||||||
|
deadline.tv_sec += timeout_sec;
|
||||||
|
|
||||||
|
unsigned long keepalives_sent = 0;
|
||||||
|
unsigned long replies_seen = 0;
|
||||||
|
Status final = STATUS_ERROR;
|
||||||
|
while (true) {
|
||||||
|
if (abort_check && abort_check())
|
||||||
|
return false;
|
||||||
|
if (!session->ssl || SSL_pending(session->ssl) == 0) {
|
||||||
|
int remaining_ms = deadline_remaining_ms(&deadline);
|
||||||
|
if (remaining_ms <= 0) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Receive timeout after %ds", timeout_sec);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
/* Only interleave a keepalive while waiting for the FIRST byte of a
|
||||||
|
* frame; once part of a frame is buffered a write could race the peer's
|
||||||
|
* reply into the middle of it. */
|
||||||
|
long long interval_ms_ll = (long long)keepalive_interval_sec * 1000LL;
|
||||||
|
int interval_ms = interval_ms_ll > INT_MAX ? INT_MAX : (int)interval_ms_ll;
|
||||||
|
int wait_ms = interval_ms < remaining_ms ? interval_ms : remaining_ms;
|
||||||
|
struct pollfd pfd = {.fd = session->read_fd, .events = POLLIN};
|
||||||
|
int poll_result = poll(&pfd, 1, wait_ms);
|
||||||
|
if (poll_result == 0) {
|
||||||
|
if (abort_check && abort_check())
|
||||||
|
return false;
|
||||||
|
if (!protocol_send_status(session, STATUS_KEEPALIVE))
|
||||||
|
return false;
|
||||||
|
keepalives_sent++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (poll_result < 0) {
|
||||||
|
if (errno == EINTR)
|
||||||
|
continue;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (pfd.revents & (POLLERR | POLLNVAL))
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
Status received;
|
||||||
|
if (!protocol_read_status_until(session, &received, &deadline))
|
||||||
|
return false;
|
||||||
|
if (!protocol_capture_error_detail(session, &received, &deadline, abort_check))
|
||||||
|
return false;
|
||||||
|
if (received == STATUS_KEEPALIVE) {
|
||||||
|
/* The receiver's answer to one of our keepalives. */
|
||||||
|
replies_seen++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
final = received;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
/* Drain the replies the receiver still owes for keepalives we sent while it
|
||||||
|
* was busy. It answers them only after the real status, so leaving them
|
||||||
|
* unread would put stale KEEPALIVE frames ahead of the next exchange and
|
||||||
|
* desynchronize the protocol. */
|
||||||
|
if (replies_seen < keepalives_sent) {
|
||||||
|
/* A short separate grace, not the (possibly exhausted) main deadline: the
|
||||||
|
terminal status already arrived, so a peer that never answers its owed
|
||||||
|
keepalives must not turn a successful ack into a reported failure. */
|
||||||
|
struct timespec drain_deadline;
|
||||||
|
clock_gettime(CLOCK_MONOTONIC, &drain_deadline);
|
||||||
|
drain_deadline.tv_sec += 1;
|
||||||
|
while (replies_seen < keepalives_sent) {
|
||||||
|
Status drained;
|
||||||
|
if (!protocol_read_status_until(session, &drained, &drain_deadline)) {
|
||||||
|
log_message(LOG_LEVEL_WARNING, "peer did not answer %lu keepalive(s); continuing",
|
||||||
|
keepalives_sent - replies_seen);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if (!protocol_capture_error_detail(session, &drained, &drain_deadline, abort_check))
|
||||||
|
return false;
|
||||||
|
if (drained != STATUS_KEEPALIVE) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Unexpected status while draining keepalive replies");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
replies_seen++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
*status = final;
|
||||||
log_debug_message(LOG_DEBUG_PROTO, "Received Status: %s", status_to_string(*status));
|
log_debug_message(LOG_DEBUG_PROTO, "Received Status: %s", status_to_string(*status));
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
@@ -635,3 +923,29 @@ bool receive_status(int fd, Status* status) {
|
|||||||
bool receive_status_timed(int fd, Status* status, int timeout_sec) {
|
bool receive_status_timed(int fd, Status* status, int timeout_sec) {
|
||||||
return protocol_receive_status_timed(legacy_session(fd, -1), status, timeout_sec);
|
return protocol_receive_status_timed(legacy_session(fd, -1), status, timeout_sec);
|
||||||
}
|
}
|
||||||
|
bool receive_status_keepalive(int fd, Status* status, int timeout_sec, int keepalive_interval_sec,
|
||||||
|
ProtocolWaitAbort abort_check) {
|
||||||
|
return protocol_receive_status_keepalive(legacy_session(fd, -1), status, timeout_sec,
|
||||||
|
keepalive_interval_sec, abort_check);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool send_error_detail(int fd, const char* message) {
|
||||||
|
if (!message)
|
||||||
|
message = "";
|
||||||
|
char bounded[MAX_ERROR_DETAIL_BYTES + 1];
|
||||||
|
size_t len = strlen(message);
|
||||||
|
if (len > MAX_ERROR_DETAIL_BYTES) {
|
||||||
|
memcpy(bounded, message, MAX_ERROR_DETAIL_BYTES);
|
||||||
|
bounded[MAX_ERROR_DETAIL_BYTES] = '\0';
|
||||||
|
message = bounded;
|
||||||
|
}
|
||||||
|
return send_status(fd, STATUS_ERROR_DETAIL) && send_str(fd, message);
|
||||||
|
}
|
||||||
|
|
||||||
|
const char* protocol_last_error(void) {
|
||||||
|
return io_error_detail;
|
||||||
|
}
|
||||||
|
|
||||||
|
void protocol_clear_last_error(void) {
|
||||||
|
io_error_detail[0] = '\0';
|
||||||
|
}
|
||||||
|
|||||||
+72
-2
@@ -9,6 +9,12 @@
|
|||||||
/* Maximum allowed string size for receive_str (64 KB) */
|
/* Maximum allowed string size for receive_str (64 KB) */
|
||||||
#define MAX_STRING_SIZE (64 * 1024)
|
#define MAX_STRING_SIZE (64 * 1024)
|
||||||
|
|
||||||
|
/* Hard cap on the optional server->client rejection detail carried by
|
||||||
|
* STATUS_ERROR_DETAIL (protocol 2.21.0). A longer message is sliced to this
|
||||||
|
* many bytes before it is sent, so a peer can never be made to retain more than
|
||||||
|
* this for a rejection and the detail frame stays a small, fixed bound. */
|
||||||
|
#define MAX_ERROR_DETAIL_BYTES 4096
|
||||||
|
|
||||||
/* Maximum uncompressed file payload accepted by the receiver's whole-file
|
/* Maximum uncompressed file payload accepted by the receiver's whole-file
|
||||||
* paths. A single whole file is charged against the per-connection memory
|
* paths. A single whole file is charged against the per-connection memory
|
||||||
* reservation (MAX_CONNECTION_MEMORY) and against the server allocation
|
* reservation (MAX_CONNECTION_MEMORY) and against the server allocation
|
||||||
@@ -52,6 +58,12 @@ typedef struct ProtocolSession {
|
|||||||
atomic_ullong total_allocated_bytes;
|
atomic_ullong total_allocated_bytes;
|
||||||
bool eight_bit_output;
|
bool eight_bit_output;
|
||||||
unsigned long long max_alloc;
|
unsigned long long max_alloc;
|
||||||
|
/* Per-session deadline (seconds) applied to every protocol send/receive by
|
||||||
|
* protocol_send_n_data / protocol_receive_n_data. Defaults to the built-in
|
||||||
|
* 60 s window; a value <= 0 falls back to that default. Set from the
|
||||||
|
* negotiated Config->timeout so --timeout is honored by the poll()-driven
|
||||||
|
* protocol I/O, not just the socket SO_RCVTIMEO/SO_SNDTIMEO. */
|
||||||
|
int io_timeout_sec;
|
||||||
} ProtocolSession;
|
} ProtocolSession;
|
||||||
|
|
||||||
typedef int Status;
|
typedef int Status;
|
||||||
@@ -126,7 +138,24 @@ enum NET_STATUS {
|
|||||||
STATUS_AUTH_CHALLENGE,
|
STATUS_AUTH_CHALLENGE,
|
||||||
STATUS_AUTH_RESPONSE,
|
STATUS_AUTH_RESPONSE,
|
||||||
STATUS_AUTH_OK,
|
STATUS_AUTH_OK,
|
||||||
STATUS_AUTH_FAILED
|
STATUS_AUTH_FAILED,
|
||||||
|
/* Optional server->client rejection detail (protocol 2.21.0). When the
|
||||||
|
* server refuses a transfer for a concrete reason it may send
|
||||||
|
* STATUS_ERROR_DETAIL followed by a length-prefixed, bounded string instead
|
||||||
|
* of a bare STATUS_ERROR. receive_status() consumes the string and maps the
|
||||||
|
* status back to STATUS_ERROR, so every pre-2.21 call site keeps working;
|
||||||
|
* callers that want the human-readable reason consult protocol_last_error().
|
||||||
|
* Appended immediately after STATUS_AUTH_FAILED so the existing wire values
|
||||||
|
* never move. */
|
||||||
|
STATUS_ERROR_DETAIL,
|
||||||
|
/* Server-contacting --dry-run (protocol 2.21.0). Sent by the receiver in
|
||||||
|
* response to a per-file STATUS_CHECK when the wire config carries
|
||||||
|
* dry_run=true and the file is NOT already up to date: it tells the sender
|
||||||
|
* the file WOULD be transferred, and the sender must NOT transmit any data
|
||||||
|
* (the receiver reads none in dry-run). STATUS_OK keeps its meaning in this
|
||||||
|
* path ("already up to date / nothing to do"). Appended after
|
||||||
|
* STATUS_ERROR_DETAIL so no existing status is renumbered. */
|
||||||
|
STATUS_DRY_RUN_TRANSFER
|
||||||
};
|
};
|
||||||
|
|
||||||
void io_set_fds(int read_fd, int write_fd);
|
void io_set_fds(int read_fd, int write_fd);
|
||||||
@@ -141,6 +170,15 @@ void protocol_session_unbind(void);
|
|||||||
void protocol_session_set_ssl(ProtocolSession* session, SSL* ssl);
|
void protocol_session_set_ssl(ProtocolSession* session, SSL* ssl);
|
||||||
void protocol_session_set_bwlimit(ProtocolSession* session, unsigned long long bytes_per_sec);
|
void protocol_session_set_bwlimit(ProtocolSession* session, unsigned long long bytes_per_sec);
|
||||||
void protocol_session_set_max_alloc(ProtocolSession* session, unsigned long long max_alloc);
|
void protocol_session_set_max_alloc(ProtocolSession* session, unsigned long long max_alloc);
|
||||||
|
/* Override the per-message send/receive deadline for this session.
|
||||||
|
* `sec` <= 0 restores the built-in 60 s default (used for --timeout=0/unset).
|
||||||
|
* An explicit long deadline (e.g. the delete-ack wait) is applied per-call by
|
||||||
|
* protocol_receive_status_timed and is unaffected by this setter. */
|
||||||
|
void protocol_session_set_io_timeout(ProtocolSession* session, int sec);
|
||||||
|
/* Effective per-message I/O deadline (seconds) for the currently-bound session,
|
||||||
|
* falling back to the built-in default. Used by the plaintext sendfile path
|
||||||
|
* which bypasses the protocol send primitive. */
|
||||||
|
int protocol_get_io_timeout_sec(void);
|
||||||
void* protocol_alloc(size_t size);
|
void* protocol_alloc(size_t size);
|
||||||
void* protocol_realloc(void* ptr, size_t size);
|
void* protocol_realloc(void* ptr, size_t size);
|
||||||
void protocol_session_set_8_bit_output(ProtocolSession* session, bool enabled);
|
void protocol_session_set_8_bit_output(ProtocolSession* session, bool enabled);
|
||||||
@@ -157,7 +195,6 @@ char* protocol_receive_str(ProtocolSession* session);
|
|||||||
bool protocol_send_str_redacted(ProtocolSession* session, const char* data);
|
bool protocol_send_str_redacted(ProtocolSession* session, const char* data);
|
||||||
char* protocol_receive_str_redacted(ProtocolSession* session);
|
char* protocol_receive_str_redacted(ProtocolSession* session);
|
||||||
bool protocol_send_data(ProtocolSession* session, const Data* data);
|
bool protocol_send_data(ProtocolSession* session, const Data* data);
|
||||||
Data* protocol_receive_data(ProtocolSession* session);
|
|
||||||
Data* protocol_receive_data_limited(ProtocolSession* session, unsigned long long maximum_size);
|
Data* protocol_receive_data_limited(ProtocolSession* session, unsigned long long maximum_size);
|
||||||
bool protocol_send_int(ProtocolSession* session, int data);
|
bool protocol_send_int(ProtocolSession* session, int data);
|
||||||
bool protocol_receive_int(ProtocolSession* session, int* data);
|
bool protocol_receive_int(ProtocolSession* session, int* data);
|
||||||
@@ -178,10 +215,43 @@ bool send_int(int file_descriptor, int data);
|
|||||||
bool receive_int(int file_descriptor, int* data);
|
bool receive_int(int file_descriptor, int* data);
|
||||||
bool send_status(int file_descriptor, Status status);
|
bool send_status(int file_descriptor, Status status);
|
||||||
bool receive_status(int file_descriptor, Status* status);
|
bool receive_status(int file_descriptor, Status* status);
|
||||||
|
/* Send STATUS_ERROR_DETAIL followed by a bounded (<= MAX_ERROR_DETAIL_BYTES)
|
||||||
|
* length-prefixed string. Over-long messages are sliced and NULL is treated
|
||||||
|
* as "". Returns false if the status or the string could not be sent. */
|
||||||
|
bool send_error_detail(int file_descriptor, const char* message);
|
||||||
|
/* Human-readable reason captured from the most recent STATUS_ERROR_DETAIL
|
||||||
|
* received on this thread, or "" when the last status was a bare STATUS_ERROR
|
||||||
|
* (or no detail was seen). Thread-local, and valid until the next non-keepalive
|
||||||
|
* status read on the same thread; a later STATUS_KEEPALIVE does NOT clear it.
|
||||||
|
* The detail body is bounded by MAX_ERROR_DETAIL_BYTES: an over-cap declared
|
||||||
|
* length is drained and yields "" (so the stream never desyncs), while an
|
||||||
|
* absurd length is a fatal framing error that fails the status read. */
|
||||||
|
const char* protocol_last_error(void);
|
||||||
|
/* Clear the thread-local last-error buffer. */
|
||||||
|
void protocol_clear_last_error(void);
|
||||||
/* receive_status with an explicit per-message deadline in seconds, instead of
|
/* receive_status with an explicit per-message deadline in seconds, instead of
|
||||||
the default RECEIVE_TIMEOUT_SEC. A reply that may legitimately take longer
|
the default RECEIVE_TIMEOUT_SEC. A reply that may legitimately take longer
|
||||||
(e.g. the early-delete ACK after a large receiver-side deletion) must use
|
(e.g. the early-delete ACK after a large receiver-side deletion) must use
|
||||||
this so the sender does not abort after the deletion already committed. */
|
this so the sender does not abort after the deletion already committed. */
|
||||||
bool receive_status_timed(int file_descriptor, Status* status, int timeout_sec);
|
bool receive_status_timed(int file_descriptor, Status* status, int timeout_sec);
|
||||||
|
|
||||||
|
/* Callback polled by protocol_receive_status_keepalive once per keepalive
|
||||||
|
interval. Return true to stop waiting (e.g. a SIGINT/SIGTERM abort flag was
|
||||||
|
set). Kept as a function pointer so the protocol layer does not depend on
|
||||||
|
client signal state. */
|
||||||
|
typedef bool (*ProtocolWaitAbort)(void);
|
||||||
|
|
||||||
|
/* Like receive_status_timed, but while the peer is silent it emits
|
||||||
|
STATUS_KEEPALIVE every keepalive_interval_sec (the receiver answers each with
|
||||||
|
STATUS_KEEPALIVE, which this function consumes and skips) so a long
|
||||||
|
server-side operation does not look like a dead connection. The total wait
|
||||||
|
is still bounded by timeout_sec; abort_check (may be NULL) is polled every
|
||||||
|
interval and, when it returns true, ends the wait immediately with false.
|
||||||
|
Runs entirely on the calling thread: the protocol send path is NOT safe for
|
||||||
|
concurrent writers, so this must not be paired with a helper thread. */
|
||||||
|
bool receive_status_keepalive(int file_descriptor, Status* status, int timeout_sec,
|
||||||
|
int keepalive_interval_sec, ProtocolWaitAbort abort_check);
|
||||||
|
bool protocol_receive_status_keepalive(ProtocolSession* session, Status* status, int timeout_sec,
|
||||||
|
int keepalive_interval_sec, ProtocolWaitAbort abort_check);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
@@ -75,6 +75,15 @@ static int parse_remote_dest(const char* dest, RemoteDest* r) {
|
|||||||
memcpy(r->host, dest, host_len);
|
memcpy(r->host, dest, host_len);
|
||||||
r->host[host_len] = '\0';
|
r->host[host_len] = '\0';
|
||||||
}
|
}
|
||||||
|
/* The user@host token is handed to ssh in option position. Reject anything
|
||||||
|
* that ssh would consume as an option (a leading '-') or an empty host, so a
|
||||||
|
* crafted destination can never inject an ssh option such as
|
||||||
|
* -oProxyCommand=... . This mirrors config_parse_ssh_dest's validation and
|
||||||
|
* is defense-in-depth for callers that bypass it. */
|
||||||
|
if (r->host[0] == '\0' || r->host[0] == '-' || r->user[0] == '-') {
|
||||||
|
remote_dest_destroy(r);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -128,7 +137,7 @@ char* ssh_build_remote_command(const char* server_path, bool old_args, char* con
|
|||||||
q++;
|
q++;
|
||||||
len++;
|
len++;
|
||||||
}
|
}
|
||||||
if (len > SIZE_MAX - q * 3 || len + q * 3 + 3 > SIZE_MAX - command_len)
|
if (q > (SIZE_MAX - len) / 3 || len + q * 3 + 3 > SIZE_MAX - command_len)
|
||||||
return NULL;
|
return NULL;
|
||||||
command_len += len + q * 3 + 3;
|
command_len += len + q * 3 + 3;
|
||||||
}
|
}
|
||||||
@@ -216,10 +225,10 @@ char** ssh_build_client_argv(const char* rsh_command, int port, const char* user
|
|||||||
nwords = 1;
|
nwords = 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Fixed tail: three -o pairs (6) + optional -p/value (2) + user@host +
|
/* Fixed tail: three -o pairs (6) + optional -p/value (2) + the "--" end of
|
||||||
* remote command + terminating NULL. */
|
* options marker + user@host + remote command + terminating NULL. */
|
||||||
int port_extra = (port > 0 && port != 22) ? 2 : 0;
|
int port_extra = (port > 0 && port != 22) ? 2 : 0;
|
||||||
size_t total = (size_t)nwords + 6 + (size_t)port_extra + 3;
|
size_t total = (size_t)nwords + 6 + (size_t)port_extra + 4;
|
||||||
char** argv = calloc(total, sizeof(char*));
|
char** argv = calloc(total, sizeof(char*));
|
||||||
if (!argv) {
|
if (!argv) {
|
||||||
for (int i = 0; i < nwords; i++)
|
for (int i = 0; i < nwords; i++)
|
||||||
@@ -253,6 +262,13 @@ char** ssh_build_client_argv(const char* rsh_command, int port, const char* user
|
|||||||
goto fail_argv;
|
goto fail_argv;
|
||||||
ac++;
|
ac++;
|
||||||
}
|
}
|
||||||
|
/* End of options: guarantees the user@host token that follows is treated as
|
||||||
|
* the destination and never re-interpreted as an ssh option, even if every
|
||||||
|
* caller-side validation were bypassed. */
|
||||||
|
argv[ac] = str_dup("--");
|
||||||
|
if (!argv[ac])
|
||||||
|
goto fail_argv;
|
||||||
|
ac++;
|
||||||
argv[ac] = str_dup(userhost);
|
argv[ac] = str_dup(userhost);
|
||||||
if (!argv[ac])
|
if (!argv[ac])
|
||||||
goto fail_argv;
|
goto fail_argv;
|
||||||
|
|||||||
+127
-11
@@ -1,4 +1,5 @@
|
|||||||
#include "transport_tcp.h"
|
#include "transport_tcp.h"
|
||||||
|
#include "daemon_limits.h"
|
||||||
#include "log.h"
|
#include "log.h"
|
||||||
#include "protocol.h"
|
#include "protocol.h"
|
||||||
#include "utils.h"
|
#include "utils.h"
|
||||||
@@ -8,6 +9,7 @@
|
|||||||
#include <netinet/in.h>
|
#include <netinet/in.h>
|
||||||
#include <netinet/tcp.h>
|
#include <netinet/tcp.h>
|
||||||
#include <openssl/ssl.h>
|
#include <openssl/ssl.h>
|
||||||
|
#include <pthread.h>
|
||||||
#include <signal.h>
|
#include <signal.h>
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
@@ -18,18 +20,45 @@
|
|||||||
|
|
||||||
static volatile sig_atomic_t g_active_connections = 0;
|
static volatile sig_atomic_t g_active_connections = 0;
|
||||||
|
|
||||||
|
/* Shared registry installed on the active server; the SIGCHLD handler needs a
|
||||||
|
* file-scope pointer so it can reclaim the dead child's slot. Set once by
|
||||||
|
* accept_loop before the fork loop (single-threaded parent). */
|
||||||
|
static DaemonLimitRegistry* g_limit_registry = NULL;
|
||||||
|
/* Slot reserved by the parent for the connection child currently being forked.
|
||||||
|
* Written before fork(), read by the child (which inherits the value). */
|
||||||
|
static int g_current_slot = DAEMON_LIMITS_NO_SLOT;
|
||||||
|
|
||||||
static void tcp_apply_socket_timeout(int fd);
|
static void tcp_apply_socket_timeout(int fd);
|
||||||
|
static void tcp_enable_nodelay_default(int fd, int family);
|
||||||
|
|
||||||
static void sigchld_handler(int sig) {
|
static void sigchld_handler(int sig) {
|
||||||
(void)sig;
|
(void)sig;
|
||||||
int saved_errno = errno;
|
int saved_errno = errno;
|
||||||
while (waitpid(-1, NULL, WNOHANG) > 0) {
|
pid_t pid;
|
||||||
|
while ((pid = waitpid(-1, NULL, WNOHANG)) > 0) {
|
||||||
if (g_active_connections > 0)
|
if (g_active_connections > 0)
|
||||||
g_active_connections--;
|
g_active_connections--;
|
||||||
|
daemon_limits_reclaim_pid(g_limit_registry, (long)pid);
|
||||||
}
|
}
|
||||||
|
/* Re-derive the occupancy counters once for the whole reap batch. The slot
|
||||||
|
* table is the source of truth, so this self-heals any count leaked by a child
|
||||||
|
* SIGKILLed mid-registration. Atomics only: async-signal-safe. */
|
||||||
|
if (g_limit_registry)
|
||||||
|
daemon_limits_recompute(g_limit_registry);
|
||||||
errno = saved_errno;
|
errno = saved_errno;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Reset a signal to its default action with sigaction (preferred over
|
||||||
|
* signal(3), whose semantics are implementation-defined). Used in the forked
|
||||||
|
* child before it can spawn any thread. */
|
||||||
|
static void reset_signal_default(int sig) {
|
||||||
|
struct sigaction action;
|
||||||
|
memset(&action, 0, sizeof(action));
|
||||||
|
action.sa_handler = SIG_DFL;
|
||||||
|
sigemptyset(&action.sa_mask);
|
||||||
|
sigaction(sig, &action, NULL);
|
||||||
|
}
|
||||||
|
|
||||||
/* Map a listen socket's address to its numeric port for logging, independent
|
/* Map a listen socket's address to its numeric port for logging, independent
|
||||||
* of whether it is an IPv4 or IPv6 sockaddr. */
|
* of whether it is an IPv4 or IPv6 sockaddr. */
|
||||||
static unsigned short server_address_port(const struct sockaddr_storage* addr) {
|
static unsigned short server_address_port(const struct sockaddr_storage* addr) {
|
||||||
@@ -107,6 +136,7 @@ Server* server_create_ex(int port, const ServerBindOptions* bind_opts) {
|
|||||||
server->ssl_ctx = NULL;
|
server->ssl_ctx = NULL;
|
||||||
server->max_connections = 100;
|
server->max_connections = 100;
|
||||||
server->active_connections = 0;
|
server->active_connections = 0;
|
||||||
|
server->limit_registry = NULL;
|
||||||
|
|
||||||
return server;
|
return server;
|
||||||
}
|
}
|
||||||
@@ -115,6 +145,20 @@ Server* server_create(int port) {
|
|||||||
return server_create_ex(port, NULL);
|
return server_create_ex(port, NULL);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void server_set_max_connections(Server* server, unsigned int max_connections) {
|
||||||
|
if (server && max_connections > 0)
|
||||||
|
server->max_connections = max_connections;
|
||||||
|
}
|
||||||
|
|
||||||
|
void server_set_limit_registry(Server* server, struct DaemonLimitRegistry* registry) {
|
||||||
|
if (server)
|
||||||
|
server->limit_registry = registry;
|
||||||
|
}
|
||||||
|
|
||||||
|
int transport_tcp_current_slot(void) {
|
||||||
|
return g_current_slot;
|
||||||
|
}
|
||||||
|
|
||||||
void server_delete(Server** server) {
|
void server_delete(Server** server) {
|
||||||
if (server == NULL || *server == NULL)
|
if (server == NULL || *server == NULL)
|
||||||
return;
|
return;
|
||||||
@@ -133,9 +177,19 @@ static void accept_loop(Server* server, void (*child_fn)(int, void*), void* chil
|
|||||||
log_perror("Could not listen on port!");
|
log_perror("Could not listen on port!");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
signal(SIGCHLD, sigchld_handler);
|
/* SIGCHLD via sigaction (not signal(3)); SA_RESTART keeps accept(2) from
|
||||||
|
* failing with EINTR, and SA_NOCLDSTOP only notifies on child exit. The
|
||||||
|
* accept loop is single-threaded at this point, so installing here cannot race
|
||||||
|
* a worker thread. */
|
||||||
|
struct sigaction chld_action;
|
||||||
|
memset(&chld_action, 0, sizeof(chld_action));
|
||||||
|
chld_action.sa_handler = sigchld_handler;
|
||||||
|
sigemptyset(&chld_action.sa_mask);
|
||||||
|
chld_action.sa_flags = SA_RESTART | SA_NOCLDSTOP;
|
||||||
|
sigaction(SIGCHLD, &chld_action, NULL);
|
||||||
|
g_limit_registry = server->limit_registry;
|
||||||
while (1) {
|
while (1) {
|
||||||
struct sockaddr_in client_addr;
|
struct sockaddr_storage client_addr;
|
||||||
socklen_t client_len = sizeof(client_addr);
|
socklen_t client_len = sizeof(client_addr);
|
||||||
int fd = accept(server->file_descriptor, (struct sockaddr*)&client_addr, &client_len);
|
int fd = accept(server->file_descriptor, (struct sockaddr*)&client_addr, &client_len);
|
||||||
if (fd < 0) {
|
if (fd < 0) {
|
||||||
@@ -143,22 +197,69 @@ static void accept_loop(Server* server, void (*child_fn)(int, void*), void* chil
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
tcp_apply_socket_timeout(fd);
|
tcp_apply_socket_timeout(fd);
|
||||||
|
tcp_enable_nodelay_default(fd, client_addr.ss_family);
|
||||||
|
char peer[128];
|
||||||
|
if (!utils_sockaddr_to_string((const struct sockaddr*)&client_addr, peer, sizeof(peer)))
|
||||||
|
snprintf(peer, sizeof(peer), "unknown");
|
||||||
if ((unsigned int)g_active_connections >= server->max_connections) {
|
if ((unsigned int)g_active_connections >= server->max_connections) {
|
||||||
log_message(LOG_LEVEL_WARNING, "Max connections (%u) reached, rejecting",
|
log_message(LOG_LEVEL_WARNING, "Max connections (%u) reached, rejecting %s",
|
||||||
server->max_connections);
|
server->max_connections, peer);
|
||||||
close(fd);
|
close(fd);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
log_message(LOG_LEVEL_INFO, "%s", log_fmt);
|
int slot = DAEMON_LIMITS_NO_SLOT;
|
||||||
|
if (server->limit_registry) {
|
||||||
|
slot = daemon_limits_claim_slot(server->limit_registry);
|
||||||
|
if (slot == DAEMON_LIMITS_NO_SLOT) {
|
||||||
|
/* The global cap bounds live children, so this only happens when the
|
||||||
|
* fixed registry is smaller than the configured cap; fail closed. */
|
||||||
|
log_message(LOG_LEVEL_WARNING, "Connection registry slots exhausted (max %u), rejecting %s",
|
||||||
|
server->max_connections, peer);
|
||||||
|
close(fd);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
log_message(LOG_LEVEL_INFO, "%s from %s", log_fmt, peer);
|
||||||
|
g_current_slot = slot;
|
||||||
|
/* Block SIGCHLD across fork() and the parent's pid publication: a child
|
||||||
|
* that exits immediately must not be reaped before its slot records its
|
||||||
|
* pid, which would leak the slot and its module/source counts. Use
|
||||||
|
* pthread_sigmask rather than sigprocmask so the behavior is well defined
|
||||||
|
* even if this process ever gains threads: the mask is per-thread, the fork
|
||||||
|
* copies only the calling thread, and the child inherits this thread's
|
||||||
|
* blocked mask until it restores `previous` below. No thread exists yet at
|
||||||
|
* this point, and none is created before the mask is restored, so the
|
||||||
|
* critical window is race-free. */
|
||||||
|
sigset_t blocked;
|
||||||
|
sigset_t previous;
|
||||||
|
sigemptyset(&blocked);
|
||||||
|
sigaddset(&blocked, SIGCHLD);
|
||||||
|
pthread_sigmask(SIG_BLOCK, &blocked, &previous);
|
||||||
pid_t pid = fork();
|
pid_t pid = fork();
|
||||||
if (pid == 0) {
|
if (pid == 0) {
|
||||||
|
pthread_sigmask(SIG_SETMASK, &previous, NULL);
|
||||||
|
/* Connection children must not run the parent's global cleanup(): it
|
||||||
|
* frees state (credentials / daemon conf) that the child's worker
|
||||||
|
* threads may still be reading and closes fd numbers the child could
|
||||||
|
* already have reused. Reset the inherited handlers so a signal
|
||||||
|
* terminates the child directly; SIGCHLD is reset too since a child
|
||||||
|
* must never reap the parent's children. This runs before the child
|
||||||
|
* spawns any thread, so it cannot race one. */
|
||||||
|
reset_signal_default(SIGINT);
|
||||||
|
reset_signal_default(SIGTERM);
|
||||||
|
reset_signal_default(SIGCHLD);
|
||||||
close(server->file_descriptor);
|
close(server->file_descriptor);
|
||||||
child_fn(fd, child_ctx);
|
child_fn(fd, child_ctx);
|
||||||
close(fd);
|
|
||||||
_exit(0);
|
_exit(0);
|
||||||
} else if (pid > 0) {
|
} else if (pid > 0) {
|
||||||
g_active_connections++;
|
g_active_connections++;
|
||||||
|
if (server->limit_registry)
|
||||||
|
daemon_limits_set_slot_pid(server->limit_registry, slot, (long)pid);
|
||||||
|
} else if (server->limit_registry) {
|
||||||
|
/* fork() failed: release the reservation so the slot is not leaked. */
|
||||||
|
daemon_limits_reclaim_slot(server->limit_registry, slot);
|
||||||
}
|
}
|
||||||
|
pthread_sigmask(SIG_SETMASK, &previous, NULL);
|
||||||
close(fd);
|
close(fd);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -169,6 +270,9 @@ struct plain_ctx {
|
|||||||
|
|
||||||
static void plain_child_fn(int fd, void* ctx) {
|
static void plain_child_fn(int fd, void* ctx) {
|
||||||
((struct plain_ctx*)ctx)->handler(fd);
|
((struct plain_ctx*)ctx)->handler(fd);
|
||||||
|
/* handler() never closes the connection fd; the child owns its single
|
||||||
|
* close here after the handler has fully torn down. */
|
||||||
|
close(fd);
|
||||||
}
|
}
|
||||||
|
|
||||||
bool server_listen(Server* server, void (*handler)(int file_descriptor)) {
|
bool server_listen(Server* server, void (*handler)(int file_descriptor)) {
|
||||||
@@ -211,6 +315,19 @@ static void tcp_apply_socket_timeout(int fd) {
|
|||||||
setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(tv));
|
setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(tv));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Enable TCP_NODELAY by default on a transfer socket: the protocol emits many
|
||||||
|
* small messages and Nagle's algorithm would otherwise coalesce/delay them.
|
||||||
|
* Best-effort only: the family guard keeps this to IP/TCP sockets, and a
|
||||||
|
* setsockopt failure is ignored. A caller-provided --sockopts TCP_NODELAY=0
|
||||||
|
* is applied afterwards on the connect path, so an explicit user choice still
|
||||||
|
* wins. */
|
||||||
|
static void tcp_enable_nodelay_default(int fd, int family) {
|
||||||
|
if (family != AF_INET && family != AF_INET6)
|
||||||
|
return;
|
||||||
|
int value = 1;
|
||||||
|
setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &value, sizeof(value));
|
||||||
|
}
|
||||||
|
|
||||||
Client* client_create() {
|
Client* client_create() {
|
||||||
Client* client = (Client*)malloc(sizeof(Client));
|
Client* client = (Client*)malloc(sizeof(Client));
|
||||||
if (client == NULL) {
|
if (client == NULL) {
|
||||||
@@ -356,6 +473,9 @@ bool tcp_connect_socket_ex(Client* client, const char* host, int port,
|
|||||||
if (client->file_descriptor < 0)
|
if (client->file_descriptor < 0)
|
||||||
continue;
|
continue;
|
||||||
|
|
||||||
|
/* Default first; a user --sockopts TCP_NODELAY=0 applied below overrides. */
|
||||||
|
tcp_enable_nodelay_default(client->file_descriptor, rp->ai_family);
|
||||||
|
|
||||||
if (opts && opts->sockopt_count > 0 &&
|
if (opts && opts->sockopt_count > 0 &&
|
||||||
!tcp_apply_sockopts(client->file_descriptor, opts->sockopts, opts->sockopt_count)) {
|
!tcp_apply_sockopts(client->file_descriptor, opts->sockopts, opts->sockopt_count)) {
|
||||||
close(client->file_descriptor);
|
close(client->file_descriptor);
|
||||||
@@ -402,10 +522,6 @@ bool tcp_connect_socket_ex(Client* client, const char* host, int port,
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool tcp_connect_socket(Client* client, const char* host, int port) {
|
|
||||||
return tcp_connect_socket_ex(client, host, port, NULL);
|
|
||||||
}
|
|
||||||
|
|
||||||
bool client_connect_ex(Client* client, const char* host, int port, const TcpConnectOptions* opts) {
|
bool client_connect_ex(Client* client, const char* host, int port, const TcpConnectOptions* opts) {
|
||||||
if (!tcp_connect_socket_ex(client, host, port, opts))
|
if (!tcp_connect_socket_ex(client, host, port, opts))
|
||||||
return false;
|
return false;
|
||||||
|
|||||||
@@ -7,6 +7,10 @@
|
|||||||
#include <stdbool.h>
|
#include <stdbool.h>
|
||||||
#include <sys/types.h>
|
#include <sys/types.h>
|
||||||
|
|
||||||
|
/* Cross-process daemon registry (daemon_limits.c). Only an opaque pointer is
|
||||||
|
* stored here so the transport layer does not depend on daemon config. */
|
||||||
|
struct DaemonLimitRegistry;
|
||||||
|
|
||||||
typedef struct Server {
|
typedef struct Server {
|
||||||
struct sockaddr_storage address;
|
struct sockaddr_storage address;
|
||||||
unsigned int address_length;
|
unsigned int address_length;
|
||||||
@@ -14,6 +18,7 @@ typedef struct Server {
|
|||||||
void* ssl_ctx;
|
void* ssl_ctx;
|
||||||
unsigned int max_connections;
|
unsigned int max_connections;
|
||||||
volatile unsigned int active_connections;
|
volatile unsigned int active_connections;
|
||||||
|
struct DaemonLimitRegistry* limit_registry;
|
||||||
} Server;
|
} Server;
|
||||||
|
|
||||||
typedef struct Client {
|
typedef struct Client {
|
||||||
@@ -45,6 +50,17 @@ typedef struct {
|
|||||||
|
|
||||||
Server* server_create_ex(int port, const ServerBindOptions* bind_opts);
|
Server* server_create_ex(int port, const ServerBindOptions* bind_opts);
|
||||||
Server* server_create(int port);
|
Server* server_create(int port);
|
||||||
|
/* Override the listener's connection cap (the global daemon `max connections`
|
||||||
|
* value). A non-positive value is ignored so the default cap stands. */
|
||||||
|
void server_set_max_connections(Server* server, unsigned int max_connections);
|
||||||
|
/* Install the shared per-module / per-source registry used by the accept loop
|
||||||
|
* to reserve a slot for each forked child. NULL disables the accounting (the
|
||||||
|
* global cap and ACLs still apply). */
|
||||||
|
void server_set_limit_registry(Server* server, struct DaemonLimitRegistry* registry);
|
||||||
|
/* Slot reserved for the connection child currently running (set by the parent
|
||||||
|
* before fork, inherited by the child). Returns DAEMON_LIMITS_NO_SLOT (-1)
|
||||||
|
* outside the accept-loop child path. */
|
||||||
|
int transport_tcp_current_slot(void);
|
||||||
bool server_listen(Server* server, void (*handler)(int file_descriptor));
|
bool server_listen(Server* server, void (*handler)(int file_descriptor));
|
||||||
void server_accept_loop(Server* server, void (*child_fn)(int, void*), void* child_ctx,
|
void server_accept_loop(Server* server, void (*child_fn)(int, void*), void* child_ctx,
|
||||||
const char* log_fmt);
|
const char* log_fmt);
|
||||||
@@ -54,7 +70,6 @@ bool client_connect_ex(Client* client, const char* host, int port, const TcpConn
|
|||||||
bool client_connect(Client* client, const char* host, int port);
|
bool client_connect(Client* client, const char* host, int port);
|
||||||
bool tcp_connect_socket_ex(Client* client, const char* host, int port,
|
bool tcp_connect_socket_ex(Client* client, const char* host, int port,
|
||||||
const TcpConnectOptions* opts);
|
const TcpConnectOptions* opts);
|
||||||
bool tcp_connect_socket(Client* client, const char* host, int port);
|
|
||||||
void client_disconnect(Client* client);
|
void client_disconnect(Client* client);
|
||||||
void client_delete(Client* client);
|
void client_delete(Client* client);
|
||||||
void tcp_set_timeouts(int timeout_sec, int contimeout_sec);
|
void tcp_set_timeouts(int timeout_sec, int contimeout_sec);
|
||||||
|
|||||||
+93
-15
@@ -4,7 +4,9 @@
|
|||||||
#include "transport_tcp.h"
|
#include "transport_tcp.h"
|
||||||
#include "utils.h"
|
#include "utils.h"
|
||||||
#include <arpa/inet.h>
|
#include <arpa/inet.h>
|
||||||
|
#include <fcntl.h>
|
||||||
#include <openssl/err.h>
|
#include <openssl/err.h>
|
||||||
|
#include <openssl/pem.h>
|
||||||
#include <openssl/ssl.h>
|
#include <openssl/ssl.h>
|
||||||
#include <signal.h>
|
#include <signal.h>
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
@@ -34,6 +36,59 @@ static void log_ssl_errors(void) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Load the TLS private key through an already-opened, no-follow descriptor so
|
||||||
|
* the owner/mode policy is checked on the SAME file object that is loaded: an
|
||||||
|
* attacker cannot swap the path between a stat() and a later open() (TOCTOU).
|
||||||
|
* The exact-owner / 0600 policy is preserved and group/other execute bits are
|
||||||
|
* rejected as well. Ownership of the descriptor passes to the BIO and is
|
||||||
|
* released exactly once by BIO_free() (BIO_CLOSE). */
|
||||||
|
static bool load_private_key_secure(SSL_CTX* ctx, const char* key) {
|
||||||
|
int fd = open(key, O_RDONLY | O_NOFOLLOW | O_CLOEXEC);
|
||||||
|
if (fd < 0) {
|
||||||
|
char* escaped = output_escape(key, false);
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Failed to open private key: %s",
|
||||||
|
escaped ? escaped : "<allocation failed>");
|
||||||
|
free(escaped);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
struct stat key_stat;
|
||||||
|
if (fstat(fd, &key_stat) != 0 || !S_ISREG(key_stat.st_mode) || key_stat.st_uid != geteuid() ||
|
||||||
|
(key_stat.st_mode & (S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH | S_IXGRP | S_IXOTH))) {
|
||||||
|
log_message(LOG_LEVEL_ERROR,
|
||||||
|
"TLS private key must be a regular file owned by the current user and private "
|
||||||
|
"(mode 0600)");
|
||||||
|
close(fd);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
BIO* bio = BIO_new_fd(fd, BIO_CLOSE);
|
||||||
|
if (!bio) {
|
||||||
|
close(fd);
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Failed to read private key");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
EVP_PKEY* pkey = PEM_read_bio_PrivateKey(bio, NULL, NULL, NULL);
|
||||||
|
BIO_free(bio); /* releases fd via BIO_CLOSE */
|
||||||
|
if (!pkey) {
|
||||||
|
char* escaped = output_escape(key, false);
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Failed to load private key: %s",
|
||||||
|
escaped ? escaped : "<allocation failed>");
|
||||||
|
free(escaped);
|
||||||
|
log_ssl_errors();
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
int use_ok = SSL_CTX_use_PrivateKey(ctx, pkey);
|
||||||
|
EVP_PKEY_free(pkey);
|
||||||
|
if (use_ok != 1) {
|
||||||
|
char* escaped = output_escape(key, false);
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Failed to use private key: %s",
|
||||||
|
escaped ? escaped : "<allocation failed>");
|
||||||
|
free(escaped);
|
||||||
|
log_ssl_errors();
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
static SSL_CTX* create_ssl_ctx(bool is_server, const char* cert, const char* key,
|
static SSL_CTX* create_ssl_ctx(bool is_server, const char* cert, const char* key,
|
||||||
const char* ca_path) {
|
const char* ca_path) {
|
||||||
if (!is_server && !ca_path) {
|
if (!is_server && !ca_path) {
|
||||||
@@ -56,24 +111,38 @@ static SSL_CTX* create_ssl_ctx(bool is_server, const char* cert, const char* key
|
|||||||
#ifdef SSL_OP_NO_RENEGOTIATION
|
#ifdef SSL_OP_NO_RENEGOTIATION
|
||||||
SSL_CTX_set_options(ctx, SSL_OP_NO_RENEGOTIATION);
|
SSL_CTX_set_options(ctx, SSL_OP_NO_RENEGOTIATION);
|
||||||
#endif
|
#endif
|
||||||
|
/* Let the server's own preference order decide the negotiated cipher rather
|
||||||
|
* than the client's, so a client cannot steer both peers into a weaker (but
|
||||||
|
* still offered) suite. */
|
||||||
|
SSL_CTX_set_options(ctx, SSL_OP_CIPHER_SERVER_PREFERENCE);
|
||||||
|
|
||||||
if (SSL_CTX_set_min_proto_version(ctx, TLS1_2_VERSION) != 1) {
|
if (SSL_CTX_set_min_proto_version(ctx, TLS1_2_VERSION) != 1) {
|
||||||
SSL_CTX_free(ctx);
|
SSL_CTX_free(ctx);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
if (SSL_CTX_set_cipher_list(ctx, "HIGH:!aNULL:!eNULL:!MD5:!RC4:!3DES") != 1) {
|
/* TLS 1.2 and below: an AEAD-only suite list. "HIGH" still includes CBC
|
||||||
|
* suites (Lucky13/POODLE-adjacent MAC-then-encrypt constructions), so restrict
|
||||||
|
* the list to ECDHE key agreement with an AEAD record cipher (AES-GCM or
|
||||||
|
* ChaCha20-Poly1305). A NULL/weak/3DES cipher is never selectable. */
|
||||||
|
if (SSL_CTX_set_cipher_list(ctx, "ECDHE+AESGCM:ECDHE+CHACHA20:!aNULL:!eNULL:!MD5:!RC4:!3DES") !=
|
||||||
|
1) {
|
||||||
SSL_CTX_free(ctx);
|
SSL_CTX_free(ctx);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
/* TLS 1.3 ciphersuites are configured separately from the TLS 1.2 and below
|
||||||
|
* cipher list above. Pin the three AEAD suites OpenSSL offers, dropping
|
||||||
|
* TLS_AES_128_CCM_SHA256 and the CCM_8 variant, and fail closed if the
|
||||||
|
* library rejects the policy. SSL_CTX_set_ciphersuites needs OpenSSL 1.1.1;
|
||||||
|
* earlier versions have no TLS 1.3, so the call is compile-guarded. */
|
||||||
|
#if OPENSSL_VERSION_NUMBER >= 0x10101000L
|
||||||
|
if (SSL_CTX_set_ciphersuites(
|
||||||
|
ctx, "TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_128_GCM_SHA256") != 1) {
|
||||||
|
SSL_CTX_free(ctx);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
if (cert && key) {
|
if (cert && key) {
|
||||||
struct stat key_stat;
|
|
||||||
if (stat(key, &key_stat) != 0 || !S_ISREG(key_stat.st_mode) || key_stat.st_uid != geteuid() ||
|
|
||||||
(key_stat.st_mode & (S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH))) {
|
|
||||||
log_message(LOG_LEVEL_ERROR, "TLS private key must be owned by the current user and private");
|
|
||||||
SSL_CTX_free(ctx);
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
if (SSL_CTX_use_certificate_file(ctx, cert, SSL_FILETYPE_PEM) <= 0) {
|
if (SSL_CTX_use_certificate_file(ctx, cert, SSL_FILETYPE_PEM) <= 0) {
|
||||||
char* escaped = output_escape(cert, false);
|
char* escaped = output_escape(cert, false);
|
||||||
log_message(LOG_LEVEL_ERROR, "Failed to load certificate: %s",
|
log_message(LOG_LEVEL_ERROR, "Failed to load certificate: %s",
|
||||||
@@ -83,12 +152,7 @@ static SSL_CTX* create_ssl_ctx(bool is_server, const char* cert, const char* key
|
|||||||
SSL_CTX_free(ctx);
|
SSL_CTX_free(ctx);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
if (SSL_CTX_use_PrivateKey_file(ctx, key, SSL_FILETYPE_PEM) <= 0) {
|
if (!load_private_key_secure(ctx, key)) {
|
||||||
char* escaped = output_escape(key, false);
|
|
||||||
log_message(LOG_LEVEL_ERROR, "Failed to load private key: %s",
|
|
||||||
escaped ? escaped : "<allocation failed>");
|
|
||||||
free(escaped);
|
|
||||||
log_ssl_errors();
|
|
||||||
SSL_CTX_free(ctx);
|
SSL_CTX_free(ctx);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
@@ -132,7 +196,16 @@ static SSL* wrap_fd_with_ssl(int fd, SSL_CTX* ctx, bool is_server, const char* h
|
|||||||
// Enable hostname verification for client connections when a hostname is provided.
|
// Enable hostname verification for client connections when a hostname is provided.
|
||||||
// Must be done before SSL_connect to take effect during the handshake.
|
// Must be done before SSL_connect to take effect during the handshake.
|
||||||
if (!is_server && hostname) {
|
if (!is_server && hostname) {
|
||||||
if (SSL_set1_host(ssl, hostname) != 1) {
|
/* An IP-literal host must be verified against the certificate's IP SAN
|
||||||
|
* (X509_check_ip_asc), not as a DNS name: SSL_set1_host would look for a
|
||||||
|
* DNS SAN that a legitimate IP-SAN certificate never carries. */
|
||||||
|
struct in_addr ipv4;
|
||||||
|
struct in6_addr ipv6;
|
||||||
|
bool is_ip_literal =
|
||||||
|
inet_pton(AF_INET, hostname, &ipv4) == 1 || inet_pton(AF_INET6, hostname, &ipv6) == 1;
|
||||||
|
int set_ok = is_ip_literal ? X509_VERIFY_PARAM_set1_ip_asc(SSL_get0_param(ssl), hostname)
|
||||||
|
: SSL_set1_host(ssl, hostname);
|
||||||
|
if (set_ok != 1) {
|
||||||
SSL_free(ssl);
|
SSL_free(ssl);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
@@ -180,13 +253,18 @@ static void tls_child_fn(int fd, void* arg) {
|
|||||||
SSL* ssl = wrap_fd_with_ssl(fd, ctx->ssl_ctx, true, NULL);
|
SSL* ssl = wrap_fd_with_ssl(fd, ctx->ssl_ctx, true, NULL);
|
||||||
if (!ssl) {
|
if (!ssl) {
|
||||||
io_set_ssl(NULL);
|
io_set_ssl(NULL);
|
||||||
|
close(fd);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
io_set_ssl(ssl);
|
io_set_ssl(ssl);
|
||||||
ctx->handler(fd);
|
ctx->handler(fd);
|
||||||
|
/* Shut the TLS layer down before releasing the fd: handler() no longer
|
||||||
|
* closes it, so SSL_shutdown still has a valid socket. The child owns the
|
||||||
|
* single fd close, performed last. */
|
||||||
SSL_shutdown(ssl);
|
SSL_shutdown(ssl);
|
||||||
SSL_free(ssl);
|
SSL_free(ssl);
|
||||||
io_set_ssl(NULL);
|
io_set_ssl(NULL);
|
||||||
|
close(fd);
|
||||||
}
|
}
|
||||||
|
|
||||||
bool server_listen_tls(Server* server, void (*handler)(int file_descriptor)) {
|
bool server_listen_tls(Server* server, void (*handler)(int file_descriptor)) {
|
||||||
|
|||||||
+505
-81
@@ -13,6 +13,7 @@
|
|||||||
#include <sys/socket.h>
|
#include <sys/socket.h>
|
||||||
#include <sys/stat.h>
|
#include <sys/stat.h>
|
||||||
#include <unistd.h>
|
#include <unistd.h>
|
||||||
|
#include <xxhash.h>
|
||||||
|
|
||||||
static int authorized_root_fd = -1;
|
static int authorized_root_fd = -1;
|
||||||
static char* authorized_root_path;
|
static char* authorized_root_path;
|
||||||
@@ -35,21 +36,41 @@ void utils_set_authorized_root_fd(int fd) {
|
|||||||
(void)utils_set_authorized_root(fd, NULL);
|
(void)utils_set_authorized_root(fd, NULL);
|
||||||
}
|
}
|
||||||
|
|
||||||
static bool path_is_within_root(const char* root, const char* path) {
|
/* Accessors for the process-global authorized root. The path pointer is
|
||||||
|
* borrowed and valid until the next setter call; the root is a single-threaded,
|
||||||
|
* set-before-worker-threads value (see server.c), so these carry no locking. */
|
||||||
|
int utils_get_authorized_root_fd(void) {
|
||||||
|
return authorized_root_fd;
|
||||||
|
}
|
||||||
|
|
||||||
|
const char* utils_get_authorized_root_path(void) {
|
||||||
|
return authorized_root_path;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool path_is_within_root(const char* root, const char* path) {
|
||||||
size_t root_len = strlen(root);
|
size_t root_len = strlen(root);
|
||||||
return strncmp(root, path, root_len) == 0 && (path[root_len] == '\0' || path[root_len] == '/');
|
return strncmp(root, path, root_len) == 0 && (path[root_len] == '\0' || path[root_len] == '/');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Open the destination root directory itself, confined to the authorized root.
|
||||||
|
* NOTE (do not merge with file_open_secure_parent): this walk opens dest_root
|
||||||
|
* (a directory that must already exist) and returns its fd, whereas
|
||||||
|
* file_open_secure_parent resolves the PARENT of a file path, optionally
|
||||||
|
* creating missing components and honouring --keep-dirlinks / --copy-as. The
|
||||||
|
* two differ in create-vs-no-create, in what path component they stop at, and
|
||||||
|
* in the extra receiver policies they apply, so they are intentionally kept
|
||||||
|
* separate. Both rely on the shared lexical path_is_within_root check. */
|
||||||
static int open_authorized_destination(const char* dest_root) {
|
static int open_authorized_destination(const char* dest_root) {
|
||||||
if (authorized_root_fd < 0 || !authorized_root_path || !dest_root ||
|
int root_fd = utils_get_authorized_root_fd();
|
||||||
!path_is_within_root(authorized_root_path, dest_root))
|
const char* root_path = utils_get_authorized_root_path();
|
||||||
|
if (root_fd < 0 || !root_path || !dest_root || !path_is_within_root(root_path, dest_root))
|
||||||
return -1;
|
return -1;
|
||||||
|
|
||||||
int dirfd = dup(authorized_root_fd);
|
int dirfd = dup(root_fd);
|
||||||
if (dirfd < 0)
|
if (dirfd < 0)
|
||||||
return -1;
|
return -1;
|
||||||
|
|
||||||
const char* relative_path = dest_root + strlen(authorized_root_path);
|
const char* relative_path = dest_root + strlen(root_path);
|
||||||
while (*relative_path == '/')
|
while (*relative_path == '/')
|
||||||
relative_path++;
|
relative_path++;
|
||||||
char* relative = str_dup(*relative_path ? relative_path : ".");
|
char* relative = str_dup(*relative_path ? relative_path : ".");
|
||||||
@@ -96,6 +117,241 @@ char* str_dup(const char* string) {
|
|||||||
return new_string;
|
return new_string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#define STR_HASH_SET_MIN_CAPACITY 16
|
||||||
|
|
||||||
|
static size_t str_hash_set_hash(const char* key, size_t len) {
|
||||||
|
return (size_t)XXH64(key, len, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Store a borrowed key. Returns 1 when a new slot was filled and 0 for a
|
||||||
|
* duplicate. */
|
||||||
|
static int str_hash_set_put(StrHashSet* set, const char* key, size_t len) {
|
||||||
|
size_t mask = set->capacity - 1;
|
||||||
|
size_t index = str_hash_set_hash(key, len) & mask;
|
||||||
|
while (true) {
|
||||||
|
StrHashSetSlot* slot = &set->slots[index];
|
||||||
|
if (!slot->key) {
|
||||||
|
slot->key = key;
|
||||||
|
set->size++;
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
if (strlen(slot->key) == len && memcmp(slot->key, key, len) == 0)
|
||||||
|
return 0;
|
||||||
|
index = (index + 1) & mask;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool str_hash_set_resize(StrHashSet* set, size_t new_capacity) {
|
||||||
|
StrHashSetSlot* old_slots = set->slots;
|
||||||
|
size_t old_capacity = set->capacity;
|
||||||
|
StrHashSetSlot* slots = calloc(new_capacity, sizeof(StrHashSetSlot));
|
||||||
|
if (!slots)
|
||||||
|
return false;
|
||||||
|
set->slots = slots;
|
||||||
|
set->capacity = new_capacity;
|
||||||
|
set->size = 0;
|
||||||
|
for (size_t i = 0; i < old_capacity; i++) {
|
||||||
|
if (old_slots[i].key)
|
||||||
|
(void)str_hash_set_put(set, old_slots[i].key, strlen(old_slots[i].key));
|
||||||
|
}
|
||||||
|
free(old_slots);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool str_hash_set_grow(StrHashSet* set) {
|
||||||
|
if (set->capacity != 0 && (set->size + 1) * 4 <= set->capacity * 3)
|
||||||
|
return true;
|
||||||
|
size_t new_capacity = set->capacity ? set->capacity * 2 : STR_HASH_SET_MIN_CAPACITY;
|
||||||
|
return str_hash_set_resize(set, new_capacity);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool str_hash_set_init(StrHashSet* set, size_t hint) {
|
||||||
|
if (!set)
|
||||||
|
return false;
|
||||||
|
set->slots = NULL;
|
||||||
|
set->capacity = 0;
|
||||||
|
set->size = 0;
|
||||||
|
size_t capacity = STR_HASH_SET_MIN_CAPACITY;
|
||||||
|
while (capacity < (hint + 1) * 2 && capacity <= SIZE_MAX / 2)
|
||||||
|
capacity *= 2;
|
||||||
|
set->slots = calloc(capacity, sizeof(StrHashSetSlot));
|
||||||
|
if (!set->slots)
|
||||||
|
return false;
|
||||||
|
set->capacity = capacity;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
void str_hash_set_free(StrHashSet* set) {
|
||||||
|
if (!set)
|
||||||
|
return;
|
||||||
|
free(set->slots);
|
||||||
|
set->slots = NULL;
|
||||||
|
set->capacity = 0;
|
||||||
|
set->size = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool str_hash_set_insert_ref(StrHashSet* set, const char* key) {
|
||||||
|
if (!set || !key)
|
||||||
|
return false;
|
||||||
|
if (!str_hash_set_grow(set))
|
||||||
|
return false;
|
||||||
|
/* put() returns 1 for a new slot and 0 for a duplicate; both are success. */
|
||||||
|
(void)str_hash_set_put(set, key, strlen(key));
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
static const StrHashSetSlot* str_hash_set_find_n(const StrHashSet* set, const char* key,
|
||||||
|
size_t len) {
|
||||||
|
if (!set || set->capacity == 0 || !key)
|
||||||
|
return NULL;
|
||||||
|
size_t mask = set->capacity - 1;
|
||||||
|
size_t index = str_hash_set_hash(key, len) & mask;
|
||||||
|
while (true) {
|
||||||
|
const StrHashSetSlot* slot = &set->slots[index];
|
||||||
|
if (!slot->key)
|
||||||
|
return NULL;
|
||||||
|
if (strlen(slot->key) == len && memcmp(slot->key, key, len) == 0)
|
||||||
|
return slot;
|
||||||
|
index = (index + 1) & mask;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
bool str_hash_set_lookup_n(const StrHashSet* set, const char* key, size_t len) {
|
||||||
|
return str_hash_set_find_n(set, key, len) != NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool str_hash_set_lookup(const StrHashSet* set, const char* key) {
|
||||||
|
if (!key)
|
||||||
|
return false;
|
||||||
|
return str_hash_set_lookup_n(set, key, strlen(key));
|
||||||
|
}
|
||||||
|
|
||||||
|
static int str_sorted_array_compare(const void* left, const void* right) {
|
||||||
|
const char* const* left_key = left;
|
||||||
|
const char* const* right_key = right;
|
||||||
|
return strcmp(*left_key, *right_key);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool str_sorted_array_build(StrSortedArray* array, const char* const* items, size_t count) {
|
||||||
|
if (!array)
|
||||||
|
return false;
|
||||||
|
array->items = NULL;
|
||||||
|
array->count = 0;
|
||||||
|
if (count == 0)
|
||||||
|
return true;
|
||||||
|
if (!items || count > SIZE_MAX / sizeof(const char*))
|
||||||
|
return false;
|
||||||
|
const char** sorted = malloc(count * sizeof(*sorted));
|
||||||
|
if (!sorted)
|
||||||
|
return false;
|
||||||
|
for (size_t i = 0; i < count; i++)
|
||||||
|
sorted[i] = items[i];
|
||||||
|
qsort(sorted, count, sizeof(*sorted), str_sorted_array_compare);
|
||||||
|
array->items = sorted;
|
||||||
|
array->count = count;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
void str_sorted_array_free(StrSortedArray* array) {
|
||||||
|
if (!array)
|
||||||
|
return;
|
||||||
|
free(array->items);
|
||||||
|
array->items = NULL;
|
||||||
|
array->count = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool str_sorted_array_contains(const StrSortedArray* array, const char* key) {
|
||||||
|
if (!array || !key || array->count == 0)
|
||||||
|
return false;
|
||||||
|
size_t lo = 0;
|
||||||
|
size_t hi = array->count;
|
||||||
|
while (lo < hi) {
|
||||||
|
size_t mid = lo + (hi - lo) / 2;
|
||||||
|
int cmp = strcmp(array->items[mid], key);
|
||||||
|
if (cmp < 0)
|
||||||
|
lo = mid + 1;
|
||||||
|
else if (cmp > 0)
|
||||||
|
hi = mid;
|
||||||
|
else
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Compare `entry` against the virtual key `key` + '/' without allocating the
|
||||||
|
* concatenation. Returns <0, 0 or >0 as `entry` sorts before, equal to, or
|
||||||
|
* after that virtual key. */
|
||||||
|
static int str_sorted_array_compare_prefix(const char* entry, const char* key, size_t key_len) {
|
||||||
|
int cmp = strncmp(entry, key, key_len);
|
||||||
|
if (cmp != 0)
|
||||||
|
return cmp;
|
||||||
|
unsigned char next = (unsigned char)entry[key_len];
|
||||||
|
if (next == '\0')
|
||||||
|
return -1; /* entry == key sorts before key + '/' */
|
||||||
|
return (int)next - (int)'/';
|
||||||
|
}
|
||||||
|
|
||||||
|
bool str_sorted_array_has_child_prefix(const StrSortedArray* array, const char* key) {
|
||||||
|
if (!array || !key || array->count == 0 || key[0] == '\0')
|
||||||
|
return false;
|
||||||
|
size_t key_len = strlen(key);
|
||||||
|
size_t lo = 0;
|
||||||
|
size_t hi = array->count;
|
||||||
|
while (lo < hi) {
|
||||||
|
size_t mid = lo + (hi - lo) / 2;
|
||||||
|
if (str_sorted_array_compare_prefix(array->items[mid], key, key_len) < 0)
|
||||||
|
lo = mid + 1;
|
||||||
|
else
|
||||||
|
hi = mid;
|
||||||
|
}
|
||||||
|
if (lo >= array->count)
|
||||||
|
return false;
|
||||||
|
const char* entry = array->items[lo];
|
||||||
|
return strncmp(entry, key, key_len) == 0 && entry[key_len] == '/';
|
||||||
|
}
|
||||||
|
|
||||||
|
bool path_index_build(PathIndex* index, const char* const* entries, size_t count) {
|
||||||
|
if (!index)
|
||||||
|
return false;
|
||||||
|
index->exact.slots = NULL;
|
||||||
|
index->exact.capacity = 0;
|
||||||
|
index->exact.size = 0;
|
||||||
|
index->sorted.items = NULL;
|
||||||
|
index->sorted.count = 0;
|
||||||
|
if (!str_hash_set_init(&index->exact, count))
|
||||||
|
return false;
|
||||||
|
if (!str_sorted_array_build(&index->sorted, entries, count)) {
|
||||||
|
str_hash_set_free(&index->exact);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
for (size_t i = 0; i < count; i++) {
|
||||||
|
if (!str_hash_set_insert_ref(&index->exact, entries[i])) {
|
||||||
|
path_index_free(index);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
void path_index_free(PathIndex* index) {
|
||||||
|
if (!index)
|
||||||
|
return;
|
||||||
|
str_hash_set_free(&index->exact);
|
||||||
|
str_sorted_array_free(&index->sorted);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool path_index_contains(const PathIndex* index, const char* path) {
|
||||||
|
return index && str_hash_set_lookup(&index->exact, path);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool path_index_contains_n(const PathIndex* index, const char* path, size_t len) {
|
||||||
|
return index && str_hash_set_lookup_n(&index->exact, path, len);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool path_index_has_descendant(const PathIndex* index, const char* path) {
|
||||||
|
return index && str_sorted_array_has_child_prefix(&index->sorted, path);
|
||||||
|
}
|
||||||
|
|
||||||
char* output_escape(const char* string, bool eight_bit_output) {
|
char* output_escape(const char* string, bool eight_bit_output) {
|
||||||
if (!string)
|
if (!string)
|
||||||
return NULL;
|
return NULL;
|
||||||
@@ -122,59 +378,155 @@ char* output_escape(const char* string, bool eight_bit_output) {
|
|||||||
return escaped;
|
return escaped;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
ssize_t utils_getdelim_bounded(FILE* stream, char** line, size_t* cap, int delim, size_t max_len) {
|
||||||
|
if (!stream || !line || !cap || max_len == 0) {
|
||||||
|
errno = EINVAL;
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
size_t limit = max_len + 1; /* content bytes plus the terminating NUL */
|
||||||
|
if (*line == NULL || *cap < 2) {
|
||||||
|
size_t initial = limit < 256 ? limit : 256;
|
||||||
|
char* buf = malloc(initial);
|
||||||
|
if (!buf)
|
||||||
|
return -1;
|
||||||
|
free(*line);
|
||||||
|
*line = buf;
|
||||||
|
*cap = initial;
|
||||||
|
}
|
||||||
|
size_t len = 0;
|
||||||
|
int c;
|
||||||
|
while ((c = getc_unlocked(stream)) != EOF) {
|
||||||
|
if (len >= max_len) {
|
||||||
|
errno = EFBIG;
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
if (len + 2 > *cap) {
|
||||||
|
size_t new_cap = *cap * 2;
|
||||||
|
if (new_cap < len + 2)
|
||||||
|
new_cap = len + 2;
|
||||||
|
if (new_cap > limit)
|
||||||
|
new_cap = limit;
|
||||||
|
char* grown = realloc(*line, new_cap);
|
||||||
|
if (!grown)
|
||||||
|
return -1;
|
||||||
|
*line = grown;
|
||||||
|
*cap = new_cap;
|
||||||
|
}
|
||||||
|
(*line)[len++] = (char)c;
|
||||||
|
if (c == delim)
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if (c == EOF && len == 0)
|
||||||
|
return 0;
|
||||||
|
(*line)[len] = '\0';
|
||||||
|
return (ssize_t)len;
|
||||||
|
}
|
||||||
|
|
||||||
/* Match a glob pattern against a string. Supported wildcards:
|
/* Match a glob pattern against a string. Supported wildcards:
|
||||||
* ? matches any single character except '/'.
|
* ? matches any single character except '/'.
|
||||||
* * matches any sequence of characters within one path component (no '/').
|
* * matches any sequence of characters within one path component (no '/').
|
||||||
* ** matches any sequence of characters, including '/' (cross-directory).
|
* ** matches any sequence of characters, including '/' (cross-directory).
|
||||||
* slash-star-star-slash is treated as a cross-directory wildcard when it appears between
|
* slash-star-star-slash is treated as a cross-directory wildcard when it appears between
|
||||||
* literals.
|
* literals.
|
||||||
*/
|
*
|
||||||
|
* The matcher is an iterative O(pattern * string) dynamic program rather than the
|
||||||
|
* original backtracking recursion: overlapping `*`/`**` wildcards made a pattern
|
||||||
|
* like `*a*a*a*...*b` run in exponential time against a long run of `a`, a CPU
|
||||||
|
* denial-of-service vector reachable from a hostile --exclude/--include pattern
|
||||||
|
* or `.rsync-filter`. The DP reasons over (pattern position, string position)
|
||||||
|
* so every state is visited once; the transitions below mirror the original
|
||||||
|
* recursion exactly. */
|
||||||
bool glob_match(const char* pattern, const char* str) {
|
bool glob_match(const char* pattern, const char* str) {
|
||||||
while (*pattern) {
|
if (!pattern || !str)
|
||||||
if (*pattern == '*') {
|
return false;
|
||||||
if (*(pattern + 1) == '*') {
|
size_t pattern_len = strlen(pattern);
|
||||||
/* globstar: match across directories */
|
size_t str_len = strlen(str);
|
||||||
pattern += 2;
|
if (pattern_len == 0)
|
||||||
if (*pattern == '\0')
|
return str_len == 0;
|
||||||
return true;
|
/* Defensive work cap: the DP is bounded by pattern*string states, but a
|
||||||
if (*pattern == '/')
|
* 64 KiB pattern against a 64 KiB path would still cost billions of steps.
|
||||||
pattern++;
|
* Treat the pattern as non-matching above the cap instead of burning CPU. */
|
||||||
while (*str) {
|
if (str_len > (SIZE_MAX / (pattern_len + 1)) - 1)
|
||||||
if (glob_match(pattern, str))
|
return false;
|
||||||
return true;
|
if ((pattern_len + 1) * (str_len + 1) > 64u * 1024u * 1024u)
|
||||||
str++;
|
return false;
|
||||||
|
|
||||||
|
size_t row_bytes = str_len + 1;
|
||||||
|
/* Rows for pattern positions i, i+1, i+2 and i+3 are live at once (the
|
||||||
|
* globstar transition can skip up to three pattern bytes). Four rotating
|
||||||
|
* rows keep memory at O(string length); a stack buffer avoids an allocation
|
||||||
|
* for the common short-leaf case. */
|
||||||
|
enum { STACK_ROW = 257 };
|
||||||
|
uint8_t stack_rows[4 * STACK_ROW];
|
||||||
|
uint8_t* rows = stack_rows;
|
||||||
|
if (row_bytes > STACK_ROW) {
|
||||||
|
rows = malloc(4 * row_bytes);
|
||||||
|
if (!rows)
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
#define GLOB_ROW(i) (rows + ((pattern_len - (i)) & 3) * row_bytes)
|
||||||
|
|
||||||
|
/* Base row: pattern position `pattern_len` matches only the string's end. */
|
||||||
|
for (size_t j = 0; j <= str_len; j++)
|
||||||
|
GLOB_ROW(pattern_len)[j] = (j == str_len) ? 1 : 0;
|
||||||
|
|
||||||
|
for (size_t i = pattern_len; i-- > 0;) {
|
||||||
|
const char pc = pattern[i];
|
||||||
|
uint8_t* cur = GLOB_ROW(i);
|
||||||
|
const uint8_t* next = GLOB_ROW(i + 1);
|
||||||
|
if (pc == '*') {
|
||||||
|
if (i + 1 < pattern_len && pattern[i + 1] == '*') {
|
||||||
|
/* Globstar: skip `**` and an optional following '/', then consume any
|
||||||
|
* (possibly empty) run of characters -- including '/'. */
|
||||||
|
size_t rest = i + 2;
|
||||||
|
if (rest < pattern_len && pattern[rest] == '/')
|
||||||
|
rest++;
|
||||||
|
const uint8_t* rest_row = GLOB_ROW(rest);
|
||||||
|
for (size_t j = str_len + 1; j-- > 0;) {
|
||||||
|
bool v = rest_row[j] != 0;
|
||||||
|
if (!v && j < str_len)
|
||||||
|
v = cur[j + 1] != 0;
|
||||||
|
cur[j] = v ? 1 : 0;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
/* Single `*`: zero characters, or one non-'/' character. */
|
||||||
|
for (size_t j = str_len + 1; j-- > 0;) {
|
||||||
|
bool v = next[j] != 0;
|
||||||
|
if (!v && j < str_len && str[j] != '/')
|
||||||
|
v = cur[j + 1] != 0;
|
||||||
|
cur[j] = v ? 1 : 0;
|
||||||
}
|
}
|
||||||
return glob_match(pattern, str);
|
|
||||||
}
|
}
|
||||||
/* single *: match within one path component */
|
} else if (pc == '?') {
|
||||||
pattern++;
|
for (size_t j = str_len + 1; j-- > 0;) {
|
||||||
while (*str && *str != '/') {
|
bool v = j < str_len && str[j] != '/' && next[j + 1] != 0;
|
||||||
if (glob_match(pattern, str))
|
cur[j] = v ? 1 : 0;
|
||||||
return true;
|
|
||||||
str++;
|
|
||||||
}
|
}
|
||||||
return glob_match(pattern, str);
|
|
||||||
} else if (*pattern == '?') {
|
|
||||||
if (!*str || *str == '/')
|
|
||||||
return false;
|
|
||||||
pattern++;
|
|
||||||
str++;
|
|
||||||
} else {
|
} else {
|
||||||
if (*pattern != *str) {
|
/* Literal: consume an equal character, or -- for a '/' immediately before
|
||||||
/* allow literal / ** / rest to match any number of directories */
|
* a globstar -- let the '/' match zero directories and continue at `**`. */
|
||||||
if (*pattern == '/' && *(pattern + 1) == '*' && *(pattern + 2) == '*') {
|
for (size_t j = str_len + 1; j-- > 0;) {
|
||||||
const char* rest = pattern + 3;
|
bool v = false;
|
||||||
if (*rest == '/')
|
if (j < str_len && str[j] == pc) {
|
||||||
|
v = next[j + 1] != 0;
|
||||||
|
} else if (pc == '/' && i + 2 < pattern_len && pattern[i + 1] == '*' &&
|
||||||
|
pattern[i + 2] == '*') {
|
||||||
|
size_t rest = i + 3;
|
||||||
|
if (rest < pattern_len && pattern[rest] == '/')
|
||||||
rest++;
|
rest++;
|
||||||
return glob_match(rest, str);
|
v = GLOB_ROW(rest)[j] != 0;
|
||||||
}
|
}
|
||||||
return false;
|
cur[j] = v ? 1 : 0;
|
||||||
}
|
}
|
||||||
pattern++;
|
|
||||||
str++;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return *str == '\0';
|
|
||||||
|
bool matched = GLOB_ROW(0)[0] != 0;
|
||||||
|
#undef GLOB_ROW
|
||||||
|
if (rows != stack_rows)
|
||||||
|
free(rows);
|
||||||
|
return matched;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool format_human_bytes(unsigned long long bytes, char* buffer, size_t buffer_size) {
|
bool format_human_bytes(unsigned long long bytes, char* buffer, size_t buffer_size) {
|
||||||
@@ -196,15 +548,23 @@ bool format_human_bytes(unsigned long long bytes, char* buffer, size_t buffer_si
|
|||||||
return written >= 0 && (size_t)written < buffer_size;
|
return written >= 0 && (size_t)written < buffer_size;
|
||||||
}
|
}
|
||||||
|
|
||||||
static bool is_dir_in_manifest(const char* rel_path, ArrayList* manifest) {
|
/* Build the keep-set index from the exact manifest entries only. A lookup of
|
||||||
size_t len = strlen(rel_path);
|
`rel` succeeds iff `rel` is a kept entry, a kept directory, or an ancestor
|
||||||
for (int i = 0; i < manifest->size; i++) {
|
directory of kept content (the old is_dir_in_manifest predicate); the sorted
|
||||||
const char* entry = (const char*)manifest->items[i];
|
view answers "is an ancestor of kept content" without materializing any
|
||||||
// Check if entry starts with rel_path + '/' or matches exactly
|
per-component prefix copy, so the index is O(manifest size) memory. */
|
||||||
if (strncmp(entry, rel_path, len) == 0 && (entry[len] == '/' || entry[len] == '\0'))
|
static bool build_keep_index(const ArrayList* manifest, PathIndex* index) {
|
||||||
return true;
|
if (!manifest || manifest->size <= 0)
|
||||||
}
|
return path_index_build(index, NULL, 0);
|
||||||
return false;
|
return path_index_build(index, (const char* const*)manifest->items, (size_t)manifest->size);
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool keep_is_dir(const PathIndex* index, const char* rel_path) {
|
||||||
|
return path_index_contains(index, rel_path) || path_index_has_descendant(index, rel_path);
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool keep_is_file(const PathIndex* index, const char* rel_path) {
|
||||||
|
return path_index_contains(index, rel_path);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* True when child_rel is, or lies below, a protected entry. A prefix "a"
|
/* True when child_rel is, or lies below, a protected entry. A prefix "a"
|
||||||
@@ -234,7 +594,7 @@ bool path_under_skip_prefix(const char* child_rel, bool at_root, const DeleteSki
|
|||||||
prefixes) mark the enclosing directory as surviving, exactly as they would
|
prefixes) mark the enclosing directory as surviving, exactly as they would
|
||||||
make a real rmdir fail with ENOTEMPTY. Stops early once *count reaches the
|
make a real rmdir fail with ENOTEMPTY. Stops early once *count reaches the
|
||||||
cap (sets *exceeds). Returns false on a traversal error. */
|
cap (sets *exceeds). Returns false on a traversal error. */
|
||||||
static bool count_extras_fd(int dirfd, const char* rel_path, ArrayList* manifest, size_t cap,
|
static bool count_extras_fd(int dirfd, const char* rel_path, const PathIndex* keep, size_t cap,
|
||||||
size_t* count, bool* exceeds, const DeleteSkipEntry* skips,
|
size_t* count, bool* exceeds, const DeleteSkipEntry* skips,
|
||||||
int skip_count, bool* survives) {
|
int skip_count, bool* survives) {
|
||||||
/* openat(dirfd, ".") opens an independent file description: a dup() would
|
/* openat(dirfd, ".") opens an independent file description: a dup() would
|
||||||
@@ -284,15 +644,15 @@ static bool count_extras_fd(int dirfd, const char* rel_path, ArrayList* manifest
|
|||||||
bool child_ok = true;
|
bool child_ok = true;
|
||||||
bool child_survives = true;
|
bool child_survives = true;
|
||||||
if (childfd >= 0) {
|
if (childfd >= 0) {
|
||||||
child_ok = count_extras_fd(childfd, child_rel, manifest, cap, count, exceeds, skips,
|
child_ok = count_extras_fd(childfd, child_rel, keep, cap, count, exceeds, skips, skip_count,
|
||||||
skip_count, &child_survives);
|
&child_survives);
|
||||||
close(childfd);
|
close(childfd);
|
||||||
} else if (errno != ENOENT) {
|
} else if (errno != ENOENT) {
|
||||||
operation_ok = false;
|
operation_ok = false;
|
||||||
}
|
}
|
||||||
if (!child_ok)
|
if (!child_ok)
|
||||||
operation_ok = false;
|
operation_ok = false;
|
||||||
if (is_dir_in_manifest(child_rel, manifest)) {
|
if (keep_is_dir(keep, child_rel)) {
|
||||||
/* A directory with kept content below it is never removed. */
|
/* A directory with kept content below it is never removed. */
|
||||||
local_survives = true;
|
local_survives = true;
|
||||||
} else if (child_survives) {
|
} else if (child_survives) {
|
||||||
@@ -308,13 +668,7 @@ static bool count_extras_fd(int dirfd, const char* rel_path, ArrayList* manifest
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
bool found = false;
|
bool found = keep_is_file(keep, child_rel);
|
||||||
for (int i = 0; i < manifest->size; i++) {
|
|
||||||
if (strcmp((char*)manifest->items[i], child_rel) == 0) {
|
|
||||||
found = true;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (!found) {
|
if (!found) {
|
||||||
if (*count >= cap) {
|
if (*count >= cap) {
|
||||||
*exceeds = true;
|
*exceeds = true;
|
||||||
@@ -330,7 +684,7 @@ static bool count_extras_fd(int dirfd, const char* rel_path, ArrayList* manifest
|
|||||||
return operation_ok;
|
return operation_ok;
|
||||||
}
|
}
|
||||||
|
|
||||||
static bool delete_extras_fd(int dirfd, const char* rel_path, ArrayList* manifest,
|
static bool delete_extras_fd(int dirfd, const char* rel_path, const PathIndex* keep,
|
||||||
size_t max_delete, size_t* deleted_count, const DeleteSkipEntry* skips,
|
size_t max_delete, size_t* deleted_count, const DeleteSkipEntry* skips,
|
||||||
int skip_count) {
|
int skip_count) {
|
||||||
/* Independent file description (see count_extras_fd). */
|
/* Independent file description (see count_extras_fd). */
|
||||||
@@ -379,15 +733,15 @@ static bool delete_extras_fd(int dirfd, const char* rel_path, ArrayList* manifes
|
|||||||
int childfd = openat(dirfd, entry->d_name, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
int childfd = openat(dirfd, entry->d_name, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||||
bool child_removed = false;
|
bool child_removed = false;
|
||||||
if (childfd >= 0) {
|
if (childfd >= 0) {
|
||||||
child_removed = delete_extras_fd(childfd, child_rel, manifest, max_delete, deleted_count,
|
child_removed = delete_extras_fd(childfd, child_rel, keep, max_delete, deleted_count, skips,
|
||||||
skips, skip_count);
|
skip_count);
|
||||||
if (!child_removed)
|
if (!child_removed)
|
||||||
operation_ok = false;
|
operation_ok = false;
|
||||||
close(childfd);
|
close(childfd);
|
||||||
} else if (errno != ENOENT) {
|
} else if (errno != ENOENT) {
|
||||||
operation_ok = false;
|
operation_ok = false;
|
||||||
}
|
}
|
||||||
if (child_removed && !is_dir_in_manifest(child_rel, manifest)) {
|
if (child_removed && !keep_is_dir(keep, child_rel)) {
|
||||||
if (*deleted_count >= max_delete) {
|
if (*deleted_count >= max_delete) {
|
||||||
operation_ok = false;
|
operation_ok = false;
|
||||||
} else {
|
} else {
|
||||||
@@ -406,13 +760,7 @@ static bool delete_extras_fd(int dirfd, const char* rel_path, ArrayList* manifes
|
|||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// Check if relative path is in manifest
|
// Check if relative path is in manifest
|
||||||
bool found = false;
|
bool found = keep_is_file(keep, child_rel);
|
||||||
for (int i = 0; i < manifest->size; i++) {
|
|
||||||
if (strcmp((char*)manifest->items[i], child_rel) == 0) {
|
|
||||||
found = true;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (!found) {
|
if (!found) {
|
||||||
if (*deleted_count >= max_delete) {
|
if (*deleted_count >= max_delete) {
|
||||||
operation_ok = false;
|
operation_ok = false;
|
||||||
@@ -436,53 +784,64 @@ static bool delete_extras_fd(int dirfd, const char* rel_path, ArrayList* manifes
|
|||||||
return operation_ok;
|
return operation_ok;
|
||||||
}
|
}
|
||||||
|
|
||||||
DeleteWalkResult delete_extras_limited(const char* dest_root, ArrayList* manifest,
|
DeleteWalkResult delete_extras_limited(const char* dest_root, const ArrayList* manifest,
|
||||||
size_t max_delete, const DeleteSkipEntry* skips,
|
size_t max_delete, const DeleteSkipEntry* skips,
|
||||||
int skip_count, size_t* deleted_out) {
|
int skip_count, size_t* deleted_out) {
|
||||||
if (deleted_out)
|
if (deleted_out)
|
||||||
*deleted_out = 0;
|
*deleted_out = 0;
|
||||||
if (!manifest)
|
if (!manifest)
|
||||||
return DELETE_WALK_ERROR;
|
return DELETE_WALK_ERROR;
|
||||||
|
/* Index the keep-set once so both passes answer membership in O(path length)
|
||||||
|
instead of scanning every manifest entry for every destination entry. */
|
||||||
|
PathIndex keep;
|
||||||
|
if (!build_keep_index(manifest, &keep))
|
||||||
|
return DELETE_WALK_ERROR;
|
||||||
int rootfd;
|
int rootfd;
|
||||||
if (authorized_root_fd >= 0) {
|
int root_fd = utils_get_authorized_root_fd();
|
||||||
if (authorized_root_path)
|
if (root_fd >= 0) {
|
||||||
|
if (utils_get_authorized_root_path())
|
||||||
rootfd = open_authorized_destination(dest_root);
|
rootfd = open_authorized_destination(dest_root);
|
||||||
else if (dest_root == NULL)
|
else if (dest_root == NULL)
|
||||||
rootfd = dup(authorized_root_fd);
|
rootfd = dup(root_fd);
|
||||||
else
|
else
|
||||||
rootfd = -1;
|
rootfd = -1;
|
||||||
} else {
|
} else {
|
||||||
rootfd = open(dest_root, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
rootfd = open(dest_root, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||||
}
|
}
|
||||||
if (rootfd < 0)
|
if (rootfd < 0) {
|
||||||
|
path_index_free(&keep);
|
||||||
return DELETE_WALK_ERROR;
|
return DELETE_WALK_ERROR;
|
||||||
|
}
|
||||||
if (max_delete != SIZE_MAX) {
|
if (max_delete != SIZE_MAX) {
|
||||||
/* Rehearse the deletion first so a run that would exceed the cap removes
|
/* Rehearse the deletion first so a run that would exceed the cap removes
|
||||||
nothing (rsync's all-or-nothing --max-delete contract). */
|
nothing (rsync's all-or-nothing --max-delete contract). */
|
||||||
size_t count = 0;
|
size_t count = 0;
|
||||||
bool exceeds = false;
|
bool exceeds = false;
|
||||||
bool survives = false;
|
bool survives = false;
|
||||||
bool counted_ok = count_extras_fd(rootfd, "", manifest, max_delete, &count, &exceeds, skips,
|
bool counted_ok = count_extras_fd(rootfd, "", &keep, max_delete, &count, &exceeds, skips,
|
||||||
skip_count, &survives);
|
skip_count, &survives);
|
||||||
if (!counted_ok) {
|
if (!counted_ok) {
|
||||||
close(rootfd);
|
close(rootfd);
|
||||||
|
path_index_free(&keep);
|
||||||
return DELETE_WALK_ERROR;
|
return DELETE_WALK_ERROR;
|
||||||
}
|
}
|
||||||
if (exceeds) {
|
if (exceeds) {
|
||||||
close(rootfd);
|
close(rootfd);
|
||||||
|
path_index_free(&keep);
|
||||||
return DELETE_WALK_LIMIT_EXCEEDED;
|
return DELETE_WALK_LIMIT_EXCEEDED;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
size_t deleted_count = 0;
|
size_t deleted_count = 0;
|
||||||
bool ok = delete_extras_fd(rootfd, "", manifest, max_delete, &deleted_count, skips, skip_count);
|
bool ok = delete_extras_fd(rootfd, "", &keep, max_delete, &deleted_count, skips, skip_count);
|
||||||
if (close(rootfd) != 0)
|
if (close(rootfd) != 0)
|
||||||
ok = false;
|
ok = false;
|
||||||
|
path_index_free(&keep);
|
||||||
if (deleted_out)
|
if (deleted_out)
|
||||||
*deleted_out = deleted_count;
|
*deleted_out = deleted_count;
|
||||||
return ok ? DELETE_WALK_OK : DELETE_WALK_ERROR;
|
return ok ? DELETE_WALK_OK : DELETE_WALK_ERROR;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool delete_extras(const char* dest_root, ArrayList* manifest) {
|
bool delete_extras(const char* dest_root, const ArrayList* manifest) {
|
||||||
return delete_extras_limited(dest_root, manifest, SIZE_MAX, NULL, 0, NULL) == DELETE_WALK_OK;
|
return delete_extras_limited(dest_root, manifest, SIZE_MAX, NULL, 0, NULL) == DELETE_WALK_OK;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -587,6 +946,71 @@ bool utils_fd_peer_is_local(int fd) {
|
|||||||
return utils_sockaddr_is_loopback((const struct sockaddr*)&peer);
|
return utils_sockaddr_is_loopback((const struct sockaddr*)&peer);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Numeric peer address of a connected fd. Only AF_INET/AF_INET6 peers are
|
||||||
|
formatted; every other descriptor/family (pipe, AF_UNIX socketpair, ...) or a
|
||||||
|
getpeername failure returns false with buf emptied. The caller must treat
|
||||||
|
that as "cannot tell". */
|
||||||
|
bool utils_fd_peer_ip(int fd, char* buf, size_t len) {
|
||||||
|
if (!buf || len == 0)
|
||||||
|
return false;
|
||||||
|
buf[0] = '\0';
|
||||||
|
if (fd < 0)
|
||||||
|
return false;
|
||||||
|
struct sockaddr_storage peer;
|
||||||
|
socklen_t peer_len = sizeof(peer);
|
||||||
|
if (getpeername(fd, (struct sockaddr*)&peer, &peer_len) != 0)
|
||||||
|
return false;
|
||||||
|
const void* src = NULL;
|
||||||
|
int family = peer.ss_family;
|
||||||
|
if (family == AF_INET) {
|
||||||
|
src = &((const struct sockaddr_in*)&peer)->sin_addr;
|
||||||
|
} else if (family == AF_INET6) {
|
||||||
|
const struct sockaddr_in6* peer6 = (const struct sockaddr_in6*)&peer;
|
||||||
|
/* A dual-stack IPv6 listener reports IPv4 peers as ::ffff:a.b.c.d. Emit
|
||||||
|
* the IPv4 form so IPv4 ACL patterns (and logs) see the real address. */
|
||||||
|
if (IN6_IS_ADDR_V4MAPPED(&peer6->sin6_addr)) {
|
||||||
|
struct in_addr v4;
|
||||||
|
memcpy(&v4, &peer6->sin6_addr.s6_addr[12], sizeof(v4));
|
||||||
|
return inet_ntop(AF_INET, &v4, buf, (socklen_t)len) != NULL;
|
||||||
|
}
|
||||||
|
src = &peer6->sin6_addr;
|
||||||
|
} else {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return inet_ntop(family, src, buf, (socklen_t)len) != NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* "ip:port" / "[ip]:port" for a connected peer, used to log the connecting
|
||||||
|
address in the accept loop. Returns false for a non-INET family. */
|
||||||
|
bool utils_sockaddr_to_string(const struct sockaddr* addr, char* buf, size_t len) {
|
||||||
|
if (!addr || !buf || len == 0)
|
||||||
|
return false;
|
||||||
|
buf[0] = '\0';
|
||||||
|
char ip[INET6_ADDRSTRLEN];
|
||||||
|
unsigned short port;
|
||||||
|
int written;
|
||||||
|
if (addr->sa_family == AF_INET) {
|
||||||
|
const struct sockaddr_in* v4 = (const struct sockaddr_in*)addr;
|
||||||
|
if (!inet_ntop(AF_INET, &v4->sin_addr, ip, sizeof(ip)))
|
||||||
|
return false;
|
||||||
|
port = ntohs(v4->sin_port);
|
||||||
|
written = snprintf(buf, len, "%s:%u", ip, port);
|
||||||
|
} else if (addr->sa_family == AF_INET6) {
|
||||||
|
const struct sockaddr_in6* v6 = (const struct sockaddr_in6*)addr;
|
||||||
|
if (!inet_ntop(AF_INET6, &v6->sin6_addr, ip, sizeof(ip)))
|
||||||
|
return false;
|
||||||
|
port = ntohs(v6->sin6_port);
|
||||||
|
written = snprintf(buf, len, "[%s]:%u", ip, port);
|
||||||
|
} else {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (written < 0 || (size_t)written >= len) {
|
||||||
|
buf[0] = '\0';
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
/* True when a client-supplied host string names a loopback destination:
|
/* True when a client-supplied host string names a loopback destination:
|
||||||
"localhost", any 127.0.0.0/8 literal, "::1", or "[::1]". */
|
"localhost", any 127.0.0.0/8 literal, "::1", or "[::1]". */
|
||||||
bool utils_host_is_loopback(const char* host) {
|
bool utils_host_is_loopback(const char* host) {
|
||||||
|
|||||||
+121
-2
@@ -4,10 +4,94 @@
|
|||||||
#include "array_list.h"
|
#include "array_list.h"
|
||||||
#include <stddef.h>
|
#include <stddef.h>
|
||||||
#include <stdbool.h>
|
#include <stdbool.h>
|
||||||
|
#include <stdio.h>
|
||||||
#include <sys/socket.h>
|
#include <sys/socket.h>
|
||||||
|
#include <sys/types.h>
|
||||||
|
|
||||||
|
/* Small open-addressing string hash set used to turn quadratic membership
|
||||||
|
* scans into O(path length) exact-match lookups (the --delete keep-set and the
|
||||||
|
* --files-from allow-set). Keys are hashed with xxHash64 (seed 0); collisions
|
||||||
|
* are resolved by linear probing over a power-of-two table that grows at 75%
|
||||||
|
* load. Keys are always borrowed from the caller and must outlive the set; the
|
||||||
|
* set never copies or owns keys, so indexing M entries costs O(M) memory. The
|
||||||
|
* set is not thread-safe for mutation, but a fully built set supports
|
||||||
|
* concurrent read-only lookups. */
|
||||||
|
typedef struct {
|
||||||
|
const char* key; /* NULL marks an empty slot */
|
||||||
|
} StrHashSetSlot;
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
StrHashSetSlot* slots;
|
||||||
|
size_t capacity; /* power of two, zero before init */
|
||||||
|
size_t size;
|
||||||
|
} StrHashSet;
|
||||||
|
|
||||||
|
/* Initialize an empty set sized for roughly `hint` entries. Returns false on
|
||||||
|
* allocation failure. */
|
||||||
|
bool str_hash_set_init(StrHashSet* set, size_t hint);
|
||||||
|
void str_hash_set_free(StrHashSet* set);
|
||||||
|
/* Insert a borrowed key (must outlive the set). A duplicate is ignored.
|
||||||
|
* Returns false on allocation failure. */
|
||||||
|
bool str_hash_set_insert_ref(StrHashSet* set, const char* key);
|
||||||
|
/* Look up a NUL-terminated key / a key of `len` bytes. */
|
||||||
|
bool str_hash_set_lookup(const StrHashSet* set, const char* key);
|
||||||
|
bool str_hash_set_lookup_n(const StrHashSet* set, const char* key, size_t len);
|
||||||
|
|
||||||
|
/* Sorted, non-owning view of NUL-terminated strings. Built from borrowed
|
||||||
|
* pointers (qsort), so indexing M entries costs O(M) memory and O(M log M)
|
||||||
|
* time; exact membership and ancestor-prefix existence are binary searches
|
||||||
|
* that never materialize a prefix copy. */
|
||||||
|
typedef struct {
|
||||||
|
const char** items; /* sorted with strcmp; borrowed, never freed */
|
||||||
|
size_t count;
|
||||||
|
} StrSortedArray;
|
||||||
|
|
||||||
|
/* Build `array` over the borrowed `items`. Only the pointer array is copied,
|
||||||
|
* never the strings. Returns false on allocation failure. */
|
||||||
|
bool str_sorted_array_build(StrSortedArray* array, const char* const* items, size_t count);
|
||||||
|
void str_sorted_array_free(StrSortedArray* array);
|
||||||
|
/* True when some item equals `key`. */
|
||||||
|
bool str_sorted_array_contains(const StrSortedArray* array, const char* key);
|
||||||
|
/* True when some item starts with `key` followed by '/' (i.e. `key` is a proper
|
||||||
|
* ancestor directory of an item). Allocates nothing. */
|
||||||
|
bool str_sorted_array_has_child_prefix(const StrSortedArray* array, const char* key);
|
||||||
|
|
||||||
|
/* Read-only membership index over exact relative paths. `exact` answers
|
||||||
|
* O(path length) equality; `sorted` answers whether any indexed path lies
|
||||||
|
* strictly below a query directory. Both borrow their keys from the caller and
|
||||||
|
* no ancestor prefix is stored as a separate string, so an index over M entries
|
||||||
|
* is O(M) memory regardless of path depth. Not thread-safe to build, but safe
|
||||||
|
* for concurrent read-only queries once built. */
|
||||||
|
typedef struct {
|
||||||
|
StrHashSet exact;
|
||||||
|
StrSortedArray sorted;
|
||||||
|
} PathIndex;
|
||||||
|
|
||||||
|
/* Build an index borrowing `entries` (which must outlive the index). Returns
|
||||||
|
* false on allocation failure, freeing any partial state. */
|
||||||
|
bool path_index_build(PathIndex* index, const char* const* entries, size_t count);
|
||||||
|
void path_index_free(PathIndex* index);
|
||||||
|
/* True when `path` is an indexed entry. */
|
||||||
|
bool path_index_contains(const PathIndex* index, const char* path);
|
||||||
|
/* Length-bounded form of path_index_contains (`path` need not be terminated). */
|
||||||
|
bool path_index_contains_n(const PathIndex* index, const char* path, size_t len);
|
||||||
|
/* True when some indexed entry lies strictly below `path` (starts with
|
||||||
|
* `path` + '/'). */
|
||||||
|
bool path_index_has_descendant(const PathIndex* index, const char* path);
|
||||||
|
|
||||||
char* str_dup(const char* string);
|
char* str_dup(const char* string);
|
||||||
char* output_escape(const char* string, bool eight_bit_output);
|
char* output_escape(const char* string, bool eight_bit_output);
|
||||||
|
/* Upper bound on one line/token read from a local list file (--files-from,
|
||||||
|
* --exclude-from/--include-from, .rsync-filter). Mirrors MAX_STRING_SIZE and
|
||||||
|
* stops a hostile multi-gigabyte line from forcing unbounded allocation. */
|
||||||
|
#define UTILS_MAX_LINE_LEN (64 * 1024)
|
||||||
|
/* Read one `delim`-terminated record from `stream` into *line (grown as needed
|
||||||
|
* and NUL-terminated), refusing to consume/allocate more than `max_len` bytes
|
||||||
|
* of content. Returns the number of bytes stored (delimiter included, matching
|
||||||
|
* getdelim), 0 at end of file, or -1 on error (errno is EFBIG when the record
|
||||||
|
* exceeds `max_len`, ENOMEM on allocation failure). *line and *cap are updated
|
||||||
|
* as the buffer grows and the caller owns *line. */
|
||||||
|
ssize_t utils_getdelim_bounded(FILE* stream, char** line, size_t* cap, int delim, size_t max_len);
|
||||||
char* path_cat(const char* path1, const char* path2);
|
char* path_cat(const char* path1, const char* path2);
|
||||||
bool glob_match(const char* pattern, const char* str);
|
bool glob_match(const char* pattern, const char* str);
|
||||||
/* Result of a bounded extra-file deletion run. */
|
/* Result of a bounded extra-file deletion run. */
|
||||||
@@ -48,14 +132,41 @@ bool path_under_skip_prefix(const char* child_rel, bool at_root, const DeleteSki
|
|||||||
and the delete pass are two separate walks, so a concurrent change between
|
and the delete pass are two separate walks, so a concurrent change between
|
||||||
them (another process adding/removing entries) can make the second pass
|
them (another process adding/removing entries) can make the second pass
|
||||||
delete a different set than the first one counted. */
|
delete a different set than the first one counted. */
|
||||||
DeleteWalkResult delete_extras_limited(const char* dest_root, ArrayList* manifest,
|
DeleteWalkResult delete_extras_limited(const char* dest_root, const ArrayList* manifest,
|
||||||
size_t max_delete, const DeleteSkipEntry* skips,
|
size_t max_delete, const DeleteSkipEntry* skips,
|
||||||
int skip_count, size_t* deleted_out);
|
int skip_count, size_t* deleted_out);
|
||||||
bool delete_extras(const char* dest_root, ArrayList* manifest);
|
bool delete_extras(const char* dest_root, const ArrayList* manifest);
|
||||||
bool utils_set_authorized_root(int fd, const char* canonical_path);
|
bool utils_set_authorized_root(int fd, const char* canonical_path);
|
||||||
/* The fd-only compatibility form is fail-closed for path-based operations;
|
/* The fd-only compatibility form is fail-closed for path-based operations;
|
||||||
* callers should use utils_set_authorized_root with the canonical identity. */
|
* callers should use utils_set_authorized_root with the canonical identity. */
|
||||||
void utils_set_authorized_root_fd(int fd);
|
void utils_set_authorized_root_fd(int fd);
|
||||||
|
/* Read accessors for the process-wide authorized root, so every secure-walk
|
||||||
|
* site consumes the single shared state instead of keeping its own copy. The
|
||||||
|
* fd is caller-owned (see the setters): it is returned verbatim, never dup'd,
|
||||||
|
* and the caller that opened it is responsible for closing it. With no root
|
||||||
|
* configured the fd accessor returns -1 and the path accessor returns NULL.
|
||||||
|
*
|
||||||
|
* The pointer returned by utils_get_authorized_root_path() is borrowed into
|
||||||
|
* process-global state and is invalidated by the next
|
||||||
|
* utils_set_authorized_root() / utils_set_authorized_root_fd() call. The fd
|
||||||
|
* and path are stored separately and read independently, so the pair is NOT
|
||||||
|
* observed atomically together; the accessors are non-reentrant and callers
|
||||||
|
* must serialize configuration (the server installs the root before any worker
|
||||||
|
* threads spawn; see utils.c). */
|
||||||
|
int utils_get_authorized_root_fd(void);
|
||||||
|
const char* utils_get_authorized_root_path(void);
|
||||||
|
/* True when `path` is `root` itself or lies directly beneath it: a lexical
|
||||||
|
* prefix test requiring the byte after `root` to be '\0' or '/'. Both `root`
|
||||||
|
* and `path` must be absolute canonical paths free of "."/".." components (the
|
||||||
|
* callers guarantee this); this is containment by string, not by resolved
|
||||||
|
* symlinks. Shared by the utils and file secure-walk root confinement. */
|
||||||
|
bool path_is_within_root(const char* root, const char* path);
|
||||||
|
/* True when `path` contains a ".." component. This is a purely lexical
|
||||||
|
* dot-dot check: an absolute path is NOT rejected here, because default
|
||||||
|
* (non-relative) transfers legitimately put the sender's absolute source path
|
||||||
|
* on the wire and the receiver re-roots it under the destination with
|
||||||
|
* path_cat(). Callers that accept a strictly relative path (e.g. batch paths)
|
||||||
|
* must reject a leading '/' themselves (see utils_valid_batch_path). */
|
||||||
bool has_path_traversal(const char* path);
|
bool has_path_traversal(const char* path);
|
||||||
bool utils_valid_batch_path(const char* path);
|
bool utils_valid_batch_path(const char* path);
|
||||||
bool format_human_bytes(unsigned long long bytes, char* buffer, size_t buffer_size);
|
bool format_human_bytes(unsigned long long bytes, char* buffer, size_t buffer_size);
|
||||||
@@ -77,5 +188,13 @@ bool append_tail_length(unsigned long long old_size, unsigned long long check_si
|
|||||||
bool utils_sockaddr_is_loopback(const struct sockaddr* addr);
|
bool utils_sockaddr_is_loopback(const struct sockaddr* addr);
|
||||||
bool utils_fd_peer_is_local(int fd);
|
bool utils_fd_peer_is_local(int fd);
|
||||||
bool utils_host_is_loopback(const char* host);
|
bool utils_host_is_loopback(const char* host);
|
||||||
|
/* Numeric peer address of a connected fd (INET6_ADDRSTRLEN is always enough).
|
||||||
|
* Returns false and leaves buf empty when the fd is not a connected INET socket
|
||||||
|
* or getpeername/inet_ntop fails. Used by the daemon host-access gate; a false
|
||||||
|
* return is "cannot tell" and must be treated as fail-closed when ACLs apply. */
|
||||||
|
bool utils_fd_peer_ip(int fd, char* buf, size_t len);
|
||||||
|
/* Format a sockaddr as "ip:port" (IPv4) or "[ip]:port" (IPv6) for logging.
|
||||||
|
* Returns false (buf emptied) for a non-INET family or a formatting failure. */
|
||||||
|
bool utils_sockaddr_to_string(const struct sockaddr* addr, char* buf, size_t len);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
+47
-17
@@ -73,13 +73,17 @@ bool xattr_list_append(FileXattrList* list, const char* name, const void* value,
|
|||||||
|
|
||||||
/* A Linux xattr name is "namespace.name" with an optional leading "trusted.",
|
/* A Linux xattr name is "namespace.name" with an optional leading "trusted.",
|
||||||
* "system.", "security.", "user.", or "trusted." prefix. We only ever touch
|
* "system.", "security.", "user.", or "trusted." prefix. We only ever touch
|
||||||
* the unprivileged "user.*" namespace and the two POSIX ACL xattrs carried in
|
* the unprivileged "user.*" namespace and, only when --acls/-A was negotiated,
|
||||||
* the "system." namespace. Everything else -- especially "security.*" (ACLs,
|
* the two POSIX ACL xattrs carried in the "system." namespace. Everything else
|
||||||
* capabilities, SELinux labels) and "trusted.*" -- is refused so a client can
|
* -- especially "security.*" (ACLs, capabilities, SELinux labels) and
|
||||||
* never compel the receiver to apply a privileged attribute it would not
|
* "trusted.*" -- is refused so a client can never compel the receiver to apply a
|
||||||
* otherwise be able to set (and which would be a local privilege escalation if
|
* privileged attribute it would not otherwise be able to set (and which would be
|
||||||
* it could). */
|
* a local privilege escalation if it could).
|
||||||
bool xattr_name_appliable(const char* name) {
|
*
|
||||||
|
* The ACL gate is deliberate: --xattrs/-X alone derives use_xattrs but must NOT
|
||||||
|
* authorize the ACL names, otherwise a -X client could plant an ACL the
|
||||||
|
* receiver never opted into (B4). */
|
||||||
|
bool xattr_name_appliable(const char* name, bool preserve_acls) {
|
||||||
if (!name || name[0] == '\0')
|
if (!name || name[0] == '\0')
|
||||||
return false;
|
return false;
|
||||||
size_t len = strlen(name);
|
size_t len = strlen(name);
|
||||||
@@ -95,15 +99,24 @@ bool xattr_name_appliable(const char* name) {
|
|||||||
if (strncmp(name, "user.", 5) == 0)
|
if (strncmp(name, "user.", 5) == 0)
|
||||||
return name[5] != '\0';
|
return name[5] != '\0';
|
||||||
if (strcmp(name, "system.posix_acl_access") == 0)
|
if (strcmp(name, "system.posix_acl_access") == 0)
|
||||||
return true;
|
return preserve_acls;
|
||||||
if (strcmp(name, "system.posix_acl_default") == 0)
|
if (strcmp(name, "system.posix_acl_default") == 0)
|
||||||
return true;
|
return preserve_acls;
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* The two POSIX ACL xattr names: the only names whose applicablity is
|
||||||
|
* conditional (they require --acls). Used by the receiver to distinguish "not
|
||||||
|
* negotiated" (drop the entry, keep user.* working for -X) from a genuinely
|
||||||
|
* disallowed namespace (hard reject). */
|
||||||
|
static bool xattr_name_is_posix_acl(const char* name) {
|
||||||
|
return name != NULL && (strcmp(name, "system.posix_acl_access") == 0 ||
|
||||||
|
strcmp(name, "system.posix_acl_default") == 0);
|
||||||
|
}
|
||||||
|
|
||||||
/* ---- SENDER: capture ---- */
|
/* ---- SENDER: capture ---- */
|
||||||
|
|
||||||
FileXattrList* xattr_capture_path(const char* path) {
|
FileXattrList* xattr_capture_path(const char* path, bool preserve_acls) {
|
||||||
if (!path)
|
if (!path)
|
||||||
return NULL;
|
return NULL;
|
||||||
ssize_t list_size = listxattr(path, NULL, 0);
|
ssize_t list_size = listxattr(path, NULL, 0);
|
||||||
@@ -130,7 +143,10 @@ FileXattrList* xattr_capture_path(const char* path) {
|
|||||||
if (name_len == 0)
|
if (name_len == 0)
|
||||||
break; /* trailing double NUL not expected; stop */
|
break; /* trailing double NUL not expected; stop */
|
||||||
offset += (ssize_t)name_len + 1;
|
offset += (ssize_t)name_len + 1;
|
||||||
if (!xattr_name_appliable(name))
|
/* Capture is sender-side: the scanner has already gated on -X/-A, so the
|
||||||
|
per-name whitelist here allows the ACL names only when --acls was
|
||||||
|
negotiated. Without it a plain -X capture never carries an ACL. */
|
||||||
|
if (!xattr_name_appliable(name, preserve_acls))
|
||||||
continue;
|
continue;
|
||||||
ssize_t value_size = getxattr(path, name, NULL, 0);
|
ssize_t value_size = getxattr(path, name, NULL, 0);
|
||||||
if (value_size < 0)
|
if (value_size < 0)
|
||||||
@@ -190,7 +206,7 @@ bool xattr_send(int fd, const FileXattrList* list) {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
FileXattrList* xattr_receive(int fd, int* ok) {
|
FileXattrList* xattr_receive(int fd, int* ok, bool preserve_acls) {
|
||||||
if (ok)
|
if (ok)
|
||||||
*ok = 0;
|
*ok = 0;
|
||||||
int count;
|
int count;
|
||||||
@@ -232,11 +248,19 @@ FileXattrList* xattr_receive(int fd, int* ok) {
|
|||||||
xattr_list_free(list);
|
xattr_list_free(list);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
if (!xattr_name_appliable(name)) {
|
bool skip = false;
|
||||||
log_message(LOG_LEVEL_ERROR, "rejected xattr block: disallowed namespace for '%s'", name);
|
if (!xattr_name_appliable(name, preserve_acls)) {
|
||||||
free(name);
|
if (!preserve_acls && xattr_name_is_posix_acl(name)) {
|
||||||
xattr_list_free(list);
|
/* -X without -A: the sender may still carry ACLs, but the receiver must
|
||||||
return NULL;
|
never apply an ACL it was not asked to preserve. Consume and drop the
|
||||||
|
entry (keeping -X compatibility) rather than failing the transfer. */
|
||||||
|
skip = true;
|
||||||
|
} else {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "rejected xattr block: disallowed namespace for '%s'", name);
|
||||||
|
free(name);
|
||||||
|
xattr_list_free(list);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
int32_t value_len32;
|
int32_t value_len32;
|
||||||
if (!receive_n_data(fd, &value_len32, sizeof(value_len32))) {
|
if (!receive_n_data(fd, &value_len32, sizeof(value_len32))) {
|
||||||
@@ -273,6 +297,12 @@ FileXattrList* xattr_receive(int fd, int* ok) {
|
|||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if (skip) {
|
||||||
|
free(value);
|
||||||
|
free(name);
|
||||||
|
budget += (size_t)name_len32 + (size_t)value_len32;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
if (!xattr_list_append(list, name, value, (size_t)value_len32)) {
|
if (!xattr_list_append(list, name, value, (size_t)value_len32)) {
|
||||||
free(value);
|
free(value);
|
||||||
free(name);
|
free(name);
|
||||||
|
|||||||
+17
-9
@@ -59,20 +59,28 @@ void xattr_list_free(FileXattrList* list);
|
|||||||
bool xattr_list_append(FileXattrList* list, const char* name, const void* value, size_t value_len);
|
bool xattr_list_append(FileXattrList* list, const char* name, const void* value, size_t value_len);
|
||||||
|
|
||||||
/* True when `name` is a well-formed xattr name AND belongs to a namespace this
|
/* True when `name` is a well-formed xattr name AND belongs to a namespace this
|
||||||
* build is authorized to apply (user.* or the two POSIX ACL xattrs). Used for
|
* build is authorized to apply. `user.*` is always accepted for -X; the two
|
||||||
* both capture and receiver-side validation. */
|
* POSIX ACL xattrs are accepted only when `preserve_acls` (--acls/-A) is set, so
|
||||||
bool xattr_name_appliable(const char* name);
|
* a plain -X run can never carry or apply an ACL the receiver did not ask for.
|
||||||
|
* Used for both capture and receiver-side validation. */
|
||||||
|
bool xattr_name_appliable(const char* name, bool preserve_acls);
|
||||||
|
|
||||||
/* Sender: read the whitelisted xattrs of `path` into a new list. Returns NULL
|
/* Sender: read the whitelisted xattrs of `path` into a new list. The POSIX ACL
|
||||||
* when the path has no appliable xattrs (or the filesystem has no xattr
|
* names are captured only when `preserve_acls` (--acls/-A) is set, so a plain
|
||||||
* support); an empty-but-valid list is never returned distinct from NULL. */
|
* -X run never carries an ACL it was not asked to preserve; `user.*` is
|
||||||
FileXattrList* xattr_capture_path(const char* path);
|
* unaffected. Returns NULL when the path has no appliable xattrs (or the
|
||||||
|
* filesystem has no xattr support); an empty-but-valid list is never returned
|
||||||
|
* distinct from NULL. */
|
||||||
|
FileXattrList* xattr_capture_path(const char* path, bool preserve_acls);
|
||||||
|
|
||||||
/* Wire: bounded serialization. xattr_send returns false on write failure; an
|
/* Wire: bounded serialization. xattr_send returns false on write failure; an
|
||||||
* empty/NULL list transmits a zero-count block. xattr_receive returns NULL and
|
* empty/NULL list transmits a zero-count block. xattr_receive returns NULL and
|
||||||
* sets *ok = 0 on any malformed / oversized / non-whitelisted entry. */
|
* sets *ok = 0 on any malformed / oversized / non-whitelisted entry. When
|
||||||
|
* `preserve_acls` is false, any POSIX ACL entries are consumed and DROPPED (so
|
||||||
|
* a -X transfer still succeeds and never applies an ACL it did not negotiate);
|
||||||
|
* a genuinely disallowed namespace is still rejected. */
|
||||||
bool xattr_send(int fd, const FileXattrList* list);
|
bool xattr_send(int fd, const FileXattrList* list);
|
||||||
FileXattrList* xattr_receive(int fd, int* ok);
|
FileXattrList* xattr_receive(int fd, int* ok, bool preserve_acls);
|
||||||
|
|
||||||
/* Receiver: apply every entry fd-relative (fsetxattr) to the just-written file
|
/* Receiver: apply every entry fd-relative (fsetxattr) to the just-written file
|
||||||
* descriptor. A per-attribute failure (e.g. ACL set refused for non-root on a
|
* descriptor. A per-attribute failure (e.g. ACL set refused for non-root on a
|
||||||
|
|||||||
+7
-1
@@ -16,7 +16,13 @@ def shared_server():
|
|||||||
Under pytest-xdist this session fixture is instantiated once per worker
|
Under pytest-xdist this session fixture is instantiated once per worker
|
||||||
process, so each worker gets its own server on an ephemeral port."""
|
process, so each worker gets its own server on an ephemeral port."""
|
||||||
server = ServerManager()
|
server = ServerManager()
|
||||||
server.start()
|
# --allow-super keeps the historical permissive super mode for a root
|
||||||
|
# receiver: the integration suite's root-only ownership/device/copy-as tests
|
||||||
|
# exercise that opted-in configuration. The secure default (a root
|
||||||
|
# standalone server without --allow-super forces SUPER_MODE_OFF) is covered
|
||||||
|
# explicitly by TestStandaloneSuperDefault in test_features.py. Non-root
|
||||||
|
# runs are unaffected by the flag.
|
||||||
|
server.start(extra_args=["--allow-super"])
|
||||||
yield server
|
yield server
|
||||||
server.stop()
|
server.stop()
|
||||||
|
|
||||||
|
|||||||
@@ -17,7 +17,11 @@ int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
|
|||||||
if (!d)
|
if (!d)
|
||||||
return 0;
|
return 0;
|
||||||
|
|
||||||
Chunk* chunk = chunk_deserialize(d, false);
|
/* Exercise both the metadata and non-metadata chunk layouts: the
|
||||||
|
metadata branch (present flag + 4-vs-72 advance) is only reachable with
|
||||||
|
use_metadata=true, so base the choice on the input rather than hardcoding
|
||||||
|
false. */
|
||||||
|
Chunk* chunk = chunk_deserialize(d, (data[0] & 1) != 0);
|
||||||
if (chunk)
|
if (chunk)
|
||||||
chunk_destroy(chunk);
|
chunk_destroy(chunk);
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,126 @@
|
|||||||
|
/*
|
||||||
|
* Fuzz the delete-manifest parser: receive_manifest_entries(int fd).
|
||||||
|
*
|
||||||
|
* The parser reads three length-delimited sections (keeps, protected prefixes,
|
||||||
|
* missing-args paths) from the connection. Feeding raw bytes alone exercises
|
||||||
|
* the "reject the first malformed count/string" fast paths, but because each
|
||||||
|
* section is self-delimiting a single bad value hides every later section.
|
||||||
|
*
|
||||||
|
* To reach the protected-prefix and missing-args parsers (the paths that drive
|
||||||
|
* actual destination deletion) we build one canonical, fully-valid manifest
|
||||||
|
* with hand-written wire framing and then feed the receiver several shapes:
|
||||||
|
*
|
||||||
|
* 1. raw : the raw fuzz bytes as the whole manifest.
|
||||||
|
* 2. keeps : the valid keep count only + the fuzz bytes, so the fuzzer
|
||||||
|
* drives the keep count and entries directly.
|
||||||
|
* 3. prot : the valid keeps section + the fuzz bytes, so the fuzzer drives
|
||||||
|
* the protected count and prefixes.
|
||||||
|
* 4. missing: the valid keeps+protected sections + the fuzz bytes, so the
|
||||||
|
* fuzzer drives the trailing missing-args section, including the
|
||||||
|
* aggregate MAX_MANIFEST_BYTES budget.
|
||||||
|
*
|
||||||
|
* The wire encoding matches receive_int (native int) and receive_wire_str
|
||||||
|
* (native size_t length prefix + body); no charset conversion is configured in
|
||||||
|
* the fuzz process, so receive_wire_str is receive_str.
|
||||||
|
*/
|
||||||
|
#include "file_receive.h"
|
||||||
|
#include "protocol.h"
|
||||||
|
#include <errno.h>
|
||||||
|
#include <fcntl.h>
|
||||||
|
#include <stdint.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
|
#include <sys/socket.h>
|
||||||
|
#include <unistd.h>
|
||||||
|
|
||||||
|
static unsigned char g_manifest[512];
|
||||||
|
static size_t g_len_after_count; /* offset of the first keep entry */
|
||||||
|
static size_t g_len_after_keeps; /* offset of the protected count */
|
||||||
|
static size_t g_len_after_protected; /* offset of the missing count */
|
||||||
|
static int g_manifest_ready;
|
||||||
|
|
||||||
|
static void append_int32(unsigned char* buf, size_t* off, int32_t value) {
|
||||||
|
memcpy(buf + *off, &value, sizeof(value));
|
||||||
|
*off += sizeof(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void append_wire_str(unsigned char* buf, size_t* off, const char* s) {
|
||||||
|
size_t n = strlen(s);
|
||||||
|
memcpy(buf + *off, &n, sizeof(n));
|
||||||
|
*off += sizeof(n);
|
||||||
|
memcpy(buf + *off, s, n);
|
||||||
|
*off += n;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void build_canonical_manifest(void) {
|
||||||
|
g_manifest_ready = 1;
|
||||||
|
size_t off = 0;
|
||||||
|
append_int32(g_manifest, &off, 2);
|
||||||
|
g_len_after_count = off;
|
||||||
|
append_wire_str(g_manifest, &off, "keep/a");
|
||||||
|
append_wire_str(g_manifest, &off, "keep/b");
|
||||||
|
g_len_after_keeps = off;
|
||||||
|
append_int32(g_manifest, &off, 1);
|
||||||
|
append_wire_str(g_manifest, &off, "excluded/prefix");
|
||||||
|
g_len_after_protected = off;
|
||||||
|
append_int32(g_manifest, &off, 1);
|
||||||
|
append_wire_str(g_manifest, &off, "missing/path");
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Best-effort non-blocking write: an oversized fuzz input is truncated rather
|
||||||
|
* than stalling the harness. */
|
||||||
|
static void write_best_effort(int fd, const void* data, size_t size) {
|
||||||
|
const unsigned char* p = data;
|
||||||
|
size_t off = 0;
|
||||||
|
while (off < size) {
|
||||||
|
ssize_t n = write(fd, p + off, size - off);
|
||||||
|
if (n > 0) {
|
||||||
|
off += (size_t)n;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (n < 0 && errno == EINTR)
|
||||||
|
continue;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Build prefix ++ data as a stream and drive receive_manifest_entries over it.
|
||||||
|
* The write half is shut down first so the parser always sees EOF instead of
|
||||||
|
* blocking on a missing frame tail. */
|
||||||
|
static void receive_stream(const unsigned char* prefix, size_t prefix_len, const uint8_t* data,
|
||||||
|
size_t size) {
|
||||||
|
int sv[2];
|
||||||
|
if (socketpair(AF_UNIX, SOCK_STREAM, 0, sv) != 0)
|
||||||
|
return;
|
||||||
|
|
||||||
|
int flags = fcntl(sv[0], F_GETFL, 0);
|
||||||
|
if (flags != -1)
|
||||||
|
(void)fcntl(sv[0], F_SETFL, flags | O_NONBLOCK);
|
||||||
|
|
||||||
|
if (prefix_len > 0)
|
||||||
|
write_best_effort(sv[0], prefix, prefix_len);
|
||||||
|
if (size > 0)
|
||||||
|
write_best_effort(sv[0], data, size);
|
||||||
|
shutdown(sv[0], SHUT_WR);
|
||||||
|
|
||||||
|
DeleteManifest* manifest = receive_manifest_entries(sv[1]);
|
||||||
|
delete_manifest_free(manifest);
|
||||||
|
|
||||||
|
close(sv[0]);
|
||||||
|
close(sv[1]);
|
||||||
|
}
|
||||||
|
|
||||||
|
int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
|
||||||
|
if (!g_manifest_ready)
|
||||||
|
build_canonical_manifest();
|
||||||
|
|
||||||
|
/* Raw bytes as the whole manifest. */
|
||||||
|
receive_stream(NULL, 0, data, size);
|
||||||
|
|
||||||
|
/* Keep the valid framing so the fuzzer reaches each later section. */
|
||||||
|
receive_stream(g_manifest, g_len_after_protected, data, size);
|
||||||
|
receive_stream(g_manifest, g_len_after_keeps, data, size);
|
||||||
|
receive_stream(g_manifest, g_len_after_count, data, size);
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
@@ -5,19 +5,19 @@
|
|||||||
#include <string.h>
|
#include <string.h>
|
||||||
|
|
||||||
int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
|
int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
|
||||||
if (size < sizeof(int) + FILE_METADATA_WIRE_SIZE)
|
/* Exercise the bounds-checked decoder on EVERY input length, including
|
||||||
return 0;
|
* records shorter than a full metadata body; the decoder must reject those
|
||||||
|
* without reading past `size`. */
|
||||||
char* buf = malloc(size);
|
char* buf = malloc(size > 0 ? size : 1);
|
||||||
if (!buf)
|
if (!buf)
|
||||||
return 0;
|
return 0;
|
||||||
memcpy(buf, data, size);
|
if (size > 0)
|
||||||
|
memcpy(buf, data, size);
|
||||||
|
|
||||||
char* original_buf = buf;
|
FileMetadata* m = metadata_from_buf((const uint8_t*)buf, size);
|
||||||
FileMetadata* m = metadata_from_buf(&buf);
|
|
||||||
if (m)
|
if (m)
|
||||||
free(m);
|
free(m);
|
||||||
|
|
||||||
free(original_buf);
|
free(buf);
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,130 @@
|
|||||||
|
/*
|
||||||
|
* Fuzz the base protocol framing: receive_str / receive_data / receive_status
|
||||||
|
* (plus the redacted string, the size-limited data and the timed-status
|
||||||
|
* variants) fed arbitrary bytes over an in-memory socketpair.
|
||||||
|
*
|
||||||
|
* Every receive primitive reads a fixed-width header (a size_t string length,
|
||||||
|
* an unsigned long long data length, an int status/int value) and then a body.
|
||||||
|
* The fuzzer attacks:
|
||||||
|
* - oversized length headers (the MAX_STRING_SIZE / MAX_DATA_PAYLOAD_SIZE
|
||||||
|
* gates must reject before allocating),
|
||||||
|
* - truncated bodies (a declared body larger than the stream must fail
|
||||||
|
* cleanly at EOF, never read uninitialised memory or leak),
|
||||||
|
* - embedded NUL bytes in strings (must be refused),
|
||||||
|
* - out-of-range status enum values (status_to_string must stay in bounds).
|
||||||
|
*
|
||||||
|
* Each entry point gets its own socketpair because a single receive consumes a
|
||||||
|
* variable number of bytes from the stream; reusing one would make the later
|
||||||
|
* calls meaningless. The write half is shut down first so a truncated frame
|
||||||
|
* always terminates at EOF instead of blocking.
|
||||||
|
*/
|
||||||
|
#include "data.h"
|
||||||
|
#include "protocol.h"
|
||||||
|
#include <errno.h>
|
||||||
|
#include <fcntl.h>
|
||||||
|
#include <stdint.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
|
#include <sys/socket.h>
|
||||||
|
#include <unistd.h>
|
||||||
|
|
||||||
|
static void write_best_effort(int fd, const void* data, size_t size) {
|
||||||
|
const unsigned char* p = data;
|
||||||
|
size_t off = 0;
|
||||||
|
while (off < size) {
|
||||||
|
ssize_t n = write(fd, p + off, size - off);
|
||||||
|
if (n > 0) {
|
||||||
|
off += (size_t)n;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (n < 0 && errno == EINTR)
|
||||||
|
continue;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Create a socketpair pre-loaded with `data`, shut down the write half and
|
||||||
|
* return the read end (which the receiver reads from). `*write_end` is also
|
||||||
|
* returned so the caller can close it. */
|
||||||
|
static int make_stream(const uint8_t* data, size_t size, int* write_end) {
|
||||||
|
int sv[2];
|
||||||
|
if (socketpair(AF_UNIX, SOCK_STREAM, 0, sv) != 0) {
|
||||||
|
*write_end = -1;
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
int flags = fcntl(sv[0], F_GETFL, 0);
|
||||||
|
if (flags != -1)
|
||||||
|
(void)fcntl(sv[0], F_SETFL, flags | O_NONBLOCK);
|
||||||
|
if (size > 0)
|
||||||
|
write_best_effort(sv[0], data, size);
|
||||||
|
shutdown(sv[0], SHUT_WR);
|
||||||
|
*write_end = sv[0];
|
||||||
|
return sv[1];
|
||||||
|
}
|
||||||
|
|
||||||
|
int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
|
||||||
|
int w;
|
||||||
|
|
||||||
|
int rd = make_stream(data, size, &w);
|
||||||
|
if (rd >= 0) {
|
||||||
|
char* s = receive_str(rd);
|
||||||
|
free(s);
|
||||||
|
close(rd);
|
||||||
|
close(w);
|
||||||
|
}
|
||||||
|
|
||||||
|
rd = make_stream(data, size, &w);
|
||||||
|
if (rd >= 0) {
|
||||||
|
char* s = receive_str_redacted(rd);
|
||||||
|
free(s);
|
||||||
|
close(rd);
|
||||||
|
close(w);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Bounded so a crafted 256 MiB length header cannot make each iteration
|
||||||
|
allocate the full MAX_DATA_PAYLOAD_SIZE under ASan; the framing logic is
|
||||||
|
identical to receive_data(), which delegates to the limited variant. */
|
||||||
|
rd = make_stream(data, size, &w);
|
||||||
|
if (rd >= 0) {
|
||||||
|
Data* d = receive_data_limited(rd, 1u << 20);
|
||||||
|
data_destroy(d);
|
||||||
|
close(rd);
|
||||||
|
close(w);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* The size-limited variant must reject anything beyond its explicit bound
|
||||||
|
* before allocating the body buffer. */
|
||||||
|
rd = make_stream(data, size, &w);
|
||||||
|
if (rd >= 0) {
|
||||||
|
Data* d = receive_data_limited(rd, 256);
|
||||||
|
data_destroy(d);
|
||||||
|
close(rd);
|
||||||
|
close(w);
|
||||||
|
}
|
||||||
|
|
||||||
|
rd = make_stream(data, size, &w);
|
||||||
|
if (rd >= 0) {
|
||||||
|
Status status = STATUS_OK;
|
||||||
|
(void)receive_status(rd, &status);
|
||||||
|
close(rd);
|
||||||
|
close(w);
|
||||||
|
}
|
||||||
|
|
||||||
|
rd = make_stream(data, size, &w);
|
||||||
|
if (rd >= 0) {
|
||||||
|
Status status = STATUS_OK;
|
||||||
|
(void)receive_status_timed(rd, &status, 1);
|
||||||
|
close(rd);
|
||||||
|
close(w);
|
||||||
|
}
|
||||||
|
|
||||||
|
rd = make_stream(data, size, &w);
|
||||||
|
if (rd >= 0) {
|
||||||
|
int value = 0;
|
||||||
|
(void)receive_int(rd, &value);
|
||||||
|
close(rd);
|
||||||
|
close(w);
|
||||||
|
}
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
@@ -0,0 +1,119 @@
|
|||||||
|
/*
|
||||||
|
* Fuzz the xattr wire block parser: xattr_receive(int fd, int* ok).
|
||||||
|
*
|
||||||
|
* The block is a count followed by that many (name_len, name, value_len, value)
|
||||||
|
* records. The receiver must reject an invalid count, an out-of-range or
|
||||||
|
* negative name/value length, an embedded NUL or non-whitelisted namespace in
|
||||||
|
* the name, an oversized value, and an aggregate payload beyond
|
||||||
|
* XATTR_TOTAL_MAX -- all without over-allocating or leaking.
|
||||||
|
*
|
||||||
|
* Raw bytes mostly stop at the first invalid count/length, so we also build a
|
||||||
|
* canonical, fully-valid two-entry block by hand and feed the receiver valid
|
||||||
|
* prefixes of it followed by the fuzz bytes. That drives the deep value-
|
||||||
|
* parsing and per-entry namespace/budget checks with attacker-controlled input.
|
||||||
|
*/
|
||||||
|
#include "protocol.h"
|
||||||
|
#include "xattr.h"
|
||||||
|
#include <errno.h>
|
||||||
|
#include <fcntl.h>
|
||||||
|
#include <stdint.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
|
#include <sys/socket.h>
|
||||||
|
#include <unistd.h>
|
||||||
|
|
||||||
|
static unsigned char g_block[512];
|
||||||
|
static size_t g_off_after_count; /* start of entry 0 */
|
||||||
|
static size_t g_off_after_entry0; /* start of entry 1 */
|
||||||
|
static size_t g_off_value0; /* start of the first value length */
|
||||||
|
static int g_block_ready;
|
||||||
|
|
||||||
|
static void append_int32(unsigned char* buf, size_t* off, int32_t value) {
|
||||||
|
memcpy(buf + *off, &value, sizeof(value));
|
||||||
|
*off += sizeof(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void append_bytes(unsigned char* buf, size_t* off, const void* p, size_t n) {
|
||||||
|
if (n > 0)
|
||||||
|
memcpy(buf + *off, p, n);
|
||||||
|
*off += n;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void build_canonical_block(void) {
|
||||||
|
g_block_ready = 1;
|
||||||
|
size_t off = 0;
|
||||||
|
append_int32(g_block, &off, 2);
|
||||||
|
g_off_after_count = off;
|
||||||
|
|
||||||
|
int32_t name0_len = (int32_t)strlen("user.foo");
|
||||||
|
append_int32(g_block, &off, name0_len);
|
||||||
|
append_bytes(g_block, &off, "user.foo", (size_t)name0_len);
|
||||||
|
g_off_value0 = off;
|
||||||
|
append_int32(g_block, &off, 3);
|
||||||
|
append_bytes(g_block, &off, "bar", 3);
|
||||||
|
|
||||||
|
g_off_after_entry0 = off;
|
||||||
|
int32_t name1_len = (int32_t)strlen("user.empty");
|
||||||
|
append_int32(g_block, &off, name1_len);
|
||||||
|
append_bytes(g_block, &off, "user.empty", (size_t)name1_len);
|
||||||
|
append_int32(g_block, &off, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void write_best_effort(int fd, const void* data, size_t size) {
|
||||||
|
const unsigned char* p = data;
|
||||||
|
size_t off = 0;
|
||||||
|
while (off < size) {
|
||||||
|
ssize_t n = write(fd, p + off, size - off);
|
||||||
|
if (n > 0) {
|
||||||
|
off += (size_t)n;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (n < 0 && errno == EINTR)
|
||||||
|
continue;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static void receive_stream(const unsigned char* prefix, size_t prefix_len, const uint8_t* data,
|
||||||
|
size_t size, bool preserve_acls) {
|
||||||
|
int sv[2];
|
||||||
|
if (socketpair(AF_UNIX, SOCK_STREAM, 0, sv) != 0)
|
||||||
|
return;
|
||||||
|
|
||||||
|
int flags = fcntl(sv[0], F_GETFL, 0);
|
||||||
|
if (flags != -1)
|
||||||
|
(void)fcntl(sv[0], F_SETFL, flags | O_NONBLOCK);
|
||||||
|
|
||||||
|
if (prefix_len > 0)
|
||||||
|
write_best_effort(sv[0], prefix, prefix_len);
|
||||||
|
if (size > 0)
|
||||||
|
write_best_effort(sv[0], data, size);
|
||||||
|
shutdown(sv[0], SHUT_WR);
|
||||||
|
|
||||||
|
int ok = 0;
|
||||||
|
FileXattrList* list = xattr_receive(sv[1], &ok, preserve_acls);
|
||||||
|
xattr_list_free(list);
|
||||||
|
|
||||||
|
close(sv[0]);
|
||||||
|
close(sv[1]);
|
||||||
|
}
|
||||||
|
|
||||||
|
int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
|
||||||
|
if (!g_block_ready)
|
||||||
|
build_canonical_block();
|
||||||
|
|
||||||
|
/* Raw bytes as the whole block. Exercise both the -X-only (no ACLs) and the
|
||||||
|
* -A (ACL names accepted) receiver gates. */
|
||||||
|
for (int acls = 0; acls < 2; acls++) {
|
||||||
|
bool preserve_acls = acls != 0;
|
||||||
|
receive_stream(NULL, 0, data, size, preserve_acls);
|
||||||
|
|
||||||
|
/* Valid framing so the fuzzer mutates the entry list, the first value and
|
||||||
|
* the second entry respectively instead of stopping at the count. */
|
||||||
|
receive_stream(g_block, g_off_after_entry0, data, size, preserve_acls);
|
||||||
|
receive_stream(g_block, g_off_value0, data, size, preserve_acls);
|
||||||
|
receive_stream(g_block, g_off_after_count, data, size, preserve_acls);
|
||||||
|
}
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
@@ -118,3 +118,15 @@ def test_batch_modes_conflict():
|
|||||||
result = subprocess.run(cmd, capture_output=True, text=True, timeout=180)
|
result = subprocess.run(cmd, capture_output=True, text=True, timeout=180)
|
||||||
assert result.returncode != 0, \
|
assert result.returncode != 0, \
|
||||||
f"expected conflict failure for {flags}: {result.stderr}"
|
f"expected conflict failure for {flags}: {result.stderr}"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_dry_run_rejects_write_batch():
|
||||||
|
"""--dry-run must not emit a batch file (it must not mutate anything)."""
|
||||||
|
if os.path.exists(BATCH_FILE):
|
||||||
|
os.unlink(BATCH_FILE)
|
||||||
|
cmd = _run(["--source-dir", SOURCE_DIR, "--dest-dir", DEST1,
|
||||||
|
"--dry-run", "--write-batch", BATCH_FILE])
|
||||||
|
result = subprocess.run(cmd, capture_output=True, text=True, timeout=180)
|
||||||
|
assert result.returncode != 0, result.stderr
|
||||||
|
assert not os.path.exists(BATCH_FILE), "dry-run must not create a batch file"
|
||||||
@@ -43,6 +43,9 @@ from common import (
|
|||||||
_find_free_port,
|
_find_free_port,
|
||||||
_wait_for_port,
|
_wait_for_port,
|
||||||
)
|
)
|
||||||
|
# The dry-run no-mutation contract is asserted with the same structural snapshot
|
||||||
|
# (mode/inode/mtime/xattr/content) the feature suite uses.
|
||||||
|
from test_features import _snapshot_tree
|
||||||
|
|
||||||
SOURCE_DIR = os.path.join(TEST_DATA_DIR, "daemon_source")
|
SOURCE_DIR = os.path.join(TEST_DATA_DIR, "daemon_source")
|
||||||
MODULE_ROOT = os.path.join(TEST_DATA_DIR, "daemon_modules")
|
MODULE_ROOT = os.path.join(TEST_DATA_DIR, "daemon_modules")
|
||||||
@@ -51,6 +54,7 @@ READONLY_MODULE = os.path.join(MODULE_ROOT, "readonly")
|
|||||||
AUTH_MODULE = os.path.join(MODULE_ROOT, "auth")
|
AUTH_MODULE = os.path.join(MODULE_ROOT, "auth")
|
||||||
TEAM_MODULE = os.path.join(MODULE_ROOT, "team")
|
TEAM_MODULE = os.path.join(MODULE_ROOT, "team")
|
||||||
OWNER_MODULE = os.path.join(MODULE_ROOT, "owner")
|
OWNER_MODULE = os.path.join(MODULE_ROOT, "owner")
|
||||||
|
DENIED_MODULE = os.path.join(MODULE_ROOT, "denied")
|
||||||
CONF_FILE = os.path.join(TEST_DATA_DIR, "fastsyncd.conf")
|
CONF_FILE = os.path.join(TEST_DATA_DIR, "fastsyncd.conf")
|
||||||
CRED_FILE = os.path.join(TEST_DATA_DIR, "fastsyncd.passwd")
|
CRED_FILE = os.path.join(TEST_DATA_DIR, "fastsyncd.passwd")
|
||||||
STARTFAIL_CONF = os.path.join(TEST_DATA_DIR, "fastsyncd_startfail.conf")
|
STARTFAIL_CONF = os.path.join(TEST_DATA_DIR, "fastsyncd_startfail.conf")
|
||||||
@@ -134,7 +138,7 @@ class DaemonManager:
|
|||||||
self._proc = None
|
self._proc = None
|
||||||
self._port = None
|
self._port = None
|
||||||
|
|
||||||
def start(self, config_path, port_override=None, extra_args=None):
|
def start(self, config_path, port_override=None, extra_args=None, log_path=None):
|
||||||
self.stop()
|
self.stop()
|
||||||
# When no override is given the daemon binds the config file's `port`
|
# When no override is given the daemon binds the config file's `port`
|
||||||
# (the plain config-port path); with an override the --dparam path.
|
# (the plain config-port path); with an override the --dparam path.
|
||||||
@@ -145,7 +149,8 @@ class DaemonManager:
|
|||||||
cmd += ["--dparam", f"port={port_override}"]
|
cmd += ["--dparam", f"port={port_override}"]
|
||||||
if extra_args:
|
if extra_args:
|
||||||
cmd += extra_args
|
cmd += extra_args
|
||||||
log_path = os.path.join(TEST_DATA_DIR, "fastsyncd.log")
|
if log_path is None:
|
||||||
|
log_path = os.path.join(TEST_DATA_DIR, "fastsyncd.log")
|
||||||
log = open(log_path, "w")
|
log = open(log_path, "w")
|
||||||
self._proc = subprocess.Popen(
|
self._proc = subprocess.Popen(
|
||||||
cmd, stdout=log, stderr=log, stdin=subprocess.DEVNULL, start_new_session=True)
|
cmd, stdout=log, stderr=log, stdin=subprocess.DEVNULL, start_new_session=True)
|
||||||
@@ -191,7 +196,7 @@ def _config_port(config_path):
|
|||||||
@pytest.fixture(scope="module", autouse=True)
|
@pytest.fixture(scope="module", autouse=True)
|
||||||
def daemon_env():
|
def daemon_env():
|
||||||
for d in (MODULE_ROOT, FILES_MODULE, READONLY_MODULE, AUTH_MODULE, TEAM_MODULE, OWNER_MODULE,
|
for d in (MODULE_ROOT, FILES_MODULE, READONLY_MODULE, AUTH_MODULE, TEAM_MODULE, OWNER_MODULE,
|
||||||
DETACH_MODULE):
|
DENIED_MODULE, DETACH_MODULE):
|
||||||
shutil.rmtree(d, ignore_errors=True)
|
shutil.rmtree(d, ignore_errors=True)
|
||||||
os.makedirs(d, exist_ok=True)
|
os.makedirs(d, exist_ok=True)
|
||||||
generate_test_files(SOURCE_DIR, full=False)
|
generate_test_files(SOURCE_DIR, full=False)
|
||||||
@@ -231,7 +236,12 @@ def daemon_env():
|
|||||||
"[owner]\n"
|
"[owner]\n"
|
||||||
"path = %s\n"
|
"path = %s\n"
|
||||||
"client owner = yes\n"
|
"client owner = yes\n"
|
||||||
% (config_port, FILES_MODULE, READONLY_MODULE, AUTH_MODULE, TEAM_MODULE, OWNER_MODULE))
|
"\n"
|
||||||
|
"[denied]\n"
|
||||||
|
"path = %s\n"
|
||||||
|
"hosts deny = 127.0.0.1\n"
|
||||||
|
% (config_port, FILES_MODULE, READONLY_MODULE, AUTH_MODULE, TEAM_MODULE, OWNER_MODULE,
|
||||||
|
DENIED_MODULE))
|
||||||
|
|
||||||
# A dedicated config for the fail-closed startup check: an auth-required
|
# A dedicated config for the fail-closed startup check: an auth-required
|
||||||
# module with no credential store must refuse to start. Its own free port
|
# module with no credential store must refuse to start. Its own free port
|
||||||
@@ -348,6 +358,33 @@ class TestDaemonRejection:
|
|||||||
assert result.returncode != 0
|
assert result.returncode != 0
|
||||||
assert self._tree_files() == before, "read-only rejection wrote under the module root"
|
assert self._tree_files() == before, "read-only rejection wrote under the module root"
|
||||||
|
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_read_only_module_allows_dry_run(self, daemon):
|
||||||
|
"""A server-contacting --dry-run IS a read-only wire operation, so a
|
||||||
|
`read only = yes` module is the safest dry-run target and must accept it
|
||||||
|
while writing nothing."""
|
||||||
|
result, _ = run_client(SOURCE_DIR, "127.0.0.1::readonly", flags=["--dry-run"],
|
||||||
|
port=daemon.port)
|
||||||
|
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||||
|
assert "Dry run:" in result.stdout, result.stdout[:200]
|
||||||
|
assert _tree_file_count(READONLY_MODULE) == 0, "read-only dry-run wrote a file"
|
||||||
|
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_module_dry_run_mutates_nothing(self, daemon):
|
||||||
|
"""A daemon-module dry-run reports would-transfer entries but leaves the
|
||||||
|
module tree structurally identical (mode/inode/mtime/xattr/content)."""
|
||||||
|
result = _push("127.0.0.1::files", daemon.port)
|
||||||
|
assert result.returncode == 0, result.stderr or result.stdout
|
||||||
|
before = _snapshot_tree(FILES_MODULE)
|
||||||
|
# --ignore-times forces every regular file to be reported as
|
||||||
|
# would-transfer, so the dry-run exercises the receiver decision rather
|
||||||
|
# than an all-skip shortcut -- while still mutating nothing.
|
||||||
|
result, _ = run_client(SOURCE_DIR, "127.0.0.1::files",
|
||||||
|
flags=["--dry-run", "--ignore-times"], port=daemon.port)
|
||||||
|
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||||
|
assert "Dry run:" in result.stdout, result.stdout[:200]
|
||||||
|
assert _snapshot_tree(FILES_MODULE) == before, "daemon dry-run mutated the module root"
|
||||||
|
|
||||||
def test_unknown_module_rejected(self, daemon):
|
def test_unknown_module_rejected(self, daemon):
|
||||||
result = _push("127.0.0.1::no-such-module", daemon.port)
|
result = _push("127.0.0.1::no-such-module", daemon.port)
|
||||||
assert result.returncode != 0
|
assert result.returncode != 0
|
||||||
@@ -368,6 +405,15 @@ class TestDaemonRejection:
|
|||||||
result = _push("127.0.0.1::/sub", daemon.port)
|
result = _push("127.0.0.1::/sub", daemon.port)
|
||||||
assert result.returncode != 0
|
assert result.returncode != 0
|
||||||
|
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_hosts_deny_rejects_loopback(self, daemon):
|
||||||
|
"""Host access control: a module with `hosts deny = 127.0.0.1` refuses a
|
||||||
|
loopback client at the config gate, before any data is exchanged."""
|
||||||
|
before = self._tree_files()
|
||||||
|
result = _push("127.0.0.1::denied", daemon.port)
|
||||||
|
assert result.returncode != 0
|
||||||
|
assert self._tree_files() == before, "host-denied connection wrote under the module root"
|
||||||
|
|
||||||
def test_dotdot_destination_rejected(self, daemon):
|
def test_dotdot_destination_rejected(self, daemon):
|
||||||
"""A '..' path expansion in the module-relative path is refused at parse
|
"""A '..' path expansion in the module-relative path is refused at parse
|
||||||
time so a client cannot escape the module root while it is still on the
|
time so a client cannot escape the module root while it is still on the
|
||||||
@@ -1193,3 +1239,80 @@ class TestDaemonTLSAuth:
|
|||||||
d.stop()
|
d.stop()
|
||||||
os.unlink(client_creds)
|
os.unlink(client_creds)
|
||||||
shutil.rmtree(cert_dir, ignore_errors=True)
|
shutil.rmtree(cert_dir, ignore_errors=True)
|
||||||
|
|
||||||
|
|
||||||
|
class TestDaemonConnectionLimits:
|
||||||
|
"""Wave 8: cross-process per-module / per-source connection caps and the
|
||||||
|
shared auth lockout. Each test boots its own daemon with a unique port so
|
||||||
|
the shared (per-daemon) registry state is isolated from the module-scoped
|
||||||
|
`daemon` fixture."""
|
||||||
|
|
||||||
|
LOCKOUT_CONF = os.path.join(TEST_DATA_DIR, "fastsyncd_lockout.conf")
|
||||||
|
CAPS_CONF = os.path.join(TEST_DATA_DIR, "fastsyncd_caps.conf")
|
||||||
|
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_auth_lockout_exempts_trusted_loopback(self):
|
||||||
|
"""`auth lockout threshold = 1`: a trusted loopback peer is EXEMPT from
|
||||||
|
the shared lockout because every local client shares the 127.0.0.1
|
||||||
|
identity, so a single wrong password must not lock out correct-password
|
||||||
|
attempts (that would be a local denial of service). The shared
|
||||||
|
per-source lockout machinery itself is covered by the daemon_limits unit
|
||||||
|
tests; this locks in the loopback policy and the absence of a stale
|
||||||
|
"locked out" log line."""
|
||||||
|
port = _find_free_port()
|
||||||
|
with open(self.LOCKOUT_CONF, "w") as f:
|
||||||
|
f.write("port = %d\n"
|
||||||
|
"auth lockout threshold = 1\n"
|
||||||
|
"auth lockout duration = 300\n"
|
||||||
|
"\n"
|
||||||
|
"[locked]\n"
|
||||||
|
"path = %s\n"
|
||||||
|
"auth users = alice\n"
|
||||||
|
% (port, AUTH_MODULE))
|
||||||
|
d = DaemonManager()
|
||||||
|
log_path = os.path.join(TEST_DATA_DIR, f"fastsyncd_lockout_{os.getpid()}.log")
|
||||||
|
try:
|
||||||
|
d.start(self.LOCKOUT_CONF, port_override=port, extra_args=["--password-file", CRED_FILE],
|
||||||
|
log_path=log_path)
|
||||||
|
log_before = os.path.getsize(log_path) if os.path.exists(log_path) else 0
|
||||||
|
# First attempt: wrong password -> a failure is logged, but a loopback
|
||||||
|
# peer is not counted toward the lockout.
|
||||||
|
wrong = _push_with_creds("127.0.0.1::locked", port, "alice", WRONG_PASS)
|
||||||
|
assert wrong.returncode != 0
|
||||||
|
# Second attempt: the correct password from the same local source must
|
||||||
|
# still be accepted (no lockout), which also runs the SCRAM handshake
|
||||||
|
# to completion in a fresh forked child.
|
||||||
|
right = _push_with_creds("127.0.0.1::locked", port, "alice", ALICE_PASS)
|
||||||
|
assert right.returncode == 0, (right.stderr or right.stdout)
|
||||||
|
time.sleep(0.3)
|
||||||
|
with open(log_path, "rb") as f:
|
||||||
|
f.seek(log_before)
|
||||||
|
tail = f.read().decode("utf-8", "replace")
|
||||||
|
assert "locked out" not in tail, tail[-400:]
|
||||||
|
finally:
|
||||||
|
d.stop()
|
||||||
|
|
||||||
|
def test_caps_keys_accepted_and_transfer_still_works(self):
|
||||||
|
"""A daemon configured with the new keys (per-host cap, lockout threshold
|
||||||
|
and duration, per-module cap) starts and serves a normal transfer."""
|
||||||
|
port = _find_free_port()
|
||||||
|
with open(self.CAPS_CONF, "w") as f:
|
||||||
|
f.write("port = %d\n"
|
||||||
|
"max connections per host = 5\n"
|
||||||
|
"auth lockout threshold = 3\n"
|
||||||
|
"auth lockout duration = 60\n"
|
||||||
|
"\n"
|
||||||
|
"[files]\n"
|
||||||
|
"path = %s\n"
|
||||||
|
"max connections = 2\n"
|
||||||
|
% (port, FILES_MODULE))
|
||||||
|
d = DaemonManager()
|
||||||
|
try:
|
||||||
|
d.start(self.CAPS_CONF, port_override=port)
|
||||||
|
result = _push("127.0.0.1::files", port)
|
||||||
|
assert result.returncode == 0, result.stderr or result.stdout
|
||||||
|
received = get_dest_received_dir(FILES_MODULE, SOURCE_DIR)
|
||||||
|
_, missing = verify_transfer(SOURCE_DIR, received)
|
||||||
|
assert not missing, f"missing: {missing[:5]}"
|
||||||
|
finally:
|
||||||
|
d.stop()
|
||||||
|
|||||||
@@ -0,0 +1,367 @@
|
|||||||
|
"""Fault injection: the server must survive truncated / corrupted protocol
|
||||||
|
frames and abrupt mid-frame disconnects, and keep serving later connections.
|
||||||
|
|
||||||
|
These tests deliberately speak raw bytes to a real server process:
|
||||||
|
|
||||||
|
* malformed frames before/inside the config handshake (oversized length
|
||||||
|
headers, truncated string bodies, outright garbage),
|
||||||
|
* a captured *valid* config frame replayed so the connection reaches the
|
||||||
|
operation loop, followed by a partial ``STATUS_MANIFEST`` frame that is cut
|
||||||
|
mid-body and dropped, and
|
||||||
|
* a real client run relayed through a proxy that truncates the stream at a
|
||||||
|
range of byte offsets and resets both ends.
|
||||||
|
|
||||||
|
After every fault the server process is asserted alive and a subsequent
|
||||||
|
ordinary transfer must complete and verify, proving the accept loop and
|
||||||
|
per-connection children recovered cleanly. All interactions are bounded by
|
||||||
|
short socket timeouts (no sleeps).
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
import select
|
||||||
|
import shutil
|
||||||
|
import socket
|
||||||
|
import struct
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import threading
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
sys.path.insert(0, os.path.dirname(__file__))
|
||||||
|
from common import ( # noqa: E402
|
||||||
|
ServerManager,
|
||||||
|
TEST_DATA_DIR,
|
||||||
|
get_dest_received_dir,
|
||||||
|
run_client,
|
||||||
|
verify_transfer,
|
||||||
|
)
|
||||||
|
|
||||||
|
PROTOCOL_VERSION = b"2.21.0"
|
||||||
|
STATUS_MANIFEST = 5
|
||||||
|
STATUS_OK = 0
|
||||||
|
|
||||||
|
SOURCE_DIR = os.path.join(TEST_DATA_DIR, "fault_src")
|
||||||
|
DEST_DIR = os.path.join(TEST_DATA_DIR, "fault_dst")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="module")
|
||||||
|
def fault_server():
|
||||||
|
"""A dedicated server so the aliveness assertions observe exactly the
|
||||||
|
process these faults were sent to."""
|
||||||
|
server = ServerManager()
|
||||||
|
server.start()
|
||||||
|
yield server
|
||||||
|
server.stop()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="module", autouse=True)
|
||||||
|
def _seed_source():
|
||||||
|
if os.path.exists(SOURCE_DIR):
|
||||||
|
shutil.rmtree(SOURCE_DIR)
|
||||||
|
os.makedirs(os.path.join(SOURCE_DIR, "nested"))
|
||||||
|
with open(os.path.join(SOURCE_DIR, "hello.txt"), "wb") as fh:
|
||||||
|
fh.write(b"fault injection payload\n" * 64)
|
||||||
|
with open(os.path.join(SOURCE_DIR, "nested", "deep.bin"), "wb") as fh:
|
||||||
|
fh.write(bytes(range(256)) * 16)
|
||||||
|
yield
|
||||||
|
shutil.rmtree(SOURCE_DIR, ignore_errors=True)
|
||||||
|
shutil.rmtree(DEST_DIR, ignore_errors=True)
|
||||||
|
|
||||||
|
|
||||||
|
def _assert_alive(server):
|
||||||
|
assert server._proc is not None, "server process missing"
|
||||||
|
assert server._proc.poll() is None, (
|
||||||
|
f"server exited with {server._proc.returncode} after fault injection"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _recover(server, label):
|
||||||
|
"""Run one ordinary transfer and verify it end-to-end."""
|
||||||
|
shutil.rmtree(DEST_DIR, ignore_errors=True)
|
||||||
|
os.makedirs(DEST_DIR)
|
||||||
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["--preserve"], port=server.port)
|
||||||
|
assert result.returncode == 0, (
|
||||||
|
f"{label}: recovery transfer failed rc={result.returncode}: "
|
||||||
|
f"{(result.stderr or result.stdout)[:200]}"
|
||||||
|
)
|
||||||
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
||||||
|
mismatches, missing = verify_transfer(SOURCE_DIR, received)
|
||||||
|
assert not missing, f"{label}: recovery missing {missing}"
|
||||||
|
assert not mismatches, f"{label}: recovery mismatch {mismatches}"
|
||||||
|
|
||||||
|
|
||||||
|
def _abrupt_close(sock):
|
||||||
|
"""Force an RST instead of a graceful FIN, the nastier mid-frame drop."""
|
||||||
|
try:
|
||||||
|
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack("ii", 1, 0))
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
try:
|
||||||
|
sock.close()
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def _raw_connect(server):
|
||||||
|
sock = socket.create_connection(("127.0.0.1", server.port), timeout=5)
|
||||||
|
sock.settimeout(5)
|
||||||
|
return sock
|
||||||
|
|
||||||
|
|
||||||
|
def _recv_exact(sock, n):
|
||||||
|
buf = b""
|
||||||
|
while len(buf) < n:
|
||||||
|
chunk = sock.recv(n - len(buf))
|
||||||
|
if not chunk:
|
||||||
|
return None
|
||||||
|
buf += chunk
|
||||||
|
return buf
|
||||||
|
|
||||||
|
|
||||||
|
# --- faults before/inside the config handshake -----------------------------
|
||||||
|
|
||||||
|
CONFIG_HANDSHAKE_FAULTS = {
|
||||||
|
"empty": b"",
|
||||||
|
# Length header claims a 1 EiB string body that never arrives.
|
||||||
|
"oversized_length": struct.pack("<Q", 1 << 60),
|
||||||
|
# A truncated 8-byte length header (only 3 bytes of it are sent).
|
||||||
|
"truncated_length_header": b"\x10\x00\x00",
|
||||||
|
# A valid version string followed by a string-length header whose body is
|
||||||
|
# deliberately truncated (mid-config-frame disconnect).
|
||||||
|
"truncated_config_body": struct.pack("<Q", len(PROTOCOL_VERSION)) + PROTOCOL_VERSION
|
||||||
|
+ struct.pack("<Q", 4096)
|
||||||
|
+ b"partial",
|
||||||
|
# Pure garbage that is not a valid frame at any offset.
|
||||||
|
"garbage": b"\xff" * 32,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class TestConfigHandshakeFaults:
|
||||||
|
def test_truncated_and_corrupt_config_frames(self, fault_server):
|
||||||
|
for name, payload in CONFIG_HANDSHAKE_FAULTS.items():
|
||||||
|
sock = _raw_connect(fault_server)
|
||||||
|
if payload:
|
||||||
|
sock.sendall(payload)
|
||||||
|
_abrupt_close(sock)
|
||||||
|
_assert_alive(fault_server)
|
||||||
|
_recover(fault_server, "config handshake faults")
|
||||||
|
|
||||||
|
|
||||||
|
# --- capture a valid config frame, then truncate a STATUS_MANIFEST ----------
|
||||||
|
|
||||||
|
|
||||||
|
class _CaptureProxy:
|
||||||
|
"""Relay one client<->server connection and record the client's config
|
||||||
|
frame (all client bytes forwarded before the server's first reply)."""
|
||||||
|
|
||||||
|
def __init__(self, target_port):
|
||||||
|
self.target = ("127.0.0.1", target_port)
|
||||||
|
self.listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||||
|
self.listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||||
|
self.listener.bind(("127.0.0.1", 0))
|
||||||
|
self.listener.listen(1)
|
||||||
|
self.listener.settimeout(20)
|
||||||
|
self.port = self.listener.getsockname()[1]
|
||||||
|
self.config_frame = None
|
||||||
|
|
||||||
|
def run(self, cmd):
|
||||||
|
def serve():
|
||||||
|
try:
|
||||||
|
client, _ = self.listener.accept()
|
||||||
|
except OSError:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
backend = socket.create_connection(self.target, timeout=10)
|
||||||
|
except OSError:
|
||||||
|
client.close()
|
||||||
|
return
|
||||||
|
client.settimeout(20)
|
||||||
|
backend.settimeout(20)
|
||||||
|
buf_c = bytearray()
|
||||||
|
seen_server = False
|
||||||
|
try:
|
||||||
|
while True:
|
||||||
|
ready, _, _ = select.select([client, backend], [], [], 20)
|
||||||
|
if not ready:
|
||||||
|
break
|
||||||
|
done = False
|
||||||
|
for sock in ready:
|
||||||
|
data = sock.recv(65536)
|
||||||
|
if not data:
|
||||||
|
done = True
|
||||||
|
continue
|
||||||
|
if sock is client:
|
||||||
|
buf_c += data
|
||||||
|
backend.sendall(data)
|
||||||
|
else:
|
||||||
|
if not seen_server:
|
||||||
|
seen_server = True
|
||||||
|
self.config_frame = bytes(buf_c)
|
||||||
|
client.sendall(data)
|
||||||
|
if done:
|
||||||
|
break
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
finally:
|
||||||
|
client.close()
|
||||||
|
backend.close()
|
||||||
|
|
||||||
|
thread = threading.Thread(target=serve)
|
||||||
|
thread.start()
|
||||||
|
result = subprocess.run(cmd, capture_output=True, text=True, timeout=60)
|
||||||
|
thread.join(20)
|
||||||
|
return result
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
try:
|
||||||
|
self.listener.close()
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope="module")
|
||||||
|
def captured_config(fault_server):
|
||||||
|
"""Capture the config frame of one real client run through a relay."""
|
||||||
|
proxy = _CaptureProxy(fault_server.port)
|
||||||
|
cmd = [
|
||||||
|
os.path.join(os.path.dirname(__file__), "..", "..", "build", "client"),
|
||||||
|
"--source-dir",
|
||||||
|
SOURCE_DIR,
|
||||||
|
"--dest-dir",
|
||||||
|
DEST_DIR,
|
||||||
|
"--save-to-disk",
|
||||||
|
"--server-port",
|
||||||
|
str(proxy.port),
|
||||||
|
]
|
||||||
|
try:
|
||||||
|
result = proxy.run(cmd)
|
||||||
|
assert result.returncode == 0, (
|
||||||
|
f"capture run failed rc={result.returncode}: "
|
||||||
|
f"{(result.stderr or result.stdout)[:200]}"
|
||||||
|
)
|
||||||
|
assert proxy.config_frame, "failed to capture the client config frame"
|
||||||
|
yield proxy.config_frame
|
||||||
|
finally:
|
||||||
|
proxy.close()
|
||||||
|
|
||||||
|
|
||||||
|
class TestTruncatedStatusFrame:
|
||||||
|
def test_partial_manifest_frame_then_drop(self, fault_server, captured_config):
|
||||||
|
sock = _raw_connect(fault_server)
|
||||||
|
sock.sendall(captured_config)
|
||||||
|
ack = _recv_exact(sock, 4)
|
||||||
|
assert ack is not None, "server closed before the config ack"
|
||||||
|
(status,) = struct.unpack("<i", ack)
|
||||||
|
assert status == STATUS_OK, f"expected STATUS_OK, got {status}"
|
||||||
|
|
||||||
|
# STATUS_MANIFEST, then only half of the keep-count int, then an RST.
|
||||||
|
sock.sendall(struct.pack("<i", STATUS_MANIFEST) + b"\x02\x00")
|
||||||
|
_abrupt_close(sock)
|
||||||
|
|
||||||
|
_assert_alive(fault_server)
|
||||||
|
_recover(fault_server, "truncated manifest frame")
|
||||||
|
|
||||||
|
def test_manifest_count_without_sections(self, fault_server, captured_config):
|
||||||
|
"""A syntactically valid STATUS_MANIFEST whose bodies never arrive."""
|
||||||
|
sock = _raw_connect(fault_server)
|
||||||
|
sock.sendall(captured_config)
|
||||||
|
assert _recv_exact(sock, 4) is not None
|
||||||
|
|
||||||
|
sock.sendall(struct.pack("<i", STATUS_MANIFEST) + struct.pack("<i", 3))
|
||||||
|
# Announce three keeps but send none; then drop.
|
||||||
|
_abrupt_close(sock)
|
||||||
|
|
||||||
|
_assert_alive(fault_server)
|
||||||
|
_recover(fault_server, "manifest body truncation")
|
||||||
|
|
||||||
|
|
||||||
|
# --- abrupt truncation of a real transfer ----------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class _TruncatingProxy:
|
||||||
|
"""Forward at most ``max_client_bytes`` from client to server, then reset
|
||||||
|
both ends mid-stream. Runs one client command (which is expected to fail)."""
|
||||||
|
|
||||||
|
def __init__(self, target_port, max_client_bytes):
|
||||||
|
self.target = ("127.0.0.1", target_port)
|
||||||
|
self.max_client_bytes = max_client_bytes
|
||||||
|
self.listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||||
|
self.listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||||
|
self.listener.bind(("127.0.0.1", 0))
|
||||||
|
self.listener.listen(1)
|
||||||
|
self.listener.settimeout(20)
|
||||||
|
self.port = self.listener.getsockname()[1]
|
||||||
|
|
||||||
|
def run(self, cmd):
|
||||||
|
def serve():
|
||||||
|
try:
|
||||||
|
client, _ = self.listener.accept()
|
||||||
|
except OSError:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
backend = socket.create_connection(self.target, timeout=10)
|
||||||
|
except OSError:
|
||||||
|
client.close()
|
||||||
|
return
|
||||||
|
# A short receive timeout bounds the case where the client has
|
||||||
|
# nothing left to send and is waiting on the server: the proxy then
|
||||||
|
# cuts the stream anyway instead of stalling the test.
|
||||||
|
client.settimeout(2)
|
||||||
|
backend.settimeout(20)
|
||||||
|
forwarded = 0
|
||||||
|
try:
|
||||||
|
while forwarded < self.max_client_bytes:
|
||||||
|
data = client.recv(65536)
|
||||||
|
if not data:
|
||||||
|
break
|
||||||
|
room = self.max_client_bytes - forwarded
|
||||||
|
take = data[:room]
|
||||||
|
backend.sendall(take)
|
||||||
|
forwarded += len(take)
|
||||||
|
if forwarded >= self.max_client_bytes:
|
||||||
|
break
|
||||||
|
except (OSError, socket.timeout):
|
||||||
|
pass
|
||||||
|
for sock in (client, backend):
|
||||||
|
try:
|
||||||
|
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack("ii", 1, 0))
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
try:
|
||||||
|
sock.close()
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
thread = threading.Thread(target=serve)
|
||||||
|
thread.start()
|
||||||
|
try:
|
||||||
|
subprocess.run(cmd, capture_output=True, text=True, timeout=30)
|
||||||
|
finally:
|
||||||
|
thread.join(20)
|
||||||
|
self.listener.close()
|
||||||
|
|
||||||
|
|
||||||
|
class TestAbruptMidTransferDisconnect:
|
||||||
|
def test_client_stream_cut_at_offsets(self, fault_server, captured_config):
|
||||||
|
"""Cut the real client stream at offsets anchored to the config frame's
|
||||||
|
actual size: mid-config, right after the config, and into the operation
|
||||||
|
stream -- each followed by an RST of both ends."""
|
||||||
|
config_len = len(captured_config)
|
||||||
|
cuts = sorted({max(1, config_len // 2), max(1, config_len - 1), config_len + 8,
|
||||||
|
config_len + 256})
|
||||||
|
for cut in cuts:
|
||||||
|
proxy = _TruncatingProxy(fault_server.port, cut)
|
||||||
|
cmd = [
|
||||||
|
os.path.join(os.path.dirname(__file__), "..", "..", "build", "client"),
|
||||||
|
"--source-dir",
|
||||||
|
SOURCE_DIR,
|
||||||
|
"--dest-dir",
|
||||||
|
DEST_DIR,
|
||||||
|
"--save-to-disk",
|
||||||
|
"--server-port",
|
||||||
|
str(proxy.port),
|
||||||
|
]
|
||||||
|
# The client is expected to fail; what matters is the server survives.
|
||||||
|
proxy.run(cmd)
|
||||||
|
_assert_alive(fault_server)
|
||||||
|
_recover(fault_server, "abrupt mid-transfer disconnects")
|
||||||
@@ -370,6 +370,384 @@ class TestDryRun:
|
|||||||
assert not mismatches, f"Mismatch: {mismatches}"
|
assert not mismatches, f"Mismatch: {mismatches}"
|
||||||
|
|
||||||
|
|
||||||
|
def _snapshot_xattrs(path):
|
||||||
|
"""Return a stable, comparable tuple of (name, value) xattr pairs.
|
||||||
|
|
||||||
|
Returns None when the platform/filesystem does not expose xattrs so both
|
||||||
|
snapshots agree on "unavailable" instead of one being treated as changed."""
|
||||||
|
try:
|
||||||
|
names = os.listxattr(path, follow_symlinks=False)
|
||||||
|
except (AttributeError, OSError):
|
||||||
|
return None
|
||||||
|
if not names:
|
||||||
|
return ()
|
||||||
|
pairs = []
|
||||||
|
for name in sorted(names):
|
||||||
|
try:
|
||||||
|
value = os.getxattr(path, name, follow_symlinks=False)
|
||||||
|
except OSError:
|
||||||
|
value = None
|
||||||
|
pairs.append((name, value))
|
||||||
|
return tuple(pairs)
|
||||||
|
|
||||||
|
|
||||||
|
def _snapshot_tree(root):
|
||||||
|
"""Return a structural snapshot of a directory tree.
|
||||||
|
|
||||||
|
Every entry (including directories) is recorded as
|
||||||
|
(inode, mtime_ns, mode, xattrs, kind-specific payload) so a dry-run that
|
||||||
|
touched a mode, inode, mtime, xattr, or content is observable. Regular
|
||||||
|
files carry their size+bytes, symlinks their target, and special entries
|
||||||
|
(FIFO/socket/device) their size only -- opening a special file could block.
|
||||||
|
Returns an empty dict for a missing root so "nothing was created" is also
|
||||||
|
observable."""
|
||||||
|
snapshot = {}
|
||||||
|
if not os.path.exists(root):
|
||||||
|
return snapshot
|
||||||
|
for dirpath, dirnames, filenames in os.walk(root):
|
||||||
|
for name in list(dirnames) + filenames:
|
||||||
|
path = os.path.join(dirpath, name)
|
||||||
|
rel = os.path.relpath(path, root)
|
||||||
|
st = os.lstat(path)
|
||||||
|
entry = [st.st_ino, st.st_mtime_ns, stat.S_IMODE(st.st_mode), _snapshot_xattrs(path)]
|
||||||
|
if stat.S_ISLNK(st.st_mode):
|
||||||
|
entry.append(("symlink", os.readlink(path)))
|
||||||
|
elif stat.S_ISREG(st.st_mode):
|
||||||
|
with open(path, "rb") as fh:
|
||||||
|
data = fh.read()
|
||||||
|
entry += [st.st_size, data]
|
||||||
|
else:
|
||||||
|
entry.append(st.st_size)
|
||||||
|
snapshot[rel] = tuple(entry)
|
||||||
|
return snapshot
|
||||||
|
|
||||||
|
|
||||||
|
class TestRemoteDryRun:
|
||||||
|
"""Server-contacting --dry-run (protocol 2.21.0): contacts the receiver,
|
||||||
|
reports what WOULD transfer/skip based on receiver state, and mutates
|
||||||
|
nothing on either side."""
|
||||||
|
|
||||||
|
def _seed(self, source):
|
||||||
|
clean_dir(source)
|
||||||
|
os.makedirs(os.path.join(source, "nested"), exist_ok=True)
|
||||||
|
with open(os.path.join(source, "keep.txt"), "wb") as f:
|
||||||
|
f.write(b"unchanged content\n")
|
||||||
|
with open(os.path.join(source, "changed.txt"), "wb") as f:
|
||||||
|
f.write(b"original content\n")
|
||||||
|
with open(os.path.join(source, "nested", "deep.txt"), "wb") as f:
|
||||||
|
f.write(b"deep file\n")
|
||||||
|
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_remote_dry_run_reports_changes_and_mutates_nothing(self, shared_server):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "remote_dry_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "remote_dry_dst")
|
||||||
|
self._seed(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
|
||||||
|
# Populate the destination with a real transfer that preserves mtimes
|
||||||
|
# (--preserve), then make exactly one file differ (content+size) and add
|
||||||
|
# a brand-new file.
|
||||||
|
result, _ = run_client(source, dest, flags=["--preserve"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"seed transfer failed: {result.stderr[:200]}"
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
|
||||||
|
with open(os.path.join(source, "changed.txt"), "wb") as f:
|
||||||
|
f.write(b"a much longer replacement payload\n")
|
||||||
|
with open(os.path.join(source, "added.txt"), "wb") as f:
|
||||||
|
f.write(b"newly added\n")
|
||||||
|
|
||||||
|
before = _snapshot_tree(received)
|
||||||
|
# --checksum must NOT read destination contents in a dry-run (B3), so
|
||||||
|
# the up-to-date decision is metadata-only. The --preserve seed made
|
||||||
|
# keep.txt and deep.txt size+mtime-identical; the dry-run must also
|
||||||
|
# transmit metadata (--preserve) for that metadata to be comparable.
|
||||||
|
result, _ = run_client(source, dest, flags=["--dry-run", "--checksum", "--preserve"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"remote dry-run failed: {result.stderr[:300]}"
|
||||||
|
assert "Dry run:" in result.stdout, result.stdout[:200]
|
||||||
|
assert "changed.txt" in result.stdout, result.stdout
|
||||||
|
assert "added.txt" in result.stdout, result.stdout
|
||||||
|
assert "keep.txt" not in result.stdout, (
|
||||||
|
f"up-to-date file must not be reported as would-transfer: {result.stdout}"
|
||||||
|
)
|
||||||
|
assert "deep.txt" not in result.stdout, result.stdout
|
||||||
|
assert _snapshot_tree(received) == before, "remote dry-run mutated the destination"
|
||||||
|
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_remote_dry_run_checksum_does_not_read_destination(self, shared_server):
|
||||||
|
"""B3: --dry-run --checksum against a read-only module must not read the
|
||||||
|
destination file's content (a 1-bit hash oracle). A same-size/same-content
|
||||||
|
file whose mtime differs is therefore reported as would-transfer because
|
||||||
|
the metadata-only decision is inconclusive, instead of being hashed and
|
||||||
|
silently skipped."""
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "remote_dry_oracle_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "remote_dry_oracle_dst")
|
||||||
|
self._seed(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
result, _ = run_client(source, dest, flags=["--preserve"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, result.stderr[:200]
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
|
||||||
|
target = os.path.join(received, "keep.txt")
|
||||||
|
# Identical size and content, but a deliberately different mtime.
|
||||||
|
os.utime(target, (1000000000, 1000000000))
|
||||||
|
before = _snapshot_tree(received)
|
||||||
|
|
||||||
|
result, _ = run_client(source, dest, flags=["--dry-run", "--checksum", "--preserve"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, result.stderr[:300]
|
||||||
|
assert "keep.txt" in result.stdout, (
|
||||||
|
f"dry-run --checksum must not read the destination to prove equality: {result.stdout}"
|
||||||
|
)
|
||||||
|
assert _snapshot_tree(received) == before, "dry-run mutated the destination"
|
||||||
|
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_remote_dry_run_into_empty_dest_creates_nothing(self, shared_server):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "remote_dry_empty_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "remote_dry_empty_dst")
|
||||||
|
self._seed(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
assert not os.path.exists(received)
|
||||||
|
|
||||||
|
result, _ = run_client(source, dest, flags=["--dry-run"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"exit {result.returncode}: {result.stderr[:300]}"
|
||||||
|
assert "keep.txt" in result.stdout
|
||||||
|
assert "changed.txt" in result.stdout
|
||||||
|
assert "deep.txt" in result.stdout
|
||||||
|
# Nowhere may the receiver have created the destination mirror.
|
||||||
|
assert not os.path.exists(received), "dry-run created directories on the receiver"
|
||||||
|
assert _snapshot_tree(received) == {}
|
||||||
|
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_remote_dry_run_mkpath_does_not_create_root(self, shared_server):
|
||||||
|
"""A wire dry_run cannot make --mkpath create anything, and it cannot
|
||||||
|
relax the precondition either: a nonexistent root is rejected (a real
|
||||||
|
run without the created root is impossible in dry-run) while nothing is
|
||||||
|
created."""
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "remote_dry_mk_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "remote_dry_mk_dst")
|
||||||
|
self._seed(source)
|
||||||
|
shutil.rmtree(dest, ignore_errors=True)
|
||||||
|
assert not os.path.exists(dest)
|
||||||
|
|
||||||
|
result, _ = run_client(source, dest, flags=["--dry-run", "--mkpath"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode != 0, "dry-run --mkpath accepted a nonexistent receive root"
|
||||||
|
assert not os.path.exists(dest), "dry-run --mkpath created the destination root"
|
||||||
|
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_remote_dry_run_with_delete_does_not_delete(self, shared_server):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "remote_dry_del_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "remote_dry_del_dst")
|
||||||
|
self._seed(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
result, _ = run_client(source, dest, port=shared_server.port)
|
||||||
|
assert result.returncode == 0, result.stderr[:200]
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
extra = os.path.join(received, "extra.txt")
|
||||||
|
with open(extra, "wb") as f:
|
||||||
|
f.write(b"must survive a dry-run delete\n")
|
||||||
|
before = _snapshot_tree(received)
|
||||||
|
|
||||||
|
for flags in (["--dry-run", "--delete"], ["--dry-run", "--delete-after"]):
|
||||||
|
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"{flags}: {result.stderr[:300]}"
|
||||||
|
assert os.path.exists(extra), f"{flags} deleted an extra in dry-run"
|
||||||
|
assert _snapshot_tree(received) == before, f"{flags} mutated the destination"
|
||||||
|
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_remote_dry_run_quiet_is_silent(self, shared_server):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "remote_dry_quiet_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "remote_dry_quiet_dst")
|
||||||
|
self._seed(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
result, _ = run_client(source, dest, flags=["-q", "--dry-run"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, result.stderr[:300]
|
||||||
|
assert result.stdout == ""
|
||||||
|
assert result.stderr == ""
|
||||||
|
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_remote_dry_run_threaded_routes_to_server(self, shared_server):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "remote_dry_mt_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "remote_dry_mt_dst")
|
||||||
|
self._seed(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
result, _ = run_client(source, dest, flags=["--dry-run", "--threads"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, result.stderr[:300]
|
||||||
|
assert "changed.txt" in result.stdout
|
||||||
|
assert _snapshot_tree(get_dest_received_dir(dest, source)) == {}
|
||||||
|
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_normal_transfer_unaffected_by_dry_run(self, shared_server):
|
||||||
|
"""A real transfer after dry-run still installs the changes."""
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "remote_dry_normal_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "remote_dry_normal_dst")
|
||||||
|
self._seed(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
run_client(source, dest, port=shared_server.port)
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
with open(os.path.join(source, "changed.txt"), "wb") as f:
|
||||||
|
f.write(b"updated payload for the real transfer\n")
|
||||||
|
run_client(source, dest, flags=["--dry-run"], port=shared_server.port)
|
||||||
|
|
||||||
|
result, _ = run_client(source, dest, port=shared_server.port)
|
||||||
|
assert result.returncode == 0, result.stderr[:200]
|
||||||
|
with open(os.path.join(received, "changed.txt"), "rb") as f:
|
||||||
|
assert f.read() == b"updated payload for the real transfer\n"
|
||||||
|
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_remote_dry_run_delay_updates_mutates_nothing(self, shared_server):
|
||||||
|
"""--delay-updates stages under the receive root; a dry-run must neither
|
||||||
|
create that staging tree nor publish anything (mode/inode/mtime intact)."""
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "remote_dry_delay_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "remote_dry_delay_dst")
|
||||||
|
self._seed(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
result, _ = run_client(source, dest, flags=["--delay-updates"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, result.stderr[:200]
|
||||||
|
|
||||||
|
with open(os.path.join(source, "changed.txt"), "wb") as f:
|
||||||
|
f.write(b"changed for delay-updates dry-run\n")
|
||||||
|
before = _snapshot_tree(dest)
|
||||||
|
result, _ = run_client(source, dest, flags=["--dry-run", "--delay-updates"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, result.stderr[:300]
|
||||||
|
assert "changed.txt" in result.stdout, result.stdout
|
||||||
|
assert _snapshot_tree(dest) == before, "delay-updates dry-run mutated the destination"
|
||||||
|
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_remote_dry_run_backup_mutates_nothing(self, shared_server):
|
||||||
|
"""--backup would rename the old file aside; a dry-run must not."""
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "remote_dry_backup_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "remote_dry_backup_dst")
|
||||||
|
self._seed(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
result, _ = run_client(source, dest, port=shared_server.port)
|
||||||
|
assert result.returncode == 0, result.stderr[:200]
|
||||||
|
|
||||||
|
with open(os.path.join(source, "changed.txt"), "wb") as f:
|
||||||
|
f.write(b"changed for backup dry-run\n")
|
||||||
|
before = _snapshot_tree(dest)
|
||||||
|
result, _ = run_client(source, dest, flags=["--dry-run", "--backup"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, result.stderr[:300]
|
||||||
|
assert "changed.txt" in result.stdout, result.stdout
|
||||||
|
assert _snapshot_tree(dest) == before, "--backup dry-run mutated the destination"
|
||||||
|
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_remote_dry_run_symlink_mutates_nothing(self, shared_server):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "remote_dry_symlink_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "remote_dry_symlink_dst")
|
||||||
|
self._seed(source)
|
||||||
|
os.symlink("changed.txt", os.path.join(source, "link"))
|
||||||
|
clean_dir(dest)
|
||||||
|
result, _ = run_client(source, dest, flags=["-a"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, result.stderr[:200]
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
assert os.path.islink(os.path.join(received, "link"))
|
||||||
|
|
||||||
|
# Re-point the source link so the entry is genuinely stale, then prove a
|
||||||
|
# dry-run leaves the destination link target, inode, and mtime untouched.
|
||||||
|
os.unlink(os.path.join(source, "link"))
|
||||||
|
os.symlink("keep.txt", os.path.join(source, "link"))
|
||||||
|
before = _snapshot_tree(dest)
|
||||||
|
result, _ = run_client(source, dest, flags=["-a", "--dry-run"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, result.stderr[:300]
|
||||||
|
assert _snapshot_tree(dest) == before, "symlink dry-run mutated the destination"
|
||||||
|
assert os.readlink(os.path.join(received, "link")) == "changed.txt"
|
||||||
|
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_remote_dry_run_hardlink_mutates_nothing(self, shared_server):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "remote_dry_hardlink_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "remote_dry_hardlink_dst")
|
||||||
|
clean_dir(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
with open(os.path.join(source, "h1.txt"), "wb") as f:
|
||||||
|
f.write(b"hardlinked payload\n")
|
||||||
|
os.link(os.path.join(source, "h1.txt"), os.path.join(source, "h2.txt"))
|
||||||
|
result, _ = run_client(source, dest, flags=["-H"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, result.stderr[:200]
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
assert os.stat(os.path.join(received, "h1.txt")).st_ino == \
|
||||||
|
os.stat(os.path.join(received, "h2.txt")).st_ino
|
||||||
|
|
||||||
|
# Change the shared inode; both names are now stale in the destination.
|
||||||
|
with open(os.path.join(source, "h1.txt"), "wb") as f:
|
||||||
|
f.write(b"changed hardlinked payload\n")
|
||||||
|
before = _snapshot_tree(dest)
|
||||||
|
result, _ = run_client(source, dest, flags=["-H", "--dry-run"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, result.stderr[:300]
|
||||||
|
assert _snapshot_tree(dest) == before, "hardlink dry-run mutated the destination"
|
||||||
|
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_remote_dry_run_fifo_special_mutates_nothing(self, shared_server):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "remote_dry_fifo_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "remote_dry_fifo_dst")
|
||||||
|
clean_dir(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
with open(os.path.join(source, "plain.txt"), "wb") as f:
|
||||||
|
f.write(b"plain\n")
|
||||||
|
os.mkfifo(os.path.join(source, "existing.fifo"))
|
||||||
|
result, _ = run_client(source, dest, flags=["--specials"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, result.stderr[:200]
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
assert stat.S_ISFIFO(os.lstat(os.path.join(received, "existing.fifo")).st_mode)
|
||||||
|
|
||||||
|
os.mkfifo(os.path.join(source, "new.fifo"))
|
||||||
|
before = _snapshot_tree(dest)
|
||||||
|
result, _ = run_client(source, dest, flags=["--specials", "--dry-run"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, result.stderr[:300]
|
||||||
|
assert not os.path.exists(os.path.join(received, "new.fifo")), \
|
||||||
|
"dry-run created a FIFO on the receiver"
|
||||||
|
assert _snapshot_tree(dest) == before, "special-node dry-run mutated the destination"
|
||||||
|
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_read_batch_with_dry_run_is_refused(self, shared_server):
|
||||||
|
"""A dry-run of a local batch apply is meaningless (and must not become a
|
||||||
|
mutation escape hatch): the CLI rejects the combination up front."""
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "remote_dry_batch_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "remote_dry_batch_dst")
|
||||||
|
self._seed(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
result, _ = run_client(source, dest, flags=["--read-batch=/nonexistent.batch", "--dry-run"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode != 0, "read-batch + dry-run was accepted"
|
||||||
|
combined = (result.stderr or "") + (result.stdout or "")
|
||||||
|
assert "cannot be combined" in combined or "--dry-run" in combined, combined[:300]
|
||||||
|
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_remote_dry_run_bad_root_fails_like_real_run(self, shared_server):
|
||||||
|
"""A wire dry_run must not relax the destination-root precondition: a
|
||||||
|
missing or non-directory root that fails a real run fails a dry-run too,
|
||||||
|
and the dry-run must not create/replace anything."""
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "remote_dry_badroot_src")
|
||||||
|
self._seed(source)
|
||||||
|
|
||||||
|
missing = os.path.join(TEST_DATA_DIR, "remote_dry_badroot_missing")
|
||||||
|
shutil.rmtree(missing, ignore_errors=True)
|
||||||
|
real, _ = run_client(source, missing, port=shared_server.port)
|
||||||
|
assert real.returncode != 0, "real run accepted a missing receive root"
|
||||||
|
assert not os.path.exists(missing), "real run created the missing root"
|
||||||
|
dry, _ = run_client(source, missing, flags=["--dry-run"], port=shared_server.port)
|
||||||
|
assert dry.returncode != 0, "dry-run accepted a missing receive root a real run rejects"
|
||||||
|
assert not os.path.exists(missing), "dry-run created the missing receive root"
|
||||||
|
|
||||||
|
fileroot = os.path.join(TEST_DATA_DIR, "remote_dry_badroot_file")
|
||||||
|
shutil.rmtree(fileroot, ignore_errors=True)
|
||||||
|
with open(fileroot, "wb") as f:
|
||||||
|
f.write(b"i am a regular file, not a directory\n")
|
||||||
|
real, _ = run_client(source, fileroot, port=shared_server.port)
|
||||||
|
assert real.returncode != 0, "real run accepted a regular-file receive root"
|
||||||
|
dry, _ = run_client(source, fileroot, flags=["--dry-run"], port=shared_server.port)
|
||||||
|
assert dry.returncode != 0, "dry-run accepted a regular-file receive root a real run rejects"
|
||||||
|
with open(fileroot, "rb") as f:
|
||||||
|
assert f.read() == b"i am a regular file, not a directory\n", \
|
||||||
|
"dry-run clobbered a regular-file receive root"
|
||||||
|
|
||||||
|
|
||||||
class TestRemoveSourceFiles:
|
class TestRemoveSourceFiles:
|
||||||
def test_removes_only_transferred_regular_files(self, shared_server):
|
def test_removes_only_transferred_regular_files(self, shared_server):
|
||||||
source = os.path.join(TEST_DATA_DIR, "remove_source")
|
source = os.path.join(TEST_DATA_DIR, "remove_source")
|
||||||
@@ -1214,8 +1592,33 @@ class TestDelete:
|
|||||||
assert not missing, f"Missing: {missing}"
|
assert not missing, f"Missing: {missing}"
|
||||||
assert not mismatches, f"Mismatch: {mismatches}"
|
assert not mismatches, f"Mismatch: {mismatches}"
|
||||||
|
|
||||||
|
@pytest.mark.ci
|
||||||
class TestProgress:
|
def test_force_cannot_replace_directory_without_allow_delete(self):
|
||||||
|
"""C2: --force is deletion authority (an incoming file may recursively
|
||||||
|
remove a non-empty destination directory tree). A server started without
|
||||||
|
--allow-delete must clear it, so the operator's delete policy cannot be
|
||||||
|
bypassed with --force."""
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "force_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "force_dst")
|
||||||
|
clean_dir(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
with open(os.path.join(source, "blocker"), "wb") as f:
|
||||||
|
f.write(b"incoming file\n")
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
blocker = os.path.join(received, "blocker")
|
||||||
|
os.makedirs(blocker)
|
||||||
|
nested = os.path.join(blocker, "nested.txt")
|
||||||
|
with open(nested, "w") as f:
|
||||||
|
f.write("survivor")
|
||||||
|
# Deliberately NO --allow-delete.
|
||||||
|
server = ServerManager()
|
||||||
|
server.start()
|
||||||
|
try:
|
||||||
|
run_client(source, dest, flags=["--force"], port=server.port)
|
||||||
|
finally:
|
||||||
|
server.stop()
|
||||||
|
assert os.path.isdir(blocker), "unauthorized --force removed a destination directory"
|
||||||
|
assert os.path.exists(nested), "unauthorized --force removed a nested file"
|
||||||
def test_progress_output(self, shared_server):
|
def test_progress_output(self, shared_server):
|
||||||
clean_dir(DEST_DIR)
|
clean_dir(DEST_DIR)
|
||||||
result, dur = run_client(
|
result, dur = run_client(
|
||||||
@@ -1239,6 +1642,20 @@ class TestProgress:
|
|||||||
assert "Stats:" in result.stderr
|
assert "Stats:" in result.stderr
|
||||||
assert "KB" in result.stderr
|
assert "KB" in result.stderr
|
||||||
|
|
||||||
|
def test_human_readable_stats_multithreaded(self, shared_server):
|
||||||
|
# The multithreaded sender shares the single-threaded --stats format,
|
||||||
|
# including --human-readable and the rate suffix.
|
||||||
|
clean_dir(DEST_DIR)
|
||||||
|
result, dur = run_client(
|
||||||
|
SOURCE_DIR, DEST_DIR,
|
||||||
|
flags=["--threads", "-h", "--stats"],
|
||||||
|
port=shared_server.port,
|
||||||
|
)
|
||||||
|
assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}"
|
||||||
|
assert "Stats:" in result.stderr
|
||||||
|
assert "KB" in result.stderr
|
||||||
|
assert "/s" in result.stderr
|
||||||
|
|
||||||
def test_human_readable_progress_multithreaded(self, shared_server):
|
def test_human_readable_progress_multithreaded(self, shared_server):
|
||||||
clean_dir(DEST_DIR)
|
clean_dir(DEST_DIR)
|
||||||
result, dur = run_client(
|
result, dur = run_client(
|
||||||
@@ -3421,6 +3838,27 @@ class TestBasisDestDirs:
|
|||||||
assert _read_file(os.path.join(received, self.ADDED)) == \
|
assert _read_file(os.path.join(received, self.ADDED)) == \
|
||||||
self._source_tree("c")[self.ADDED], "added file not transferred"
|
self._source_tree("c")[self.ADDED], "added file not transferred"
|
||||||
|
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_dry_run_compare_dest_does_not_read_basis(self, shared_server):
|
||||||
|
# A dry-run --compare-dest must never read/hash the basis file: doing so
|
||||||
|
# is a 1-bit content oracle against the client-supplied digest. Even a
|
||||||
|
# byte-identical basis with a matching size+mtime is therefore reported
|
||||||
|
# as would-transfer, and nothing is created.
|
||||||
|
source = self._make_source("basis_dry_src", {self.UNCHANGED: b"stable content v1\n"})
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "basis_dry_dst")
|
||||||
|
clean_dir(dest)
|
||||||
|
self._seed_basis(dest, source, "drybasis", {self.UNCHANGED: b"stable content v1\n"})
|
||||||
|
before = _snapshot_tree(dest)
|
||||||
|
result, _ = run_client(source, dest,
|
||||||
|
flags=["--compare-dest=drybasis", "--dry-run"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, \
|
||||||
|
f"dry-run compare-dest failed: {result.stderr[:300]}"
|
||||||
|
assert self.UNCHANGED in result.stdout, (
|
||||||
|
"dry-run compare-dest silently skipped: receiver read the basis content"
|
||||||
|
)
|
||||||
|
assert _snapshot_tree(dest) == before, "dry-run compare-dest mutated the destination"
|
||||||
|
|
||||||
def test_compare_dest_content_mismatch_forces_transfer(self, shared_server):
|
def test_compare_dest_content_mismatch_forces_transfer(self, shared_server):
|
||||||
# The basis holds a file with a DIFFERENT body: even though it shares
|
# The basis holds a file with a DIFFERENT body: even though it shares
|
||||||
# the mtime pin, the xxHash check fails and the data must be sent.
|
# the mtime pin, the xxHash check fails and the data must be sent.
|
||||||
@@ -4217,6 +4655,61 @@ class TestSuperPrivilege:
|
|||||||
f"--no-super must suppress fake-super's owner replay: uid={st.st_uid} gid={st.st_gid}"
|
f"--no-super must suppress fake-super's owner replay: uid={st.st_uid} gid={st.st_gid}"
|
||||||
|
|
||||||
|
|
||||||
|
class TestStandaloneSuperDefault:
|
||||||
|
"""C3: a privileged (root) STANDALONE server without --allow-super forces
|
||||||
|
SUPER_MODE_OFF, so a client cannot make it create device nodes, write raw
|
||||||
|
devices, apply ownership, or use --copy-as. The shared_server fixture opts in
|
||||||
|
with --allow-super to keep the historical behavior available to the existing
|
||||||
|
root-only tests; these tests start their own un-opted server."""
|
||||||
|
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_copy_as_refused_without_allow_super(self):
|
||||||
|
"""--copy-as is a client-chosen-ownership request and must be refused by
|
||||||
|
a standalone server that did not opt in with --allow-super (on a non-root
|
||||||
|
receiver it is refused for lack of privilege either way)."""
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "super_default_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "super_default_dst")
|
||||||
|
clean_dir(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
with open(os.path.join(source, "f.txt"), "wb") as f:
|
||||||
|
f.write(b"no copy-as\n")
|
||||||
|
server = ServerManager()
|
||||||
|
server.start() # deliberately no --allow-super
|
||||||
|
try:
|
||||||
|
result, _ = run_client(source, dest,
|
||||||
|
flags=["--preserve", "--copy-as=@65534:@65534"],
|
||||||
|
port=server.port)
|
||||||
|
finally:
|
||||||
|
server.stop()
|
||||||
|
assert result.returncode != 0, (
|
||||||
|
"standalone server accepted --copy-as without --allow-super"
|
||||||
|
)
|
||||||
|
|
||||||
|
@pytest.mark.skipif(os.geteuid() != 0, reason="root can create the source device node")
|
||||||
|
def test_devices_skipped_without_allow_super(self):
|
||||||
|
"""Root standalone server without --allow-super must skip device-node
|
||||||
|
creation even for a client --devices request (the run still succeeds and
|
||||||
|
the regular file transfers)."""
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "super_default_dev_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "super_default_dev_dst")
|
||||||
|
clean_dir(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
with open(os.path.join(source, "plain.txt"), "wb") as f:
|
||||||
|
f.write(b"regular\n")
|
||||||
|
os.mknod(os.path.join(source, "null"), stat.S_IFCHR | 0o666, os.makedev(1, 3))
|
||||||
|
server = ServerManager()
|
||||||
|
server.start() # deliberately no --allow-super
|
||||||
|
try:
|
||||||
|
result, _ = run_client(source, dest, flags=["--devices"], port=server.port)
|
||||||
|
finally:
|
||||||
|
server.stop()
|
||||||
|
assert result.returncode == 0, f"exit {result.returncode}: {(result.stderr or '')[:200]}"
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
assert not os.path.lexists(os.path.join(received, "null")), (
|
||||||
|
"root standalone server created a device node without --allow-super"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class TestHardLinks:
|
class TestHardLinks:
|
||||||
"""-H/--hard-links: source files sharing an inode are re-created as hard
|
"""-H/--hard-links: source files sharing an inode are re-created as hard
|
||||||
links to one another on the destination (dedup preserved, first copy
|
links to one another on the destination (dedup preserved, first copy
|
||||||
|
|||||||
@@ -94,14 +94,14 @@ def _seed_protocol_source(source):
|
|||||||
class TestProtocol:
|
class TestProtocol:
|
||||||
@pytest.mark.ci
|
@pytest.mark.ci
|
||||||
def test_protocol_current_version_accepted(self, shared_server):
|
def test_protocol_current_version_accepted(self, shared_server):
|
||||||
"""--protocol=2.19.0 (the current PROTOCOL_VERSION) is accepted and the
|
"""--protocol=2.21.0 (the current PROTOCOL_VERSION) is accepted and the
|
||||||
transfer completes normally."""
|
transfer completes normally."""
|
||||||
source = os.path.join(TEST_DATA_DIR, "proto_ok_src")
|
source = os.path.join(TEST_DATA_DIR, "proto_ok_src")
|
||||||
dest = os.path.join(TEST_DATA_DIR, "proto_ok_dst")
|
dest = os.path.join(TEST_DATA_DIR, "proto_ok_dst")
|
||||||
shutil.rmtree(dest, ignore_errors=True)
|
shutil.rmtree(dest, ignore_errors=True)
|
||||||
os.makedirs(dest)
|
os.makedirs(dest)
|
||||||
_seed_protocol_source(source)
|
_seed_protocol_source(source)
|
||||||
result, _ = run_client(source, dest, flags=["--protocol=2.19.0"],
|
result, _ = run_client(source, dest, flags=["--protocol=2.21.0"],
|
||||||
port=shared_server.port)
|
port=shared_server.port)
|
||||||
assert result.returncode == 0, \
|
assert result.returncode == 0, \
|
||||||
f"--protocol current run failed: {(result.stderr or result.stdout)[:400]}"
|
f"--protocol current run failed: {(result.stderr or result.stdout)[:400]}"
|
||||||
@@ -118,7 +118,7 @@ class TestProtocol:
|
|||||||
shutil.rmtree(dest, ignore_errors=True)
|
shutil.rmtree(dest, ignore_errors=True)
|
||||||
os.makedirs(dest)
|
os.makedirs(dest)
|
||||||
_seed_protocol_source(source)
|
_seed_protocol_source(source)
|
||||||
for bad in ("2.18.0", "2.17.0", "2.15.0", "2.16.0", "216", "31"):
|
for bad in ("2.20.0", "2.19.0", "2.18.0", "2.17.0", "2.15.0", "2.16.0", "216", "31"):
|
||||||
result, _ = run_client(source, dest, flags=[f"--protocol={bad}"],
|
result, _ = run_client(source, dest, flags=[f"--protocol={bad}"],
|
||||||
port=shared_server.port)
|
port=shared_server.port)
|
||||||
assert result.returncode != 0, f"--protocol={bad} should be rejected"
|
assert result.returncode != 0, f"--protocol={bad} should be rejected"
|
||||||
|
|||||||
@@ -9,12 +9,15 @@
|
|||||||
#include "test_credentials.h"
|
#include "test_credentials.h"
|
||||||
#include "test_data.h"
|
#include "test_data.h"
|
||||||
#include "test_daemon_conf.h"
|
#include "test_daemon_conf.h"
|
||||||
|
#include "test_daemon_limits.h"
|
||||||
#include "test_delay_updates.h"
|
#include "test_delay_updates.h"
|
||||||
#include "test_delta.h"
|
#include "test_delta.h"
|
||||||
#include "test_file.h"
|
#include "test_file.h"
|
||||||
|
#include "test_file_list.h"
|
||||||
#include "test_file_sendfile.h"
|
#include "test_file_sendfile.h"
|
||||||
#include "test_fuzz_smoke.h"
|
#include "test_fuzz_smoke.h"
|
||||||
#include "test_glob.h"
|
#include "test_glob.h"
|
||||||
|
#include "test_hardlink.h"
|
||||||
#include "test_iconv.h"
|
#include "test_iconv.h"
|
||||||
#include "test_log.h"
|
#include "test_log.h"
|
||||||
#include "test_metadata.h"
|
#include "test_metadata.h"
|
||||||
@@ -22,7 +25,9 @@
|
|||||||
#include "test_multiprocessing.h"
|
#include "test_multiprocessing.h"
|
||||||
#include "test_property.h"
|
#include "test_property.h"
|
||||||
#include "test_protocol.h"
|
#include "test_protocol.h"
|
||||||
|
#include "test_protocol_error.h"
|
||||||
#include "test_queue.h"
|
#include "test_queue.h"
|
||||||
|
#include "test_receiver_timeout.h"
|
||||||
#include "test_robustness.h"
|
#include "test_robustness.h"
|
||||||
#include "test_scanner.h"
|
#include "test_scanner.h"
|
||||||
#include "test_server.h"
|
#include "test_server.h"
|
||||||
@@ -61,10 +66,13 @@ int main() {
|
|||||||
RUN_TEST(test_delta);
|
RUN_TEST(test_delta);
|
||||||
RUN_TEST(test_data);
|
RUN_TEST(test_data);
|
||||||
RUN_TEST(test_protocol);
|
RUN_TEST(test_protocol);
|
||||||
|
RUN_TEST(test_protocol_error);
|
||||||
|
RUN_TEST(test_receiver_timeout);
|
||||||
RUN_TEST(test_metadata);
|
RUN_TEST(test_metadata);
|
||||||
RUN_TEST(test_glob);
|
RUN_TEST(test_glob);
|
||||||
RUN_TEST(test_iconv);
|
RUN_TEST(test_iconv);
|
||||||
RUN_TEST(test_file);
|
RUN_TEST(test_file);
|
||||||
|
RUN_TEST(test_file_list);
|
||||||
RUN_TEST(test_trust_sender);
|
RUN_TEST(test_trust_sender);
|
||||||
RUN_TEST(test_delay_updates);
|
RUN_TEST(test_delay_updates);
|
||||||
RUN_TEST(test_file_sendfile);
|
RUN_TEST(test_file_sendfile);
|
||||||
@@ -80,10 +88,12 @@ int main() {
|
|||||||
RUN_TEST(test_client_cli);
|
RUN_TEST(test_client_cli);
|
||||||
RUN_TEST(test_server);
|
RUN_TEST(test_server);
|
||||||
RUN_TEST(test_daemon_conf);
|
RUN_TEST(test_daemon_conf);
|
||||||
|
RUN_TEST(test_daemon_limits);
|
||||||
RUN_TEST(test_motd);
|
RUN_TEST(test_motd);
|
||||||
RUN_TEST(test_server_cli);
|
RUN_TEST(test_server_cli);
|
||||||
RUN_TEST(test_fuzz_smoke);
|
RUN_TEST(test_fuzz_smoke);
|
||||||
RUN_TEST(test_xattr);
|
RUN_TEST(test_xattr);
|
||||||
|
RUN_TEST(test_hardlink);
|
||||||
|
|
||||||
printf("\n\033[1;36m=== TEST SUMMARY ===\033[0m\n");
|
printf("\n\033[1;36m=== TEST SUMMARY ===\033[0m\n");
|
||||||
printf("Total Tests Run: %d\n", tests_run);
|
printf("Total Tests Run: %d\n", tests_run);
|
||||||
|
|||||||
+19
-1
@@ -1,6 +1,7 @@
|
|||||||
#include "test_array_list.h"
|
#include "test_array_list.h"
|
||||||
#include "array_list.h"
|
#include "array_list.h"
|
||||||
#include "test_utils.h"
|
#include "test_utils.h"
|
||||||
|
#include <limits.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
|
|
||||||
static int destroyer_calls = 0;
|
static int destroyer_calls = 0;
|
||||||
@@ -9,7 +10,7 @@ static void test_destroyer(void* item) {
|
|||||||
free(item);
|
free(item);
|
||||||
}
|
}
|
||||||
|
|
||||||
void test_array_list() {
|
static void test_array_list_basic() {
|
||||||
ArrayList* list = array_list_create(free);
|
ArrayList* list = array_list_create(free);
|
||||||
EXPECT_NOT_NULL(list);
|
EXPECT_NOT_NULL(list);
|
||||||
EXPECT_EQ_INT(list->size, 0);
|
EXPECT_EQ_INT(list->size, 0);
|
||||||
@@ -54,3 +55,20 @@ void test_array_list() {
|
|||||||
array_list_delete(list);
|
array_list_delete(list);
|
||||||
EXPECT_EQ_INT(destroyer_calls, 106);
|
EXPECT_EQ_INT(destroyer_calls, 106);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* A capacity that would overflow `capacity * 2` must be refused instead of
|
||||||
|
* wrapping into signed-overflow UB; array_list_add surfaces the failure. */
|
||||||
|
static void test_array_list_extend_overflow_guard() {
|
||||||
|
ArrayList* list = array_list_create(NULL);
|
||||||
|
EXPECT_NOT_NULL(list);
|
||||||
|
list->capacity = INT_MAX / 2 + 1;
|
||||||
|
list->size = list->capacity;
|
||||||
|
EXPECT_FALSE(array_list_add(list, NULL));
|
||||||
|
list->size = 0;
|
||||||
|
array_list_delete(list);
|
||||||
|
}
|
||||||
|
|
||||||
|
void test_array_list() {
|
||||||
|
test_array_list_basic();
|
||||||
|
test_array_list_extend_overflow_guard();
|
||||||
|
}
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user