Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 TapTap
|
64b8a1a950 |
@@ -1,10 +0,0 @@
|
||||
BasedOnStyle: LLVM
|
||||
IndentWidth: 2
|
||||
ColumnLimit: 100
|
||||
PointerAlignment: Left
|
||||
AllowShortFunctionsOnASingleLine: None
|
||||
SortIncludes: false
|
||||
AllowShortIfStatementsOnASingleLine: false
|
||||
AllowShortLoopsOnASingleLine: false
|
||||
BinPackArguments: true
|
||||
BinPackParameters: true
|
||||
+36
-100
@@ -1,137 +1,73 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, dev]
|
||||
pull_request:
|
||||
workflow_dispatch:
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
lint:
|
||||
runs-on: ubuntu-latest
|
||||
container: gitea.tap-tap.win/taptap/fastsync-ci:v10
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: clang-format check
|
||||
run: find src/ tests/ -name '*.c' -o -name '*.h' | xargs clang-format --dry-run --Werror
|
||||
|
||||
- name: cppcheck
|
||||
run: cppcheck --enable=warning,style,performance,portability --suppress=missingIncludeSystem --error-exitcode=1 --inline-suppr src/ tests/
|
||||
|
||||
# Fast PR gate: build + unit tests + a representative subset of integration
|
||||
# tests (marked `ci`), parallelized with pytest-xdist. Only the full coverage
|
||||
# jobs below (sanitizers/fuzz/coverage/valgrind and the FULL integration
|
||||
# suite) run on merge to dev/main, so PR CI stays well under ~3 minutes.
|
||||
build-and-test:
|
||||
runs-on: ubuntu-latest
|
||||
container: gitea.tap-tap.win/taptap/fastsync-ci:v10
|
||||
needs: lint
|
||||
container: gitea.tap-tap.win/taptap/fastsync-ci:v6
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Configure
|
||||
run: cmake -B build -S . -DSTRICT_WARNINGS=ON
|
||||
run: cmake -B build -S .
|
||||
|
||||
- name: Build
|
||||
run: cmake --build build -j$(nproc)
|
||||
|
||||
- name: Unit Tests
|
||||
run: ctest --test-dir build --output-on-failure -j$(nproc)
|
||||
run: ./build/tests
|
||||
|
||||
- name: Integration Tests (PR smoke subset)
|
||||
if: github.event_name == 'pull_request'
|
||||
run: python3 -m pytest tests/integration/ -n 4 --dist=load -m ci --durations=25 --tb=short -q
|
||||
- name: Integration Tests
|
||||
run: python3 -m pytest tests/ -v --tb=short
|
||||
|
||||
- name: Integration Tests (full suite)
|
||||
if: github.event_name == 'push'
|
||||
run: python3 -m pytest tests/integration/ -n 4 --dist=load -m "not setpriv" --durations=25 --tb=short -q
|
||||
|
||||
sanitizers:
|
||||
sanitizer:
|
||||
runs-on: ubuntu-latest
|
||||
container: gitea.tap-tap.win/taptap/fastsync-ci:v10
|
||||
needs: lint
|
||||
if: github.event_name == 'push'
|
||||
strategy:
|
||||
matrix:
|
||||
sanitizer: [address, undefined]
|
||||
container: gitea.tap-tap.win/taptap/fastsync-ci:v6
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Configure
|
||||
run: cmake -B build-${{ matrix.sanitizer }} -S . -DSANITIZER=${{ matrix.sanitizer }}
|
||||
- name: Configure with ASan + UBSan
|
||||
run: >
|
||||
cmake -B build -S .
|
||||
-DCMAKE_C_FLAGS="-fsanitize=address,undefined -fno-omit-frame-pointer -g"
|
||||
-DCMAKE_EXE_LINKER_FLAGS="-fsanitize=address,undefined"
|
||||
|
||||
- name: Build
|
||||
run: cmake --build build-${{ matrix.sanitizer }} -j$(nproc)
|
||||
run: cmake --build build -j$(nproc)
|
||||
|
||||
- name: Unit Tests
|
||||
run: ctest --test-dir build-${{ matrix.sanitizer }} --output-on-failure
|
||||
run: ./build/tests
|
||||
|
||||
fuzz-build:
|
||||
- name: Integration Tests
|
||||
run: LSAN_OPTIONS=suppressions=.lsan-suppressions.txt python3 -m pytest tests/ -v --tb=short
|
||||
|
||||
clang-tidy:
|
||||
runs-on: ubuntu-latest
|
||||
container: gitea.tap-tap.win/taptap/fastsync-ci:v10
|
||||
needs: lint
|
||||
if: github.event_name == 'push'
|
||||
container: gitea.tap-tap.win/taptap/fastsync-ci:v6
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Configure (clang + fuzz)
|
||||
run: CC=clang CXX=clang++ cmake -B build-fuzz -S . -DENABLE_FUZZ=ON
|
||||
|
||||
- name: Build fuzz targets
|
||||
run: cmake --build build-fuzz -j$(nproc)
|
||||
|
||||
- name: Smoke fuzz targets
|
||||
- name: Install clang-tidy
|
||||
run: |
|
||||
for target in build-fuzz/fuzz_*; do
|
||||
[ -x "$target" ] || continue
|
||||
timeout 10s "$target" -runs=100 -max_total_time=5
|
||||
done
|
||||
apt-get update && apt-get install -y clang-tidy 2>/dev/null || \
|
||||
echo "::warning title=clang-tidy-skip::clang-tidy not available in container, skipping static analysis"
|
||||
|
||||
coverage:
|
||||
runs-on: ubuntu-latest
|
||||
container: gitea.tap-tap.win/taptap/fastsync-ci:v10
|
||||
needs: lint
|
||||
if: github.event_name == 'push'
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
- name: Configure (generate compile_commands.json)
|
||||
run: cmake -B build -S .
|
||||
|
||||
- name: Configure
|
||||
run: cmake -B build -S . -DENABLE_COVERAGE=ON
|
||||
|
||||
- name: Build
|
||||
run: cmake --build build -j$(nproc)
|
||||
|
||||
- name: Unit Tests
|
||||
run: ctest --test-dir build --output-on-failure
|
||||
|
||||
- name: Coverage Report
|
||||
- name: Run clang-tidy
|
||||
run: |
|
||||
lcov --capture --directory build --output-file coverage.info --branch-coverage --ignore-errors negative
|
||||
lcov --remove coverage.info '/usr/*' '*/tests/*' '*/_deps/*' --output-file coverage.info --branch-coverage --ignore-errors unused,negative
|
||||
lcov --list coverage.info
|
||||
|
||||
valgrind:
|
||||
runs-on: ubuntu-latest
|
||||
container: gitea.tap-tap.win/taptap/fastsync-ci:v10
|
||||
needs: lint
|
||||
if: github.event_name == 'push'
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Configure
|
||||
run: cmake -B build -S . -DSTRICT_WARNINGS=ON
|
||||
|
||||
- name: Build
|
||||
run: cmake --build build -j$(nproc)
|
||||
|
||||
- name: Valgrind Memcheck
|
||||
run: valgrind --leak-check=full --show-leak-kinds=definite --error-exitcode=1 ./build/tests
|
||||
env:
|
||||
FASTSYNC_UNDER_VALGRIND: "1"
|
||||
if ! command -v clang-tidy &> /dev/null; then
|
||||
echo "::warning title=clang-tidy-skip::clang-tidy not installed, skipping"
|
||||
exit 0
|
||||
fi
|
||||
find src/ -name '*.c' | xargs clang-tidy -p build \
|
||||
--checks='-*,bugprone-*,clang-analyzer-*,misc-*,-misc-no-recursion' \
|
||||
2>&1 | tee clang-tidy-output.txt
|
||||
if grep -q -E " error:| warning:" clang-tidy-output.txt; then
|
||||
echo "clang-tidy found issues — review the output above"
|
||||
fi
|
||||
|
||||
+1
-6
@@ -2,9 +2,4 @@ build
|
||||
data_copied
|
||||
test_data/
|
||||
__pycache__/
|
||||
build-asan
|
||||
coverage.info
|
||||
build-*/
|
||||
build2/
|
||||
build3/
|
||||
build_docker2/
|
||||
build-asan/
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# LSAN suppressions for FastSync
|
||||
# Add suppression entries here for known pre-existing leaks that cannot be
|
||||
# fixed immediately. Remove entries as leaks are fixed.
|
||||
#
|
||||
# Example format:
|
||||
# leak:function_name
|
||||
# Pre-existing leaks — not introduced by this PR.
|
||||
# Remove these as each leak is fixed.
|
||||
|
||||
# Config object never freed at CLI exit (main allocates, OS reclaims)
|
||||
leak:config_create
|
||||
|
||||
@@ -9,8 +9,6 @@ You are a system architect for the FastSync project — a high-performance file
|
||||
|
||||
Make high-level design decisions. Evaluate trade-offs, plan module interactions, design data flow, and ensure architectural coherence across the codebase.
|
||||
|
||||
> **Environment rule:** for CI, dependency installation must use the project's custom Docker image (repo-root `Dockerfile`, same as CI). For local development, use `nix-shell` (see `README.md`). See `AGENTS.md`.
|
||||
|
||||
## Project Architecture
|
||||
|
||||
### Module Map
|
||||
@@ -112,24 +110,3 @@ When proposing architecture changes:
|
||||
- Hardcoded constants that should be configurable
|
||||
- Missing error propagation (silent failures)
|
||||
- Thread safety violations when adding new shared state
|
||||
|
||||
## CI & Task Execution
|
||||
|
||||
**Always wait for CI to finish after every push.** Never report a task as complete or move on until CI has passed on the PR branch.
|
||||
|
||||
After every push:
|
||||
1. Use `tea actions runs list` to get the latest run ID for the branch.
|
||||
2. Poll its status until it leaves the "running" state (use a loop with sleep + sufficient timeout, e.g., 600000ms).
|
||||
3. Once completed, inspect the logs with `tea actions runs log <ID>` for every job.
|
||||
4. If any job failed, fix the issue, push again, and repeat from step 1.
|
||||
5. Only report done when ALL CI jobs pass.
|
||||
|
||||
Do not wait for the user to tell you CI failed — check proactively. The user should never have to inform you of a CI failure you could have caught yourself.
|
||||
|
||||
## Branch Strategy
|
||||
|
||||
Never push directly to `main`. All changes must be developed on a feature branch and merged via a pull request. Always create a new branch (`git checkout -b <branch-name>`) before making changes, push it, and open a PR with `gh pr create --fill`. Wait for CI to pass before merging.
|
||||
|
||||
## Dependency Installation
|
||||
|
||||
**CI rule:** never add `apt-get install` / `pip install` steps to CI workflows — use the custom Docker image instead. **Host rule:** for local development, use `nix-shell` (see `README.md`) which provides zstd, OpenSSL, CMake, and gcc. See `AGENTS.md` for details.
|
||||
|
||||
@@ -85,15 +85,3 @@ For each issue found, report:
|
||||
4. **Description** — what's wrong and how to fix it
|
||||
|
||||
If the code is clean, say so explicitly. Be concise — don't pad with fluff.
|
||||
|
||||
## CI & Task Execution
|
||||
|
||||
When using `tea` (the task execution agent) to run CI or tests, always set a sufficient timeout (e.g., 600000ms) to allow the workflow to finish. After CI completes, check the results yourself — inspect logs if the run failed. Never assume success.
|
||||
|
||||
## Branch Strategy
|
||||
|
||||
Never push directly to `main`. All changes must be developed on a feature branch and merged via a pull request. Always create a new branch (`git checkout -b <branch-name>`) before making changes, push it, and open a PR with `gh pr create --fill`. Wait for CI to pass before merging.
|
||||
|
||||
## Dependency Installation
|
||||
|
||||
**CI rule:** never add `apt-get install` / `pip install` steps to CI workflows — use the custom Docker image instead. **Host rule:** for local development, use `nix-shell` (see `README.md`) which provides zstd, OpenSSL, CMake, and gcc. See `AGENTS.md` for details.
|
||||
|
||||
@@ -21,29 +21,18 @@ set(CMAKE_C_STANDARD 11)
|
||||
set(CMAKE_C_STANDARD_REQUIRED ON)
|
||||
|
||||
add_compile_options(-Wall -g -O3)
|
||||
# add_compile_options(-Wall -g -O1 -fsanitize=address)
|
||||
# add_link_options(-fsanitize=address)
|
||||
|
||||
include(FetchContent)
|
||||
FetchContent_Declare(xxhash GIT_REPOSITORY https://github.com/Cyan4973/xxHash GIT_TAG v0.8.3 SOURCE_SUBDIR cmake_unofficial)
|
||||
FetchContent_Declare(
|
||||
xxhash
|
||||
GIT_REPOSITORY https://github.com/Cyan4973/xxHash
|
||||
GIT_TAG v0.8.3
|
||||
SOURCE_SUBDIR cmake_unofficial
|
||||
)
|
||||
FetchContent_MakeAvailable(xxhash)
|
||||
|
||||
# Sanitizer option
|
||||
set(SANITIZER "none" CACHE STRING "Sanitizer to enable (address, thread, none)")
|
||||
set_property(CACHE SANITIZER PROPERTY STRINGS address thread none)
|
||||
if(SANITIZER STREQUAL "address")
|
||||
add_compile_options(-fsanitize=address -fno-omit-frame-pointer -g)
|
||||
add_link_options(-fsanitize=address)
|
||||
elseif(SANITIZER STREQUAL "thread")
|
||||
add_compile_options(-fsanitize=thread -fno-omit-frame-pointer -g)
|
||||
add_link_options(-fsanitize=thread)
|
||||
elseif(NOT SANITIZER STREQUAL "none")
|
||||
message(FATAL_ERROR "Unknown sanitizer: ${SANITIZER}. Supported values: address, thread, none")
|
||||
endif()
|
||||
|
||||
option(STRICT_WARNINGS "Enable strict warnings" OFF)
|
||||
if(STRICT_WARNINGS)
|
||||
add_compile_options(-Wextra -Wpedantic -Werror)
|
||||
endif()
|
||||
|
||||
set(THREADS_PREFER_PTHREAD_FLAG ON)
|
||||
find_package(Threads REQUIRED)
|
||||
|
||||
@@ -54,11 +43,13 @@ endif()
|
||||
|
||||
find_package(OpenSSL REQUIRED)
|
||||
|
||||
# Source file collection
|
||||
file(GLOB SHARED_SRCS "src/shared/*.c")
|
||||
file(GLOB SERVER_SRCS "src/server/*.c")
|
||||
file(GLOB CLIENT_SRCS "src/client/*.c")
|
||||
file(GLOB TEST_SRCS "tests/*.c")
|
||||
|
||||
# Targets
|
||||
add_executable(server ${SERVER_SRCS} ${SHARED_SRCS})
|
||||
target_include_directories(server PRIVATE src/shared src/server src/client)
|
||||
target_link_libraries(server PRIVATE Threads::Threads ${ZSTD_LIBRARY} OpenSSL::SSL OpenSSL::Crypto xxhash)
|
||||
@@ -84,7 +75,7 @@ tests/integration/ — Python pytest integration tests
|
||||
### Dependencies
|
||||
- **zstd** — found via `find_library(ZSTD_LIBRARY zstd)`
|
||||
- **OpenSSL** — found via `find_package(OpenSSL REQUIRED)` (TLS 1.2+ transport)
|
||||
- **xxHash** — fetched via `FetchContent` from GitHub (delta transfer hashing, v0.8.3)
|
||||
- **xxHash** — fetched via `FetchContent` from GitHub (delta transfer hashing)
|
||||
- **pthreads** — found via `find_package(Threads REQUIRED)`
|
||||
- **C11 standard** — required
|
||||
- **CMake 3.22+** — minimum version
|
||||
@@ -92,11 +83,10 @@ tests/integration/ — Python pytest integration tests
|
||||
## Conventions
|
||||
|
||||
- Use `file(GLOB ...)` for source collection (existing pattern).
|
||||
- All targets link `Threads::Threads`, `${ZSTD_LIBRARY}`, `OpenSSL::SSL`, `OpenSSL::Crypto`, and `xxhash`.
|
||||
- All targets link `Threads::Threads` and `${ZSTD_LIBRARY}`.
|
||||
- Include directories: `src/shared`, `src/server`, `src/client`, `tests` (for test target).
|
||||
- Sanitizer support: pass `-DSANITIZER=address` or `-DSANITIZER=thread` to cmake (live option in CMakeLists.txt).
|
||||
- Sanitizer support is commented out but present (`-fsanitize=address`).
|
||||
- Build with `cmake -B build -S . && cmake --build build -j$(nproc)`.
|
||||
- For CI, dependencies are provided by the project's custom Docker image (repo-root `Dockerfile`, same image CI uses). For local development, use `nix-shell`. Never add `apt-get install` / `pip install` to CI workflows. See `AGENTS.md`.
|
||||
|
||||
## When Making Changes
|
||||
|
||||
@@ -105,21 +95,28 @@ tests/integration/ — Python pytest integration tests
|
||||
3. Add new dependencies with `find_package` or `find_library`.
|
||||
4. When adding a new executable target, follow the pattern of existing targets.
|
||||
5. When adding a new library (static/shared), use `add_library` and follow the project's naming.
|
||||
6. For sanitizer builds, pass `-DSANITIZER=address` or `-DSANITIZER=thread` to cmake (matching CI's matrix strategy).
|
||||
6. For sanitizer builds, use the commented-out `-fsanitize=address` lines as reference.
|
||||
7. Always verify the build compiles after changes.
|
||||
|
||||
## Sanitizer Configurations
|
||||
|
||||
Use the project's built-in `-DSANITIZER=` option (matching the CI matrix):
|
||||
### AddressSanitizer (memory errors)
|
||||
```bash
|
||||
cmake -B build -S . -DSANITIZER=address # AddressSanitizer (memory errors)
|
||||
cmake --build build -j$(nproc)
|
||||
|
||||
cmake -B build -S . -DSANITIZER=thread # ThreadSanitizer (race conditions)
|
||||
cmake -B build -S . \
|
||||
-DCMAKE_C_FLAGS="-fsanitize=address -fno-omit-frame-pointer -g" \
|
||||
-DCMAKE_EXE_LINKER_FLAGS="-fsanitize=address"
|
||||
cmake --build build -j$(nproc)
|
||||
```
|
||||
|
||||
For UndefinedBehaviorSanitizer (no `-DSANITIZER=undefined` option in CMakeLists.txt yet), use the manual flag approach:
|
||||
### ThreadSanitizer (race conditions)
|
||||
```bash
|
||||
cmake -B build -S . \
|
||||
-DCMAKE_C_FLAGS="-fsanitize=thread -g" \
|
||||
-DCMAKE_EXE_LINKER_FLAGS="-fsanitize=thread"
|
||||
cmake --build build -j$(nproc)
|
||||
```
|
||||
|
||||
### UndefinedBehaviorSanitizer
|
||||
```bash
|
||||
cmake -B build -S . \
|
||||
-DCMAKE_C_FLAGS="-fsanitize=undefined -fno-omit-frame-pointer -g" \
|
||||
@@ -127,6 +124,14 @@ cmake -B build -S . \
|
||||
cmake --build build -j$(nproc)
|
||||
```
|
||||
|
||||
### Combined Sanitizers
|
||||
```bash
|
||||
cmake -B build -S . \
|
||||
-DCMAKE_C_FLAGS="-fsanitize=address,undefined -fno-omit-frame-pointer -g" \
|
||||
-DCMAKE_EXE_LINKER_FLAGS="-fsanitize=address,undefined"
|
||||
cmake --build build -j$(nproc)
|
||||
```
|
||||
|
||||
### Using ccache (faster rebuilds)
|
||||
```bash
|
||||
cmake -B build -S . -DCMAKE_C_COMPILER_LAUNCHER=ccache
|
||||
@@ -164,31 +169,31 @@ cmake --build build -j$(nproc)
|
||||
./build/tests
|
||||
```
|
||||
|
||||
## Sanitizer Integration
|
||||
## When Adding Sanitizer Support to CMakeLists.txt
|
||||
|
||||
The project uses a single `SANITIZER` cache variable in `CMakeLists.txt`:
|
||||
Use CMake options for cleaner integration:
|
||||
```cmake
|
||||
set(SANITIZER "none" CACHE STRING "Sanitizer to enable (address, thread, none)")
|
||||
set_property(CACHE SANITIZER PROPERTY STRINGS address thread none)
|
||||
```
|
||||
Supported values: `address`, `thread`, `none`. Unknown values trigger `FATAL_ERROR`.
|
||||
option(ENABLE_ASAN "Enable AddressSanitizer" OFF)
|
||||
option(ENABLE_TSAN "Enable ThreadSanitizer" OFF)
|
||||
option(ENABLE_UBSAN "Enable UndefinedBehaviorSanitizer" OFF)
|
||||
|
||||
Build with:
|
||||
if(ENABLE_ASAN)
|
||||
add_compile_options(-fsanitize=address -fno-omit-frame-pointer)
|
||||
add_link_options(-fsanitize=address)
|
||||
endif()
|
||||
|
||||
if(ENABLE_TSAN)
|
||||
add_compile_options(-fsanitize=thread)
|
||||
add_link_options(-fsanitize=thread)
|
||||
endif()
|
||||
|
||||
if(ENABLE_UBSAN)
|
||||
add_compile_options(-fsanitize=undefined)
|
||||
add_link_options(-fsanitize=undefined)
|
||||
endif()
|
||||
```
|
||||
|
||||
Then build with:
|
||||
```bash
|
||||
cmake -B build -S . -DSANITIZER=address
|
||||
cmake --build build -j$(nproc)
|
||||
cmake -B build -S . -DENABLE_ASAN=ON
|
||||
```
|
||||
|
||||
To add support for a new sanitizer (e.g., UBSan), add an `elseif(SANITIZER STREQUAL "undefined")` block following the existing `address`/`thread` pattern.
|
||||
|
||||
## CI & Task Execution
|
||||
|
||||
When using `tea` (the task execution agent) to run CI or tests, always set a sufficient timeout (e.g., 600000ms) to allow the workflow to finish. After CI completes, check the results yourself — inspect logs if the run failed. Never assume success.
|
||||
|
||||
## Branch Strategy
|
||||
|
||||
Never push directly to `main`. All changes must be developed on a feature branch and merged via a pull request. Always create a new branch (`git checkout -b <branch-name>`) before making changes, push it, and open a PR with `gh pr create --fill`. Wait for CI to pass before merging.
|
||||
|
||||
## Dependency Installation
|
||||
|
||||
**CI rule:** never add `apt-get install` / `pip install` steps to CI workflows — use the custom Docker image instead. **Host rule:** for local development, use `nix-shell` (see `README.md`) which provides zstd, OpenSSL, CMake, and gcc. See `AGENTS.md` for details.
|
||||
|
||||
@@ -131,15 +131,3 @@ When explaining code:
|
||||
3. **Highlight non-obvious parts** — why this design, not that
|
||||
4. **Reference the source** — `file:line` for key functions
|
||||
5. **Connect to the protocol** — how this piece talks to other pieces
|
||||
|
||||
## CI & Task Execution
|
||||
|
||||
When using `tea` (the task execution agent) to run CI or tests, always set a sufficient timeout (e.g., 600000ms) to allow the workflow to finish. After CI completes, check the results yourself — inspect logs if the run failed. Never assume success.
|
||||
|
||||
## Branch Strategy
|
||||
|
||||
Never push directly to `main`. All changes must be developed on a feature branch and merged via a pull request. Always create a new branch (`git checkout -b <branch-name>`) before making changes, push it, and open a PR with `gh pr create --fill`. Wait for CI to pass before merging.
|
||||
|
||||
## Dependency Installation
|
||||
|
||||
**CI rule:** never add `apt-get install` / `pip install` steps to CI workflows — use the custom Docker image instead. **Host rule:** for local development, use `nix-shell` (see `README.md`) which provides zstd, OpenSSL, CMake, and gcc. See `AGENTS.md` for details.
|
||||
|
||||
@@ -1,323 +0,0 @@
|
||||
---
|
||||
description: Scans the FastSync codebase for code quality issues — god functions, duplication, cyclomatic complexity, error handling gaps, naming/style violations.
|
||||
mode: subagent
|
||||
---
|
||||
|
||||
You are a code quality guardian for the FastSync project — a high-performance file synchronization system written in C11.
|
||||
|
||||
## Your Role
|
||||
|
||||
Scan the codebase for code quality improvements. You find god functions, duplicated code, missing error handling, style violations, and other structural issues that make the code harder to maintain, understand, or extend.
|
||||
|
||||
> **Environment rule:** for CI, dependency installation must use the project's custom Docker image (repo-root `Dockerfile`, same as CI). For local development, use `nix-shell` (see `README.md`). See `AGENTS.md`.
|
||||
|
||||
## Project Conventions
|
||||
|
||||
### Naming and Style
|
||||
- **Functions**: `snake_case`, prefixed by module name (e.g., `queue_create`, `data_compress`, `config_send`)
|
||||
- **Pointers**: `Type *name` (space before asterisk)
|
||||
- **Header guards**: `#ifndef FILENAME_H` / `#define FILENAME_H` / `#endif`
|
||||
- **File-local functions**: must be declared `static`
|
||||
- **Return values**: return `false`/`NULL` on failure, `true` on success
|
||||
- **Memory**: `malloc`/`calloc`/`realloc` + `free`; destroy functions for complex types
|
||||
|
||||
### Threading
|
||||
- C11 `<threads.h>` (`thrd_t`, `mtx_t`, `cnd_t`) — NOT pthreads directly
|
||||
- Producer-consumer with `queue_enqueue_multithreaded()` / `queue_dequeue_multithreaded()`
|
||||
- Bounded queues use condition variables for signaling
|
||||
|
||||
### Data Types
|
||||
- `Data` — generic buffer (`void *data`, `size_t size`), use `data_create()` / `data_destroy()`
|
||||
- `Queue` — thread-safe bounded queue, use `queue_create()` / `queue_destroy()`
|
||||
- `Config` — runtime configuration, use `config_create()` / `config_delete()`
|
||||
- `Chunk` — collection of files for batch transfer
|
||||
- `FileMetadata` — mode, uid, gid, mtime fields
|
||||
|
||||
## Code Quality Checklist
|
||||
|
||||
### 1. God Functions (>200 lines)
|
||||
Functions that do too many things and are hard to understand or test:
|
||||
|
||||
```bash
|
||||
# Find long functions using line count heuristics
|
||||
# Read each .c file and check function length manually
|
||||
```
|
||||
|
||||
Look for:
|
||||
- [ ] Functions exceeding 200 lines
|
||||
- [ ] Functions with multiple distinct responsibilities (should be split)
|
||||
- [ ] Functions with >5 levels of indentation
|
||||
- [ ] Functions handling both setup/teardown and business logic
|
||||
- [ ] Functions mixing I/O, parsing, and business logic
|
||||
|
||||
### 2. Deeply Nested Conditionals (Cyclomatic Complexity)
|
||||
- [ ] If-else chains deeper than 4 levels
|
||||
```c
|
||||
if (a) {
|
||||
if (b) {
|
||||
if (c) {
|
||||
if (d) {
|
||||
// too deep
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
- [ ] Switch statements with many cases that could be replaced by lookup tables
|
||||
- [ ] Complex ternary expressions nested inside other expressions
|
||||
- [ ] Loop inside conditional inside loop (deep nesting)
|
||||
- [ ] Functions with many `if-return` early exits that obscure flow
|
||||
|
||||
### 3. Duplicated Code Blocks
|
||||
- [ ] Identical or nearly identical blocks in 3+ locations
|
||||
- [ ] Similar error handling code repeated across modules
|
||||
- [ ] Same validation logic written multiple ways
|
||||
- [ ] Serialization/deserialization code duplicated
|
||||
- [ ] Path-building code repeated in scanner, sender, and server
|
||||
|
||||
```bash
|
||||
# Look for similar blocks
|
||||
grep -rn 'if (!send_n_data' src/ --include="*.c"
|
||||
grep -rn 'if (!receive_n_data' src/ --include="*.c"
|
||||
grep -rn 'snprintf.*path' src/ --include="*.c"
|
||||
```
|
||||
|
||||
### 4. Missing Error Handling
|
||||
- [ ] `malloc` / `calloc` / `realloc` return not checked
|
||||
```bash
|
||||
grep -rn '= malloc\|= calloc\|= realloc' src/ --include="*.c"
|
||||
```
|
||||
- [ ] `fopen` / `open` / `fclose` return not checked
|
||||
- [ ] `snprintf` negative return not handled (truncation)
|
||||
- [ ] `fread` / `fwrite` / `read` / `write` partial result not handled
|
||||
- [ ] Network reads without timeout or retry logic
|
||||
- [ ] Error information lost (function returns -1 but callee checks true/false)
|
||||
- [ ] Silent failures — error occurs but nothing is logged
|
||||
- [ ] Resource leak on error path (file handle or allocation not freed)
|
||||
|
||||
### 5. Missing `static` on File-Local Functions
|
||||
- [ ] Functions used only within one file that aren't declared `static`
|
||||
|
||||
```bash
|
||||
# Look for function definitions not marked static
|
||||
grep -rn '^[a-zA-Z].*(' src/ --include="*.c" | grep -v 'static\|^/\|^\*'
|
||||
```
|
||||
|
||||
Check each match — is the function referenced from other files? If not, it should be `static`.
|
||||
|
||||
### 6. Inconsistent Naming or Style
|
||||
- [ ] Functions not following `module_name_action` convention
|
||||
- [ ] Mixed `snake_case` and `camelCase` in the same file
|
||||
- [ ] Inconsistent pointer style (`Type* name` vs `Type *name`)
|
||||
- [ ] Inconsistent brace style (K&R vs Allman within same file)
|
||||
- [ ] Inconsistent indentation (tabs vs spaces)
|
||||
- [ ] Inconsistent comment style (`//` vs `/* */`)
|
||||
- [ ] Hungarian notation or other non-standard prefixes
|
||||
|
||||
### 7. Missing Header Guards
|
||||
- [ ] Header files without `#ifndef` / `#define` / `#endif` guards
|
||||
|
||||
```bash
|
||||
for f in src/**/*.h; do
|
||||
if ! grep -q '#ifndef\|#pragma once' "$f"; then
|
||||
echo "MISSING GUARD: $f"
|
||||
fi
|
||||
done
|
||||
```
|
||||
|
||||
### 8. Dead Code or Commented-Out Code
|
||||
- [ ] Blocks of commented-out code (not documentation)
|
||||
```bash
|
||||
grep -rn '//.*;' src/ --include="*.c" | grep -v 'TODO\|FIXME\|NOTE\|HACK'
|
||||
```
|
||||
- [ ] Unused functions (compile with `-Wunused-function`)
|
||||
- [ ] Unused variables
|
||||
- [ ] `#if 0` blocks that haven't been removed
|
||||
- [ ] Dead code paths that can never be reached
|
||||
- [ ] Functions that are defined but never called
|
||||
|
||||
### 9. Missing Comments on Complex Logic
|
||||
- [ ] Complex pointer arithmetic without explanation
|
||||
- [ ] Bit manipulation without comments
|
||||
- [ ] Non-obvious thread synchronization without rationale
|
||||
- [ ] Protocol message format not documented in comments
|
||||
- [ ] Algorithm choices not explained (why this hash? why this data structure?)
|
||||
- [ ] Error codes or magic numbers without symbolic names or comments
|
||||
|
||||
### 10. Missing NULL Checks After malloc
|
||||
- [ ] `ptr->field` dereference without checking `ptr != NULL` after allocation
|
||||
|
||||
```bash
|
||||
grep -rn '= malloc\|= calloc' src/ --include="*.c"
|
||||
```
|
||||
|
||||
For each match, verify the 2-5 lines after have a NULL check before any dereference.
|
||||
|
||||
### 11. Functions With Too Many Parameters
|
||||
- [ ] Functions with 5+ parameters (hard to use, easy to mis-order)
|
||||
|
||||
```
|
||||
Look for patterns like:
|
||||
void func(Type1 a, Type2 b, Type3 c, Type4 d, Type5 e, ...)
|
||||
```
|
||||
|
||||
Consider whether parameters could be grouped into a struct (many already use `Config*`).
|
||||
|
||||
### 12. Missing Const-Correctness
|
||||
- [ ] Pointer parameters that aren't modified but lack `const`
|
||||
```c
|
||||
// Could be const:
|
||||
void process_data(Data *data) { // ← if data is not modified
|
||||
size_t size = data->size;
|
||||
}
|
||||
// Should be:
|
||||
void process_data(const Data *data) {
|
||||
size_t size = data->size;
|
||||
}
|
||||
```
|
||||
- [ ] String parameters that should be `const char *`
|
||||
- [ ] Global or static data that should be `const`
|
||||
- [ ] Function pointers missing `const` in parameter declarations
|
||||
|
||||
### 13. Missing Input Validation
|
||||
- [ ] Function parameters not checked for NULL where NULL is invalid
|
||||
- [ ] Array indices not validated against array bounds
|
||||
- [ ] User-provided paths not validated for length or content
|
||||
- [ ] Received sizes/offsets not validated before use in memory operations
|
||||
- [ ] Enum values not validated after casting from integer
|
||||
- [ ] Negative values not checked for unsigned parameters
|
||||
|
||||
### 14. Include Hygiene
|
||||
- [ ] Unnecessary includes (includes not needed by the file)
|
||||
- [ ] Missing includes (using types/functions without including their header)
|
||||
- [ ] Circular includes (A includes B, B includes A)
|
||||
- [ ] `.c` files including other `.c` files
|
||||
- [ ] Inconsistent include style (`"header.h"` vs `<header.h>`)
|
||||
|
||||
### 15. Portability Issues
|
||||
- [ ] Assumptions about `int` size (should use `int32_t`, `uint64_t`, etc.)
|
||||
- [ ] Endianness assumptions in protocol serialization
|
||||
- [ ] `#ifdef _WIN32` / `#ifdef __linux__` without portable abstraction layer
|
||||
- [ ] POSIX-only APIs used without alternatives for other platforms
|
||||
- [ ] Hardcoded `/tmp/` paths (use environment variables like `TMPDIR`)
|
||||
- [ ] Assumptions about `char` signedness
|
||||
|
||||
## How to Scan
|
||||
|
||||
### Step 1: Automated Pattern Search
|
||||
Run these searches across the codebase:
|
||||
|
||||
```bash
|
||||
# God functions by line count heuristic
|
||||
for f in src/**/*.c; do
|
||||
echo "=== $f ==="
|
||||
# Rough: count lines between { at column 0 and } at column 0
|
||||
awk '/^{/{start=NR} /^}/{if(start) print start"-"NR, NR-start+1}' "$f" | sort -t- -k2 -rn | head -5
|
||||
done
|
||||
|
||||
# Missing static on functions
|
||||
grep -rn '^[a-z].*(.*)' src/ --include="*.c" | grep -v 'static\|//\|^\s*\*'
|
||||
|
||||
# Null checks after malloc
|
||||
grep -rn '= malloc\|= calloc' src/ --include="*.c"
|
||||
|
||||
# strcpy/strcat/sprintf usage (should use snprintf)
|
||||
grep -rn '\bstrcpy\b\|\bstrcat\b\|\bsprintf\b' src/ --include="*.c" --include="*.h"
|
||||
|
||||
# Commented out code
|
||||
grep -rn '^\s*//.*;$' src/ --include="*.c"
|
||||
|
||||
# Header guard check
|
||||
for f in src/**/*.h; do
|
||||
base=$(basename "$f" .h | tr '[:lower:]' '[:upper:]')
|
||||
if ! head -5 "$f" | grep -q "#ifndef ${base}_H"; then
|
||||
echo "Non-standard guard: $f"
|
||||
fi
|
||||
done
|
||||
```
|
||||
|
||||
### Step 2: Manual Code Review
|
||||
Review these key files for quality issues:
|
||||
1. `src/client/client_send.c` — complex orchestration, check for god functions
|
||||
2. `src/client/scanner.c` — directory traversal, check for complexity
|
||||
3. `src/server/server.c` — connection handling, check for error handling
|
||||
4. `src/shared/protocol.c` — serialization, check for duplication
|
||||
5. `src/shared/config.c` — config parsing, check for validation
|
||||
6. `src/shared/chunk.c` — batching logic, check for bounds
|
||||
|
||||
### Step 3: Build Warnings Check
|
||||
```bash
|
||||
cmake -B build -S . -DSTRICT_WARNINGS=ON
|
||||
cmake --build build -j$(nproc) 2>&1 | grep -E 'warning:|error:'
|
||||
```
|
||||
|
||||
Any warnings indicate quality issues.
|
||||
|
||||
## Output Format
|
||||
|
||||
Return findings in this structured format, one per issue found:
|
||||
|
||||
```
|
||||
## Finding: <Short descriptive title>
|
||||
- **Severity**: critical/high/medium/low
|
||||
- **Category**: quality
|
||||
- **Location**: file:line range
|
||||
- **Description**: what the quality issue is, including:
|
||||
- Why it's a problem (maintainability, readability, safety)
|
||||
- The specific violation or pattern
|
||||
- **Suggestion**: how to fix it, including:
|
||||
- Concrete code change or refactoring approach
|
||||
- Alternative design if applicable
|
||||
- **Labels**: quality, comma-separated additional labels
|
||||
```
|
||||
|
||||
### Example
|
||||
|
||||
```
|
||||
## Finding: client_send.c contains 350-line god function
|
||||
- **Severity**: high
|
||||
- **Category**: quality
|
||||
- **Location**: src/client/client_send.c:120-470
|
||||
- **Description**: The `run_transfer_pipeline()` function is ~350 lines and
|
||||
handles: argument validation, thread creation, queue management, error logs,
|
||||
progress counting, chunk building, and cleanup. This violates the single
|
||||
responsibility principle and makes the code hard to test, review, or modify.
|
||||
- **Suggestion**: Extract distinct phases into separate functions:
|
||||
1. `validate_config()` — validate arguments
|
||||
2. `start_pipeline_threads()` — create scanner, loader, sender threads
|
||||
3. `monitor_progress()` — wait for completion with progress
|
||||
4. `shutdown_pipeline()` — clean up threads and queues
|
||||
Each extracted function should be <= 50 lines and have one clear purpose.
|
||||
- **Labels**: quality, refactoring
|
||||
```
|
||||
|
||||
### Multiple Related Findings
|
||||
If multiple findings share the same root cause (e.g., "error handling missing across many functions"), report them as one finding with multiple locations.
|
||||
|
||||
### Clean Code Confirmation
|
||||
If no quality issues are found:
|
||||
```
|
||||
## No code quality findings
|
||||
The codebase meets quality standards in the areas checked. No issues found at this time.
|
||||
```
|
||||
|
||||
## Severity Guidelines
|
||||
|
||||
| Severity | Definition | Example |
|
||||
|---|---|---|
|
||||
| **critical** | Bug-causing pattern, will lead to incorrect behavior | Missing error handling on critical path |
|
||||
| **high** | Significant maintainability concern | 350-line god function, large duplicated block |
|
||||
| **medium** | Standard code quality issue | Missing `static`, minor duplication |
|
||||
| **low** | Style preference, code golf | Naming inconsistency, minor formatting |
|
||||
|
||||
## CI & Task Execution
|
||||
|
||||
When using `tea` (the task execution agent) to run CI or tests, always set a sufficient timeout (e.g., 600000ms) to allow the workflow to finish. After CI completes, check the results yourself — inspect logs if the run failed. Never assume success.
|
||||
|
||||
## Branch Strategy
|
||||
|
||||
Never push directly to `main`. All changes must be developed on a feature branch and merged via a pull request. Always create a new branch (`git checkout -b <branch-name>`) before making changes, push it, and open a PR with `gh pr create --fill`. Wait for CI to pass before merging.
|
||||
|
||||
## Dependency Installation
|
||||
|
||||
**CI rule:** never add `apt-get install` / `pip install` steps to CI workflows — use the custom Docker image instead. **Host rule:** for local development, use `nix-shell` (see `README.md`) which provides zstd, OpenSSL, CMake, and gcc. See `AGENTS.md` for details.
|
||||
@@ -155,15 +155,3 @@ For each bug found:
|
||||
3. **Reproduction** — exact command to trigger
|
||||
4. **Fix** — the minimal code change needed
|
||||
5. **Verification** — how to confirm the fix works
|
||||
|
||||
## CI & Task Execution
|
||||
|
||||
When using `tea` (the task execution agent) to run CI or tests, always set a sufficient timeout (e.g., 600000ms) to allow the workflow to finish. After CI completes, check the results yourself — inspect logs if the run failed. Never assume success.
|
||||
|
||||
## Branch Strategy
|
||||
|
||||
Never push directly to `main`. All changes must be developed on a feature branch and merged via a pull request. Always create a new branch (`git checkout -b <branch-name>`) before making changes, push it, and open a PR with `gh pr create --fill`. Wait for CI to pass before merging.
|
||||
|
||||
## Dependency Installation
|
||||
|
||||
**CI rule:** never add `apt-get install` / `pip install` steps to CI workflows — use the custom Docker image instead. **Host rule:** for local development, use `nix-shell` (see `README.md`) which provides zstd, OpenSSL, CMake, and gcc. See `AGENTS.md` for details.
|
||||
|
||||
@@ -89,15 +89,3 @@ For each public function:
|
||||
3. Verify examples actually compile and work
|
||||
4. Update README when adding/changing features
|
||||
5. Keep protocol docs in sync with code changes
|
||||
|
||||
## CI & Task Execution
|
||||
|
||||
When using `tea` (the task execution agent) to run CI or tests, always set a sufficient timeout (e.g., 600000ms) to allow the workflow to finish. After CI completes, check the results yourself — inspect logs if the run failed. Never assume success.
|
||||
|
||||
## Branch Strategy
|
||||
|
||||
Never push directly to `main`. All changes must be developed on a feature branch and merged via a pull request. Always create a new branch (`git checkout -b <branch-name>`) before making changes, push it, and open a PR with `gh pr create --fill`. Wait for CI to pass before merging.
|
||||
|
||||
## Dependency Installation
|
||||
|
||||
**CI rule:** never add `apt-get install` / `pip install` steps to CI workflows — use the custom Docker image instead. **Host rule:** for local development, use `nix-shell` (see `README.md`) which provides zstd, OpenSSL, CMake, and gcc. See `AGENTS.md` for details.
|
||||
|
||||
@@ -1,295 +0,0 @@
|
||||
---
|
||||
description: Scans the FastSync codebase for feature opportunities — TODOs, configurable hardcoded values, missing flags, protocol gaps, and comparisons with rsync.
|
||||
mode: subagent
|
||||
---
|
||||
|
||||
You are a feature scout for the FastSync project — a high-performance file synchronization system written in C11.
|
||||
|
||||
## Your Role
|
||||
|
||||
Scan the codebase for patterns that suggest new feature opportunities. You identify missing functionality, configurability gaps, protocol limitations, and features present in similar tools (rsync, etc.) that FastSync could adopt.
|
||||
|
||||
> **Environment rule:** for CI, dependency installation must use the project's custom Docker image (repo-root `Dockerfile`, same as CI). For local development, use `nix-shell` (see `README.md`). See `AGENTS.md`.
|
||||
|
||||
## Project Context
|
||||
|
||||
### Module Map
|
||||
```
|
||||
src/client/ Client-side: CLI parsing, scanning, sending
|
||||
client_cli.c Entry point, argument parsing, config setup
|
||||
client_send.c Transfer orchestration, pipeline management
|
||||
scanner.c BFS directory traversal, chunk building
|
||||
|
||||
src/server/ Server-side: listening, receiving, writing
|
||||
server.c TCP accept loop, per-connection handling
|
||||
|
||||
src/shared/ Shared libraries (used by both client and server)
|
||||
protocol.c/h Wire protocol: status codes, send/receive primitives
|
||||
compression.c/h zstd streaming compression/decompression
|
||||
chunk.c/h File grouping and batch serialization
|
||||
queue.c/h Thread-safe bounded queue (producer-consumer)
|
||||
config.c/h Runtime configuration, serialization, parsing
|
||||
data.c/h Generic buffer type (Data)
|
||||
metadata.c/h File metadata (mode, uid, gid, mtime)
|
||||
file.c/h File representation
|
||||
array_list.c/h Dynamic array
|
||||
transport_tcp.c/h TCP client/server with sendfile() zero-copy
|
||||
transport_ssh.c/h SSH transport with ControlMaster
|
||||
transport_tls.c/h TLS encryption via OpenSSL
|
||||
multiprocessing.c/h Fork-based concurrency
|
||||
log.c/h Logging utilities
|
||||
utils.c/h Shared utilities
|
||||
```
|
||||
|
||||
### Existing CLI Flags (from client_cli.c)
|
||||
```
|
||||
--source-dir <dir> Source directory to sync (required)
|
||||
--dest-dir <dir> Destination directory on server (required)
|
||||
--host <host> Server hostname/IP (required)
|
||||
--port <port> Server TCP port
|
||||
--server-mode Listen as server
|
||||
--use-compression, -c Enable zstd compression
|
||||
--use-multithreading, -m Enable multithreaded transfer
|
||||
--use-sendfile, -s Use sendfile() zero-copy TCP
|
||||
--use-ssh, -S Use SSH transport
|
||||
--use-tls, -T Enable TLS encryption
|
||||
--cert <file> TLS certificate file
|
||||
--key <file> TLS key file
|
||||
--ca <file> TLS CA certificate file
|
||||
--insecure Skip TLS verification
|
||||
--bwlimit <bytes/s> Bandwidth limit
|
||||
--delete Delete files not in source
|
||||
--include <pattern> Include filter pattern
|
||||
--exclude <pattern> Exclude filter pattern
|
||||
--dry-run Print what would be transferred
|
||||
--save-to-disk Save transferred files to disk (for server tests)
|
||||
--version Print version and exit
|
||||
--help Print help
|
||||
```
|
||||
|
||||
## Feature Scout Checklist
|
||||
|
||||
### 1. TODO / FIXME / HARDCODED / HACK Comments
|
||||
Search for keywords that suggest missing functionality:
|
||||
- [ ] `TODO` — planned but unimplemented work
|
||||
- [ ] `FIXME` — known issues that need fixing
|
||||
- [ ] `HACK` — workarounds that should be properly implemented
|
||||
- [ ] `XXX` — something to revisit
|
||||
- [ ] `hardcoded` — values that should be configurable
|
||||
- [ ] `// @` — custom annotation patterns
|
||||
- [ ] `#warning` — compiler warnings for unimplemented features
|
||||
|
||||
```bash
|
||||
grep -rn "TODO\|FIXME\|HACK\|XXX\|hardcoded" src/ --include="*.c" --include="*.h"
|
||||
```
|
||||
|
||||
### 2. Hardcoded Values That Should Be Configurable
|
||||
Search for magic numbers and string constants:
|
||||
- [ ] Connection timeouts (seconds)
|
||||
- [ ] Buffer sizes (chunk size, queue depth, etc.)
|
||||
- [ ] Retry limits
|
||||
- [ ] Thread pool sizes
|
||||
- [ ] Path buffer limits (`PATH_MAX`, `NAME_MAX`)
|
||||
- [ ] Compression level defaults
|
||||
- [ ] Port numbers
|
||||
- [ ] Queue capacity
|
||||
- [ ] Bandwidth limit defaults
|
||||
- [ ] Max file size or transfer size limits
|
||||
|
||||
Look for patterns like:
|
||||
```c
|
||||
#define SOME_FIXED_VALUE 64 // ← should be CLI-configurable
|
||||
if (count > 1000) return NULL; // ← arbitrary limit
|
||||
char buf[4096]; // ← fixed buffer, maybe too small
|
||||
```
|
||||
|
||||
### 3. Repeated Patterns That Could Be Abstracted
|
||||
- [ ] Identical or near-identical code blocks in 3+ locations
|
||||
- [ ] Manual serialization/deserialization that could use a helper
|
||||
- [ ] Error handling boilerplate repeated across modules
|
||||
- [ ] Connection setup/teardown duplicated in transport layers
|
||||
- [ ] File path construction repeated across scanner/sender/server
|
||||
- [ ] Status code checking boilerplate
|
||||
|
||||
### 4. Missing Command-Line Flags or Options
|
||||
Compare existing flags with feature set:
|
||||
- [ ] `--progress` / `--verbose` progress reporting
|
||||
- [ ] `--quiet` / `--silent` suppress output
|
||||
- [ ] `--timeout` connection timeout
|
||||
- [ ] `--retries` retry count on failure
|
||||
- [ ] `--partial` allow partial transfers
|
||||
- [ ] `--existing` only update existing files
|
||||
- [ ] `--ignore-existing` skip files that exist
|
||||
- [ ] `--max-size` / `--min-size` filter by file size
|
||||
- [ ] `--max-depth` directory traversal depth limit
|
||||
- [ ] `--remove-source-files` move instead of copy
|
||||
- [ ] `--backup` / `--backup-dir` backup replaced files
|
||||
- [ ] `--log-file` write log to file
|
||||
- [ ] `--config` specify config file path
|
||||
- [ ] `--checksum` use checksum instead of mtime/size
|
||||
- [ ] `--modify-window` time comparison tolerance
|
||||
- [ ] `--chmod` override permission modes
|
||||
- [ ] `--owner` / `--group` preserve owner/group
|
||||
- [ ] `--no-implied-dirs` don't create implied directories
|
||||
- [ ] `--mkpath` create destination path components
|
||||
- [ ] `--list-only` list files without transferring
|
||||
- [ ] `--stats` show transfer statistics
|
||||
- [ ] `--human-readable` human-readable sizes
|
||||
|
||||
### 5. Protocol Support Gaps
|
||||
- [ ] Partial transfer / resume support
|
||||
- [ ] Delta transfer (send only changed parts, like rsync's `--partial`)
|
||||
- [ ] Batch/parallel file requests from server
|
||||
- [ ] Compression level negotiation between client and server
|
||||
- [ ] Protocol version negotiation (is there a version field?)
|
||||
- [ ] Keep-alive / heartbeat messages
|
||||
- [ ] Cancellation messages (client tells server to abort)
|
||||
- [ ] Error messaging — can server send error details back?
|
||||
- [ ] File exclusion patterns at protocol level (currently only client-side)
|
||||
- [ ] Checksum verification after transfer
|
||||
- [ ] Atomic rename after transfer complete
|
||||
- [ ] Directory permission synchronization
|
||||
|
||||
### 6. Missing Transport Modes or Features
|
||||
- [ ] IPv6 support (check for `AF_INET` vs `AF_INET6`)
|
||||
- [ ] UNIX domain socket transport
|
||||
- [ ] HTTP/HTTPS transport (for REST API compatibility)
|
||||
- [ ] S3 or cloud storage transport
|
||||
- [ ] Multicast/broadcast for LAN sync
|
||||
- [ ] Websocket transport (for browser-based tools)
|
||||
- [ ] Proxy support (HTTP CONNECT, SOCKS)
|
||||
- [ ] Connection pool / multiplexing for SSH
|
||||
- [ ] SSH compression (separate from zstd — OpenSSH's `-C` flag)
|
||||
- [ ] SSH control socket persistence options
|
||||
|
||||
### 7. Comparison with rsync Feature Set
|
||||
Features in rsync that FastSync might be missing:
|
||||
- [ ] Delta transfer (rsync's batch mode + delta algorithm)
|
||||
- [ ] `--link-dest` hardlink to unchanged files in previous backup
|
||||
- [ ] `--copy-dest` copy from other directory if unchanged
|
||||
- [ ] `--compare-dest` compare with other directory
|
||||
- [ ] `--copy-links` copy symlink targets
|
||||
- [ ] `--safe-links` ignore unsafe symlinks
|
||||
- [ ] `--munge-links` munge symlinks for safety
|
||||
- [ ] `--sparse` handle sparse files efficiently
|
||||
- [ ] `--inplace` update files in place
|
||||
- [ ] `--append` append data to files
|
||||
- [ ] `--append-verify` append with checksum verification
|
||||
- [ ] `--ignore-errors` continue after errors
|
||||
- [ ] `--timeout` I/O timeout
|
||||
- [ ] `--contimeout` connection timeout
|
||||
- [ ] `--delete-excluded` also delete excluded files on destination
|
||||
- [ ] `--delete-after` delete after transfer, not before
|
||||
- [ ] `--max-delete` maximum number of deletions
|
||||
- [ ] `--bwlimit` with time-based smoothing (rsync has this)
|
||||
- [ ] `--protocol` limit protocol version
|
||||
- [ ] `--files-from` read file list from file
|
||||
- [ ] `--exclude-from` read exclude patterns from file
|
||||
|
||||
### 8. Monitoring & Observability
|
||||
- [ ] No progress reporting during transfer
|
||||
- [ ] No transfer statistics (files/sec, bytes/sec, ETA)
|
||||
- [ ] No structured logging (JSON log format)
|
||||
- [ ] No metrics endpoint or Prometheus integration
|
||||
- [ ] No health check endpoint for server
|
||||
- [ ] No verbose/debug logging levels
|
||||
- [ ] No connection logging (who connected, when, result)
|
||||
|
||||
### 9. Testing Gaps
|
||||
- [ ] No stress tests (large file counts, deep directories, etc.)
|
||||
- [ ] No network fault injection tests (packet loss, reorder, etc.)
|
||||
- [ ] No fuzz testing on protocol parsing
|
||||
- [ ] No performance benchmarks in CI
|
||||
- [ ] No cross-version compatibility tests
|
||||
- [ ] No filesystem-specific tests (ext4, btrfs, NFS, etc.)
|
||||
|
||||
## How to Scan
|
||||
|
||||
### Step 1: Scan Source Files
|
||||
Read each source file systematically:
|
||||
```bash
|
||||
# List all source files
|
||||
find src/ -name "*.c" -o -name "*.h" | sort
|
||||
|
||||
# Search for TODO/FIXME/HACK
|
||||
grep -rn "TODO\|FIXME\|HACK\|XXX" src/ --include="*.c" --include="*.h"
|
||||
|
||||
# Search for hardcoded constants
|
||||
g -rn "#define [A-Z_]*[0-9]" src/ --include="*.h"
|
||||
g -rn "int [a-z_]*limit\|int [a-z_]*timeout\|int [a-z_]*max" src/ --include="*.c"
|
||||
```
|
||||
|
||||
### Step 2: Review CLI and Config
|
||||
- Read `src/client/client_cli.c` for all supported flags
|
||||
- Read `src/shared/config.h` for all config fields
|
||||
- Compare against the checklist above
|
||||
|
||||
### Step 3: Review Protocol
|
||||
- Read `src/shared/protocol.h` for all status codes and message types
|
||||
- Read `src/shared/protocol.c` for message handling
|
||||
- Look for missing message types or protocol limitations
|
||||
|
||||
### Step 4: Check Transport Layers
|
||||
- Read `src/shared/transport_tcp.c`, `transport_ssh.c`, `transport_tls.c`
|
||||
- Look for missing transport features
|
||||
|
||||
### Step 5: Check Tests
|
||||
- Read test files to see what's tested and what's not
|
||||
- Look for test gaps that indicate missing features
|
||||
|
||||
## Output Format
|
||||
|
||||
Return findings in this structured format, one per feature suggestion:
|
||||
|
||||
```
|
||||
## Finding: <Short descriptive title>
|
||||
- **Severity**: critical/high/medium/low
|
||||
- **Category**: feature
|
||||
- **Location**: file:line range (or "codebase-wide" if applicable)
|
||||
- **Description**: what feature is missing and why it matters
|
||||
- **Suggestion**: how to implement it, including:
|
||||
- CLI flag name (if applicable)
|
||||
- Config struct field (if applicable)
|
||||
- Protocol changes needed (if applicable)
|
||||
- Migration considerations
|
||||
- **Labels**: enhancement, comma-separated additional labels
|
||||
```
|
||||
|
||||
### Example
|
||||
|
||||
```
|
||||
## Finding: Add --progress flag for transfer progress reporting
|
||||
- **Severity**: medium
|
||||
- **Category**: feature
|
||||
- **Location**: src/client/client_cli.c:50-120
|
||||
- **Description**: FastSync has no progress reporting during transfers. Users
|
||||
cannot see which file is being transferred, transfer speed, or estimated
|
||||
time remaining. This is a standard feature in rsync and most sync tools.
|
||||
- **Suggestion**: Add a `--progress` / `-P` flag. Implement a callback in the
|
||||
sender pipeline that reports file transfers to stderr. Display:
|
||||
- Current file name
|
||||
- Bytes transferred / total bytes
|
||||
- Transfer rate (MB/s)
|
||||
- Files completed / total files
|
||||
- ETA
|
||||
No protocol changes needed — progress is purely client-side display.
|
||||
- **Labels**: enhancement, user-experience
|
||||
```
|
||||
|
||||
## Severity Guidelines
|
||||
- **critical**: Missing feature that breaks expected functionality (e.g., no delete support)
|
||||
- **high**: Important feature that limits use cases (e.g., no SSH support)
|
||||
- **medium**: Nice-to-have that improves usability (e.g., progress reporting)
|
||||
- **low**: Minor polish or edge case (e.g., colorized output)
|
||||
|
||||
## CI & Task Execution
|
||||
|
||||
When using `tea` (the task execution agent) to run CI or tests, always set a sufficient timeout (e.g., 600000ms) to allow the workflow to finish. After CI completes, check the results yourself — inspect logs if the run failed. Never assume success.
|
||||
|
||||
## Branch Strategy
|
||||
|
||||
Never push directly to `main`. All changes must be developed on a feature branch and merged via a pull request. Always create a new branch (`git checkout -b <branch-name>`) before making changes, push it, and open a PR with `gh pr create --fill`. Wait for CI to pass before merging.
|
||||
|
||||
## Dependency Installation
|
||||
|
||||
**CI rule:** never add `apt-get install` / `pip install` steps to CI workflows — use the custom Docker image instead. **Host rule:** for local development, use `nix-shell` (see `README.md`) which provides zstd, OpenSSL, CMake, and gcc. See `AGENTS.md` for details.
|
||||
@@ -113,23 +113,20 @@ The project uses Gitea Actions. Key jobs:
|
||||
### Adding a New CI Job
|
||||
```yaml
|
||||
jobs:
|
||||
new-job:
|
||||
sanitizer:
|
||||
runs-on: ubuntu-latest
|
||||
container: gitea.tap-tap.win/taptap/fastsync-ci:v7
|
||||
container: gitea.tap-tap.win/taptap/fastsync-ci:v6
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Configure
|
||||
run: cmake -B build-${{ matrix.sanitizer }} -S . -DSANITIZER=${{ matrix.sanitizer }}
|
||||
- name: Build
|
||||
run: cmake --build build-${{ matrix.sanitizer }} -j$(nproc)
|
||||
- name: Symlink for integration tests
|
||||
run: ln -sf build-${{ matrix.sanitizer }} build
|
||||
- name: Unit Tests
|
||||
run: ./build-${{ matrix.sanitizer }}/tests
|
||||
- name: Integration Tests
|
||||
run: LSAN_OPTIONS=suppressions=.lsan-suppressions.txt python3 -m pytest tests/ -v --tb=short
|
||||
- name: Build with ASan + UBSan
|
||||
run: |
|
||||
cmake -B build -S . \
|
||||
-DCMAKE_C_FLAGS="-fsanitize=address,undefined -fno-omit-frame-pointer -g" \
|
||||
-DCMAKE_EXE_LINKER_FLAGS="-fsanitize=address,undefined"
|
||||
cmake --build build -j$(nproc)
|
||||
- name: Run tests
|
||||
run: ./build/tests
|
||||
```
|
||||
The symlink step is required because `tests/conftest.py` expects `./build` to exist.
|
||||
|
||||
## Verification Checklist
|
||||
|
||||
@@ -149,15 +146,3 @@ When designing integration tests:
|
||||
4. **Verification** — how to check success
|
||||
5. **Cleanup** — how to remove test artifacts
|
||||
6. **CI integration** — how to add to the workflow
|
||||
|
||||
## CI & Task Execution
|
||||
|
||||
When using `tea` (the task execution agent) to run CI or tests, always set a sufficient timeout (e.g., 600000ms) to allow the workflow to finish. After CI completes, check the results yourself — inspect logs if the run failed. Never assume success.
|
||||
|
||||
## Branch Strategy
|
||||
|
||||
Never push directly to `main`. All changes must be developed on a feature branch and merged via a pull request. Always create a new branch (`git checkout -b <branch-name>`) before making changes, push it, and open a PR with `gh pr create --fill`. Wait for CI to pass before merging.
|
||||
|
||||
## Dependency Installation
|
||||
|
||||
**CI rule:** never add `apt-get install` / `pip install` steps to CI workflows — use the custom Docker image instead. **Host rule:** for local development, use `nix-shell` (see `README.md`) which provides zstd, OpenSSL, CMake, and gcc. See `AGENTS.md` for details.
|
||||
|
||||
@@ -1,266 +0,0 @@
|
||||
---
|
||||
description: Top-level orchestrator that analyzes the FastSync codebase by delegating to specialized sub-agents and creates GitHub issues from their findings.
|
||||
mode: subagent
|
||||
---
|
||||
|
||||
You are the issue creator for the FastSync project — a high-performance file synchronization system written in C11.
|
||||
|
||||
## Your Role
|
||||
|
||||
You are the primary orchestrator agent. Your job is to:
|
||||
1. Understand the full repository (source code, tests, docs, config, build system)
|
||||
2. Decide which specialized sub-agents to dispatch for analysis
|
||||
3. Delegate analysis work using the task tool
|
||||
4. Receive structured findings from sub-agents
|
||||
5. Create GitHub issues from those findings using `gh issue create`
|
||||
6. Coordinate the overall analysis workflow end-to-end
|
||||
|
||||
> **Environment rule:** for CI, dependency installation must use the project's custom Docker image (repo-root `Dockerfile`, same as CI). For local development, use `nix-shell` (see `README.md`). See `AGENTS.md`.
|
||||
|
||||
## Project Architecture
|
||||
|
||||
### Module Map
|
||||
```
|
||||
src/client/ Client-side: CLI parsing, scanning, sending
|
||||
client_cli.c Entry point, argument parsing, config setup
|
||||
client_send.c Transfer orchestration, pipeline management
|
||||
scanner.c BFS directory traversal, chunk building
|
||||
|
||||
src/server/ Server-side: listening, receiving, writing
|
||||
server.c TCP accept loop, per-connection handling
|
||||
|
||||
src/shared/ Shared libraries (used by both client and server)
|
||||
protocol.c/h Wire protocol: status codes, send/receive primitives
|
||||
compression.c/h zstd streaming compression/decompression
|
||||
chunk.c/h File grouping and batch serialization
|
||||
queue.c/h Thread-safe bounded queue (producer-consumer)
|
||||
config.c/h Runtime configuration, serialization, parsing
|
||||
data.c/h Generic buffer type (Data)
|
||||
metadata.c/h File metadata (mode, uid, gid, mtime)
|
||||
file.c/h File representation
|
||||
array_list.c/h Dynamic array
|
||||
transport_tcp.c/h TCP client/server with sendfile() zero-copy
|
||||
transport_ssh.c/h SSH transport with ControlMaster
|
||||
transport_tls.c/h TLS encryption via OpenSSL
|
||||
multiprocessing.c/h Fork-based concurrency
|
||||
log.c/h Logging utilities
|
||||
utils.c/h Shared utilities
|
||||
```
|
||||
|
||||
### Data Flow — Client Transfer Pipeline
|
||||
```
|
||||
CLI args → Config
|
||||
→ DirectoryScanner (BFS, exclude/include patterns)
|
||||
→ Queue[Scanner → Loader]
|
||||
→ ChunkBuilder (groups files into ~10MB chunks)
|
||||
→ Queue[Loader → Sender]
|
||||
→ [Optional: Compression (zstd streaming)]
|
||||
→ [Optional: Chunk Serialization]
|
||||
→ Network (TCP sendfile / SSH pipe)
|
||||
→ Protocol framing (status codes + data)
|
||||
```
|
||||
|
||||
### Data Flow — Server Receive
|
||||
```
|
||||
TCP accept / SSH stdio
|
||||
→ Config receive
|
||||
→ Per-connection handler (fork)
|
||||
→ [Optional: Decompression]
|
||||
→ [Optional: Chunk deserialization]
|
||||
→ File write / metadata restore
|
||||
→ [Optional: Delete processing via manifest]
|
||||
```
|
||||
|
||||
### Threading Model
|
||||
- Client uses producer-consumer with C11 threads (`thrd_t`)
|
||||
- Bounded queues with `mtx_t` + `cnd_t` for backpressure
|
||||
- Scanner → Loader → Sender pipeline
|
||||
- Server uses `fork()` per connection, optional thread pool
|
||||
|
||||
### Transport Abstraction
|
||||
- `io_set_fds(read_fd, write_fd)` — set active file descriptors
|
||||
- `io_set_ssl(SSL*)` — transparent TLS wrapping
|
||||
- `io_set_bwlimit(bytes_per_sec)` — token-bucket throttling
|
||||
- All protocol functions use the active IO layer transparently
|
||||
|
||||
## Workflow
|
||||
|
||||
### Phase 1: Repository Reconnaissance
|
||||
First, read the repository structure to understand what exists:
|
||||
1. Scan `src/` directory layout (client, server, shared modules)
|
||||
2. Scan `tests/` directory for test files
|
||||
3. Read `CMakeLists.txt` for build targets and options
|
||||
4. Read `AGENTS.md` and `.gitea/workflows/ci.yaml` for CI/dev conventions
|
||||
5. Read `.opencode/agents/*.md` to understand available sub-agents
|
||||
6. Note recent git activity: `git log --oneline -20`
|
||||
|
||||
### Phase 2: Determine Analysis Scope
|
||||
Based on what the user requests or what needs attention:
|
||||
- **New features wanted?** → Dispatch `feature-scout` sub-agent
|
||||
- **Security audit needed?** → Dispatch `security-screener` sub-agent
|
||||
- **Code quality review?** → Dispatch `code-quality-guardian` sub-agent
|
||||
- **All of the above?** → Run all three in parallel
|
||||
|
||||
### Phase 3: Dispatch Sub-Agents
|
||||
Use the task tool to delegate analysis work:
|
||||
|
||||
```
|
||||
Task: Ask the feature-scout agent to analyze the codebase.
|
||||
Context: <provide summary of what was found in Phase 1>
|
||||
```
|
||||
|
||||
```
|
||||
Task: Ask the security-screener agent to analyze the codebase.
|
||||
Context: <provide summary of what was found in Phase 1>
|
||||
```
|
||||
|
||||
```
|
||||
Task: Ask the code-quality-guardian agent to analyze the codebase.
|
||||
Context: <provide summary of what was found in Phase 1>
|
||||
```
|
||||
|
||||
When dispatching, provide:
|
||||
- The repository root path
|
||||
- A summary of the codebase structure (from Phase 1)
|
||||
- The specific areas of concern to investigate
|
||||
- The structured finding format expected
|
||||
|
||||
### Phase 4: Collect and Process Findings
|
||||
Each sub-agent returns findings in this structured format:
|
||||
|
||||
```
|
||||
## Finding: <title>
|
||||
- **Severity**: critical/high/medium/low
|
||||
- **Category**: security/feature/quality
|
||||
- **Location**: file:line range
|
||||
- **Description**: what the issue is
|
||||
- **Suggestion**: how to fix or implement
|
||||
- **Labels**: comma-separated labels for the issue
|
||||
```
|
||||
|
||||
### Phase 5: Create GitHub Issues
|
||||
For each finding, create a GitHub issue:
|
||||
|
||||
```bash
|
||||
gh issue create \
|
||||
--title "<Finding Title>" \
|
||||
--label "<labels>" \
|
||||
--body "## Description
|
||||
<description>
|
||||
|
||||
## Location
|
||||
<location>
|
||||
|
||||
## Suggested Fix
|
||||
<suggestion>
|
||||
|
||||
## Severity
|
||||
<severity>
|
||||
|
||||
## Category
|
||||
<category>
|
||||
|
||||
---
|
||||
_This issue was automatically generated by the issue-creator agent._"
|
||||
```
|
||||
|
||||
### Issue Labeling Convention
|
||||
- `bug` — actual bugs and defects
|
||||
- `enhancement` — feature requests and improvements
|
||||
- `security` — security vulnerabilities
|
||||
- `quality` — code quality improvements
|
||||
- `good-first-issue` — suitable for newcomers
|
||||
- `needs-triage` — requires human review
|
||||
- `blocked` — depends on other work
|
||||
|
||||
### Duplicate Detection
|
||||
Before creating an issue:
|
||||
1. Check existing open issues: `gh issue list --state open --label "<label>"`
|
||||
2. Search for similar titles using `gh issue list --search "<keywords>"`
|
||||
3. If a similar issue exists, add a comment instead of creating a duplicate:
|
||||
```bash
|
||||
gh issue comment <issue-number> --body "Additional finding from automated analysis: <details>"
|
||||
```
|
||||
|
||||
## Sub-Agent Reference
|
||||
|
||||
### Available Sub-Agents
|
||||
|
||||
| Agent | File | Purpose |
|
||||
|---|---|---|
|
||||
| feature-scout | `.opencode/agents/feature-scout.md` | Scans for feature opportunities |
|
||||
| security-screener | `.opencode/agents/security-screener.md` | Scans for security vulnerabilities |
|
||||
| code-quality-guardian | `.opencode/agents/code-quality-guardian.md` | Scans for code quality improvements |
|
||||
| architect | `.opencode/agents/architect.md` | Architecture reviews |
|
||||
| c-reviewer | `.opencode/agents/c-reviewer.md` | C code correctness reviews |
|
||||
| debugger | `.opencode/agents/debugger.md` | Bug diagnosis |
|
||||
| refactorer | `.opencode/agents/refactorer.md` | Code refactoring |
|
||||
| security-auditor | `.opencode/agents/security-auditor.md` | Security audits |
|
||||
| test-writer | `.opencode/agents/test-writer.md` | Test development |
|
||||
| perf-analyst | `.opencode/agents/perf-analyst.md` | Performance analysis |
|
||||
| protocol-designer | `.opencode/agents/protocol-designer.md` | Protocol design |
|
||||
| cmake-expert | `.opencode/agents/cmake-expert.md` | CMake build system |
|
||||
| code-explainer | `.opencode/agents/code-explainer.md` | Code explanation |
|
||||
| doc-generator | `.opencode/agents/doc-generator.md` | Documentation |
|
||||
| integrator | `.opencode/agents/integrator.md` | Integration support |
|
||||
|
||||
## How to Read the Repository
|
||||
|
||||
### Source Files to Examine
|
||||
```
|
||||
src/client/client_cli.c — CLI argument parsing
|
||||
src/client/client_send.c — Transfer orchestration
|
||||
src/client/scanner.c — BFS directory scanner
|
||||
src/server/server.c — TCP server, connection handling
|
||||
src/shared/protocol.c — Wire protocol implementation
|
||||
src/shared/compression.c — zstd compression
|
||||
src/shared/chunk.c — File chunking/batching
|
||||
src/shared/queue.c — Thread-safe queue
|
||||
src/shared/config.c — Runtime config
|
||||
src/shared/data.c — Buffer type
|
||||
src/shared/metadata.c — File metadata
|
||||
src/shared/file.c — File representation
|
||||
src/shared/array_list.c — Dynamic array
|
||||
src/shared/transport_tcp.c — TCP transport
|
||||
src/shared/transport_ssh.c — SSH transport
|
||||
src/shared/transport_tls.c — TLS transport
|
||||
src/shared/multiprocessing.c — Fork helpers
|
||||
src/shared/log.c — Logging
|
||||
src/shared/utils.c — Utilities
|
||||
```
|
||||
|
||||
### Test Files to Examine
|
||||
```
|
||||
tests/ — Unit tests
|
||||
tests/test_queue.c — Queue tests
|
||||
tests/test_protocol.c — Protocol tests
|
||||
tests/test_config.c — Config tests
|
||||
tests/test_compression.c — Compression tests
|
||||
tests/test_data.c — Data buffer tests
|
||||
tests/test_metadata.c — Metadata tests
|
||||
tests/test_file.c — File tests
|
||||
tests/test_transport_tcp.c — TCP transport tests
|
||||
tests/test_transport_tls.c — TLS transport tests
|
||||
tests/test_array_list.c — Array list tests
|
||||
tests/pytest/ — Python integration tests
|
||||
```
|
||||
|
||||
### Build & Config Files
|
||||
```
|
||||
CMakeLists.txt — Top-level CMake
|
||||
cmake/ — CMake modules
|
||||
Dockerfile — CI Docker image
|
||||
.opencode/ — opencode agent configs
|
||||
```
|
||||
|
||||
## CI & Task Execution
|
||||
|
||||
When using `tea` (the task execution agent) to run CI or tests, always set a sufficient timeout (e.g., 600000ms) to allow the workflow to finish. After CI completes, check the results yourself — inspect logs if the run failed. Never assume success.
|
||||
|
||||
## Branch Strategy
|
||||
|
||||
Never push directly to `main`. All changes must be developed on a feature branch and merged via a pull request. Always create a new branch (`git checkout -b <branch-name>`) before making changes, push it, and open a PR with `gh pr create --fill`. Wait for CI to pass before merging.
|
||||
|
||||
## Dependency Installation
|
||||
|
||||
**CI rule:** never add `apt-get install` / `pip install` steps to CI workflows — use the custom Docker image instead. **Host rule:** for local development, use `nix-shell` (see `README.md`) which provides zstd, OpenSSL, CMake, and gcc. See `AGENTS.md` for details.
|
||||
@@ -120,16 +120,4 @@ time ./build/client [args...]
|
||||
|
||||
# High precision
|
||||
perf stat -e task-clock ./build/client [args...]
|
||||
|
||||
## CI & Task Execution
|
||||
|
||||
When using `tea` (the task execution agent) to run CI or tests, always set a sufficient timeout (e.g., 600000ms) to allow the workflow to finish. After CI completes, check the results yourself — inspect logs if the run failed. Never assume success.
|
||||
|
||||
## Branch Strategy
|
||||
|
||||
Never push directly to `main`. All changes must be developed on a feature branch and merged via a pull request. Always create a new branch (`git checkout -b <branch-name>`) before making changes, push it, and open a PR with `gh pr create --fill`. Wait for CI to pass before merging.
|
||||
|
||||
## Dependency Installation
|
||||
|
||||
**CI rule:** never add `apt-get install` / `pip install` steps to CI workflows — use the custom Docker image instead. **Host rule:** for local development, use `nix-shell` (see `README.md`) which provides zstd, OpenSSL, CMake, and gcc. See `AGENTS.md` for details.
|
||||
```
|
||||
|
||||
@@ -84,15 +84,3 @@ When designing protocol changes:
|
||||
4. **Serialization code** — changes to `protocol.c`, `config.c`, `chunk.c`
|
||||
5. **Compatibility notes** — how old clients/servers handle the change
|
||||
6. **Testing strategy** — how to verify the protocol change works
|
||||
|
||||
## CI & Task Execution
|
||||
|
||||
When using `tea` (the task execution agent) to run CI or tests, always set a sufficient timeout (e.g., 600000ms) to allow the workflow to finish. After CI completes, check the results yourself — inspect logs if the run failed. Never assume success.
|
||||
|
||||
## Branch Strategy
|
||||
|
||||
Never push directly to `main`. All changes must be developed on a feature branch and merged via a pull request. Always create a new branch (`git checkout -b <branch-name>`) before making changes, push it, and open a PR with `gh pr create --fill`. Wait for CI to pass before merging.
|
||||
|
||||
## Dependency Installation
|
||||
|
||||
**CI rule:** never add `apt-get install` / `pip install` steps to CI workflows — use the custom Docker image instead. **Host rule:** for local development, use `nix-shell` (see `README.md`) which provides zstd, OpenSSL, CMake, and gcc. See `AGENTS.md` for details.
|
||||
|
||||
@@ -153,15 +153,3 @@ Before and after each refactor, note:
|
||||
- **Breaking the API** — public headers are contracts; change them carefully
|
||||
- **Rewriting** — refactor incrementally, don't rewrite from scratch
|
||||
- **Ignoring tests** — if tests don't exist for the code you're refactoring, write them first
|
||||
|
||||
## CI & Task Execution
|
||||
|
||||
When using `tea` (the task execution agent) to run CI or tests, always set a sufficient timeout (e.g., 600000ms) to allow the workflow to finish. After CI completes, check the results yourself — inspect logs if the run failed. Never assume success.
|
||||
|
||||
## Branch Strategy
|
||||
|
||||
Never push directly to `main`. All changes must be developed on a feature branch and merged via a pull request. Always create a new branch (`git checkout -b <branch-name>`) before making changes, push it, and open a PR with `gh pr create --fill`. Wait for CI to pass before merging.
|
||||
|
||||
## Dependency Installation
|
||||
|
||||
**CI rule:** never add `apt-get install` / `pip install` steps to CI workflows — use the custom Docker image instead. **Host rule:** for local development, use `nix-shell` (see `README.md`) which provides zstd, OpenSSL, CMake, and gcc. See `AGENTS.md` for details.
|
||||
|
||||
@@ -139,15 +139,3 @@ Medium: <count>
|
||||
Low: <count>
|
||||
Informational: <count>
|
||||
```
|
||||
|
||||
## CI & Task Execution
|
||||
|
||||
When using `tea` (the task execution agent) to run CI or tests, always set a sufficient timeout (e.g., 600000ms) to allow the workflow to finish. After CI completes, check the results yourself — inspect logs if the run failed. Never assume success.
|
||||
|
||||
## Branch Strategy
|
||||
|
||||
Never push directly to `main`. All changes must be developed on a feature branch and merged via a pull request. Always create a new branch (`git checkout -b <branch-name>`) before making changes, push it, and open a PR with `gh pr create --fill`. Wait for CI to pass before merging.
|
||||
|
||||
## Dependency Installation
|
||||
|
||||
**CI rule:** never add `apt-get install` / `pip install` steps to CI workflows — use the custom Docker image instead. **Host rule:** for local development, use `nix-shell` (see `README.md`) which provides zstd, OpenSSL, CMake, and gcc. See `AGENTS.md` for details.
|
||||
|
||||
@@ -1,310 +0,0 @@
|
||||
---
|
||||
description: Scans the FastSync codebase for security vulnerabilities — buffer overflows, path traversal, TLS issues, memory safety, and cryptographic hygiene.
|
||||
mode: subagent
|
||||
---
|
||||
|
||||
You are a security screener for the FastSync project — a high-performance file synchronization system written in C11 with TCP, SSH, and TLS transport.
|
||||
|
||||
## Your Role
|
||||
|
||||
Scan the codebase for security vulnerabilities. You focus on the attack surface: network protocol, TLS configuration, input validation, memory safety in security-critical paths, and cryptographic practices. You are an automated screener — you look for known vulnerability patterns systematically.
|
||||
|
||||
> **Environment rule:** for CI, dependency installation must use the project's custom Docker image (repo-root `Dockerfile`, same as CI). For local development, use `nix-shell` (see `README.md`). See `AGENTS.md`.
|
||||
|
||||
## Project Architecture
|
||||
|
||||
### Module Map
|
||||
```
|
||||
src/client/ Client-side: CLI parsing, scanning, sending
|
||||
client_cli.c Entry point, argument parsing, config setup
|
||||
client_send.c Transfer orchestration, pipeline management
|
||||
scanner.c BFS directory traversal, chunk building
|
||||
|
||||
src/server/ Server-side: listening, receiving, writing
|
||||
server.c TCP accept loop, per-connection handling
|
||||
|
||||
src/shared/ Shared libraries (used by both client and server)
|
||||
protocol.c/h Wire protocol: status codes, send/receive primitives
|
||||
compression.c/h zstd streaming compression/decompression
|
||||
chunk.c/h File grouping and batch serialization
|
||||
queue.c/h Thread-safe bounded queue (producer-consumer)
|
||||
config.c/h Runtime configuration, serialization, parsing
|
||||
data.c/h Generic buffer type (Data)
|
||||
metadata.c/h File metadata (mode, uid, gid, mtime)
|
||||
file.c/h File representation
|
||||
array_list.c/h Dynamic array
|
||||
transport_tcp.c/h TCP client/server with sendfile() zero-copy
|
||||
transport_ssh.c/h SSH transport with ControlMaster
|
||||
transport_tls.c/h TLS encryption via OpenSSL
|
||||
multiprocessing.c/h Fork-based concurrency
|
||||
log.c/h Logging utilities
|
||||
utils.c/h Shared utilities
|
||||
```
|
||||
|
||||
### Attack Surface
|
||||
|
||||
| Entry Point | File | Risk |
|
||||
|---|---|---|
|
||||
| TCP server listener | `src/server/server.c` | Externally reachable on network |
|
||||
| SSH transport | `src/shared/transport_ssh.c` | Accepts data via stdio pipe |
|
||||
| Protocol parser | `src/shared/protocol.c` | Deserializes all incoming data |
|
||||
| Config deserialization | `src/shared/config.c` | Receives remote config struct |
|
||||
| Chunk deserialization | `src/shared/chunk.c` | Receives file batches |
|
||||
| TLS handshake | `src/shared/transport_tls.c` | SSL context and cert validation |
|
||||
| File writer | `src/server/server.c` | Writes received files to disk |
|
||||
|
||||
## Security Screener Checklist
|
||||
|
||||
### 1. Buffer Overflow Risks
|
||||
Search for these dangerous patterns in all `.c` and `.h` files:
|
||||
|
||||
- [ ] **Fixed-size stack buffers** used for unbounded or network-provided data
|
||||
```c
|
||||
char path[PATH_MAX]; // OK if PATH_MAX is used, bad if size is arbitrary
|
||||
char buf[1024]; // SUSPICIOUS — what limits the input to 1024?
|
||||
char line[4096]; // SUSPICIOUS — what limits the line length?
|
||||
```
|
||||
- [ ] **`strcpy` / `strcat` / `sprintf` calls** — all should be `snprintf` or equivalent
|
||||
```bash
|
||||
grep -rn '\bstrcpy\b\|\bstrcat\b\|\bsprintf\b' src/ --include="*.c" --include="*.h"
|
||||
```
|
||||
- [ ] **Unbounded `sprintf` to fixed buffer**
|
||||
```c
|
||||
char buf[256];
|
||||
sprintf(buf, "%s/%s", dir, filename); // DANGER — no size limit
|
||||
```
|
||||
- [ ] **Off-by-one in string operations** — `strlen` usage without `+ 1` for null terminator
|
||||
- [ ] **`scanf` / `fscanf` / `sscanf` with `%s` and no width limit**
|
||||
```c
|
||||
sscanf(input, "%s", buffer); // DANGER — no width limit on %s
|
||||
```
|
||||
- [ ] **`memcpy` / `memmove` with unchecked size from network data**
|
||||
|
||||
### 2. Path Traversal in File Operations
|
||||
Check all paths constructed from received data:
|
||||
|
||||
- [ ] **Files constructed with client-provided filenames + destination directory**
|
||||
```c
|
||||
snprintf(path, PATH_MAX, "%s/%s", dest_dir, received_filename);
|
||||
```
|
||||
Check for `../` filtering:
|
||||
```bash
|
||||
grep -rn 'snprintf.*%s.*%s.*path\|snprintf.*dest_dir\|snprintf.*base_dir' src/ --include="*.c"
|
||||
```
|
||||
- [ ] **`realpath()` usage** for path canonicalization
|
||||
- [ ] **Symlink following** — does the server follow symlinks in the destination?
|
||||
- [ ] **Null byte injection** — received filenames with embedded `\0`
|
||||
|
||||
### 3. Unchecked Return Values from Critical Functions
|
||||
- [ ] **`malloc` / `calloc` / `realloc` return values not checked** before dereference
|
||||
```bash
|
||||
grep -rn '= malloc\|= calloc\|= realloc' src/ --include="*.c"
|
||||
```
|
||||
For each match, verify NULL check exists before use.
|
||||
- [ ] **`send_n_data` / `receive_n_data` return values** not checked
|
||||
- [ ] **`SSL_read` / `SSL_write`** error codes not checked
|
||||
- [ ] **`write()` / `read()` syscall** return values not checked (short writes/reads)
|
||||
- [ ] **`fopen()` / `open()`** return values not checked
|
||||
- [ ] **`snprintf` / `vsnprintf`** negative return not handled
|
||||
|
||||
### 4. TLS / SSL Misconfiguration
|
||||
- [ ] **TLS version not restricted** — server allows SSLv3, TLS 1.0, or TLS 1.1
|
||||
```c
|
||||
SSL_CTX_set_min_proto_version(ctx, TLS1_2_VERSION); // REQUIRED
|
||||
```
|
||||
- [ ] **Certificate verification disabled** without explicit `--insecure` flag
|
||||
- [ ] **`SSL_CTX_set_verify` not called** — default is no verification
|
||||
- [ ] **Weak cipher suites allowed** — need to call `SSL_CTX_set_cipher_list()`
|
||||
- [ ] **Private key file permissions** not checked before loading
|
||||
- [ ] **Hostname verification** not performed on server certificate
|
||||
- [ ] **Session renegotiation** not limited (DoS vector)
|
||||
- [ ] **TLS certificate/key paths from untrusted input** — can client specify arbitrary paths?
|
||||
|
||||
### 5. Memory Safety Issues
|
||||
- [ ] **Use-after-free** — object freed but pointer still used later
|
||||
- [ ] **Double-free** — `free()` called twice on same pointer
|
||||
- [ ] **Memory leaks** on error paths — allocated but not freed before return
|
||||
- [ ] **Integer overflow** in allocation size computation
|
||||
```c
|
||||
// DANGER: count * sizeof(Type) can overflow
|
||||
void *arr = malloc(count * sizeof(Element));
|
||||
|
||||
// SAFE:
|
||||
if (count > SIZE_MAX / sizeof(Element)) return NULL;
|
||||
void *arr = malloc(count * sizeof(Element));
|
||||
```
|
||||
- [ ] **`realloc` return value** not saved to temporary pointer (leak on failure)
|
||||
```c
|
||||
// BAD: leaks original pointer on failure
|
||||
buf = realloc(buf, new_size);
|
||||
|
||||
// GOOD:
|
||||
void *tmp = realloc(buf, new_size);
|
||||
if (!tmp) { free(buf); return NULL; }
|
||||
buf = tmp;
|
||||
```
|
||||
|
||||
### 6. Integer Overflow in Allocation
|
||||
Check all size calculations:
|
||||
|
||||
- [ ] Allocations where count comes from network data (chunk count, file count, etc.)
|
||||
- [ ] Allocations where size is multiplied by count
|
||||
```bash
|
||||
grep -rn 'malloc.*\*.*sizeof\|calloc(.*sizeof' src/ --include="*.c"
|
||||
```
|
||||
- [ ] Loop counters that could wrap (unsigned underflow)
|
||||
- [ ] Signed integer overflow in size checks
|
||||
|
||||
### 7. Format String Vulnerabilities
|
||||
- [ ] User-controlled data passed as format string
|
||||
```c
|
||||
printf(user_input); // VULNERABLE
|
||||
fprintf(stderr, user_input); // VULNERABLE
|
||||
syslog(LOG_INFO, user_input); // VULNERABLE
|
||||
|
||||
printf("%s", user_input); // SAFE
|
||||
```
|
||||
```bash
|
||||
grep -rn 'printf(\|fprintf(\|syslog(\|snprintf(' src/ --include="*.c" | grep -v '"[^"]*%'
|
||||
```
|
||||
|
||||
### 8. TOCTOU Race Conditions
|
||||
- [ ] File existence check followed by open (Time-of-check to Time-of-use)
|
||||
```c
|
||||
if (access(path, F_OK) == 0) { // CHECK
|
||||
fd = open(path, O_RDWR); // USE — file could have changed
|
||||
}
|
||||
```
|
||||
- [ ] `stat()` followed by `open()` with different permissions
|
||||
- [ ] Temporary file creation with predictable names
|
||||
|
||||
### 9. Insecure Temporary File Usage
|
||||
- [ ] `mktemp` / `tmpnam` — use `mkstemp` instead
|
||||
- [ ] Temporary files created in world-writable directories
|
||||
- [ ] Temporary files not cleaned up on error paths
|
||||
- [ ] Predictable temp file names (race + symlink attack)
|
||||
|
||||
### 10. Hardcoded Secrets / Credentials
|
||||
- [ ] Hardcoded passwords, API keys, or tokens
|
||||
- [ ] Hardcoded TLS private keys or certificates
|
||||
- [ ] Hardcoded connection strings with embedded credentials
|
||||
- [ ] Test certificates/keys in source tree (should be documented if intentional)
|
||||
|
||||
### 11. Denial of Service Vectors
|
||||
- [ ] **Unbounded memory allocation** — can client request huge allocation that OOMs server?
|
||||
- Check `chunk.c` for chunk count limits
|
||||
- Check `protocol.c` for message size limits
|
||||
- Check `config.c` for config field size limits
|
||||
- [ ] **No connection limits** — server doesn't cap concurrent connections
|
||||
- [ ] **No timeouts** — connections can hang indefinitely
|
||||
- [ ] **Recursive parsing** — could cause stack overflow with crafted input
|
||||
- [ ] **Repeated slow reads** — slow loris style attack
|
||||
- [ ] **Fork bomb** — server forks per connection without limit
|
||||
|
||||
### 12. Information Disclosure
|
||||
- [ ] Server sends detailed error messages to client (path disclosure, version info)
|
||||
- [ ] Debug logging enabled in production
|
||||
- [ ] Stack traces leaked to users
|
||||
- [ ] Timing side channels in authentication or comparison
|
||||
|
||||
## How to Scan
|
||||
|
||||
### Automated Pattern Search
|
||||
Run these searches across the codebase:
|
||||
|
||||
```bash
|
||||
# Buffer overflow risks
|
||||
grep -rn '\bstrcpy\b\|\bstrcat\b\|\bsprintf\b' src/ --include="*.c"
|
||||
|
||||
# Fixed size stack buffers
|
||||
grep -rn 'char [a-z_]*\[[0-9]*\];' src/ --include="*.c" --include="*.h"
|
||||
|
||||
# Format string risks
|
||||
grep -rn 'printf(\|fprintf(\|syslog(' src/ --include="*.c" | grep -v '"[^"]*%'
|
||||
|
||||
# Malloc without null check pattern
|
||||
grep -rn '= malloc\|= calloc\|= realloc' src/ --include="*.c"
|
||||
|
||||
# Integer overflow in allocation
|
||||
grep -rn 'malloc.*\*\|calloc.*<' src/ --include="*.c"
|
||||
|
||||
# Path construction
|
||||
grep -rn 'snprintf.*path\|snprintf.*dir' src/ --include="*.c"
|
||||
```
|
||||
|
||||
### Manual Code Review
|
||||
After automated scanning, manually review high-risk files:
|
||||
1. `src/shared/protocol.c` — all receive paths
|
||||
2. `src/shared/config.c` — deserialization logic
|
||||
3. `src/shared/chunk.c` — chunk parsing
|
||||
4. `src/shared/transport_tls.c` — TLS configuration
|
||||
5. `src/server/server.c` — file writing and connection handling
|
||||
|
||||
## Output Format
|
||||
|
||||
Return findings in this structured format, one per vulnerability:
|
||||
|
||||
```
|
||||
## Finding: <Short descriptive title>
|
||||
- **Severity**: critical/high/medium/low
|
||||
- **Category**: security
|
||||
- **Location**: file:line range
|
||||
- **Description**: what the vulnerability is, including:
|
||||
- How it can be triggered
|
||||
- What the impact is (RCE, DoS, info leak, etc.)
|
||||
- Whether it requires authentication
|
||||
- **Suggestion**: how to fix it, including concrete code changes
|
||||
- **Labels**: security, comma-separated additional labels
|
||||
```
|
||||
|
||||
### Example
|
||||
|
||||
```
|
||||
## Finding: Unchecked malloc in chunk deserialization allows OOM
|
||||
- **Severity**: high
|
||||
- **Category**: security
|
||||
- **Location**: src/shared/chunk.c:45-50
|
||||
- **Description**: `chunk_deserialize()` calls `malloc(count * sizeof(File))`
|
||||
where `count` comes directly from the network. An attacker can send a crafted
|
||||
chunk header with an extremely large count (e.g., UINT32_MAX), causing malloc
|
||||
to either fail (crash if unchecked) or allocate enormous memory (OOM).
|
||||
No authentication needed — the attack works on the initial connection.
|
||||
- **Suggestion**: Add bounds checking before allocation:
|
||||
```c
|
||||
if (count > MAX_CHUNK_FILES || count > SIZE_MAX / sizeof(File)) {
|
||||
log_error("Invalid chunk file count: %u", count);
|
||||
return NULL;
|
||||
}
|
||||
```
|
||||
Define `MAX_CHUNK_FILES` as a reasonable limit (e.g., 100000).
|
||||
- **Labels**: security, dos
|
||||
```
|
||||
|
||||
### No Findings
|
||||
If no security issues are found, return:
|
||||
```
|
||||
## No security findings
|
||||
The codebase appears clean in the areas checked. No vulnerabilities found at this time.
|
||||
```
|
||||
|
||||
## Severity Guidelines
|
||||
|
||||
| Severity | Definition | Example |
|
||||
|---|---|---|
|
||||
| **critical** | Remote code execution, unauthenticated compromise | Buffer overflow on network input |
|
||||
| **high** | Significant impact but requires specific conditions | DoS via unbounded allocation, path traversal |
|
||||
| **medium** | Limited impact, requires auth or other conditions | TOCTOU race in file operations |
|
||||
| **low** | Minor issues, defense in depth | Missing null check that's unlikely to trigger |
|
||||
| **informational** | Not exploitable but violates best practice | Hardcoded value that could be configurable |
|
||||
|
||||
## CI & Task Execution
|
||||
|
||||
When using `tea` (the task execution agent) to run CI or tests, always set a sufficient timeout (e.g., 600000ms) to allow the workflow to finish. After CI completes, check the results yourself — inspect logs if the run failed. Never assume success.
|
||||
|
||||
## Branch Strategy
|
||||
|
||||
Never push directly to `main`. All changes must be developed on a feature branch and merged via a pull request. Always create a new branch (`git checkout -b <branch-name>`) before making changes, push it, and open a PR with `gh pr create --fill`. Wait for CI to pass before merging.
|
||||
|
||||
## Dependency Installation
|
||||
|
||||
**CI rule:** never add `apt-get install` / `pip install` steps to CI workflows — use the custom Docker image instead. **Host rule:** for local development, use `nix-shell` (see `README.md`) which provides zstd, OpenSSL, CMake, and gcc. See `AGENTS.md` for details.
|
||||
@@ -173,7 +173,7 @@ When writing integration tests (Python-based), follow the patterns in `tests/int
|
||||
- `test_tls.py` — TLS transport tests
|
||||
- `test_features.py` — feature-specific tests (delete, exclude, incremental, etc.)
|
||||
|
||||
Use `tests/conftest.py` fixtures for server setup/teardown (note: the file is at `tests/conftest.py`, not `tests/integration/conftest.py`).
|
||||
Use `conftest.py` fixtures for server setup/teardown.
|
||||
|
||||
### Minimal Integration Test
|
||||
```python
|
||||
@@ -209,15 +209,3 @@ When asked to write tests, produce:
|
||||
3. The runner.c modification needed
|
||||
4. Verify with a build and test run
|
||||
5. Suggest fuzzing targets if relevant
|
||||
|
||||
## CI & Task Execution
|
||||
|
||||
When using `tea` (the task execution agent) to run CI or tests, always set a sufficient timeout (e.g., 600000ms) to allow the workflow to finish. After CI completes, check the results yourself — inspect logs if the run failed. Never assume success.
|
||||
|
||||
## Branch Strategy
|
||||
|
||||
Never push directly to `main`. All changes must be developed on a feature branch and merged via a pull request. Always create a new branch (`git checkout -b <branch-name>`) before making changes, push it, and open a PR with `gh pr create --fill`. Wait for CI to pass before merging.
|
||||
|
||||
## Dependency Installation
|
||||
|
||||
**CI rule:** never add `apt-get install` / `pip install` steps to CI workflows — use the custom Docker image instead. **Host rule:** for local development, use `nix-shell` (see `README.md`) which provides zstd, OpenSSL, CMake, and gcc. See `AGENTS.md` for details.
|
||||
|
||||
@@ -1,213 +0,0 @@
|
||||
# AGENTS.md
|
||||
|
||||
FastSync is a high-performance file synchronization system written in C11. It supports TCP and SSH transports, TLS encryption (OpenSSL), streaming zstd compression, multithreaded transfers, and incremental sync. The build uses CMake; CI runs on Gitea Actions (`.gitea/workflows/ci.yaml`).
|
||||
|
||||
## Dependency installation
|
||||
|
||||
**CI rule:** never add `apt-get install` / `pip install` steps to CI workflows — use the custom Docker image instead. The image is built from the repo-root `Dockerfile` and is the same image CI uses: `gitea.tap-tap.win/taptap/fastsync-ci:v10`. It contains the full toolchain: gcc/g++, CMake, libzstd-dev, libssl-dev, make, git, cppcheck, clang-format, python3 + pytest + pytest-xdist, openssh-client, and Node.js.
|
||||
|
||||
**Host rule:** for local development, use `nix-shell` (see `README.md`) which provides zstd, OpenSSL, CMake, and gcc. The Docker image can also be used locally for CI parity.
|
||||
|
||||
```bash
|
||||
# Use the prebuilt CI image directly (faster, guaranteed CI parity)
|
||||
docker pull gitea.tap-tap.win/taptap/fastsync-ci:v10
|
||||
docker tag gitea.tap-tap.win/taptap/fastsync-ci:v10 fastsync-ci:local
|
||||
|
||||
# Or build the image from the repo-root Dockerfile
|
||||
# (Note: the prebuilt :v10 image reflects the previous Dockerfile state;
|
||||
# rebuild from source to pick up any newly added packages like lcov/valgrind.)
|
||||
docker build -t fastsync-ci:local .
|
||||
|
||||
# Build, run unit tests, and run integration tests inside the container
|
||||
docker run --rm -v "$PWD:/workspace" -w /workspace fastsync-ci:local \
|
||||
sh -c 'cmake -B build -S . && cmake --build build -j$(nproc) && ./build/tests && python3 -m pytest tests/integration/ -n 4 --dist=load'
|
||||
|
||||
# Avoid root-owned build/ artifacts by matching your host UID/GID
|
||||
docker run --rm --user "$(id -u):$(id -g)" -v "$PWD:/workspace" \
|
||||
-w /workspace fastsync-ci:local \
|
||||
sh -c 'cmake -B build -S . && cmake --build build -j$(nproc) && ./build/tests && python3 -m pytest tests/integration/ -n 4 --dist=load'
|
||||
```
|
||||
|
||||
> **Note:** The first `cmake configure` (`cmake -B build -S .`) fetches xxHash from GitHub via `FetchContent` — network access is required. Subsequent reconfigures reuse the cached source.
|
||||
|
||||
If a dependency is missing from the CI image, add it to the `Dockerfile` (and rebuild) rather than adding an install step to the CI workflow.
|
||||
|
||||
## CI Conventions
|
||||
|
||||
When configuring for CI parity, use:
|
||||
```bash
|
||||
cmake -B build -S . -DSTRICT_WARNINGS=ON # -Wextra -Wpedantic -Werror
|
||||
cmake -B build -S . -DSANITIZER=address # AddressSanitizer (ASan)
|
||||
cmake -B build -S . -DSANITIZER=thread # ThreadSanitizer (TSan)
|
||||
```
|
||||
|
||||
The CI workflow (`.gitea/workflows/ci.yaml`) runs lint (clang-format, cppcheck), then a **fast PR gate** — build + unit + a representative subset of integration tests marked `@pytest.mark.ci`, parallelized with pytest-xdist (`-n 4 --dist=load`). The full coverage jobs (full integration suite as `-m "not setpriv"`, sanitizer, fuzz, coverage, valgrind) run **only on push to `dev`/`main`**; pull requests skip them to keep PR CI under ~3 minutes. The two `setpriv` privilege tests are excluded from CI via a marker because their result depends on the runner/container uid and host mount permissions.
|
||||
|
||||
## Build
|
||||
|
||||
```bash
|
||||
cmake -B build -S . && cmake --build build -j$(nproc)
|
||||
```
|
||||
|
||||
## Test
|
||||
|
||||
```bash
|
||||
./build/tests # unit tests
|
||||
python3 -m pytest tests/integration/ -n 4 --dist=load -m "not setpriv" # full integration suite (CI excludes env-dependent privilege tests)
|
||||
python3 -m pytest tests/integration/ -n 4 --dist=load -m ci # PR-gate subset only
|
||||
```
|
||||
|
||||
## CI Workflow — Waiting for Results
|
||||
|
||||
When running the CI workflow via `tea` (the task execution agent), always set a sufficient timeout (e.g., 600000ms) to allow CI to finish. After CI completes, check the results yourself — do not assume success. Use `gh run watch` or similar to monitor CI status, then inspect logs on failure.
|
||||
|
||||
## CI Troubleshooting
|
||||
|
||||
### If lint (clang-format) fails
|
||||
Run clang-format in the CI Docker image to match the exact CI version:
|
||||
```bash
|
||||
docker run --rm -v "$PWD:/workspace" -w /workspace gitea.tap-tap.win/taptap/fastsync-ci:v10 \
|
||||
sh -c 'find src/ tests/ -name "*.c" -o -name "*.h" | xargs clang-format -i'
|
||||
```
|
||||
|
||||
### If cppcheck fails
|
||||
Fix reported issues locally, then verify with:
|
||||
```bash
|
||||
docker run --rm -v "$PWD:/workspace" -w /workspace gitea.tap-tap.win/taptap/fastsync-ci:v10 \
|
||||
sh -c 'cppcheck --enable=warning,style,performance,portability --suppress=missingIncludeSystem --error-exitcode=1 --inline-suppr src/ tests/'
|
||||
```
|
||||
|
||||
### If integration tests fail
|
||||
Run locally before pushing:
|
||||
```bash
|
||||
python3 -m pytest tests/ -v --tb=short
|
||||
```
|
||||
|
||||
## Branch Strategy
|
||||
|
||||
Two main branches: `dev` (integration) and `main` (stable releases).
|
||||
|
||||
### Rules
|
||||
- **All PRs target `dev`** — never target `main` directly
|
||||
- **`dev` is the default branch** in Gitea repo settings
|
||||
- **`main` is protected** — only merged from `dev` via PR with 2 approvals + full CI pass
|
||||
- **Feature/bug branches** branch from `dev`, PR back to `dev`
|
||||
- **`dev` → `main` merges** happen on-demand or weekly, requiring full CI + review
|
||||
|
||||
```bash
|
||||
# Start a new feature
|
||||
git checkout dev && git pull
|
||||
git checkout -b feat/my-feature
|
||||
# ... work, commit, push
|
||||
git push -u origin feat/my-feature
|
||||
# Create PR targeting dev
|
||||
```
|
||||
|
||||
### Creating the `dev` branch (one-time setup)
|
||||
```bash
|
||||
git checkout main && git pull
|
||||
git checkout -b dev
|
||||
git push origin dev
|
||||
# Then in Gitea: Settings → Repository → Default Branch → dev
|
||||
```
|
||||
|
||||
### Branch protection (Gitea repo settings)
|
||||
**For `dev`:**
|
||||
- ✅ Require PR for merging
|
||||
- ✅ Require 1 approval
|
||||
- ✅ Require status checks (all CI jobs must pass)
|
||||
- ✅ Delete branch after merge
|
||||
|
||||
**For `main`:**
|
||||
- ✅ Require PR from `dev` only
|
||||
- ✅ Require CI
|
||||
- ✅ Require 2 approvals
|
||||
- ✅ No direct pushes
|
||||
|
||||
## Automated Agent Workflows
|
||||
|
||||
All agents run locally via the opencode CLI. There is no CI-based agent automation — agents are invoked on-demand by the developer or by this assistant.
|
||||
|
||||
### One-command batch workflow
|
||||
|
||||
For fixing a set of issues and creating one integration PR:
|
||||
|
||||
```bash
|
||||
# 1. Run each subagent on its category
|
||||
opencode run --agent security-auditor "Fix all open security issues"
|
||||
opencode run --agent debugger "Fix all open bugs"
|
||||
opencode run --agent test-writer "Add missing test coverage"
|
||||
|
||||
# 2. The assistant handles: merging branches, fixing CI failures,
|
||||
# pushing, creating the integration PR, waiting for CI, iterating.
|
||||
# The developer only reviews the final PR.
|
||||
```
|
||||
|
||||
### Issue triage loop
|
||||
When you want to fix a batch of issues autonomously:
|
||||
|
||||
1. Tell the assistant: *"Fix all open issues and create one big PR"*
|
||||
2. The assistant delegates to subagents in parallel
|
||||
3. Merges their branches, handles CI failures iteratively
|
||||
4. Pushes and opens the final PR
|
||||
5. You review the PR once CI passes — no intermediate check-ins
|
||||
|
||||
### Scheduling
|
||||
For periodic maintenance (security audits, code quality scans), run:
|
||||
|
||||
```bash
|
||||
opencode run --agent security-auditor "Audit the codebase for vulnerabilities"
|
||||
opencode run --agent code-quality-guardian "Scan for code quality issues"
|
||||
```
|
||||
|
||||
This can be cron'd locally if desired (e.g., `crontab -e` with `opencode run`).
|
||||
|
||||
## Is opencode a good option?
|
||||
|
||||
**Yes, for FastSync's needs.** The hybrid model works well:
|
||||
- opencode's 17 specialized agents handle deep code analysis, fixes, tests, and reviews
|
||||
- The assistant orchestrates subagents, merges branches, and iterates on CI
|
||||
- You only review the final output
|
||||
|
||||
The key limitation: opencode is session-based, not a persistent daemon. But for the "fix all issues, one PR" workflow, this is fine — the assistant runs the full pipeline in one shot. Persistent webhook-driven automation isn't available for Gitea, but the one-shot batch approach is simpler and gives you full control over what gets merged.
|
||||
|
||||
### Recommendations for this project
|
||||
- **Do** use the batch pattern: delegate to subagents, let the assistant merge + iterate CI, review once
|
||||
- **Don't** try to run opencode in Gitea Actions — the CI container doesn't have your LLM keys or the interactive context agents need
|
||||
- **If** you want fully hands-off periodic scans, set up a local cron job or systemd timer that runs `opencode run` and posts results to Gitea via API
|
||||
|
||||
## Gitea API & tea CLI
|
||||
|
||||
### Check CI status via API
|
||||
```bash
|
||||
TOKEN="<token>"
|
||||
curl -s -H "Authorization: token $TOKEN" \
|
||||
"https://gitea.tap-tap.win/api/v1/repos/TapTap/FastSync/actions/runs?limit=5" \
|
||||
| python3 -c "
|
||||
import json,sys; d=json.load(sys.stdin)
|
||||
for r in d.get('workflow_runs',[]):
|
||||
path = r.get('path','')
|
||||
prn = path.split('@')[1].replace('refs/pull/','').replace('/head','') if '@' in path else ''
|
||||
print(f'PR #{prn}: sha={r[\"head_sha\"][:8]} {r[\"status\"]} {r.get(\"conclusion\",\"\")}')
|
||||
"
|
||||
```
|
||||
|
||||
### Post review comments
|
||||
```bash
|
||||
curl -s -X POST -H "Authorization: token $TOKEN" -H "Content-Type: application/json" \
|
||||
-d '{"body":"MARKDOWN_REVIEW_BODY"}' \
|
||||
"https://gitea.tap-tap.win/api/v1/repos/TapTap/FastSync/issues/<PR_NUMBER>/comments"
|
||||
```
|
||||
|
||||
### Use tea for PR operations
|
||||
```bash
|
||||
tea pr list --repo TapTap/FastSync
|
||||
tea pr close <number> --repo TapTap/FastSync
|
||||
```
|
||||
|
||||
## Common pitfalls
|
||||
|
||||
- **Per-thread SSL context**: `io_ssl` is stored per-thread (`static __thread SSL* io_ssl`). Each thread that performs protocol I/O must call `io_set_ssl()` to install its own SSL object before using `send_*` / `receive_*` primitives. The main thread's SSL context is not automatically inherited by worker threads.
|
||||
- **SSL WANT_READ/WANT_WRITE retry**: Always retry on `SSL_ERROR_WANT_READ` and `SSL_ERROR_WANT_WRITE` in `send_n_data`/`receive_n_data`. Removing these breaks TLS multithreaded transfers.
|
||||
- **clang-format version**: The CI image uses clang-format 18. Always format inside the CI Docker container for exact match.
|
||||
- **Merge order matters**: Merge the most comprehensive branch first, then smaller ones, to minimize conflicts when creating a combined branch.
|
||||
@@ -1,65 +0,0 @@
|
||||
# Changelog
|
||||
|
||||
All notable changes to FastSync are documented here. Versions match
|
||||
`PROTOCOL_VERSION` (printed by `fastsync --version`); the client and server must
|
||||
run the same version because the handshake is strict.
|
||||
|
||||
## [2.19.0] - 2026-09-12
|
||||
|
||||
### Security
|
||||
|
||||
- **Daemon authentication rewritten as SCRAM-SHA-256 challenge/response**
|
||||
(`STATUS_AUTH_CHALLENGE` → `STATUS_AUTH_RESPONSE` → `STATUS_AUTH_OK`/`STATUS_AUTH_FAILED`),
|
||||
replacing the old replayable static `SHA-256(password)` bearer credential.
|
||||
Each proof is bound to a fresh per-connection server nonce plus a client
|
||||
nonce, so a captured response can never be reused.
|
||||
- **Salted verifier store.** `--password-file`/`--early-input` now hold
|
||||
`user:$fastsync$1$pbkdf2-sha256$<iters>$<salt>$<stored_key>$<server_key>`
|
||||
(PBKDF2-HMAC-SHA256, default 600000 iterations, range 100000–10000000). The
|
||||
legacy `user:SHA256HEX` form is hard-rejected; there is no auto-upgrade.
|
||||
Generate stores offline with `fastsync-server --hash-credentials FILE
|
||||
[--iterations N]`.
|
||||
- **Username-enumeration hardening.** Unknown/off-list users are answered with a
|
||||
dummy verifier whose salt is a deterministic per-username value
|
||||
(`HMAC-SHA256(dummy_key, username)`), using the store-wide uniform iteration
|
||||
count and a constant-time full-length membership scan. The dummy key is
|
||||
persisted in an owner-only `<store>.dummykey` sidecar (atomic publish, exact
|
||||
mode 0600) so challenges are stable across restarts.
|
||||
- **Verified transport for auth-required modules.** A module with `auth users`
|
||||
accepts credentials only over verified TLS whose client certificate matches
|
||||
`--client-cn`, or — when `--allow-unauthenticated` is explicitly set —
|
||||
plaintext from a loopback peer. Remote plaintext is refused before any
|
||||
challenge. Clients must use `--tls` to send `--password-file` credentials to a
|
||||
non-loopback daemon; `--client-cn` is mandatory with `--tls`.
|
||||
- **Secret hygiene.** The plaintext password, derived keys, nonces/proofs and
|
||||
the dummy key are wiped from memory on every path and never logged.
|
||||
- Carried-over hardening: `-K` TOCTOU-safe directory walk
|
||||
(`openat(O_NOFOLLOW)` per component), always shell-quoted SSH remote path,
|
||||
TLS compression/renegotiation disabled, race-free (open-then-`fstat`)
|
||||
`--password-file`/`--early-input` checks, log-injection escaping, and lazy
|
||||
protocol debug escaping.
|
||||
|
||||
### Added
|
||||
|
||||
- `fastsync-server --hash-credentials FILE [--iterations N]` offline tool.
|
||||
- `<store>.dummykey` sidecar (auto-created, owner-only, 0600).
|
||||
- Integration tests for auth replay rejection, malformed frames, legacy-store
|
||||
refusal, and the loopback/TLS transport policy; fuzz targets for config
|
||||
receive and daemon-auth parsing.
|
||||
|
||||
### Changed
|
||||
|
||||
- **Protocol version 2.18.0 → 2.19.0 (breaking).** The config-frame auth block
|
||||
is now `[present][username]` (digest removed) and the auth challenge/response
|
||||
frames are interleaved between the config frame and its `STATUS_OK`. A 2.19.0
|
||||
client and a 2.18.0 server (or vice versa) fail cleanly at the handshake.
|
||||
- Daemon modules declaring `auth users` require a configured credential store at
|
||||
startup (fail closed); operators regenerate stores from plaintext with
|
||||
`--hash-credentials`.
|
||||
|
||||
### Notes
|
||||
|
||||
- First tagged release. FastSync implements rsync-compatible file
|
||||
synchronization over TCP and SSH with TLS (OpenSSL), streaming zstd
|
||||
compression, multithreaded transfers, and incremental sync. See
|
||||
[RSYNC_COMPAT.md](RSYNC_COMPAT.md) for the flag-parity matrix.
|
||||
+9
-66
@@ -1,42 +1,15 @@
|
||||
cmake_minimum_required(VERSION 3.22)
|
||||
|
||||
project(FastFileTransfer VERSION 2.19.0)
|
||||
project(FastFileTransfer)
|
||||
|
||||
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
|
||||
set(CMAKE_C_STANDARD 11)
|
||||
set(CMAKE_C_STANDARD_REQUIRED ON)
|
||||
|
||||
add_compile_options(-Wall -g -O3)
|
||||
# add_compile_options(-Wall -g -O1 -fsanitize=address)
|
||||
|
||||
# --- Sanitizer option ---
|
||||
set(SANITIZER "none" CACHE STRING "Sanitizer to enable (address, thread, undefined, none)")
|
||||
set_property(CACHE SANITIZER PROPERTY STRINGS address thread undefined none)
|
||||
|
||||
if(SANITIZER STREQUAL "address")
|
||||
add_compile_options(-fsanitize=address -fno-omit-frame-pointer -g)
|
||||
add_link_options(-fsanitize=address)
|
||||
elseif(SANITIZER STREQUAL "thread")
|
||||
add_compile_options(-fsanitize=thread -fno-omit-frame-pointer -g)
|
||||
add_link_options(-fsanitize=thread)
|
||||
elseif(SANITIZER STREQUAL "undefined")
|
||||
add_compile_options(-fsanitize=undefined -fno-omit-frame-pointer -g)
|
||||
add_link_options(-fsanitize=undefined)
|
||||
elseif(NOT SANITIZER STREQUAL "none")
|
||||
message(FATAL_ERROR "Unknown sanitizer: ${SANITIZER}. Supported values: address, thread, undefined, none")
|
||||
endif()
|
||||
|
||||
# --- Strict warnings option ---
|
||||
option(STRICT_WARNINGS "Enable strict warnings (Wextra, Wpedantic, Werror)" OFF)
|
||||
if(STRICT_WARNINGS)
|
||||
add_compile_options(-Wextra -Wpedantic -Werror)
|
||||
endif()
|
||||
|
||||
# --- Coverage option ---
|
||||
option(ENABLE_COVERAGE "Enable gcov coverage" OFF)
|
||||
if(ENABLE_COVERAGE)
|
||||
add_compile_options(--coverage -fprofile-arcs -ftest-coverage -O0 -g)
|
||||
add_link_options(--coverage)
|
||||
endif()
|
||||
# add_link_options(-fsanitize=address)
|
||||
|
||||
include(FetchContent)
|
||||
FetchContent_Declare(
|
||||
@@ -58,49 +31,19 @@ endif()
|
||||
find_package(OpenSSL REQUIRED)
|
||||
|
||||
file(GLOB SHARED_SRCS "src/shared/*.c")
|
||||
set(FILE_STORE_SRCS "${CMAKE_CURRENT_SOURCE_DIR}/src/shared/file_store.c")
|
||||
list(REMOVE_ITEM SHARED_SRCS ${FILE_STORE_SRCS})
|
||||
file(GLOB SERVER_SRCS "src/server/*.c")
|
||||
set(SERVER_RECEIVER_SRCS src/server/receiver.c)
|
||||
file(GLOB CLIENT_SRCS "src/client/*.c")
|
||||
file(GLOB TEST_SRCS "tests/*.c")
|
||||
|
||||
# --- Main executables ---
|
||||
add_executable(server ${SERVER_SRCS} ${SHARED_SRCS} ${FILE_STORE_SRCS})
|
||||
add_executable(server ${SERVER_SRCS} ${SHARED_SRCS})
|
||||
target_include_directories(server PRIVATE src/shared src/server src/client)
|
||||
target_link_libraries(server PRIVATE Threads::Threads ${ZSTD_LIBRARY} OpenSSL::SSL OpenSSL::Crypto xxhash)
|
||||
|
||||
add_executable(client ${CLIENT_SRCS} ${SHARED_SRCS} ${FILE_STORE_SRCS} ${SERVER_RECEIVER_SRCS})
|
||||
add_executable(client ${CLIENT_SRCS} ${SHARED_SRCS})
|
||||
target_include_directories(client PRIVATE src/shared src/server src/client)
|
||||
target_link_libraries(client PRIVATE Threads::Threads ${ZSTD_LIBRARY} OpenSSL::SSL OpenSSL::Crypto xxhash)
|
||||
|
||||
# --- Testing ---
|
||||
enable_testing()
|
||||
add_executable(tests ${TEST_SRCS} ${SHARED_SRCS} src/client/scanner.c)
|
||||
target_include_directories(tests PRIVATE tests src/shared src/server src/client)
|
||||
target_link_libraries(tests PRIVATE Threads::Threads ${ZSTD_LIBRARY} OpenSSL::SSL OpenSSL::Crypto xxhash)
|
||||
|
||||
# Common test libraries
|
||||
set(TEST_LIBS Threads::Threads ${ZSTD_LIBRARY} OpenSSL::SSL OpenSSL::Crypto xxhash)
|
||||
set(TEST_INCLUDES tests src/shared src/server src/client)
|
||||
|
||||
# Monolithic test binary (backward compatible)
|
||||
file(GLOB TEST_SRCS "tests/test_*.c" "tests/runner.c")
|
||||
add_executable(tests ${TEST_SRCS} ${SHARED_SRCS} ${FILE_STORE_SRCS} ${SERVER_RECEIVER_SRCS} src/client/scanner.c src/client/change_list.c src/client/client_cli.c src/client/client_validation.c src/client/usage.c src/server/server_cli.c)
|
||||
target_include_directories(tests PRIVATE ${TEST_INCLUDES})
|
||||
target_compile_definitions(tests PRIVATE FASTSYNC_TEST_BUILD)
|
||||
target_link_libraries(tests PRIVATE ${TEST_LIBS})
|
||||
add_test(NAME unit_all COMMAND tests)
|
||||
|
||||
# --- Fuzz targets (requires clang) ---
|
||||
option(ENABLE_FUZZ "Build fuzz targets (requires clang)" OFF)
|
||||
if(ENABLE_FUZZ)
|
||||
if(NOT CMAKE_C_COMPILER_ID MATCHES "Clang")
|
||||
message(FATAL_ERROR "ENABLE_FUZZ requires Clang (compiler is ${CMAKE_C_COMPILER_ID})")
|
||||
endif()
|
||||
file(GLOB FUZZ_SRCS "tests/fuzz/*.c")
|
||||
foreach(FUZZ_SRC ${FUZZ_SRCS})
|
||||
get_filename_component(FUZZ_NAME ${FUZZ_SRC} NAME_WE)
|
||||
add_executable(${FUZZ_NAME} ${FUZZ_SRC} ${SHARED_SRCS} ${FILE_STORE_SRCS} ${SERVER_RECEIVER_SRCS})
|
||||
target_include_directories(${FUZZ_NAME} PRIVATE ${TEST_INCLUDES})
|
||||
target_compile_options(${FUZZ_NAME} PRIVATE -fsanitize=fuzzer,address,undefined -fno-omit-frame-pointer)
|
||||
target_link_options(${FUZZ_NAME} PRIVATE -fsanitize=fuzzer,address,undefined)
|
||||
target_link_libraries(${FUZZ_NAME} PRIVATE ${TEST_LIBS})
|
||||
endforeach()
|
||||
endif()
|
||||
|
||||
+3
-4
@@ -1,9 +1,8 @@
|
||||
FROM ubuntu:24.04
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
gcc g++ make libc6-dev cmake libzstd-dev libssl-dev git ca-certificates curl cppcheck clang-format \
|
||||
python3 python3-pip python3-venv openssl openssh-client \
|
||||
lcov valgrind clang libclang-rt-18-dev && \
|
||||
pip3 install --break-system-packages pytest pytest-xdist && \
|
||||
gcc g++ make libc6-dev cmake libzstd-dev libssl-dev git ca-certificates curl \
|
||||
python3 python3-pip python3-venv openssl openssh-client && \
|
||||
pip3 install --break-system-packages pytest && \
|
||||
curl -fsSL https://deb.nodesource.com/setup_20.x | bash - && \
|
||||
apt-get install -y --no-install-recommends nodejs && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
@@ -1,145 +1,97 @@
|
||||
#FastSync
|
||||
# FastSync
|
||||
|
||||
FastSync is a high-performance file synchronization tool designed to become a
|
||||
drop-in replacement for common `rsync` workflows. It keeps the familiar
|
||||
source/destination model and rsync-style options while adding optional
|
||||
multithreading, streaming zstd compression, chunking, zero-copy TCP transfers,
|
||||
and native TCP/TLS transports.
|
||||
A high-performance file synchronization system with SSH and TCP transport, TLS encryption, streaming zstd compression, multithreaded transfer, incremental sync, metadata preservation, and rsync-compatible CLI flags.
|
||||
|
||||
The release version is FastSync's client/server protocol version (printed by
|
||||
`fastsync --version`); client and server must match. See
|
||||
[CHANGELOG.md](CHANGELOG.md) for the history.
|
||||
## Technical Overview
|
||||
|
||||
The compatibility target is straightforward:
|
||||
1. **Dual transport**: custom TCP client-server or SSH subprocess (rsync-style `user@host:/path`)
|
||||
2. **TLS encryption**: OpenSSL-based TLS 1.2+ for encrypted TCP connections
|
||||
3. **Chunked file transfer**: files grouped into configurable-size chunks (default ~10 MB)
|
||||
4. **Streaming zstd compression** (levels 1–22) using `ZSTD_compressStream2`
|
||||
5. **Multithreading**: producer-consumer pipeline with thread-safe queues (scanner → loader → sender)
|
||||
6. **Incremental sync**: skip files unchanged since last transfer (compares size + mtime)
|
||||
7. **Metadata preservation**: `mode`, `uid`, `gid`, `mtime` restored on disk when enabled
|
||||
8. **`sendfile()` zero-copy** on TCP (~2× faster on loopback)
|
||||
9. **SSH ControlMaster** for connection reuse across repeated invocations
|
||||
10. **Bandwidth limiting**: token-bucket throttling (`--bwlimit`)
|
||||
11. **`--delete`**: receiver removes files not present in sender manifest
|
||||
12. **`--exclude` / `--include`**: glob-pattern filename filtering
|
||||
|
||||
- Existing rsync commands should keep the same meaning.
|
||||
- FastSync-only performance options should be additive and optional.
|
||||
- A normal compatibility-mode transfer should prioritize rsync filesystem
|
||||
semantics over maximum throughput.
|
||||
## System Architecture
|
||||
|
||||
FastSync currently speaks its own protocol to `fastsync-server`. SSH mode
|
||||
starts that server remotely; it does not yet interoperate with an unmodified
|
||||
rsync client or rsync daemon. See [Compatibility Status](#compatibility-status)
|
||||
for the current boundary.
|
||||
### Client
|
||||
- Recursively scans source directories (BFS), supports exclude and include patterns
|
||||
- Groups files into chunks (configurable size)
|
||||
- Streaming zstd compression with configurable level
|
||||
- Chunk serialization (compact binary format) or per-file transfer
|
||||
- Incremental transfer: sends file metadata to server, skips unchanged files
|
||||
- Manifests all sent paths when `--delete` is active
|
||||
- Sends via TCP `sendfile()` or SSH pipe
|
||||
- Optional progress display with throughput
|
||||
- Bandwidth limiting via token-bucket algorithm
|
||||
|
||||
## Why FastSync
|
||||
### Server
|
||||
- TCP mode: listens on configurable port (default 8080); SSH mode: runs via `--stdio`
|
||||
- TLS mode: wraps TCP connections with OpenSSL
|
||||
- Receives and reassembles files
|
||||
- Decompresses (streaming zstd), deserializes, restores metadata
|
||||
- Handles incremental checks: compares size + mtime against destination files
|
||||
- Processes `STATUS_MANIFEST` for `--delete`: walks destination tree, removes extras
|
||||
- Per-connection concurrency via `fork()`
|
||||
- Thread pool for parallel processing
|
||||
|
||||
FastSync uses a producer-consumer transfer pipeline and can combine several
|
||||
optimizations for large or high-latency transfers:
|
||||
## Protocol Details
|
||||
|
||||
- Multithreaded scanning, loading, and sending.
|
||||
- Streaming zstd compression with levels 1 through 22.
|
||||
- Configurable file chunking and compact chunk serialization.
|
||||
- `sendfile()` zero-copy transfers over TCP.
|
||||
- Batched incremental checks to reduce round trips.
|
||||
- Optional block-level delta transfer for FastSync peers.
|
||||
- Bandwidth limiting, progress reporting, statistics, and backups.
|
||||
- TCP, SSH, and TLS transports.
|
||||
- Atomic temporary-file writes by default.
|
||||
### Status Codes
|
||||
| Code | Meaning |
|
||||
|------|---------|
|
||||
| `STATUS_OK` | Operation successful |
|
||||
| `STATUS_ERROR` | Error occurred |
|
||||
| `STATUS_FINISHED` | Transfer complete |
|
||||
| `STATUS_NEXT` | Ready for next file (per-file mode) |
|
||||
| `STATUS_CHUNK` | Following data is a serialized chunk |
|
||||
| `STATUS_MANIFEST` | Following data is a file manifest (for `--delete`) |
|
||||
| `STATUS_CHECK` | Incremental check: client sends file path + size + mtime, server responds with OK (skip) or NEXT (send) |
|
||||
|
||||
These optimizations are disabled or selected independently. Users can start
|
||||
with rsync-style commands and add FastSync options when they are useful.
|
||||
### Wire Format — Metadata
|
||||
|
||||
## Compatibility Status
|
||||
When `use_metadata` is enabled (`-M`), each file entry carries a 4-byte `present` flag followed by five fields (`mode`, `uid`, `gid`, `mtime_sec`, `mtime_nsec`). When disabled globally, no metadata bytes are sent — zero wire overhead.
|
||||
|
||||
FastSync is currently an rsync-compatible CLI in progress, not a complete
|
||||
replacement for every rsync feature or protocol mode.
|
||||
### Transfer Flow
|
||||
```
|
||||
Config → (STATUS_NEXT | STATUS_CHUNK | STATUS_CHECK)* → [STATUS_MANIFEST] → STATUS_FINISHED → STATUS_OK
|
||||
```
|
||||
|
||||
### Working today
|
||||
### Protocol Version
|
||||
|
||||
- Recursive directory scanning.
|
||||
- Rsync-style source and destination arguments.
|
||||
- SSH transport using `user@host:destination` paths below the remote authorized root.
|
||||
- TCP client/server transfers.
|
||||
- Dry runs, excludes, includes, size filters, backups, statistics, and
|
||||
bandwidth limiting.
|
||||
- Incremental size/mtime checks and optional xxHash64 content checks.
|
||||
- FastSync-native delta transfer for changed files.
|
||||
- Optional mode and timestamp preservation.
|
||||
- Delete manifests with server-side delete authorization.
|
||||
- Temporary-file writes with atomic rename by default.
|
||||
- Path traversal checks and destination-root confinement.
|
||||
`1.1.0` — server and client must match. Mismatch results in `STATUS_ERROR`.
|
||||
|
||||
### Not yet equivalent to rsync
|
||||
|
||||
- The FastSync wire protocol is not the rsync wire protocol.
|
||||
- SSH mode requires `fastsync-server` on the remote host.
|
||||
- Archive mode does not yet provide all of rsync's `-rlptgoD` behavior.
|
||||
- Symlink transfer is incomplete; link targets are not yet recreated in all
|
||||
modes.
|
||||
- Owner/group, ACL, xattr, and hard-link handling is incomplete or
|
||||
unavailable.
|
||||
- Device and special-file preservation is implemented with documented
|
||||
divergences: recreated device nodes require `CAP_MKNOD` on the receiver (a
|
||||
non-root receiver skips the entry), and sockets cannot be recreated (FIFOs
|
||||
are).
|
||||
- Sparse-file hole preservation (`-S`, `--sparse`) is implemented receiver-side:
|
||||
long all-zero runs are written as holes (no wire change; the full file image
|
||||
is already in memory).
|
||||
- `--partial`, `--partial-dir`, `-P`, `--append`, and `--append-verify` keep
|
||||
the write atomic (temp + rename). With `--partial`, a failed/interrupted write
|
||||
now retains the already-written temp at the destination path (best-effort) so
|
||||
a later `--append`/`--append-verify` run can resume it.
|
||||
- `--dirs` is not implemented. Its compatibility aliases `--old-dirs` and
|
||||
`--old-d` are recognized but rejected explicitly rather than silently using
|
||||
FastSync's recursive directory behavior.
|
||||
- Short-option names are now rsync-parity (Phase 7 Wave A): FastSync's former
|
||||
collisions were renamed (`-j`/`--threads`, `--preserve`, `--sendfile`,
|
||||
`--chunk-serialization`, `--timeout`, `--ssh-port`), so `-m`, `-M`, `-f`,
|
||||
`-s`, `-T`, `-p`, `-c`, `-a`, and `-z` follow rsync. See `RSYNC_COMPAT.md`.
|
||||
|
||||
The detailed flag matrix is maintained in
|
||||
[`RSYNC_COMPAT.md`](RSYNC_COMPAT.md). It distinguishes implemented,
|
||||
partial, alternate, and planned behavior.
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Build
|
||||
## Command-Line Arguments
|
||||
|
||||
### Client
|
||||
|
||||
| Argument | Description |
|
||||
|----------|-------------|
|
||||
| Positional | `<source> <dest>` — automatic SSH detection if dest contains `:` |
|
||||
| `-c, --checksum` | Verify content by checksum instead of size+mtime |
|
||||
| `-z, --compress [level]` | Enable streaming zstd compression (level 1–22, default 5) |
|
||||
| `-a, --archive` | rsync archive mode (`-rlptgoD`): links, metadata, devices and specials (not compression/multithreading) |
|
||||
| `-j, --threads` | Multithreading mode |
|
||||
| `-m` | rsync `--prune-empty-dirs` (short form now rsync-parity) |
|
||||
| `--chunk-serialization` | Chunk serialization (batch all files per chunk; long form only) |
|
||||
| `-s` | rsync `--secluded-args` compatibility no-op (remote SSH argv is already injection-safe) |
|
||||
| `--sendfile` | Sendfile zero-copy. Incompatible with compression / chunk serialization. TCP only. Long form only. |
|
||||
| `--preserve` | Preserve supported file metadata (mode and mtime; ownership and atime are unsupported) |
|
||||
| `-c [level]` | Compression with optional level (1–22, default 5) |
|
||||
| `-z [level]` | Alias for `-c` |
|
||||
| `-a, --archive` | Archive mode: enables `-c -m -M` (no `-s`) |
|
||||
| `-m` | Multithreading mode |
|
||||
| `-s` | Chunk serialization (batch all files per chunk) |
|
||||
| `-f, --sendfile` | Sendfile zero-copy. Incompatible with `-c` / `-s`. TCP only. |
|
||||
| `-M, --preserve` | Preserve file metadata (mode, uid, gid, mtime) |
|
||||
| `-n, --dry-run` | Scan and print what would be transferred |
|
||||
| `-p, --perms` | Preserve permission bits (part of the metadata bundle) |
|
||||
| `--ssh-port <port>` | SSH port (default: 22) |
|
||||
| `-p <port>` | SSH port (default: 22) |
|
||||
| `-v, --verbose` | Enable debug logging |
|
||||
| `-q, --quiet` | Suppress non-error output |
|
||||
| `--progress` | Show real-time transfer speed |
|
||||
| `-P` | Enables partial-transfer mode + progress output; interrupted writes retain the already-written temp for resumption |
|
||||
| `--delete` | Delete files on receiver not present in source (default timing: delete-after, i.e. only after the whole transfer succeeded) |
|
||||
| `--delete-before` | Delete extras before the transfer starts (implies `--delete`) |
|
||||
| `--delete-during`, `--del` | Delete extras once the keep-set is known, before data is applied (implies `--delete`) |
|
||||
| `--delete-delay` | Delete extras only after a successful transfer (implies `--delete`) |
|
||||
| `--delete-after` | Explicit delete-after timing (implies `--delete`) |
|
||||
| `--delete` | Delete files on receiver not present in source |
|
||||
| `--exclude <pattern>` | Exclude files matching glob pattern (repeatable) |
|
||||
| `--exclude-from <file>` | Read exclude patterns from a file (one per line) |
|
||||
| `--include <pattern>` | Only transfer files matching glob pattern (repeatable, whitelist) |
|
||||
| `--max-size <n>` | Skip files larger than n bytes |
|
||||
| `--min-size <n>` | Skip files smaller than n bytes |
|
||||
| `--max-alloc <SIZE>` | Maximum single allocation (binary units: B, K, M, G, T, P, E; default 1G) |
|
||||
| `--incremental` | Skip files unchanged since last transfer (size + mtime). Auto-enables `--preserve`. Incompatible with `--chunk-serialization`. |
|
||||
| `--existing` | Skip files not already present at the destination; update existing files normally. |
|
||||
| `--incremental` | Skip files unchanged since last transfer (size + mtime). Auto-enables `--preserve`. Incompatible with `-s`. |
|
||||
| `--bwlimit <KB/s>` | Bandwidth limit in kilobytes per second |
|
||||
| `--chunk-size <n>` | Chunk size in bytes (default: 10485760) |
|
||||
| `--timeout <sec>` | I/O timeout in seconds (default: 30) |
|
||||
| `--contimeout <sec>` | Connection timeout in seconds (default: 10) |
|
||||
| `--backup` | Backup existing destination files before overwriting |
|
||||
| `--backup-dir <dir>` | Target directory for backups (requires `--backup`) |
|
||||
| `--stats` | Print transfer statistics at end (bytes, files, timing) |
|
||||
| `-h, --human-readable` | Format transfer byte sizes with binary units |
|
||||
| `--max-depth <n>` | Maximum directory depth to recurse (0 = unlimited, default: 0) |
|
||||
| `--log-file <path>` | Write log messages to file instead of stderr |
|
||||
| `--source-dir <path>` | Source directory (overrides `FASTSYNC_SOURCE_DIR`) |
|
||||
| `--dest-dir <path>` | Server destination directory (overrides `FASTSYNC_DEST_DIR`) |
|
||||
| `--save-to-disk` | Write received files to disk |
|
||||
@@ -149,7 +101,6 @@ partial, alternate, and planned behavior.
|
||||
| `--cert <path>` | TLS certificate file (PEM) |
|
||||
| `--key <path>` | TLS private key file (PEM) |
|
||||
| `--ca <path>` | TLS CA certificate file for verification (PEM) |
|
||||
| `--client-cn <name>` | TLS client certificate common name; mandatory with `--tls` (a TLS connection always verifies the client CN) |
|
||||
|
||||
### Server
|
||||
|
||||
@@ -161,9 +112,6 @@ partial, alternate, and planned behavior.
|
||||
| `--cert <path>` | TLS certificate file (PEM) |
|
||||
| `--key <path>` | TLS private key file (PEM) |
|
||||
| `--ca <path>` | TLS CA certificate file for verification (PEM) |
|
||||
| `--destination-root <path>` | Authorized destination root (default: `.`) |
|
||||
| `--allow-delete` | Permit manifest deletion |
|
||||
| `--allow-unauthenticated` | Permit plaintext TCP clients. For an `auth users` module this opts in **loopback plaintext only**; remote auth still requires verified TLS, so the flag never permits remote plaintext auth. |
|
||||
| `-v, --verbose` | Enable debug logging |
|
||||
| `--help` | Show help |
|
||||
|
||||
@@ -174,77 +122,28 @@ partial, alternate, and planned behavior.
|
||||
| `FASTSYNC_SOURCE_DIR` | — | Source directory fallback |
|
||||
| `FASTSYNC_DEST_DIR` | — | Destination directory fallback |
|
||||
| `FASTSYNC_SAVE_TO_DISK` | `false` | Disk persistence fallback |
|
||||
| `FASTSYNC_SSH_PORT` | `22` | Default SSH port |
|
||||
| `FASTSYNC_SERVER_HOST` | `127.0.0.1` | Default server host |
|
||||
| `FASTSYNC_SERVER_PORT` | `8080` | Default server port |
|
||||
| `FASTSYNC_TLS_CERT` | — | Default TLS certificate path |
|
||||
| `FASTSYNC_TLS_KEY` | — | Default TLS private key path |
|
||||
| `FASTSYNC_TLS_CA` | — | Default TLS CA certificate path |
|
||||
|
||||
## Implementation Details
|
||||
|
||||
### Data Structures
|
||||
1. **Chunk** — collection of files (~10 MB total by default)
|
||||
2. **File** — path, content (`Data`), optional `FileMetadata` pointer
|
||||
3. **FileMetadata** — `mode`, `uid`, `gid`, `mtime_sec`, `mtime_nsec`;
|
||||
uid / gid are advisory wire fields and are never applied by the receiver;
|
||||
atime is unsupported
|
||||
4. **Config** — runtime parameters (transported over wire, TLS settings excluded). Includes `timeout`, `contimeout`, `quiet`, `backup`, `backup_dir`, `stats`, `max_depth`, `log_file`, `queue_size`.
|
||||
3. **FileMetadata** — `mode`, `uid`, `gid`, `mtime_sec`, `mtime_nsec`
|
||||
4. **Config** — runtime parameters (transported over wire, TLS settings excluded)
|
||||
5. **Queue** — thread-safe bounded queue with condition variables
|
||||
6. **DirectoryScanner** — recursive BFS traversal with exclude and include pattern support, max-depth enforcement
|
||||
6. **DirectoryScanner** — recursive BFS traversal with exclude and include pattern support
|
||||
|
||||
### Key Algorithms
|
||||
1. **File scanning** — BFS directory traversal;
|
||||
entries matched against exclude and include patterns,
|
||||
max - depth enforced 2. * *Chunking ** — files accumulated until `chunk_size` threshold,
|
||||
then flushed 3. *
|
||||
*Compression ** — streaming zstd
|
||||
via `ZSTD_compressStream2` / `ZSTD_decompressStream` 4. *
|
||||
*Network protocol ** — status -
|
||||
code - driven exchange with metadata packing,
|
||||
keep - alive,
|
||||
and abort support 5. * *Incremental check ** — client sends `STATUS_CHECK` + path + size +
|
||||
mtime and,
|
||||
with `--checksum`, XXH64 content checksum; server compares against destination. Can be batched via `STATUS_CHECK_BATCH` for reduced round-trips.
|
||||
1. **File scanning** — BFS directory traversal; entries matched against exclude and include patterns
|
||||
2. **Chunking** — files accumulated until `chunk_size` threshold, then flushed
|
||||
3. **Compression** — streaming zstd via `ZSTD_compressStream2` / `ZSTD_decompressStream`
|
||||
4. **Network protocol** — status-code-driven exchange with metadata packing
|
||||
5. **Incremental check** — client sends `STATUS_CHECK` + path + size + mtime; server compares against destination
|
||||
6. **Bandwidth limiting** — token-bucket algorithm with `nanosleep` throttling on 64 KB write chunks
|
||||
7. **Metadata restoration** — `chmod()`, `chown()`, `utimensat()` on the receiving side
|
||||
8. **`--delete`** — sender tracks all sent paths;
|
||||
receiver walks destination tree and removes unlisted files / directories 9. *
|
||||
*SSH transport *
|
||||
* — `socketpair()` + `fork()` + `execvp("ssh",
|
||||
...)` with `ControlMaster` and port support
|
||||
10. *
|
||||
*TLS transport ** — OpenSSL `SSL_CTX` with TLS
|
||||
1.2 minimum,
|
||||
mutual CA verification,
|
||||
transparent `SSL_read`/`SSL_write` via `io_set_ssl()` 11. *
|
||||
*Path traversal protection ** — `has_path_traversal()` rejects any file path
|
||||
containing `..` components,
|
||||
preventing directory escape attacks 12. *
|
||||
*Connection limiting ** — server tracks active connections and rejects
|
||||
new ones beyond `max_connections` (default 100)13. *
|
||||
*Keep
|
||||
- alive ** — idle connections receive periodic `STATUS_KEEPALIVE` to detect half
|
||||
- open TCP connections 14. * *Abort handling ** — `SIGINT` sets an abort flag; the next protocol operation sends `STATUS_ABORT` for clean server cleanup
|
||||
15. **Atomic writes** — files are written to a `.tmp` suffix then atomically renamed via `rename()`, preventing partial files
|
||||
16. **Backup** — before overwriting, existing files are moved to `--backup-dir` (or same directory with `~` suffix) preserving the original
|
||||
|
||||
## Security Features
|
||||
|
||||
### Path Traversal Protection
|
||||
All received file paths are validated by `has_path_traversal()` before any disk operation. Any path containing `..` components is rejected with `STATUS_ERROR`, preventing directory escape attacks.
|
||||
|
||||
### TLS Certificate Verification
|
||||
TLS requires `--ca` and performs mutual TLS verification (`SSL_VERIFY_PEER` with depth 4). Connections without certificate verification are rejected.
|
||||
|
||||
### Connection Limits
|
||||
The server enforces a maximum of 100 concurrent connections (configurable via `max_connections` in `Server`). When the limit is reached, new connections are immediately rejected and closed.
|
||||
|
||||
### Abort Handling
|
||||
If the client receives `SIGINT` (Ctrl+C) during a transfer, it sends `STATUS_ABORT` to the server. The server then cleans up temporary files and exits the child process, preventing incomplete files from remaining on disk.
|
||||
|
||||
### Atomic Writes
|
||||
Received files are written to a temporary path (suffixed with `.tmp`) and then atomically renamed to the final filename via `rename()`. This prevents partial or corrupted files from appearing at the destination if the transfer is interrupted.
|
||||
8. **`--delete`** — sender tracks all sent paths; receiver walks destination tree and removes unlisted files/directories
|
||||
9. **SSH transport** — `socketpair()` + `fork()` + `execvp("ssh", ...)` with `ControlMaster` and port support
|
||||
10. **TLS transport** — OpenSSL `SSL_CTX` with TLS 1.2 minimum, optional CA verification, transparent `SSL_read`/`SSL_write` via `io_set_ssl()`
|
||||
|
||||
## Build Requirements
|
||||
|
||||
@@ -270,368 +169,110 @@ nix-shell # provides zstd, openssl, cmake, gcc
|
||||
## Building
|
||||
|
||||
```bash
|
||||
cmake -B build -S .
|
||||
cmake --build build -j$(nproc)
|
||||
cmake -B build -S . && cmake --build build -j$(nproc)
|
||||
```
|
||||
|
||||
With Nix:
|
||||
## Running
|
||||
|
||||
### Server (TCP mode)
|
||||
```bash
|
||||
nix-shell
|
||||
cmake -B build -S .
|
||||
cmake --build build -j$(nproc)
|
||||
./build/server
|
||||
```
|
||||
|
||||
### SSH transfer
|
||||
|
||||
The remote host must have `fastsync-server` available in `PATH`, or use
|
||||
`--fastsync-server-path`. SSH starts `fastsync-server --stdio` in its remote
|
||||
working directory, so use a destination below that directory unless the
|
||||
remote server is otherwise configured with a matching authorized root.
|
||||
|
||||
### Server with TLS
|
||||
```bash
|
||||
ssh user@host 'mkdir -p destination'
|
||||
./build/client /path/to/source user@host:destination
|
||||
./build/server --tls --cert server.pem --key server-key.pem
|
||||
```
|
||||
|
||||
### TCP transfer
|
||||
|
||||
Start the FastSync server:
|
||||
### Server via SSH
|
||||
Place the `fastsync-server` binary in the remote `$PATH`. The client runs `ssh user@host fastsync-server --stdio` automatically when an SSH-style destination is given.
|
||||
|
||||
### Client — SSH (rsync-style)
|
||||
```bash
|
||||
./build/server --destination-root /path/to -p 8080
|
||||
./build/client /path/to/send user@host:/path/to/receive
|
||||
```
|
||||
|
||||
Then run the client:
|
||||
|
||||
### Client — TCP
|
||||
```bash
|
||||
./build/client --server-host 127.0.0.1 --server-port 8080 \
|
||||
--source-dir /path/to/source --dest-dir /path/to/destination \
|
||||
--save-to-disk
|
||||
./build/client --source-dir /path/to/send --dest-dir /path/to/receive --save-to-disk
|
||||
```
|
||||
|
||||
Plain TCP requires the explicit `--allow-unauthenticated` server option. Use TLS for
|
||||
authenticated network connections.
|
||||
|
||||
### TLS transfer
|
||||
### Client — TCP with TLS
|
||||
```bash
|
||||
./build/server --destination-root /path/to --tls --cert server.pem --key server-key.pem -p 8443
|
||||
./build/client --tls --cert client.pem --key client-key.pem --ca ca.pem \
|
||||
--server-host example.com --server-port 8443 \
|
||||
--source-dir /path/to/source --dest-dir /path/to/destination \
|
||||
--save-to-disk
|
||||
--source-dir /path/to/send --dest-dir /path/to/receive --save-to-disk
|
||||
```
|
||||
|
||||
## Common Workflows
|
||||
|
||||
These examples show the intended rsync-style workflow. Options marked as
|
||||
FastSync-native are optional performance or transport extensions.
|
||||
|
||||
### Common Options
|
||||
```bash
|
||||
#Basic synchronization
|
||||
./build/client /source/ /destination/
|
||||
# Archive mode (compression + multithreading + metadata)
|
||||
./build/client -a /path/to/send user@host:/path
|
||||
|
||||
#Archive - style synchronization(current FastSync archive behavior)
|
||||
./build/client -a /source/ user@host:destination/
|
||||
# Dry run
|
||||
./build/client -n /path/to/send /path/to/receive
|
||||
|
||||
#Preview a transfer without changing the destination
|
||||
./build/client -n /source/ /destination/
|
||||
# With progress and custom chunk size
|
||||
./build/client --progress --chunk-size 2097152 /src user@host:/dst
|
||||
|
||||
#Exclude temporary and object files
|
||||
./build/client --exclude '*.tmp' --exclude '*.o' \
|
||||
/source/ user@host:destination/
|
||||
# Exclude temporary files + delete extras on receiver
|
||||
./build/client --exclude "*.tmp" --exclude "*.o" --delete /src user@host:/dst
|
||||
|
||||
#Remove destination entries not present in the source
|
||||
./build/client --delete /source/ user@host:destination/
|
||||
# Incremental sync (skip unchanged files)
|
||||
./build/client --incremental /src user@host:/dst
|
||||
|
||||
#Skip unchanged files using size and modification time
|
||||
./build/client --incremental /source/ user@host:destination/
|
||||
# Bandwidth limit to 1 MB/s
|
||||
./build/client --bwlimit 1024 /src user@host:/dst
|
||||
|
||||
#Verify content when size and time are not sufficient
|
||||
./build/client --incremental --checksum /source/ user@host:destination/
|
||||
|
||||
#Preserve supported mode and timestamp metadata
|
||||
./build/client -M /source/ user@host:destination/
|
||||
|
||||
#Keep backups of overwritten destination files
|
||||
./build/client --backup --backup-dir backups \
|
||||
/source/ user@host:destination/
|
||||
# All features
|
||||
./build/client -a --progress --chunk-size 5242880 --exclude "*.log" --delete /src /dst
|
||||
```
|
||||
|
||||
## FastSync Extensions
|
||||
|
||||
FastSync-native options are intended to add performance or operational
|
||||
features without changing the meaning of ordinary compatibility options.
|
||||
|
||||
| Option | Purpose |
|
||||
|---|---|
|
||||
| `-j`, `--threads` | Enable the multithreaded scanner/loader/sender pipeline. |
|
||||
| `-z [level]`, `--compress [level]` | Enable streaming zstd compression, levels 1-22. |
|
||||
| `--compress-level <n>` | Set the zstd compression level. |
|
||||
| `--zc <alg>` | Alias for `--compress-choice`. FastSync supports `zstd` and `none`. |
|
||||
| `--zl <n>` | Alias for `--compress-level`. |
|
||||
| `--skip-compress <list>` | Skip compression for comma-separated suffixes; incompatible with `--chunk-serialization`. |
|
||||
| `--compress-threads <n>` | Use `n` zstd compression workers. Requires compression and a zstd build with threaded support; the setting affects sender CPU work only. |
|
||||
| `--chunk-size <bytes>` | Set the transfer chunk size. |
|
||||
| `--chunk-serialization` | Enable FastSync chunk serialization (long form only; `-s` is rsync's `--secluded-args`). |
|
||||
| `--sendfile` | Use TCP `sendfile()` zero-copy transfer. Incompatible with compression and chunk serialization. Long form only. |
|
||||
| `--delta` | Use FastSync-native block delta transfer. Requires `--incremental`. |
|
||||
| `--delta-block <bytes>` | Set the FastSync delta block size (`--block-size` is an alias). |
|
||||
| `--delta-max <bytes>` | Limit files eligible for FastSync delta transfer. |
|
||||
| `--server-host <host>` | Select the TCP server host. |
|
||||
| `--server-port <port>` | Select the TCP server port. |
|
||||
| `--tls` | Enable TLS for TCP transport. |
|
||||
| `--bwlimit <KB/s>` | Apply token-bucket bandwidth limiting. |
|
||||
| `--progress` | Show transfer progress and throughput. |
|
||||
| `--stats` | Print transfer statistics. |
|
||||
| `--timeout <seconds>` | Set I/O timeout. |
|
||||
| `--contimeout <seconds>` | Set connection timeout. |
|
||||
|
||||
Short-option conflicts with rsync have been resolved for the CLI namespace
|
||||
(Phase 7): `-c` is now rsync's `--checksum`, `-m` is `--prune-empty-dirs`, `-M`
|
||||
is `--remote-option`, `-f` is `--filter`, `-s` is `--secluded-args`, `-p` is
|
||||
`--perms`, and `-T` is `--temp-dir`. FastSync's own flags were renamed to
|
||||
long-form-only or new shorts: multithreading is `-j`/`--threads`, metadata
|
||||
is `--preserve`, sendfile is `--sendfile`, chunk serialization is
|
||||
`--chunk-serialization`, timeout is `--timeout`, and SSH port is `--ssh-port`.
|
||||
`-a`/`--archive` is now real rsync archive (`-rlptgoD`).
|
||||
|
||||
`--secluded-args` (and its short form `-s`) is accepted as a compatibility
|
||||
no-op. It does not change FastSync's transport or protocol behavior, because
|
||||
remote SSH argv is already built injection-safe.
|
||||
|
||||
## Client Options
|
||||
|
||||
### Selection and transfer
|
||||
|
||||
| Option | Description |
|
||||
|---|---|
|
||||
| `-a`, `--archive` | rsync archive mode (`-rlptgoD`): links, metadata, devices and specials. |
|
||||
| `-n`, `--dry-run` | Scan and report without writing files. |
|
||||
| `--delete` | Request removal of destination entries absent from the source. The server must allow deletion. Default timing is delete-after: extras are removed only after the whole transfer succeeded. |
|
||||
| `--delete-before` | Delete extras before the transfer starts (implies `--delete`). |
|
||||
| `--delete-during`, `--del` | Delete extras once the keep-set manifest is known, before data is applied (implies `--delete`; early mode, same engine behaviour as `--delete-before`). |
|
||||
| `--delete-delay` | Delete extras only after a successful transfer (implies `--delete`; commit mode, same behaviour as `--delete-after`). |
|
||||
| `--delete-after` | Explicit delete-after timing: delete only after the transfer succeeded (implies `--delete`). |
|
||||
| `--exclude <pattern>` | Exclude matching paths. Repeatable. |
|
||||
| `--include <pattern>` | Include matching paths. Repeatable. |
|
||||
| `--exclude-from <file>` | Read exclude patterns from a file. |
|
||||
| `--include-from <file>` | Read include patterns from a file. |
|
||||
| `--max-size <bytes>` | Skip files larger than the limit. |
|
||||
| `--min-size <bytes>` | Skip files smaller than the limit. |
|
||||
| `--max-depth <n>` | Limit recursive scanning depth;
|
||||
zero means unlimited.| | `--incremental` | Skip files matching destination size and mtime.|
|
||||
| `--checksum` | Include xxHash64 content checks in incremental comparisons.| | `--backup` |
|
||||
Back up overwritten files.| | `--backup - dir<dir>` | Store backups under a separate directory.|
|
||||
| `--suffix<suffix>` | Set the backup filename suffix.| | `--partial` |
|
||||
Select partial - transfer handling. On failed/interrupted writes the
|
||||
already-written temp file is retained (best-effort) for resumption.|
|
||||
With `--partial --partial-dir <dir>`, completed files are written under the
|
||||
partial directory and installed atomically. | | `--partial - dir<dir>` |
|
||||
Set a relative partial - transfer directory below the server destination root.
|
||||
Use with `--partial`. |
|
||||
| `--inplace` | Write directly to the destination instead of using a temporary file. |
|
||||
|
||||
### Metadata and links
|
||||
|
||||
| Option | Description |
|
||||
|---|---|
|
||||
| `--preserve` | Preserve supported file metadata, currently mode and modification time (long form only). |
|
||||
| `-l`, `--links` | Request symlink preservation;
|
||||
link-target transfer remains incomplete. |
|
||||
| `--copy-links` | Copy symlink referents. |
|
||||
| `--safe-links` | Skip symlinks that point outside the transfer tree. |
|
||||
| `--copy-unsafe-links` | Copy unsafe symlink referents. |
|
||||
| `-S`, `--sparse` | Sparse-file handling: receiver preserves holes (zero runs are written as holes; no wire change). |
|
||||
|
||||
### Output and logging
|
||||
|
||||
| Option | Description |
|
||||
|---|---|
|
||||
| `-v`, `--verbose` | Enable debug logging. |
|
||||
| `--progress` | Show live transfer progress. |
|
||||
| `--stats` | Print transfer statistics. |
|
||||
| `--log-file <path>` | Write log output to a file. |
|
||||
| `-V`, `--version` | Print the FastSync protocol version. |
|
||||
| `--help` | Print command usage. |
|
||||
|
||||
### Paths and transport
|
||||
|
||||
| Option | Description |
|
||||
|---|---|
|
||||
| `--ssh-port <port>` | SSH port for the SSH transport (default: 22). Note the short `-p` is now rsync's `--perms`. |
|
||||
| `--fastsync-server-path <path>` | Remote FastSync server path for SSH mode. |
|
||||
| `--source-dir <path>` | Set the source directory explicitly. |
|
||||
| `--dest-dir <path>` | Set the destination directory explicitly. |
|
||||
| `--save-to-disk` | Enable server-side disk persistence. |
|
||||
| `--server-host <host>` | TCP server address. |
|
||||
| `--server-port <port>` | TCP server port. |
|
||||
| `--tls` | Enable TLS. Requires `--cert` and `--key`. |
|
||||
| `--cert <path>` | TLS certificate file. |
|
||||
| `--key <path>` | TLS private key file. |
|
||||
| `--ca <path>` | CA file for peer verification. |
|
||||
|
||||
## Server Options
|
||||
|
||||
| Option | Description |
|
||||
|---|---|
|
||||
| `--stdio` | Serve one SSH connection over standard input/output. |
|
||||
| `-p <port>` | TCP listen port. |
|
||||
| `--tls` | Enable TLS. |
|
||||
| `--cert <path>` | TLS certificate file. |
|
||||
| `--key <path>` | TLS private key file. |
|
||||
| `--ca <path>` | CA file for peer verification. |
|
||||
| `--destination-root <path>` | Confine received files to this server-side root;
|
||||
defaults to the current directory. |
|
||||
| `--allow-delete` | Permit client delete manifests. Deletion is refused by default. |
|
||||
| `-v`, `--verbose` | Enable debug logging. |
|
||||
| `--help` | Print server usage. |
|
||||
|
||||
## Architecture
|
||||
|
||||
### Client
|
||||
|
||||
- Recursively scans the source tree with include, exclude, size, and depth
|
||||
filters.
|
||||
- Sends individual files or serialized chunks.
|
||||
- Performs incremental checks and optional content checksums.
|
||||
- Uses a multithreaded producer-consumer pipeline when requested.
|
||||
- Sends over TCP, TLS-wrapped TCP, or an SSH subprocess.
|
||||
- Supports progress, statistics, backups, timeouts, and bandwidth limiting.
|
||||
|
||||
### Server
|
||||
|
||||
- Runs as a TCP listener or one-shot SSH `--stdio` server.
|
||||
- Receives and reassembles files and decompresses streaming zstd data.
|
||||
- Applies supported metadata and writes files through a confined destination
|
||||
root.
|
||||
- Uses temporary files and atomic rename by default.
|
||||
- Handles delete manifests only when explicitly authorized.
|
||||
- Enforces connection, message-size, and path-safety limits.
|
||||
|
||||
## Protocol and Security
|
||||
|
||||
FastSync protocol version `2.19.0` is shared by the client and server. The
|
||||
current protocol is sender-driven and includes configuration negotiation,
|
||||
including the maximum allocation limit, incremental checks, checksums,
|
||||
manifests, keep-alives, abort handling, per-file remove-source results, and
|
||||
FastSync-native delta messages.
|
||||
Client and server versions must currently match exactly.
|
||||
|
||||
Daemon modules that declare `auth users` authenticate with a SCRAM-SHA-256-style
|
||||
challenge/response against a salted PBKDF2 verifier store: no password and no
|
||||
replayable bearer credential crosses the wire or is stored on the daemon. All
|
||||
store entries share one iteration count, and an unknown user is answered with a
|
||||
deterministic per-username dummy challenge, so probing the daemon cannot
|
||||
enumerate users. Store lines are generated with
|
||||
`fastsync-server --hash-credentials <plaintext-file>` (see `RSYNC_COMPAT.md`);
|
||||
redirect that output to an owner-only (mode 0600) file, and note that legacy
|
||||
`user:SHA256HEX` stores are rejected. FastSync also maintains an owner-only
|
||||
(mode 0600) `<store>.dummykey` sidecar next to the store: it holds the store-wide
|
||||
dummy key, is auto-created on first load, and must be preserved across daemon
|
||||
restarts so the dummy challenge for an unknown user stays stable (the key is
|
||||
never regenerated while the sidecar exists). The sidecar is secret material and
|
||||
must be protected like the credential store: keep it owner-only (mode 0600) and
|
||||
include it with the store in backups and credential rotation. If the sidecar
|
||||
cannot be created (a process-substitution/FIFO store path such as `/dev/fd/N`, a
|
||||
read-only filesystem, a missing directory, or a create, write, fsync, link, or
|
||||
fchmod failure), the daemon logs a warning and uses a transient key, so the
|
||||
cross-restart guarantee does not hold for those deployments. One residual is
|
||||
accepted: the store
|
||||
iteration count is observable pre-auth by design, since the miss path must match
|
||||
a hit.
|
||||
|
||||
An `auth users` module accepts credentials only when one of two conditions
|
||||
holds: (a) the connection is an encrypted, verified TLS connection whose client
|
||||
certificate matches the server's `--client-cn`, or (b) the connection is
|
||||
plaintext from a loopback peer **and** the operator explicitly passed
|
||||
`--allow-unauthenticated`. A remote plaintext peer is refused before any
|
||||
challenge is sent, and `--allow-unauthenticated` never permits remote plaintext
|
||||
auth: remote peers still require verified TLS regardless of the flag. Clients
|
||||
sending daemon credentials with `--password-file` to a non-loopback daemon must
|
||||
therefore use `--tls`; the client rejects a non-local plaintext credential
|
||||
destination before any network I/O. Daemon modules are a `--daemon`-only
|
||||
feature: the SSH `--stdio` path never loads a daemon config and is not an auth
|
||||
transport for them.
|
||||
|
||||
Because the loopback allowance trusts whichever peer the kernel reports as
|
||||
`127.0.0.1`, it assumes nothing relays remote connections to the daemon. A local
|
||||
TCP forwarder or a TLS-terminating proxy in front of an auth-module listener
|
||||
makes remote clients appear as loopback and bypasses the mutual-TLS identity
|
||||
check, so do not front an auth-module listener with such a relay. `--tls` always
|
||||
mandates `--client-cn`, so a TLS connection to an auth-required module always
|
||||
has its client CN verified (`--client-cn` matches the certificate's CN only, not
|
||||
a subjectAltName, which is acceptable for a private CA).
|
||||
|
||||
TLS provides encrypted TCP transport. Supplying `--ca` enables certificate
|
||||
verification; without it, traffic is encrypted but peer identity is not
|
||||
verified. Use certificate verification for deployments where authentication
|
||||
matters. The default TCP transport is not encrypted.
|
||||
|
||||
The receiver protects its destination root with path validation, `openat()`
|
||||
directory traversal, `O_NOFOLLOW`, temporary files, and atomic renames. Delete
|
||||
operations require the server's explicit `--allow-delete` policy.
|
||||
|
||||
## Compatibility Roadmap
|
||||
|
||||
The project will reach the drop-in replacement goal in stages:
|
||||
|
||||
1. Correct rsync option meanings, including short options, combined options,
|
||||
and `--option=value` syntax.
|
||||
2. Add differential tests that compare FastSync and rsync contents, metadata,
|
||||
links, deletes, filters, dry runs, and exit codes.
|
||||
3. Make `-a` implement the expected recursive, links, permissions, times,
|
||||
owner/group, and supported special-file behavior.
|
||||
4. Complete symlink, sparse-file, metadata, delete-policy, and resumable-write
|
||||
semantics.
|
||||
5. Add rsync remote-shell and daemon protocol interoperability.
|
||||
6. Keep FastSync performance options as negotiated, optional extensions.
|
||||
|
||||
The exhaustive implementation matrix and compatibility notes are in
|
||||
[`RSYNC_COMPAT.md`](RSYNC_COMPAT.md).
|
||||
|
||||
## Testing
|
||||
|
||||
Run the unit test binary:
|
||||
|
||||
```bash
|
||||
# Unit tests (7 suites)
|
||||
./build/tests
|
||||
|
||||
# Integration + benchmark suite
|
||||
python3 test.py
|
||||
```
|
||||
|
||||
Run the Python integration suite:
|
||||
The benchmark prints throughput metrics, best configuration, and speedup vs rsync.
|
||||
|
||||
```bash
|
||||
python3 -m pytest tests/
|
||||
```
|
||||
## Performance Considerations
|
||||
|
||||
For stricter local validation:
|
||||
1. Chunk size (~10 MB default) balances memory and transfer efficiency
|
||||
2. Compression level trades CPU for bandwidth
|
||||
3. `sendfile()` bypasses userspace — ~2× faster on localhost for large files
|
||||
4. Multithreading scales with core count
|
||||
5. Metadata transfer adds negligible overhead (~24 bytes per file when enabled)
|
||||
6. SSH socketpair buffer set to 1 MB for improved pipe throughput
|
||||
7. SSH ControlMaster reuses connections across repeated invocations
|
||||
8. Incremental sync eliminates redundant transfers entirely
|
||||
9. Bandwidth limiting uses token-bucket with nanosleep for accurate throttling
|
||||
|
||||
```bash
|
||||
cmake -B build-strict -S . -DSTRICT_WARNINGS=ON
|
||||
cmake --build build-strict -j$(nproc)
|
||||
cmake -B build-asan -S . -DSANITIZER=address
|
||||
cmake --build build-asan -j$(nproc)
|
||||
```
|
||||
## Benchmark Results
|
||||
|
||||
The benchmark tool compares FastSync configurations with rsync under
|
||||
controlled local and network conditions:
|
||||
25 MB of mixed file sizes over `localhost` with disk I/O throttled (reads ≤ 15 MB/s, writes ≤ 10 MB/s) and network emulation via `tc netem`. Each test was run 3×; the median is reported below.
|
||||
|
||||
```bash
|
||||
python3 benchmark/bench.py --help
|
||||
```
|
||||
### LAN (1000 Mbit, 20 ms ±1 ms, 0.1% loss)
|
||||
|
||||
Benchmark results measure transfer performance only. They do not establish
|
||||
rsync protocol or filesystem-semantic compatibility.
|
||||
| Configuration | Time | vs rsync (archive) | vs rsync (compress) |
|
||||
|---|---|---|---|
|
||||
| **Best: `-m -c`** | **0.20 s** | **11.2× faster** | **3.6× faster** |
|
||||
| Compression (`-c`) | 0.31 s | 7.3× faster | 2.3× faster |
|
||||
| Standard | 1.27 s | 1.8× faster | — |
|
||||
| rsync (archive) | 2.27 s | — | — |
|
||||
| rsync (archive + compress) | 0.72 s | — | — |
|
||||
|
||||
## Performance Guidance
|
||||
### WAN (100 Mbit, 50 ms ±10 ms, 1% loss)
|
||||
|
||||
- Use `-m` for workloads with many files or enough CPU parallelism.
|
||||
- Use `-c` or `-z` when network bandwidth is more constrained than CPU.
|
||||
- Tune `--chunk-size` for file sizes, memory limits, and network latency.
|
||||
- Use `-f` for large uncompressed TCP transfers where zero-copy I/O helps.
|
||||
- Use `--incremental` to avoid retransmitting unchanged files.
|
||||
- Use `--delta` for changed files when both endpoints are FastSync peers.
|
||||
- Use `--bwlimit` when sharing a link with other traffic.
|
||||
| Configuration | Time | vs rsync (archive) | vs rsync (compress) |
|
||||
|---|---|---|---|
|
||||
| **Best: `-m -c`** | **0.39 s** | **44.8× faster** | **3.8× faster** |
|
||||
| Compression (`-c`) | 0.64 s | 27.3× faster | 2.3× faster |
|
||||
| Standard | 7.12 s | 2.4× faster | — |
|
||||
| rsync (archive) | 17.44 s | — | — |
|
||||
| rsync (archive + compress) | 1.47 s | — | — |
|
||||
|
||||
Always validate the compatibility behavior required by a deployment before
|
||||
replacing an existing rsync job.
|
||||
Compression reduces the data on the wire enough that the transfer becomes latency-bound rather than bandwidth-bound. On WAN, the best configuration runs 10.8× faster than the theoretical limit for uncompressed data, since zstd shrinks the 25 MB payload to a fraction of its original size over the wire.
|
||||
|
||||
-886
@@ -1,886 +0,0 @@
|
||||
# Rsync Feature Compatibility
|
||||
|
||||
This document maps rsync's full feature set to FastSync's current implementation status.
|
||||
|
||||
## Summary
|
||||
|
||||
| Status | Count | Description |
|
||||
|--------|-------|-------------|
|
||||
| ✅ Implemented | 143 | Feature works end-to-end |
|
||||
| 🔀 Alt Arg | 0 | Functionality exists but under different flag/semantics |
|
||||
| ⛔ Impossible/Divergence | 4 | Flag is a documented divergence or cannot be implemented on any portable filesystem call |
|
||||
| ⚠️ Partial | 0 | Flag parsed/stored but behavior incomplete |
|
||||
| 🔄 Compatibility No-op | 0 | Flag is accepted for CLI compatibility but has no effect |
|
||||
| ❌ Not Implemented | 0 | Flag not recognized or no behavior |
|
||||
| **Total** | **147** | |
|
||||
|
||||
---
|
||||
|
||||
## 1. General Options
|
||||
|
||||
| Flag | Rsync Description | FastSync Status | Notes |
|
||||
|------|-------------------|-----------------|-------|
|
||||
| `-a`, `--archive` | Archive mode is -rlptgoD | ✅ Implemented | Phase 7 Wave A: real rsync archive. `-a`/`--archive` now implies `--links` + metadata (perms/times/group/owner as FastSync's broad bundle) + `--devices` + `--specials`. FastSync is always recursive, so no `-r` is needed. It no longer implies compression or multithreading (those moved to `-z`/`-j`). The short-option namespace is now rsync-parity (see the Phase 7 note) |
|
||||
| `-v`, `--verbose` | Increase verbosity | ✅ Implemented | Sets `log_level=DEBUG` |
|
||||
| `-q`, `--quiet` | Suppress non-error messages | ✅ Implemented | Suppresses client output while preserving errors |
|
||||
| `--help` | Show help | ✅ Implemented | Prints usage and exits; `-h` is not accepted |
|
||||
| `-V`, `--version` | Print version | ✅ Implemented | |
|
||||
| `--info=FLAGS` | Fine-grained info verbosity | ✅ Implemented | Supports `copy`, `misc`, `skip`, `stats`, `all`, and `none`; explicit flags override `--verbose`, and `none` suppresses info output; unsupported names are rejected |
|
||||
| `--debug=FLAGS` | Fine-grained debug verbosity | ✅ Implemented | `io`, `proto`, `pack`, and `util` are supported; `--debug=help` lists flags; other rsync categories are rejected |
|
||||
| `--stderr=MODE` | Change stderr output mode | ⛔ Impossible/Divergence | `errors` (default) and `all` are supported; `client` is rejected with a clear error (`--stderr=client is not supported`) because FastSync has no rsync client-message channel — the rejection itself is the documented behavior (Phase 7 Wave B decision). The modes that exist work; the missing rsync channel cannot be emulated without a wire change |
|
||||
| `--no-motd` | Suppress daemon MOTD | ✅ Implemented | Client-only display switch (Wave C): the daemon still sends the configured `motd file` on a `host::module/path` connection; the client reads and discards the frame without showing it. Without the flag the MOTD is printed to stdout after the config/auth handshake and escaped so control bytes cannot inject terminal sequences |
|
||||
| `--exclude=PATTERN` | Exclude files matching pattern | ✅ Implemented | Glob matching in scanner |
|
||||
| `--include=PATTERN` | Include files matching pattern | ✅ Implemented | Glob matching in scanner |
|
||||
| `-C`, `--cvs-exclude` | Auto-ignore CVS files | ✅ Implemented | Applies the well-known rsync default exclude set as exclude rules during scanning (RCS SCCS CVS CVS.adm RCSLOG cvslog.* tags TAGS .make.state .nse_depinfo *~ #* .#* ,* _$* *$ *.old *.bak *.BAK *.orig *.rej .del-* *.a *.olb *.o *.obj *.so *.exe *.Z *.elc *.ln core .svn/ .git/ .hg/ .bzr/); `.git/`-style repo dirs are pruned without descending |
|
||||
|
||||
## 2. Modifying Output
|
||||
|
||||
| Flag | Rsync Description | FastSync Status | Notes |
|
||||
|------|-------------------|-----------------|-------|
|
||||
| `--stats` | Give transfer stats | ✅ Implemented | Prints file/byte counts |
|
||||
| `-h`, `--human-readable` | Human-readable numbers | ✅ Implemented | Formats transfer byte sizes using binary units |
|
||||
| `-i`, `--itemize-changes` | Per-file change summary | ✅ Implemented | Prints rsync-style `>f+++++++++` lines to stdout only for files actually sent (also under `-j`/`--threads`); unchanged files print nothing, matching single-`-i` behavior |
|
||||
| `--progress` | Show progress | ✅ Implemented | Progress callback in sender |
|
||||
| `-P` | Same as --partial --progress | ✅ Implemented | Phase 7 Wave B: `-P` parses to `--partial` + `--progress`. On a failed/interrupted write the receiver now retains the already-written temp file at the destination path (best-effort rename instead of unlink when configured), so a later `--append`/`--append-verify` run can resume it; `--partial-dir` still stages completed files under the confined partial dir and installs them atomically. The retention never runs when `--partial` is off, when no data was actually written, or under `--ignore-existing`/`--existing` (the destination is not ours to overwrite), and it only ever renames the already-written temp (never a corrupt blend; a failed rename falls back to the normal unlink). See the `-S`/`--sparse` interplay note (a retained sparse temp has full logical size) |
|
||||
| `--out-format=FORMAT` | Custom output format | ✅ Implemented | Per-transfer template on stdout; tokens `%f` `%n` `%l` `%b` `%M` `%%` (`%b` is the source length, always `== %l`; post-compression/delta wire bytes are not counted); unknown escapes preserved |
|
||||
| `--log-file=FILE` | Log to file | ✅ Implemented | `log_file` config field |
|
||||
| `--log-file-format=FMT` | Log format | ✅ Implemented | Requires `--log-file`; writes one template line per transferred file using the same token set as `--out-format` (including `%b` `==` source length) |
|
||||
| `--8-bit-output`, `-8` | Leave high-bit chars unescaped | ✅ Implemented | Applies to displayed paths and protocol debug output |
|
||||
| `--list-only` | List files instead of copying | ✅ Implemented | `ls -l`-style listing of files that would be transferred; scans the source only, contacts no server, writes nothing; also works with `-n` |
|
||||
|
||||
## 3. File Selection
|
||||
|
||||
| Flag | Rsync Description | FastSync Status | Notes |
|
||||
|------|-------------------|-----------------|-------|
|
||||
| `--exclude-from=FILE` | Read exclude patterns from file | ✅ Implemented | Reads patterns from file |
|
||||
| `--include-from=FILE` | Read include patterns from file | ✅ Implemented | Reads patterns from file |
|
||||
| `--filter=RULE` | Add file-filtering rule | ✅ Implemented | Long option only: rsync's short `-f` conflicts with FastSync sendfile (see FastSync-specific list), so `-f` is not reassigned. Supported subset: `+`/`-` include/exclude, implicit-exclude patterns, `include`/`exclude` word forms, a leading `/` anchor (to the transfer root, or to a `.rsync-filter` file's directory), and a trailing `/` for dir-only rules; first match wins with a default of include inside the filter layer. Filters are an independent layer from `--exclude`/`--include` (an entry must pass both). Rejected with a clear error (no silent no-ops): `merge`/`dir-merge`/`hide`/`show`/`protect`/`risk`/`clear` words, rules that begin with `:`/`.`/`!` (merge/dir-merge/list-clear shorthands), and include/exclude modifiers other than `/` (`! C s r p x`) |
|
||||
| `--files-from=FILE` | Read source file list from file | ✅ Implemented | Entries are paths relative to the source root (leading `./` stripped, `..`/absolute entries rejected at parse time, blank lines ignored; NUL-delimited with `-0`). A listed regular file is transferred; a listed directory transfers its whole subtree (FastSync recursion is always on, unlike rsync's non-recursive default). Non-listed paths and their subtrees are pruned by the scanner. A listed entry that does not exist under the source (and an empty list) is a hard error reported before any transfer, unless `--ignore-missing-args` / `--delete-missing-args` is given (see the Safety & Security rows): those flags downgrade the listed-but-missing case to a skip and, for `--delete-missing-args`, a destination deletion; an empty list stays a hard error in every mode. Listing `.` (whole tree) and empty listed directories are fine. Scalability note: `file_list_affects` is O(list size) per scanned entry, so a very large `--files-from` list against a huge tree is quadratic; lists are typically small enough that this is acceptable, but it is the documented bound. The delete manifest still derives from what was actually sent, so `--delete` stays consistent with the subset |
|
||||
| `-0`, `--from0` | Delimit *-from files with NULs | ✅ Implemented | `--files-from` entries become NUL-delimited; the flag may appear before or after `--files-from` on the command line. NUL mode preserves entry bytes exactly (trailing CR/LF are part of the name; only newline mode trims them) |
|
||||
| `--max-size=SIZE` | Skip files larger than SIZE | ✅ Implemented | `max_size` in scanner |
|
||||
| `--min-size=SIZE` | Skip files smaller than SIZE | ✅ Implemented | `min_size` in scanner |
|
||||
| `-I`, `--ignore-times` | Don't skip files matching size+time | ✅ Implemented | `ignore_times` config field (crosses the wire). Disables the size+mtime quick-check in the `--incremental` per-file handshake and the basis-dir quick-match, forcing the file to be transferred rather than skipped as unchanged. Receiver-side policy: `match_by_metadata` (file_receive.c) is bypassed, so the receiver never replies `STATUS_OK` for a matching size+mtime. Requires `--incremental` to have the handshake to act on (rsync does its quick check by default; FastSync's `-I`/`--size-only`/`--modify-window` only take effect under `--incremental`, exactly like they take effect through the basis check) |
|
||||
| `--size-only` | Skip based on size only | ✅ Implemented | With `--incremental`, ignores mtime |
|
||||
| `-@`, `--modify-window=NUM` | Mod-time comparison accuracy | ✅ Implemented | Whole-second tolerance with nanosecond-aware comparisons |
|
||||
| `--existing` | Skip creating new files on receiver | ✅ Implemented | Existing destination files continue through normal update handling |
|
||||
| `--ignore-existing` | Skip updating existing files | ✅ Implemented | `ignore_existing` config field (crosses the wire; receiver-side policy). For a destination entry that already exists, the receiver skips the write: in the regular-file path, existing/delay-updates-staged, hardlink-sibling, and special/device handlers all return `FILE_SAVE_SKIPPED` without overwriting (passed as `no_replace` to the write engine), and `--backup` is disabled for skipped files. Note: it is applied at write time, so an existing dest whose size+mtime differ still has its data (or delta) transmitted before the write is discarded — functionally correct, bandwidth-suboptimal vs rsync, which short-circuits earlier. Like rsync, it does not apply to directories/symlinks (those return before the block). Combines with `-j`/`--threads` and `--delay-updates`. See Phase-4/— notes below |
|
||||
| `--remove-source-files` | Sender removes regular files after confirmed transfer | ✅ Implemented | |
|
||||
| `-x`, `--one-file-system` | Do not cross filesystem boundaries | ✅ Implemented | Sender scanner captures the root device and skips descending into mount-point crossings (`st_dev` differs); cross-filesystem mount-point subdirectories are dropped entirely, matching rsync |
|
||||
| `-F` | Add the default `.rsync-filter` rules | ✅ Implemented | Reads one filter rule per line from each directory's `.rsync-filter` file during traversal and applies it to that directory's subtree; the current directory's rules are evaluated before its ancestors', so deeper files override shallower ones and per-directory files override the command-line `--filter`/`-C` base by default (matching rsync's first-match-wins precedence); `.rsync-filter` files are never transferred. The rsync `-FF` behavior (also `.cvsignore`) is out of scope; unsupported rule types inside the file abort with a clear error |
|
||||
|
||||
## 4. Directory Options
|
||||
|
||||
| Flag | Rsync Description | FastSync Status | Notes |
|
||||
|------|-------------------|-----------------|-------|
|
||||
| `-r`, `--recursive` | Recurse into directories | ✅ Implemented | Default behavior |
|
||||
| `-R`, `--relative` | Use relative path names | ✅ Implemented | Meaningful together with `--files-from` (FastSync's default full-tree scan always mirrors the full source argument path below the destination root, so -R does not change it). With `-R` + `--files-from` each listed entry is transmitted under its bare relative destination path: an entry `sub/x.txt` lands at `<dest>/sub/x.txt` (its leading components preserved) instead of under the `<dest>/<full source path>` mirror. Only the path sent on the wire changes; the client still reads the absolute source path, and the delete manifest derives from the sent (relative) paths so `--delete` and `--remove-source-files` stay consistent in both layouts. Works single-threaded and under `-j`/`--threads` (including chunk serialization) |
|
||||
| `--no-implied-dirs` | Don't send implied dirs with -R | ✅ Implemented | Client-side, meaningful only with `-R` + `--files-from`. rsync would normally create the ancestor directories implied by a listed file so it can be written; with `--no-implied-dirs` a listed file whose parent directory is not itself (or via an ancestor) explicitly listed cannot be placed, and FastSync fails the whole run up front with a clear error (`--no-implied-dirs: cannot place file '...': parent directory '...' is not explicitly listed`). Listing the directory (or an ancestor of it, or the whole tree `.`) permits the file. In every other mode the option has no effect. FastSync has no per-entry skip channel, so the rsync "omit the file" case is surfaced as a hard pre-transfer error |
|
||||
| `-d`, `--dirs`, `--old-dirs`, `--old-d` | Transfer dirs without recursing | ✅ Implemented | `-d <dir>` transmits an explicit directory entry for the source-root directory, so the destination mirror is created empty and nothing is descended into. With `--files-from` exactly the listed items are transferred: a listed directory is created empty (no descent) and a listed file is transferred with its content; the dest layout follows the same -R rules as plain files. A new wire frame (`STATUS_MKDIR`) carries each directory entry — the path and, when `--preserve`/`-a` (metadata mode) is negotiated, the directory's metadata; the receiver creates it with the same confined mkdir-parent semantics as regular writes, in single-threaded and `-j`/`--threads` receivers (chunk serialization carries a per-entry type marker). Directory entries appear in the delete manifest so `--delete` prunes correctly. Directory TIMES are transmitted (the `STATUS_DIR_TIMES` frame carries every traversed source directory's captured times, including `--dirs` entries) and applied by the receiver at the END of the transfer, after all children and the delete/publication phases, so a later child write cannot clobber a directory's mtime (`-O`/`--omit-dir-times` skips this application). FastSync divergences: directory modes/ownership are still not applied (only times are), and empty directories are still never created (a `STATUS_DIR_TIMES` entry is record-only), filter/`--exclude` rules are not re-applied to the listed dirs mode (there is no descent during which they would apply), and `-d` never creates the intermediate directories between the destination root and a listed file beyond the usual on-demand parent creation. Under `--delay-updates` only regular files are staged: directory entries are created immediately, so a delayed run that fails part way can leave the already-created empty directories behind (matching rsync, which also creates directories as it processes the file list and only delays regular-file data) |
|
||||
| `--mkpath` | Create missing path components | ✅ Implemented | Wire option (client → server). At connection start the server creates the client's destination root directory (and any missing leading components below its own authorized root) when `--mkpath` is set, failing the connection cleanly if it cannot. Without `--mkpath` a destination root that does not exist yet is rejected up front (rsync semantics), so the flag is the only way to transfer into a not-yet-created destination directory. Creation is confined by the same secure mkdir walk as file writes (`O_NOFOLLOW`, no `..`) |
|
||||
|
||||
## 5. Transfer Modifications
|
||||
|
||||
| Flag | Rsync Description | FastSync Status | Notes |
|
||||
|------|-------------------|-----------------|-------|
|
||||
| `-u`, `--update` | Skip files newer on receiver | ✅ Implemented | `update` config field (crosses the wire; receiver-side policy, implies `-M` metadata). Before writing a regular file, the receiver checks `file_destination_is_newer_secure()` (via `stat_is_newer`, second-then-nanosecond strict `>` on the existing destination) and skips the write when the destination is newer than the source (`FILE_SAVE_SKIPPED`); equal-or-older destination (or a newer source) is transferred normally. Applied at write time on the regular-file, delay-updates-staged, hardlink-sibling, and special/device paths. Only regular destinations can be guarded (the newer-check requires `S_ISREG`), and like the other write-time policies it does not short-circuit the data transfer for a differing-size dest. `--remove-source-files` correctly respects the receiver's skip outcome so a skipped source is not removed |
|
||||
| `--inplace` | Update files in-place | ✅ Implemented | Direct write mode |
|
||||
| `--append` | Append data to shorter files | ✅ Implemented | Tail-only resume. When an existing destination file is SHORTER than the source, the receiver negotiates a resume offset with the sender and only the tail is transferred; the receiver rebuilds the full file (retained prefix + tail) and installs it through the normal atomic store path, so the result is byte-identical to the source whenever the retained prefix matches. Plain `--append` does NOT content-verify that prefix (rsync parity): a destination whose prefix differs from the source is resumed anyway, so the result (wrong prefix + correct tail) is NOT byte-identical and the file is effectively left corrupt — the documented rsync-parity risk (use `--append-verify` when the prefix cannot be trusted). Non-content attributes (permissions/ownership/mtime, via `-M`) are still applied. Requires the per-file `STATUS_CHECK` handshake, so it implies `--incremental`; it takes precedence over block delta for a growing file and falls back to delta/full when the destination is not shorter. Incompatible with `-s` (chunk serialization) and `--whole-file` (both rejected up front so the mode never silently degrades to a full transfer). Combines with `--inplace`, `--partial`/`--partial-dir`, and `--delay-updates` (the reconstructed full file flows through those paths unchanged). Divergence: rsync appends in place; FastSync reconstructs and atomically installs, so an interrupted or failed resume never leaves a half-written file at the destination (no corruption window), and `--append` is thus safe to use with the normal atomic path — not only with in-place writes |
|
||||
| `--append-verify` | Append with old-data checksum | ✅ Implemented | Like `--append`, but the retained prefix IS verified before resuming: the sender transmits the source prefix checksum and the receiver compares it to the xxHash64 of the retained destination prefix; on a match only the tail is transferred, on a MISMATCH the run falls back to a clean full transfer so the result is always a byte-identical source copy (never a corrupt prefix+tail blend). Wire/protocol: the append handshake adds `STATUS_APPEND` / `STATUS_APPEND_SIG` / `STATUS_APPEND_OK` / `STATUS_APPEND_DATA` frames and `PROTOCOL_VERSION` was bumped **2.9.0 → 2.10.0** (peers must match, and both must be 2.10.0 or the run fails the version check). Same implications/incompatibilities as `--append`; when both spellings are given `--append-verify` wins (the safer semantics). See the Phase-3 append notes below |
|
||||
| `-W`, `--whole-file` | Copy whole file (no delta) | ✅ Implemented | `whole_file` config field. Forces a full (whole-file) copy, disabling the block-level delta machinery: the sender only sends `STATUS_NEXT` + full data (client_send.c) and the receiver never requests a delta signature/reconstruction — the receiver's `try_delta = use_delta && !whole_file && ...` short-circuits. `whole_file` crosses the wire folded into `use_delta` (the wire carries `use_delta && !whole_file`), so no separate field/bump is needed. Delta is opt-in (`--delta` needs `--incremental`); `-W` additionally makes `--fuzzy` inert (no similar-file delta basis). `--append`/`--append-verify` are incompatible with `-W` and rejected up front (both sides). See the delta/append notes below |
|
||||
| `--block-size=SIZE` | Force checksum block-size | ✅ Implemented | Phase 7 Wave B: `--block-size` is an alias for `--delta-block`; both set `config->delta_block_size` (default `DELTA_BLOCK_SIZE_DEFAULT`, bounds `DELTA_BLOCK_SIZE_MIN..MAX`, out-of-range values are rejected with the default kept). The value is genuinely honored by the delta engine end-to-end: `delta_signature_create_seeded(old, size, config->delta_block_size, seed)` on the sender and receiver, `delta_apply(old, ...)` with the same size, so a non-default block size changes the block count of every signature the harnesses exchange (verified by unit + integration tests) |
|
||||
|
||||
## 6. Destination Handling
|
||||
|
||||
| Flag | Rsync Description | FastSync Status | Notes |
|
||||
|------|-------------------|-----------------|-------|
|
||||
| `-n`, `--dry-run` | Trial run with no changes | ✅ Implemented | `dry_run` config field |
|
||||
| `-b`, `--backup` | Make backups of overwritten files | ✅ Implemented | Backup before overwrite |
|
||||
| `--backup-dir=DIR` | Backup directory hierarchy | ✅ Implemented | `backup_dir` config field |
|
||||
| `--suffix=SUFFIX` | Backup suffix (default ~) | ✅ Implemented | `suffix` config field |
|
||||
| `--delay-updates` | Put updated files in place at end | ✅ Implemented | Successfully received files are staged under a private 0700 `.fastsync-stage` dir inside the receive root and atomically renamed into their final destinations only after the whole transfer (manifest/delete handling included) succeeds, just before the success/outcome frame is sent. The delete walker deliberately skips the staging dir at the receive root, so `--delete` removes genuine extras but never the staged files (deletion runs before publication; rsync's delete-after ordering is not implemented). `--existing`/`--ignore-existing`/`--update` decide against the final destination path at stage time; `--backup` moves the old file aside at publication. Incompatible with `--inplace` and with `--backup-dir=.fastsync-stage` (the internal staging name is reserved; both are rejected). The staging dir name is fixed, so two simultaneous delayed transfers to the same destination root are serialized with an exclusive advisory lock held for the whole transfer: the second session fails cleanly instead of corrupting the first. Aborting or failing before publication installs nothing and removes the staging tree; a crash between stage and publish leaves staged leftovers that the next delayed run wipes at start (process death releases the lock). A stage→publish failure aborts the transfer (best-effort cleanup of the not-yet-published staged files; already-published files are not rolled back). Works in single-threaded and `-j`/`--threads` modes |
|
||||
| `-T`, `--temp-dir=DIR` | Create temporary files in DIR | ✅ Implemented | `--temp-dir` with the rsync short `-T` (Phase 7 Wave A; the timeout alias moved to long-only `--timeout`). Scratch dir is resolved under the receive root; temp copies use a unique name there and are atomically renamed into place. If the scratch dir and destination are on different filesystems the atomic rename fails with EXDEV and the file save fails, which aborts the whole transfer (FastSync has no per-file skip/resume on a save error; rsync's non-atomic copy fallback is deliberately not used). `--inplace` and `--partial-dir` writes bypass the scratch dir |
|
||||
|
||||
## 7. Deletion
|
||||
|
||||
| Flag | Rsync Description | FastSync Status | Notes |
|
||||
|------|-------------------|-----------------|-------|
|
||||
| `--delete` | Delete extraneous files from dest | ✅ Implemented | `use_delete` config field. Deletion is always derived from the transmitted keep-set manifest of the paths the sender sent/keeps (never from unchecked input), runs through the symlink-safe walker bounded by `MAX_SERVER_DELETE_COUNT`, and skips the `.fastsync-stage` staging dir under `--delay-updates`. FastSync's default timing when no timing flag is given is **delete-after** (extras are removed only once the whole transfer succeeded) — intentionally NOT rsync's `--del`/delete-during default, to preserve FastSync's commit-style safety. By default the destination mirror of a path the source scan pruned (filter/exclude/size rules) is **protected** from deletion — matching rsync, which does not delete excluded files under `--delete`; `--delete-excluded` opts back into deleting them (see below). The bounded deletion is **all-or-nothing**: if the destination holds more extras than the effective bound (a client `--max-delete=NUM` or the 100000-entry server bound) nothing is deleted and the run fails with a distinct error instead of silently truncating |
|
||||
| `--delete-before` | Delete before transfer | ✅ Implemented | Implies `--delete`. The sender runs a full source pre-scan (paths only) and transmits the keep-set manifest BEFORE any file data; the receiver validates it, removes every destination entry not listed (all-or-nothing bounded walk, staging-dir skip, protected prefixes honored), then acks `STATUS_OK`. The sender only starts streaming after the deletion committed, or aborts if the receiver reported a deletion error. By definition the deletions already happened when a later transfer phase fails — rsync's delete-before is destructive the same way; a subsequent failure does not restore the removed files. Divergence: the keep-set is the pre-scan snapshot, so a file that appears on the source between the pre-scan and the data pass is still transferred but was not protected from deletion |
|
||||
| `--del`, `--delete-during` | Delete during transfer | ✅ Implemented | Both spellings accepted; imply `--delete`. FastSync streams the source in a single directory scan and has no per-directory generator pass, so deletions cannot be interleaved per-directory the way rsync's delete-during does. `--delete-during` therefore selects the same early engine mode as `--delete-before` (manifest transmitted before any data, extras removed and acknowledged before data is applied); observable success/failure behaviour equals `--delete-before`. That is the documented divergence from rsync, where `--del` is the default meaning of `--delete` |
|
||||
| `--delete-delay` | Find deletions during, delete after | ✅ Implemented | Implies `--delete`. Commit-mode timing: extras are removed only after the whole transfer succeeded. rsync's delete-delay records the deletion list during its scan and applies it at the end; FastSync never snapshots the destination while data flows (the keep-set is the transmitted manifest and the destination is listed only at deletion time), so `--delete-delay` is implemented as the same end-of-transfer commit as `--delete-after` with identical safety. That is the documented divergence |
|
||||
| `--delete-after` | Delete after transfer | ✅ Implemented | Implies `--delete`. The delete-after timing is also what plain `--delete` does: the keep-set manifest closes the data stream and the receiver commits the bounded deletion only after the terminal `STATUS_FINISHED` proves the whole transfer (every data frame received and stored) succeeded. A failed or aborted transfer removes nothing |
|
||||
| `--delete-excluded` | Also delete excluded files | ✅ Implemented | `delete_excluded` config field. Under `--delete` FastSync now protects (rsync's default) the destination mirror of paths the sender's source scan pruned by user-selection rules — the `--filter`/`-F`/`-C` layer, the legacy `--exclude`/`--include` layer, and `--max-size`/`--min-size`. The sender transmits those concrete pruned paths as **protected prefixes** in the delete-manifest frame (see the Phase-3 notes below); the walker never descends into or removes them. `--delete-excluded` opts back in: the sender sends an empty protected list, so the excluded destination mirrors become ordinary extras and are removed. Divergences (documented): protection is derived only from what the source scan actually pruned — a stray destination-only file that happens to match an exclude rule is not protected (FastSync never re-applies rules to the destination, keeping deletion sender-derived), and `--files-from` subset pruning stays keep-set-only (an unlisted source path is treated as absent and its mirror is deletable, matching the `--files-from` delete note below). The two are orthogonal: `--delete-excluded` removes filter-excluded mirrors; it does not make `--files-from` prune things |
|
||||
| `--max-delete=NUM` | Max files to delete | ✅ Implemented | `max_delete` config field (default -1 = no client limit; 0 = delete nothing). NUM bounds a `--delete` run with rsync's all-or-nothing semantics: the receiver rehearses the deletion first and, if the destination holds more than NUM extras, deletes NOTHING and fails the transfer with a distinct `--max-delete` error. A run at or below NUM deletes exactly the extras. NUM only applies together with `--delete` (it is inert otherwise, matching rsync). The hard server bound `MAX_SERVER_DELETE_COUNT` (100000) still caps the walk; a NUM above it never raises that cap, and exceeding the server bound is its own all-or-nothing error. Directories count toward the limit (each removed empty directory is one deletion), like rsync |
|
||||
| `--ignore-errors` | Delete even with I/O errors | ✅ Implemented | Sender-side, client-only config field. rsync suppresses `--delete` when the transfer had I/O errors; FastSync's equivalent is a source-scan I/O error (an unreadable directory, e.g. EACCES): by default the scan aborts the run so no deletion happens. With `--ignore-errors` the scan continues past the unreadable directory, the readable tree is transferred and the deletion still runs (the mirror of the unreadable directory is treated as an extra). The run still exits non-zero (the error is reported, matching rsync's error status). Divergence: without the flag FastSync aborts the whole run on the scan error, whereas rsync transfers the rest of the tree and merely skips the deletion; both leave the deletion undone |
|
||||
| `--force` | Force deletion of non-empty dirs | ✅ Implemented | `force_delete` receiver config field (crosses the wire). rsync's `--force` lets an incoming non-directory replace a destination directory; FastSync implements exactly that: when a regular file is written to a path that is currently a (possibly non-empty) destination directory, `--force` removes that directory tree first — confined to the receive root and symlink-safe (O_NOFOLLOW fd walk, symlinks removed by name, never followed) — so the atomic install can place the file. Without `--force` such a write fails and the run aborts. Divergence: `--force` acts on the immediate-install path only; under `--delay-updates` a blocking directory is not cleared (publication renames over regular files) |
|
||||
| `-m`, `--prune-empty-dirs` | Prune empty dir chains | ✅ Implemented | `-m`/`--prune-empty-dirs` (Phase 7 Wave A freed the rsync short `-m`; FastSync multithreading is now `-j`/`--threads`). FastSync's recursive transfer records directory times but never CREATES an empty directory (a `STATUS_DIR_TIMES` entry is record-only, and `--dirs` empty entries are pruned by this flag), so empty directories are inherently never transferred (which is rsync's `-m` behavior) and truly-empty destination directory chains are removed by `--delete` regardless of this flag. The flag's additional real effect is on the `--dirs` explicit directory-entry generator: a plain `-d <empty-dir>` run omits the empty source directory's entry, so nothing is created at the destination (no `STATUS_MKDIR`, no `-i`/`--out-format` change line, and an existing empty mirror becomes an extra that `--delete` prunes). Explicitly `--files-from`-listed directories always pass through (documented `--files-from` behavior). A directory that still holds an excluded-but-protected file survives, matching the `--delete-excluded` default |
|
||||
|
||||
**Deletion-timing implementation notes (Phase 3):** the delete flags above are
|
||||
real. Two new config booleans (`delete_during`, `delete_delay`) join the already
|
||||
serialized `delete_before`/`delete_after`, so the on-the-wire config layout
|
||||
changed and `PROTOCOL_VERSION` was bumped **2.7.0 → 2.8.0** (peers must match).
|
||||
The `STATUS_MANIFEST` frame is count-delimited and position-independent: the
|
||||
receiver commits the deletion either when the manifest arrives (early modes:
|
||||
`--delete-before`/`--delete-during`, which additionally acknowledge with
|
||||
`STATUS_OK` before data flows) or after the terminal `STATUS_FINISHED` proves
|
||||
the whole transfer succeeded (commit modes: plain `--delete`/`--delete-after`/
|
||||
`--delete-delay`). Timing is chosen purely from the config, so server policy
|
||||
(`--allow-delete` off) still disables deletion without deadlocking the early
|
||||
manifest ack. `--delete-delay` and `--delete-during` are each implemented as
|
||||
the closest safe approximation their engine mode allows; the divergences are
|
||||
noted in the rows above.
|
||||
|
||||
**Deletion-policy notes (Phase 3, delete-policy wave):** this wave made the
|
||||
deletion family real — `--delete-excluded`, `--max-delete`, `--ignore-errors`,
|
||||
`--force`, `--prune-empty-dirs` — and, to support them, the `STATUS_MANIFEST`
|
||||
frame now carries **two sections**: the keep-set paths followed by a list of
|
||||
**protected prefixes** (destination-relative paths the source scan pruned by
|
||||
user-selection rules, which the walker must never delete unless
|
||||
`--delete-excluded` opted out). Two config booleans were added for the wave:
|
||||
`force_delete` (crosses the wire; the receiver clears a directory that blocks an
|
||||
incoming file) and `ignore_errors` (client-only; the sender's scan continues
|
||||
past an unreadable directory). `max_delete`'s default became -1 ("no client
|
||||
limit"). These wire/layout changes bumped `PROTOCOL_VERSION` **2.8.0 → 2.9.0**
|
||||
(peers must match). All four wire additions — `force_delete`,
|
||||
`delete_excluded`, `prune_empty_dirs`, `max_delete` — round-trip unchanged and
|
||||
are validated on receive.
|
||||
|
||||
**Missing-args note (Phase 3, missing-args wave):** `--ignore-missing-args` and
|
||||
`--delete-missing-args` are implemented as described in the Safety & Security
|
||||
rows. Wire impact: the `STATUS_MANIFEST` frame now carries a **third section** —
|
||||
a list of destination-relative **exact-delete paths** (the missing entries'
|
||||
mirrors) — and the config frame gained a `delete_missing_args` boolean
|
||||
(`ignore_missing_args` stays client-only, exactly like `ignore_errors`). These
|
||||
wire/layout changes bumped `PROTOCOL_VERSION` **2.9.0 → 2.10.0** (peers must
|
||||
match). The receiver validates the third section identically to the keep-set
|
||||
(non-empty, relative, traversal-free; `MAX_MANIFEST_ENTRIES` per section, a
|
||||
single `MAX_MANIFEST_BYTES` budget shared across all three). On commit the
|
||||
receiver runs the exact-path deletions FIRST (`manifest_delete_missing_args`:
|
||||
confined per-path unlink/rmdir, deep removal only under `--force`/`--delete`,
|
||||
staging/basis protected, never blocked by the protected-prefix list) and then
|
||||
the ordinary extras walk when `--delete` is active (`manifest_delete_all`). A
|
||||
client may request the exact-path deletions without `--delete`; the server's
|
||||
`--allow-delete` policy gates them exactly like `--delete`, so an unauthorized
|
||||
server ignores the request while the missing entries are still skipped.
|
||||
|
||||
The deletion walker is now **all-or-nothing**: before any unlink it rehearses
|
||||
the deletion (an fd-relative walk identical to the delete pass, counting every
|
||||
regular file it would unlink and every directory it would remove) and refuses to
|
||||
start when the extras exceed the effective bound — a client `--max-delete=NUM`
|
||||
below the hard bound, or the hard `MAX_SERVER_DELETE_COUNT` (100000) bound
|
||||
itself. Previously the walker removed up to `MAX_SERVER_DELETE_COUNT` extras and
|
||||
then reported an error (a truncated deletion); it now removes nothing and fails
|
||||
with an error naming the bound. Directories count toward the bound. A directory
|
||||
that still holds entries the walker leaves in place (a protected excluded file,
|
||||
a kept manifest entry, a symlink) is left behind rather than failing the run —
|
||||
matching rsync's "cannot delete non-empty directory" behaviour. The
|
||||
all-or-nothing guarantee holds only while the destination is not concurrently
|
||||
modified: rehearsal and delete are two separate walks, so a concurrent change
|
||||
between them (another process adding or removing destination entries) can make
|
||||
the actual deletion diverge from the counted set.
|
||||
|
||||
Manifest size: the sender's keep-set and protected-prefix collections (streaming
|
||||
or early pre-scan) are unbounded, but the receiver rejects a manifest beyond
|
||||
`MAX_MANIFEST_ENTRIES` (1 048 576 entries, applied to EACH section — a frame can
|
||||
therefore total up to 2 097 152 entries) / `MAX_MANIFEST_BYTES` (16 MB of paths,
|
||||
counted across BOTH sections) as a hard protocol error. A heavily filtered
|
||||
source whose exclusion list grows large thus fails the run cleanly on the
|
||||
receiver (STATUS_ERROR) instead of being silently truncated. In the commit
|
||||
modes this only means the deletion is refused after the data already arrived; in
|
||||
the early modes (`--delete-before`/`--delete-during`) the manifest is the first
|
||||
frame, so an oversized keep-set or protected list aborts the whole transfer
|
||||
BEFORE any data is sent. Keep the source tree small enough for the receiver's
|
||||
manifest caps when using the early timing.
|
||||
|
||||
Early-delete ACK wait: after committing a large deletion (up to
|
||||
`MAX_SERVER_DELETE_COUNT` removals) the receiver's `STATUS_OK`/`STATUS_ERROR`
|
||||
reply can legitimately take much longer than a normal round trip, so the sender
|
||||
waits for that single ACK with an extended explicit deadline (1 hour) instead
|
||||
of the default 60 s per-message receive window. A receiver that is genuinely
|
||||
gone still aborts the wait via connection close/error; the extended bound only
|
||||
protects against aborting after the deletion already committed on the receiver.
|
||||
|
||||
Flag-conflict policy: unlike rsync's last-one-wins behaviour, every deletion
|
||||
timing flag implies `--delete`, and combining a timing flag with `--no-delete`
|
||||
(in either argument order) — or more than one timing flag — is rejected as a
|
||||
configuration error rather than silently resolved. Note the check is
|
||||
order-independent because it runs over the fully parsed config. The deletion
|
||||
POLICY flags (`--delete-excluded`, `--max-delete`, `--ignore-errors`, `--force`)
|
||||
do NOT imply `--delete`; without `--delete` they are inert (matching rsync).
|
||||
|
||||
**Append-resume notes (Phase 3, append wave):** `--append` and `--append-verify`
|
||||
are real. Both are negotiated when an existing destination file is found to be
|
||||
**shorter** than the source during the per-file `STATUS_CHECK`; the receiver
|
||||
replies with a new `STATUS_APPEND` frame carrying the resume offset (the prefix
|
||||
length it already holds) instead of `STATUS_NEXT`/`STATUS_DELTA_SIGNATURE`.
|
||||
The sender transmits ONLY the tail. For `--append-verify` it first sends the
|
||||
source's prefix xxHash64 in a `STATUS_APPEND_SIG` frame; the receiver compares
|
||||
it to the retained prefix and answers `STATUS_APPEND_OK` (transfer the tail) or
|
||||
`STATUS_NEXT` (prefix mismatch → the sender falls back to a byte-exact full
|
||||
transfer). The tail arrives in a `STATUS_APPEND_DATA` frame (compression and
|
||||
metadata still apply). The receiver then rebuilds the full file in memory
|
||||
(prefix + tail) and routes it through the existing atomic store engine, so all
|
||||
of `--inplace`, `--partial`/`--partial-dir`, `--delay-updates`, `--backup`,
|
||||
`--existing`/`--ignore-existing`/`--update` and delete-manifest behaviour is
|
||||
unchanged and the result is a byte-identical source copy (given a matching
|
||||
prefix). These new frames changed the wire, so `PROTOCOL_VERSION` was bumped
|
||||
**2.9.0 → 2.10.0** (peers must match; the pre-existing `append`/`append_verify`
|
||||
config booleans already crossed the wire). CLI: both flags imply `--incremental`
|
||||
(the handshake needs it); they are incompatible with `-s` (chunk serialization)
|
||||
and `--whole-file` (both rejected up front, never a silent full transfer); when
|
||||
both spellings are given `--append-verify` wins. The FastSync divergence from
|
||||
rsync is intentional and safer: rsync appends in place, whereas FastSync
|
||||
reconstructs the whole file and atomically installs it, so an interrupted or
|
||||
failed resume never leaves a partial/corrupt file at the destination — this is
|
||||
why plain `--append` works on the normal atomic path, not only with `--inplace`.
|
||||
|
||||
## 8. Metadata Preservation
|
||||
|
||||
| Flag | Rsync Description | FastSync Status | Notes |
|
||||
|------|-------------------|-----------------|-------|
|
||||
| `-M`, `--preserve` | Preserve file metadata | ✅ Implemented | Mode, uid, gid, mtime |
|
||||
| `-p`, `--perms` | Preserve permissions | ✅ Implemented | Phase 7 Wave A: `-p`/`--perms` now preserve permission bits, folded into FastSync's broad metadata bundle (`--preserve`); the SSH port moved to `--ssh-port`. rsync-parity short form |
|
||||
| `-o`, `--owner` | Preserve owner | ✅ Implemented | Part of -M |
|
||||
| `-g`, `--group` | Preserve group | ✅ Implemented | Part of -M |
|
||||
| `-t`, `--times` | Preserve modification times | ✅ Implemented | Part of -M |
|
||||
| `-E`, `--executability` | Preserve executability | ✅ Implemented | Preserves executable permission bits (implies metadata preservation) |
|
||||
| `--chmod=CHMOD` | Affect file permissions | ✅ Implemented | Supports numeric and symbolic `ugo` `rwx` changes; retains receiver safety masking |
|
||||
| `-A`, `--acls` | Preserve ACLs | ✅ Implemented | Implemented on Linux via the POSIX-ACL xattr representation: the sender captures the `system.posix_acl_access` / `system.posix_acl_default` xattrs into the same bounded whitelisted set as `-X`, transmits them per-file, and the receiver re-applies them fd-relative. Setting an ACL the receiver is not permitted to set (non-root on a file it does not own, unsupported filesystem) is logged and skipped, never fatal. libacl is **not** required. Only the `system.posix_acl_*` namespaces plus `user.*` are ever applied; privileged namespaces are never applied (see the Phase-4 xattr/ACL notes below). Implies metadata transmission |
|
||||
| `-X`, `--xattrs` | Preserve extended attributes | ✅ Implemented | Preserves unprivileged `user.*` extended attributes (Linux `listxattr`/`getxattr` on capture, `fsetxattr` on the written destination fd). Both capture (sender) and application (receiver) are restricted to the `user.*` namespace and the two POSIX ACL xattrs, so a client can **never** force a `security.*`/`trusted.*`/privileged attribute onto the destination; the receiver independently re-validates every incoming name against this whitelist and rejects anything else. Payloads are bounded (per-name ≤255B, per-value ≤1MiB, per-file count ≤256 total bytes ≤4MiB) on both ends, and an oversized/malformed frame is a clean protocol rejection (no OOM). Applied fd-relative to the exact written file. Implies metadata transmission. Incompatible with `-s` (chunk serialization), rejected up front (see the notes); a `--link-dest`/`-H` hard-link copy fallback re-applies the attributes so they are not dropped when a link is refused |
|
||||
| `-H`, `--hard-links` | Preserve hard links | ✅ Implemented | Files on the source that share an inode (`st_dev`+`st_ino`, e.g. a `cp -al` tree) are re-created as hard links to one another on the destination, so duplicate links stay deduplicated and only the first member's data is sent (later members are transmitted as payload-less `STATUS_HARDLINK` frames). The receiver links each sibling to the first member's installed file with an atomic link + rename; on `link()` failure it falls back to a byte-identical local copy of the first member, never a partial/corrupt file. Requires the sequential scan for ordering (the first member is always emitted and installed before any sibling is linked). Works single-threaded and under `-j`/`--threads`, `--inplace`, `--delay-updates` (links staged and published by rename) and `--partial`. Crosses the wire (`preserve_hard_links` bool; `PROTOCOL_VERSION` bumped **2.11.0 → 2.12.0**, peers must match). Incompatible with `-s` (chunk serialization) and `--append`/`--append-verify`, rejected up front with a distinct error. See the Phase-4 hard-links notes below |
|
||||
| `-D` | Same as --devices --specials | ✅ Implemented | Implies `--devices --specials`. `-D` was unassigned in FastSync (verified: no collision), so it is free to imply both device-node and special-file preservation. See the `--devices`/`--specials` rows and the Phase-4 devices notes below |
|
||||
| `--devices` | Preserve device files | ✅ Implemented | Recreates char/block device nodes on the destination via `mknod` instead of transferring content. Type + rdev are validated strictly (S_IFMT from the transmitted mode; major/minor range-checked, non-negative), and creation is **privilege-gated**: `mknod` needs `CAP_MKNOD`, so a non-root receiver (CI runs via setpriv as non-root) logs a warning and **skips the device entry safely** — the whole transfer never aborts just because the node could not be made. The node is created fd-relative below the receive root (`mknodat` on the confined secure parent), so it can never be placed outside the authorized root, never follows a symlink, and never replaces an existing directory. Only a char/block mode is honored. Crosses the wire (a new `STATUS_SPECIAL` frame carries the path + metadata mode + rdev; `PROTOCOL_VERSION` bumped **2.12.0 → 2.13.0**). Divergence: per-entry skip (not a hard error) when the receiver lacks `CAP_MKNOD`, documented in the Phase-4 devices notes |
|
||||
| `--specials` | Preserve special files | ⛔ Impossible/Divergence | **FIFO recreation works**: FIFOs are recreated on the destination via `mkfifo` (unprivileged, so this is a real, assertable behavior under CI). **Only socket recreation is impossible**: a socket entry can be created only by `bind(2)` on a live socket, not by any filesystem call, so a source socket is skipped with an explicit note. That one unsupported node kind is why the flag is classified Impossible/Divergence even though FIFO recreation itself works; its normal path is otherwise complete. FIFO creation is privileged-gated only in the sense of graceful skip on any permission failure. Node creation is confined below the receive root (`mkfifoat` on the secure fd-relative parent; no `..`, no symlink follow). Crosses the wire like `--devices` (the `STATUS_SPECIAL` frame; `PROTOCOL_VERSION` bumped **2.12.0 → 2.13.0**). See the Phase-4 devices notes |
|
||||
| `--copy-devices` | Copy device contents as file | ✅ Implemented | Copy a device's CONTENT into an ordinary regular file on the destination instead of recreating the node — non-privileged and safe. FastSync scans a device/FIFO as a regular file: its reported size (`st_size`, typically 0 for char devices and FIFOs) is copied, so a FIFO or a non-readable device becomes an empty (or size-bounded) regular file. The default data path is size-bounded and never blocks (it sends exactly `st_size` bytes, never an unbounded pseudo-device stream); with `--sendfile`, a non-regular source (FIFO/device) is detected from its `stat` mode and falls back to that same buffered read, so `--copy-devices --sendfile` cannot hang either. The run always succeeds and never crashes on such input. **Deliberate, safe divergence from rsync's dd-like unbounded device read.** See the Phase-4 devices notes |
|
||||
| `--write-devices` | Write to devices as files | ✅ Implemented | Write the received data directly into an **existing** device node on the destination instead of creating a regular file. Restricted and best-effort: the destination must already exist and be a char/block device (opened only under the confined receive root, with `O_NOFOLLOW` + `O_NONBLOCK`); a missing, symlinked, FIFO-with-no-reader (`ENXIO`), non-device destination, or any write failure is **skipped with a warning** rather than allowed, so a run can never clobber the system, never blocks on a special-file target, and never aborts on an unusable target. See the Phase-4 devices notes |
|
||||
| `-U`, `--atimes` | Preserve access times | ✅ Implemented | Captures the source access time (from the scanner's pre-read stat, so it is not clobbered by reading the file for transfer) and transmits it over the wire; the receiver restores it together with the mtime via `futimens`/`utimensat`. Implies metadata transmission (the times travel inside the `-M` metadata payload), but does not enable ownership application (that stays opt-in via the identity flags). Wire: new `atime` fields on the metadata frame + a `preserve_atimes` config boolean; `PROTOCOL_VERSION` bumped **2.11.0 → 2.12.0** |
|
||||
| `-N`, `--crtimes` | Preserve create times | ⛔ Impossible/Divergence | Birth-times cannot be set by any portable filesystem call (`utimensat`/`futimens` only set atime/mtime), so this row is an explicit **Impossible/Divergence** (Phase 7 Wave B). Capture + transmit stays: `statx(STATX_BTIME)` on Linux records the source birth time as a wire field; the receiver logs a debug note that it cannot be applied and continues — never failing the transfer and never pretending it worked. On platforms without `statx` it parses as a documented no-op (flag accepted; nothing is captured). Implies metadata transmission. Wire: new `crtime` fields + a `preserve_crtimes` config boolean; `PROTOCOL_VERSION` bumped **2.11.0 → 2.12.0** (see the Phase-4 metadata-time notes) |
|
||||
| `-O`, `--omit-dir-times` | Omit dirs from --times | ✅ Implemented | Real modifier now that FastSync preserves directory times. With metadata on, the scanner captures every traversed source directory's mtime (and atime under `-U`) and the sender transmits them in trailing `STATUS_DIR_TIMES` frame(s) **after all file data and the optional delete manifest** (chunked at the receiver's `MAX_MANIFEST_ENTRIES` per-frame cap); a dir-time entry only RECORDS metadata and never creates the directory, so empty source directories stay untransferred. The receiver defers applying them until its delete / `--delay-updates` publication phases have committed, so writing or removing a child never clobbers a parent directory's mtime (rsync applies directory times at the end for exactly this reason). When `-O` is set (the boolean crosses the wire) the receiver does not apply any of them; without `-O` an `-a`/`--preserve` transfer now restores directory times (reversing the old "never preserves dir times" divergence). Wire change: the terminal `STATUS_DIR_TIMES` frame; `PROTOCOL_VERSION` bumped **2.16.0 → 2.17.0** |
|
||||
| `-J`, `--omit-link-times` | Omit symlinks from --times | ✅ Implemented | Real modifier now that FastSync preserves symlink times. Symlink entries already carried their metadata on `STATUS_SYMLINK`; the receiver now applies it with **no-follow primitives only** (`utimensat(..., AT_SYMLINK_NOFOLLOW)`, plus best-effort `fchmodat(..., AT_SYMLINK_NOFOLLOW)` and policy-gated `fchownat(..., AT_SYMLINK_NOFOLLOW)`), so the link itself is stamped without ever dereferencing it, confined fd-relative below the authorized receive root. A symlink has no children, so the times are applied immediately at creation. When `-J` is set (the boolean crosses the wire) the receiver skips the timestamps (mode/ownership are unaffected); without `-J` an `-a`/`-l` transfer restores symlink mtimes. Wire change alongside `-O`: the shared `STATUS_DIR_TIMES` frame; `PROTOCOL_VERSION` bumped **2.16.0 → 2.17.0** |
|
||||
| `--super` | Receiver attempts super-user activities | ✅ Implemented | Phase 7 Wave E: receiver-side **safe-subset + clear-refusal** privilege model, tri-state `super_mode` (auto/on/off). `--super` **permits** the receiver to attempt super-user activities — ownership application and char/block device-node creation — that are already confined fd-relative below the authorized receive root; `--no-super` **forbids** them even when the receiver is root; the default (`auto`) preserves the pre-existing **best-effort** behavior of *attempting* them (not only when already root: an unprivileged attempt is refused by the kernel and skipped per entry, matching FastSync's history). The server additionally accepts an operator-level `--no-super` veto that forces `OFF` for every connection it accepts (so it also refuses any client `--copy-as`/`--super`); the `--fake-super` owner replay and the `--write-devices` write path are gated by the same policy. **FastSync never elevates**: no `setuid`/`seteuid`/`setgid` is ever called, and `--super` never bypasses the confinement floor (`file_open_secure_parent`, `O_NOFOLLOW`, root checks) — it only permits an attempt that is already confined. `--super` does **not** imply `--numeric-ids` and never enables client-chosen ownership on its own: ownership is applied only when an explicit identity policy (`--usermap`/`--groupmap`/`--chown`/`--numeric-ids`/`--copy-as`) is also given. A non-root receiver given `--super` logs exactly one warning at activation and each confined attempt is then refused by the kernel and skipped per entry (never aborts); `--no-super` suppresses ownership, char/block `mknod`, `--write-devices` and the fake-super owner replay, while unprivileged FIFO creation is unaffected. Wire: one trailing `super_mode` int on the config frame (validated 0..2), sent **before** the `--copy-as` block (fixed order: super int, then copy-as presence int + ids); `PROTOCOL_VERSION` bumped **2.17.0 → 2.18.0**. **Documented divergence from rsync:** rsync's `--super` runs the receiver with elevated privilege; FastSync only permits a confined attempt and never elevates |
|
||||
| `--fake-super` | Store/recover privileged attrs via xattrs | ✅ Implemented | Phase 7 Wave B: full record **and replay**. The receiver writes the source `uid:gid:mode:mtime_sec:mtime_nsec` into a reserved `user.fastsync.stat` xattr on each written file (best-effort, fd-relative, format unchanged), then immediately re-applies it via `fake_super_restore_fd`: `fchown` (only where privileged — a non-root EPERM/EACCES is skipped silently, matching FastSync's identity philosophy), `fchmod`, and `futimens`. The OWNER leg is additionally skipped unless an explicit ownership identity policy (`--numeric-ids`/`--usermap`/`--groupmap`/`--chown`/`--copy-as`) is active — `--fake-super` on its own only *records* the source owner and must not act as an un-gated chown primitive — when `--no-super` forbids super-user activities (even for root), or when an active `--copy-as` is authoritative, so the recorded source owner can never override a forced `--copy-as` owner; the xattr record is still stored/replayed for a later privileged restore and mode/mtime still apply, so unprivileged `--fake-super` keeps working. The restored mode goes through the same sanitization as the normal metadata path (group/other write bits are never granted, so a recorded 0666 restores as 0644), so fake-super replay can never grant group/other-write that plain `--preserve` would refuse. Absence or a malformed record is a silent no-op, never fatal. The recording format diverges from rsync's `user.rsync.%stat%`; no cross-tool conversion is attempted. Implies metadata transmission so the source uid/gid/mode/mtime are available. Both it and `-X`/`-A` are incompatible with `-s` (chunk serialization), rejected up front |
|
||||
| `--open-noatime` | Avoid changing access time when opening files | ✅ Implemented | Sender-side policy: the sender opens source files with `O_NOATIME` (Linux) when reading them for transfer, so the open/read does NOT bump the source's on-disk access time. Degrades safely when `O_NOATIME` is unavailable (not defined) or refused (`EPERM`, since it needs `CAP_FOWNER` or file ownership): the code falls back to a normal open, so the data always transfers — only the atime-bump is skipped. It does not itself capture/preserve atime; it only avoids modifying it. **Client-only, never crosses the wire.** Exposed as `file_open_for_read()` and applied to both the buffered data path and the sendfile path |
|
||||
| `--numeric-ids` | Do not map uid/gid by name | ✅ Implemented | Ownership is applied through FastSync's opt-in identity path (see the Phase-4 identity notes below). `--numeric-ids` is a mapping-policy modifier: when applying ownership it uses the transmitted numeric uid/gid directly, skipping the name lookup. Without an ownership-affecting option it is inert (FastSync only applies ownership when the user opts in). It does not need `-M` to be parsed, but ownership is only applied when metadata (hence the source uid/gid) is actually transmitted (see the notes) |
|
||||
| `--usermap=STRING` | Map usernames | ✅ Implemented | Opt-in ownership application. rsync subset implemented: comma-separated `FROM:TO` rules evaluated in order, first match wins; `FROM`/`TO` are group/user names (resolved on the SOURCE machine at parse time), `*` (FROM matches any id / TO = the receiving process's current euid), and an `@N` or bare `N` numeric id. Rules are carried over the wire as resolved numeric id pairs; the receiver applies a matching rule (else falls back to `--chown`, `--numeric-ids`, then a best-effort name lookup) via an fd-relative `fchown`. Malformed/unresolvable specs are rejected with a clear error, never a silent no-op. Implies metadata preservation so the source uid/gid travel. Only effective when the receiver can actually change ownership (root or membership); otherwise it warns and continues |
|
||||
| `--groupmap=STRING` | Map group names | ✅ Implemented | Same rsync subset and semantics as `--usermap` but for the group (gid) side and the group databases. See the Phase-4 identity notes |
|
||||
| `--chown=USER:GROUP` | Map owner and group | ✅ Implemented | Opt-in ownership override applied receiver-side. Forms: `USER:GROUP`, `USER` (owner only), `:GROUP` (group only); a `*` for USER/GROUP means the current/root user or group as appropriate; an `@N`/bare `N` numeric id is accepted. A `:` inside a name may be escaped as `\:`. Equivalent to a trailing `*:*` usermap+groupmap rule (so an explicit `--usermap`/`--groupmap` match wins). Malformed or unresolvable specs are clear parse errors. Implies metadata preservation. Only effective when the receiver has permission to chown; otherwise it warns and continues (rsync parity) |
|
||||
| `--copy-as=USER[:GROUP]` | Perform the copy as another user/group | ✅ Implemented | Safe-subset implementation, an explicit divergence from rsync's **real identity switching**. rsync makes the receiving process actually assume USER/GROUP (setuid/setgid); FastSync's receiver is multithreaded, so a real credential drop would be unsafe and is never attempted — FastSync never calls `setuid`/`seteuid`/`setgid`. Instead the receiver FORCES the ownership of every entry it writes to `copy_as_uid`/`copy_as_gid` through the existing confined, fd-relative identity path (the same `fchown`/`fchownat` mechanism as `--chown`/`--usermap`/`--groupmap`; symlinks use `fchownat(..., AT_SYMLINK_NOFOLLOW)`, and directories — including intermediate parents created implicitly while writing a nested file — and char/block/FIFO nodes are owned no-follow too, so a directory never keeps the receiver's owner while its children get the target owner), with `--copy-as` at the **highest priority** — it beats usermap/groupmap/`--chown`/`--numeric-ids` and the best-effort name lookup. This REQUIRES a privileged (root) receiver: an unprivileged receiver REFUSES the whole transfer up front at the config handshake (`server_module_gate`, running inside `config_receive_with_validate` before the `STATUS_OK` ack) with a clear error and no file data exchanged — never a silent wrong-ownership result. A server running with an operator `--no-super` veto also refuses it, and a **daemon** refuses `--copy-as`, like every other client-chosen-ownership request (`--numeric-ids`/`--chown`/`--usermap`/`--groupmap`/`--fake-super`/explicit `--super`), unless the selected module opts in with `client owner = yes`; without that per-module opt-in a daemon must not honor an arbitrary client-selected owner (the standalone listener and SSH `--stdio` server keep honoring these for their single operator-authorized root). `--fake-super` interaction: `--copy-as` is authoritative, so the recorded source owner is never replayed over the forced target owner. If the ownership apply still fails with EPERM/EACCES (capability-restricted root, root-squash, read-only mount) the failure is logged at ERROR and the **entry is reported as failed** rather than written with the wrong owner, which fails the transfer (fail-fast) so overall success is never reported with the wrong owner. USER is resolved on the client against the user database (a name, an `@N`/bare `N` numeric id, or `*` meaning the client's current euid); when `:GROUP` is present it is resolved against the group database (`*` meaning the client's egid). **Group-default rule:** when the group is omitted FastSync uses the user's primary gid (`getpwuid(uid)->pw_gid`); a numeric id with no local passwd entry has no primary gid to look up, so `gid` falls back to `uid` (documented divergence). Malformed/empty/unresolvable specs are clear parse errors, never a silent no-op. Never elevates privileges and never bypasses the confined receive root. Implies metadata preservation (the source uid/gid must be transmitted). Wire: a new trailing config-frame block **sent after** the `--super` int (presence int, then the two int32 ids, both validated `>= 0` on receive; the ids are also rejected if they do not fit int32 at CLI parse time); `PROTOCOL_VERSION` bumped **2.17.0 → 2.18.0** |
|
||||
|
||||
**Phase-4 metadata-time notes:** `-U/--atimes`, `-N/--crtimes`,
|
||||
`-O/--omit-dir-times`, `-J/--omit-link-times`, and `--open-noatime` are new.
|
||||
They change the wire: the per-file metadata frame grows `atime_valid` +
|
||||
`atime_sec` + `atime_nsec` and `crtime_valid` + `crtime_sec` + `crtime_nsec`
|
||||
(appended after the existing mode/uid/gid/mtime fields, preserving the exact
|
||||
positions of every pre-existing field), and the config frame grows four
|
||||
booleans — `preserve_atimes`, `preserve_crtimes`, `omit_dir_times`,
|
||||
`omit_link_times` — that CROSS the wire so the receiver knows what to apply /
|
||||
suppress. `--open-noatime` is **client-only** and is never serialized (it only
|
||||
governs the sender's source reads). `PROTOCOL_VERSION` was bumped **2.11.0 →
|
||||
2.12.0** (peers must match, exactly as prior phases did).
|
||||
|
||||
**Client-vs-wire split:** `-U` and `-N` affect both the sender (capture) and the
|
||||
receiver (apply), so they and their metadata fields cross the wire;
|
||||
`-O`/`-J` are receiver-side preferences and cross as config booleans;
|
||||
`--open-noatime` is purely a client/sender open flag and stays off the wire
|
||||
(mirroring the existing convention where `ignore_errors` is client-only while
|
||||
`force_delete` crosses the wire).
|
||||
|
||||
**Phase-4 xattr/ACL notes (`-X/--xattrs`, `-A/--acls`, `--fake-super`):** these
|
||||
are new in protocol 2.13.0 and add a bounded per-file xattr block to the
|
||||
per-file metadata frame (count + each `name`/`value`, sent only when xattr
|
||||
transport is enabled, i.e. with zero overhead on unaffected runs). The config
|
||||
frame carries `preserve_xattrs`, `preserve_acls` (in the existing file-options
|
||||
block) and a trailing `fake_super` boolean — all CROSS the wire so the receiver
|
||||
knows the negotiated behavior; the derived `use_xattrs` flag is recomputed on
|
||||
the receiver. `PROTOCOL_VERSION` was bumped **2.12.0 → 2.13.0** (peers must
|
||||
match, exactly as prior phases did).
|
||||
|
||||
- **Security model (both `-X` and `-A`):** only `user.*` and the
|
||||
`system.posix_acl_access` / `system.posix_acl_default` namespaces are ever
|
||||
captured (sender) or applied (receiver). `security.*` (SELinux, capabilities,
|
||||
...), `trusted.*`, and all other `system.*` attributes are never transmitted
|
||||
or applied, so a client can never compel the receiver to set a privileged
|
||||
xattr. The receiver re-validates each incoming name against this whitelist
|
||||
even though the sender already filtered, so a malicious/compromised sender's
|
||||
`security.capability` payload is rejected outright (a clean protocol error),
|
||||
never applied.
|
||||
- **Bounds / memory safety:** per-name length ≤ 255 B, per-value ≤ 1 MiB,
|
||||
per-file count ≤ 256 names, per-file name+value total ≤ 4 MiB. Both the
|
||||
sender (during capture) and the receiver (during receive) enforce these; an
|
||||
oversized or malformed frame is rejected, never a large allocation.
|
||||
- **Confined application:** xattrs are applied with `fsetxattr` on the exact
|
||||
just-written destination file fd (before the atomic rename), never on a
|
||||
caller-controlled path; this is the same confinement as mode/time restore.
|
||||
The `--link-dest` / `-H` hard-link copy fallback (a byte copy when `link()`
|
||||
is refused) also re-applies the incoming (or, for `-H`, the first member's)
|
||||
xattrs and the `--fake-super` stat, so attributes are preserved rather than
|
||||
silently dropped when the link fails.
|
||||
- **Reserved fake-super key is receiver-only:** the `user.fastsync.stat` key is
|
||||
excluded from sender capture AND from receiver application, so it can only be
|
||||
written by the receiver's own `--fake-super` handling. A source file that
|
||||
already carries such a record is never forwarded on a plain `-X` run, so it
|
||||
cannot be spoofed to mislead a later privileged restore.
|
||||
- **`-A` requires no libacl** — ACLs travel as the `system.posix_acl_*` xattrs.
|
||||
Applying an ACL is owner-privileged: `fsetxattr` failure (e.g. non-root,
|
||||
unsupported filesystem) is logged (collapsed to one line per file) and never
|
||||
fatal.
|
||||
- **`--fake-super`**: see the row above; the reserved key is `user.fastsync.stat`
|
||||
with the documented `uid:gid:mode:mtime_sec:mtime_nsec` (mode octal) format.
|
||||
It is honest but partial — there is no replay, and it does not interoperate
|
||||
with rsync's `user.rsync.%stat%`.
|
||||
- **Chunk serialization (`-s`) incompatibility:** the per-file xattr block rides
|
||||
the streaming per-file frame, which `-s` replaces with a fixed buffer format,
|
||||
so `-X` / `-A` combined with `-s` is rejected up front on both ends (mirroring
|
||||
the existing `-H` + `-s` rejection) rather than silently dropping attributes.
|
||||
|
||||
**atime capture does not clobber the source atime:** the sender records the
|
||||
access time from the **same pre-read stat the scanner already took** (inside
|
||||
`file_metadata_create`), before any file data is read for transfer. So `-U`
|
||||
alone captures the correct atime even without `--open-noatime`. `--open-noatime`
|
||||
is orthogonal: it keeps the source's on-disk atime from being bumped by the read
|
||||
that actually ships the data (only honoured where `O_NOATIME` works; it degrades
|
||||
to a normal open otherwise, so the data always transfers).
|
||||
|
||||
**crtime handling:** `-N` captures the source birth time via `statx`/`STATX_BTIME`
|
||||
(guarded `#ifdef STATX_BTIME` on Linux) and transmits it. On the receiver, **no
|
||||
portable setter exists** (`utimensat` can only set atime/mtime), so the receiver
|
||||
deliberately does **not** apply it: it logs a debug note and continues — it never
|
||||
fails the transfer and never pretends the crtime was applied. This is the
|
||||
explicit, documented unsupported-attribute handling. On platforms without
|
||||
`statx` the flag is accepted but nothing is captured (a documented no-op).
|
||||
|
||||
**omit-dir-times / omit-link-times:** `-O` and `-J` are **real modifiers** as of
|
||||
P7 Wave D (`🔄 → ✅ Implemented`). FastSync now preserves directory mtimes
|
||||
(captured by the scanner, transmitted in trailing `STATUS_DIR_TIMES` frame(s),
|
||||
applied only after all children and the delete/publication phases) and symlink
|
||||
mtime/owner/mode (no-follow `utimensat`/`fchownat`/`fchmodat` at link creation).
|
||||
`-O` makes the receiver skip the directory-time set; `-J` makes it skip the
|
||||
symlink timestamps (ownership/mode application is unaffected and stays governed
|
||||
by the identity opt-in). Both config booleans already crossed the wire. See the
|
||||
`-O`/`-J` rows and the Wave D note below.
|
||||
|
||||
**-U/-N and -M interaction:** because FastSync carries all metadata (mode, uid,
|
||||
gid, mtime, and now atime/crtime) in one bounded payload that is only sent when
|
||||
metadata transmission is on, `-U` and `-N` imply metadata transmission (the
|
||||
times travel inside that payload). They do **not** enable ownership application,
|
||||
which remains opt-in strictly through the identity flags (`--numeric-ids` /
|
||||
`--usermap` / `--groupmap` / `--chown`).
|
||||
|
||||
**Phase-4 identity notes:** `--numeric-ids`, `--usermap`, `--groupmap`, and
|
||||
`--chown` are real. They introduce a **controlled, opt-in, privilege-gated**
|
||||
ownership-application path on the receiver: plain `-M`/`--preserve` still does
|
||||
NOT apply client-supplied ownership (FastSync's deliberate conservative
|
||||
default, byte-for-byte backward compatible); ownership is only attempted once a
|
||||
client explicitly requests an ownership-affecting option. Application goes
|
||||
through an fd-relative `fchown()` in the receiver's metadata-restore path (after
|
||||
the file is fully written, before timestamps are set), so it is confined and
|
||||
symlink-safe — never a path-based `chown`. When the receiver lacks permission
|
||||
(typically non-root, e.g. the CI `nobody` user) `EPERM`/`EACCES` is logged as a
|
||||
warning and the transfer CONTINUES with exit status success, matching rsync.
|
||||
A no-op default means existing transfers are unaffected.
|
||||
|
||||
Resolution of the destination uid/gid on the receiver: a matching
|
||||
`--usermap`/`--groupmap` rule wins; else the matching `--chown` side; else, with
|
||||
`--numeric-ids`, the transmitted numeric id is used raw (no name lookup); else a
|
||||
best-effort name lookup on the receiver's own account databases (skipped when
|
||||
the transmitted id has no name present there). `--chown` enforces the receiver
|
||||
side and is validated at parse time (malformed specs are clear errors, never a
|
||||
silent no-op).
|
||||
|
||||
Wire/version: the config frame gained `numeric_ids`, `chown_uid_set`,
|
||||
`chown_uid`, `chown_gid_set`, `chown_gid`, and the `usermap`/`groupmap` tables
|
||||
(count-delimited lists of resolved int32 FROM/TO id pairs), so
|
||||
`PROTOCOL_VERSION` was bumped **2.10.0 → 2.11.0** (peers must match). All new
|
||||
fields cross `config_send`/`config_receive` with full symmetry and are validated
|
||||
on receive (bounded map sizes below `MAX_IDENTITY_MAP`, ids `>=` the `-1`
|
||||
sentinels).
|
||||
|
||||
Documented divergences from rsync: because FastSync transmits only numeric
|
||||
uid/gid (not names) on the wire, name-based values (`--usermap`/`--groupmap`
|
||||
names, `--chown` names) are resolved to numbers at CLI parse time against the
|
||||
**client (sender) machine's** account databases; this reproduces rsync's
|
||||
semantics on a shared-account source/destination and is documented for a
|
||||
genuinely different destination. The interesting named-value subset is
|
||||
supported (`*` FROM wildcard, `*` TO = current user, `@N`/bare-`N` numerics); a
|
||||
lone-`@` "use the FROM value unchanged" rsync form is not implemented. Also
|
||||
unlike rsync, plain `-M` never applies ownership and `--usermap`/`--groupmap`/
|
||||
`--chown` each imply metadata preservation so the source uid/gid actually travel
|
||||
(the flags only take effect where ownership is being preserved/applied).
|
||||
|
||||
**Phase-4 hard-links notes:** `-H`/`--hard-links` is real and introduces a
|
||||
deduplicating wire path for files whose source entries share a filesystem inode.
|
||||
On the sender, the scanner records each distinct `(st_dev, st_ino)` encounter and
|
||||
assigns it a stable, run-local link-group id (`HardLinkTable`, mutex-guarded so a
|
||||
multi-threaded scan could share one instance). The FIRST member of a group is
|
||||
transferred normally and carries the data; each later (sibling) member is
|
||||
transmitted as a payload-less `STATUS_HARDLINK` frame carrying its destination
|
||||
path, the group id, and the first member's destination-relative wire path.
|
||||
Ordering is guaranteed by forcing the sequential scanner whenever `-H` is on
|
||||
(even under `-j`/`--threads`), so the first member is always emitted — and, on the receiver's
|
||||
single write thread, installed — before any of its siblings; the receiver is
|
||||
therefore always able to link to an already-present first member, including the
|
||||
"first member already up-to-date/skipped" case (the sibling links to or copies
|
||||
the existing file). Asymmetric existence policies are handled gracefully: under
|
||||
`--existing`, if the first member's destination is absent (so it is skipped) but
|
||||
a sibling's own destination already exists, that existing sibling is left in
|
||||
place rather than the transfer aborting on the missing first member. The receiver
|
||||
installs each sibling beneath its confined root
|
||||
as an atomic hard link (temp link + rename); when `link()` fails (cross-device,
|
||||
filesystem refuses links) it falls back to a byte-identical local copy of the
|
||||
first member, never a partial/corrupt file. `--delay-updates` stages each sibling
|
||||
as a hard link to the first member's STAGED file, so publication's renames
|
||||
preserve the shared inode; `--inplace` and `--partial` are unaffected (a sibling
|
||||
is a fresh link/copy). Because a hard link shares an inode, metadata is applied
|
||||
exactly once on the first member and never re-written through the sibling (whose
|
||||
members are byte-identical by construction), so all members agree.
|
||||
|
||||
Wire/version: `PROTOCOL_VERSION` was bumped **2.11.0 → 2.12.0** (peers must
|
||||
match). The config frame already carried the `preserve_hard_links` boolean
|
||||
(round-trips through `config_send`/`config_receive`); the only new wire element
|
||||
is the `STATUS_HARDLINK` frame described above. Incompatibilities (rejected up
|
||||
front with a distinct error on the client, and re-checked on receive): `-H` with
|
||||
`-s` chunk serialization (the chunk wire has no per-file hard-link info) and `-H`
|
||||
with `--append`/`--append-verify` (a payload-less sibling cannot be tail-resumed).
|
||||
|
||||
**Phase-4 devices notes:** `--devices`, `--specials`, `-D`, `--copy-devices`,
|
||||
and `--write-devices` are new. They change the wire: the config frame grows three
|
||||
booleans — `preserve_specials`, `copy_devices`, `write_devices` — that CROSS the
|
||||
wire (`preserve_devices` already existed), and a new `STATUS_SPECIAL` frame (used
|
||||
by `--devices`/`--specials`/`-D`) carries a special/device entry: the destination
|
||||
path, the metadata frame (whose mode's S_IFMT bits carry the node kind, requiring
|
||||
the flags to imply metadata transmission), and two int32 `rdev` major/minor
|
||||
fields. The chunk-serialized wire (`-s`) grows a matching per-file special
|
||||
marker + rdev so `--devices/--specials` also work under `-s`. `PROTOCOL_VERSION`
|
||||
was bumped **2.12.0 → 2.13.0** (peers must match, exactly as prior phases did).
|
||||
|
||||
**Privilege gating (the crux):** making a device node requires `CAP_MKNOD` (root).
|
||||
CI runs the integration suite as a NON-ROOT user (via setpriv), so `mknod` fails
|
||||
with `EPERM`. The receiver treats this as a graceful, logged *skip of the entry*
|
||||
returned as a success/skip outcome — the whole transfer NEVER aborts just because
|
||||
the environment cannot create the node. `mkfifo` (FIFOs) is unprivileged, so
|
||||
`--specials` FIFO creation is a real, assertable behavior under CI; sockets cannot
|
||||
be recreated by any standard filesystem call and are skipped with an explicit
|
||||
note. The "device actually created" integration assertions are guarded to run
|
||||
only as root. User-facing expectation: point `--devices` at devices and a
|
||||
non-root receiver will faithfully skip them while transferring everything else.
|
||||
|
||||
**Confinement & validation:** a special/device node is created with
|
||||
`mknodat`/`mkfifoat` on the parent directory opened fd-relative below the receive
|
||||
root (`file_open_secure_parent`: `O_NOFOLLOW`, no `..` components, root-checked),
|
||||
so a node can never be created outside the authorized destination root and never
|
||||
through a symlinked parent. The transmitted type is derived ONLY from the
|
||||
validated S_IFMT bits of the metadata mode (char/block/FIFO honored, socket
|
||||
skipped, regular/dir rejected as an invalid special), and the transmitted rdev is
|
||||
validated both on the wire (`file_receive_special`, `chunk_deserialize`) and at
|
||||
the creation site (`file_special_rdev_valid`): a negative, oversize, or
|
||||
non-device-carrying rdev is rejected outright (receiver aborts the frame), and a
|
||||
node is never replaced over an existing directory or unrelated entry (a matching
|
||||
existing node is left in place). `--write-devices` is the deliberately restricted
|
||||
danger path: it only ever opens an existing char/block node under the confined
|
||||
root, and every failure mode (missing, non-device, write error, EPERM) is a
|
||||
warning + skip, never a system-clobbering write or an abort.
|
||||
|
||||
**Documented divergences (honest subset):**
|
||||
- A device entry the receiver cannot create (missing `CAP_MKNOD`) is *skipped*,
|
||||
not a transfer failure — rsync under the same conditions would error.
|
||||
- `--copy-devices` copies the device's *reported size* (typically 0 for char
|
||||
devices/FIFOs) into a regular file and never reads an unbounded pseudo-device;
|
||||
this is the safe, non-hanging alternative to rsync's dd-like read.
|
||||
- `--write-devices` requires the device to already exist at the destination and
|
||||
never creates it; unsupported/inaccessible targets are skipped, not written.
|
||||
- Ownership is not applied to recreated nodes (identity `fchown` needs an fd and
|
||||
would require opening the node); permissions and mtime are applied at
|
||||
creation / via `utimensat`.
|
||||
|
||||
## 9. Symlink Handling
|
||||
|
||||
| Flag | Rsync Description | FastSync Status | Notes |
|
||||
|------|-------------------|-----------------|-------|
|
||||
| `-l`, `--links` | Copy symlinks as symlinks | ✅ Implemented | A symlink is transmitted as a real symlink: its target string crosses the wire (a new `STATUS_SYMLINK` frame / chunk entry type) and the receiver creates it with `symlinkat` beneath the receive root. This makes the previously-`-l`-included-but-targetless symlink handling complete. See the Phase-4 symlink-trust notes |
|
||||
| `-L`, `--copy-links` | Transform symlink to referent | ✅ Implemented | `copy_links` config field |
|
||||
| `--copy-unsafe-links` | Transform unsafe symlinks | ✅ Implemented | `copy_unsafe_links` config field |
|
||||
| `--safe-links` | Ignore symlinks outside tree | ✅ Implemented | `safe_links` config field |
|
||||
| `--munge-links` | Munge symlinks for safety | ✅ Implemented | Sender rewrites each transmitted symlink target with a `#SYMLINK/` marker; a target that could escape the receive root (absolute or containing `..`) is never transmitted (contained/skipped); the receiver strips the marker to restore the real target. See the Phase-4 symlink-trust notes |
|
||||
| `-k`, `--copy-dirlinks` | Transform symlink to dir | ✅ Implemented | A symlink whose referent is a directory is dereferenced and recursed as a real directory; a symlink to a regular file stays a symlink. Sender-side only. See the Phase-4 symlink-trust notes |
|
||||
| `-K`, `--keep-dirlinks` | Treat symlinked dir as dir | ✅ Implemented | On the receiver, an existing destination symlink-to-a-directory is used as that directory (followed) instead of being replaced; it is followed only when it resolves to a directory that stays beneath the receive root. See the Phase-4 symlink-trust notes |
|
||||
|
||||
**Phase-4 symlink-trust notes:** `-l/--links`, `-k/--copy-dirlinks`,
|
||||
`-K/--keep-dirlinks`, and `--munge-links` form the "symlink trust boundaries"
|
||||
row. Making all three new flags have an observable, security-sane effect
|
||||
required transmitting symlink targets, so FastSync's `-l/--links` is now real:
|
||||
a symlink-type entry carries its target on the wire (a new `STATUS_SYMLINK`
|
||||
frame for the per-file path, and a new entry type `2` in the `-s` chunk
|
||||
serializer) and the receiver creates it with `symlinkat` under an `O_NOFOLLOW`
|
||||
parent walk, never following the target. Wire changes: `STATUS_SYMLINK`,
|
||||
the chunk entry type `2`, a per-entry symlink-target string, and two new config
|
||||
booleans that CROSS the wire — `munge_links` and `keep_dirlinks`; `PROTOCOL_VERSION`
|
||||
was bumped **2.12.0 → 2.13.0** (peers must match, exactly as prior phases did).
|
||||
|
||||
**Per-flag semantics and divergences.**
|
||||
- **`-l/--links`** copies a symlink as a symlink: the scanner `readlink`s the
|
||||
target, the sender transmits it, and the receiver `symlinkat`s it. FastSync
|
||||
`-l` never preserved symlink targets before (the flag was documented partial
|
||||
and, in fact, tried to read the referent as file data); it now does, matching
|
||||
rsync. Divergences: because the receiver enforces the symlink containment
|
||||
predicate unconditionally, a plain `-l` sync **refuses to round-trip a
|
||||
legitimate absolute symlink target** (it is dropped, never created pointing
|
||||
outside the root — see the `--munge-links` note for the symmetric trust
|
||||
boundary); a relative in-root target is copied as-is. As of P7 Wave D FastSync
|
||||
also applies the symlink's own metadata with no-follow primitives
|
||||
(`utimensat`/`fchownat`/`fchmodat` with `AT_SYMLINK_NOFOLLOW`), so `-J` is a
|
||||
real omit switch rather than a no-op.
|
||||
- **`-k/--copy-dirlinks`** (sender): a symlink whose referent is a directory is
|
||||
dereferenced and recursed into as a real directory; a symlink to a regular
|
||||
file (or any non-directory) is kept as a symlink. This is rsync's `-k`. When
|
||||
`-L/--copy-links` or `--safe-links`/`--copy-unsafe-links` are active, their
|
||||
(dereference) semantics take precedence, so `-k` is subsumed exactly as in
|
||||
rsync.
|
||||
- **`-K/--keep-dirlinks`** (receiver, crosses the wire): when a directory is to
|
||||
be created (on-demand parent creation for a child write) and the destination
|
||||
path is already an existing symlink that resolves to a directory *within* the
|
||||
receive root, that symlinked directory is used (followed) instead of being
|
||||
replaced by a real directory; new entries are written beneath it. The follow
|
||||
is confined: it only happens where `realpath` of the symlink resolves to a
|
||||
still-within-root real directory, so a malicious link pointing outside the
|
||||
root is never followed. Scope: `-K` acts on the write path (parent/`mkdir`
|
||||
creation); the delete walker still never follows symlinks (a documented
|
||||
divergence for `--delete` over an existing symlinked dir). Without `-K` the
|
||||
destination symlink is not followed (the O_NOFOLLOW walk fails the write),
|
||||
which is the safe default.
|
||||
- **`--munge-links`** (sender security rewrite; crosses the wire so the receiver
|
||||
unmunges): every transmitted symlink target is prefixed with the marker
|
||||
`#SYMLINK/`; the receiver strips the marker (only when the negotiated
|
||||
`munge_links` policy is on — a plain `-l` run never strips the prefix, so a
|
||||
source symlink that genuinely begins with `#SYMLINK/` round-trips verbatim)
|
||||
and restores the exact real target. The trust boundary is **symmetric and
|
||||
enforced receiver-side**, independent of the sender: `file_symlink_at_secure`
|
||||
refuses any target that `file_symlink_target_contained` rejects (absolute
|
||||
`/...` or relative with a `..` component), and `file_save_to_disk_full`
|
||||
contains such an entry (skipped) rather than materializing it. A deliberate confinement trade-off: because the receiver
|
||||
enforces containment unconditionally, a plain `-l` (no `--munge-links`) sync
|
||||
*refuses to round-trip a legitimate absolute symlink target* — such target is
|
||||
dropped, never created pointing outside the root. This is a stricter subset of
|
||||
rsync: rsync stores munged targets on the RECEIVING side and depends on both
|
||||
ends running `--munge-links`; FastSync additionally enforces the containment
|
||||
predicate at the receiver regardless of what the sender transmitted. When no
|
||||
symlink is being transmitted (`-l`/`-k`/`-a` off) `--munge-links` has nothing
|
||||
to rewrite and is inert. -*K/`--keep-dirlinks` policy is installed per
|
||||
connection at config-accept (stable for the whole transfer, never racy under
|
||||
`-j`/`--threads`), and only ever follows an in-root symlink-to-directory.*
|
||||
|
||||
**Compatibility (byte-identical when all three are absent):** `-k`, `-K` and
|
||||
`--munge-links` are opt-in. Without them the scanner's link handling, the wire
|
||||
frames, and the receiver's writes are unchanged for every other option set, so a
|
||||
run that previously worked continues to behave identically. `-l/--links` itself
|
||||
now transmits targets (the prior behavior was broken/partial); its status moved
|
||||
`⚠️ Partial → ✅ Implemented`.
|
||||
|
||||
## 10. Sparse & Device
|
||||
|
||||
| Flag | Rsync Description | FastSync Status | Notes |
|
||||
|------|-------------------|-----------------|-------|
|
||||
| `-S`, `--sparse` | Sparse block handling | ✅ Implemented | Phase 7 Wave B: real hole preservation with no wire change. The receiver's sparse-aware writer (`write_all_sparse`, next to `write_all` in `src/shared/file.c` and `src/shared/file_store.c`) walks the in-memory file image and emits any all-zero run ≥ 4096 bytes as a hole via `lseek(SEEK_CUR)` (the pre-size `ftruncate` guarantees the offset bookkeeping and logical size), `ftruncate(size)` after the last run pins the final size even with a hole tail. Wired into both the atomic temp+rename store and `--inplace` when `sparse` is set; the non-sparse path is byte-identical to before. **Sparse wins over `--preallocate`** (posix_fallocate is skipped when sparse is set, so the holes are not re-allocated). Interplay note: under `--partial` a retained sparse temp already has the full logical size (trailing content is holes), so `--append`'s "shorter destination" resume does not re-run; the retained file is still valid and a normal re-transfer (or `-W`/delta) repairs it — documented so the combination is never surprising |
|
||||
| `--preallocate` | Allocate dest files before writing | ✅ Implemented | The receiver preallocates the destination file's full expected space before any data is written, so a transfer that would overflow disk fails fast at allocation time (a clean error, not a half-written file) and the file is laid out contiguously, avoiding fragmentation. Crosses the wire (the config frame carries a `preallocate` boolean; `PROTOCOL_VERSION` bumped **2.10.0 → 2.11.0**, peers must match) so the sender knows the receiver will preallocate and the receiver performs it. **Allocation approach:** `posix_fallocate()` is preferred because it reserves *real* disk blocks (true fail-fast on ENOSPC), falling back to plain `ftruncate()` only when the filesystem reports the allocation is unsupported (`EOPNOTSUPP`/`ENOSYS`); `ftruncate` still extends the logical size so the intent degrades gracefully. **Fallback/error semantics:** `EOPNOTSUPP`/`ENOSYS` → clean fallback to `ftruncate` (best-effort, preallocates the logical size and never fails a transfer on filesystems that lack `posix_fallocate`); a genuine allocation failure (`ENOSPC`/`EDQUOT`/`EFBIG`/…) aborts the file/receive with a distinct `preallocate failed ... transfer aborted` error — it does **not** fall back to a normal non-preallocated write, preserving the fail-fast purpose. **Size-known requirement:** preallocation only runs when the final size is already known up front (the normal regular-file case); unknown-length data is skipped (never failed). **Orthogonality:** applies uniformly across the atomic temp+rename store path, `--inplace`, `--partial`/`--partial-dir`, `--delay-updates` (the staged temp file is preallocated before data flows) and the `--link-dest` copy fallback; it neither implies nor conflicts with `-s`, `--append`, or delta. rsync-divergence: rsync signals that `--preallocate` is ignored with `--sparse`; FastSync gives **sparse precedence** — when both are set, `posix_fallocate` is skipped so the holes the sparse writer creates are not re-allocated (the `ftruncate` presize sizing stays), matching the intent of "sparse wins". See the Phase-4 preallocate notes below |
|
||||
|
||||
**Preallocate notes (Phase 4, preallocate wave):** `--preallocate` is implemented as a real receiver-side allocation of the destination file's space before data is written. It is a plain boolean config flag that crosses the wire (serialized in the config frame's selection-options block, mirroring `--inplace`/`--append`/`--force`), so the run requires matching ends: `PROTOCOL_VERSION` was bumped **2.10.0 → 2.11.0** (peers must match or the version check fails). The allocation is performed on the exact destination fd, immediately after it is opened, before any bytes are streamed; `posix_fallocate` (and the `ftruncate` fallback) leave the fd's file offset untouched, so the subsequent data write at offset 0 is unaffected and complete. Because FastSync writes each file's byte payload in one in-memory batch, the "full expected size" is exactly the known `data_size`, which is what gets preallocated. Unknown-length/streamed payloads are skipped rather than failed. A failed allocation logs a distinct `preallocate failed` error and aborts the file (the atomic temp is unlinked, the inplace target is left untrimmed) so the run fails cleanly and never silently degrades to a non-preallocated write — preserving rsync's fail-fast intent on a full disk.
|
||||
|
||||
|
||||
## 11. Checksum & Comparison
|
||||
|
||||
| Flag | Rsync Description | FastSync Status | Notes |
|
||||
|------|-------------------|-----------------|-------|
|
||||
| `--checksum` | Skip based on checksum | ✅ Implemented | With `--incremental`, compares per-file whole-file content digests to skip unchanged files. The digest algorithm is `xxh64` with seed 0 by default and is selectable via `--checksum-choice`/`--cc` (xxh64/xxhash or md5) and `--checksum-seed=NUM` (see those rows); `-c` remains compression |
|
||||
| `--checksum-choice=STR`, `--cc=STR` | Choose checksum algorithm | ✅ Implemented | Real algorithm selection for the per-file whole-file digest used by the `--incremental`/`--checksum` handshake and by the basis-dir content verification. FastSync genuinely supports `xxh64` (the default, exact xxHash64, seeded by `--checksum-seed`) and `md5` (via OpenSSL EVP); `xxhash` is accepted as rsync's spelling of xxHash64. Any other name (md4/sha1/sha256/crc32/none/…) is rejected with a clear error at parse time — never a silent no-op. `--cc` is the alias (`--cc=ALG` and space forms both parse). The algorithm id and seed cross the wire with the config frame, so the receiver hashes its on-disk old file with the SAME algorithm+seed the sender used and both agree on a match; the sender's digest and the receiver's comparison live in the per-file `STATUS_CHECK` handshake, which now carries a length-prefixed, bounded (1..16 byte) digest instead of a fixed 64-bit value, and the receiver pins the received length to the negotiated algorithm's digest length (defense-in-depth: a mismatched/malicious length only forces a safe re-transfer). Note: `md5` is a FIPS-non-approved algorithm, so under an OpenSSL build with FIPS mode enabled `--checksum-choice=md5` fails loudly rather than silently falling back. Protocol/layout: `PROTOCOL_VERSION` bumped **2.9.0 → 2.10.0** (peers must match). Defaults preserve the pre-existing behavior byte-for-byte (xxh64, seed 0). Like rsync, the choice only takes effect where a whole-file digest is actually computed (`--checksum` on, or a basis-dir flag); it does not itself enable `--checksum`. Closely-related divergence: the delta BLOCK strong checksum (§11 delta) stays xxHash32 — `--checksum-choice` selects only the whole-file digest, matching rsync where the per-block checksum is independent of the whole-file checksum choice |
|
||||
| `--compare-dest=DIR` | Compare dest files relative to DIR | ✅ Implemented | DIR is a receiver-side basis relative to the destination root (confined below it; absolute/`..`/`.` rejected, `//` collapsed and trailing `/` dropped). On the receiver's per-file check (implies `--incremental`) an exact match = same size + mtime (unless `--size-only`; `-I` disables matching) **and** equal xxHash64 of the sender's file; a match suppresses the data transfer. compare-dest never copies: it only skips a file the destination does **not** already hold (sparse destination, rsync parity), and is consulted before the normal delta/full paths. Repeatable; searched in command-line order, first match wins. Divergences: when the destination already holds a *different* version rsync deletes it but FastSync instead transfers the data (keeps the mirror complete; never deletes without `--delete`); attribute-only differences on a match are not re-applied (data is skipped so the sender never sends metadata); content is verified by xxHash64, stricter than rsync's default quick check. Sizing: FastSync's whole-file payload limit is 256 MiB on **every** transfer path (not basis-specific); rsync applies basis dirs to arbitrary sizes, so FastSync refuses a basis run whose source contains a larger file up front with a clear error before any transfer. Wire: a basis-count field is always present on the config frame (protocol 2.9.0, so clients and servers must both be 2.9.0) |
|
||||
| `--copy-dest=DIR` | Include copies of unchanged files | ✅ Implemented | Same basis rules as `--compare-dest`, but an exact match materializes a **local copy** of the DIR file into the destination (via the normal atomic temp+rename store path, so `--existing`/`--ignore-existing`/`--update`/`--backup`/`--delay-updates` all still apply) instead of transferring data. Repeatable; command-line order = priority. Content is xxHash64-verified before the copy. Divergences: a basis-hit destination keeps the basis file's own mode/uid/gid and mtime (the sender sends no metadata on a skip), so with `--size-only` its mtime can differ from the source and attribute-only differences are copied with the basis attributes rather than rsync's "copy + fix attributes". Requires `--incremental` (implied); incompatible with `-s`. Wire: protocol 2.9.0 |
|
||||
| `--link-dest=DIR` | Hardlink to files when unchanged | ✅ Implemented | Same basis rules as `--copy-dest`, but an exact match installs an atomic **hard link** to the DIR file (temp hard link + rename) so no data or disk space is used; where the link is impossible (basis on another filesystem, filesystem refuses links) it falls back cleanly to a byte-identical local copy, never a corrupt/partial file. `--delay-updates` stages the link and publishes by rename, so the final entry stays a real hard link. Repeatable (searched in command-line order, first match wins). Content is xxHash64-verified before linking. Divergences and caveats: an already up-to-date destination file is not re-linked to a basis file (only files that would otherwise be written are linked); a link keeps the basis inode's own mode/uid/gid and mtime — metadata is never written through the shared inode (that would mutate the basis file), so a later `--inplace` run that rewrites such a destination path **will mutate the basis snapshot** through the shared inode (use `--copy-dest` when the destination must stay independently writable); with `--size-only` the linked mtime can differ from the source; a `--remove-source-files` source satisfied by a basis dir is treated as skipped and therefore **retained** (never removed); basis dirs are excluded from `--delete`. Requires `--incremental` (implied); incompatible with `-s`. Wire: protocol 2.9.0 |
|
||||
| `-y`, `--fuzzy`, `--no-fuzzy` | Find similar file for basis | ✅ Implemented | `-y/--fuzzy` is a pure bandwidth optimization on the existing receiver-driven delta path: when a file must be transferred and the destination holds no usable content at the exact path (file absent, or the destination file is outside the delta engine's size bounds), the receiver searches the SAME destination directory for an existing regular file whose basename is similar to the incoming name and uses it as the delta basis, so the sender transmits only the differences instead of the whole file. The output is always byte-exact regardless of which (or whether any) basis is chosen. Decision location: the receiver performs the candidate search inside `receive_incremental_check` and sends the normal `STATUS_DELTA_SIGNATURE`; the sender never learns the basis was a different file, so no new frame type or sender logic was needed — only the config frame grew a `fuzzy` boolean, so `PROTOCOL_VERSION` was bumped **2.8.0 → 2.9.0** (peers must match). Similarity heuristic (deterministic, simpler than rsync's deliberately-fuzzy matching, and documented precisely): candidates are the target's sibling entries in its destination directory, opened `O_NOFOLLOW`/`AT_SYMLINK_NOFOLLOW` under the confined root (symlinks never followed; nothing outside the destination root is ever read or hashed); dotfiles, directories, the target's own name, and the `.fastsync-stage`/temp scratch names are excluded; like the ordinary delta path, the block signature the receiver transmits is derived from on-disk content it may not otherwise send, so a negotiated `--fuzzy` run exposes the destination's sibling files (at block granularity) to the sender as a known-plaintext oracle — the same information class as the normal delta handshake over the file being replaced; the size gate is the delta engine's own bounds (both files ≥ 16 KiB, ≤ `--delta-max`, ratio ≤ 10×) rather than rsync's ~1.5× size window; the name gate is a Levenshtein edit distance between the basenames accepted only when ≤ half the length of the longer basename; the single best candidate (smallest distance, tie-break size closest to the incoming file then lexicographically smaller basename) is read; the directory scan is capped at 4096 entries so a pathological directory cannot stall a transfer. When fuzzy applies: only to files the receiver would otherwise send whole — the destination's own file is always preferred as the delta basis when it exists and fits the delta size bounds, so fuzzy does NOT replace an existing-but-different destination basis; FastSync's 10× delta size-ratio bound means an existing destination file that is too far away in size still lets the fuzzy search run. When no similar candidate exists the transfer falls back to the normal whole-file transfer. rsync-divergence note: rsync's own matching uses a fuzzy name/size rule set; FastSync implements the closest safe deterministic approximation above. Because FastSync's delta machinery is off by default (rsync's is on), `--fuzzy` implies `--incremental` + `--delta` (unless `--whole-file`/`-W` or an explicit `--no-delta` switched delta off, in which case fuzzy is inert — matching rsync where `--whole-file` makes fuzzy irrelevant). Unlike the basis-dir options, `--fuzzy` honors an explicit `--no-incremental` (it does not force the handshake back on); an explicit `--no-incremental` also suppresses the delta implication so no invalid `--delta requires --incremental` config results. `--no-fuzzy` negates it. All surrounding semantics are untouched: a fuzzy-reconstructed file is stored as a normal file, so `--remove-source-files`, itemize/`-i`, `--stats`, `--backup`, `--delay-updates`, `--existing`/`--ignore-existing`/`--update` behave exactly as for a whole-file transfer (the fuzzy delta does not skip the file) |
|
||||
|
||||
## 12. Compression
|
||||
|
||||
| Flag | Rsync Description | FastSync Status | Notes |
|
||||
|------|-------------------|-----------------|-------|
|
||||
| `-z`, `--compress` | Compress file data | ✅ Implemented | Always uses zstd (rsync supports multiple algorithms — a documented divergence, selectable via `--compress-choice`). Phase 7 Wave A: `-z` is now the compression short form; `-c` is rsync's `--checksum` |
|
||||
| `--compress-choice=STR`, `--zc=STR` | Choose compression algorithm | ✅ Implemented | FastSync supports `zstd` and `none` |
|
||||
| `--compress-level=NUM`, `--zl=NUM` | Set compression level | ✅ Implemented | 1-22, default 5 |
|
||||
| `--compress-threads=NUM` | Set compression threads | ✅ Implemented | `compression_threads` config field (client-only; does not cross the wire). Sets the number of worker threads used by the zstd compression pool to NUM (1..64; 0/garbage/oversized rejected up front). Accepted in both `--compress-threads=NUM` and two-argument `--compress-threads NUM` forms. Composes with `-z`/compression; under the `-j`/`--threads` multithreaded pipeline it parallelizes compressed chunk encoding. See test_tcp.py `-z --compress-threads=2` and test_client_cli.c |
|
||||
| `--skip-compress=LIST` | Skip compress for suffixes | ✅ Implemented | Comma-separated, case-insensitive suffix list; empty list skips none; incompatible with FastSync chunk serialization (`-s`) |
|
||||
|
||||
## 13. Connectivity
|
||||
|
||||
| Flag | Rsync Description | FastSync Status | Notes |
|
||||
|------|-------------------|-----------------|-------|
|
||||
| `-e`, `--rsh=COMMAND` | Remote shell to use | ✅ Implemented | `-e`/`--rsh` (and `--rsh=COMMAND`) select the remote-shell program used to build the SSH child argv, overriding the default `ssh`. The command is whitespace-split into the leading argv words so rsync's `-e "ssh -p 2222"` works; the standard `-o` family, an optional `-p` port, `user@host` and the quoted remote command (`fastsync-server --stdio`) follow. Stored in the `rsh_command` config field. **Client-only, never crosses the wire** (it is a launch concern, not a handshake property) |
|
||||
| `--rsync-path=PROGRAM` | rsync binary on remote | ✅ Implemented | Alias for `--fastsync-server-path`: both write the `fastsync_server_path` config field used as the remote-side server program (always quoted as one remote-shell word), which CROSSES the wire as before. Kept separate from `--rsh`, which names the local connecting program |
|
||||
| `--port=PORT` | Alternate daemon port | ✅ Implemented | rsync's daemon-port flag maps to the client-side `server_port` config field: a client connects to a TCP/TLS server (incl. `host::module/path` daemon destinations) with `--server-port`, and the `fastsync-server --daemon` listener's port is taken from its config's `port` key (default 873) or overridden by `--dparam port=` / `-p` |
|
||||
| `--sockopts=OPTIONS` | Custom TCP options | ✅ Implemented | Comma-separated allowlist of `OPT=VAL` applied via `setsockopt` after `socket()` before `connect()`/`bind()`. Only `TCP_NODELAY`, `SO_KEEPALIVE`, `SO_REUSEADDR` (0/1) and `SO_RCVBUF`/`SO_SNDBUF` (byte count) are accepted; an unknown option name or a bad value is rejected up front, never silently ignored. A value is required for every option (`OPT=VAL`; a bare name is an error). Applied to the outgoing TCP and TLS client socket; absent by default. `SockOptEntry`/`sockopts` config fields. Local socket concern: never crosses the wire |
|
||||
| `--blocking-io` | Use blocking I/O for remote shell | ✅ Implemented | With `--blocking-io` the SSH-transport socketpair socket is left without `SO_RCVTIMEO`/`SO_SNDTIMEO`, so the transfer blocks naturally; by default it gets the same read/write timeout as the TCP transport (see `--timeout`). `blocking_io` config bool. **Client-only, never crosses the wire** |
|
||||
| `--outbuf=N\|L\|B` | Set output buffering | ✅ Implemented | `N` (none/unbuffered) → `_IONBF`, `L` (line) → `_IOLBF`, `B` (block, the default) → `_IOFBF` via `setvbuf` on stdout and stderr. Garbage values are rejected. `outbuf` config field (`OutbufMode`). **Client-only, never crosses the wire** |
|
||||
| `--address=ADDRESS` | Bind address for outgoing socket | ✅ Implemented | Binds the outgoing client socket to a local source address before `connect()` (resolved with the same `-4`/`-6` family hints as the destination). Local socket concern: never crosses the wire |
|
||||
| `-4`, `--ipv4` | Prefer IPv4 | ✅ Implemented | Forces `AF_INET` in the `getaddrinfo` hints for client destination/source resolution and the server bind (see the Phase 5, Wave B note). Mutually exclusive with `-6` |
|
||||
| `-6`, `--ipv6` | Prefer IPv6 | ✅ Implemented | Forces `AF_INET6` in the `getaddrinfo` hints for client destination/source resolution and the server bind. Mutually exclusive with `-4` |
|
||||
| `--remote-option=OPT`, `-M` | Send an option only to the remote side | ✅ Implemented | Each value is appended to the remote server invocation over SSH as an individually single-quote-escaped shell word in `ssh_build_remote_command()`. Values are validated (non-empty, no control characters) and shell metacharacters cannot break out of the quoting (`;`, `&`, `|`, <code>`</code>, `$`, `(`, `)`, quotes are neutralized), so a value cannot inject an arbitrary remote command and a subsequent `--` on the client line cannot be turned into one. The options never cross the binary config frame. Phase 7 Wave A: the short `-M` form is now available (as `-M OPT` and `-M=OPT`), matching rsync; metadata mode moved to long-only `--preserve` |
|
||||
|
||||
## 14. Daemon Mode
|
||||
|
||||
| Flag | Rsync Description | FastSync Status | Notes |
|
||||
|------|-------------------|-----------------|-------|
|
||||
| `--daemon` | Run as rsync daemon | ✅ Implemented | Wave A: a real persistent listener. `fastsync-server --daemon --config FILE` (plus `--no-detach` to stay foreground; without it the listener detaches to the background after binding) reads a FastSync-native module config file and serves each connection confined to the requested module's `path` root (never a client-chosen root; every client-chosen-ownership/super-user request (`--numeric-ids`/`--chown`/`--usermap`/`--groupmap`/`--fake-super`/`--copy-as`/explicit `--super`) is refused unless the module opts in with `client owner = yes`, and the operator `--no-super` veto is honored). TCP/TLS via the existing `--tls` stack; plaintext still requires `--allow-unauthenticated` (same secure default as the standalone server). Client destinations use rsync's `host::module/path` form. Wire/protocol: the config frame gained a trailing daemon-module string and `PROTOCOL_VERSION` was bumped **2.14.0 → 2.15.0** (see the Daemon Mode notes below). Daemon mode is built in FastSync's own protocol/config grammar, not rsync's SMB/daemon option encoding |
|
||||
| `--config=FILE` | Alternate rsyncd.conf file | ✅ Implemented | Wave A: selects the daemon config file. Default when omitted (in `--daemon` mode): `~/.config/fastsync/fastsyncd.conf` if it exists, else `/etc/fastsyncd.conf`. The grammar is FastSync-native (documented in the Daemon Mode notes below) and strictly rejects unknown keys so a typo can never silently change what a module serves; requires `--daemon` |
|
||||
| `--dparam=OVERRIDE` | Override global daemon config | ✅ Implemented | Wave A: overrides one global scalar from the command line (`--dparam port=8734` and `--dparam=KEY=VALUE` both work). Limited to the global scalar keys the grammar defines (`port`, `motd file`, `address`); keys are case-insensitive and unknown keys/invalid values are rejected. Requires `--daemon` |
|
||||
| `--no-detach` | Don't detach from parent | ✅ Implemented | Wave A: with `--daemon`, keeps the listener in the foreground (what integration tests use). Without it the daemonizes (fork/setsid, stdio redirected to /dev/null) after the listening socket is bound. Requires `--daemon` |
|
||||
| `--password-file=FILE` | Read daemon password from file | ✅ Implemented | A7 daemon auth. Client: `--password-file` supplies `user:password` for a `host::module/path` destination (the username is taken from this file, so `user@host::module` stays rejected); the literal password is held client-side only for the SCRAM handshake and wiped at teardown. Server (`fastsync-server --daemon --password-file FILE`): the salted-PBKDF2 verifier store that modules with `auth users` are verified against. **Neither the password nor any replayable bearer value crosses the wire or is stored server-side** — the store holds a per-user salt plus derived keys, and the daemon proves the secret with a per-connection nonce challenge. The file must be private to its owner: both the client and server verify the exact inode they read (open-then-`fstat`, so the check cannot be raced) and refuse a `--password-file`/`--early-input` that is not owned by the current user or grants any group/other permission bit (mode 0600), mirroring the TLS private-key check. A process-substitution pipe (`--early-input <(vault ...)`) is still accepted when it satisfies those checks. See the Daemon Mode notes below for the file formats and the plaintext/TLS caveat |
|
||||
| `--early-input=FILE` | Use FILE for daemon early exec | ✅ Implemented | Server-only (requires `--daemon`): a second credential-store file, same new-format grammar as `--password-file`, read before the listener accepts connections (a secrets-manager / process-substitution source). Its entries layer over `--password-file`: byte-identical verifiers dedupe, a conflicting verifier for the same user is a startup error. A daemon whose modules declare `auth users` must be given at least one of the two, or it refuses to start (fail closed) |
|
||||
| `--hash-credentials=FILE`, `--iterations N` | Hash a plaintext credential file | ✅ Implemented | Server-only offline tool (A7): reads the `user:password` lines of FILE (same owner-only 0600 check) and prints one new-format store line per entry to stdout, then exits. `--iterations` sets the PBKDF2 work factor (default 600000, range 100000–10000000). Dependency-free and does not run a listener. Use its output as `--password-file` for `--daemon`. There is no auto-upgrade: a legacy store line is hard-rejected by the loader and must be regenerated |
|
||||
|
||||
**Daemon Mode notes (Wave A protocol 2.15.0; A7 auth protocol 2.19.0; MOTD no bump):** FastSync daemon mode is supported in FastSync's own protocol/config grammar, not rsync's SMB/daemon option encoding.
|
||||
|
||||
- **Config grammar** (`fastsyncd.conf`): line-based; an implicit global section first, then `[module]` sections. Keys are case-insensitive, values are trimmed and may be wrapped in one layer of double quotes (`path = "/srv/my dir"`). `#` and `;` at the start of a line (after leading whitespace) are full-line comments; inline comments and `\` continuations are not supported. Lines are bounded (4096 chars). Global keys: `port` (default 873), `motd file` (the daemon sends its bounded, escaped content to a client after the module gate/auth accepts, unless the client passes `--no-motd`), `address` (optional bind address). Module keys: `path` (required; the daemon-side authorized root for that module), `read only` (yes/no/true/false/1/0, default no), `client owner` (yes/no/true/false/1/0, default no; opts the module into client-chosen ownership — see below), `auth users` (comma list). **Unknown keys and malformed lines are parse-and-reject errors** (never silently ignored), so a typo cannot change what a module serves.
|
||||
- **Module selection & confinement:** the client requests a module with an rsync-style `host::module[/path]` destination. The module name crosses the wire as a trailing string on the config frame (bumping `PROTOCOL_VERSION` 2.14.0 → 2.15.0; the bump is required because the config-frame layout changed and the strict same-version handshake is what prevents a peer from desynchronizing on the new trailing field). The daemon looks the module up in ITS OWN config and uses the module's `path` as the authorized root through the exact same `configure_authorization` confinement the standalone server applies to `--destination-root` (`file_open_secure_parent`, `has_path_traversal`, `path_is_within`); the client never supplies the root, every client-chosen-ownership/super-user request is refused unless the module declares `client owner = yes` (the daemon's per-module opt-in, see below), and the operator `--no-super` veto forces super-user activities off for every daemon connection. The client's `/path` part is relative inside the module and is rejected if absolute or if it contains `..`. Unknown modules are refused before any data moves (the run fails cleanly at the config handshake). An absolute destination and a module request against a non-daemon server are also refused.
|
||||
- **`client owner` (client-chosen-ownership opt-in):** by default a daemon module refuses every request that would let the client pick an owner or ask for super-user activities — `--numeric-ids`, `--chown`, `--usermap`/`--groupmap`, `--fake-super`, `--copy-as`, and an explicit `--super` — at the config handshake (before `STATUS_OK`), because a daemon has no per-module opt-in for client-chosen ownership and any anonymous client could otherwise force arbitrary owner ids inside the module root. `client owner = yes` opts a single module in, allowing those requests within that module's root (the standalone listener and the SSH `--stdio` server always honor them for their single operator-authorized root). Without the opt-in the daemon also forces super-user **device** activity off for that connection — char/block device-node creation (`--devices`) and `--write-devices` — even under the default `AUTO` mode, so a non-opted module can never be made to `mknod` or write a raw device; those entries are skipped (not refused) so an ordinary `-a` push still succeeds without device nodes. The opt-in does **not** lift the privilege requirement: `--copy-as` still needs a root receiver, and the operator `--no-super` veto still forces super-user activities off for every connection. The daemon logs a prominent startup warning for each `client owner = yes` module so the operator's deliberate choice is visible.
|
||||
- **`read only` safe default:** every network transfer FastSync currently supports is a push that writes under the module root, so a `read only` module refuses the connection (clear server log "module is read only"; the client exits non-zero, nothing is transferred). A future pull/list operation can be opened up when it exists; the knob is already stored.
|
||||
- **`auth users` (A7 SCRAM-SHA-256 authentication):** a module that declares `auth users` requires the client to present credentials. The config frame carries ONLY the username; the daemon answers an auth-required module with `STATUS_AUTH_CHALLENGE` (PBKDF2 iteration count, 16-byte salt, 32-byte server nonce), the client answers with `STATUS_AUTH_RESPONSE` (fresh 32-byte client nonce + a 32-byte ClientProof), and the daemon accepts only when the proof verifies **and** the username is **on the module's `auth users` list** and has a store entry, replying `STATUS_AUTH_OK` with a 32-byte ServerSignature the client verifies before proceeding. Verification is constant-time over fixed 32-byte keys (the compare runs even for a miss), username membership uses a constant-time full-length scan, and an unknown/off-list user still receives a challenge and runs the same math against a dummy verifier: a deterministic per-username salt (`HMAC-SHA256(store dummy key, username)`), the store-wide uniform iteration count and dummy keys. Re-probing the same unknown username therefore yields an identical salt and iteration count while a different username yields a different salt, so there is no user-enumeration or timing oracle. The daemon logs the username but **never the password, proof or keys**. A module WITHOUT `auth users` stays open (legitimate rsync configuration); credentials sent to such a module are ignored. Read-only is orthogonal: even a correctly authenticated push to a `read only` module is still refused (all FastSync network transfers write). Fail-closed policy: a daemon whose config declares `auth users` on any module refuses to start unless a credential store was given (`--password-file` and/or `--early-input`); a missing or empty store is never silently treated as "open". A failed handshake (missing credentials, unknown/off-list user, wrong proof or malformed data) yields a single generic `STATUS_AUTH_FAILED` and the daemon closes before any data moves. The dummy key is persisted in an owner-only `<store_path>.dummykey` sidecar (auto-created on first load, mode 0600) so the dummy salt stays stable across daemon restarts, closing the restart-gated enumeration channel. The sidecar is secret material and must be protected like the credential store (owner-only 0600, included with the store in backups and rotation). It must be preserved across restarts for that guarantee; if it cannot be created (a process-substitution/FIFO store path such as `/dev/fd/N`, a read-only filesystem, a missing directory, or a create/write/fsync/link/fchmod failure), the daemon logs a warning and uses a transient per-run key, so unknown-user challenges change across restarts and the cross-restart guarantee does not hold for that deployment. One residual is accepted: the store iteration count is observable pre-auth by design, since the miss path must match a hit. **Transport policy (hardening A7-3/S1):** an auth-required module accepts credentials only when either (a) the connection is an encrypted, verified TLS connection whose client certificate matches `--client-cn`, or (b) the connection is plaintext from a loopback TCP peer **and** the operator explicitly passed `--allow-unauthenticated`. A remote plaintext peer, and a loopback plaintext peer without that flag, are refused at the config gate before any challenge is sent; `--allow-unauthenticated` never permits remote plaintext auth (remote peers still require verified TLS). Daemon modules are a `--daemon`-only feature — the SSH `--stdio` path never loads a daemon config and is not an auth transport for them. Because the loopback allowance trusts whichever peer the kernel reports as `127.0.0.1`, it assumes nothing relays remote connections to the daemon: a local TCP forwarder or TLS-terminating proxy in front of an auth-module listener makes remote clients appear as loopback and bypasses the mutual-TLS identity check, so do not front an auth-module listener with such a relay.
|
||||
- **Credential store format:** server `--password-file`/`--early-input` files are line-based `user:$fastsync$1$pbkdf2-sha256$<iters>$<salt_b64>$<stored_key_b64>$<server_key_b64>`, one per line (standard base64; 16-byte salt, 32-byte keys; `iters` in `[100000, 10000000]`, default 600000). Every entry in the resulting store must agree on `iters` (a store whose entries disagree, or where a layered `--early-input` disagrees with `--password-file`, is rejected). Generate lines with `fastsync-server --hash-credentials FILE [--iterations N]`; the emitted lines are secret material, so redirect them to an owner-only (mode 0600) file (the tool warns on stderr if stdout is a group/other-accessible regular file). Blank lines and lines starting with `#`/`;` are comments; the parser is strict (a malformed line fails the whole load, so a typo can never let a different set of users in). **The legacy `user:SHA256HEX` form is hard-rejected** with an actionable "legacy" error; there is no auto-upgrade, so a replayable bearer digest can never be loaded by a 2.19.0 daemon. The client `--password-file` holds `user:password` on its first meaningful line (the literal password, used only for the handshake then burned); keep both files readable only by their owner (mode 0600). Per-username wire length is bounded (256 chars) and every decoded salt/key length is validated. Loading the store also maintains an owner-only `<store_path>.dummykey` sidecar (auto-created, mode 0600, exactly 32 bytes) holding the store-wide dummy key that shapes unknown-user challenges; persist it across daemon restarts so those challenges stay stable, and treat a sidecar with the wrong owner, a mode other than exactly 0600, the wrong size or the wrong type as a fatal load error (fail closed). If the sidecar cannot be created (e.g. a process-substitution store path such as `/dev/fd/N`, a read-only filesystem, a missing directory, or a create/write/fsync/link/fchmod failure), the daemon logs a warning and uses a transient per-run key, so the cross-restart stability guarantee does not hold there.
|
||||
- **Plaintext caveat:** an auth-required module is refused, **before any challenge is sent**, unless the connection is encrypted and verified TLS whose client certificate matches the server's `--client-cn`, or it is plaintext from a loopback TCP peer **and** the operator passed `--allow-unauthenticated`. A remote plaintext peer, and a loopback plaintext peer without that flag, never receive a challenge, and `--allow-unauthenticated` never permits remote plaintext auth (remote peers still require verified TLS). On the loopback plaintext transport that remains permitted, a local sniffer could still read the challenge and response and mount an **offline dictionary attack** against a weak password, so use `--tls` for any real deployment. `--client-cn` matches the certificate CN only (not a subjectAltName), which is acceptable for a private CA. Clients sending daemon credentials with `--password-file` to a non-loopback daemon must use `--tls`; the client rejects such a destination before any network I/O. Unlike the old challenge-less exchange there is **no replay**: the proof is bound to the fresh per-connection server nonce, so a captured `STATUS_AUTH_RESPONSE` cannot be reused on another connection (an integration test proxies the daemon and proves this). TLS client-CN (`--client-cn`) is an independent transport identity check and composes with password auth; because `--tls` already mandates `--client-cn`, a TLS auth connection always verifies the client CN, so both checks necessarily apply together on such a connection.
|
||||
- **Wire/protocol:** the config-frame auth block is now `[int present][str_redacted username]` (the old digest field is gone), and the frame stream gains the challenge/response (`STATUS_AUTH_CHALLENGE` → `STATUS_AUTH_RESPONSE` → `STATUS_AUTH_OK`/`STATUS_AUTH_FAILED`) between the config frame and the `STATUS_OK` ack. Both are wire-layout changes, so `PROTOCOL_VERSION` is bumped **2.18.0 → 2.19.0** (see the A7 note in `src/shared/config.h`); the strict same-version handshake keeps a 2.19 client and a 2.18 server from desynchronizing.
|
||||
- **Client side:** `host::module/path` selects the TCP transport and connects to `--server-port`; `host:path` stays the SSH transport; plain paths stay local TCP. The daemon username comes from `--password-file` (first `user:password` line), and `--password-file` without a `host::module/path` destination is a client error (fail fast). A `user@host::module` form is rejected with a pointer to `--password-file`. The client's plaintext password is wiped from memory (`config_burn_auth`) at transfer teardown.
|
||||
- **MOTD (Wave C):** a daemon configured with a global `motd file` sends that file's content as the first server→client string frame after the config-frame STATUS_OK ack (rsync sends the MOTD as the first thing from the server at the start of a daemon connection). Only the daemon listener path (`host::module`) gets a MOTD; the `--stdio` SSH path never sends or reads one. The server reads the file bounded to 4096 bytes and treats an absent/unreadable file as "no MOTD" (an empty frame, never an error). The exchange is server→client only and does **not** bump `PROTOCOL_VERSION`: every 2.15.0 daemon client reads the frame after the ack, so sender and receiver stay in lockstep (see the Wave C note in `src/shared/config.h`). `--no-motd` is the client-side suppression switch: the client still reads (consumes) the frame to keep the stream in sync but does not display it. The MOTD is printed to stdout with control bytes (ESC included) escaped octal-style while newlines/tabs are preserved, so a hostile server cannot inject terminal escape sequences.
|
||||
- **Merge note:** the Wave A module bump (2.15.0) and the MOTD wave did not bump the version, but the A7 auth redesign is a genuine wire-layout change and owns the 2.18.0 → 2.19.0 bump (see the A7 note in `src/shared/config.h`).
|
||||
|
||||
## 15. Safety & Security
|
||||
|
||||
| Flag | Rsync Description | FastSync Status | Notes |
|
||||
|------|-------------------|-----------------|-------|
|
||||
| Path escape detection | Ensure files stay within root | ✅ Implemented | `has_path_traversal()` + realpath |
|
||||
| Symlink-safe delete | Skip symlinks in delete walk | ✅ Implemented | `delete_extras_walk()` |
|
||||
| Protocol version check | Verify compatible versions | ✅ Implemented | `config_receive()` |
|
||||
| Max data/string/chunk sizes | Prevent OOM attacks | ✅ Implemented | Per-message limits |
|
||||
| Per-connection memory limit | 1GB per connection | ✅ Implemented | `MAX_CONNECTION_MEMORY` |
|
||||
| `--max-alloc=SIZE` | Limit a single memory allocation | ✅ Implemented | Caps the largest single allocation; binary units, default 1G |
|
||||
| `--trust-sender` | Trust remote sender's file list | ✅ Implemented | Long-form-only, receiver-local policy that never crosses the wire. The receiver skips its redundant up-front re-validation of the incoming file list (empty/`..` path rejection and the escaping-symlink-target containment), trusting the sender instead of double-checking (fewer checks, faster, potentially unsafe, matching rsync). Off by default. The low-level fd-relative confinement primitives (`file_open_secure_parent`, the O_NOFOLLOW parent walk, leaf/destination confinement) are deliberately KEPT even under `--trust-sender`, so a hostile sender still cannot write or link outside the authorized root (see Phase-5 notes below) |
|
||||
| `--old-args` | Disable modern arg protection | ✅ Implemented | SSH-only; accepted for CLI compatibility but is now a **documented no-op**: FastSync always single-quote-escapes the remote server path and each `--remote-option` value (`ssh_build_remote_command`), so a metacharacter-bearing `--rsync-path` can never be interpreted by the remote shell. The flag no longer disables that quoting (the old raw-construction behavior was an injection foot-gun and is removed); the safety-relevant behavior is identical either way |
|
||||
| `--ignore-missing-args` | Ignore missing source args | ✅ Implemented | FastSync has a single source-root argument (which always exists), so the "explicitly requested source arguments" are the `--files-from` entries and the flags only ever apply there (inert without `--files-from`, like `-R`). Without the flag a listed-but-missing entry stays a hard pre-transfer error (nothing is transferred). With it each missing entry is skipped: nothing is sent for it, it never enters the keep-set, and the run succeeds for the rest — an all-missing non-empty list succeeds transferring nothing, matching rsync. `--dirs` + `--files-from` missing entries are skipped the same way. Every skipped entry is logged and a per-run warning names the count, so the handling is never a silent no-op. Divergences: an EMPTY `--files-from` file stays a hard error in every mode (no argument was requested at all; rsync likewise reports "no source files specified"); missing-arg skipping only applies to the pre-transfer list validation, so an entry that is present at preflight and vanishes mid-transfer still fails (matching rsync, whose flag "does not affect subsequent vanished-file errors"); `--no-ignore-missing-args` is not a supported negation |
|
||||
| `--delete-missing-args` | Delete missing source args | ✅ Implemented | Implies `--ignore-missing-args` (order-independent) and additionally removes each missing entry's destination mirror receiver-side. The mirror is computed exactly like a present sibling's wire path: the bare relative entry under `-R`, otherwise the full source-mirror path below the destination root. rsync parity, verified against the man page: it does **not** imply `--delete` generally and is "independent of any other type of delete processing" — unrelated destination extras are untouched unless `--delete` is also present. Composition with `--delete` + timing: the exact-path deletions commit with the manifest, early for `--delete-before`/`--delete-during`, else only after a fully-successful transfer (delete-after/commit). A non-empty directory mirror is removed only when `--force` or `--delete` is in effect (otherwise it is left with a warning and the run continues, like rsync); an absent mirror is a no-op. An explicitly listed missing arg is a user request, not an excluded file: its deletion is never blocked by the filter-exclusion protection of excluded destination mirrors (a mirror sitting inside a filter-excluded directory is still removed). Safety/policy: gated by the server `--allow-delete` policy like `--delete`; the request paths cross the wire only in the delete-manifest frame and are confined by the same receiver validation as the keep-set (non-empty, relative, traversal-free, bounded by the per-section/per-frame manifest caps); the `--delay-updates` staging directory and basis snapshots are protected exactly as in the extras walker. Divergence: the missing-args deletions are not counted toward `--max-delete` (they are explicit per-path requests, not discovered extras). See the Phase-3 wire note below for the `PROTOCOL_VERSION` bump |
|
||||
|
||||
## 16. Batch Operations
|
||||
|
||||
| Flag | Rsync Description | FastSync Status | Notes |
|
||||
|------|-------------------|-----------------|-------|
|
||||
| `--write-batch=FILE` | Write batched update to file | ✅ Implemented | Phase-6 residual-batch (client-only): runs the normal live transfer AND additionally emits a self-contained single-file batch of the whole source tree. The batch is a magic/format-version header followed by length-prefixed `chunk_serialize` blobs (full file images), replayable byte-identically by `--read-batch` on another machine with no source/server. `--write-batch` drives the single-threaded transfer path (the multithreaded path consumes the config before the separate batch scan pass). See the Phase-6 batch note below |
|
||||
| `--only-write-batch=FILE` | Write batch without updating dest | ✅ Implemented | Phase-6 residual-batch: emits the self-contained batch FILE only — NO destination update, NO server connection. Requires a source (scans it and serializes the full tree to FILE). Same single-file format as `--write-batch`, so the file is re-appliable via `--read-batch=FILE DEST`. See the Phase-6 batch note below |
|
||||
| `--read-batch=FILE` | Read batched update from file | ✅ Implemented | Phase-6 residual-batch: applies a previously written batch FILE locally to the destination. NO source and NO server — positional args are the destination only. Reads the magic/version header, then length-prefixed records, `chunk_deserialize`, and applies each via the confined `file_save_to_disk_full` path (same O_NOFOLLOW / `..`-rejection / root-confinement as the network receiver, so an attacker-controlled batch cannot escape the destination root). Malformed/truncated/oversized/traversal records are rejected cleanly. See the Phase-6 batch note below |
|
||||
|
||||
## 17. Advanced
|
||||
|
||||
| Flag | Rsync Description | FastSync Status | Notes |
|
||||
|------|-------------------|-----------------|-------|
|
||||
| `--stop-after=MINS` | Stop after N minutes | ✅ Implemented | Client-only sender stop deadline (Phase 6): computing `--stop-after=MINS` (a positive minute count; 0/negative/garbage rejected) and `--stop-at=TIME` (`HH:MM`, `HH:MM:SS`, or `now+N[smhd]`; a past time stops immediately). The transfer stops ELEGANTLY at the next chunk boundary: everything already fully sent is kept and applied, the run returns 0, and --delete (late/delete-after timing) does NOT wipe the destination — when the scan is cut short the partial keep-set manifest is suppressed with a warning (the delete walk is skipped rather than acting on an incomplete keep-set, so unscanned source mirrors survive). `--delete-before`/`--delete-during` still run their complete pre-scan (which ignores the deadline). Local client-only fields: never serialized into the wire config frame, so no PROTOCOL_VERSION bump. `--stop-after` uses CLOCK_MONOTONIC; `--stop-at` uses the wall clock. Works single-threaded and under `-j`/`--threads` (multithreaded). Divergence: rsync computes `--stop-after` from the run start; FastSync likewise. When both are given, the earlier of the two deadlines wins (checked per iteration). See the Phase-6 stop notes below |
|
||||
| `--stop-at=TIME` | Stop at specified time | ✅ Implemented | Same feature as `--stop-after` (deadline transfer stop), absolute wall-clock form (`HH:MM[:SS]` or `now+N[smhd]`). See the row above and the Phase-6 stop notes |
|
||||
| `--fsync` | Fsync every written file before publication | ✅ Implemented | |
|
||||
| `--protocol=NUM` | Force older protocol version | ✅ Implemented | Forces the wire protocol version for this transfer. FastSync has exactly ONE wire format (`PROTOCOL_VERSION`, currently 2.19.0) with no downgrade/backward-compat code paths, so `--protocol=2.19.0` is accepted (it sets the version claim the client sends, which the server already requires to match exactly) and **every other value is rejected up front** with a clear error before any connection — it does not and cannot speak an older or virtual wire format. Divergence from rsync (which negotiates a range and downgrades to an integer 0..31): FastSync's honest contract is force-to-the-one-supported-value; a genuine downgrade would require a per-version compatibility layer that does not exist. Client-only; the server-side exact-match check is unchanged. `--protocol=2.18.0`/`2.18`/`2.17.0`/`2.16.0`/`2.15.0`/`216`/`31`/garbage are all rejected. See the Phase-6 protocol note below |
|
||||
| `--iconv=CONVERT_SPEC` | Charset conversion | ✅ Implemented | Charset conversion of FILE NAMES (not content) at the protocol boundary via iconv(3): `--iconv=LOCAL[,REMOTE]` — the sender converts each local filename LOCAL→REMOTE before transmitting, and the receiver converts each wire filename REMOTE→LOCAL before creating/writing. The full CONVERT_SPEC is serialized into the config frame as a new trailing string field so the peer knows the wire charset; **PROTOCOL_VERSION bumped 2.15.0 → 2.16.0**. `LOCAL[,REMOTE]` parse: single charset ⇒ LOCAL==REMOTE (identity both ways); garbage rejected up front. Validation probes BOTH directions (a spec that only opens one way is refused, as is a NUL-emitting target charset like utf-16/utf-32/ucs-2, since filenames cannot contain NUL). An unrepresentable name (EILSEQ/EINVAL) fails that path cleanly with a logged `--iconv: cannot convert file name ...` and is never written mangled/truncated. Conversion is applied at EVERY wire-path site (regular/MKDIR/hardlink path+target/symlink path+target/SPECIAL, the delete manifest, the incremental-check path, and the `-s`/`chunk_serialize` embedded blob path), on both client and server (`--iconv` is also a server/daemon option). Zero overhead when unset. See the Phase-6 iconv notes below |
|
||||
| `--checksum-seed=NUM` | Set checksum seed | ✅ Implemented | Sets the seed for FastSync's whole-file xxHash64 digest (full 64-bit seed) and for the delta path's per-block xxHash32 strong checksum (low 32 bits of the seed). An explicit seed deterministically changes every computed digest on BOTH endpoints (sender and receiver share the seed via the config frame, protocol 2.10.0), so identical runs with the same seed skip the same files and a changed seed changes the digests — the explicit-seed path that makes xxHash comparisons deterministic. `--checksum-choice=md5` has no seed and ignores it (documented). The value is a strict decimal 0..2⁶⁴-1 (blank, signed, or non-numeric values are rejected). Like rsync, a seed only matters where a digest is actually computed (`--checksum` or a basis-dir run, or a delta transfer); it does not by itself enable `--checksum`/`--delta`. Divergence from rsync: the default is seed 0, and FastSync never randomizes the seed (rsync uses a random per-transfer seed when `--checksum-seed` is unset); FastSync's unset default therefore reproduces its historical byte-for-byte behavior |
|
||||
| `--secluded-args`, `-s` | Use protocol to send args | ⛔ Impossible/Divergence | Accepted for CLI compatibility (including the rsync short `-s`, Phase 7 Wave A) but a documented **no-op / divergence**. rsync's `-s` protects arguments from shell expansion by shipping them over the protocol; FastSync never passes remote arguments through a shell expansion boundary in the first place — its SSH transport builds the remote argv as **single-quote-escaped shell words** (`ssh_build_remote_command`), so the injection/leak that `-s` guards against does not exist and there is nothing to "seclude". Implementing a true arg-send protocol would mean replacing the argv-based SSH launch with an in-band argument channel, a large redesign of the transport that buys no security here. Chunk serialization remains the long-only `--chunk-serialization`. |
|
||||
| `--no-OPTION` | Turn off implied option | ✅ Supported | Supported boolean FastSync options and archive-implied options; unsafe or value-taking options are rejected. |
|
||||
|
||||
---
|
||||
|
||||
## Implementation Difficulty Plan
|
||||
|
||||
**Phase 5 notes (remote-option wave):** `--remote-option=OPT` (long form only) and `--trust-sender` landed here.
|
||||
- `--remote-option` is CLIENT-only and never serialized into the binary config frame. On the SSH transport the client forwards each value to the remote server by appending it to the remote command line in `ssh_build_remote_command()`, after ` --stdio`, as an individually single-quoted shell word (`'...'` with `'\''` for embedded quotes). Values are validated at CLI parse time (non-empty; no ASCII control characters) and rejected otherwise, and a non-conforming value is refused again in the command builder, so shell metacharacters (`;`, `&`, `|`, backticks, `$()`, quotes) can never break out of the quoting to inject an unrelated remote command — including after a client-side `--` separator, whose arguments are never forwarded anyway. Because the remote options affect the *remote server invocation*, not the transmitted config, the wire frame layout is unchanged, but `PROTOCOL_VERSION` was bumped **2.13.0 → 2.14.0** as the Phase-5 lockstep release marker (a 2.14 client against a 2.13 server fails the version check cleanly rather than the old server rejecting an unfamiliar forwarded argv later). Divergence: rsync's short `-M` form of `--remote-option` was intentionally NOT implemented at that time because `-M` was FastSync metadata mode; **Phase 7 Wave A later freed `-M` for `--remote-option` and moved metadata to long-only `--preserve`** (see the Sending Options table).
|
||||
- `--trust-sender` is a receiver-local policy: it never crosses the wire (the sender's value is never serialized, so a wire peer can never enable it). On the receiving process it skips the up-front re-validation of the incoming file list (empty/`..` path rejection and the escaping-symlink-target containment), trusting the sender's list instead of double-checking — fewer checks, faster, and potentially unsafe, matching rsync. It is OFF by default (`config.trust_sender`). As a deliberate safety floor, the low-level fd-relative confinement primitives are NOT disabled: `file_open_secure_parent()` (O_NOFOLLOW walk, `..` rejection, root containment) and leaf/destination confinement still hold, so even under `--trust-sender` a hostile sender cannot write or create a symlink outside the authorized root — the relaxation only removes the redundant list-layer double-checks, never the root-confinement guarantees.
|
||||
|
||||
The estimates below cover the currently unimplemented features in this document. They assume one engineer familiar with the codebase, include implementation and focused tests, and exclude production rollout time. A feature should not be marked implemented until its behavior is tested in both local and SSH/TCP paths where applicable.
|
||||
|
||||
> **Note:** This plan is a superset snapshot written while several of the listed features were still outstanding. The Summary matrix above is the authoritative record of what is already shipped (for example quiet/info/debug output, `--existing`, `--remove-source-files`, `-h`, and `--size-only` are now implemented on `dev`). Treat the phases as sequencing guidance for the work that remains unimplemented.
|
||||
|
||||
| Effort | Typical duration | Meaning |
|
||||
|--------|------------------|---------|
|
||||
| XS | 0.5-1 day | CLI alias or a local formatting/validation change |
|
||||
| S | 1-3 days | Isolated behavior with little or no protocol change |
|
||||
| M | 3-7 days | Cross-cutting client, server, or scanner behavior |
|
||||
| L | 1-3 weeks | Protocol, filesystem, privilege, or compatibility work |
|
||||
| XL | 3+ weeks | New transfer mode, daemon subsystem, or broad interoperability effort |
|
||||
|
||||
### Phase 1: Low-Risk CLI and Local Behavior
|
||||
|
||||
These are the best first changes because they require limited wire-format work and can be tested with existing transfer fixtures.
|
||||
|
||||
| Features | Effort | Implementation plan |
|
||||
|----------|--------|--------------------|
|
||||
| `--quiet`, `-q`; `--human-readable`, `-h`; `--8-bit-output`, `-8`; `--stderr=MODE`; `--info=FLAGS`; `--debug=FLAGS` | S | Extend logging and output formatting without changing transferred data. |
|
||||
| `--no-OPTION`; `--old-args`; `--secluded-args`, `-s` | M | Add option implication/negation and safely serialize or protect remote arguments. `-s` currently has FastSync-specific semantics and needs a compatibility decision. |
|
||||
| `-P`; `--del`; `--old-dirs`, `--old-d`; `--cc`; `--zc`; `--zl` | XS | Add aliases and composed behaviors after the underlying options exist. |
|
||||
| `--whole-file`, `-W`; `--ignore-times`, `-I`; `--size-only`; `--modify-window`, `-@`; `--update`, `-u` | S | Extend the existing incremental comparison decision. |
|
||||
| `--existing`; `--ignore-existing`; `--remove-source-files` | S | Add scanner/receiver eligibility checks and remove successfully synchronized source files. |
|
||||
| `--executability`, `-E`; `--chmod=CHMOD` | M | Apply permission transformations safely while preserving current metadata behavior. |
|
||||
| `--skip-compress=LIST`; `--compress-threads=NUM` | S | Make compression selection configurable and validate the thread setting against zstd behavior. |
|
||||
| `--max-alloc=SIZE`; `--fsync` | S | Reuse existing allocation limits and add an explicit durability step after file writes. |
|
||||
|
||||
### Phase 2: Filesystem Selection and Update Semantics
|
||||
|
||||
These features are moderate because they affect traversal, temporary files, manifests, or the receiver's update policy.
|
||||
|
||||
| Features | Effort | Implementation plan |
|
||||
|----------|--------|--------------------|
|
||||
| `--one-file-system`, `-x` | M | Track the source device during scanner traversal and skip mount-point crossings. |
|
||||
| `--relative`, `-R`; `--no-implied-dirs`; `--dirs`, `-d`; `--mkpath` | M | Extend path-list construction and destination directory creation while preserving traversal safety. |
|
||||
| `--temp-dir`, `-T` | M | Separate temporary-file placement from FastSync's timeout alias and define collision, permissions, and cleanup rules. |
|
||||
| `--delay-updates` | L | Stage all successful updates and publish them at completion, including crash and cancellation cleanup. |
|
||||
| `--files-from=FILE`; `--from0`, `-0`; `--filter=RULE`, `-f`; `-F`; `--cvs-exclude`, `-C` | L | Build a complete filter/parser layer and integrate it with scanner pruning, manifests, and delete behavior. `-f` conflicts with FastSync sendfile mode. |
|
||||
| `--list-only`; `--itemize-changes`, `-i`; `--out-format=FORMAT`; `--log-file-format=FMT` | M | Add a structured change-event model so output modes share one source of truth. |
|
||||
|
||||
### Phase 3: Deletion, Comparison, and Delta Compatibility
|
||||
|
||||
These features require careful interaction with manifests, incremental checks, backups, and the existing delta protocol.
|
||||
|
||||
| Features | Effort | Implementation plan |
|
||||
|----------|--------|--------------------|
|
||||
| `--delete-during`; `--delete-before`; `--delete-after`; `--delete-delay`; `--del` | L | Add deletion timing to the transfer state machine and ensure failures cannot remove files unexpectedly. |
|
||||
| `--delete-excluded`; `--max-delete=NUM`; `--ignore-errors`; `--force`; `--prune-empty-dirs`, `-m` | M | Extend delete walks with policy limits, error handling, empty-directory pruning, and the `-m` short-flag conflict. |
|
||||
| `--ignore-missing-args`; `--delete-missing-args` | M | Distinguish missing source arguments from traversal errors and apply explicit deletion policy. |
|
||||
| `--compare-dest=DIR`; `--copy-dest=DIR`; `--link-dest=DIR` | L | Add alternate basis roots and hard-link handling, including metadata and cross-filesystem failures. |
|
||||
| `--fuzzy`, `-y`; `--no-fuzzy` | L | Index candidate files and select a safe similar basis without making transfer time unbounded. |
|
||||
| `--append`; `--append-verify` | M | Negotiate file length and verify the retained prefix before resuming. |
|
||||
| `--checksum-choice=STR`, `--cc`; `--checksum-seed=NUM` | M | Negotiate checksum algorithms/seeds and preserve compatibility with existing xxHash checks. |
|
||||
|
||||
### Phase 4: Metadata, Links, and Devices
|
||||
|
||||
These features are platform-sensitive and need Linux permission, ACL, xattr, and special-file integration tests.
|
||||
|
||||
| Features | Effort | Implementation plan |
|
||||
|----------|--------|--------------------|
|
||||
| `--numeric-ids`; `--usermap=STRING`; `--groupmap=STRING`; `--chown=USER:GROUP` | L | Define identity mapping, privilege failures, and wire representation before applying ownership. |
|
||||
| `--open-noatime`; `--atimes`, `-U`; `--crtimes`, `-N`; `--omit-dir-times`, `-O`; `--omit-link-times`, `-J` | L | Extend metadata capture/apply with platform capability checks and explicit unsupported-attribute handling. |
|
||||
| `--acls`, `-A`; `--xattrs`, `-X`; `--fake-super` | XL | Add portable serialization, size limits, privilege behavior, and security tests for ACL/xattr data. |
|
||||
| `--hard-links`, `-H` | L | Preserve inode relationships across the file list and coordinate hard-link creation order. |
|
||||
| `--munge-links`; `--copy-dirlinks`, `-k`; `--keep-dirlinks`, `-K` | L | Define symlink trust boundaries and receiver-side directory/link collision behavior. |
|
||||
| `--devices`; `--specials`; `-D`; `--copy-devices`; `--write-devices` | XL | Add privileged special-file handling with strict type, path, and authorization checks. |
|
||||
| `--super`; `--copy-as=USER[:GROUP]` | XL | Requires a deliberate privilege model, identity switching, and refusal paths; do not implement by blindly elevating the process. |
|
||||
| `--preallocate` | S | Use platform allocation APIs before writes and fall back cleanly when unsupported. |
|
||||
|
||||
### Phase 5: Connectivity and Daemon Compatibility
|
||||
|
||||
These options affect process startup, authentication, sockets, and remote execution. They should follow the filesystem and protocol work rather than being added as parser-only flags.
|
||||
|
||||
| Features | Effort | Implementation plan |
|
||||
|----------|--------|--------------------|
|
||||
| `--rsh=COMMAND`, `-e`; `--rsync-path=PROGRAM`; `--blocking-io`; `--outbuf=N\|L\|B` | M | ✅ Wave A implemented (see the Connectivity table above). SSH argv construction is generalized: `-e`/`--rsh` replaces the hardcoded `ssh` program (whitespace-split, so `-e "ssh -p 2222"` works), `--rsync-path` aliases the existing `fastsync_server_path`, `--blocking-io` drops the SSH socket timeouts, and `--outbuf` maps N/L/B onto `setvbuf`. All four are client-only launch concerns and never cross the wire. |
|
||||
| `--address=ADDRESS`; `--ipv4`, `-4`; `--ipv6`, `-6`; `--sockopts=OPTIONS`; `--port=PORT` daemon semantics | M | Add explicit socket-family/bind configuration and validate it independently for TCP client and daemon modes. |
|
||||
|
||||
**Phase 5, Wave B (socket/bind) shipping note:** `--sockopts` adds a strict allowlisted `OPT=VAL` socket-option layer applied with correct per-option value types; `--address` binds the outgoing client socket to a local source address; `-4`/`-6` pin the address family via `getaddrinfo` hints on both the client connect and the server bind; and the server bind now honors `--address` plus `-4`/`-6` (falling back to the historical IPv4 `INADDR_ANY` when none are given). All of these are local socket concerns and none cross the wire config frame (only `--port` maps to `server_port`).
|
||||
| `--remote-option=OPT`, `-M`; `--trust-sender` | L | Add authenticated remote-option/config negotiation and reject unsafe sender-controlled values. `-M` conflicts with FastSync metadata mode. |
|
||||
| `--daemon`; `--config=FILE`; `--dparam=OVERRIDE`; `--no-detach`; `--password-file=FILE`; `--early-input=FILE`; `--no-motd` | XL | Implement a real daemon lifecycle, module configuration, authentication, privilege separation, and process management. |
|
||||
|
||||
**Phase 5, Wave A (rsh/ssh) shipping note:** the SSH transport no longer hardcodes `ssh`. `-e`/`--rsh=COMMAND` selects the remote-shell program (whitespace-split into the leading child argv words), `--rsync-path=PROGRAM` aliases `--fastsync-server-path`, `--blocking-io` removes the SSH-socketpair `SO_RCVTIMEO`/`SO_SNDTIMEO` timeouts (by default they now match the TCP transport so a wedged shell cannot hang forever), and `--outbuf=N|L|B` maps onto `setvbuf` (`_IONBF`/`_IOLBF`/`_IOFBF`, garbage rejected). All four are client-only launch concerns and never cross the wire.
|
||||
|
||||
**Phase 5, Wave C (remote-option/trust-sender) shipping note (PROTOCOL 2.13.0 → 2.14.0):** `--remote-option=OPT` (long form only; the short `-M` is intentionally left as FastSync metadata mode — documented divergence) appends each validated value to the remote server invocation over SSH as an individually single-quote-escaped shell word, so shell metacharacters cannot break out and a `--` can never be turned into injection; options never cross the binary config frame. `--trust-sender` is a receiver-local policy (never serialized, so a wire peer can't enable it): when requested on the server (via `--remote-option=--trust-sender`), it removes only the redundant receiver/save-layer path re-checking; the low-level floor (`file_open_secure_parent`'s `..` rejection, the O_NOFOLLOW parent walk, leaf/destination confinement) stays enforced. Off by default. The wire config-frame layout is unchanged; the bump reflects that a 2.14 sender composing remote options requires a 2.14 receiver to honor them.
|
||||
|
||||
### Phase 6: Batch, Encoding, and Protocol Interoperability
|
||||
|
||||
These are the hardest compatibility items because they require durable formats or behavior that must interoperate with rsync itself.
|
||||
|
||||
| Features | Effort | Implementation plan |
|
||||
|----------|--------|--------------------|
|
||||
| `--write-batch=FILE`; `--only-write-batch=FILE`; `--read-batch=FILE` | XL | ✅ Implemented (see the Batch Operations table and Phase-6 batch note below): a versioned self-contained single-file residual-batch format, persisted via the existing chunk codec, with replay, corruption, and partial-application safety tests |
|
||||
| `--protocol=NUM` | XL | ✅ Implemented (see the Advanced table and Phase-6 protocol note below): protocol-version forcing without weakening current validation; FastSync's single lockstep wire format means only the current `PROTOCOL_VERSION` is accepted, and everything else is rejected up-front |
|
||||
| `--iconv=CONVERT_SPEC` | L | ✅ Implemented (see the Advanced table and Phase-6 iconv notes below): filename charset conversion at the wire boundary with expansion/overflow safety and invalid-sequence test coverage |
|
||||
| `--stop-after=MINS`; `--stop-at=TIME` | M | ✅ Implemented (see the Advanced table and Phase-6 stop notes below): deadline propagation and safe early stop with --delete safety |
|
||||
| `--early-input=FILE`; `--password-file=FILE` | M | Securely read startup credentials/input with permission checks and no secret disclosure in logs. |
|
||||
|
||||
**Phase 6, Wave A (stop deadline) shipping note:** `--stop-after=MINS` and `--stop-at=TIME` are client-only sender stop deadlines. `--stop-after` takes a positive minute count (0/negative/garbage rejected); `--stop-at` takes `HH:MM`, `HH:MM:SS`, or `now+N[smhd]` (a past time stops immediately, a garbage spec is rejected at parse time). The deadline is computed once at the start of the transfer (CLOCK_MONOTONIC for `--stop-after`, wall clock via `time()` for `--stop-at`) and checked at every chunk boundary in both the single-threaded `send_files` loop and the multithreaded `send_chunks_multithreaded` path, and inside the scanner loops so a busy scan itself stops. When it fires, the transfer stops ELEGANTLY: the in-flight chunk completes, the existing completion tail runs (summary, `disconnect`), and the run returns 0 — exactly like rsync's clean early stop. Because the deadline is client-only and never crosses the wire config frame, no PROTOCOL_VERSION bump is required. The safety-critical interaction is with `--delete`: FastSync streams while scanning, so a deadline can cut the source scan short and yield a PARTIAL keep-set manifest; committing that would make the receiver delete destination mirrors of source files not yet scanned. So the sender tracks `scan_stopped_early` and, when it is true on the late/delete-after (`--delete`/`--delete-after`/`--delete-delay`) path, SUPPRESSES the keep-set manifest (logs a warning) so no deletion happens from an incomplete set — this is the safe direction (preserves data; the delete simply does not run). `--delete-before`/`--delete-during` are unaffected: their complete pre-scan runs before any data and ignores the deadline (a stop can be exceeded by that pre-scan). Under `-j`/`--threads` the stop is symmetric and the scanner thread's still-in-progress manifest appends can never race the tail because the tail does not read the manifest on the early-stop path.
|
||||
|
||||
**Phase 6, Wave B (iconv) shipping note (PROTOCOL 2.15.0 → 2.16.0):** `--iconv=LOCAL[,REMOTE]` converts file NAMES at the wire boundary (never content). The full CONVERT_SPEC is serialized into the config frame as a new trailing string field (empty→NULL canonicalized), so both ends share the same wire charset interpretation; this required the PROTOCOL bump because the frame is a strict ordered sequence and a peer that does not parse the new trailing field would desynchronize. Each end derives LOCAL (its own charset) and REMOTE (the wire charset): the sender opens LOCAL→REMOTE and converts every transmitted filename; the receiver opens REMOTE→LOCAL and converts every received filename before creating/writing. Conversion is applied at every wire-path site (regular/MKDIR/hardlink path+target/symlink path+target/SPECIAL, the delete manifest keep/protected/missing entries, the incremental-check path, and the embedded `-s`/chunk-blob path). A name it cannot convert (EILSEQ/EINVAL) is failed cleanly with a logged `--iconv: cannot convert file name ...` and is never written truncated/mangled. Validation probes both directions up front (both the sender local→remote and the receiver remote→local, and, for a server/daemon with its own `--iconv`, the client-REMOTE→server-LOCAL pair) so an unusable spec is rejected before the connection rather than mid-transfer, and NUL-emitting target charsets (utf-16/utf-32/ucs-2) are refused because filenames cannot contain NUL. Divergence documented upstream: the receiver does NOT half-swap; the wire charset always comes from the sender's REMOTE half, so a server whose local charset differs from the client's LOCAL must declare it with its own `--iconv`. Conversion is process-global and runs on a single thread per process (sender thread / receiver-loop thread), initialized before worker threads start and freed after they join.
|
||||
|
||||
**Phase 6, Wave C (protocol-version) shipping note (no PROTOCOL_VERSION change):** `--protocol=NUM` lets the client force the wire protocol version for a transfer. FastSync's protocol is a single lockstep format: the config frame is a strict ordered sequence and the server requires the client's version string to equal `PROTOCOL_VERSION` exactly (`config_receive_with_validate`, src/shared/config.c) — there are no older-format code paths and no downgrade/negotiation machinery, so a lower/higher/virtual version can never be spoken. The honest contract is therefore: `--protocol=2.19.0` (the current `PROTOCOL_VERSION`, as of the A7 auth redesign) is accepted and stored into the client's `version` claim (which `config_send` already transmits), and every other value — `2.18.0`, `2.18`, `2.17.0`, `2.16.0`, `2.15.0`, `3.0.0`, rsync-integer spellings like `216`/`31`, garbage, empty — is rejected up front in `validate_config()` before any connection, with a clear error that FastSync supports only its current wire protocol and cannot speak an older or virtual one. Implementation is client-only: a server-side `--protocol` is intentionally not added because the server has no negotiation (it only enforces exact match), and it could only ever be the current version. This preserves (and slightly tightens) existing validation: the client now also refuses to launch with a version it cannot actually speak, rather than only the server rejecting it later. A genuine downgrade would require a per-version compatibility layer for every frame/feature added since (append 2.10, preallocate 2.11, hardlinks 2.12, devices/specials/symlink-trust/xattr 2.13, remote-option 2.14, daemon module/auth 2.15, iconv 2.16, dir/symlink times 2.17, privilege flags --super/--copy-as 2.18, SCRAM daemon auth 2.19) and is intentionally out of scope — documented divergences from rsync's integer-negotiated downgrade remain.
|
||||
|
||||
**Phase-1/2 selection-and-update status correction (docs):** `-I/--ignore-times`, `--size-only`, `-@/--modify-window`, `--existing`, `--ignore-existing`, `-u/--update`, `-W/--whole-file`, and `--compress-threads` were previously listed as not-implemented in this document but are in fact fully implemented and tested on `dev`. This pass corrects the matrix to match the code. The realistic model of these is that FastSync is a *sender-driven* whole-tree copy, so the size+mtime quick-check and all three receiver-policy skips (`--existing`, `--ignore-existing`, `-u`) are evaluated against the **destination** on the receiver side, and their booleans cross the wire in the config frame. `-I`/`--size-only`/`--modify-window` modify the `--incremental` per-file `STATUS_CHECK` handshake's match predicate (`-I` disables the mtime leg and forces transfer; `--size-only` drops only the mtime leg; `--modify-window` adds tolerance to `metadata_mtime_matches`); they require `--incremental` (or a basis dir) to have a handshake to affect, mirroring how they only matter where a quick-check exists in rsync. `--existing`/`--ignore-existing`/`-u` are receiver write-time policies (skipping the write / newer-destination guard) applied across the regular-file, `--delay-updates`-staged, hardlink-sibling, and special/device paths; `-u` implies `-M` metadata and uses a second-then-nanosecond strict `>` newer check; both correctly influence `--remove-source-files` (a skipped source is not removed). `-W/--whole-file` disables block-level delta (opt-in via `--delta`), folded into the wire `use_delta` so no protocol bump was needed, and makes `--fuzzy` inert; `--append`/`--append-verify` are rejected with `-W`. `--compress-threads=NUM` (1..64, client-only, never crosses the wire) sizes the zstd compression worker pool. No code was changed by this correction; the implementation had landed in earlier merge waves (feat/ignore-times, feat/ignore-existing via the newer `file_to_disk_secure_no_replace`/`linkat EEXIST` path, feat/size-only, feat/modify-window, feat/whole-file, feat/update, compression-threads).
|
||||
|
||||
**Phase 6, Wave D (batch) shipping note (no PROTOCOL_VERSION change):** FastSync batch mode is a **client-only, self-contained "residual batch"**: a single file `MAGIC "FSTRESBATCH" + format version 1 + metadata flag`, followed by length-prefixed `chunk_serialize` blobs that store full file images (regular files, dirs, symlinks, specials). It is NOT a raw capture of the live wire, because FastSync's protocol is per-file interactive (`STATUS_CHECK`/`STATUS_DELTA_SIGNATURE`/`STATUS_APPEND` handshake), so a raw sender-stream tee is not deterministically replayable against an arbitrary destination. Storing full residuals via the existing, fuzz-tested chunk codec makes `--read-batch` replay byte-identically by construction. `--write-batch=FILE` runs the normal live transfer AND emits the batch from a separate deterministic scan pass; `--only-write-batch=FILE` emits the batch only (no destination, no server); `--read-batch=FILE DEST` applies it locally (no source, no server; DEST is the only positional arg). Because batch is a local driver concern, it never crosses the wire: no new config-frame field and no `PROTOCOL_VERSION` bump (mirroring `--stop-after`/`--protocol`/`--compress-threads`). The READ side is hardened against untrusted/attacker-controlled batch files: magic+version validated before any record, per-record length bounds checked before allocation (64 MB cap), clean-EOF-after-prefix and truncated/oversized records rejected, and every applied path goes through the same confined `file_save_to_disk_full` machinery as the network receiver (O_NOFOLLOW fd-walk, `..`-rejection, root confinement — a malicious `../` or absolute/symlink path cannot escape the destination root; this was security-reviewed and valgrind/ASan-clean). Divergences from rsync: (1) the batch carries the FULL residual (complete file images) rather than rsync's update-only delta stream — always byte-correct but larger; (2) per-file data is capped at the chunk codec's ~64 MB (`BATCH_MAX_RECORD`), so very large files may be refused by the batch writer with a clean error (never a corrupt/truncated batch); (3) hard-links and xattr/ACL blocks are not represented by `chunk_serialize`, so `-H`/`-X`/`-A` are out of scope for batch; (4) there is no companion `.sh`/`.rsync_argvs` — the batch is invoked directly (`fastsync --read-batch=FILE DEST`, `--only-write-batch=FILE SOURCE`); (5) `--write-batch` drives the single-threaded transfer path. Integration/`-M` note: metadata is captured in the batch when `-M` is used and persisted in the header so it applies consistently regardless of the reading process's own `-M`.
|
||||
|
||||
### Phase 7: CLI-Namespace Parity, Filesystem/Output Completion, and Privilege (Final)
|
||||
|
||||
These are the last compatibility items and the closing phase toward rsync flag parity. Per the project decision: every rsync flag (short **and** long) that is *possible* gets real rsync-parity behavior; anything physically impossible becomes an explicit **Impossible/Divergence** status (accepted for CLI compatibility, safely inert, with coverage tests proving that); and the two privilege flags (`--super`, `--copy-as`) adopt the deliberately-scoped **safe-subset + clear-refusal** model rather than blind elevation. The remaining `⚠️ Partial`, `🔄 Compatibility No-op`, `🔀 Alt Arg`, and `❌ Not Implemented` rows in the Summary are this phase's scope. All Wave A renames are **client-side only** (the wire config fields `use_compression`/`use_metadata`/`use_sendfile`/`use_chunk_serialization` are unchanged), so they require **no `PROTOCOL_VERSION` bump**.
|
||||
|
||||
**Wave A — CLI namespace parity (rename colliding FastSync short flags) — ✅ implemented.** This freed the short letters rsync needs and made the three `🔀 Alt Arg` rows real. `-c`→`--checksum`, `-m`→`--prune-empty-dirs`, `-M`→`--remote-option`, `-f`→`--filter`, `-s`→`--secluded-args`, `-p`→`--perms`, `-T`→`--temp-dir`, `-a`/`--archive`→real `-rlptgoD`. FastSync's own flags moved to long-form-only or new shorts: `-j`/`--threads` (multithreading), `--preserve` (metadata), `--sendfile`, `--chunk-serialization`, `--timeout`, `--ssh-port`. The server's independent little CLI keeps `-p` as its port. All client-side, no wire change, no `PROTOCOL_VERSION` bump. Unit tests 37/37, full integration 400 passed, cppcheck and clang-format clean. Known Wave-A limitation: `--no-perms`/`--no-compress`-style negation of the newly-aliased shorts is not wired into the negatable set (only the long-form `--preserve`/`--compress`/`--no-links` negations exist); `--archive --no-perms` is consequently not supported yet — a minor deviation from rsync, acceptable for Wave A.
|
||||
|
||||
| FastSync flag today | rsync wants that name | Proposed rename |
|
||||
|---------------------|----------------------|-----------------|
|
||||
| `-c` / `--compress` | `-c` = `--checksum` | compression is already aliased as `-z`/`--compress` (rsync parity!) → drop the `-c` short, keep `--compress`/`-z` |
|
||||
| `-m` / `--multithreading` | `-m` = `--prune-empty-dirs` | → `-j` / `--threads` |
|
||||
| `-M` / `--preserve` | `-M` = `--remote-option` | → `--preserve` (long-only) |
|
||||
| `-f` / `--sendfile` | `-f` = `--filter` | → `--sendfile` (long-only) |
|
||||
| `-s` / `--chunk-serialization` | `-s` = `--secluded-args`/`--protect-args` | → `--chunk-serialization` (long-only) |
|
||||
| `-p` (SSH port) | `-p` = `--perms` | → `--port` (long-only; `--server-port` already exists) |
|
||||
| `-T` / `--timeout` | `-T` = `--temp-dir` | → `--timeout` (long-only) |
|
||||
| `-a` / `--archive` (= `-c -m -M`) | `-a` = `-rlptgoD` | → becomes **real rsync `-a`** after the renames |
|
||||
|
||||
**Wave B — Output & filesystem completion (✅ implemented).** `-S`/`--sparse` (`⚠️→✅`): real hole preservation — a sparse-aware writer (`write_all_sparse`) skips all-zero runs ≥ 4096 bytes with `lseek(SEEK_CUR)` and `ftruncate`s the final size, wired into both the atomic temp+rename store and `--inplace` receiver-side with **no wire change** (the full file image is already in memory; the ftruncate presize is kept). `-P` (`⚠️→✅`): interrupted-write retention — on a save failure after data reached the temp fd, `--partial` now renames the already-written temp to the destination path (best-effort; falls through to the normal unlink on failure, never retains when `--partial` is off) so a later `--append`/`--append-verify` run can resume. `--block-size=SIZE` (`⚠️→✅`): promoted after verification — `--block-size` is now an alias for `--delta-block`, both set `config->delta_block_size`, which the delta engine already honored end-to-end (`delta_signature_create_seeded` + `delta_apply`); out-of-range values keep the default. `--fake-super` (`⚠️→✅`): added `fake_super_restore_fd` to parse and re-apply the recorded `user.fastsync.stat` record fd-relative (fchown best-effort/non-root skipped, fchmod, futimens); a save under `--fake-super` now re-applies the recorded attrs instead of only recording them, with the recording format unchanged. `--stderr=client` (`⚠️→⛔ Impossible/Divergence`): FastSync has no rsync client-message channel, and `client` is rejected at CLI parse — the rejection is the documented behavior (unit-tested). `-N`/`--crtimes` (`⚠️→⛔ Impossible/Divergence`): birth-times cannot be set by any portable fs call (`utimensat` sets only atime/mtime); capture/transmit stays, setting is impossible, the flag is accepted and safely inert. Review-hardening (post-eval): fake-super replay applies the mode through the same sanitization as the normal metadata path (group/other write bits are never granted); `--sparse` takes precedence over `--preallocate` (posix_fallocate skipped so holes survive); `--partial` retention is disabled for `--no_replace` (ignore/existing) and only marks a write-attempt after the actual write begins; `--block-size=SIZE`/`--delta-block=SIZE` inline forms are accepted.
|
||||
|
||||
**Wave C — Devices & special files (finalize statuses + tests) (✅ implemented).** The four special-file rows are finalized with coverage tests. `--devices`, `--copy-devices`, and `--write-devices` are **✅ Implemented**, each with a documented, safety-driven divergence: device-node creation is privilege-gated, so a receiver without `CAP_MKNOD` skips that entry with a warning (a per-entry skip, never a transfer failure); `--copy-devices` copies a device/FIFO's reported size into an ordinary regular file (a size-bounded safe divergence from rsync's unbounded dd-like read); `--write-devices` writes only into an existing char/block node under the confined receive root and skips every unusable target rather than clobbering or aborting. `--specials` is classified **⛔ Impossible/Divergence** for one reason only: **FIFO recreation works** (unprivileged `mkfifo`, asserted under CI), but **sockets cannot be recreated by any standard filesystem call**, so a source socket is skipped with an explicit note. Tests assert FIFO recreation, the safe socket skip, the regular-file result of `--copy-devices`, the skipped/missing and non-device `--write-devices` targets, and (root-gated) real device-node creation; a root runner additionally drops the receiver to an unprivileged user to assert the `CAP_MKNOD` skip is graceful.
|
||||
|
||||
**Wave D — Times superstructure & arg-protection no-ops (✅ implemented, `--secluded-args` ⛔).** `-O`/`--omit-dir-times` and `-J`/`--omit-link-times` are now **real modifiers** (both `🔄 → ✅ Implemented`), reversing the old "never preserves directory/symlink times" divergence:
|
||||
|
||||
- **Directory times.** The recursive scanner captures every traversed source directory's metadata (mtime, plus atime under `-U`) into a per-transfer list — two paths are covered: the sequential `DirectoryScanner` captures each opened directory (including the transfer root), and the parallel scanner captures both the root in `parallel_scanner_create_with_options` and each worker's subdirectories in `open_next_directory` (appends are guarded by a mutex shared with the sender's pipeline context). The sender transmits them in trailing `STATUS_DIR_TIMES` frames (each: int count + count × (wire path, metadata) pairs) sent **after all file data and after the optional delete manifest**, just before `STATUS_FINISHED`. A tree larger than `MAX_MANIFEST_ENTRIES` (1 048 576) directories is chunked into repeated frames, each within the receiver's per-frame bound. A dir-time entry is RECORD-ONLY (`file->dir_time_only`): `file_save_to_disk_full` returns `FILE_SAVE_SKIPPED` without creating anything, so a source directory that was empty (or pruned by `-m/--prune-empty-dirs`) is never resurrected. The receiver accumulates received directory metadata in a `DirTimeList` and applies it only at the very end — after the entire stream, after the commit-style `--delete` deletion, and after `--delay-updates` publication — because creating or removing a child bumps the parent's mtime. Application is fd-relative/walk-confined (`file_open_secure_parent` + `utimensat(..., AT_SYMLINK_NOFOLLOW)`) and best-effort per entry: an absent path (an intentionally uncreated empty dir) is skipped QUIETLY and only a real existing directory is stamped. `-O` (config boolean, already on the wire) makes the receiver skip the whole set. The single-threaded sink applies in `receiver_send_success_frame`; the `-j`/`--threads` sink accumulates in `write_thread` and server.c applies after both threads join and the deletion commits.
|
||||
- **Symlink times/owner/mode.** `STATUS_SYMLINK` already carried metadata; the receiver now applies it with no-follow primitives only: `utimensat(..., AT_SYMLINK_NOFOLLOW)`, best-effort `fchmodat(..., AT_SYMLINK_NOFOLLOW)` (honest no-op where unsupported, e.g. Linux), and policy-gated `fchownat(..., AT_SYMLINK_NOFOLLOW)` via a new `identity_apply_ownership_link` that shares the identity resolver with the fd path. `-J` suppresses only the timestamps; ownership stays governed by the identity opt-in (`--numeric-ids`/`--usermap`/`--groupmap`/`--chown`) exactly like regular files. A symlink has no children, so this is applied immediately at creation.
|
||||
- **Wire:** the shared `STATUS_DIR_TIMES` frame (and metadata on `STATUS_MKDIR` for `--dirs` entries) is a frame-sequence change, so `PROTOCOL_VERSION` was bumped **2.16.0 → 2.17.0**; every version-sensitive test (`--protocol` accepted/rejected values) was updated. The config-frame layout itself is unchanged (the omit booleans already crossed). Non-metadata and `--no-preserve` transfers send no `STATUS_DIR_TIMES` frame and no directory metadata, keeping them byte-identical.
|
||||
|
||||
`--secluded-args` (`🔄 → ⛔ Impossible/Divergence`): a true arg-send protocol would replace the argv-based SSH launch with an in-band channel, and FastSync already builds the remote SSH argv injection-safe (single-quote-escaped shell words), so there is no argument-leak to close; the already-safe behavior is documented in the row and no transport change is made.
|
||||
|
||||
**Wave E (LAST) — Privilege: `--super`/`--no-super` and `--copy-as=USER[:GROUP]` (✅ implemented).** FastSync adopts a **safe-subset + clear-refusal** privilege model: it never blind-elevates and never calls `setuid`/`seteuid`/`setgid`. All privileged operations remain fd-relative and confined below the authorized receive root.
|
||||
|
||||
`--super`/`--no-super` set a receiver-side tri-state `Config->super_mode` (`SUPER_MODE_AUTO`/`ON`/`OFF`). `privilege_super_permitted()` / `privilege_super_mode_permitted()` (src/shared/identity.c) return true for `ON` and `AUTO` (AUTO preserves FastSync's historical best-effort attempt, where the kernel refuses an unprivileged call and the caller skips it) and false only for `OFF`. The gate covers every super-user activity FastSync performs: ownership application (`identity_apply_ownership`/`_link`), char/block device-node creation (`file_save_special_to_disk`), writes into an existing device (`--write-devices`), and the `--fake-super` owner replay. Unprivileged FIFO creation is deliberately unaffected. `--super` does **not** imply `--numeric-ids`: ownership is applied only when an explicit identity policy (`--usermap`/`--groupmap`/`--chown`/`--numeric-ids`/`--copy-as`) is also given. `--no-super` suppresses those activities even for a root receiver. A non-root receiver given `--super` logs one warning at activation (`identity_set_active`); each confined attempt is then refused by the kernel and skipped, never aborting. The confinement floor is unchanged (`file_open_secure_parent`, `O_NOFOLLOW`, root/path checks). Operator control: the server CLI accepts `--no-super`, a veto that forces `OFF` for every connection, refuses any client `--copy-as`, and neutralizes an explicit `--super` (the connection is accepted but no super-user activity is attempted). On a daemon, a module that has not opted in with `client owner = yes` additionally has super-user device activity forced off (see the Daemon Mode notes).
|
||||
|
||||
`--copy-as=USER[:GROUP]` is the safe subset. FastSync's receiver is multithreaded, so a real credential switch is unsafe; instead the receiver forces the ownership of **every entry it writes** — regular files, symlinks, directories (including implicitly-created parents), and special nodes — to the resolved target ids through the confined fd-relative identity path. USER is resolved on the client (name, `@N`/bare N, or `*` = client euid); when `:GROUP` is omitted the user's primary gid is used (falling back to `gid == uid` for a numeric id with no local passwd entry). It requires a privileged (root) receiver: an unprivileged receiver refuses the whole transfer at the config handshake, before `STATUS_OK`, so no data is ever written with the wrong ownership. A `--copy-as` chown failure on a capability-restricted root is logged at ERROR (never silently downgraded). `--copy-as` implies metadata (`--no-preserve` is rejected) and `--fake-super` cannot override it. Daemon policy: a `--daemon` receiver refuses **every** client-chosen-ownership / super-user request — `--numeric-ids`, `--chown`, `--usermap`/`--groupmap`, `--fake-super`, `--copy-as`, and explicit `--super` — unless the selected module opts in with `client owner = yes`; without that per-module opt-in any client could force arbitrary ownership inside the module root (the standalone listener and the SSH-launched `--stdio` server, which each serve one operator-authorized root, honor these requests). A `--copy-as` chown failure on a capability-restricted root marks the entry as failed rather than reporting success with the wrong owner.
|
||||
|
||||
**Wire:** two trailing config-frame blocks after the `--iconv` spec, in fixed order — `send_privilege_options`/`receive_privilege_options` (one `super_mode` int, validated `0..2`), then `send_copy_as_options`/`receive_copy_as_options` (presence int + two int32 ids, validated `>= 0`, with `copy_as_set ⇒ use_metadata`). `PROTOCOL_VERSION` bumped **2.17.0 → 2.18.0**. **Divergences from rsync:** rsync's `--super` elevates the receiver and `--copy-as` actually switches its credentials; FastSync never elevates and only permits/forwards confined attempts, and `--copy-as` forces ownership rather than switching identity.
|
||||
|
||||
**Post-Phase-7 Summary (after Waves A–E).** ✅143 / 🔀0 / ⛔4 / ⚠️0 / 🔄0 / ❌0 = 147. The 3 `🔀 Alt Arg` rows (`-a`, `-p`, `-z`) are ✅ (Wave A). All 10 prior `⚠️ Partial` rows are resolved to ✅ (`-S`, `-P`, `--block-size`, `--fake-super`, `--devices`, `--copy-devices`, `--write-devices`) or ⛔ (`--stderr=client`, `-N/--crtimes`, `--specials` for the impossible socket case). The 3 `🔄 Compatibility No-op` rows are resolved: `-O`/`-J` are now real ✅ (Wave D), `--secluded-args` is ⛔. The **Impossible/Divergence** bucket holds the 4 physically-impossible/divergent flags: `--stderr=client`, `-N/--crtimes`, `--specials` (sockets), `--secluded-args`. The last two `❌ Not Implemented` rows — `--super` and `--copy-as=USER[:GROUP]` — are now ✅ (Wave E). **No `❌ Not Implemented` rows remain.**
|
||||
|
||||
### Recommended Delivery Order
|
||||
|
||||
1. Resolve short-option conflicts (`-m`, `-M`, `-T`, `-f`, `-s`) and define the compatibility contract.
|
||||
2. Implement Phase 1 comparison, update, output, and alias features with unit and integration coverage.
|
||||
3. Implement Phase 2 traversal/filtering and Phase 3 deletion semantics.
|
||||
4. Implement metadata and link features that are safe on the supported platforms.
|
||||
5. Treat daemon mode, special files, batch mode, and protocol-version compatibility as separate projects.
|
||||
|
||||
The existing priority list below is a feature shortlist, not an implementation schedule; this plan supersedes it for effort and sequencing.
|
||||
|
||||
---
|
||||
|
||||
## Recommendations: Top Features to Implement Next
|
||||
|
||||
Ranked by user demand, implementation complexity, and interoperability impact (_status reflects current `dev`_):
|
||||
|
||||
| Priority | Feature | Effort | Impact |
|
||||
|----------|---------|--------|--------|
|
||||
| 1 | `--whole-file` / `-W` | Low | High — users expect opt-out of delta — **✅ implemented** |
|
||||
| 2 | `--ignore-times` / `-I` | Low | Medium — useful for forcing re-transfer — **✅ implemented** |
|
||||
| 3 | `--size-only` | Low | Medium — common migration scenario — **✅ implemented** |
|
||||
| 4 | `--ignore-existing` | Low | Medium — common sync patterns — **✅ implemented** |
|
||||
| 5 | `--existing` | Low | Medium — common sync patterns — **✅ implemented** |
|
||||
| 6 | `--remove-source-files` | Low | High — common for moves/backup |
|
||||
| 7 | `--delete-during` | Medium | High — performance improvement |
|
||||
| 8 | `--delay-updates` | Medium | High — atomic updates |
|
||||
| 9 | `--chmod` | Low | Medium — permission flexibility |
|
||||
| 10 | `--executability` / `-E` | Low | Low — simple flag |
|
||||
| 11 | `--skip-compress` | Low | Medium — performance tuning |
|
||||
|
||||
---
|
||||
|
||||
## FastSync-Specific Features (Not in rsync)
|
||||
|
||||
| Feature | Description |
|
||||
|---------|-------------|
|
||||
| `-j` / `--threads` | Multithreaded pipeline (scanner/loader/sender) (renamed from `-m` in Phase 7 Wave A; `-m` is now rsync `--prune-empty-dirs`) |
|
||||
| `--chunk-serialization` | Chunk serialization mode (long form only; `-s` is now rsync `--secluded-args`) |
|
||||
| `--sendfile` | Zero-copy sendfile() syscall (TCP only) (long form only; `-f` is now rsync `--filter`) |
|
||||
| `-z [level]` / `--compress` | zstd compression level (1-22) (`-c` is now rsync `--checksum`) |
|
||||
| `--chunk-size` | Configurable chunk size |
|
||||
| `--tls` | TLS encryption (mutual auth) |
|
||||
| `--fastsync-server-path` | Path to fastsync-server binary |
|
||||
| `--server-host` / `--server-port` | Direct TCP connection |
|
||||
| Incremental sync | Skip unchanged files (size+mtime) |
|
||||
| Delta transfer | Block-level delta for changed files |
|
||||
+1
-2
@@ -1,11 +1,10 @@
|
||||
{
|
||||
"$schema": "https://opencode.ai/config.json",
|
||||
"instructions": ["AGENTS.md"],
|
||||
"permission": {
|
||||
"bash": {
|
||||
"*": "allow",
|
||||
"git push origin main": "deny",
|
||||
"git push main": "deny"
|
||||
"git push main": "ask"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
-10
@@ -1,10 +0,0 @@
|
||||
[pytest]
|
||||
; Fast integration subset run on every pull request (see .gitea/workflows/ci.yaml).
|
||||
markers =
|
||||
ci: fast, representative integration tests run on the PR CI gate
|
||||
setpriv: privilege-dependent tests (drop to an unprivileged user); excluded
|
||||
from CI because their result depends on the runner/container uid and the
|
||||
host mount permissions, but run locally as root
|
||||
daemon_detach: real double-fork backgrounding path (--daemon without
|
||||
--no-detach); slower/fragile, so it runs in the full suite but not the
|
||||
fast PR gate
|
||||
@@ -1,329 +0,0 @@
|
||||
#include "change_list.h"
|
||||
#include "utils.h"
|
||||
#include <limits.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/stat.h>
|
||||
#include <time.h>
|
||||
|
||||
/* Itemize code emitted for a transferred regular file.
|
||||
*
|
||||
* Layout (rsync-compatible 11-char item): `>f` marks a regular file that was
|
||||
* transferred to the remote host; the trailing nine markers are, in order,
|
||||
* c(hecksum) s(ize) t(ime) p(erms) o(wner) g(roup) u(ser/acl) a(ttrs) x(attrs).
|
||||
* Every marker is `+` (FastSync does not compare each attribute on the
|
||||
* receiving side, so a sent file is reported as fully updated). Files that
|
||||
* are already up to date print no line at all, matching rsync's single -i
|
||||
* which only itemizes changes.
|
||||
*
|
||||
* Because the scanner only yields regular-file transfer candidates, `>d`
|
||||
* (directory) lines are never produced; directories are not transferred as
|
||||
* items by FastSync. */
|
||||
#define ITEMIZE_SENT_FILE ">f+++++++++"
|
||||
|
||||
typedef struct {
|
||||
char* data;
|
||||
size_t length;
|
||||
size_t capacity;
|
||||
} StrBuf;
|
||||
|
||||
static void strbuf_free(StrBuf* buf) {
|
||||
if (buf == NULL)
|
||||
return;
|
||||
free(buf->data);
|
||||
buf->data = NULL;
|
||||
buf->length = 0;
|
||||
buf->capacity = 0;
|
||||
}
|
||||
|
||||
static bool strbuf_reserve(StrBuf* buf, size_t extra) {
|
||||
if (buf->length > SIZE_MAX - extra - 1)
|
||||
return false;
|
||||
size_t need = buf->length + extra + 1;
|
||||
if (need <= buf->capacity)
|
||||
return true;
|
||||
size_t capacity = buf->capacity > 0 ? buf->capacity : 32;
|
||||
while (capacity < need) {
|
||||
if (capacity > SIZE_MAX / 2) {
|
||||
capacity = need;
|
||||
break;
|
||||
}
|
||||
capacity *= 2;
|
||||
}
|
||||
char* grown = realloc(buf->data, capacity);
|
||||
if (!grown)
|
||||
return false;
|
||||
buf->data = grown;
|
||||
buf->capacity = capacity;
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool strbuf_append_char(StrBuf* buf, char c) {
|
||||
if (!strbuf_reserve(buf, 1))
|
||||
return false;
|
||||
buf->data[buf->length++] = c;
|
||||
buf->data[buf->length] = '\0';
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool strbuf_append(StrBuf* buf, const char* text) {
|
||||
if (text == NULL)
|
||||
return true;
|
||||
size_t length = strlen(text);
|
||||
if (!strbuf_reserve(buf, length))
|
||||
return false;
|
||||
memcpy(buf->data + buf->length, text, length);
|
||||
buf->length += length;
|
||||
buf->data[buf->length] = '\0';
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool strbuf_append_ull(StrBuf* buf, unsigned long long value) {
|
||||
char digits[32];
|
||||
int written = snprintf(digits, sizeof(digits), "%llu", value);
|
||||
if (written < 0 || (size_t)written >= sizeof(digits))
|
||||
return false;
|
||||
return strbuf_append(buf, digits);
|
||||
}
|
||||
|
||||
static bool strbuf_append_longlong(StrBuf* buf, long long value) {
|
||||
char digits[32];
|
||||
int written = snprintf(digits, sizeof(digits), "%lld", value);
|
||||
if (written < 0 || (size_t)written >= sizeof(digits))
|
||||
return false;
|
||||
return strbuf_append(buf, digits);
|
||||
}
|
||||
|
||||
bool change_list_enabled(const Config* config) {
|
||||
return config != NULL && (config->itemize_changes || config->out_format != NULL ||
|
||||
(config->log_file != NULL && config->log_file_format != NULL));
|
||||
}
|
||||
|
||||
char* change_render_itemize(const ChangeEvent* event) {
|
||||
if (event == NULL || event->decision != CHANGE_SENT)
|
||||
return str_dup("");
|
||||
const char* code = event->is_directory ? ">d+++++++++" : ITEMIZE_SENT_FILE;
|
||||
StrBuf line = {0};
|
||||
bool ok = strbuf_append(&line, code) && strbuf_append(&line, " ") &&
|
||||
strbuf_append(&line, event->path != NULL ? event->path : "");
|
||||
if (!ok) {
|
||||
strbuf_free(&line);
|
||||
return NULL;
|
||||
}
|
||||
return line.data;
|
||||
}
|
||||
|
||||
static const char* leaf_name(const char* path) {
|
||||
if (path == NULL)
|
||||
return "";
|
||||
const char* slash = strrchr(path, '/');
|
||||
return slash != NULL && slash[1] != '\0' ? slash + 1 : path;
|
||||
}
|
||||
|
||||
char* change_render_format(const char* format, const ChangeEvent* event) {
|
||||
if (format == NULL)
|
||||
return NULL;
|
||||
StrBuf line = {0};
|
||||
bool ok = true;
|
||||
for (const char* p = format; *p != '\0' && ok;) {
|
||||
if (*p != '%') {
|
||||
ok = strbuf_append_char(&line, *p);
|
||||
p++;
|
||||
continue;
|
||||
}
|
||||
char token = p[1];
|
||||
if (token == '\0') {
|
||||
ok = strbuf_append_char(&line, '%');
|
||||
break;
|
||||
}
|
||||
switch (token) {
|
||||
case '%':
|
||||
ok = strbuf_append_char(&line, '%');
|
||||
break;
|
||||
case 'f':
|
||||
ok = strbuf_append(&line, event->path != NULL ? event->path : "");
|
||||
break;
|
||||
case 'n':
|
||||
ok = strbuf_append(&line, leaf_name(event->path));
|
||||
break;
|
||||
case 'l':
|
||||
ok = strbuf_append_ull(&line, event->size);
|
||||
break;
|
||||
case 'b':
|
||||
ok = strbuf_append_ull(&line, event->bytes_sent);
|
||||
break;
|
||||
case 'M':
|
||||
ok = strbuf_append_longlong(&line, (long long)event->mtime_sec);
|
||||
break;
|
||||
default:
|
||||
/* Unknown escape sequences are preserved verbatim. */
|
||||
ok = strbuf_append_char(&line, '%') && strbuf_append_char(&line, token);
|
||||
break;
|
||||
}
|
||||
p += 2;
|
||||
}
|
||||
if (!ok) {
|
||||
strbuf_free(&line);
|
||||
return NULL;
|
||||
}
|
||||
if (line.data == NULL) {
|
||||
line.data = str_dup("");
|
||||
if (!line.data)
|
||||
return NULL;
|
||||
}
|
||||
return line.data;
|
||||
}
|
||||
|
||||
/* Format a mode as an `ls -l` permission string, e.g. `-rw-r--r--`. */
|
||||
static void mode_to_ls_string(mode_t mode, char out[11]) {
|
||||
out[0] = S_ISDIR(mode) ? 'd'
|
||||
: S_ISLNK(mode) ? 'l'
|
||||
: S_ISCHR(mode) ? 'c'
|
||||
: S_ISBLK(mode) ? 'b'
|
||||
: S_ISFIFO(mode) ? 'p'
|
||||
: S_ISSOCK(mode) ? 's'
|
||||
: '-';
|
||||
mode_t bits = mode & 07777;
|
||||
out[1] = (bits & S_IRUSR) ? 'r' : '-';
|
||||
out[2] = (bits & S_IWUSR) ? 'w' : '-';
|
||||
out[3] = (bits & S_IXUSR) ? (bits & S_ISUID ? 's' : 'x') : (bits & S_ISUID ? 'S' : '-');
|
||||
out[4] = (bits & S_IRGRP) ? 'r' : '-';
|
||||
out[5] = (bits & S_IWGRP) ? 'w' : '-';
|
||||
out[6] = (bits & S_IXGRP) ? (bits & S_ISGID ? 's' : 'x') : (bits & S_ISGID ? 'S' : '-');
|
||||
out[7] = (bits & S_IROTH) ? 'r' : '-';
|
||||
out[8] = (bits & S_IWOTH) ? 'w' : '-';
|
||||
out[9] = (bits & S_IXOTH) ? (bits & S_ISVTX ? 't' : 'x') : (bits & S_ISVTX ? 'T' : '-');
|
||||
out[10] = '\0';
|
||||
}
|
||||
|
||||
char* change_render_list_line(mode_t mode, unsigned long long size, time_t mtime,
|
||||
const char* path) {
|
||||
char permission[11];
|
||||
mode_to_ls_string(mode, permission);
|
||||
char date[32];
|
||||
struct tm broken_down;
|
||||
if (localtime_r(&mtime, &broken_down) != NULL) {
|
||||
if (strftime(date, sizeof(date), "%Y/%m/%d %H:%M:%S", &broken_down) == 0)
|
||||
snprintf(date, sizeof(date), "?");
|
||||
} else {
|
||||
snprintf(date, sizeof(date), "?");
|
||||
}
|
||||
StrBuf line = {0};
|
||||
char size_field[32];
|
||||
int written = snprintf(size_field, sizeof(size_field), "%llu", size);
|
||||
if (written < 0 || (size_t)written >= sizeof(size_field)) {
|
||||
strbuf_free(&line);
|
||||
return NULL;
|
||||
}
|
||||
bool ok = strbuf_append(&line, permission) && strbuf_append_char(&line, ' ') &&
|
||||
strbuf_append(&line, size_field) && strbuf_append_char(&line, ' ') &&
|
||||
strbuf_append(&line, date) && strbuf_append_char(&line, ' ') &&
|
||||
strbuf_append(&line, path != NULL ? path : "");
|
||||
if (!ok) {
|
||||
strbuf_free(&line);
|
||||
return NULL;
|
||||
}
|
||||
return line.data;
|
||||
}
|
||||
|
||||
static void print_escaped_line(FILE* stream, const char* line, bool eight_bit_output) {
|
||||
char* escaped = output_escape(line, eight_bit_output);
|
||||
if (escaped != NULL) {
|
||||
fprintf(stream, "%s\n", escaped);
|
||||
free(escaped);
|
||||
} else {
|
||||
fprintf(stream, "%s\n", line);
|
||||
}
|
||||
fflush(stream);
|
||||
}
|
||||
|
||||
void change_emit(const Config* config, const ChangeEvent* event) {
|
||||
if (event == NULL || !change_list_enabled(config))
|
||||
return;
|
||||
if (event->decision == CHANGE_UP_TO_DATE)
|
||||
return;
|
||||
bool to_stdout = config->itemize_changes || config->out_format != NULL;
|
||||
bool to_log = config->log_file != NULL && config->log_file_format != NULL;
|
||||
if (to_stdout) {
|
||||
char* line = config->out_format != NULL ? change_render_format(config->out_format, event)
|
||||
: change_render_itemize(event);
|
||||
if (line != NULL) {
|
||||
print_escaped_line(stdout, line, config->eight_bit_output);
|
||||
free(line);
|
||||
}
|
||||
}
|
||||
if (to_log) {
|
||||
char* line = change_render_format(config->log_file_format, event);
|
||||
if (line != NULL) {
|
||||
print_escaped_line(config->log_file, line, config->eight_bit_output);
|
||||
free(line);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static bool format_uses_mtime(const char* format) {
|
||||
if (format == NULL)
|
||||
return false;
|
||||
/* Mirror change_render_format's tokenizer: "%%" is a literal percent (so
|
||||
* "%%M" does NOT expand %M) and unknown "%X" escapes consume both chars.
|
||||
* This keeps the optional stat() fallback below in step with the renderer. */
|
||||
for (const char* p = format; *p != '\0';) {
|
||||
if (*p != '%') {
|
||||
p++;
|
||||
continue;
|
||||
}
|
||||
char token = p[1];
|
||||
if (token == '\0')
|
||||
break;
|
||||
if (token == 'M')
|
||||
return true;
|
||||
p += 2;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
void change_emit_file_sent(const Config* config, const File* file) {
|
||||
if (file == NULL || !change_list_enabled(config))
|
||||
return;
|
||||
ChangeEvent event;
|
||||
memset(&event, 0, sizeof(event));
|
||||
/* The displayed path is the one transmitted (with -R + --files-from this is
|
||||
the bare relative destination path); the metadata fallback below still
|
||||
stats the local absolute path. */
|
||||
event.path = file_wire_path(file);
|
||||
event.decision = CHANGE_SENT;
|
||||
event.is_directory = false;
|
||||
event.size = file->data != NULL ? file->data->size : 0;
|
||||
/* FastSync has no wire-byte counter yet, so %b reports the source length
|
||||
* that had to be delivered (always equal to %l); the actual bytes written
|
||||
* to the socket (compressed/delta) are not measured. */
|
||||
event.bytes_sent = event.size;
|
||||
if (file->metadata != NULL) {
|
||||
event.mtime_sec = file->metadata->mtime_sec;
|
||||
} else if (format_uses_mtime(config->out_format) || format_uses_mtime(config->log_file_format)) {
|
||||
/* Best-effort fallback for %M when no metadata was captured (no -M): the
|
||||
* path is stat()ed just to fill the field, and any failure leaves 0. */
|
||||
struct stat st;
|
||||
if (file->path != NULL && stat(file->path, &st) == 0)
|
||||
event.mtime_sec = st.st_mtime;
|
||||
}
|
||||
change_emit(config, &event);
|
||||
}
|
||||
|
||||
/* Build and emit a CHANGE_SENT event for an explicit directory entry (-d). */
|
||||
void change_emit_dir_sent(const Config* config, const File* file) {
|
||||
if (file == NULL || !change_list_enabled(config))
|
||||
return;
|
||||
ChangeEvent event;
|
||||
memset(&event, 0, sizeof(event));
|
||||
event.path = file_wire_path(file);
|
||||
event.decision = CHANGE_SENT;
|
||||
event.is_directory = true;
|
||||
event.size = 0;
|
||||
event.bytes_sent = 0;
|
||||
if (file->metadata != NULL)
|
||||
event.mtime_sec = file->metadata->mtime_sec;
|
||||
change_emit(config, &event);
|
||||
}
|
||||
@@ -1,78 +0,0 @@
|
||||
#ifndef CHANGE_LIST_H
|
||||
#define CHANGE_LIST_H
|
||||
|
||||
#include "config.h"
|
||||
#include "file_types.h"
|
||||
#include <stdbool.h>
|
||||
#include <sys/stat.h>
|
||||
#include <time.h>
|
||||
|
||||
/*
|
||||
* Shared per-file change-event / output model (rsync --itemize-changes,
|
||||
* --out-format, --log-file-format, and --list-only all render from here).
|
||||
*
|
||||
* FastSync is a push-style tool: the client sends files from the source tree
|
||||
* to a server that writes them under the destination root. Events are
|
||||
* emitted by whichever code path decides a file's fate (the single-threaded
|
||||
* send loop and the `-m` sender thread both call the same per-file sender), so
|
||||
* all change events are emitted by exactly one thread and itemize/out-format
|
||||
* lines never interleave with each other. They may still interleave with
|
||||
* legacy log messages (log.c) that share the same stdout/log-file stream.
|
||||
*/
|
||||
|
||||
typedef enum {
|
||||
CHANGE_SENT, /* file data (full or delta) was transmitted */
|
||||
CHANGE_UP_TO_DATE, /* receiver already had an identical file; skipped */
|
||||
} ChangeDecision;
|
||||
|
||||
typedef struct {
|
||||
const char* path; /* full source path */
|
||||
ChangeDecision decision;
|
||||
bool is_directory;
|
||||
unsigned long long size; /* source file length in bytes */
|
||||
/* The number of bytes reported for a sent file. FastSync has no wire-byte
|
||||
* counter, so this is always the source length (== size / %l); actual
|
||||
* post-compression/delta bytes on the wire are not counted. */
|
||||
unsigned long long bytes_sent;
|
||||
time_t mtime_sec; /* 0 when unknown */
|
||||
} ChangeEvent;
|
||||
|
||||
/* True when any output mode is active and per-file events matter. */
|
||||
bool change_list_enabled(const Config* config);
|
||||
|
||||
/* Render the rsync-style itemize line for a transferred file:
|
||||
* `>f+++++++++ <path>`
|
||||
* The 11-char code is `>f` (regular file transferred to the remote host)
|
||||
* followed by c/s/t/p/o/g/u/a/x markers that are all `+` (value will be set
|
||||
* / differs) because FastSync does not separately compare checksums, size,
|
||||
* mtime, perms, owner, group, uid, acl, or xattr on the receiving side, so a
|
||||
* sent file is reported as fully updated. Up-to-date files print no line
|
||||
* (rsync single `-i` only shows changes). Caller frees the result. */
|
||||
char* change_render_itemize(const ChangeEvent* event);
|
||||
|
||||
/* Expand an --out-format/--log-file-format template. Tokens:
|
||||
* %f full source path %b "bytes sent" == the source length (%l);
|
||||
* %n leaf (base) name actual post-compression/delta wire bytes
|
||||
* %l file length in bytes are not counted
|
||||
* %M mtime in whole seconds %% a literal percent sign
|
||||
* Unknown %X sequences are preserved verbatim. Caller frees the result. */
|
||||
char* change_render_format(const char* format, const ChangeEvent* event);
|
||||
|
||||
/* Render one --list-only long-listing entry:
|
||||
* `-rw-r--r-- 12 2026/09/06 10:00:00 <path>`
|
||||
* (ls -l style columns; mtime in the local time zone). Caller frees it. */
|
||||
char* change_render_list_line(mode_t mode, unsigned long long size, time_t mtime, const char* path);
|
||||
|
||||
/* Emit an event to every active destination:
|
||||
* stdout: --itemize-changes line, or the --out-format expansion when set;
|
||||
* log file: the --log-file-format expansion (requires --log-file).
|
||||
* CHANGE_UP_TO_DATE events produce no output. */
|
||||
void change_emit(const Config* config, const ChangeEvent* event);
|
||||
|
||||
/* Build and emit a CHANGE_SENT event for a file the client just sent. */
|
||||
void change_emit_file_sent(const Config* config, const File* file);
|
||||
|
||||
/* Build and emit a CHANGE_SENT event for an explicit directory entry (-d). */
|
||||
void change_emit_dir_sent(const Config* config, const File* file);
|
||||
|
||||
#endif
|
||||
+242
-1743
File diff suppressed because it is too large
Load Diff
+280
-2146
File diff suppressed because it is too large
Load Diff
@@ -5,12 +5,11 @@
|
||||
#include "config.h"
|
||||
#include "transport_tcp.h"
|
||||
|
||||
int send_chunk(Client* client, Chunk* chunk, Config* config);
|
||||
int send_files(Config* config);
|
||||
/* Takes ownership only when *config is set to NULL on return. */
|
||||
int send_files_multithreaded(Config** config);
|
||||
/* Phase 6 residual-batch (client-only). See client_send.c. */
|
||||
int write_batch_from_source(const Config* config, const char* batch_path);
|
||||
int apply_batch_to_dest(const Config* config, const char* batch_path, const char* dest_root);
|
||||
extern char *server_host;
|
||||
extern int server_port;
|
||||
|
||||
int send_chunk(Client *client, Chunk *chunk, Config *config);
|
||||
int send_files(Config *config);
|
||||
int send_files_multithreaded(Config *config);
|
||||
|
||||
#endif
|
||||
|
||||
@@ -1,194 +0,0 @@
|
||||
#include "client_validation.h"
|
||||
#include "charset.h"
|
||||
#include "delay_updates.h"
|
||||
#include "log.h"
|
||||
#include "usage.h"
|
||||
#include "utils.h"
|
||||
#include <string.h>
|
||||
#include <stdio.h>
|
||||
|
||||
/* Validate config after parsing. Returns true if valid. */
|
||||
bool validate_config(const Config* config) {
|
||||
/* Phase 6 residual-batch modes relax the normal source+destination pair: the
|
||||
batch driver is local and needs only what it consumes. --only-write-batch
|
||||
emits a batch from the source (no destination, no server);
|
||||
--read-batch applies a batch to the destination (no source, no server);
|
||||
--write-batch runs the live transfer AND emits a batch, so it keeps the
|
||||
full pair. */
|
||||
bool write_batch = config->write_batch != NULL;
|
||||
bool only_write_batch = config->only_write_batch != NULL;
|
||||
bool read_batch = config->read_batch != NULL;
|
||||
if ((write_batch && only_write_batch) || (write_batch && read_batch) ||
|
||||
(only_write_batch && read_batch)) {
|
||||
log_message(LOG_LEVEL_ERROR,
|
||||
"--write-batch, --only-write-batch, and --read-batch are mutually exclusive");
|
||||
return false;
|
||||
}
|
||||
if (read_batch) {
|
||||
if (!config->receive_root_directory) {
|
||||
log_message(LOG_LEVEL_ERROR, "--read-batch requires a destination directory");
|
||||
print_usage();
|
||||
return false;
|
||||
}
|
||||
} else if (only_write_batch) {
|
||||
if (!config->send_directory) {
|
||||
log_message(LOG_LEVEL_ERROR, "--only-write-batch requires a source directory");
|
||||
print_usage();
|
||||
return false;
|
||||
}
|
||||
} else if (!config->send_directory || !config->receive_root_directory) {
|
||||
log_message(LOG_LEVEL_ERROR, "source and destination directories are required");
|
||||
print_usage();
|
||||
return false;
|
||||
}
|
||||
if (config_has_basis(config) && config->use_chunk_serialization) {
|
||||
log_message(LOG_LEVEL_ERROR,
|
||||
"--compare-dest/--copy-dest/--link-dest require per-file incremental checks and "
|
||||
"cannot be combined with -s (chunk serialization)");
|
||||
return false;
|
||||
}
|
||||
if (config->use_sendfile && (config->use_chunk_serialization || config->use_compression)) {
|
||||
log_message(LOG_LEVEL_ERROR, "-f/--sendfile cannot be combined with -c (compression) or -s "
|
||||
"(chunk serialization)");
|
||||
return false;
|
||||
}
|
||||
if (config->compression_threads > 0 && !config->use_compression) {
|
||||
log_message(LOG_LEVEL_ERROR, "--compress-threads requires compression (-c or -z)");
|
||||
return false;
|
||||
}
|
||||
if (config->transport == TRANSPORT_SSH && config->use_sendfile) {
|
||||
log_message(LOG_LEVEL_ERROR, "-f/--sendfile is not supported with SSH transport");
|
||||
return false;
|
||||
}
|
||||
if (config->use_incremental && config->use_chunk_serialization) {
|
||||
log_message(LOG_LEVEL_ERROR, "--incremental is not supported with -s (chunk serialization)");
|
||||
return false;
|
||||
}
|
||||
/* -4 and -6 are mutually exclusive: a socket address family cannot be both. */
|
||||
if (config->ipv4 && config->ipv6) {
|
||||
log_message(LOG_LEVEL_ERROR, "-4/--ipv4 and -6/--ipv6 are mutually exclusive");
|
||||
return false;
|
||||
}
|
||||
if (config->skip_compress_set && config->use_chunk_serialization) {
|
||||
log_message(LOG_LEVEL_ERROR,
|
||||
"--skip-compress cannot be combined with -s (chunk serialization)");
|
||||
return false;
|
||||
}
|
||||
if (config->use_delta && !config->whole_file && !config->use_incremental) {
|
||||
log_message(LOG_LEVEL_ERROR, "--delta requires --incremental");
|
||||
return false;
|
||||
}
|
||||
if (config->use_delta && !config->whole_file && config->use_chunk_serialization) {
|
||||
log_message(LOG_LEVEL_ERROR, "--delta cannot be combined with -s (chunk serialization)");
|
||||
return false;
|
||||
}
|
||||
if (config->use_delta && !config->whole_file && config->use_sendfile) {
|
||||
log_message(LOG_LEVEL_ERROR, "--delta cannot be combined with -f (sendfile)");
|
||||
return false;
|
||||
}
|
||||
/* --append / --append-verify resume a shorter existing destination by
|
||||
transmitting only the tail. The resume needs the per-file STATUS_CHECK
|
||||
handshake (so the dest length is learned), which chunk serialization -s
|
||||
disables; and whole-file is the opposite intent (send everything), so the
|
||||
two would silently make the resume pointless. Both are rejected up front
|
||||
rather than silently degrading to a full transfer. */
|
||||
if ((config->append || config->append_verify) && config->use_chunk_serialization) {
|
||||
log_message(LOG_LEVEL_ERROR,
|
||||
"--append/--append-verify require the per-file incremental check and cannot be "
|
||||
"combined with -s (chunk serialization)");
|
||||
return false;
|
||||
}
|
||||
if ((config->append || config->append_verify) && config->whole_file) {
|
||||
log_message(LOG_LEVEL_ERROR,
|
||||
"--append/--append-verify are incompatible with --whole-file (which forces a "
|
||||
"full transfer)");
|
||||
return false;
|
||||
}
|
||||
/* --hard-links/-H transmits each later group member as a dedicated per-file
|
||||
STATUS_HARDLINK frame, which chunk serialization -s does not support; and a
|
||||
hard-links sibling carries no payload, so the tail-resume of --append is
|
||||
meaningless for it. Both combinations are rejected up front rather than
|
||||
silently degrading. */
|
||||
if (config->preserve_hard_links && config->use_chunk_serialization) {
|
||||
log_message(LOG_LEVEL_ERROR,
|
||||
"--hard-links/-H cannot be combined with -s (chunk serialization)");
|
||||
return false;
|
||||
}
|
||||
/* -X/-A ride the per-file metadata frame; the buffer-based chunk-serialization
|
||||
wire format does not carry the xattr block, so the pair is rejected up front
|
||||
(mirroring -H + -s) rather than silently dropping attributes. */
|
||||
if ((config->preserve_xattrs || config->preserve_acls) && config->use_chunk_serialization) {
|
||||
log_message(LOG_LEVEL_ERROR,
|
||||
"--xattrs/-X and --acls/-A cannot be combined with -s (chunk serialization)");
|
||||
return false;
|
||||
}
|
||||
if (config->preserve_hard_links && (config->append || config->append_verify)) {
|
||||
log_message(LOG_LEVEL_ERROR,
|
||||
"--hard-links/-H cannot be combined with --append/--append-verify");
|
||||
return false;
|
||||
}
|
||||
if (config->log_file_format && !config->log_file) {
|
||||
log_message(LOG_LEVEL_ERROR, "--log-file-format requires --log-file");
|
||||
return false;
|
||||
}
|
||||
if (config->use_tls) {
|
||||
if (!config->tls_cert || !config->tls_key || !config->tls_ca) {
|
||||
log_message(LOG_LEVEL_ERROR, "--tls requires --cert, --key, and --ca");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
/* Daemon credentials (A7, protocol 2.19.0): a --password-file would send the
|
||||
username in the clear and derive a SCRAM proof a network sniffer could
|
||||
attack offline, so it is only allowed over TLS (which itself mandates a
|
||||
verified --cert/--key/--ca set above) or to a loopback destination. A
|
||||
remote plaintext daemon is refused here, before any network I/O. */
|
||||
if (config->password_file && !config->use_tls && !utils_host_is_loopback(config->server_host)) {
|
||||
log_message(LOG_LEVEL_ERROR, "sending daemon credentials to a non-local server requires --tls");
|
||||
return false;
|
||||
}
|
||||
if (config->delay_updates && config->inplace) {
|
||||
log_message(LOG_LEVEL_ERROR, "--delay-updates does not work with --inplace");
|
||||
return false;
|
||||
}
|
||||
if (config->delay_updates && delay_updates_staging_name_conflict(config->backup_dir)) {
|
||||
log_message(LOG_LEVEL_ERROR,
|
||||
"--backup-dir is reserved when --delay-updates is active (used for the internal "
|
||||
"staging directory)");
|
||||
return false;
|
||||
}
|
||||
if (!config_has_valid_delete_timing(config)) {
|
||||
log_message(LOG_LEVEL_ERROR,
|
||||
"--delete-before/--delete-during/--delete-delay/--delete-after select the delete "
|
||||
"timing; at most one may be given and each implies --delete");
|
||||
return false;
|
||||
}
|
||||
/* --iconv: reject a malformed CONVERT_SPEC or an unsupported charset name at
|
||||
startup (a probe iconv_open is attempted), so a typo'd charset never fails
|
||||
the run mid-transfer with per-file errors. */
|
||||
if (!charset_spec_valid(config->iconv_spec)) {
|
||||
log_message(LOG_LEVEL_ERROR,
|
||||
"--iconv requires LOCAL[,REMOTE] charset names supported by iconv");
|
||||
return false;
|
||||
}
|
||||
/* --protocol: FastSync has exactly one wire format, so the forced version
|
||||
must equal the current PROTOCOL_VERSION exactly. Rejected here, before any
|
||||
network I/O, rather than letting the server hit its own mismatch check. */
|
||||
if (strcmp(config->version, PROTOCOL_VERSION) != 0) {
|
||||
log_message(LOG_LEVEL_ERROR,
|
||||
"--protocol must be %s (FastSync supports only its current wire "
|
||||
"protocol version and cannot speak an older or virtual one)",
|
||||
PROTOCOL_VERSION);
|
||||
return false;
|
||||
}
|
||||
/* --copy-as pushes the source ids through the metadata path (it implies
|
||||
--preserve). A later --no-preserve would clear use_metadata, leaving the
|
||||
transfer with nothing to chown while the receiver gate would still pass.
|
||||
Refuse the combination up front rather than silently chowning nothing. */
|
||||
if (config->copy_as_set && !config->use_metadata) {
|
||||
log_message(LOG_LEVEL_ERROR,
|
||||
"--copy-as requires metadata preservation and cannot be combined with "
|
||||
"--no-preserve");
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
@@ -1,9 +0,0 @@
|
||||
#ifndef CLIENT_VALIDATION_H
|
||||
#define CLIENT_VALIDATION_H
|
||||
|
||||
#include "config.h"
|
||||
#include <stdbool.h>
|
||||
|
||||
bool validate_config(const Config* config);
|
||||
|
||||
#endif
|
||||
+72
-1643
File diff suppressed because it is too large
Load Diff
+8
-230
@@ -2,248 +2,26 @@
|
||||
#define SCANNER_H
|
||||
|
||||
#include "chunk.h"
|
||||
#include "file_list.h"
|
||||
#include "filter.h"
|
||||
#include "hardlink.h"
|
||||
#include "protocol.h"
|
||||
#include "queue.h"
|
||||
#include "stop_condition.h"
|
||||
#include <dirent.h>
|
||||
#include <stdbool.h>
|
||||
#include <stdatomic.h>
|
||||
#include <sys/types.h>
|
||||
#include <threads.h>
|
||||
|
||||
typedef struct {
|
||||
Queue *directories;
|
||||
DIR *current_dir;
|
||||
char *current_path;
|
||||
bool use_metadata;
|
||||
/* Phase 4 metadata capture: -U/--atimes and -N/--crtimes tell the scanner to
|
||||
* capture the source access / birth time into each entry's FileMetadata. */
|
||||
bool preserve_atimes;
|
||||
bool preserve_crtimes;
|
||||
/* Phase 4 xattrs: when preserve_xattrs || preserve_acls is set the scanner
|
||||
* captures each regular file's whitelisted xattr set onto the File. */
|
||||
bool preserve_xattrs;
|
||||
bool preserve_acls;
|
||||
unsigned long long chunk_size;
|
||||
char** exclude_patterns;
|
||||
char **exclude_patterns;
|
||||
int exclude_count;
|
||||
char** include_patterns;
|
||||
char **include_patterns;
|
||||
int include_count;
|
||||
unsigned long long max_size;
|
||||
unsigned long long min_size;
|
||||
int max_depth;
|
||||
int num_threads;
|
||||
bool follow_symlinks;
|
||||
bool copy_links;
|
||||
bool safe_links;
|
||||
bool copy_unsafe_links;
|
||||
/* Phase 4 symlink-trust sender options: -k/--copy-dirlinks (dereference a
|
||||
* symlink to a directory as a directory, keeping symlinks-to-files as
|
||||
* symlinks) and --munge-links (rewrite each transmitted symlink target with a
|
||||
* marker; escaping targets are never transmitted). Both are client/sender
|
||||
* side only and never serialized to the wire (keep_dirlinks is the
|
||||
* receiver-side counterpart). */
|
||||
bool copy_dirlinks;
|
||||
bool munge_links;
|
||||
bool checksum;
|
||||
bool one_file_system;
|
||||
/* Phase 4 special/devices: whether device nodes (--devices) and special files
|
||||
* (--specials) are preserved via recreation, and whether --copy-devices
|
||||
* copies a device's content as an ordinary regular file. */
|
||||
bool preserve_devices;
|
||||
bool preserve_specials;
|
||||
bool copy_devices;
|
||||
/* Phase 2 (files-from / filter layer). All pointers are shared read-only
|
||||
* across scanner instances and worker threads; ownership stays with the
|
||||
* caller (client_send). */
|
||||
const FileListSet* file_list; /* --files-from allow-set, or NULL */
|
||||
const FilterRuleList* base_filters; /* command-line + -C rules, or NULL */
|
||||
bool per_dir_filters; /* -F: read .rsync-filter per directory */
|
||||
bool dirs; /* -d/--dirs: transfer dir entries, no recursion */
|
||||
bool relative; /* -R/--relative (dest rel paths, with --files-from) */
|
||||
/* --prune-empty-dirs (long only): in --dirs mode an empty source directory's
|
||||
explicit entry is omitted from the transfer file list (so nothing is
|
||||
created at the destination and it can be pruned by --delete); explicitly
|
||||
--files-from-listed directories always pass through. Recursive transfers
|
||||
never emit empty directories, so the flag has no additional effect there. */
|
||||
bool prune_empty_dirs;
|
||||
/* Delete-excluded protection sink (optional): when non-NULL the scanner
|
||||
* appends the destination-relative path of every entry it prunes because a
|
||||
* USER SELECTION rule excluded it (--filter/-C/per-dir rules, the legacy
|
||||
* --exclude/--include layer, and --max-size/--min-size). The sender turns
|
||||
* this list into the manifest's protected prefixes so `--delete` leaves the
|
||||
* destination mirror of excluded source paths alone (rsync's default), and
|
||||
* empties it when --delete-excluded opts back into deleting them. NOT
|
||||
* recorded for --files-from subset pruning (whose delete semantics stay
|
||||
* keep-set-only) or for -R/--files-from relative wire paths. When
|
||||
* `excluded_mutex` is non-NULL it is taken around every append (the parallel
|
||||
* scanner shares one list across its worker threads). */
|
||||
ArrayList* excluded_paths;
|
||||
mtx_t* excluded_mutex;
|
||||
/* --ignore-errors: an unreadable directory during the scan is recorded as an
|
||||
* I/O error and skipped instead of aborting the scan. Client-only. */
|
||||
bool ignore_io_errors;
|
||||
/* --ignore-missing-args (implied by --delete-missing-args): an explicitly
|
||||
* --files-from-listed entry that does not exist under the source is skipped
|
||||
* instead of failing (the --dirs generator is the only scanner path that
|
||||
* observes a listed-but-missing entry). */
|
||||
bool ignore_missing_args;
|
||||
/* --hard-links (-H): shared, mutable (mutex-guarded) link-group detection
|
||||
* table, NULL when -H is off. Owned by the caller (client_send), shared
|
||||
* read-only here; the parallel scanner passes it unchanged to every worker so
|
||||
* one table detects every group across all subdirectories. */
|
||||
HardLinkTable* hardlinks;
|
||||
/* Phase 6: optional sender stop deadline. When non-NULL the scanner checks
|
||||
* it at natural loop boundaries and stops emitting chunks once reached
|
||||
* (without marking the scan as failed), so a busy scan itself stops early.
|
||||
* Client-only, never serialized to the wire. */
|
||||
const StopCondition* stop_condition;
|
||||
/* P7 Wave D (protocol 2.17.0): directory-time capture sink. When
|
||||
* `capture_dir_times` is true the recursive scan appends one is_dir File
|
||||
* (with metadata, no payload) per source directory it traverses to
|
||||
* `dir_entries`, so the sender can transmit trailing STATUS_DIR_TIMES
|
||||
* frame(s) and the receiver can apply directory mtimes AFTER all children
|
||||
* are written. `dir_entries_mutex` (optional) guards the list
|
||||
* for the parallel scanner's shared worker threads; the caller owns both.
|
||||
* The --dirs generator does not use this (its directory entries carry their
|
||||
* metadata inline through STATUS_MKDIR). */
|
||||
bool capture_dir_times;
|
||||
ArrayList* dir_entries;
|
||||
mtx_t* dir_entries_mutex;
|
||||
} ScannerOptions;
|
||||
|
||||
/* Internal per-scanner filter state. FilterNode chains represent the ordered
|
||||
* per-directory .rsync-filter rules that apply below a directory. */
|
||||
typedef struct FilterNode FilterNode;
|
||||
|
||||
typedef struct {
|
||||
Queue* directories;
|
||||
DIR* current_dir;
|
||||
char* current_path;
|
||||
bool use_metadata;
|
||||
bool preserve_atimes;
|
||||
bool preserve_crtimes;
|
||||
bool preserve_xattrs;
|
||||
bool preserve_acls;
|
||||
unsigned long long chunk_size;
|
||||
char** exclude_patterns;
|
||||
int exclude_count;
|
||||
char** include_patterns;
|
||||
int include_count;
|
||||
unsigned long long max_size;
|
||||
unsigned long long min_size;
|
||||
int max_depth;
|
||||
int current_depth;
|
||||
bool follow_symlinks;
|
||||
bool copy_links;
|
||||
bool safe_links;
|
||||
bool copy_unsafe_links;
|
||||
bool copy_dirlinks;
|
||||
bool munge_links;
|
||||
bool checksum;
|
||||
bool one_file_system;
|
||||
dev_t root_dev;
|
||||
bool failed;
|
||||
/* Phase 4 special/devices (see ScannerOptions). */
|
||||
bool preserve_devices;
|
||||
bool preserve_specials;
|
||||
bool copy_devices;
|
||||
/* Phase 2 (files-from / filter layer). */
|
||||
char* root_path; /* transfer root (fs path) for rel computation */
|
||||
char* current_rel; /* rel path of the open directory ("" == root) */
|
||||
bool at_seed_dir; /* next open is the seed directory */
|
||||
FilterNode* seed_node; /* inherited context of the seed dir, or NULL */
|
||||
FilterNode* current_node; /* filter context of the open directory */
|
||||
ArrayList* filter_nodes; /* owned FilterNode arena (may be NULL) */
|
||||
const FileListSet* file_list;
|
||||
const FilterRuleList* base_filters;
|
||||
bool per_dir_filters;
|
||||
/* --dirs / -R state for the directory-entry generator (dirs_mode replaces
|
||||
the recursive scan). */
|
||||
bool dirs_mode;
|
||||
bool relative_mode; /* file_list && relative: send bare relative wire paths */
|
||||
bool prune_empty_dirs;
|
||||
bool dirs_root_emitted;
|
||||
int list_index;
|
||||
ArrayList* dirs_batch; /* owned when non-NULL */
|
||||
unsigned long long dirs_batch_size;
|
||||
/* Excluded-path sink (see ScannerOptions). `excluded_mutex` is shared across
|
||||
parallel worker threads. */
|
||||
ArrayList* excluded_paths;
|
||||
mtx_t* excluded_mutex;
|
||||
/* --ignore-errors: continue past unreadable directories (records io_error). */
|
||||
bool ignore_io_errors;
|
||||
/* --ignore-missing-args: --dirs listed-but-missing entries are skipped, not
|
||||
fatal (see ScannerOptions.ignore_missing_args). */
|
||||
bool ignore_missing_args;
|
||||
/* A directory could not be opened (I/O error, e.g. EACCES). With
|
||||
--ignore-errors the scan continues past it and the caller decides what to
|
||||
do; `failed` is reserved for fatal errors that always abort the scan. */
|
||||
bool io_error;
|
||||
/* --hard-links (-H): shared link-group detection table (see ScannerOptions).
|
||||
NULL when -H is off. */
|
||||
HardLinkTable* hardlinks;
|
||||
/* Phase 6: sender stop deadline (from ScannerOptions). */
|
||||
const StopCondition* stop_condition;
|
||||
/* P7 Wave D directory-time capture (see ScannerOptions). */
|
||||
bool capture_dir_times;
|
||||
ArrayList* dir_entries;
|
||||
mtx_t* dir_entries_mutex;
|
||||
} DirectoryScanner;
|
||||
|
||||
typedef struct {
|
||||
Queue* result_queue;
|
||||
mtx_t result_mutex;
|
||||
cnd_t result_not_empty;
|
||||
cnd_t result_not_full;
|
||||
int num_threads;
|
||||
int expected_threads;
|
||||
int created_threads;
|
||||
thrd_t* threads;
|
||||
bool done;
|
||||
bool failed;
|
||||
/* A worker skipped an unreadable directory under --ignore-errors (non-fatal). */
|
||||
bool io_error;
|
||||
atomic_bool cancelled;
|
||||
int completed;
|
||||
Chunk* initial_chunk;
|
||||
ProtocolSession* allocation_session;
|
||||
FilterNode* root_filter_node; /* root .rsync-filter context (owned by ps) */
|
||||
} ParallelScanner;
|
||||
|
||||
DirectoryScanner* directory_scanner_create(const char* root_directory, bool use_metadata,
|
||||
unsigned long long chunk_size, char** exclude_patterns,
|
||||
int exclude_count, char** include_patterns,
|
||||
int include_count, unsigned long long max_size,
|
||||
unsigned long long min_size, int max_depth,
|
||||
bool follow_symlinks, bool copy_links, bool safe_links,
|
||||
bool copy_unsafe_links, bool checksum);
|
||||
DirectoryScanner* directory_scanner_create_with_options(const char* root_directory,
|
||||
const ScannerOptions* options);
|
||||
Chunk* directory_scanner_next(DirectoryScanner* scanner);
|
||||
bool directory_scanner_failed(const DirectoryScanner* scanner);
|
||||
void directory_scanner_destroy(DirectoryScanner* scanner);
|
||||
|
||||
/* --one-file-system (-x) decision: a directory entry may be descended into
|
||||
* only when the option is disabled or the entry lives on the same device as
|
||||
* the transfer root. Exposed so tests can exercise the rule directly. */
|
||||
bool scanner_same_filesystem(bool one_file_system, dev_t root_device, dev_t entry_device);
|
||||
|
||||
/* Relative path of an on-disk path below `root` ("" == the root itself, NULL
|
||||
* when `fs_path` is not under `root`). Handles trailing slashes and a root of
|
||||
* "/". Exposed so tests can exercise the mapping directly. */
|
||||
char* scanner_path_relative(const char* root, const char* fs_path);
|
||||
|
||||
ParallelScanner* parallel_scanner_create_with_options(const char* root_directory,
|
||||
const ScannerOptions* options,
|
||||
ProtocolSession* allocation_session);
|
||||
Chunk* parallel_scanner_next(ParallelScanner* scanner);
|
||||
bool parallel_scanner_failed(const ParallelScanner* scanner);
|
||||
bool parallel_scanner_had_io_error(const ParallelScanner* scanner);
|
||||
void parallel_scanner_destroy(ParallelScanner* scanner);
|
||||
|
||||
/* True when a directory could not be opened during the scan (an I/O error,
|
||||
recorded even when --ignore-errors keeps the scan going past it). */
|
||||
bool directory_scanner_had_io_error(const DirectoryScanner* scanner);
|
||||
DirectoryScanner *directory_scanner_create(char *root_directory, bool use_metadata, unsigned long long chunk_size, char **exclude_patterns, int exclude_count, char **include_patterns, int include_count, unsigned long long max_size, unsigned long long min_size);
|
||||
Chunk *directory_scanner_next(DirectoryScanner *scanner);
|
||||
void directory_scanner_destroy(DirectoryScanner *scanner);
|
||||
|
||||
#endif
|
||||
|
||||
@@ -1,298 +0,0 @@
|
||||
#include "usage.h"
|
||||
#include <stdio.h>
|
||||
#include <delta.h>
|
||||
#include <chunk.h>
|
||||
|
||||
void print_usage(void) {
|
||||
printf("Usage:\n");
|
||||
printf(" fastsync [options] <source> <destination>\n");
|
||||
printf(" fastsync [options] --source-dir <src> --dest-dir <dst>\n");
|
||||
printf("\n");
|
||||
printf("Destination formats:\n");
|
||||
printf(" user@host:/path SSH transport (rsync-style)\n");
|
||||
printf(" host:/path SSH transport (current user)\n");
|
||||
printf(" host::module/path Daemon TCP transport (fastsync-server --daemon);\n");
|
||||
printf(" module names a server-side module, path is relative\n");
|
||||
printf(" within it (connect with --server-port)\n");
|
||||
printf(" /local/path TCP transport (requires server on localhost:8080)\n");
|
||||
printf("\n");
|
||||
printf("Options:\n");
|
||||
printf(" -c, --checksum Verify content by checksum instead of size+mtime\n");
|
||||
printf(" -z, --compress [level] Enable compression (level 1-22, default 5)\n");
|
||||
printf(" -a, --archive rsync archive mode (-rlptgoD): links, metadata,\n");
|
||||
printf(" devices and specials (not compression/multithreading)\n");
|
||||
printf(" -n, --dry-run Show what would be transferred\n");
|
||||
printf(" --remove-source-files Remove regular source files after successful transfer\n");
|
||||
printf(" -p, --perms Preserve permission bits (part of the metadata bundle)\n");
|
||||
printf(" --ssh-port <port> SSH port (default: 22)\n");
|
||||
printf(" -e, --rsh <command> Remote shell to launch on the client for the SSH\n");
|
||||
printf(" transport (default: ssh). The command may include\n");
|
||||
printf(" arguments, e.g. -e \"ssh -p 2222\"\n");
|
||||
printf(" --rsync-path <path> Alias for --fastsync-server-path (path to the\n");
|
||||
printf(" fastsync server binary on the remote side)\n");
|
||||
printf(" --blocking-io Leave the SSH transport socket without read/write\n");
|
||||
printf(" timeouts so it blocks naturally\n");
|
||||
printf(" --outbuf=MODE stdout/stderr buffering: N (none/unbuffered),\n");
|
||||
printf(" L (line-buffered), or B (block-buffered, default)\n");
|
||||
printf(" --progress Show transfer progress\n");
|
||||
printf(" -P Partial mode with progress (retention incomplete)\n");
|
||||
printf(" -8, --8-bit-output Leave high-bit characters unescaped in output\n");
|
||||
printf(" --iconv=LOCAL[,REMOTE] Convert file-NAME charsets at the wire boundary:\n");
|
||||
printf(" LOCAL is the charset of our file names, REMOTE is the\n");
|
||||
printf(" remote side's charset (defaults to LOCAL). Names are\n");
|
||||
printf(" converted before transmission and back on receipt; a\n");
|
||||
printf(" name that cannot be represented in the target charset\n");
|
||||
printf(" fails that transfer cleanly (rsync-compatible)\n");
|
||||
printf(" --protocol=NUM Force the wire protocol version (must equal the current\n");
|
||||
printf(" PROTOCOL_VERSION; FastSync cannot speak older/virtual\n");
|
||||
printf(" wire formats)\n");
|
||||
printf(" --write-batch=FILE Run the normal live transfer AND also emit a\n");
|
||||
printf(" self-contained batch file of the whole source tree\n");
|
||||
printf(" (implies the single-threaded transfer path)\n");
|
||||
printf(" --only-write-batch=FILE\n");
|
||||
printf(" Emit the batch file only (no destination, no server)\n");
|
||||
printf(" --read-batch=FILE Apply the batch file to the destination (no source, no\n");
|
||||
printf(" server); takes only the destination as an argument\n");
|
||||
printf(" --delete Delete files on receiver not in source\n");
|
||||
printf(" (default timing: delete only after the whole\n");
|
||||
printf(" transfer has succeeded)\n");
|
||||
printf(" --delete-before Delete extras before the transfer starts\n");
|
||||
printf(" (implies --delete)\n");
|
||||
printf(" --delete-during Delete extras once the keep-set manifest is known,\n");
|
||||
printf(" before the data is applied (implies --delete)\n");
|
||||
printf(" --del Alias for --delete-during\n");
|
||||
printf(" --delete-delay Delete extras only after a successful transfer\n");
|
||||
printf(" (implies --delete)\n");
|
||||
printf(" --delete-after Delete only after the whole transfer succeeded\n");
|
||||
printf(" (the default --delete timing; implies --delete)\n");
|
||||
printf(" --delete-excluded Also delete destination files that were excluded on\n");
|
||||
printf(" the source (default protects them, matching rsync)\n");
|
||||
printf(" --max-delete=NUM Never delete more than NUM destination entries per run;\n");
|
||||
printf(" if the extras would exceed NUM, nothing is deleted and\n");
|
||||
printf(" the run fails with a clear error (implies --delete only\n");
|
||||
printf(" when used with it)\n");
|
||||
printf(" --ignore-errors Continue (and still delete) when a source directory is\n");
|
||||
printf(" unreadable during the scan, instead of aborting with no\n");
|
||||
printf(" deletion\n");
|
||||
printf(" --force A file may replace a destination directory by removing\n");
|
||||
printf(" that (non-empty) directory first\n");
|
||||
printf(" --ignore-missing-args A --files-from entry that does not exist under the\n");
|
||||
printf(" source is silently skipped instead of failing the run\n");
|
||||
printf(" --delete-missing-args Implies --ignore-missing-args; also deletes each missing\n");
|
||||
printf(" entry's destination mirror receiver-side. Independent of\n");
|
||||
printf(" --delete (it does not imply --delete; a non-empty directory\n");
|
||||
printf(" mirror is removed only with --force or --delete)\n");
|
||||
printf(" -m, --prune-empty-dirs Do not transfer empty directory entries (--dirs mode);\n");
|
||||
printf(" recursive transfers never send empty dirs\n");
|
||||
printf(" Note: each timing flag implies --delete. Combining a timing flag with\n");
|
||||
printf(" --no-delete (in either order) is rejected as a config error.\n");
|
||||
printf(" --ignore-existing Skip files that already exist on receiver\n");
|
||||
printf(" --delay-updates Put updated files into place only at the end of transfer\n");
|
||||
printf(" --dirs, -d, --old-dirs, --old-d Transfer the named directory entries without\n");
|
||||
printf(" recursing into their contents (-d <dir> mirrors the source\n");
|
||||
printf(" directory empty; with --files-from listed dirs are created\n");
|
||||
printf(" empty and listed files are transferred)\n");
|
||||
printf(" -R, --relative With --files-from, preserve each listed entry's relative path\n");
|
||||
printf(" below the destination root instead of mirroring the full\n");
|
||||
printf(" source path (no effect without --files-from)\n");
|
||||
printf(" --no-implied-dirs With -R --files-from, refuse to place a listed file whose\n");
|
||||
printf(" parent directory is not itself listed\n");
|
||||
printf(" --mkpath Create the destination root directory on the server when it\n");
|
||||
printf(" does not exist yet\n");
|
||||
printf(" --exclude <pattern> Exclude files matching pattern\n");
|
||||
printf(" --include <pattern> Only include files matching pattern\n");
|
||||
printf(" --exclude-from <file> Read exclude patterns from file\n");
|
||||
printf(" --include-from <file> Read include patterns from file\n");
|
||||
printf(" --files-from <file> Read the source file list from FILE (paths relative to the "
|
||||
"source root)\n");
|
||||
printf(" -0, --from0 Entries in --files-from are NUL-delimited\n");
|
||||
printf(" -f, --filter=RULE rsync-style filter rule (+/- include/exclude; repeatable;\n");
|
||||
printf(" both --filter=RULE and the -f RULE / -f=RULE short forms work)\n");
|
||||
printf(" -C, --cvs-exclude Auto-ignore common CVS/SCM files (.git/, .svn/, *.o, *~, ...)\n");
|
||||
printf(" -F Apply per-directory .rsync-filter files during the scan\n");
|
||||
printf(" --max-size <n> Skip files larger than n bytes\n");
|
||||
printf(" --min-size <n> Skip files smaller than n bytes\n");
|
||||
printf(" --max-alloc <SIZE> Maximum single allocation (default: 1G)\n");
|
||||
printf(" --incremental Skip files unchanged since last transfer\n");
|
||||
printf(" --size-only Skip incremental files matching in size, ignoring mtime\n");
|
||||
printf(" -I, --ignore-times Transfer files even when size and mtime match\n");
|
||||
printf(" -@, --modify-window <sec> Modification time tolerance\n");
|
||||
printf(" -u, --update Skip files newer than the source on receiver\n");
|
||||
printf(" --existing Skip files not already present at destination\n");
|
||||
printf(" --compare-dest <dir> Treat DIR (relative to destination root) as an extra\n");
|
||||
printf(" comparison basis: unchanged files are not transferred\n");
|
||||
printf(" (requires --incremental, which is implied)\n");
|
||||
printf(" --copy-dest <dir> Like --compare-dest, but copies the unchanged file from DIR\n");
|
||||
printf(" into the destination instead of transferring its data\n");
|
||||
printf(" --link-dest <dir> Like --copy-dest, but hard-links the unchanged file from DIR\n");
|
||||
printf(" into the destination (repeatable; earlier DIRs win)\n");
|
||||
printf(" --checksum-choice, --cc <alg> Whole-file checksum algorithm for --incremental/\n");
|
||||
printf(" --checksum compares (xxh64/xxhash or md5; default xxh64 with\n");
|
||||
printf(" seed 0). The seed comes from --checksum-seed\n");
|
||||
printf(" --checksum-seed <num> Seed for the whole-file xxHash64 digest (and the delta\n");
|
||||
printf(" block strong hash, low 32 bits); md5 ignores the seed. The\n");
|
||||
printf(" digest algorithm and seed must match on sender and receiver\n");
|
||||
printf(" --delta Delta transfer for changed files (requires --incremental)\n");
|
||||
printf(" -W, --whole-file Transfer changed files without delta processing\n");
|
||||
printf(" -y, --fuzzy Use a similar-named file already in the destination\n");
|
||||
printf(" directory as the delta basis when the destination has no\n");
|
||||
printf(" usable file at the exact path (saves bandwidth; implies\n");
|
||||
printf(" --incremental and --delta; inert with --whole-file,\n");
|
||||
printf(" --no-delta, or --no-incremental)\n");
|
||||
printf(" --no-fuzzy Disable --fuzzy\n");
|
||||
printf(" --delta-block <n>, --block-size <n>\n");
|
||||
printf(" Delta block size in bytes (default: %d)\n", DELTA_BLOCK_SIZE_DEFAULT);
|
||||
printf(" --delta-max <n> Max file size for delta transfer (default: %llu)\n",
|
||||
DELTA_MAX_FILE_SIZE);
|
||||
printf(" -j, --threads Enable multithreading\n");
|
||||
printf(" --chunk-serialization Enable chunk serialization (long form only)\n");
|
||||
printf(" -s, --secluded-args Protect-args compatibility option (no effect; remote\n");
|
||||
printf(" SSH argv is already built injection-safe)\n");
|
||||
printf(" --sendfile Enable sendfile zero-copy (TCP only; long form only)\n");
|
||||
printf(" --compress-choice <alg> Compression algorithm (default: zstd)\n");
|
||||
printf(" --zc <alg> Alias for --compress-choice\n");
|
||||
printf(" -v, --verbose Enable debug logging\n");
|
||||
printf(" -q, --quiet Suppress non-error output\n");
|
||||
printf(" --debug=FLAGS Fine-grained debug logging (use --debug=help for flags)\n");
|
||||
printf(" --info=FLAGS Fine-grained info: copy,misc,skip,stats,all,none\n");
|
||||
printf(" none suppresses info even with --verbose\n");
|
||||
printf(" --preserve Preserve file metadata (long form only)\n");
|
||||
printf(" -E, --executability Preserve executable permission bits\n");
|
||||
printf(" -X, --xattrs Preserve user extended attributes (user.* only;\n");
|
||||
printf(" privileged security.*/trusted.* namespaces are\n");
|
||||
printf(" never captured or applied)\n");
|
||||
printf(" -A, --acls Preserve POSIX ACLs (the system.posix_acl_* xattrs;\n");
|
||||
printf(" setting an ACL the receiver is not permitted to\n");
|
||||
printf(" set is warned and skipped, never fatal)\n");
|
||||
printf(" --fake-super Store the source uid/gid/mode/mtime in a reserved\n");
|
||||
printf(" user.fastsync.stat xattr on each written file and\n");
|
||||
printf(" re-apply it (fd-relative) on a privileged run; the\n");
|
||||
printf(" recording format diverges from rsync's user.rsync.%%stat%%\n");
|
||||
printf(" --super Permit the receiver to attempt super-user activities\n");
|
||||
printf(" (char/block device-node creation, --write-devices)\n");
|
||||
printf(" within the confined receive root. Never elevates\n");
|
||||
printf(" privileges and never bypasses confinement; ownership\n");
|
||||
printf(" is still applied only with an explicit identity flag\n");
|
||||
printf(" (--numeric-ids/--chown/--usermap/--groupmap/--copy-as)\n");
|
||||
printf(" --no-super Forbid those super-user activities even when the\n");
|
||||
printf(" receiver is running as root\n");
|
||||
printf(" --chmod <changes> Modify transferred permissions (rsync syntax)\n");
|
||||
printf(" --numeric-ids Do not map uid/gid by name: use the source numeric\n");
|
||||
printf(" ids directly when applying ownership\n");
|
||||
printf(" --usermap=MAP Map usernames when applying ownership: comma-separated\n");
|
||||
printf(" FROM:TO rules, first match wins. FROM/TO are names\n");
|
||||
printf(" (resolved on the source machine), * (match any /\n");
|
||||
printf(" current user), or @N numeric ids. e.g. *:nobody\n");
|
||||
printf(" --groupmap=MAP Map group names when applying ownership (same syntax)\n");
|
||||
printf(" --chown=USER:GROUP Override the ownership of transferred files. Forms:\n");
|
||||
printf(" USER:GROUP, USER (owner only), :GROUP (group only); a\n");
|
||||
printf(" value of * means the current/root user as appropriate.\n");
|
||||
printf(" Names resolve on the source machine; @N for numerics.\n");
|
||||
printf(" (Metadata is enabled with --preserve; -M now means\n");
|
||||
printf(" rsync's --remote-option.)\n");
|
||||
printf(" --copy-as=USER[:GROUP] Force every written entry (files, dirs, symlinks\n");
|
||||
printf(" and special nodes) to USER[:GROUP], resolved on the\n");
|
||||
printf(" source machine like --chown. Requires a privileged\n");
|
||||
printf(" (root) receiver and implies --preserve; an\n");
|
||||
printf(" unprivileged receiver refuses the transfer. Never\n");
|
||||
printf(" switches process credentials (safe-subset; see\n");
|
||||
printf(" RSYNC_COMPAT.md). A daemon refuses it.\n");
|
||||
printf(" --chunk-size <n> Chunk size in bytes (default: %d)\n", DEFAULT_CHUNK_SIZE);
|
||||
printf(" --source-dir <path> Source directory\n");
|
||||
printf(" --dest-dir <path> Destination directory\n");
|
||||
printf(" --save-to-disk Write received files to disk\n");
|
||||
printf(" --server-host <ip> Server IP address (default: 127.0.0.1)\n");
|
||||
printf(" --server-port <n> Server port (default: 8080)\n");
|
||||
printf(" --password-file <f> Authenticate a host::module/path daemon destination.\n");
|
||||
printf(" The file's first user:password line supplies the\n");
|
||||
printf(" username and password (only a SHA-256 digest of the\n");
|
||||
printf(" password is sent; keep the file mode 0600)\n");
|
||||
printf(" --no-motd Suppress display of the daemon's MOTD (the server\n");
|
||||
printf(" still sends it; the client just does not show it)\n");
|
||||
printf(" --bwlimit <KB/s> Bandwidth limit in kilobytes per second\n");
|
||||
printf(" --tls Enable TLS encryption\n");
|
||||
printf(" --cert <path> TLS certificate file (PEM)\n");
|
||||
printf(" --key <path> TLS private key file (PEM)\n");
|
||||
printf(" --ca <path> TLS CA certificate file (PEM)\n");
|
||||
printf(" --timeout <sec> I/O timeout in seconds (default: 30; long form only)\n");
|
||||
printf(" --contimeout <sec> Connection timeout in seconds (default: 10)\n");
|
||||
printf(" --stop-after=MINS Stop the transfer after MINS minutes (a positive\n");
|
||||
printf(" integer); whatever was already transferred is kept\n");
|
||||
printf(" --stop-at=TIME Stop at an absolute time: HH:MM, HH:MM:SS, or\n");
|
||||
printf(" now+N[smhd] (a time already in the past stops the\n");
|
||||
printf(" transfer immediately; client-only). An early stop\n");
|
||||
printf(" skips the late --delete keep-set so it cannot delete\n");
|
||||
printf(" source mirrors that were not yet scanned\n");
|
||||
printf(" --address <ip> Bind the outgoing client socket to this source address\n");
|
||||
printf(" -4, --ipv4 Force IPv4 for destination resolution\n");
|
||||
printf(" -6, --ipv6 Force IPv6 for destination resolution\n");
|
||||
printf(" --sockopts=OPTS Comma-separated OPT=VAL socket options applied before connect:\n");
|
||||
printf(" TCP_NODELAY, SO_KEEPALIVE, SO_RCVBUF, SO_SNDBUF, SO_REUSEADDR\n");
|
||||
printf(" --backup Backup existing files before overwriting\n");
|
||||
printf(" --backup-dir <dir> Directory for backups (requires --backup)\n");
|
||||
printf(" --suffix <str> Backup suffix (default: ~)\n");
|
||||
printf(" --stats Print transfer statistics at end\n");
|
||||
printf(" -i, --itemize-changes Print an rsync-style per-file change line\n");
|
||||
printf(" --out-format=FORMAT Output format for changed files (%%f %%n %%l %%b %%M %%%%)\n");
|
||||
printf(" --list-only List source files instead of transferring\n");
|
||||
printf(" --log-file-format=FORMAT Per-file log line format (needs --log-file)\n");
|
||||
printf(" -h, --human-readable Print byte sizes in human-readable form\n");
|
||||
printf(" --max-depth <n> Maximum directory depth (0=unlimited)\n");
|
||||
printf(" -x, --one-file-system Do not cross filesystem boundaries\n");
|
||||
printf(" --log-file <path> Write log messages to file\n");
|
||||
printf(" --stderr=MODE Route logging to stderr: errors or all\n");
|
||||
printf(" --partial Keep partial files on interrupted transfer\n");
|
||||
printf(" --partial-dir <dir> Directory for partial files\n");
|
||||
printf(" -T, --temp-dir <dir> Scratch dir for temp files before atomic install\n");
|
||||
printf(" --fastsync-server-path <path>\n");
|
||||
printf(" Path to fastsync-server on remote (default: fastsync-server)\n");
|
||||
printf(" --old-args Accepted for rsync CLI compatibility; no effect (the\n");
|
||||
printf(" remote server path is always safely quoted now)\n");
|
||||
printf(" -M, --remote-option=OPT Append OPT to the REMOTE server invocation over SSH\n");
|
||||
printf(" (repeatable; each value is single-quote-escaped on the remote\n");
|
||||
printf(" command line; empty values and values with control characters\n");
|
||||
printf(" are rejected; -M OPT, -M=OPT and --remote-option=OPT work)\n");
|
||||
printf(" --trust-sender Trust the remote sender's file list: the receiver skips its\n");
|
||||
printf(" own up-front path-traversal/containment re-validation of the\n");
|
||||
printf(" incoming file list (fewer checks, faster, potentially unsafe).\n");
|
||||
printf(" Local receiver policy: never sent to the peer, off by default\n");
|
||||
printf(" -l, --links Copy symlinks as symlinks\n");
|
||||
printf(" --copy-links Transform symlinks into referent files\n");
|
||||
printf(" --safe-links Skip symlinks that point outside transfer tree\n");
|
||||
printf(" --copy-unsafe-links Only transform unsafe symlinks into referent files\n");
|
||||
printf(" -k, --copy-dirlinks Transform symlinks to directories into real dirs\n");
|
||||
printf(" -K, --keep-dirlinks Keep an existing symlink-to-dir as that dir\n");
|
||||
printf(" --munge-links Munge symlink targets on the wire (sender)\n");
|
||||
printf(" -H, --hard-links Preserve hard-link relationships across the transfer\n");
|
||||
printf(" -S, --sparse Handle sparse files efficiently\n");
|
||||
printf(
|
||||
" -D Preserve device and special files (implies --devices --specials)\n");
|
||||
printf(
|
||||
" --devices Recreate device nodes on the destination (privileged; skipped when\n");
|
||||
printf(" the receiver lacks CAP_MKNOD)\n");
|
||||
printf(" --specials Recreate special files (FIFOs) on the destination (sockets "
|
||||
"skipped)\n");
|
||||
printf(" --copy-devices Copy a source device's content as a regular file instead\n");
|
||||
printf(" --write-devices Write received data into an existing destination device node\n");
|
||||
printf(" --inplace Update files in-place (no temp+rename)\n");
|
||||
printf(
|
||||
" --preallocate Allocate destination file space up front (fail-fast on full disk)\n");
|
||||
printf(" --append Resume a shorter destination by appending only its tail\n");
|
||||
printf(" (prefix is not verified; requires --incremental)\n");
|
||||
printf(" --append-verify Like --append, but verifies the retained prefix checksum\n");
|
||||
printf(" before appending (falls back to a full transfer on mismatch)\n");
|
||||
printf(" --fsync Fsync every written file before publication\n");
|
||||
printf(" --compress-level <n> Compression level (default: 5)\n");
|
||||
printf(" --zl <n> Alias for --compress-level\n");
|
||||
printf(" --skip-compress=LIST Skip compression for comma-separated suffixes\n");
|
||||
printf(" --compress-threads <n> Compression worker threads (requires zstd threaded support)\n");
|
||||
printf(" --no-OPTION Disable a supported boolean option\n");
|
||||
printf(" --help Show this help\n");
|
||||
printf(" -V, --version Show version\n");
|
||||
}
|
||||
|
||||
void print_debug_usage(void) {
|
||||
printf("Supported debug flags: IO,PROTO,PACK,UTIL,ALL,NONE\n");
|
||||
printf("Flags may be comma-separated, for example: --debug=io,proto\n");
|
||||
printf("Other rsync debug flags are unsupported and rejected.\n");
|
||||
}
|
||||
@@ -1,7 +0,0 @@
|
||||
#ifndef USAGE_H
|
||||
#define USAGE_H
|
||||
|
||||
void print_usage(void);
|
||||
void print_debug_usage(void);
|
||||
|
||||
#endif
|
||||
@@ -1,402 +0,0 @@
|
||||
#include "receiver.h"
|
||||
|
||||
#include "charset.h"
|
||||
#include "chunk.h"
|
||||
#include "config.h"
|
||||
#include "delay_updates.h"
|
||||
#include "file.h"
|
||||
#include "file_receive.h"
|
||||
#include "log.h"
|
||||
#include "metadata.h"
|
||||
#include "protocol.h"
|
||||
#include "utils.h"
|
||||
#include <stdlib.h>
|
||||
#include <sys/stat.h>
|
||||
|
||||
bool receiver_outcomes_append(ReceiverOutcomes* outcomes, unsigned char code) {
|
||||
if (!outcomes)
|
||||
return false;
|
||||
if (outcomes->count == outcomes->capacity) {
|
||||
size_t new_capacity = outcomes->capacity == 0 ? 64 : outcomes->capacity * 2;
|
||||
if (new_capacity < outcomes->capacity)
|
||||
return false;
|
||||
unsigned char* grown = realloc(outcomes->entries, new_capacity);
|
||||
if (!grown)
|
||||
return false;
|
||||
outcomes->entries = grown;
|
||||
outcomes->capacity = new_capacity;
|
||||
}
|
||||
outcomes->entries[outcomes->count++] = code;
|
||||
return true;
|
||||
}
|
||||
|
||||
void receiver_outcomes_destroy(ReceiverOutcomes* outcomes) {
|
||||
if (!outcomes)
|
||||
return;
|
||||
free(outcomes->entries);
|
||||
outcomes->entries = NULL;
|
||||
outcomes->count = 0;
|
||||
outcomes->capacity = 0;
|
||||
}
|
||||
|
||||
/* End-of-transfer success frame. When --remove-source-files was negotiated
|
||||
each processed data file is acknowledged first (STATUS_NEXT = written,
|
||||
STATUS_OK = skipped) so the sender never removes a source the receiver did
|
||||
not actually store. The frame always ends with a plain STATUS_OK. */
|
||||
bool receiver_send_final_success(int fd, const Config* config, const ReceiverOutcomes* outcomes) {
|
||||
if (!config->remove_source_files)
|
||||
return send_status(fd, STATUS_OK);
|
||||
size_t count = outcomes ? outcomes->count : 0;
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
Status per_file = outcomes->entries[i] == FILE_SAVE_WRITTEN ? STATUS_NEXT : STATUS_OK;
|
||||
if (!send_status(fd, per_file))
|
||||
return false;
|
||||
}
|
||||
return send_status(fd, STATUS_OK);
|
||||
}
|
||||
|
||||
static bool receiver_process_chunk(Chunk* chunk, const ReceiverSink* sink) {
|
||||
if (!chunk || !sink || !sink->store_file)
|
||||
return false;
|
||||
for (int i = 0; i < chunk->element_count; i++) {
|
||||
File* file = chunk->items[i];
|
||||
if (!file) {
|
||||
chunk_destroy(chunk);
|
||||
return false;
|
||||
}
|
||||
chunk->items[i] = NULL;
|
||||
if (!sink->store_file(file, sink->context)) {
|
||||
chunk_destroy(chunk);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
chunk_destroy(chunk);
|
||||
return true;
|
||||
}
|
||||
|
||||
/* P7 Wave D: read one STATUS_DIR_TIMES frame (a count followed by that many
|
||||
* (path, metadata) directory entries) and route every entry through the regular
|
||||
* store_file sink. A dir-time entry is RECORD-ONLY (file->dir_time_only): the
|
||||
* sink accumulates its metadata for end-of-transfer application but creates
|
||||
* nothing, so an empty/pruned source directory is never resurrected. A large
|
||||
* tree arrives as repeated frames, each bounded by MAX_MANIFEST_ENTRIES; a
|
||||
* malformed count or entry is a hard error. */
|
||||
static bool receiver_process_dir_times(int fd, const Config* config, const ReceiverSink* sink) {
|
||||
int count;
|
||||
if (!receive_int(fd, &count) || count < 0 || count > MAX_MANIFEST_ENTRIES)
|
||||
return false;
|
||||
for (int i = 0; i < count; i++) {
|
||||
File* dir = file_receive_dir_time(fd, config);
|
||||
if (!dir || !sink->store_file(dir, sink->context))
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool receiver_process_batch(Config* config, int file_descriptor) {
|
||||
int count;
|
||||
if (config->checksum || !receive_int(file_descriptor, &count) || count < 0 ||
|
||||
count > MAX_MANIFEST_ENTRIES)
|
||||
return false;
|
||||
for (int i = 0; i < count; i++) {
|
||||
char* check_path = receive_wire_str(file_descriptor);
|
||||
if (!check_path)
|
||||
return false;
|
||||
unsigned long long check_size;
|
||||
long long check_mtime;
|
||||
long long check_mtime_nsec;
|
||||
if (!receive_n_data(file_descriptor, &check_size, sizeof(check_size)) ||
|
||||
!receive_n_data(file_descriptor, &check_mtime, sizeof(check_mtime)) ||
|
||||
!receive_n_data(file_descriptor, &check_mtime_nsec, sizeof(check_mtime_nsec)) ||
|
||||
check_mtime_nsec < 0 || check_mtime_nsec >= 1000000000LL) {
|
||||
free(check_path);
|
||||
send_status(file_descriptor, STATUS_ERROR);
|
||||
return false;
|
||||
}
|
||||
/* --trust-sender: accept a ``..``/absolute check path (a trusted sender's
|
||||
odd-but-legit entry) and defer containment to the secure stat below;
|
||||
an empty path is still always rejected. */
|
||||
if (check_path[0] == '\0' ||
|
||||
(!file_get_trust_sender() && !utils_valid_batch_path(check_path))) {
|
||||
free(check_path);
|
||||
send_status(file_descriptor, STATUS_ERROR);
|
||||
return false;
|
||||
}
|
||||
if (check_size > MAX_RECEIVE_WHOLE_FILE_SIZE) {
|
||||
free(check_path);
|
||||
send_status(file_descriptor, STATUS_ERROR);
|
||||
return false;
|
||||
}
|
||||
char* full_path = path_cat(config->receive_root_directory, check_path);
|
||||
if (!full_path) {
|
||||
free(check_path);
|
||||
send_status(file_descriptor, STATUS_ERROR);
|
||||
return false;
|
||||
}
|
||||
struct stat st;
|
||||
bool has_old = file_stat_secure(full_path, &st);
|
||||
long long old_mtime_nsec = 0;
|
||||
if (has_old) {
|
||||
#ifdef __linux__
|
||||
old_mtime_nsec = st.st_mtim.tv_nsec;
|
||||
#endif
|
||||
}
|
||||
bool match = !config->ignore_times && has_old && (unsigned long long)st.st_size == check_size &&
|
||||
metadata_mtime_matches(st.st_mtime, old_mtime_nsec, (time_t)check_mtime,
|
||||
(long)check_mtime_nsec, config->modify_window);
|
||||
bool sent = send_status(file_descriptor, match ? STATUS_OK : STATUS_NEXT);
|
||||
free(full_path);
|
||||
free(check_path);
|
||||
if (!sent)
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
int receiver_process(Config* config, int file_descriptor, const ReceiverSink* sink) {
|
||||
return receiver_process_pending(config, file_descriptor, sink, NULL);
|
||||
}
|
||||
|
||||
/* Runs the whole receive loop. The delete manifest may legitimately arrive
|
||||
either FIRST (--delete-before / --delete-during: the sender transmits the
|
||||
validated keep-set before any file data) or LAST (plain --delete /
|
||||
--delete-after / --delete-delay: the manifest closes the data stream). In
|
||||
the early modes the receiver deletes as soon as the manifest has been read
|
||||
and acknowledges with STATUS_OK so the sender only starts streaming once the
|
||||
deletion has committed (or failed); in the late modes the manifest is held
|
||||
and the deletion is committed only after the terminal STATUS_FINISHED proves
|
||||
the whole transfer succeeded. See receiver_process_pending() for how the -m
|
||||
receiver defers that commit until its disk writer has drained. */
|
||||
int receiver_process_pending(Config* config, int file_descriptor, const ReceiverSink* sink,
|
||||
DeleteManifest** pending_manifest) {
|
||||
Status status;
|
||||
if (!receive_status(file_descriptor, &status))
|
||||
return -1;
|
||||
bool early_delete = config_delete_timing_early(config);
|
||||
/* Parked keep-set for the late/commit timing. Every exit path below frees it
|
||||
exactly once; the only exception is the successful FINISHED handoff, which
|
||||
transfers ownership to *pending_manifest (used by the -m receiver). */
|
||||
DeleteManifest* deferred_manifest = NULL;
|
||||
while (status == STATUS_NEXT || status == STATUS_CHUNK || status == STATUS_CHECK ||
|
||||
status == STATUS_KEEPALIVE || status == STATUS_ABORT || status == STATUS_CHECK_BATCH ||
|
||||
status == STATUS_MKDIR || status == STATUS_MANIFEST || status == STATUS_HARDLINK ||
|
||||
status == STATUS_SYMLINK || status == STATUS_SPECIAL || status == STATUS_DIR_TIMES) {
|
||||
if (status == STATUS_KEEPALIVE) {
|
||||
if (!send_status(file_descriptor, STATUS_KEEPALIVE))
|
||||
goto fail;
|
||||
goto next_status;
|
||||
}
|
||||
if (status == STATUS_ABORT) {
|
||||
log_message(LOG_LEVEL_INFO, "Received abort from client, cleaning up");
|
||||
goto fail;
|
||||
}
|
||||
if (status == STATUS_CHECK) {
|
||||
bool skipped;
|
||||
File* file = receive_incremental_check(file_descriptor, config, &skipped);
|
||||
if (!skipped && (!file || !sink->store_file(file, sink->context)))
|
||||
goto receive_error;
|
||||
} else if (status == STATUS_CHUNK) {
|
||||
Chunk* chunk = receive_chunk_data(file_descriptor, config);
|
||||
if (!chunk || !receiver_process_chunk(chunk, sink))
|
||||
goto receive_error;
|
||||
} else if (status == STATUS_CHECK_BATCH) {
|
||||
if (!receiver_process_batch(config, file_descriptor))
|
||||
goto fail;
|
||||
goto next_status;
|
||||
} else if (status == STATUS_MKDIR) {
|
||||
File* dir = file_receive_directory(file_descriptor, config);
|
||||
if (!dir || !sink->store_file(dir, sink->context))
|
||||
goto receive_error;
|
||||
} else if (status == STATUS_DIR_TIMES) {
|
||||
if (!receiver_process_dir_times(file_descriptor, config, sink))
|
||||
goto receive_error;
|
||||
} else if (status == STATUS_HARDLINK) {
|
||||
File* file = file_receive_hardlink(file_descriptor);
|
||||
if (!file || !sink->store_file(file, sink->context))
|
||||
goto receive_error;
|
||||
} else if (status == STATUS_SYMLINK) {
|
||||
File* sym = file_receive_symlink(file_descriptor, config);
|
||||
if (!sym || !sink->store_file(sym, sink->context))
|
||||
goto receive_error;
|
||||
} else if (status == STATUS_SPECIAL) {
|
||||
File* file = file_receive_special(file_descriptor);
|
||||
if (!file || !sink->store_file(file, sink->context))
|
||||
goto receive_error;
|
||||
} else if (status == STATUS_MANIFEST) {
|
||||
DeleteManifest* manifest = receive_manifest_entries(file_descriptor);
|
||||
if (!manifest)
|
||||
goto fail; /* receive_manifest_entries already sent STATUS_ERROR */
|
||||
if (early_delete) {
|
||||
/* --delete-before / --delete-during: the manifest is authoritative the
|
||||
moment it arrives, before any file data. Delete now and acknowledge
|
||||
so the sender only starts streaming once the deletion committed (or
|
||||
failed). This is the rsync delete-before/delete-during window: a
|
||||
later transfer failure does not restore these deletions. */
|
||||
bool deletion_ok = (config->use_delete || config->delete_missing_args)
|
||||
? manifest_delete_all(config, manifest)
|
||||
: true;
|
||||
delete_manifest_free(manifest);
|
||||
if (!deletion_ok) {
|
||||
send_status(file_descriptor, STATUS_ERROR);
|
||||
goto fail;
|
||||
}
|
||||
if (!send_status(file_descriptor, STATUS_OK))
|
||||
goto fail;
|
||||
} else if (config->use_delete || config->delete_missing_args) {
|
||||
/* Plain --delete / --delete-after / --delete-delay and the
|
||||
--delete-missing-args exact-path deletions: hold the manifest and
|
||||
commit it only after STATUS_FINISHED. */
|
||||
if (deferred_manifest) {
|
||||
log_message(LOG_LEVEL_ERROR, "Received a second delete manifest");
|
||||
delete_manifest_free(deferred_manifest);
|
||||
deferred_manifest = NULL;
|
||||
delete_manifest_free(manifest);
|
||||
send_status(file_descriptor, STATUS_ERROR);
|
||||
goto fail;
|
||||
}
|
||||
deferred_manifest = manifest;
|
||||
} else {
|
||||
delete_manifest_free(manifest);
|
||||
}
|
||||
goto next_status;
|
||||
} else {
|
||||
File* file = file_receive(config, file_descriptor);
|
||||
if (!file) {
|
||||
log_message(LOG_LEVEL_ERROR, "Failed to receive file");
|
||||
goto receive_error;
|
||||
}
|
||||
if (!sink->store_file(file, sink->context))
|
||||
goto receive_error;
|
||||
}
|
||||
next_status:
|
||||
if (!receive_status(file_descriptor, &status))
|
||||
goto receive_error;
|
||||
}
|
||||
if (status != STATUS_FINISHED) {
|
||||
log_message(LOG_LEVEL_ERROR, "Did not receive FINISHED Status");
|
||||
goto receive_error;
|
||||
}
|
||||
/* Commit-style (late) deletion: every data frame has been received and the
|
||||
sender proved the whole tree with STATUS_FINISHED. The single-threaded
|
||||
receiver stores files synchronously, so everything is on disk here and the
|
||||
deletion can be committed before the --delay-updates publication in
|
||||
send_success (the walker skips the staging dir, so staged files are never
|
||||
treated as extras). The -m receiver passes `pending_manifest` because its
|
||||
disk writer may still be draining; the caller commits after the writer has
|
||||
joined so no extra file is removed unless the transfer is known to have
|
||||
succeeded. */
|
||||
if (deferred_manifest) {
|
||||
if (pending_manifest) {
|
||||
*pending_manifest = deferred_manifest;
|
||||
deferred_manifest = NULL;
|
||||
} else {
|
||||
bool deletion_ok = manifest_delete_all(config, deferred_manifest);
|
||||
delete_manifest_free(deferred_manifest);
|
||||
deferred_manifest = NULL;
|
||||
if (!deletion_ok) {
|
||||
send_status(file_descriptor, STATUS_ERROR);
|
||||
goto fail;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (sink->send_success) {
|
||||
if (sink->send_success_frame) {
|
||||
if (!sink->send_success_frame(file_descriptor, sink->context))
|
||||
goto fail;
|
||||
} else if (!send_status(file_descriptor, STATUS_OK)) {
|
||||
goto fail;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
|
||||
fail:
|
||||
/* Failure exits that must not (or already did) report a STATUS_ERROR. The
|
||||
parked keep-set is dropped: never commit a deletion for a failed stream. */
|
||||
if (deferred_manifest) {
|
||||
delete_manifest_free(deferred_manifest);
|
||||
deferred_manifest = NULL;
|
||||
}
|
||||
return -1;
|
||||
|
||||
receive_error:
|
||||
if (deferred_manifest) {
|
||||
delete_manifest_free(deferred_manifest);
|
||||
deferred_manifest = NULL;
|
||||
}
|
||||
if (sink->send_error)
|
||||
send_status(file_descriptor, STATUS_ERROR);
|
||||
return -1;
|
||||
}
|
||||
|
||||
/* ---- Single-threaded sink (used by receiver_receive_files) ---- */
|
||||
|
||||
typedef struct {
|
||||
Config* config;
|
||||
ReceiverOutcomes outcomes;
|
||||
/* P7 Wave D: directory metadata accumulated during the stream, applied only
|
||||
after the whole transfer (and its delete/publication phases) has run so a
|
||||
child write never clobbers a directory mtime. */
|
||||
DirTimeList dir_times;
|
||||
} ReceiverSaveContext;
|
||||
|
||||
static bool receiver_save_file(File* file, void* context_pointer) {
|
||||
ReceiverSaveContext* context = context_pointer;
|
||||
FileSaveResult result = FILE_SAVE_ERROR;
|
||||
if (!context->config->save_to_disk) {
|
||||
/* Nothing is stored; report the file as not-written so a
|
||||
--remove-source-files sender keeps its source. */
|
||||
result = FILE_SAVE_SKIPPED;
|
||||
} else {
|
||||
result = file_save_to_disk_full(context->config->receive_root_directory, file, context->config);
|
||||
}
|
||||
/* A directory's times are deferred, never applied inline: collect the
|
||||
metadata now and apply it at the end. -O/--omit-dir-times is honored by
|
||||
dir_time_list_apply's caller (see receiver_send_success_frame). */
|
||||
if (result != FILE_SAVE_ERROR && file->is_dir && file->metadata &&
|
||||
context->config->use_metadata && !context->config->omit_dir_times &&
|
||||
!dir_time_list_add(&context->dir_times, file->path, file->metadata)) {
|
||||
file_destroy(file);
|
||||
return false;
|
||||
}
|
||||
if (result != FILE_SAVE_ERROR && context->config->remove_source_files && !file->is_dir &&
|
||||
!file->is_special && !file->skip &&
|
||||
!receiver_outcomes_append(&context->outcomes, (unsigned char)result)) {
|
||||
file_destroy(file);
|
||||
return false;
|
||||
}
|
||||
file_destroy(file);
|
||||
return result != FILE_SAVE_ERROR;
|
||||
}
|
||||
|
||||
static bool receiver_send_success_frame(int fd, void* context_pointer) {
|
||||
ReceiverSaveContext* context = context_pointer;
|
||||
/* --delay-updates: the whole protocol stream (including manifest/delete
|
||||
handling, which ran inside receiver_process) has succeeded and every
|
||||
staged file was fully written. Publish them atomically now, before the
|
||||
success/outcome frame tells a --remove-source-files sender it may delete
|
||||
its sources. */
|
||||
if (context->config->delay_updates && context->config->delay_context) {
|
||||
if (!delay_updates_publish(context->config->delay_context, context->config)) {
|
||||
send_status(fd, STATUS_ERROR);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
/* P7 Wave D: every child is now written and the delete / --delay-updates
|
||||
phases have committed, so it is finally safe to stamp directory times.
|
||||
This runs after the deferred deletion because receiver_process commits it
|
||||
before calling this success frame. */
|
||||
dir_time_list_apply(&context->dir_times, context->config->receive_root_directory);
|
||||
return receiver_send_final_success(fd, context->config, &context->outcomes);
|
||||
}
|
||||
|
||||
int receiver_receive_files(Config* config, int file_descriptor) {
|
||||
ReceiverSaveContext context = {.config = config, .outcomes = {0}};
|
||||
dir_time_list_init(&context.dir_times);
|
||||
ReceiverSink sink = {receiver_save_file, &context, true, true, receiver_send_success_frame};
|
||||
int ret = receiver_process(config, file_descriptor, &sink);
|
||||
if (ret != 0 && config->delay_updates && config->delay_context)
|
||||
delay_updates_cleanup(config->delay_context);
|
||||
receiver_outcomes_destroy(&context.outcomes);
|
||||
dir_time_list_free(&context.dir_times);
|
||||
return ret;
|
||||
}
|
||||
@@ -1,48 +0,0 @@
|
||||
#ifndef RECEIVER_H
|
||||
#define RECEIVER_H
|
||||
|
||||
#include "config.h"
|
||||
#include "file.h"
|
||||
#include "file_receive.h"
|
||||
|
||||
typedef bool (*ReceiverFileSink)(File* file, void* context);
|
||||
|
||||
/* Ordered per-file save outcomes for one connection. One entry is appended
|
||||
for every data-bearing file the receiver processes (in the order the files
|
||||
were sent) so the sender of a --remove-source-files transfer can be told
|
||||
which sources were actually written versus skipped on the receiver. */
|
||||
typedef struct {
|
||||
unsigned char* entries; /* FILE_SAVE_WRITTEN or FILE_SAVE_SKIPPED */
|
||||
size_t count;
|
||||
size_t capacity;
|
||||
} ReceiverOutcomes;
|
||||
|
||||
typedef bool (*ReceiverSuccessFrame)(int fd, void* context);
|
||||
|
||||
typedef struct {
|
||||
ReceiverFileSink store_file;
|
||||
void* context;
|
||||
bool send_error;
|
||||
bool send_success;
|
||||
/* Emits the end-of-transfer success frame. When the sender requested
|
||||
--remove-source-files this includes one per-file status per processed
|
||||
data file followed by the final STATUS_OK; otherwise just STATUS_OK. */
|
||||
ReceiverSuccessFrame send_success_frame;
|
||||
} ReceiverSink;
|
||||
|
||||
bool receiver_outcomes_append(ReceiverOutcomes* outcomes, unsigned char code);
|
||||
void receiver_outcomes_destroy(ReceiverOutcomes* outcomes);
|
||||
bool receiver_send_final_success(int fd, const Config* config, const ReceiverOutcomes* outcomes);
|
||||
|
||||
int receiver_process(Config* config, int file_descriptor, const ReceiverSink* sink);
|
||||
/* receiver_process with an escape hatch for the commit-style (late) deletion:
|
||||
when `pending_manifest` is non-NULL the receiver does NOT delete at
|
||||
STATUS_FINISHED itself; instead it stores the owned keep-set manifest there
|
||||
(leaving *pending_manifest untouched on early modes/errors) so the caller can
|
||||
commit the deletion only after its disk writer has fully drained. Pass NULL
|
||||
to keep the default behaviour (delete before the success frame). */
|
||||
int receiver_process_pending(Config* config, int file_descriptor, const ReceiverSink* sink,
|
||||
DeleteManifest** pending_manifest);
|
||||
int receiver_receive_files(Config* config, int file_descriptor);
|
||||
|
||||
#endif
|
||||
+120
-983
File diff suppressed because it is too large
Load Diff
@@ -1,281 +0,0 @@
|
||||
#include "server_cli.h"
|
||||
#include "charset.h"
|
||||
#include "credentials.h"
|
||||
#include "utils.h"
|
||||
#include <limits.h>
|
||||
#include <stdarg.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/socket.h>
|
||||
|
||||
static void set_error(char* err, size_t err_size, const char* fmt, ...) {
|
||||
if (!err || err_size == 0)
|
||||
return;
|
||||
va_list args;
|
||||
va_start(args, fmt);
|
||||
vsnprintf(err, err_size, fmt, args);
|
||||
va_end(args);
|
||||
}
|
||||
|
||||
void server_cli_options_default(ServerCliOptions* opts) {
|
||||
if (!opts)
|
||||
return;
|
||||
memset(opts, 0, sizeof(*opts));
|
||||
opts->destination_root = ".";
|
||||
opts->port = 8080;
|
||||
opts->bind_family = AF_UNSPEC;
|
||||
}
|
||||
|
||||
static bool arg_is(const char* arg, const char* name) {
|
||||
return strcmp(arg, name) == 0;
|
||||
}
|
||||
|
||||
/* Match "--opt" against "--opt=value" / separate-value forms; on the "=" form
|
||||
* *value receives the inline value. Returns true when the argument is the
|
||||
* named option in either form. */
|
||||
static bool arg_has_value(const char* arg, const char* name, const char** value) {
|
||||
if (strcmp(arg, name) == 0)
|
||||
return true; /* separate form; caller takes the next argv slot */
|
||||
size_t name_len = strlen(name);
|
||||
if (strncmp(arg, name, name_len) == 0 && arg[name_len] == '=') {
|
||||
*value = arg + name_len + 1;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static int parse_port_arg(const char* value, int* port, char* err, size_t err_size) {
|
||||
char* end;
|
||||
long p = strtol(value, &end, 10);
|
||||
if (*end != '\0' || p <= 0 || p > 65535) {
|
||||
char* escaped = output_escape(value, false);
|
||||
set_error(err, err_size, "invalid port '%s' (must be 1-65535)",
|
||||
escaped ? escaped : "<allocation failed>");
|
||||
free(escaped);
|
||||
return -1;
|
||||
}
|
||||
*port = (int)p;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int server_cli_parse(int argc, char* argv[], ServerCliOptions* opts, char* err, size_t err_size) {
|
||||
if (err && err_size)
|
||||
err[0] = '\0';
|
||||
server_cli_options_default(opts);
|
||||
|
||||
for (int i = 1; i < argc; i++) {
|
||||
const char* inline_value = NULL;
|
||||
if (arg_is(argv[i], "--help")) {
|
||||
opts->show_help = true;
|
||||
return 1;
|
||||
} else if (arg_is(argv[i], "--stdio")) {
|
||||
opts->stdio_mode = true;
|
||||
} else if (arg_is(argv[i], "--daemon")) {
|
||||
opts->daemon_mode = true;
|
||||
} else if (arg_is(argv[i], "--no-detach")) {
|
||||
opts->no_detach = true;
|
||||
} else if (arg_is(argv[i], "-v") || arg_is(argv[i], "--verbose")) {
|
||||
opts->verbose = true;
|
||||
} else if (arg_is(argv[i], "--tls")) {
|
||||
opts->use_tls = true;
|
||||
} else if (arg_is(argv[i], "--cert")) {
|
||||
if (i + 1 >= argc) {
|
||||
set_error(err, err_size, "missing argument for --cert");
|
||||
return -1;
|
||||
}
|
||||
opts->tls_cert = argv[++i];
|
||||
} else if (arg_is(argv[i], "--key")) {
|
||||
if (i + 1 >= argc) {
|
||||
set_error(err, err_size, "missing argument for --key");
|
||||
return -1;
|
||||
}
|
||||
opts->tls_key = argv[++i];
|
||||
} else if (arg_is(argv[i], "--ca")) {
|
||||
if (i + 1 >= argc) {
|
||||
set_error(err, err_size, "missing argument for --ca");
|
||||
return -1;
|
||||
}
|
||||
opts->tls_ca = argv[++i];
|
||||
} else if (arg_is(argv[i], "--client-cn")) {
|
||||
if (i + 1 >= argc) {
|
||||
set_error(err, err_size, "missing argument for --client-cn");
|
||||
return -1;
|
||||
}
|
||||
opts->client_cn = argv[++i];
|
||||
} else if (arg_is(argv[i], "--destination-root")) {
|
||||
if (i + 1 >= argc) {
|
||||
set_error(err, err_size, "missing argument for --destination-root");
|
||||
return -1;
|
||||
}
|
||||
opts->destination_root = argv[++i];
|
||||
opts->destination_root_set = true;
|
||||
} else if (arg_has_value(argv[i], "--password-file", &inline_value)) {
|
||||
if (!inline_value) {
|
||||
if (i + 1 >= argc) {
|
||||
set_error(err, err_size, "missing argument for --password-file");
|
||||
return -1;
|
||||
}
|
||||
inline_value = argv[++i];
|
||||
}
|
||||
opts->password_file = inline_value;
|
||||
} else if (arg_has_value(argv[i], "--early-input", &inline_value)) {
|
||||
if (!inline_value) {
|
||||
if (i + 1 >= argc) {
|
||||
set_error(err, err_size, "missing argument for --early-input");
|
||||
return -1;
|
||||
}
|
||||
inline_value = argv[++i];
|
||||
}
|
||||
opts->early_input_file = inline_value;
|
||||
} else if (arg_has_value(argv[i], "--hash-credentials", &inline_value)) {
|
||||
if (!inline_value) {
|
||||
if (i + 1 >= argc) {
|
||||
set_error(err, err_size, "missing argument for --hash-credentials");
|
||||
return -1;
|
||||
}
|
||||
inline_value = argv[++i];
|
||||
}
|
||||
opts->hash_credentials_file = inline_value;
|
||||
} else if (arg_has_value(argv[i], "--iterations", &inline_value)) {
|
||||
if (!inline_value) {
|
||||
if (i + 1 >= argc) {
|
||||
set_error(err, err_size, "missing argument for --iterations");
|
||||
return -1;
|
||||
}
|
||||
inline_value = argv[++i];
|
||||
}
|
||||
char* end = NULL;
|
||||
long n = strtol(inline_value, &end, 10);
|
||||
if (!end || *end != '\0' || n < (long)CREDENTIAL_MIN_ITERS ||
|
||||
n > (long)CREDENTIAL_MAX_ITERS) {
|
||||
set_error(err, err_size, "--iterations must be in [%u,%u], got '%s'", CREDENTIAL_MIN_ITERS,
|
||||
CREDENTIAL_MAX_ITERS, inline_value);
|
||||
return -1;
|
||||
}
|
||||
opts->hash_iterations = (uint32_t)n;
|
||||
opts->hash_iterations_set = true;
|
||||
} else if (arg_is(argv[i], "--address")) {
|
||||
if (i + 1 >= argc) {
|
||||
set_error(err, err_size, "missing argument for --address");
|
||||
return -1;
|
||||
}
|
||||
opts->bind_address = argv[++i];
|
||||
} else if (arg_is(argv[i], "-4") || arg_is(argv[i], "--ipv4")) {
|
||||
if (opts->bind_family == AF_INET6) {
|
||||
set_error(err, err_size, "--ipv4 and --ipv6 are mutually exclusive");
|
||||
return -1;
|
||||
}
|
||||
opts->bind_family = AF_INET;
|
||||
} else if (arg_is(argv[i], "-6") || arg_is(argv[i], "--ipv6")) {
|
||||
if (opts->bind_family == AF_INET) {
|
||||
set_error(err, err_size, "--ipv4 and --ipv6 are mutually exclusive");
|
||||
return -1;
|
||||
}
|
||||
opts->bind_family = AF_INET6;
|
||||
} else if (arg_is(argv[i], "--allow-delete")) {
|
||||
opts->allow_delete = true;
|
||||
} else if (arg_is(argv[i], "--trust-sender")) {
|
||||
opts->trust_sender = true;
|
||||
} else if (arg_is(argv[i], "--no-super")) {
|
||||
opts->no_super = true;
|
||||
} else if (arg_is(argv[i], "--allow-unauthenticated")) {
|
||||
opts->allow_unauthenticated = true;
|
||||
} else if (arg_has_value(argv[i], "--iconv", &inline_value)) {
|
||||
if (!inline_value) {
|
||||
if (i + 1 >= argc) {
|
||||
set_error(err, err_size, "missing argument for --iconv");
|
||||
return -1;
|
||||
}
|
||||
inline_value = argv[++i];
|
||||
}
|
||||
opts->iconv_spec = inline_value;
|
||||
} else if (arg_is(argv[i], "-p")) {
|
||||
if (i + 1 >= argc) {
|
||||
set_error(err, err_size, "missing argument for -p");
|
||||
return -1;
|
||||
}
|
||||
opts->port_set = true;
|
||||
if (parse_port_arg(argv[++i], &opts->port, err, err_size) != 0)
|
||||
return -1;
|
||||
} else {
|
||||
if (arg_has_value(argv[i], "--config", &inline_value)) {
|
||||
if (!inline_value) {
|
||||
if (i + 1 >= argc) {
|
||||
set_error(err, err_size, "missing argument for --config");
|
||||
return -1;
|
||||
}
|
||||
inline_value = argv[++i];
|
||||
}
|
||||
opts->config_path = inline_value;
|
||||
} else if (arg_has_value(argv[i], "--dparam", &inline_value)) {
|
||||
if (!inline_value) {
|
||||
if (i + 1 >= argc) {
|
||||
set_error(err, err_size, "missing argument for --dparam");
|
||||
return -1;
|
||||
}
|
||||
inline_value = argv[++i];
|
||||
}
|
||||
const char** grown =
|
||||
realloc((char**)opts->dparams, (size_t)(opts->dparam_count + 1) * sizeof(const char*));
|
||||
if (!grown) {
|
||||
set_error(err, err_size, "out of memory parsing --dparam");
|
||||
return -1;
|
||||
}
|
||||
opts->dparams = grown;
|
||||
opts->dparams[opts->dparam_count++] = inline_value;
|
||||
} else if (argv[i][0] == '-') {
|
||||
char* escaped = output_escape(argv[i], false);
|
||||
set_error(err, err_size, "unknown option: %s", escaped ? escaped : "<allocation failed>");
|
||||
free(escaped);
|
||||
return -1;
|
||||
} else {
|
||||
set_error(err, err_size, "unexpected argument '%s'", argv[i]);
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Cross-mode validation. */
|
||||
if (opts->stdio_mode && opts->daemon_mode) {
|
||||
set_error(err, err_size, "--stdio and --daemon are mutually exclusive");
|
||||
return -1;
|
||||
}
|
||||
if (opts->daemon_mode && opts->destination_root_set) {
|
||||
set_error(err, err_size,
|
||||
"--destination-root cannot be combined with --daemon (module paths "
|
||||
"replace it)");
|
||||
return -1;
|
||||
}
|
||||
if (!opts->daemon_mode &&
|
||||
(opts->config_path != NULL || opts->dparam_count > 0 || opts->no_detach ||
|
||||
opts->password_file != NULL || opts->early_input_file != NULL)) {
|
||||
set_error(err, err_size,
|
||||
"--config, --dparam, --no-detach, --password-file, and --early-input require "
|
||||
"--daemon");
|
||||
return -1;
|
||||
}
|
||||
if (opts->hash_credentials_file != NULL && (opts->daemon_mode || opts->stdio_mode)) {
|
||||
set_error(err, err_size, "--hash-credentials cannot be combined with --daemon or --stdio");
|
||||
return -1;
|
||||
}
|
||||
if (opts->hash_iterations_set && opts->hash_credentials_file == NULL) {
|
||||
set_error(err, err_size, "--iterations requires --hash-credentials");
|
||||
return -1;
|
||||
}
|
||||
/* --iconv: reject a malformed CONVERT_SPEC or an unsupported charset name at
|
||||
startup (a probe iconv_open is attempted). */
|
||||
if (opts->iconv_spec != NULL && !charset_spec_valid(opts->iconv_spec)) {
|
||||
set_error(err, err_size, "--iconv requires LOCAL[,REMOTE] charset names supported by iconv");
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
void server_cli_options_free(ServerCliOptions* opts) {
|
||||
if (!opts)
|
||||
return;
|
||||
free((char**)opts->dparams);
|
||||
opts->dparams = NULL;
|
||||
opts->dparam_count = 0;
|
||||
}
|
||||
@@ -1,68 +0,0 @@
|
||||
#ifndef SERVER_CLI_H
|
||||
#define SERVER_CLI_H
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
|
||||
/* Parsed fastsync-server command line. All string members are borrowed
|
||||
* pointers into the original argv (valid for the life of the argv array the
|
||||
* caller passed to server_cli_parse); dparams points at the raw --dparam
|
||||
* argument strings. No member owns heap memory. */
|
||||
typedef struct ServerCliOptions {
|
||||
bool stdio_mode; /* --stdio */
|
||||
bool daemon_mode; /* --daemon */
|
||||
bool no_detach; /* --no-detach */
|
||||
bool verbose; /* -v / --verbose */
|
||||
bool show_help; /* --help */
|
||||
bool use_tls; /* --tls */
|
||||
const char* tls_cert; /* --cert */
|
||||
const char* tls_key; /* --key */
|
||||
const char* tls_ca; /* --ca */
|
||||
const char* client_cn; /* --client-cn */
|
||||
bool destination_root_set; /* an explicit --destination-root was given */
|
||||
const char* destination_root; /* --destination-root value ("." if unset) */
|
||||
bool port_set; /* an explicit -p was given */
|
||||
int port; /* -p value (default 8080 when unset) */
|
||||
const char* config_path; /* --config value, or NULL */
|
||||
const char* password_file; /* --password-file value, or NULL (daemon) */
|
||||
const char* early_input_file; /* --early-input value, or NULL (daemon) */
|
||||
/* --hash-credentials=FILE: read `user:password` lines from FILE and print
|
||||
* new-format credential-store lines to stdout, then exit. Standalone mode
|
||||
* (mutually exclusive with --daemon/--stdio). */
|
||||
const char* hash_credentials_file;
|
||||
bool hash_iterations_set; /* an explicit --iterations was given */
|
||||
uint32_t hash_iterations; /* --iterations value (default CREDENTIAL_DEFAULT_ITERS) */
|
||||
const char** dparams; /* raw --dparam override strings */
|
||||
int dparam_count;
|
||||
const char* bind_address; /* --address */
|
||||
int bind_family; /* AF_UNSPEC / AF_INET / AF_INET6 */
|
||||
bool allow_delete; /* --allow-delete */
|
||||
bool trust_sender; /* --trust-sender */
|
||||
bool allow_unauthenticated; /* --allow-unauthenticated */
|
||||
/* --no-super: operator veto forcing SUPER_MODE_OFF for every connection, so
|
||||
* the receiver never attempts super-user activities (ownership application,
|
||||
* device-node creation) even when running as root. Applies to --stdio and
|
||||
* --daemon alike; also makes the server refuse any client --copy-as. */
|
||||
bool no_super; /* --no-super */
|
||||
/* --iconv=CONVERT_SPEC: the server's own LOCAL charset declaration. The
|
||||
* client's full spec rides the wire config frame anyway; when the server is
|
||||
* started with its own --iconv, its LOCAL half overrides the local charset
|
||||
* the client assumed so the server converts received names to ITS charset.
|
||||
* Borrowed pointer into argv (never owns heap). */
|
||||
const char* iconv_spec; /* --iconv value, or NULL */
|
||||
} ServerCliOptions;
|
||||
|
||||
/* Parse argc/argv into *opts. Zero-initialize *opts before calling (or use
|
||||
* server_cli_options_default). Returns:
|
||||
* 1 -- --help was requested (opts->show_help set; caller prints usage).
|
||||
* 0 -- parsed successfully.
|
||||
* -1 -- invalid arguments (err is filled with the reason).
|
||||
*/
|
||||
void server_cli_options_default(ServerCliOptions* opts);
|
||||
int server_cli_parse(int argc, char* argv[], ServerCliOptions* opts, char* err, size_t err_size);
|
||||
/* Release the only heap the parsed options own (the dparams pointer array; the
|
||||
* strings it points at are borrowed from argv and are not freed). Safe to
|
||||
* call on a zero-initialized/defaulted struct. */
|
||||
void server_cli_options_free(ServerCliOptions* opts);
|
||||
#endif
|
||||
+15
-19
@@ -1,18 +1,16 @@
|
||||
#include "log.h"
|
||||
#include "array_list.h"
|
||||
#include "protocol.h"
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
ArrayList* array_list_create(void (*item_destroyer)(void* item)) {
|
||||
ArrayList* list = (ArrayList*)protocol_alloc(sizeof(ArrayList));
|
||||
ArrayList *array_list_create(void (*item_destroyer)(void *item)) {
|
||||
ArrayList *list = (ArrayList *)malloc(sizeof(ArrayList));
|
||||
if (list == NULL) {
|
||||
log_perror("ERROR: Could not allocate memory for array list struct");
|
||||
perror("ERROR: Could not allocate memory for array list struct");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
list->items = protocol_alloc(INITIAL_ARRAY_SIZE * sizeof(void*));
|
||||
list->items = malloc(INITIAL_ARRAY_SIZE * sizeof(void *));
|
||||
if (list->items == NULL) {
|
||||
free(list);
|
||||
return NULL;
|
||||
@@ -23,7 +21,7 @@ ArrayList* array_list_create(void (*item_destroyer)(void* item)) {
|
||||
return list;
|
||||
}
|
||||
|
||||
void array_list_delete(ArrayList* array_list) {
|
||||
void array_list_delete(ArrayList *array_list) {
|
||||
if (array_list == NULL)
|
||||
return;
|
||||
if (array_list->item_destroyer != NULL) {
|
||||
@@ -36,15 +34,14 @@ void array_list_delete(ArrayList* array_list) {
|
||||
free(array_list);
|
||||
}
|
||||
|
||||
static bool array_list_extend(ArrayList* array_list) {
|
||||
if (array_list == NULL)
|
||||
return false;
|
||||
bool array_list_extend(ArrayList *array_list) {
|
||||
if (array_list == NULL) return false;
|
||||
int new_capacity = array_list->capacity * 2;
|
||||
if (new_capacity == 0)
|
||||
new_capacity = INITIAL_ARRAY_SIZE;
|
||||
void* new_items = protocol_realloc(array_list->items, new_capacity * sizeof(void*));
|
||||
void *new_items = realloc(array_list->items, new_capacity * sizeof(void *));
|
||||
if (new_items == NULL) {
|
||||
log_perror("ERROR: Could not reallocate memory for array list items");
|
||||
perror("ERROR: Could not reallocate memory for array list items");
|
||||
return false;
|
||||
}
|
||||
array_list->items = new_items;
|
||||
@@ -52,9 +49,8 @@ static bool array_list_extend(ArrayList* array_list) {
|
||||
return true;
|
||||
}
|
||||
|
||||
bool array_list_add(ArrayList* array_list, void* item) {
|
||||
if (array_list == NULL)
|
||||
return false;
|
||||
bool array_list_add(ArrayList *array_list, void *item) {
|
||||
if (array_list == NULL) return false;
|
||||
if (array_list->capacity == array_list->size) {
|
||||
if (!array_list_extend(array_list))
|
||||
return false;
|
||||
@@ -64,15 +60,15 @@ bool array_list_add(ArrayList* array_list, void* item) {
|
||||
return true;
|
||||
}
|
||||
|
||||
void** array_list_to_array(const ArrayList* array_list) {
|
||||
void **array_list_to_array(ArrayList *array_list) {
|
||||
if (array_list == NULL) {
|
||||
return NULL;
|
||||
}
|
||||
void** array = protocol_alloc(array_list->size * sizeof(void*));
|
||||
void **array = malloc(array_list->size * sizeof(void *));
|
||||
if (array == NULL) {
|
||||
log_perror("Could not malloc space for array from array list!");
|
||||
perror("Could not malloc space for array from array list!");
|
||||
return NULL;
|
||||
}
|
||||
memcpy(array, array_list->items, array_list->size * sizeof(void*));
|
||||
memcpy(array, array_list->items, array_list->size * sizeof(void *));
|
||||
return array;
|
||||
}
|
||||
|
||||
@@ -6,15 +6,16 @@
|
||||
#define INITIAL_ARRAY_SIZE 100
|
||||
|
||||
typedef struct ArrayList {
|
||||
void** items;
|
||||
void **items;
|
||||
int size;
|
||||
int capacity;
|
||||
void (*item_destroyer)(void* item);
|
||||
void (*item_destroyer)(void *item);
|
||||
} ArrayList;
|
||||
|
||||
ArrayList* array_list_create(void (*item_destroyer)(void* item));
|
||||
void array_list_delete(ArrayList* array_list);
|
||||
bool array_list_add(ArrayList* array_list, void* item);
|
||||
void** array_list_to_array(const ArrayList* array_list);
|
||||
ArrayList *array_list_create(void (*item_destroyer)(void *item));
|
||||
void array_list_delete(ArrayList *array_list);
|
||||
bool array_list_extend(ArrayList *array_list);
|
||||
bool array_list_add(ArrayList *array_list, void *item);
|
||||
void **array_list_to_array(ArrayList *array_list);
|
||||
|
||||
#endif
|
||||
|
||||
@@ -1,160 +0,0 @@
|
||||
#include "batch.h"
|
||||
#include "data.h"
|
||||
#include "file.h"
|
||||
#include "file_receive.h"
|
||||
#include "log.h"
|
||||
#include <errno.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <unistd.h>
|
||||
|
||||
/* Serialization metadata mode for the batch stream, captured from the config at
|
||||
* batch_write_header time. The header persists it into the file so a batch is
|
||||
* self-describing: batch_read_apply re-reads it from the file (not from the
|
||||
* reading config), so a batch written with -M is applied identically by an
|
||||
* invoking process regardless of its own -M setting. The batch driver is a
|
||||
* single sequential scan pass within one thread, so this module-level flag is
|
||||
* safe. */
|
||||
static bool batch_metadata_mode = false;
|
||||
|
||||
static bool write_all_bytes(int fd, const void* data, size_t size) {
|
||||
const unsigned char* p = (const unsigned char*)data;
|
||||
size_t done = 0;
|
||||
while (done < size) {
|
||||
ssize_t n = write(fd, p + done, size - done);
|
||||
if (n < 0 && errno == EINTR)
|
||||
continue;
|
||||
if (n <= 0)
|
||||
return false;
|
||||
done += (size_t)n;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool batch_write_header(int fd, const Config* config) {
|
||||
if (fd < 0)
|
||||
return false;
|
||||
batch_metadata_mode = config != NULL && config->use_metadata;
|
||||
if (!write_all_bytes(fd, BATCH_MAGIC, BATCH_MAGIC_LEN))
|
||||
return false;
|
||||
unsigned char version = BATCH_FORMAT_VERSION;
|
||||
if (!write_all_bytes(fd, &version, 1))
|
||||
return false;
|
||||
unsigned char mode = batch_metadata_mode ? 1 : 0;
|
||||
return write_all_bytes(fd, &mode, 1);
|
||||
}
|
||||
|
||||
bool batch_write_chunk(int fd, Chunk* chunk) {
|
||||
if (fd < 0 || chunk == NULL)
|
||||
return false;
|
||||
Data* serialized = chunk_serialize(chunk, batch_metadata_mode);
|
||||
if (serialized == NULL)
|
||||
return false;
|
||||
bool ok = false;
|
||||
unsigned long long length = (unsigned long long)serialized->size;
|
||||
if (length > BATCH_MAX_RECORD) {
|
||||
log_message(LOG_LEVEL_ERROR, "batch: record size %llu exceeds the %llu-byte cap", length,
|
||||
(unsigned long long)BATCH_MAX_RECORD);
|
||||
} else if (write_all_bytes(fd, &length, sizeof(length)) &&
|
||||
(length == 0 || write_all_bytes(fd, serialized->data, (size_t)length))) {
|
||||
ok = true;
|
||||
}
|
||||
data_destroy(serialized);
|
||||
return ok;
|
||||
}
|
||||
|
||||
/* Read exactly `size` bytes. Returns true on success. On reaching EOF, sets
|
||||
* *clean_eof only when no bytes had been read yet (a clean boundary) and returns
|
||||
* that value, so a truncated record (EOF mid-read) yields false. */
|
||||
static bool read_exact(int fd, void* data, size_t size, bool* clean_eof) {
|
||||
unsigned char* p = (unsigned char*)data;
|
||||
size_t done = 0;
|
||||
while (done < size) {
|
||||
ssize_t n = read(fd, p + done, size - done);
|
||||
if (n < 0 && errno == EINTR)
|
||||
continue;
|
||||
if (n == 0) {
|
||||
if (clean_eof)
|
||||
*clean_eof = done == 0;
|
||||
return done == 0;
|
||||
}
|
||||
if (n < 0)
|
||||
return false;
|
||||
done += (size_t)n;
|
||||
}
|
||||
if (clean_eof)
|
||||
*clean_eof = false;
|
||||
return true;
|
||||
}
|
||||
|
||||
int batch_read_apply(int fd, const Config* config, const char* dest_root) {
|
||||
if (fd < 0 || dest_root == NULL || dest_root[0] == '\0')
|
||||
return -1;
|
||||
|
||||
char magic[BATCH_MAGIC_LEN];
|
||||
bool eof = false;
|
||||
if (!read_exact(fd, magic, BATCH_MAGIC_LEN, &eof) || eof ||
|
||||
memcmp(magic, BATCH_MAGIC, BATCH_MAGIC_LEN) != 0) {
|
||||
log_message(LOG_LEVEL_ERROR, "batch: malformed header (bad magic)");
|
||||
return -1;
|
||||
}
|
||||
unsigned char version;
|
||||
if (!read_exact(fd, &version, 1, &eof) || eof || version != BATCH_FORMAT_VERSION) {
|
||||
log_message(LOG_LEVEL_ERROR, "batch: malformed header (bad or missing format version)");
|
||||
return -1;
|
||||
}
|
||||
unsigned char mode;
|
||||
if (!read_exact(fd, &mode, 1, &eof) || eof || (mode != 0 && mode != 1)) {
|
||||
log_message(LOG_LEVEL_ERROR, "batch: malformed header (bad metadata flag)");
|
||||
return -1;
|
||||
}
|
||||
bool use_metadata = mode == 1;
|
||||
|
||||
while (1) {
|
||||
unsigned long long length;
|
||||
if (!read_exact(fd, &length, sizeof(length), &eof)) {
|
||||
log_message(LOG_LEVEL_ERROR, "batch: truncated length prefix");
|
||||
return -1;
|
||||
}
|
||||
if (eof)
|
||||
break; /* clean end of stream */
|
||||
if (length == 0 || length > BATCH_MAX_RECORD) {
|
||||
log_message(LOG_LEVEL_ERROR, "batch: rejected record length %llu (valid range 1..%llu)",
|
||||
length, (unsigned long long)BATCH_MAX_RECORD);
|
||||
return -1;
|
||||
}
|
||||
char* record = (char*)malloc((size_t)length);
|
||||
if (record == NULL) {
|
||||
log_message(LOG_LEVEL_ERROR, "batch: could not allocate a %llu-byte record", length);
|
||||
return -1;
|
||||
}
|
||||
if (!read_exact(fd, record, (size_t)length, &eof) || eof) {
|
||||
log_message(LOG_LEVEL_ERROR, "batch: truncated chunk record");
|
||||
free(record);
|
||||
return -1;
|
||||
}
|
||||
Data* data = data_create(record, (size_t)length);
|
||||
if (data == NULL)
|
||||
return -1; /* data_create frees `record` on failure */
|
||||
Chunk* chunk = chunk_deserialize(data, use_metadata);
|
||||
data_destroy(data);
|
||||
if (chunk == NULL) {
|
||||
log_message(LOG_LEVEL_ERROR, "batch: rejected malformed chunk record");
|
||||
return -1;
|
||||
}
|
||||
for (int i = 0; i < chunk->element_count; i++) {
|
||||
File* file = chunk->items[i];
|
||||
chunk->items[i] = NULL;
|
||||
if (file == NULL)
|
||||
continue;
|
||||
FileSaveResult result = file_save_to_disk_full(dest_root, file, config);
|
||||
file_destroy(file);
|
||||
if (result == FILE_SAVE_ERROR) {
|
||||
chunk_destroy(chunk);
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
chunk_destroy(chunk);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
#ifndef BATCH_H
|
||||
#define BATCH_H
|
||||
#include "chunk.h"
|
||||
#include "config.h"
|
||||
|
||||
/* Phase 6 residual-batch codec. A residual batch is a self-contained
|
||||
* single-file record of a whole source tree: a magic+format-version header
|
||||
* followed by length-prefixed chunk blobs (each built with chunk_serialize),
|
||||
* byte-identical by construction. The batch is a client-only driver feature:
|
||||
* it never crosses the wire, so there is no PROTOCOL_VERSION bump and no server
|
||||
* change. */
|
||||
|
||||
#define BATCH_MAGIC "FSTRESBATCH"
|
||||
#define BATCH_MAGIC_LEN 11
|
||||
#define BATCH_FORMAT_VERSION 1
|
||||
/* Max size of a single length-prefixed record (a whole serialized chunk,
|
||||
* which can span several files). A single source file near the 64 MB wire
|
||||
* limit plus per-file headers can produce a record slightly over 64 MB, so a
|
||||
* large file just under the wire cap may be refused by the batch writer; this
|
||||
* is documented upstream and the failure is clean (the partial batch is
|
||||
* unlinked), never a truncated/corrupt batch. */
|
||||
#define BATCH_MAX_RECORD (64ULL * 1024 * 1024)
|
||||
|
||||
bool batch_write_header(int fd, const Config* config);
|
||||
bool batch_write_chunk(int fd, Chunk* chunk);
|
||||
int batch_read_apply(int fd, const Config* config, const char* dest_root);
|
||||
|
||||
#endif
|
||||
@@ -1,384 +0,0 @@
|
||||
#include "charset.h"
|
||||
#include "log.h"
|
||||
#include "protocol.h"
|
||||
#include "utils.h"
|
||||
#include <errno.h>
|
||||
#include <iconv.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
typedef struct {
|
||||
iconv_t cd;
|
||||
} CharsetConversion;
|
||||
|
||||
/* Process-wide wire conversion descriptor (one direction per process: a client
|
||||
* only sends, a server only receives). CONCURRENCY CONTRACT: iconv_t is not
|
||||
* guaranteed thread-safe, so every conversion MUST run on a single thread at a
|
||||
* time. This holds today -- on the client the conversions run on the sender
|
||||
* thread (in the -m pipeline chunk_serialize/send happen on the sender thread
|
||||
* only), on the server on the receive-loop thread; the descriptor is
|
||||
* initialized on one thread before any transfer thread spawns and torn down
|
||||
* (charset_wire_free) only after all threads have joined. Do not add a
|
||||
* concurrent conversion path (e.g. parallel chunk serialization) without
|
||||
* guarding access with a mutex. */
|
||||
static CharsetConversion* g_wire_conv;
|
||||
|
||||
/* Grow *buf to double capacity, freeing it on failure. realloc preserves the
|
||||
* already-written prefix, so the caller only tracks its write offset. */
|
||||
static bool grow_charset_buffer(char** buf, size_t* cap) {
|
||||
size_t new_cap = *cap * 2;
|
||||
if (new_cap <= *cap) {
|
||||
free(*buf);
|
||||
*buf = NULL;
|
||||
return false;
|
||||
}
|
||||
char* grown = realloc(*buf, new_cap);
|
||||
if (!grown) {
|
||||
free(*buf);
|
||||
*buf = NULL;
|
||||
return false;
|
||||
}
|
||||
*buf = grown;
|
||||
*cap = new_cap;
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Throw away any pending shift state so a subsequent conversion starts clean.
|
||||
* The flush output is discarded; for the stateless single-byte/UTF charsets
|
||||
* this feature targets it is a no-op. */
|
||||
static void charset_conversion_reset(const CharsetConversion* conv) {
|
||||
char scratch[64];
|
||||
char* sp = scratch;
|
||||
size_t sl = sizeof(scratch);
|
||||
(void)iconv(conv->cd, NULL, NULL, &sp, &sl);
|
||||
}
|
||||
|
||||
int charset_spec_parse(const char* spec, char** local_out, char** remote_out) {
|
||||
if (!local_out || !remote_out)
|
||||
return -1;
|
||||
*local_out = NULL;
|
||||
*remote_out = NULL;
|
||||
if (!spec || spec[0] == '\0')
|
||||
return -1;
|
||||
char* dup = str_dup(spec);
|
||||
if (!dup)
|
||||
return -1;
|
||||
char* comma = strchr(dup, ',');
|
||||
if (comma) {
|
||||
if (comma == dup || comma[1] == '\0') {
|
||||
free(dup);
|
||||
return -1;
|
||||
}
|
||||
*comma = '\0';
|
||||
*local_out = str_dup(dup);
|
||||
*remote_out = str_dup(comma + 1);
|
||||
free(dup);
|
||||
} else {
|
||||
*local_out = str_dup(dup);
|
||||
*remote_out = str_dup(dup);
|
||||
free(dup);
|
||||
}
|
||||
if (!*local_out || !*remote_out) {
|
||||
free(*local_out);
|
||||
free(*remote_out);
|
||||
*local_out = NULL;
|
||||
*remote_out = NULL;
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
void* charset_conversion_open(const char* from_charset, const char* to_charset) {
|
||||
if (!from_charset || !to_charset)
|
||||
return NULL;
|
||||
iconv_t cd = iconv_open(to_charset, from_charset);
|
||||
if (cd == (iconv_t)-1)
|
||||
return NULL;
|
||||
CharsetConversion* conv = malloc(sizeof(CharsetConversion));
|
||||
if (!conv) {
|
||||
iconv_close(cd);
|
||||
return NULL;
|
||||
}
|
||||
conv->cd = cd;
|
||||
return conv;
|
||||
}
|
||||
|
||||
void charset_conversion_close(void* conversion) {
|
||||
if (!conversion)
|
||||
return;
|
||||
CharsetConversion* conv = (CharsetConversion*)conversion;
|
||||
iconv_close(conv->cd);
|
||||
free(conv);
|
||||
}
|
||||
|
||||
/* Probe a single conversion direction: the from/to charsets both open AND a
|
||||
* representative ASCII name converts to a byte string containing no embedded
|
||||
* NUL (so a target charset like UTF-16 that emits NUL bytes for ordinary ASCII
|
||||
* names is rejected up front -- such an output would be silently truncated by
|
||||
* the C-string wire helpers). */
|
||||
static bool direction_probe_valid(const char* from, const char* to) {
|
||||
if (!from || !to)
|
||||
return false;
|
||||
void* conv = charset_conversion_open(from, to);
|
||||
if (!conv)
|
||||
return false;
|
||||
bool ok = true;
|
||||
char input = 'a';
|
||||
char* in_ptr = &input;
|
||||
size_t in_left = 1;
|
||||
char out_buf[64];
|
||||
char* out_ptr = out_buf;
|
||||
size_t out_left = sizeof(out_buf);
|
||||
if (iconv(((CharsetConversion*)conv)->cd, &in_ptr, &in_left, &out_ptr, &out_left) == (size_t)-1)
|
||||
ok = false;
|
||||
char flush_buf[64];
|
||||
char* flush_ptr = flush_buf;
|
||||
size_t flush_left = sizeof(flush_buf);
|
||||
if (ok &&
|
||||
iconv(((CharsetConversion*)conv)->cd, NULL, NULL, &flush_ptr, &flush_left) == (size_t)-1)
|
||||
ok = false;
|
||||
size_t produced = (size_t)(out_ptr - out_buf);
|
||||
if (ok && produced > 0 && memchr(out_buf, '\0', produced) != NULL)
|
||||
ok = false;
|
||||
charset_conversion_close(conv);
|
||||
return ok;
|
||||
}
|
||||
|
||||
bool charset_pair_valid(const char* local, const char* remote) {
|
||||
/* Both ends convert in opposite directions with the same two charsets, so a
|
||||
* valid spec must open (and be NUL-free) in BOTH directions: the sender
|
||||
* opens local->remote, the receiver opens remote->local. */
|
||||
return direction_probe_valid(local, remote) && direction_probe_valid(remote, local);
|
||||
}
|
||||
|
||||
bool charset_spec_valid(const char* spec) {
|
||||
if (!spec)
|
||||
return true;
|
||||
char* local;
|
||||
char* remote;
|
||||
if (charset_spec_parse(spec, &local, &remote) != 0)
|
||||
return false;
|
||||
bool ok = charset_pair_valid(local, remote);
|
||||
free(local);
|
||||
free(remote);
|
||||
return ok;
|
||||
}
|
||||
|
||||
bool charset_spec_valid_direction(const char* from_charset, const char* to_charset) {
|
||||
return direction_probe_valid(from_charset, to_charset);
|
||||
}
|
||||
|
||||
/* The receiver's real conversion is wire(client REMOTE) -> server-local (the
|
||||
* server's own --iconv LOCAL half, or the client's LOCAL half when the server
|
||||
* has no --iconv). A dedicated pre-ack check so an impossible direction is
|
||||
* rejected before the connection instead of refusing mid-transfer. */
|
||||
bool charset_wire_receiver_spec_valid(const char* spec, const char* server_spec) {
|
||||
if (!spec)
|
||||
return true;
|
||||
char* local;
|
||||
char* remote;
|
||||
if (charset_spec_parse(spec, &local, &remote) != 0)
|
||||
return false;
|
||||
const char* wire = remote;
|
||||
const char* target_local = local;
|
||||
char* server_local = NULL;
|
||||
char* server_remote = NULL;
|
||||
if (server_spec) {
|
||||
if (charset_spec_parse(server_spec, &server_local, &server_remote) != 0) {
|
||||
free(local);
|
||||
free(remote);
|
||||
return false;
|
||||
}
|
||||
target_local = server_local;
|
||||
}
|
||||
bool ok = charset_spec_valid_direction(wire, target_local);
|
||||
free(server_local);
|
||||
free(server_remote);
|
||||
free(local);
|
||||
free(remote);
|
||||
return ok;
|
||||
}
|
||||
|
||||
char* charset_convert(const void* conversion, const char* in, int* err_out) {
|
||||
if (!conversion || !in)
|
||||
return NULL;
|
||||
const CharsetConversion* conv = (const CharsetConversion*)conversion;
|
||||
size_t in_len = strlen(in);
|
||||
size_t cap = in_len + 16;
|
||||
char* out = malloc(cap);
|
||||
if (!out)
|
||||
return NULL;
|
||||
size_t in_left = in_len;
|
||||
char* in_ptr = (char*)in;
|
||||
size_t out_used = 0;
|
||||
|
||||
while (in_left > 0) {
|
||||
char* out_ptr = out + out_used;
|
||||
size_t out_left = cap - out_used;
|
||||
if (iconv(conv->cd, &in_ptr, &in_left, &out_ptr, &out_left) == (size_t)-1) {
|
||||
if (errno != E2BIG) {
|
||||
if (err_out)
|
||||
*err_out = errno;
|
||||
charset_conversion_reset(conv);
|
||||
free(out);
|
||||
return NULL;
|
||||
}
|
||||
/* Output exhausted but input remains. E2BIG does not roll the output
|
||||
pointer back: the bytes iconv already emitted before the failure must
|
||||
be preserved, so advance out_used before growing. */
|
||||
out_used = (size_t)(out_ptr - out);
|
||||
if (!grow_charset_buffer(&out, &cap))
|
||||
return NULL;
|
||||
continue;
|
||||
}
|
||||
out_used = (size_t)(out_ptr - out);
|
||||
}
|
||||
|
||||
/* Flush any pending shift state (a no-op for the stateless single-byte and
|
||||
UTF charsets this feature targets, but keeps the descriptor clean). */
|
||||
for (;;) {
|
||||
char* out_ptr = out + out_used;
|
||||
size_t out_left = cap - out_used;
|
||||
if (iconv(conv->cd, NULL, NULL, &out_ptr, &out_left) == (size_t)-1) {
|
||||
if (errno != E2BIG) {
|
||||
if (err_out)
|
||||
*err_out = errno;
|
||||
charset_conversion_reset(conv);
|
||||
free(out);
|
||||
return NULL;
|
||||
}
|
||||
out_used = (size_t)(out_ptr - out);
|
||||
if (!grow_charset_buffer(&out, &cap))
|
||||
return NULL;
|
||||
continue;
|
||||
}
|
||||
out_used = (size_t)(out_ptr - out);
|
||||
break;
|
||||
}
|
||||
|
||||
/* A successful iconv call may legitimately consume the whole buffer (output
|
||||
exactly fills cap), leaving no room for the terminator: guarantee headroom
|
||||
before the final write. */
|
||||
if (out_used >= cap && !grow_charset_buffer(&out, &cap))
|
||||
return NULL;
|
||||
|
||||
/* Defense in depth: a target charset that emits embedded NUL bytes would
|
||||
truncate at the first NUL in the C-string wire helpers; fail cleanly
|
||||
(validation already rejects such charsets up front). */
|
||||
if (memchr(out, '\0', out_used) != NULL) {
|
||||
if (err_out)
|
||||
*err_out = EILSEQ;
|
||||
charset_conversion_reset(conv);
|
||||
free(out);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
out[out_used] = '\0';
|
||||
return out;
|
||||
}
|
||||
|
||||
bool charset_wire_init_sender(const char* spec) {
|
||||
charset_wire_free();
|
||||
if (!spec)
|
||||
return true;
|
||||
char* local;
|
||||
char* remote;
|
||||
if (charset_spec_parse(spec, &local, &remote) != 0)
|
||||
return false;
|
||||
void* conv = charset_conversion_open(local, remote);
|
||||
free(local);
|
||||
free(remote);
|
||||
if (!conv)
|
||||
return false;
|
||||
g_wire_conv = (CharsetConversion*)conv;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool charset_wire_init_receiver(const char* spec, const char* server_spec) {
|
||||
charset_wire_free();
|
||||
if (!spec)
|
||||
return true;
|
||||
char* local;
|
||||
char* remote;
|
||||
if (charset_spec_parse(spec, &local, &remote) != 0)
|
||||
return false;
|
||||
/* The wire charset is the client spec's REMOTE half; the local charset is
|
||||
* the client spec's LOCAL half unless the server was itself started with
|
||||
* --iconv naming a different local charset (the server halves above never
|
||||
* travel, so the server's own flag is the only way its local charset can
|
||||
* differ from what the client assumed). */
|
||||
const char* wire = remote;
|
||||
const char* target_local = local;
|
||||
char* server_local = NULL;
|
||||
char* server_remote = NULL;
|
||||
if (server_spec) {
|
||||
if (charset_spec_parse(server_spec, &server_local, &server_remote) != 0) {
|
||||
free(local);
|
||||
free(remote);
|
||||
return false;
|
||||
}
|
||||
target_local = server_local;
|
||||
}
|
||||
void* conv = charset_conversion_open(wire, target_local);
|
||||
free(server_local);
|
||||
free(server_remote);
|
||||
free(local);
|
||||
free(remote);
|
||||
if (!conv)
|
||||
return false;
|
||||
g_wire_conv = (CharsetConversion*)conv;
|
||||
return true;
|
||||
}
|
||||
|
||||
void charset_wire_free(void) {
|
||||
if (g_wire_conv) {
|
||||
charset_conversion_close(g_wire_conv);
|
||||
g_wire_conv = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
bool charset_wire_active(void) {
|
||||
return g_wire_conv != NULL;
|
||||
}
|
||||
|
||||
char* charset_wire_apply(const char* path) {
|
||||
if (!g_wire_conv)
|
||||
return str_dup(path);
|
||||
return charset_convert(g_wire_conv, path, NULL);
|
||||
}
|
||||
|
||||
static void charset_convert_failure_log(const char* path) {
|
||||
char* escaped = output_escape(path, false);
|
||||
log_message(LOG_LEVEL_ERROR, "--iconv: cannot convert file name '%s' to the target charset",
|
||||
escaped ? escaped : "<unprintable>");
|
||||
free(escaped);
|
||||
}
|
||||
|
||||
bool send_wire_str(int file_descriptor, const char* local_path) {
|
||||
if (!g_wire_conv)
|
||||
return send_str(file_descriptor, local_path);
|
||||
char* wire = charset_wire_apply(local_path);
|
||||
if (!wire) {
|
||||
charset_convert_failure_log(local_path);
|
||||
return false;
|
||||
}
|
||||
bool ok = send_str(file_descriptor, wire);
|
||||
free(wire);
|
||||
return ok;
|
||||
}
|
||||
|
||||
char* receive_wire_str(int file_descriptor) {
|
||||
char* raw = receive_str(file_descriptor);
|
||||
if (!raw)
|
||||
return NULL;
|
||||
if (!g_wire_conv)
|
||||
return raw;
|
||||
char* local = charset_convert(g_wire_conv, raw, NULL);
|
||||
if (!local) {
|
||||
charset_convert_failure_log(raw);
|
||||
free(raw);
|
||||
return NULL;
|
||||
}
|
||||
free(raw);
|
||||
return local;
|
||||
}
|
||||
@@ -1,85 +0,0 @@
|
||||
#ifndef CHARSET_H
|
||||
#define CHARSET_H
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <stddef.h>
|
||||
|
||||
/* --iconv=CONVERT_SPEC file-name charset conversion (rsync compatibility).
|
||||
*
|
||||
* CONVERT_SPEC is "LOCAL[,REMOTE]": LOCAL is the charset of our own file
|
||||
* names, REMOTE is the charset of the remote side's file names and defaults
|
||||
* to LOCAL when the comma half is omitted. The sender converts every local
|
||||
* path from LOCAL to REMOTE before it goes on the wire; the receiver converts
|
||||
* every received path back from REMOTE to LOCAL. A NULL/disabled spec means
|
||||
* identity with zero overhead (the common path never consults iconv).
|
||||
*
|
||||
* All helpers are friendly to the strict cold path: the wire conversion state
|
||||
* is process-global (one direction per process -- a client only sends, a
|
||||
* server only receives) and is initialized once, before any path is
|
||||
* serialized, so conversion compiles to a single non-NULL check when disabled.
|
||||
*/
|
||||
|
||||
/* Parse CONVERT_SPEC into malloc'd LOCAL and REMOTE charset names (caller
|
||||
* frees both). REMOTE is a separate copy of LOCAL when no comma is present.
|
||||
* Returns 0 on success, -1 on a malformed spec (empty halves / missing value /
|
||||
* allocation failure); nothing is allocated on the -1 path. Both output
|
||||
* pointers are REQUIRED (non-NULL). */
|
||||
int charset_spec_parse(const char* spec, char** local_out, char** remote_out);
|
||||
|
||||
/* True when a CONVERT_SPEC is well-formed AND its charsets are usable for this
|
||||
* feature: each pair opens in a probe iconv_open in BOTH directions (a sender
|
||||
* converts local->remote, the receiver converts remote->local) and converting
|
||||
* a representative ASCII name emits no embedded NUL byte (a UTF-16-style NUL
|
||||
* emitter would be silently truncated by the C-string wire helpers). A typo'd
|
||||
* charset name is therefore rejected at startup, not mid-run. NULL (iconv
|
||||
* disabled) is always valid. */
|
||||
bool charset_spec_valid(const char* spec);
|
||||
|
||||
/* Probe a concrete from->to conversion pair without keeping the descriptor:
|
||||
* both charsets open AND a representative ASCII name converts with no embedded
|
||||
* NUL. Used for direction-specific validation (e.g. the receiver's exact
|
||||
* wire->local direction including a server-side charset override). */
|
||||
bool charset_spec_valid_direction(const char* from_charset, const char* to_charset);
|
||||
bool charset_pair_valid(const char* local, const char* remote);
|
||||
|
||||
/* One-shot conversion of a NUL-terminated input to a malloc'd NUL-terminated
|
||||
* result, or NULL on failure. On failure *err_out (when non-NULL) receives
|
||||
* the iconv errno (EILSEQ/EINVAL = the input is not representable in the
|
||||
* target charset). The caller must free the result. */
|
||||
char* charset_convert(const void* conversion, const char* in, int* err_out);
|
||||
|
||||
/* Open a conversion descriptor for direction from_charset -> to_charset.
|
||||
* Returns NULL (errno = EINVAL) when a charset name is unsupported. Freed
|
||||
* with charset_conversion_close. */
|
||||
void* charset_conversion_open(const char* from_charset, const char* to_charset);
|
||||
void charset_conversion_close(void* conversion);
|
||||
|
||||
/* Process-wide wire conversion. charset_wire_init_sender (client side) opens
|
||||
* LOCAL->REMOTE; charset_wire_init_receiver (server side) opens
|
||||
* wire(REMOTE)->server-local. server_spec is the server's own --iconv, whose
|
||||
* LOCAL half may override the local charset the client assumed; NULL reuses
|
||||
* the client spec's LOCAL half. Both return false on an unsupported spec.
|
||||
* The state is freed with charset_wire_free. */
|
||||
bool charset_wire_init_sender(const char* spec);
|
||||
bool charset_wire_init_receiver(const char* spec, const char* server_spec);
|
||||
void charset_wire_free(void);
|
||||
bool charset_wire_active(void);
|
||||
|
||||
/* Pre-ack receiver-direction sanity (see charset_wire_init_receiver): true
|
||||
* when the exact wire->server-local conversion the receiver will use (client
|
||||
* spec's REMOTE half into the server's own LOCAL half, or the client's LOCAL
|
||||
* half when the server has no --iconv) opens and produces NUL-free output. */
|
||||
bool charset_wire_receiver_spec_valid(const char* spec, const char* server_spec);
|
||||
|
||||
/* Convert a path across the wire in the process direction. Returns a malloc'd
|
||||
* string, or NULL when the name cannot be represented in the target charset. */
|
||||
char* charset_wire_apply(const char* path);
|
||||
|
||||
/* Convenience wire string I/O: encode+send_str / receive_str+decode. Both
|
||||
* return false/NULL (logging a clear --iconv error) on conversion failure, so
|
||||
* an unconvertible path FAILS the transfer cleanly instead of silently sending
|
||||
* a mangled name. */
|
||||
bool send_wire_str(int file_descriptor, const char* local_path);
|
||||
char* receive_wire_str(int file_descriptor);
|
||||
|
||||
#endif
|
||||
@@ -1,75 +0,0 @@
|
||||
#include "checksum.h"
|
||||
#include <openssl/evp.h>
|
||||
#include <string.h>
|
||||
#include <strings.h>
|
||||
|
||||
/* delta.c owns the single XXH_IMPLEMENTATION that provides the xxHash symbols
|
||||
* for the whole binary; this TU only needs the declarations. */
|
||||
#include <xxhash.h>
|
||||
|
||||
bool checksum_digest(ChecksumAlgo algo, uint64_t seed, const void* data, size_t size, uint8_t* out,
|
||||
size_t out_capacity, size_t* out_len) {
|
||||
if (!out || !out_len || out_capacity < CHECKSUM_MAX_DIGEST_LEN)
|
||||
return false;
|
||||
if (data == NULL && size != 0)
|
||||
return false;
|
||||
|
||||
if (algo == CHECKSUM_ALGO_XXH64) {
|
||||
uint64_t digest = XXH64(data, size, seed);
|
||||
memcpy(out, &digest, sizeof(digest));
|
||||
*out_len = sizeof(digest);
|
||||
return true;
|
||||
}
|
||||
|
||||
if (algo == CHECKSUM_ALGO_MD5) {
|
||||
/* md5 takes no seed; the caller's seed is deliberately ignored (documented
|
||||
* in RSYNC_COMPAT.md). OpenSSL's one-shot EVP_Digest needs a non-NULL
|
||||
* buffer even for an empty input, so map a NULL data + size==0 to an empty
|
||||
* buffer. */
|
||||
static const uint8_t empty = 0;
|
||||
const void* input = data ? data : ∅
|
||||
unsigned int digest_len = 0;
|
||||
if (EVP_Digest(input, size, out, &digest_len, EVP_md5(), NULL) != 1)
|
||||
return false;
|
||||
if (digest_len > out_capacity)
|
||||
return false;
|
||||
*out_len = digest_len;
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
int checksum_algo_from_name(const char* name) {
|
||||
if (!name)
|
||||
return -1;
|
||||
if (strcasecmp(name, "xxh64") == 0 || strcasecmp(name, "xxhash") == 0)
|
||||
return (int)CHECKSUM_ALGO_XXH64;
|
||||
if (strcasecmp(name, "md5") == 0)
|
||||
return (int)CHECKSUM_ALGO_MD5;
|
||||
return -1;
|
||||
}
|
||||
|
||||
const char* checksum_algo_name(ChecksumAlgo algo) {
|
||||
switch (algo) {
|
||||
case CHECKSUM_ALGO_XXH64:
|
||||
return "xxh64";
|
||||
case CHECKSUM_ALGO_MD5:
|
||||
return "md5";
|
||||
}
|
||||
return "<unknown>";
|
||||
}
|
||||
|
||||
bool checksum_algo_valid(int algo) {
|
||||
return algo == (int)CHECKSUM_ALGO_XXH64 || algo == (int)CHECKSUM_ALGO_MD5;
|
||||
}
|
||||
|
||||
uint8_t checksum_digest_len(ChecksumAlgo algo) {
|
||||
switch (algo) {
|
||||
case CHECKSUM_ALGO_XXH64:
|
||||
return 8;
|
||||
case CHECKSUM_ALGO_MD5:
|
||||
return 16;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
@@ -1,45 +0,0 @@
|
||||
#ifndef CHECKSUM_H
|
||||
#define CHECKSUM_H
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
|
||||
/* Whole-file content-digest algorithms selectable with --checksum-choice and
|
||||
* seeded with --checksum-seed. The ids are the values actually placed on the
|
||||
* wire (config frame), so they must be kept stable and validated on receive.
|
||||
* CHECKSUM_ALGO_XXH64 == 0 is the default and is byte-for-byte what FastSync
|
||||
* computed before these options existed (xxHash64 with seed 0). */
|
||||
typedef enum { CHECKSUM_ALGO_XXH64 = 0, CHECKSUM_ALGO_MD5 = 1 } ChecksumAlgo;
|
||||
|
||||
/* md5 digest is 16 bytes, the longest supported. */
|
||||
#define CHECKSUM_MAX_DIGEST_LEN 16
|
||||
|
||||
/* Compute the whole-file digest of the first `size` bytes of `data`.
|
||||
*
|
||||
* - CHECKSUM_ALGO_XXH64: xxHash64(data, size, seed) (full 64-bit seed).
|
||||
* - CHECKSUM_ALGO_MD5: md5(data, size) via OpenSSL EVP.
|
||||
* md5 has no seed, so `seed` is ignored (documented).
|
||||
* - `size == 0` hashes the empty input (plus its seed), not a NULL input.
|
||||
*
|
||||
* Writes up to `out_capacity` bytes into `out`, storing the digest length in
|
||||
* *out_len. Returns false on NULL out* or when the digest would not fit.
|
||||
* Never writes more than CHECKSUM_MAX_DIGEST_LEN bytes. */
|
||||
bool checksum_digest(ChecksumAlgo algo, uint64_t seed, const void* data, size_t size, uint8_t* out,
|
||||
size_t out_capacity, size_t* out_len);
|
||||
|
||||
/* Resolve a --checksum-choice string (case-insensitive) to an algorithm id.
|
||||
* Accepts "xxh64" and "xxhash" (both map to CHECKSUM_ALGO_XXH64, rsync's
|
||||
* xxhash spelling) and "md5". Returns -1 for any unsupported name. */
|
||||
int checksum_algo_from_name(const char* name);
|
||||
|
||||
/* Canonical name of an algorithm (used in CLI error messages). */
|
||||
const char* checksum_algo_name(ChecksumAlgo algo);
|
||||
|
||||
/* True when `algo` is a supported id (used by config receive validation). */
|
||||
bool checksum_algo_valid(int algo);
|
||||
|
||||
/* Digest length in bytes for an algorithm (xxx64 = 8, md5 = 16). */
|
||||
uint8_t checksum_digest_len(ChecksumAlgo algo);
|
||||
|
||||
#endif /* CHECKSUM_H */
|
||||
@@ -1,90 +0,0 @@
|
||||
#include "chmod.h"
|
||||
#include <stddef.h>
|
||||
#include <string.h>
|
||||
|
||||
static bool parse_clause(mode_t* mode, const char* begin, const char* end) {
|
||||
const char* p = begin;
|
||||
unsigned who = 0;
|
||||
while (p < end && strchr("ugoa", *p)) {
|
||||
if (*p == 'a')
|
||||
who = 7;
|
||||
else
|
||||
who |= *p == 'u' ? 1U : (*p == 'g' ? 2U : 4U);
|
||||
p++;
|
||||
}
|
||||
if (who == 0)
|
||||
who = 7;
|
||||
if (p == end || (*p != '+' && *p != '-' && *p != '='))
|
||||
return false;
|
||||
char operation = *p++;
|
||||
mode_t bits = 0;
|
||||
while (p < end) {
|
||||
mode_t bit;
|
||||
switch (*p++) {
|
||||
case 'r':
|
||||
bit = 4;
|
||||
break;
|
||||
case 'w':
|
||||
bit = 2;
|
||||
break;
|
||||
case 'x':
|
||||
bit = 1;
|
||||
break;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
bits |= bit;
|
||||
}
|
||||
for (unsigned class_index = 0; class_index < 3; class_index++) {
|
||||
unsigned class_bit = 1U << class_index;
|
||||
if (!(who & class_bit))
|
||||
continue;
|
||||
mode_t shift = (mode_t)((2U - class_index) * 3U);
|
||||
mode_t mask = (mode_t)(7U << shift);
|
||||
mode_t class_bits = (mode_t)(bits << shift);
|
||||
if (operation == '+')
|
||||
*mode |= class_bits;
|
||||
else if (operation == '-')
|
||||
*mode &= ~class_bits;
|
||||
else
|
||||
*mode = (*mode & ~mask) | class_bits;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool chmod_apply(mode_t mode, const char* spec, mode_t* result) {
|
||||
if (!spec || !*spec || !result)
|
||||
return false;
|
||||
bool numeric = true;
|
||||
size_t length = strlen(spec);
|
||||
if (length > 4)
|
||||
numeric = false;
|
||||
for (size_t i = 0; i < length && numeric; i++)
|
||||
numeric = spec[i] >= '0' && spec[i] <= '7';
|
||||
if (numeric) {
|
||||
if (length == 0 || length > 4)
|
||||
return false;
|
||||
mode_t parsed = 0;
|
||||
for (size_t i = 0; i < length; i++)
|
||||
parsed = (mode_t)((parsed << 3) | (spec[i] - '0'));
|
||||
*result = parsed;
|
||||
return true;
|
||||
}
|
||||
|
||||
mode_t changed = mode;
|
||||
const char* begin = spec;
|
||||
while (*begin) {
|
||||
const char* end = strchr(begin, ',');
|
||||
if (!end)
|
||||
end = begin + strlen(begin);
|
||||
if (!parse_clause(&changed, begin, end))
|
||||
return false;
|
||||
if (*end == '\0')
|
||||
break;
|
||||
begin = end + 1;
|
||||
if (!*begin)
|
||||
return false;
|
||||
}
|
||||
*result = changed;
|
||||
return true;
|
||||
}
|
||||
@@ -1,10 +0,0 @@
|
||||
#ifndef CHMOD_H
|
||||
#define CHMOD_H
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <sys/stat.h>
|
||||
|
||||
/* Apply the supported rsync --chmod syntax to a permission mode. */
|
||||
bool chmod_apply(mode_t mode, const char* spec, mode_t* result);
|
||||
|
||||
#endif
|
||||
+53
-380
@@ -1,12 +1,9 @@
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
#include <limits.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "array_list.h"
|
||||
#include "charset.h"
|
||||
#include "chunk.h"
|
||||
#include "compression.h"
|
||||
#include "data.h"
|
||||
@@ -14,33 +11,18 @@
|
||||
#include "log.h"
|
||||
#include "metadata.h"
|
||||
#include "protocol.h"
|
||||
#include "utils.h"
|
||||
|
||||
/* Maximum individual file data size within a chunk (64 MB) */
|
||||
#define MAX_FILE_DATA_SIZE (64ULL * 1024 * 1024)
|
||||
#define MAX_FILES_PER_CHUNK 65536U
|
||||
|
||||
Chunk* chunk_create(File** items, int element_count) {
|
||||
if (element_count < 0 || (element_count > 0 && items == NULL))
|
||||
return NULL;
|
||||
Chunk* chunk = (Chunk*)protocol_alloc(sizeof(Chunk));
|
||||
Chunk *chunk_create(File **items, int element_count) {
|
||||
Chunk *chunk = (Chunk *)malloc(sizeof(Chunk));
|
||||
if (chunk == NULL) {
|
||||
log_perror("ERROR: Could not allocate memory for chunk structure");
|
||||
perror("ERROR: Could not allocate memory for chunk structure");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (element_count == 0) {
|
||||
chunk->items = NULL;
|
||||
} else {
|
||||
if ((size_t)element_count > SIZE_MAX / sizeof(File*)) {
|
||||
free(chunk);
|
||||
return NULL;
|
||||
}
|
||||
chunk->items = (File**)protocol_alloc((size_t)element_count * sizeof(File*));
|
||||
if (chunk->items == NULL) {
|
||||
free(chunk);
|
||||
return NULL;
|
||||
}
|
||||
chunk->items = (File **)malloc(element_count * sizeof(File *));
|
||||
if (chunk->items == NULL) {
|
||||
free(chunk);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
for (int i = 0; i < element_count; i++) {
|
||||
@@ -50,11 +32,11 @@ Chunk* chunk_create(File** items, int element_count) {
|
||||
return chunk;
|
||||
}
|
||||
|
||||
void chunk_destroy(void* item) {
|
||||
void chunk_destroy(void *item) {
|
||||
if (item == NULL) {
|
||||
return;
|
||||
}
|
||||
Chunk* chunk = (Chunk*)item;
|
||||
Chunk *chunk = (Chunk *)item;
|
||||
for (int i = 0; i < chunk->element_count; ++i) {
|
||||
if (chunk->items[i] != NULL) {
|
||||
file_destroy(chunk->items[i]);
|
||||
@@ -64,112 +46,31 @@ void chunk_destroy(void* item) {
|
||||
free(chunk);
|
||||
}
|
||||
|
||||
/* --iconv: a chunk blob carries wire-charset path/target bytes. Encode the
|
||||
* sender-side path (a no-op copy when iconv is disabled) so the blob is in the
|
||||
* same charset as every other wire string. */
|
||||
static char* chunk_encode_wire(const char* path) {
|
||||
if (!charset_wire_active())
|
||||
return str_dup(path);
|
||||
return charset_wire_apply(path);
|
||||
static unsigned long long per_file_serialize_size(File *file, bool use_metadata) {
|
||||
return sizeof(size_t) + strlen(file->path) +
|
||||
(use_metadata ? sizeof(int) + (file->metadata ? FILE_METADATA_WIRE_SIZE : 0) : 0) +
|
||||
sizeof(size_t) + file->data->size;
|
||||
}
|
||||
|
||||
static unsigned long long per_file_serialize_size(File* file, bool use_metadata) {
|
||||
unsigned long long size = sizeof(size_t);
|
||||
char* wire_path = chunk_encode_wire(file_wire_path(file));
|
||||
if (!wire_path)
|
||||
return 0;
|
||||
size_t path_len = strlen(wire_path);
|
||||
free(wire_path);
|
||||
unsigned long long metadata_size =
|
||||
use_metadata ? sizeof(int) + (file->metadata ? FILE_METADATA_WIRE_SIZE : 0) : 0;
|
||||
if ((unsigned long long)path_len > ULLONG_MAX - size)
|
||||
return 0;
|
||||
size += path_len;
|
||||
if (metadata_size > ULLONG_MAX - size)
|
||||
return 0;
|
||||
size += metadata_size;
|
||||
/* Entry type marker: 0 = regular file, 1 = explicit directory entry,
|
||||
2 = symlink entry (carries its target string), 3 = special/device node
|
||||
(recreated by the receiver). */
|
||||
if (sizeof(int) > ULLONG_MAX - size)
|
||||
return 0;
|
||||
size += sizeof(int);
|
||||
/* A special node also carries its rdev major/minor. */
|
||||
if (file->is_special) {
|
||||
if (2 * sizeof(int32_t) > ULLONG_MAX - size)
|
||||
return 0;
|
||||
size += 2 * sizeof(int32_t);
|
||||
}
|
||||
if (sizeof(size_t) > ULLONG_MAX - size)
|
||||
return 0;
|
||||
size += sizeof(size_t);
|
||||
if ((unsigned long long)file->data->size > ULLONG_MAX - size)
|
||||
return 0;
|
||||
size += file->data->size;
|
||||
/* Symlink entries append the target string (length-prefixed). */
|
||||
if (file->is_symlink) {
|
||||
char* wire_target = chunk_encode_wire(file->symlink_target ? file->symlink_target : "");
|
||||
if (!wire_target)
|
||||
return 0;
|
||||
size_t target_len = strlen(wire_target);
|
||||
free(wire_target);
|
||||
if (sizeof(size_t) > ULLONG_MAX - size)
|
||||
return 0;
|
||||
size += sizeof(size_t);
|
||||
if ((unsigned long long)target_len > ULLONG_MAX - size)
|
||||
return 0;
|
||||
size += target_len;
|
||||
}
|
||||
return size;
|
||||
}
|
||||
|
||||
Data* chunk_serialize(Chunk* chunk, bool use_metadata) {
|
||||
if (!chunk || chunk->element_count < 0 || (chunk->element_count > 0 && chunk->items == NULL))
|
||||
return NULL;
|
||||
Data *chunk_serialize(Chunk *chunk, bool use_metadata) {
|
||||
unsigned long long data_size = 0;
|
||||
for (int i = 0; i < chunk->element_count; i++) {
|
||||
if (!chunk->items[i] || !chunk->items[i]->path || !chunk->items[i]->data ||
|
||||
(chunk->items[i]->data->size > 0 && !chunk->items[i]->data->data) ||
|
||||
chunk->items[i]->path[0] == '\0' || has_path_traversal(chunk->items[i]->path) ||
|
||||
(file_wire_path(chunk->items[i]))[0] == '\0')
|
||||
return NULL;
|
||||
unsigned long long file_size = per_file_serialize_size(chunk->items[i], use_metadata);
|
||||
if (file_size == 0 || file_size > ULLONG_MAX - data_size || data_size + file_size > SIZE_MAX)
|
||||
return NULL;
|
||||
data_size += file_size;
|
||||
data_size += per_file_serialize_size(chunk->items[i], use_metadata);
|
||||
}
|
||||
Data* data = data_create_empty(data_size);
|
||||
Data *data = data_create_empty(data_size);
|
||||
if (data == NULL) {
|
||||
log_message(LOG_LEVEL_ERROR, "Could not allocate memory for chunk serialization");
|
||||
log_message(LOG_LEVEL_ERROR,
|
||||
"Could not allocate memory for chunk serialization");
|
||||
return NULL;
|
||||
}
|
||||
char* data_pointer = data->data;
|
||||
char *data_pointer = data->data;
|
||||
for (int i = 0; i < chunk->element_count; i++) {
|
||||
File* file = chunk->items[i];
|
||||
char* wire_path = chunk_encode_wire(file_wire_path(file));
|
||||
if (wire_path == NULL) {
|
||||
data_destroy(data);
|
||||
return NULL;
|
||||
}
|
||||
size_t path_len = strlen(wire_path);
|
||||
File *file = chunk->items[i];
|
||||
size_t path_len = strlen(file->path);
|
||||
memcpy(data_pointer, &path_len, sizeof(size_t));
|
||||
data_pointer += sizeof(size_t);
|
||||
memcpy(data_pointer, wire_path, path_len);
|
||||
memcpy(data_pointer, file->path, path_len);
|
||||
data_pointer += path_len;
|
||||
free(wire_path);
|
||||
|
||||
int entry_type = file->is_dir ? 1 : (file->is_symlink ? 2 : (file->is_special ? 3 : 0));
|
||||
memcpy(data_pointer, &entry_type, sizeof(int));
|
||||
data_pointer += sizeof(int);
|
||||
|
||||
if (file->is_special) {
|
||||
int32_t special_major = file->rdev_major;
|
||||
int32_t special_minor = file->rdev_minor;
|
||||
memcpy(data_pointer, &special_major, sizeof(special_major));
|
||||
data_pointer += sizeof(special_major);
|
||||
memcpy(data_pointer, &special_minor, sizeof(special_minor));
|
||||
data_pointer += sizeof(special_minor);
|
||||
}
|
||||
|
||||
if (use_metadata)
|
||||
metadata_to_buf(&data_pointer, file->metadata);
|
||||
@@ -177,356 +78,128 @@ Data* chunk_serialize(Chunk* chunk, bool use_metadata) {
|
||||
size_t file_data_size = file->data->size;
|
||||
memcpy(data_pointer, &file_data_size, sizeof(size_t));
|
||||
data_pointer += sizeof(size_t);
|
||||
if (file_data_size > 0)
|
||||
memcpy(data_pointer, file->data->data, file_data_size);
|
||||
memcpy(data_pointer, file->data->data, file_data_size);
|
||||
data_pointer += file_data_size;
|
||||
|
||||
if (file->is_symlink) {
|
||||
char* wire_target = chunk_encode_wire(file->symlink_target ? file->symlink_target : "");
|
||||
if (wire_target == NULL) {
|
||||
data_destroy(data);
|
||||
return NULL;
|
||||
}
|
||||
size_t target_len = strlen(wire_target);
|
||||
memcpy(data_pointer, &target_len, sizeof(size_t));
|
||||
data_pointer += sizeof(size_t);
|
||||
if (target_len > 0)
|
||||
memcpy(data_pointer, wire_target, target_len);
|
||||
data_pointer += target_len;
|
||||
free(wire_target);
|
||||
}
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
Chunk* chunk_deserialize(Data* data, bool use_metadata) {
|
||||
if (!data || (!data->data && data->size != 0))
|
||||
return NULL;
|
||||
ArrayList* files = array_list_create(file_destroy);
|
||||
if (files == NULL)
|
||||
return NULL;
|
||||
char* data_pointer = data->data;
|
||||
Chunk *chunk_deserialize(Data *data, bool use_metadata) {
|
||||
ArrayList *files = array_list_create(file_destroy);
|
||||
char *data_pointer = data->data;
|
||||
size_t remaining_size = data->size;
|
||||
|
||||
while (remaining_size > 0) {
|
||||
if ((unsigned int)files->size >= MAX_FILES_PER_CHUNK) {
|
||||
log_message(LOG_LEVEL_ERROR, "Chunk contains too many files");
|
||||
array_list_delete(files);
|
||||
return NULL;
|
||||
}
|
||||
if (remaining_size < sizeof(size_t)) {
|
||||
log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for path length");
|
||||
array_list_delete(files);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
size_t path_len;
|
||||
memcpy(&path_len, data_pointer, sizeof(size_t));
|
||||
size_t path_len = *(size_t *)data_pointer;
|
||||
data_pointer += sizeof(size_t);
|
||||
remaining_size -= sizeof(size_t);
|
||||
|
||||
if (path_len > SIZE_MAX - 1 || remaining_size < path_len) {
|
||||
if (remaining_size < path_len) {
|
||||
log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for path");
|
||||
array_list_delete(files);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (path_len == SIZE_MAX) {
|
||||
array_list_delete(files);
|
||||
return NULL;
|
||||
}
|
||||
char* path = protocol_alloc(path_len + 1);
|
||||
char *path = malloc(path_len + 1);
|
||||
if (path == NULL) {
|
||||
log_perror("Could not allocate memory for file path");
|
||||
perror("Could not allocate memory for file path");
|
||||
array_list_delete(files);
|
||||
return NULL;
|
||||
}
|
||||
memcpy(path, data_pointer, path_len);
|
||||
path[path_len] = '\0';
|
||||
if (memchr(path, '\0', path_len) != NULL) {
|
||||
free(path);
|
||||
array_list_delete(files);
|
||||
return NULL;
|
||||
}
|
||||
data_pointer += path_len;
|
||||
remaining_size -= path_len;
|
||||
|
||||
/* --iconv: the blob holds the wire charset; translate it to the receiver's
|
||||
local charset before validation and creation so the destination gets the
|
||||
local name. A name that cannot be decoded fails the file cleanly. */
|
||||
if (charset_wire_active()) {
|
||||
char* local_path = charset_wire_apply(path);
|
||||
free(path);
|
||||
if (local_path == NULL) {
|
||||
log_message(LOG_LEVEL_ERROR,
|
||||
"--iconv: received chunk file name cannot be converted to the local charset");
|
||||
array_list_delete(files);
|
||||
return NULL;
|
||||
}
|
||||
path = local_path;
|
||||
path_len = strlen(path);
|
||||
}
|
||||
|
||||
if (path_len == 0 || has_path_traversal(path)) {
|
||||
free(path);
|
||||
array_list_delete(files);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
File* file = file_create(path);
|
||||
File *file = file_create(path);
|
||||
free(path);
|
||||
if (file == NULL) {
|
||||
array_list_delete(files);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (remaining_size < sizeof(int)) {
|
||||
log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for entry type");
|
||||
file_destroy(file);
|
||||
array_list_delete(files);
|
||||
return NULL;
|
||||
}
|
||||
int entry_type;
|
||||
memcpy(&entry_type, data_pointer, sizeof(int));
|
||||
if (entry_type != 0 && entry_type != 1 && entry_type != 2 && entry_type != 3) {
|
||||
log_message(LOG_LEVEL_ERROR, "Invalid chunk format: bad entry type");
|
||||
file_destroy(file);
|
||||
array_list_delete(files);
|
||||
return NULL;
|
||||
}
|
||||
file->is_dir = entry_type == 1;
|
||||
file->is_symlink = entry_type == 2;
|
||||
file->is_special = entry_type == 3;
|
||||
data_pointer += sizeof(int);
|
||||
remaining_size -= sizeof(int);
|
||||
|
||||
if (file->is_special) {
|
||||
if (remaining_size < 2 * (int32_t)sizeof(int32_t)) {
|
||||
log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for special rdev");
|
||||
file_destroy(file);
|
||||
array_list_delete(files);
|
||||
return NULL;
|
||||
}
|
||||
int32_t special_major, special_minor;
|
||||
memcpy(&special_major, data_pointer, sizeof(special_major));
|
||||
data_pointer += sizeof(special_major);
|
||||
memcpy(&special_minor, data_pointer, sizeof(special_minor));
|
||||
data_pointer += sizeof(special_minor);
|
||||
remaining_size -= 2 * sizeof(int32_t);
|
||||
/* Reject an out-of-range/negative rdev here as a malformed chunk (the
|
||||
same 0xffff / 0x00ffffff bounds file_special_rdev_valid uses), so a
|
||||
bogus large-but-positive rdev is refused cleanly instead of being
|
||||
deferred to the creation site where it would abort after the frame. */
|
||||
if (special_major < 0 || special_minor < 0 || special_major > 0xffff ||
|
||||
special_minor > 0x00ffffff) {
|
||||
log_message(LOG_LEVEL_ERROR, "Invalid chunk format: out-of-range special rdev");
|
||||
file_destroy(file);
|
||||
array_list_delete(files);
|
||||
return NULL;
|
||||
}
|
||||
file->rdev_major = special_major;
|
||||
file->rdev_minor = special_minor;
|
||||
}
|
||||
|
||||
if (use_metadata) {
|
||||
if (remaining_size < sizeof(int)) {
|
||||
log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for metadata");
|
||||
file_destroy(file);
|
||||
array_list_delete(files);
|
||||
return NULL;
|
||||
}
|
||||
// Peek at present flag to determine total size needed before reading
|
||||
int present_flag;
|
||||
memcpy(&present_flag, data_pointer, sizeof(int));
|
||||
if ((present_flag != 0 && present_flag != 1) ||
|
||||
(present_flag == 1 && remaining_size < sizeof(int) + FILE_METADATA_WIRE_SIZE)) {
|
||||
log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for metadata body");
|
||||
file_destroy(file);
|
||||
array_list_delete(files);
|
||||
return NULL;
|
||||
}
|
||||
file->metadata = metadata_from_buf(&data_pointer);
|
||||
remaining_size -= sizeof(int);
|
||||
if (present_flag == 1) {
|
||||
if (file->metadata == NULL) {
|
||||
file_destroy(file);
|
||||
array_list_delete(files);
|
||||
return NULL;
|
||||
}
|
||||
if (file->metadata)
|
||||
remaining_size -= FILE_METADATA_WIRE_SIZE;
|
||||
}
|
||||
}
|
||||
|
||||
if (remaining_size < sizeof(size_t)) {
|
||||
log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for data size");
|
||||
file_destroy(file);
|
||||
array_list_delete(files);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
size_t file_data_size;
|
||||
memcpy(&file_data_size, data_pointer, sizeof(size_t));
|
||||
size_t file_data_size = *(size_t *)data_pointer;
|
||||
data_pointer += sizeof(size_t);
|
||||
remaining_size -= sizeof(size_t);
|
||||
|
||||
if (remaining_size < file_data_size) {
|
||||
log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for file content");
|
||||
file_destroy(file);
|
||||
array_list_delete(files);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
// Reject individual file data larger than the maximum allowed size.
|
||||
if (file_data_size > MAX_FILE_DATA_SIZE) {
|
||||
log_message(LOG_LEVEL_ERROR, "File data size %zu exceeds maximum %llu", file_data_size,
|
||||
(unsigned long long)MAX_FILE_DATA_SIZE);
|
||||
file_destroy(file);
|
||||
array_list_delete(files);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
size_t allocation_size = file_data_size > 0 ? file_data_size : 1;
|
||||
void* file_data = protocol_alloc(allocation_size);
|
||||
void *file_data = malloc(file_data_size);
|
||||
if (file_data == NULL) {
|
||||
log_perror("Could not allocate memory for file data");
|
||||
file_destroy(file);
|
||||
perror("Could not allocate memory for file data");
|
||||
array_list_delete(files);
|
||||
return NULL;
|
||||
}
|
||||
memcpy(file_data, data_pointer, file_data_size);
|
||||
Data* replacement = data_create(file_data, file_data_size);
|
||||
if (replacement == NULL) {
|
||||
file_destroy(file);
|
||||
array_list_delete(files);
|
||||
return NULL;
|
||||
}
|
||||
data_destroy(file->data);
|
||||
file->data = replacement;
|
||||
file->data = data_create(file_data, file_data_size);
|
||||
data_pointer += file_data_size;
|
||||
remaining_size -= file_data_size;
|
||||
|
||||
if (file->is_symlink) {
|
||||
if (remaining_size < sizeof(size_t)) {
|
||||
log_message(LOG_LEVEL_ERROR, "Invalid chunk format: not enough data for symlink target");
|
||||
file_destroy(file);
|
||||
array_list_delete(files);
|
||||
return NULL;
|
||||
}
|
||||
size_t target_len;
|
||||
memcpy(&target_len, data_pointer, sizeof(size_t));
|
||||
data_pointer += sizeof(size_t);
|
||||
remaining_size -= sizeof(size_t);
|
||||
if (target_len == 0 || remaining_size < target_len) {
|
||||
log_message(LOG_LEVEL_ERROR, "Invalid chunk format: bad symlink target");
|
||||
file_destroy(file);
|
||||
array_list_delete(files);
|
||||
return NULL;
|
||||
}
|
||||
char* target = protocol_alloc(target_len + 1);
|
||||
if (!target) {
|
||||
log_perror("Could not allocate memory for symlink target");
|
||||
file_destroy(file);
|
||||
array_list_delete(files);
|
||||
return NULL;
|
||||
}
|
||||
memcpy(target, data_pointer, target_len);
|
||||
target[target_len] = '\0';
|
||||
if (memchr(target, '\0', target_len) != NULL) {
|
||||
free(target);
|
||||
file_destroy(file);
|
||||
array_list_delete(files);
|
||||
return NULL;
|
||||
}
|
||||
/* The symlink target also rides the wire charset; decode it to the local
|
||||
charset like the path (a target is a path). */
|
||||
if (charset_wire_active()) {
|
||||
char* local_target = charset_wire_apply(target);
|
||||
free(target);
|
||||
if (local_target == NULL) {
|
||||
log_message(LOG_LEVEL_ERROR,
|
||||
"--iconv: received chunk symlink target cannot be converted to the local "
|
||||
"charset");
|
||||
file_destroy(file);
|
||||
array_list_delete(files);
|
||||
return NULL;
|
||||
}
|
||||
target = local_target;
|
||||
}
|
||||
file->symlink_target = target;
|
||||
data_pointer += target_len;
|
||||
remaining_size -= target_len;
|
||||
}
|
||||
|
||||
if (!array_list_add(files, file)) {
|
||||
file_destroy(file);
|
||||
array_list_delete(files);
|
||||
return NULL;
|
||||
}
|
||||
array_list_add(files, file);
|
||||
}
|
||||
|
||||
File** file_array = (File**)array_list_to_array(files);
|
||||
if (files->size > 0 && file_array == NULL) {
|
||||
array_list_delete(files);
|
||||
return NULL;
|
||||
}
|
||||
Chunk* chunk = chunk_create(file_array, files->size);
|
||||
File **file_array = (File **)array_list_to_array(files);
|
||||
Chunk *chunk = chunk_create(file_array, files->size);
|
||||
|
||||
free(file_array);
|
||||
if (chunk == NULL) {
|
||||
array_list_delete(files);
|
||||
return NULL;
|
||||
}
|
||||
files->item_destroyer = NULL;
|
||||
array_list_delete(files);
|
||||
|
||||
return chunk;
|
||||
}
|
||||
|
||||
Data* chunk_compress(Chunk* chunk, int compression_level, bool use_metadata) {
|
||||
return chunk_compress_with_threads(chunk, compression_level, use_metadata, 0);
|
||||
}
|
||||
|
||||
Data* chunk_compress_with_threads(Chunk* chunk, int compression_level, bool use_metadata,
|
||||
int compression_threads) {
|
||||
Data *chunk_compress(Chunk *chunk, int compression_level, bool use_metadata) {
|
||||
log_message(LOG_LEVEL_DEBUG, "Starting to compress chunk");
|
||||
Data* serialized = chunk_serialize(chunk, use_metadata);
|
||||
if (serialized == NULL)
|
||||
return NULL;
|
||||
Data* compressed = data_compress_with_threads(serialized, compression_level, compression_threads);
|
||||
Data *serialized = chunk_serialize(chunk, use_metadata);
|
||||
if (serialized == NULL) return NULL;
|
||||
Data *compressed = data_compress(serialized, compression_level);
|
||||
data_destroy(serialized);
|
||||
if (compressed == NULL)
|
||||
return NULL;
|
||||
log_debug_message(LOG_DEBUG_PACK, "Chunk successfully compressed");
|
||||
if (compressed == NULL) return NULL;
|
||||
log_message(LOG_LEVEL_DEBUG, "Chunk successfully compressed");
|
||||
return compressed;
|
||||
}
|
||||
|
||||
Chunk* receive_chunk_data(int fd, const Config* config) {
|
||||
Data* chunk_data = receive_data_limited(fd, MAX_CHUNK_SIZE);
|
||||
Chunk *receive_chunk_data(int fd, Config *config) {
|
||||
Data *chunk_data = receive_data(fd);
|
||||
if (chunk_data == NULL) {
|
||||
log_message(LOG_LEVEL_ERROR, "Failed to receive chunk data");
|
||||
return NULL;
|
||||
}
|
||||
Data* data_to_process = chunk_data;
|
||||
Data *data_to_process = chunk_data;
|
||||
if (config->use_compression) {
|
||||
data_to_process = data_decompress_limited(chunk_data, MAX_CHUNK_SIZE);
|
||||
data_to_process = data_decompress(chunk_data);
|
||||
data_destroy(chunk_data);
|
||||
if (data_to_process == NULL) {
|
||||
log_message(LOG_LEVEL_ERROR, "Failed to decompress chunk");
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
// Reject chunks larger than the maximum allowed size to prevent OOM.
|
||||
if (data_to_process->size > MAX_CHUNK_SIZE) {
|
||||
log_message(LOG_LEVEL_ERROR, "Chunk size %zu exceeds maximum %llu", data_to_process->size,
|
||||
(unsigned long long)MAX_CHUNK_SIZE);
|
||||
data_destroy(data_to_process);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
Chunk* chunk = chunk_deserialize(data_to_process, config->use_metadata);
|
||||
Chunk *chunk = chunk_deserialize(data_to_process, config->use_metadata);
|
||||
data_destroy(data_to_process);
|
||||
if (chunk == NULL)
|
||||
log_message(LOG_LEVEL_ERROR, "Failed to deserialize chunk, skipping");
|
||||
return chunk;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
+7
-9
@@ -10,17 +10,15 @@
|
||||
#define DESIRED_CHUNK_SIZE (10 * 1024 * 1024)
|
||||
|
||||
typedef struct {
|
||||
File** items;
|
||||
File **items;
|
||||
int element_count;
|
||||
} Chunk;
|
||||
|
||||
Chunk* chunk_create(File** items, int element_count);
|
||||
void chunk_destroy(void* chunk);
|
||||
Data* chunk_serialize(Chunk* chunk, bool use_metadata);
|
||||
Chunk* chunk_deserialize(Data* data, bool use_metadata);
|
||||
Data* chunk_compress(Chunk* chunk, int compression_level, bool use_metadata);
|
||||
Data* chunk_compress_with_threads(Chunk* chunk, int compression_level, bool use_metadata,
|
||||
int compression_threads);
|
||||
Chunk* receive_chunk_data(int fd, const Config* config);
|
||||
Chunk *chunk_create(File **items, int element_count);
|
||||
void chunk_destroy(void *chunk);
|
||||
Data *chunk_serialize(Chunk *chunk, bool use_metadata);
|
||||
Chunk *chunk_deserialize(Data *data, bool use_metadata);
|
||||
Data *chunk_compress(Chunk *chunk, int compression_level, bool use_metadata);
|
||||
Chunk *receive_chunk_data(int fd, Config *config);
|
||||
|
||||
#endif
|
||||
|
||||
+25
-128
@@ -1,98 +1,24 @@
|
||||
#include "compression.h"
|
||||
#include "data.h"
|
||||
#include "log.h"
|
||||
#include "protocol.h"
|
||||
#include <stdlib.h>
|
||||
#include <limits.h>
|
||||
#include <stdint.h>
|
||||
#include <string.h>
|
||||
#include <strings.h>
|
||||
#include <unistd.h>
|
||||
#include <zstd.h>
|
||||
#include "stdlib.h"
|
||||
#include "zstd.h"
|
||||
|
||||
#define INITIAL_DECOMPRESS_BUF_SIZE (1024 * 1024)
|
||||
#define MAX_DECOMPRESSED_SIZE (100ULL * 1024 * 1024) /* 100 MB hard ceiling */
|
||||
|
||||
static char* SKIP_COMPRESSION_EXTENSIONS[] = {".jpg", ".jpeg", ".png", ".gif", ".mp4", ".mkv",
|
||||
".zip", ".gz", ".xz", ".zst", NULL};
|
||||
|
||||
bool compression_should_skip(const char* path) {
|
||||
return compression_should_skip_with_suffixes(path, NULL, -1);
|
||||
}
|
||||
|
||||
bool compression_should_skip_with_suffixes(const char* path, char* const* suffixes, int count) {
|
||||
if (!path)
|
||||
return false;
|
||||
const char* dot = strrchr(path, '.');
|
||||
if (!dot)
|
||||
return false;
|
||||
if (count < 0) {
|
||||
suffixes = SKIP_COMPRESSION_EXTENSIONS;
|
||||
count = 0;
|
||||
while (SKIP_COMPRESSION_EXTENSIONS[count])
|
||||
count++;
|
||||
}
|
||||
for (int i = 0; i < count; i++) {
|
||||
if (strcasecmp(dot, suffixes[i]) == 0)
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
Data* data_compress(Data* data_to_compress, int compression_level) {
|
||||
return data_compress_with_threads(data_to_compress, compression_level, 0);
|
||||
}
|
||||
|
||||
Data* data_compress_with_threads(Data* data_to_compress, int compression_level,
|
||||
int compression_threads) {
|
||||
if (!data_to_compress || (!data_to_compress->data && data_to_compress->size != 0) ||
|
||||
compression_threads < 0 || compression_threads > COMPRESSION_MAX_THREADS)
|
||||
return NULL;
|
||||
Data *data_compress(Data *data_to_compress, int compression_level) {
|
||||
log_message(LOG_LEVEL_DEBUG, "Starting to compress data");
|
||||
size_t dst_size = ZSTD_compressBound(data_to_compress->size);
|
||||
Data* compressed_data = data_create_empty(dst_size);
|
||||
if (compressed_data == NULL)
|
||||
return NULL;
|
||||
Data *compressed_data = data_create_empty(dst_size);
|
||||
if (compressed_data == NULL) return NULL;
|
||||
|
||||
ZSTD_CCtx* cctx = ZSTD_createCCtx();
|
||||
ZSTD_CCtx *cctx = ZSTD_createCCtx();
|
||||
if (!cctx) {
|
||||
log_message(LOG_LEVEL_ERROR, "Failed to create ZSTD compression context");
|
||||
data_destroy(compressed_data);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
size_t zret = ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compression_level);
|
||||
if (ZSTD_isError(zret)) {
|
||||
log_message(LOG_LEVEL_ERROR, "Failed to set compression level: %s", ZSTD_getErrorName(zret));
|
||||
ZSTD_freeCCtx(cctx);
|
||||
data_destroy(compressed_data);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (compression_threads > 0) {
|
||||
long online_cpus = sysconf(_SC_NPROCESSORS_ONLN);
|
||||
int available_threads = online_cpus > 0 && online_cpus < compression_threads
|
||||
? (int)online_cpus
|
||||
: compression_threads;
|
||||
zret = ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, available_threads);
|
||||
if (ZSTD_isError(zret)) {
|
||||
log_message(LOG_LEVEL_ERROR, "Failed to set compression threads: %s",
|
||||
ZSTD_getErrorName(zret));
|
||||
ZSTD_freeCCtx(cctx);
|
||||
data_destroy(compressed_data);
|
||||
return NULL;
|
||||
}
|
||||
/* Streaming compression needs the source size before threaded mode can end a frame. */
|
||||
zret = ZSTD_CCtx_setPledgedSrcSize(cctx, data_to_compress->size);
|
||||
if (ZSTD_isError(zret)) {
|
||||
log_message(LOG_LEVEL_ERROR, "Failed to set compression source size: %s",
|
||||
ZSTD_getErrorName(zret));
|
||||
ZSTD_freeCCtx(cctx);
|
||||
data_destroy(compressed_data);
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
ZSTD_inBuffer input = {data_to_compress->data, data_to_compress->size, 0};
|
||||
ZSTD_outBuffer output = {compressed_data->data, dst_size, 0};
|
||||
|
||||
@@ -100,7 +26,8 @@ Data* data_compress_with_threads(Data* data_to_compress, int compression_level,
|
||||
do {
|
||||
ret = ZSTD_compressStream2(cctx, &output, &input, ZSTD_e_end);
|
||||
if (ZSTD_isError(ret)) {
|
||||
log_message(LOG_LEVEL_ERROR, "Compression failed: %s", ZSTD_getErrorName(ret));
|
||||
log_message(LOG_LEVEL_ERROR, "Compression failed: %s",
|
||||
ZSTD_getErrorName(ret));
|
||||
ZSTD_freeCCtx(cctx);
|
||||
data_destroy(compressed_data);
|
||||
return NULL;
|
||||
@@ -110,50 +37,32 @@ Data* data_compress_with_threads(Data* data_to_compress, int compression_level,
|
||||
compressed_data->size = output.pos;
|
||||
ZSTD_freeCCtx(cctx);
|
||||
|
||||
log_debug_message(LOG_DEBUG_UTIL, "Data succesfully compressed from %zu to %zu",
|
||||
data_to_compress->size, compressed_data->size);
|
||||
log_message(LOG_LEVEL_DEBUG, "Data succesfully compressed from %zu to %zu",
|
||||
data_to_compress->size, compressed_data->size);
|
||||
return compressed_data;
|
||||
}
|
||||
|
||||
Data* data_decompress_limited(Data* compressed_data, size_t maximum_size) {
|
||||
if (!compressed_data || (!compressed_data->data && compressed_data->size != 0) ||
|
||||
maximum_size == 0)
|
||||
return NULL;
|
||||
log_debug_message(LOG_DEBUG_UTIL, "Start to decompress data");
|
||||
unsigned long long dst_size =
|
||||
ZSTD_getFrameContentSize(compressed_data->data, compressed_data->size);
|
||||
Data *data_decompress(Data *compressed_data) {
|
||||
log_message(LOG_LEVEL_DEBUG, "Start to decompress data");
|
||||
unsigned long long dst_size = ZSTD_getFrameContentSize(
|
||||
compressed_data->data, compressed_data->size);
|
||||
if (ZSTD_isError(dst_size)) {
|
||||
log_message(LOG_LEVEL_ERROR, "Failed to get decompressed size: %s",
|
||||
ZSTD_getErrorName(dst_size));
|
||||
return NULL;
|
||||
}
|
||||
|
||||
// ZSTD_CONTENTSIZE_UNKNOWN (~2^64) can cause massive allocation;
|
||||
// fall back to a conservative estimate (3x compressed size) when unknown.
|
||||
if (dst_size == ZSTD_CONTENTSIZE_UNKNOWN) {
|
||||
if (compressed_data->size > ULLONG_MAX / 3)
|
||||
return NULL;
|
||||
dst_size = compressed_data->size * 3;
|
||||
if (dst_size < INITIAL_DECOMPRESS_BUF_SIZE)
|
||||
dst_size = INITIAL_DECOMPRESS_BUF_SIZE;
|
||||
}
|
||||
unsigned long long hard_limit =
|
||||
maximum_size < MAX_DECOMPRESSED_SIZE ? maximum_size : MAX_DECOMPRESSED_SIZE;
|
||||
if (dst_size > hard_limit) {
|
||||
log_message(LOG_LEVEL_ERROR, "Declared decompressed size exceeds %llu bytes", hard_limit);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
ZSTD_DCtx* dctx = ZSTD_createDCtx();
|
||||
ZSTD_DCtx *dctx = ZSTD_createDCtx();
|
||||
if (!dctx) {
|
||||
log_message(LOG_LEVEL_ERROR, "Failed to create ZSTD decompression context");
|
||||
log_message(LOG_LEVEL_ERROR,
|
||||
"Failed to create ZSTD decompression context");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
size_t buf_size = (dst_size > 0) ? (size_t)dst_size : INITIAL_DECOMPRESS_BUF_SIZE;
|
||||
if (buf_size > maximum_size)
|
||||
buf_size = maximum_size;
|
||||
Data* uncompressed_data = data_create_empty(buf_size);
|
||||
size_t buf_size = (!ZSTD_isError(dst_size) && dst_size > 0)
|
||||
? (size_t)dst_size
|
||||
: INITIAL_DECOMPRESS_BUF_SIZE;
|
||||
Data *uncompressed_data = data_create_empty(buf_size);
|
||||
if (!uncompressed_data) {
|
||||
log_message(LOG_LEVEL_ERROR, "Failed to allocate decompression buffer");
|
||||
ZSTD_freeDCtx(dctx);
|
||||
@@ -167,23 +76,15 @@ Data* data_decompress_limited(Data* compressed_data, size_t maximum_size) {
|
||||
do {
|
||||
ret = ZSTD_decompressStream(dctx, &output, &input);
|
||||
if (ZSTD_isError(ret)) {
|
||||
log_message(LOG_LEVEL_ERROR, "Decompression failed: %s", ZSTD_getErrorName(ret));
|
||||
log_message(LOG_LEVEL_ERROR, "Decompression failed: %s",
|
||||
ZSTD_getErrorName(ret));
|
||||
ZSTD_freeDCtx(dctx);
|
||||
data_destroy(uncompressed_data);
|
||||
return NULL;
|
||||
}
|
||||
if (ret > 0 && output.pos == output.size) {
|
||||
if (buf_size >= hard_limit || buf_size > SIZE_MAX / 2) {
|
||||
log_message(LOG_LEVEL_ERROR, "Decompressed data exceeds %llu bytes",
|
||||
(unsigned long long)MAX_DECOMPRESSED_SIZE);
|
||||
ZSTD_freeDCtx(dctx);
|
||||
data_destroy(uncompressed_data);
|
||||
return NULL;
|
||||
}
|
||||
buf_size *= 2;
|
||||
if (buf_size > hard_limit)
|
||||
buf_size = (size_t)hard_limit;
|
||||
void* new_data = protocol_realloc(uncompressed_data->data, buf_size);
|
||||
void *new_data = realloc(uncompressed_data->data, buf_size);
|
||||
if (!new_data) {
|
||||
log_message(LOG_LEVEL_ERROR, "Failed to grow decompression buffer");
|
||||
ZSTD_freeDCtx(dctx);
|
||||
@@ -199,10 +100,6 @@ Data* data_decompress_limited(Data* compressed_data, size_t maximum_size) {
|
||||
uncompressed_data->size = output.pos;
|
||||
ZSTD_freeDCtx(dctx);
|
||||
|
||||
log_debug_message(LOG_DEBUG_UTIL, "Decompressed data successfully");
|
||||
log_message(LOG_LEVEL_DEBUG, "Decompressed data successfully");
|
||||
return uncompressed_data;
|
||||
}
|
||||
|
||||
Data* data_decompress(Data* compressed_data) {
|
||||
return data_decompress_limited(compressed_data, MAX_DECOMPRESSED_SIZE);
|
||||
}
|
||||
|
||||
@@ -2,16 +2,8 @@
|
||||
#define COMPRESSION_H
|
||||
|
||||
#include "data.h"
|
||||
#include <stdbool.h>
|
||||
|
||||
#define COMPRESSION_MAX_THREADS 64
|
||||
|
||||
Data* data_compress(Data* data_to_compress, int compression_level);
|
||||
Data* data_compress_with_threads(Data* data_to_compress, int compression_level,
|
||||
int compression_threads);
|
||||
Data* data_decompress(Data* compressed_data);
|
||||
Data* data_decompress_limited(Data* compressed_data, size_t maximum_size);
|
||||
bool compression_should_skip(const char* path);
|
||||
bool compression_should_skip_with_suffixes(const char* path, char* const* suffixes, int count);
|
||||
Data *data_compress(Data *data_to_compress, int compression_level);
|
||||
Data *data_decompress(Data *compressed_data);
|
||||
|
||||
#endif
|
||||
|
||||
+107
-1373
File diff suppressed because it is too large
Load Diff
+24
-698
@@ -1,735 +1,61 @@
|
||||
#ifndef CONFIG_H
|
||||
#define CONFIG_H
|
||||
|
||||
#include "array_list.h"
|
||||
#include "checksum.h"
|
||||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <time.h>
|
||||
|
||||
typedef enum { TRANSPORT_TCP, TRANSPORT_SSH } TransportType;
|
||||
|
||||
/* --outbuf stdout/stderr buffering style (client-only launch concern, never
|
||||
* crosses the wire). OUTBUF_BLOCK is the default, matching the stdio default
|
||||
* (fully buffered when output is not a terminal). */
|
||||
typedef enum {
|
||||
OUTBUF_BLOCK = 0, /* _IOFBF */
|
||||
OUTBUF_LINE, /* _IOLBF */
|
||||
OUTBUF_NONE /* _IONBF */
|
||||
} OutbufMode;
|
||||
|
||||
/* Receiver-side staging state for --delay-updates. Forward-declared here so
|
||||
Config can carry it; the concrete type lives in delay_updates.h. */
|
||||
typedef struct DelayUpdatesContext DelayUpdatesContext;
|
||||
|
||||
/* Alternate basis-directory modes (--compare-dest / --copy-dest /
|
||||
* --link-dest). Each flag adds one entry to the ordered Config->basis_dirs
|
||||
* list; the receiver consults entries in command-line order and stops at the
|
||||
* first exact match, mirroring rsync's basis-dir priority rules. */
|
||||
typedef enum {
|
||||
BASIS_DEST_NONE = 0,
|
||||
BASIS_DEST_COMPARE, /* compare only: never copies, never materializes */
|
||||
BASIS_DEST_COPY, /* local copy of the matched basis file */
|
||||
BASIS_DEST_LINK /* hard link to the matched basis file */
|
||||
} BasisDestType;
|
||||
|
||||
typedef struct BasisDest {
|
||||
BasisDestType type;
|
||||
char* path; /* relative to the destination root (receiver-confined) */
|
||||
} BasisDest;
|
||||
|
||||
/* One resolved FROM:TO identity-mapping rule (--usermap / --groupmap). Both
|
||||
* fields are numeric ids. IDENTITY_MATCH_ANY (-1) in `from` is rsync's '*'
|
||||
* wildcard (matches any transmitted id); IDENTITY_CURRENT (-1) in `to` makes
|
||||
* the receiver resolve the receiving process's own current euid/egid at apply
|
||||
* time. Names are resolved to numbers at parse time on the client (see
|
||||
* identity.h for the exact subset). */
|
||||
typedef struct {
|
||||
int32_t from;
|
||||
int32_t to;
|
||||
} IdentityMap;
|
||||
|
||||
/* --sockopts=OPTIONS allowlist. Only these option names are accepted; anything
|
||||
* else is rejected (never silently ignored). TCP_NODELAY, SO_KEEPALIVE and
|
||||
* SO_REUSEADDR are boolean options (value 0/1); SO_RCVBUF and SO_SNDBUF take a
|
||||
* non-negative byte count. All are applied as int-sized setsockopt values. */
|
||||
typedef enum {
|
||||
SOCKOPT_TCP_NODELAY = 0,
|
||||
SOCKOPT_SO_KEEPALIVE,
|
||||
SOCKOPT_SO_RCVBUF,
|
||||
SOCKOPT_SO_SNDBUF,
|
||||
SOCKOPT_SO_REUSEADDR,
|
||||
SOCKOPT_COUNT
|
||||
} SockOptId;
|
||||
|
||||
typedef struct {
|
||||
SockOptId id; /* allowlist index */
|
||||
int value; /* 0/1 for booleans, byte count for SO_RCVBUF/SO_SNDBUF */
|
||||
} SockOptEntry;
|
||||
TRANSPORT_TCP,
|
||||
TRANSPORT_SSH
|
||||
} TransportType;
|
||||
|
||||
typedef struct Config {
|
||||
char* version;
|
||||
char* send_directory;
|
||||
char* receive_root_directory;
|
||||
char *version;
|
||||
char *send_directory;
|
||||
char *receive_root_directory;
|
||||
bool save_to_disk;
|
||||
bool use_multithreading;
|
||||
bool use_chunk_serialization;
|
||||
bool use_compression;
|
||||
bool use_sendfile;
|
||||
bool use_metadata;
|
||||
bool use_executability;
|
||||
bool metadata_explicitly_disabled;
|
||||
bool show_progress;
|
||||
bool dry_run;
|
||||
bool remove_source_files;
|
||||
bool use_delete;
|
||||
int compression_level;
|
||||
int compression_threads;
|
||||
unsigned long long chunk_size;
|
||||
int ssh_port;
|
||||
TransportType transport;
|
||||
char* ssh_destination;
|
||||
/* Daemon module selection (Wave A, protocol 2.15.0). Client-composed from a
|
||||
* host::module/path destination; NULL or "" means "no module" (the ordinary
|
||||
* standalone-server path). Crosses the wire as a trailing config-frame
|
||||
* string so the daemon can look the module up in its own config and confine
|
||||
* the connection to the module's root (never a client-chosen root). */
|
||||
char* module;
|
||||
/* Daemon password authentication (A7 remediation, protocol 2.19.0).
|
||||
* Client-composed from a --password-file whose first meaningful line is
|
||||
* `user:password`: the client sends ONLY the username in the config frame
|
||||
* (auth_user); the literal password is kept in auth_password CLIENT-SIDE for
|
||||
* the duration of the SCRAM challenge/response and is NEVER serialized. Both
|
||||
* are NULL when the client has no credentials to present; a module WITHOUT
|
||||
* `auth users` stays open and the server ignores any credentials that do
|
||||
* arrive (the client sends them opportunistically and the server decides). */
|
||||
char* auth_user;
|
||||
char* auth_password;
|
||||
/* Client-only path of --password-file (never crosses the wire; it is read to
|
||||
* populate auth_user/auth_password before connecting). */
|
||||
char* password_file;
|
||||
char* fastsync_server_path;
|
||||
/* --iconv=CONVERT_SPEC (protocol 2.16.0, rsync compatibility): convert the
|
||||
* charset of FILE NAMES at the wire boundary. CONVERT_SPEC is
|
||||
* "LOCAL[,REMOTE]": LOCAL is the charset of our own file names, REMOTE is
|
||||
* the remote side's charset and defaults to LOCAL. The sender converts
|
||||
* every path LOCAL->REMOTE before transmitting it; the receiver converts
|
||||
* every received path back REMOTE->LOCAL before creating/writing it. The
|
||||
* FULL SPEC crosses the wire as a trailing config-frame string so each end
|
||||
* derives its own LOCAL and the wire (REMOTE) charset symmetrically. NULL
|
||||
* (or "") means no conversion: identity with zero overhead. See charset.c
|
||||
* and the PROTOCOL_VERSION note below. */
|
||||
char* iconv_spec;
|
||||
char** exclude_patterns;
|
||||
char *ssh_destination;
|
||||
char **exclude_patterns;
|
||||
int exclude_count;
|
||||
char** include_patterns;
|
||||
char **include_patterns;
|
||||
int include_count;
|
||||
unsigned long long max_size;
|
||||
unsigned long long min_size;
|
||||
unsigned long long max_alloc;
|
||||
bool use_incremental;
|
||||
bool ignore_times;
|
||||
bool size_only;
|
||||
bool use_delta;
|
||||
bool whole_file;
|
||||
/* -y/--fuzzy: when a file must be transferred and the destination holds no
|
||||
* usable file at the exact path, the receiver may reuse a SIMILAR-named
|
||||
* existing regular file in the same destination directory as the delta
|
||||
* basis so the sender transmits only the differences. Crosses the wire
|
||||
* (the receiver performs the candidate search); the CLI implies
|
||||
* --incremental + --delta because the similar-basis only matters on the
|
||||
* receiver-driven delta path. Off by default. */
|
||||
bool fuzzy;
|
||||
int modify_window;
|
||||
uint32_t delta_block_size;
|
||||
unsigned long long delta_max_file_size;
|
||||
bool use_tls;
|
||||
char* server_host;
|
||||
int server_port;
|
||||
char* tls_cert;
|
||||
char* tls_key;
|
||||
char* tls_ca;
|
||||
int timeout;
|
||||
int contimeout;
|
||||
bool quiet;
|
||||
bool backup;
|
||||
char* backup_dir;
|
||||
bool stats;
|
||||
int max_depth;
|
||||
FILE* log_file;
|
||||
int queue_size;
|
||||
bool follow_symlinks;
|
||||
bool partial;
|
||||
|
||||
// Issue #120: Symlink handling
|
||||
bool copy_links;
|
||||
bool safe_links;
|
||||
bool copy_unsafe_links;
|
||||
/* Phase 4 symlink-trust. -k/--copy-dirlinks and --munge-links are
|
||||
* CLIENT/sender-side only (they decide how the SENDER scans and rewrites
|
||||
* symlinks; the receiver never reads them), so they never cross the wire.
|
||||
* -K/--keep-dirlinks is a RECEIVER-side policy (follow an in-root destination
|
||||
* symlink-to-directory as a directory) and CROSSES the wire along with
|
||||
* --munge-links (so the receiver knows to unmunge). */
|
||||
bool copy_dirlinks; /* client-only, sender-side (-k) */
|
||||
bool munge_links; /* crosses the wire */
|
||||
bool keep_dirlinks; /* crosses the wire (-K) */
|
||||
|
||||
// Issue #121: Extended metadata preservation
|
||||
bool preserve_hard_links;
|
||||
bool preserve_acls;
|
||||
bool preserve_xattrs;
|
||||
bool preserve_devices;
|
||||
bool preserve_sparse;
|
||||
/* Phase 4 special/devices: preserve special files (FIFOs, sockets) and device
|
||||
* nodes on the destination by recreating them (mknod/mkfifo) instead of
|
||||
* transferring content. preserve_specials mirrors rsync --specials (the
|
||||
* special-file half of -D); preserve_devices mirrors --devices (the device
|
||||
* half of -D); both CROSS the wire so the receiver knows a special/device
|
||||
* entry must be recreated rather than written as a regular file. */
|
||||
bool preserve_specials;
|
||||
/* --copy-devices: copy the CONTENT of a source device as an ordinary regular
|
||||
* file on the destination (rsync's non-privileged safe mode), instead of
|
||||
* recreating the device node. CROSSES the wire (receiver treats the entry as
|
||||
* a regular file, which is the default, so this is belt-and-braces). */
|
||||
bool copy_devices;
|
||||
/* --write-devices: write the received data directly INTO an existing device
|
||||
* node on the destination instead of creating a regular file. Dangeroud;
|
||||
* see RSYNC_COMPAT.md for the tight gating. CROSSES the wire. */
|
||||
bool write_devices;
|
||||
|
||||
// Issue #122: Output/logging options
|
||||
bool itemize_changes;
|
||||
char* out_format;
|
||||
char* log_file_format;
|
||||
int info_level;
|
||||
int debug_level;
|
||||
bool list_only;
|
||||
bool human_readable;
|
||||
bool eight_bit_output;
|
||||
|
||||
// Issue #127: Transfer modes
|
||||
bool existing;
|
||||
bool ignore_existing;
|
||||
bool update;
|
||||
bool inplace;
|
||||
bool delay_updates;
|
||||
bool use_fsync;
|
||||
bool append;
|
||||
bool append_verify;
|
||||
/* --preallocate: allocates the destination file's full expected space up
|
||||
* front (before any data is written) so a transfer that would overflow disk
|
||||
* fails fast at allocation time and the file is laid out contiguously,
|
||||
* avoiding fragmentation. Receiver-side, crosses the wire. */
|
||||
bool preallocate;
|
||||
|
||||
// Issue #128: Extended delete options
|
||||
/* --delete-excluded: also delete destination entries that were excluded on
|
||||
* the source. Default (off) matches rsync: excluded paths are protected from
|
||||
* deletion. Crosses the wire (the sender encodes the choice by whether it
|
||||
* transmits a protected-prefix list with the keep-set manifest). */
|
||||
bool delete_excluded;
|
||||
bool delete_after;
|
||||
/* --max-delete=NUM: the receiver refuses to delete more than NUM entries per
|
||||
* run (all-or-nothing: when the extras would exceed NUM nothing is removed and
|
||||
* the transfer fails with a distinct error). -1 == no client limit (the
|
||||
* server hard bound MAX_SERVER_DELETE_COUNT still applies). */
|
||||
int max_delete;
|
||||
/* --ignore-errors (client-only, never serialized): a sender-side source I/O
|
||||
* error (an unreadable directory during the scan) normally aborts the run so
|
||||
* no deletion happens; with --ignore-errors the scan continues and the
|
||||
* (partial) keep-set is still transmitted so the deletion runs. */
|
||||
bool ignore_errors;
|
||||
/* --force (receiver-side): a regular file may replace a destination
|
||||
* directory by removing that (possibly non-empty, symlink-safe) directory
|
||||
* tree first, instead of failing the write. Crosses the wire. */
|
||||
bool force_delete;
|
||||
/* --ignore-missing-args (client-only, never serialized): a --files-from
|
||||
* entry that does not exist under the source is silently skipped instead of
|
||||
* failing the run. Sender-side only: nothing is sent for it and it never
|
||||
* enters the keep-set. Implied by --delete-missing-args. */
|
||||
bool ignore_missing_args;
|
||||
/* --delete-missing-args: implies --ignore-missing-args; additionally each
|
||||
* missing entry's destination mirror (computed like a present entry's wire
|
||||
* path) is deleted receiver-side. Crosses the wire and is gated by the
|
||||
* server's --allow-delete policy like --delete. rsync-parity: independent
|
||||
* of ordinary --delete processing (it does not imply --delete); a non-empty
|
||||
* directory mirror is only removed with --force or --delete in effect, and
|
||||
* the missing-args deletions are not counted toward --max-delete. */
|
||||
bool delete_missing_args;
|
||||
|
||||
// Issue #129: Advanced file selection. These fields are CLIENT-ONLY: they are
|
||||
// never serialized to the wire (the receiver must not learn them).
|
||||
ArrayList* filters; /* --filter=RULE rule strings, in order */
|
||||
char* files_from; /* --files-from path (may be NULL) */
|
||||
void* files_from_set; /* parsed FileListSet* allow-set, or NULL */
|
||||
bool from0; /* -0/--from0: NUL-delimited *-from files */
|
||||
bool cvs_exclude; /* -C/--cvs-exclude: standard CVS ignore set */
|
||||
bool per_dir_filter; /* -F: apply per-directory .rsync-filter files */
|
||||
bool prune_empty_dirs;
|
||||
bool one_file_system; /* -x/--one-file-system: do not cross filesystem boundaries */
|
||||
/* -R/--relative: crosses the wire; with --files-from listed entries keep
|
||||
* their bare relative destination path (no source-root mirror prefix). */
|
||||
bool relative;
|
||||
/* --no-implied-dirs: client-only. With -R + --files-from, refuse to place a
|
||||
* listed file whose ancestor directory is not itself explicitly listed. */
|
||||
bool no_implied_dirs;
|
||||
/* -d/--dirs: client-only. Transfer the directory entries named by the
|
||||
* source argument / --files-from list without recursing into contents. */
|
||||
bool dirs;
|
||||
/* --mkpath: crosses the wire. Tells the server to create the destination
|
||||
* root directory (and missing leading components below its authorized root)
|
||||
* at connection start instead of requiring it to already exist. */
|
||||
bool mkpath;
|
||||
|
||||
// Issue #130: Remote shell/connection options
|
||||
/* -e/--rsh: the remote-shell program used to establish the SSH transport.
|
||||
* NULL means the default "ssh". Client-only launch concern: NEVER crosses
|
||||
* the wire (it is not meaningful to the daemon/server handshake). */
|
||||
char* rsh_command;
|
||||
/* --blocking-io: leave the SSH transport socket without
|
||||
* SO_RCVTIMEO/SO_SNDTIMEO so it blocks naturally instead of timing out.
|
||||
* Client-only launch concern: NEVER crosses the wire. */
|
||||
bool blocking_io;
|
||||
/* --outbuf mode (OutbufMode): stdout/stderr buffering. Client-only launch
|
||||
* concern: NEVER crosses the wire. */
|
||||
int outbuf;
|
||||
bool old_args;
|
||||
char* temp_dir;
|
||||
/* --remote-option=OPT (Phase 5, long form only): one or more extra command-line
|
||||
* options to append to the REMOTE server invocation over SSH. CLIENT-ONLY:
|
||||
* they are composed into the remote command line by ssh_build_remote_command()
|
||||
* (each valid word is shell-escaped with the same quoting boundary as the
|
||||
* server path), and are NEVER serialized into the binary config frame. They
|
||||
* do NOT cross the wire and are never parsed on the receiver process. */
|
||||
char** remote_options;
|
||||
int remote_option_count;
|
||||
/* Alternate basis directories, ordered by command-line appearance. Each
|
||||
* entry's type selects compare/copy/link behavior on an exact match. These
|
||||
* cross the wire so the receiver can consult them; they are interpreted
|
||||
* relative to the destination root and confined there. */
|
||||
BasisDest* basis_dirs;
|
||||
int basis_count;
|
||||
|
||||
// PR #174: Partial transfer resumption
|
||||
char* partial_dir;
|
||||
|
||||
// PR #178: Backup versioning
|
||||
char* suffix;
|
||||
|
||||
// PR #179: Delete policies
|
||||
bool delete_before;
|
||||
|
||||
/* rsync deletion-timing family (real from Phase 3). At most one of
|
||||
delete_before / delete_during / delete_delay / delete_after may be set, and
|
||||
only together with use_delete (the CLI implies --delete for each of them).
|
||||
delete_before and delete_during select the EARLY engine mode: the keep-set
|
||||
manifest is transmitted before any file data and extras are removed then,
|
||||
acknowledged, before the first data byte. delete_delay and delete_after
|
||||
select the LATE commit mode: extras are removed only after the whole
|
||||
transfer has succeeded (plain --delete keeps this mode). The exact
|
||||
semantics and the divergences from rsync are documented in RSYNC_COMPAT.md
|
||||
and in config_delete_timing_early() below. */
|
||||
bool delete_during;
|
||||
bool delete_delay;
|
||||
|
||||
// PR #181: IPv6 and bind address
|
||||
char* address;
|
||||
char* bind_address;
|
||||
bool ipv6;
|
||||
bool ipv4;
|
||||
/* --sockopts=OPTIONS (Phase 5, Wave B): strict allowlist of TCP/socket
|
||||
* options applied via setsockopt after socket() and before connect()/bind().
|
||||
* These are LOCAL socket concerns: they never cross the wire config frame.
|
||||
* .address is the outgoing/source bind address (--address); .bind_address is
|
||||
* reserved for daemon-side binding and is not wired yet. */
|
||||
SockOptEntry* sockopts;
|
||||
int sockopt_count;
|
||||
|
||||
// PR #182: Daemon/server mode
|
||||
bool daemon;
|
||||
char* daemon_config;
|
||||
bool server_mode;
|
||||
/* --no-motd (Wave C): CLIENT-ONLY, never crosses the wire. Suppresses
|
||||
* DISPLAY of the daemon's MOTD; the daemon still sends the MOTD frame, so
|
||||
* the client reads and discards it to keep the stream in sync. rsync's
|
||||
* --no-motd is likewise a client-side display switch. Default false (the
|
||||
* MOTD is shown when a daemon offers one). */
|
||||
bool no_motd;
|
||||
|
||||
// PR #183: Checksum comparison
|
||||
bool checksum;
|
||||
|
||||
// PR #184: Compression algorithm negotiation
|
||||
char* compress_choice;
|
||||
char* chmod_spec;
|
||||
|
||||
/* --checksum-choice / --cc and --checksum-seed. checksum_algo is the id of
|
||||
* the whole-file content-digest algorithm used by the per-file --incremental
|
||||
* handshake (sender computes it, receiver compares it to skip unchanged
|
||||
* files) and by the basis-dir content verification. checksum_seed is passed
|
||||
* to xxHash64 (and to the delta block strong hash, low 32 bits); md5 has no
|
||||
* seed so it is ignored there. Both cross the wire: the receiver MUST hash
|
||||
* the on-disk old file with the same algorithm and seed to reach a matching
|
||||
* digest. Defaults (XXH64 / seed 0) reproduce the pre-existing behavior
|
||||
* byte-for-byte. */
|
||||
int checksum_algo; /* ChecksumAlgo, default CHECKSUM_ALGO_XXH64 */
|
||||
uint64_t checksum_seed; /* default 0 */
|
||||
|
||||
char** skip_compress_suffixes;
|
||||
int skip_compress_count;
|
||||
bool skip_compress_set;
|
||||
|
||||
// Issue #131: Identity mapping. These configure whether and how the receiver
|
||||
// applies ownership when it is actually preserved/applied. ALL of them cross
|
||||
// the wire (protocol 2.11.0) so the receiver resolves and applies ownership
|
||||
// with the exact policy the client requested. Plain -M/--preserve still does
|
||||
// NOT apply ownership (FastSync's deliberate conservative default); it is
|
||||
// only attempted when at least one of these is set (see identity.h).
|
||||
/* --numeric-ids: no name lookup, use the transmitted numeric ids raw. */
|
||||
bool numeric_ids;
|
||||
/* --chown USER (owner) override; IDENTITY_CURRENT = the receiver's euid. */
|
||||
bool chown_uid_set;
|
||||
int32_t chown_uid;
|
||||
/* --chown :GROUP (group) override; IDENTITY_CURRENT = the receiver's egid. */
|
||||
bool chown_gid_set;
|
||||
int32_t chown_gid;
|
||||
/* --usermap / --groupmap entries, in order (first match wins). */
|
||||
IdentityMap* usermap;
|
||||
int usermap_count;
|
||||
IdentityMap* groupmap;
|
||||
int groupmap_count;
|
||||
|
||||
/* --super / --no-super (P7 Wave E, protocol 2.18.0): receiver-side privilege
|
||||
* policy for super-user activities confined below the authorized receive
|
||||
* root. SUPER_MODE_AUTO (default) preserves the pre-existing best-effort
|
||||
* behavior: the confined super-user operation is ALWAYS attempted and an
|
||||
* unprivileged attempt is refused by the kernel and skipped per entry.
|
||||
* SUPER_MODE_ON (--super) explicitly REQUESTS those activities (char/block
|
||||
* device-node creation, --write-devices); it does NOT imply --numeric-ids and
|
||||
* never enables ownership application on its own. SUPER_MODE_OFF
|
||||
* (--no-super) FORBIDS them even when running as root. FastSync NEVER
|
||||
* elevates privileges (no setuid/seteuid/setgid) and never bypasses the
|
||||
* fd-relative confinement (file_open_secure_parent, O_NOFOLLOW, root checks);
|
||||
* --super only permits an attempt that is already confined. Crosses the wire
|
||||
* as a trailing int so the receiver can enforce the policy. See
|
||||
* privilege_super_permitted() and identity_ownership_requested() in
|
||||
* identity.h. */
|
||||
int super_mode;
|
||||
|
||||
// Receiver-side runtime staging registry for --delay-updates. Never sent
|
||||
// over the wire and never set on the sender side.
|
||||
DelayUpdatesContext* delay_context;
|
||||
|
||||
// Phase 4: metadata time preservation. -U/--atimes and -N/--crtimes capture
|
||||
// and transmit the source access / birth time (both sender and receiver
|
||||
// effect, so they CROSS the wire). --omit-dir-times/-O and
|
||||
// --omit-link-times/-J are receiver-side prefs (CROSS the wire). Their
|
||||
// exact capture/transmit/apply semantics are documented in RSYNC_COMPAT.md.
|
||||
/* -U/--atimes: preserve source access times on the destination. */
|
||||
bool preserve_atimes;
|
||||
/* -N/--crtimes: capture+transmit source birth time; see RSYNC_COMPAT for the
|
||||
* receiver not-applied divergence. */
|
||||
bool preserve_crtimes;
|
||||
/* -O/--omit-dir-times: do not apply mtimes to directories. */
|
||||
bool omit_dir_times;
|
||||
/* -J/--omit-link-times: do not apply times to symlinks. */
|
||||
bool omit_link_times;
|
||||
/* --open-noatime: CLIENT-ONLY (never crosses the wire). The sender opens
|
||||
* source files with O_NOATIME so reading for transfer does not bump the
|
||||
* source access time. */
|
||||
bool open_noatime;
|
||||
|
||||
// Phase 4: xattr / ACL / fake-super preservation.
|
||||
/* -X/--xattrs and -A/--acls toggle the sender's capture and the receiver's
|
||||
* application of per-file extended attributes (xattrs). Both cross the wire:
|
||||
* the sender only transmits the bounded, whitelisted attribute set it
|
||||
* captures and the receiver re-validates namespaces/sizes before applying
|
||||
* fd-relative. With neither set (the default) no xattr block is sent, so the
|
||||
* wire is byte-identical to prior protocol versions for unaffected runs. */
|
||||
/* true when preserve_xattrs || preserve_acls; the sender/receiver gate the
|
||||
* xattr wire block on this single flag. */
|
||||
bool use_xattrs;
|
||||
/* --fake-super: receiver-only. When set, each written file additionally gets
|
||||
* a reserved user.fastsync.stat xattr recording the source uid/gid/mode/mtime
|
||||
* so a later privileged restore could re-apply them. Crosses the wire. */
|
||||
bool fake_super;
|
||||
/* --copy-as=USER[:GROUP] (P7 Wave E, protocol 2.18.0). Safe-subset
|
||||
* implementation, a documented divergence from rsync's real identity switch:
|
||||
* the receiver does NOT change its process credentials (FastSync's receiver
|
||||
* is multithreaded, so a setuid/seteuid drop would be unsafe). Instead the
|
||||
* receiver FORCES the ownership of every entry it writes to copy_as_uid /
|
||||
* copy_as_gid through the existing confined, fd-relative identity path
|
||||
* (fchown/fchownat), which REQUIRES receiver privilege (root); an
|
||||
* unprivileged receiver REFUSES the whole transfer up front at the config
|
||||
* handshake (never a silent wrong-ownership result). All three fields CROSS
|
||||
* the wire as a trailing config-frame block so the receiver learns the
|
||||
* requested ids; see the PROTOCOL_VERSION note below. */
|
||||
bool copy_as_set;
|
||||
int32_t copy_as_uid;
|
||||
int32_t copy_as_gid;
|
||||
|
||||
// Phase 5: --trust-sender
|
||||
/* Long-form-only, receiver-local policy. rsync's --trust-sender tells the
|
||||
* receiving side to trust that the sender already produced a sane file list,
|
||||
* relaxing the receiver's own up-front re-validation of every incoming path.
|
||||
* In FastSync the receiver normally double-checks each transmitted file-list
|
||||
* entry (empty / ".." path-traversal rejection) and refuses to materialize a
|
||||
* symlink whose target could escape the receive root. When trust_sender is
|
||||
* set, those redundant list-level re-checks are SKIPPED: the receiving side
|
||||
* trusts the sender's list instead of re-validating it (fewer checks, faster,
|
||||
* potentially unsafe, matching rsync). It is a LOCAL receiver policy and is
|
||||
* NEVER serialized into the config frame (it exists only on the process that
|
||||
* actually receives the file list). Even under trust_sender the low-level
|
||||
* fd-relative confinement primitives (file_open_secure_parent, the O_NOFOLLOW
|
||||
* parent walk, leaf/destination confinement) are deliberately KEPT as a hard
|
||||
* floor, so a hostile sender still cannot write or link outside the
|
||||
* authorized root (see the phase-5 notes in RSYNC_COMPAT.md). Off by
|
||||
* default; only relaxes validation when explicitly requested. */
|
||||
bool trust_sender;
|
||||
|
||||
// Phase 6: --stop-after / --stop-at
|
||||
/* Client-only sender-side transfer stop deadlines. --stop-after=MINS stops
|
||||
* the transfer after a number of elapsed minutes (checked against
|
||||
* CLOCK_MONOTONIC so clock changes do not skew it); --stop-at=TIME stops at
|
||||
* an absolute wall-clock time (HH:MM, HH:MM:SS, or now+N[smhd]). At the
|
||||
* deadline the run stops elegantly at the next chunk/file boundary and the
|
||||
* completion tail still runs (exit 0). Both are LOCAL to the sending
|
||||
* process and are NEVER serialized into the config frame. */
|
||||
int stop_after_mins; /* --stop-after=MINS minutes; 0 when unset */
|
||||
time_t stop_at; /* --stop-at=... absolute wall-clock deadline */
|
||||
bool stop_at_set; /* true when --stop-at was given */
|
||||
|
||||
// Phase 6: --write-batch / --only-write-batch / --read-batch
|
||||
/* Client-only residual-batch paths. A residual batch is a self-contained
|
||||
* single-file record of the whole source tree (full file images using the
|
||||
* chunk codec), independent of any live server. --write-batch=FILE runs the
|
||||
* normal live transfer AND additionally emits the batch FILE;
|
||||
* --only-write-batch=FILE emits FILE only (no destination, no server);
|
||||
* --read-batch=FILE applies FILE to the destination (no source, no server).
|
||||
* All three are LOCAL to the driving process and are NEVER serialized into
|
||||
* the config frame (the batch paths bypass the transport entirely). */
|
||||
char* write_batch; /* --write-batch=FILE path, or NULL */
|
||||
char* only_write_batch; /* --only-write-batch=FILE path, or NULL */
|
||||
char* read_batch; /* --read-batch=FILE path, or NULL */
|
||||
char *tls_cert;
|
||||
char *tls_key;
|
||||
char *tls_ca;
|
||||
} Config;
|
||||
|
||||
/* Phase 5 (remote-option wave): 2.13.0 -> 2.14.0.
|
||||
*
|
||||
* WHY the bump, grounded in the wire: the binary config-frame layout is
|
||||
* UNCHANGED by this wave (neither --remote-option nor --trust-sender adds a
|
||||
* serialized field; see the field comments above). --remote-option is
|
||||
* forwarded to the remote server over the SSH remote-command line
|
||||
* (ssh_build_remote_command) and --trust-sender is a purely local receiver
|
||||
* policy, so there is no new frame byte to negotiate. The bump is still the
|
||||
* correct release marker for Phase 5 because the client-to-server INVOCATION
|
||||
* surface changed: a client that composes remote-options expects a server that
|
||||
* knows how to honor them, and the only safe way to express "this feature set
|
||||
* is one coordinated release" is the strict same-version handshake FastSync
|
||||
* already performs for every release. A 2.14 client against a 2.13 server
|
||||
* fails the version check cleanly up front (rather than the remote server
|
||||
* rejecting an unfamiliar forwarded argv at a confusing later point), which is
|
||||
* exactly what the lockstep convention of this project requires. */
|
||||
/* Daemon Wave A: 2.14.0 -> 2.15.0.
|
||||
*
|
||||
* WHY the bump, grounded in the wire: this wave really does add a serialized
|
||||
* field to the binary config frame. The client sends its requested daemon
|
||||
* module name (Config->module) as a new trailing string on the frame (sent
|
||||
* after the Phase-4 xattr block and before the STATUS_OK/STATUS_ERROR ack, in
|
||||
* config_send/config_receive), and the daemon reads it to select which module
|
||||
* root confines the connection. Any config-frame layout change must bump the
|
||||
* protocol version because a peer that does not parse the new trailing bytes
|
||||
* would desynchronize on the frame boundary; the strict same-version handshake
|
||||
* (config_receive rejects a mismatched version before parsing anything else)
|
||||
* is what keeps a 2.15 client and a 2.14 server from ever reaching that state.
|
||||
*
|
||||
* NOTE: daemon module-selection bump owned by Wave A (2.15.0); later daemon
|
||||
* waves (auth, motd) must not bump PROTOCOL_VERSION. Wave B (auth) added the
|
||||
* credential fields (auth_user + password digest) as further trailing
|
||||
* config-frame strings AFTER the Wave A module string, with a presence int
|
||||
* prefix. This is not a new frame version: sender and receiver of a 2.15.0
|
||||
* build always read and write the same full layout (the strict same-version
|
||||
* handshake rejects any other version before a byte of the frame is parsed),
|
||||
* so a peer can never desynchronize on the added tail. The 2.15.0 release
|
||||
* ships Wave A + Wave B together; the bump stays owned by Wave A.
|
||||
*
|
||||
* Wave C (MOTD) adds NO config-frame field and no version bump either. On the
|
||||
* daemon listener path only, the server sends one MOTD string frame AFTER the
|
||||
* config-frame STATUS_OK (server.c handler), and every 2.15.0 daemon client
|
||||
* reads that frame right after the ack (client_send.c) -- symmetric
|
||||
* server->client in every build, so the strict same-version handshake keeps the
|
||||
* two peers in lockstep and nothing can desynchronize. The --stdio SSH path
|
||||
* sends/reads no MOTD at all.
|
||||
*
|
||||
* --iconv Wave (P6): 2.15.0 -> 2.16.0.
|
||||
*
|
||||
* WHY the bump, grounded in the wire: the --iconv feature adds a serialized
|
||||
* field to the binary config frame. The client sends the full CONVERT_SPEC
|
||||
* (Config->iconv_spec) as a new trailing string AFTER the Wave A/B daemon-auth
|
||||
* block (in config_send/config_receive), so the receiver knows the wire charset
|
||||
* (the REMOTE half) before the first file name arrives. Any config-frame
|
||||
* layout change must bump the protocol version: a peer that does not parse the
|
||||
* new trailing bytes would desynchronize on the frame boundary, and the strict
|
||||
* same-version handshake (config_receive rejects a mismatched version before
|
||||
* parsing anything else) is what keeps a 2.16 client and a 2.15 server from
|
||||
* ever reaching that state.
|
||||
*
|
||||
* Times Wave (P7 Wave D): 2.16.0 -> 2.17.0.
|
||||
*
|
||||
* WHY the bump, grounded in the wire: this wave makes -O/--omit-dir-times and
|
||||
* -J/--omit-link-times REAL by adding directory and symlink time preservation.
|
||||
* The config-frame LAYOUT is unchanged (the omit flags already crossed the
|
||||
* wire), but the FRAME STREAM gains a new terminal frame: after all file data
|
||||
* and the optional delete manifest, the sender transmits STATUS_DIR_TIMES
|
||||
* frame(s) (each a count followed by (path, metadata) pairs, chunked so no
|
||||
* frame exceeds the receiver's MAX_MANIFEST_ENTRIES bound) carrying every
|
||||
* source directory's captured times, so the receiver can apply them AFTER all of a
|
||||
* directory's children have been written (writing a child bumps the parent's
|
||||
* mtime). Symlink entries already carry their metadata on the STATUS_SYMLINK
|
||||
* frame; the receiver now applies it (utimensat/lchown with
|
||||
* AT_SYMLINK_NOFOLLOW) unless -J is set. Any change to the frame sequence must
|
||||
* bump the protocol version: a 2.16 peer that does not know STATUS_DIR_TIMES
|
||||
* would desynchronize on the unknown frame, and the strict same-version
|
||||
* handshake (config_receive rejects a mismatched version before parsing
|
||||
* anything else) is what keeps a 2.17 client and a 2.16 server from ever
|
||||
* reaching that state.
|
||||
*
|
||||
* Privilege Wave (P7 Wave E): 2.17.0 -> 2.18.0.
|
||||
*
|
||||
* WHY the bump, grounded in the wire: this wave adds the receiver-side
|
||||
* privilege flags --super/--no-super and --copy-as=USER[:GROUP]. The
|
||||
* config-frame layout gains two new trailing blocks AFTER the --iconv
|
||||
* CONVERT_SPEC string, in this fixed order: (1) send_privilege_options /
|
||||
* receive_privilege_options send one int (Config->super_mode, 0..2), then
|
||||
* (2) send_copy_as_options / receive_copy_as_options send a presence int and,
|
||||
* when set, the target uid and gid (both int32). The receiver uses
|
||||
* super_mode to decide whether it may attempt super-user activities
|
||||
* (ownership application, char/block device-node creation) already confined
|
||||
* below the authorized receive root, and the copy-as ids to force the
|
||||
* ownership of every entry it writes (the safe-subset --copy-as model). The
|
||||
* receiver REQUIRES privilege for copy-as: an unprivileged receiver refuses
|
||||
* the transfer at the config handshake (server_module_gate) instead of silently
|
||||
* ignoring the flag. Any config-frame layout change must bump the protocol
|
||||
* version: a peer that does not parse the new trailing bytes would
|
||||
* desynchronize on the frame boundary, and the strict same-version handshake
|
||||
* (config_receive rejects a mismatched version before parsing anything else) is
|
||||
* what keeps a 2.18 client and a 2.17 server from ever reaching that state.
|
||||
* --super never elevates privileges; it only permits a confined attempt, and
|
||||
* --copy-as never switches process credentials (see RSYNC_COMPAT.md).
|
||||
*
|
||||
* A7 Auth Wave: 2.18.0 -> 2.19.0.
|
||||
*
|
||||
* WHY the bump, grounded in the wire: the daemon auth block on the config frame
|
||||
* loses the hard-wired password digest (it becomes `[int present][str_redacted
|
||||
* username]`), and the frame stream gains the SCRAM challenge/response
|
||||
* (STATUS_AUTH_CHALLENGE -> STATUS_AUTH_RESPONSE -> STATUS_AUTH_OK) between the
|
||||
* config frame and the STATUS_OK ack. A 2.18 peer would desynchronize on both
|
||||
* the shorter auth block and the new status frames, so the strict same-version
|
||||
* handshake (config_receive rejects a mismatched version before parsing
|
||||
* anything else) is what keeps a 2.19 client and a 2.18 server from ever
|
||||
* reaching that state. SECURITY: a 2.19 store holds a salted PBKDF2 verifier
|
||||
* and cannot verify (and refuses to load) a legacy unsalted-SHA-256 store line,
|
||||
* so an old bearer digest can never be replayed against a 2.19 daemon. */
|
||||
#define PROTOCOL_VERSION "2.19.0"
|
||||
#define PROTOCOL_VERSION "1.2.0"
|
||||
#define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024)
|
||||
/* Upper bound on total basis-dir entries (rsync caps --link-dest at 20). */
|
||||
#define MAX_BASIS_DIRS 64
|
||||
|
||||
/* Identity-mapping sentinels and bounds (see identity.h for semantics).
|
||||
* IDENTITY_MATCH_ANY is a usermap/groupmap FROM '*' (matches any id);
|
||||
* IDENTITY_CURRENT is a chown / map TO '*' (resolve to the receiver's current
|
||||
* euid/egid at apply time). */
|
||||
#define IDENTITY_MATCH_ANY (-1)
|
||||
#define IDENTITY_CURRENT (-1)
|
||||
#define MAX_IDENTITY_MAP 128
|
||||
|
||||
/* --super / --no-super tri-state (Config->super_mode). AUTO (default) and ON
|
||||
* both permit a confined super-user attempt (AUTO preserves FastSync's
|
||||
* historical best-effort behavior; an unprivileged attempt is refused by the
|
||||
* kernel and skipped per entry); OFF forbids the attempt even for root. See
|
||||
* privilege_super_mode_permitted() in identity.h. */
|
||||
#define SUPER_MODE_AUTO 0
|
||||
#define SUPER_MODE_ON 1
|
||||
#define SUPER_MODE_OFF 2
|
||||
|
||||
Config* config_create(void);
|
||||
void config_delete(Config* config);
|
||||
|
||||
/* Wipe the client-side plaintext auth password (and username) from a Config
|
||||
* before it is freed or handed off. Safe on a NULL/empty Config and idempotent
|
||||
* (it clears the pointers after burning). config_delete calls this
|
||||
* automatically; a caller that drops a Config earlier may call it explicitly. */
|
||||
void config_burn_auth(Config* config);
|
||||
|
||||
bool config_send(int file_descriptor, const Config* config);
|
||||
Config* config_receive(int file_descriptor);
|
||||
bool config_is_remote_dest(const char* s);
|
||||
void config_parse_ssh_dest(Config* config);
|
||||
|
||||
/* A ConfigValidateFunc may return this sentinel to tell
|
||||
* config_receive_with_validate that the callback ALREADY sent a terminal status
|
||||
* frame (e.g. STATUS_AUTH_FAILED, then closed) and the frame must be abandoned
|
||||
* without an additional STATUS_ERROR. A normal rejection returns a message
|
||||
* string (logged, then STATUS_ERROR); NULL accepts. */
|
||||
#define CONFIG_VALIDATE_ALREADY_TERMINATED ((const char*)-1)
|
||||
|
||||
/* Server-side config-frame gate (daemon module selection, Wave A). A server
|
||||
* that needs to make an accept/reject decision about a received Config BEFORE
|
||||
* it sends the STATUS_OK ack (so a rejected connection is refused cleanly with
|
||||
* no data transferred) passes a callback here; it runs after the frame parses
|
||||
* and validates but before the STATUS_OK/STATUS_ERROR ack. Return NULL to
|
||||
* accept the connection; return a non-NULL message to reject it (the message
|
||||
* is logged server-side and STATUS_ERROR is sent in place of STATUS_OK), or the
|
||||
* CONFIG_VALIDATE_ALREADY_TERMINATED sentinel when the callback already sent
|
||||
* its own terminal status. The callback runs in the connection's own process,
|
||||
* so it may set up per-module process state (e.g. the authorized root) and
|
||||
* drive the daemon auth handshake. context is an opaque caller pointer. */
|
||||
typedef const char* (*ConfigValidateFunc)(const Config* config, void* context);
|
||||
Config* config_receive_with_validate(int file_descriptor, ConfigValidateFunc validate,
|
||||
void* context);
|
||||
|
||||
/* Daemon-destination (host::module[/path]) helpers, Wave A. config_is_remote_dest
|
||||
* recognizes the ordinary rsync-style single-colon host:path form used by the
|
||||
* SSH transport; config_is_daemon_dest recognizes the double-colon form that
|
||||
* selects a daemon module over TCP. config_parse_transport_dest is the single
|
||||
* entry point main() uses: it parses a :: destination as a daemon TCP
|
||||
* destination (host -> server_host, module -> config->module, path ->
|
||||
* receive_root_directory) and otherwise falls back to the existing SSH
|
||||
* host:path handling. */
|
||||
bool config_is_daemon_dest(const char* s);
|
||||
/* Returns 1 when the destination was daemon syntax and was parsed, 0 when it
|
||||
* is not daemon syntax (nothing changed), -1 on an invalid daemon destination
|
||||
* (a message is logged and config is left untouched). */
|
||||
int config_parse_daemon_dest(Config* config);
|
||||
/* Returns 1/0/-1 mirroring config_parse_daemon_dest when the destination is
|
||||
* daemon syntax; otherwise runs the existing SSH host:path parse and returns
|
||||
* 0. */
|
||||
int config_parse_transport_dest(Config* config);
|
||||
|
||||
/* True when the negotiated delete timing performs the extra-file deletion
|
||||
* BEFORE the transfer data (--delete-before / --delete-during). The flag is
|
||||
* a pure function of the config and is used identically on the sender (to pick
|
||||
* the manifest-first frame order) and the receiver (to delete when the early
|
||||
* manifest arrives). When false the deletion is committed only after the whole
|
||||
* transfer succeeded (--delete / --delete-after / --delete-delay). */
|
||||
bool config_delete_timing_early(const Config* config);
|
||||
/* Delete-timing sanity: with deletion enabled at most one timing flag may be
|
||||
* set (none = the default delete-after commit timing); without deletion no
|
||||
* timing flag may be set (each timing flag implies --delete). */
|
||||
bool config_has_valid_delete_timing(const Config* config);
|
||||
/* True when at least one --compare-dest/--copy-dest/--link-dest was set. */
|
||||
bool config_has_basis(const Config* config);
|
||||
/* Append one basis-dir entry. Returns 0 on success, -1 on allocation failure. */
|
||||
int config_basis_append(Config* config, BasisDestType type, const char* path);
|
||||
/* Validate a client-provided basis-dir path (relative, confined, non-empty). */
|
||||
bool config_basis_path_valid(const char* path);
|
||||
|
||||
/* Parse and validate a --sockopts=OPTIONS comma-separated "OPT=VAL" list into a
|
||||
* malloc'd array of at most *out_count entries. Returns 0 on success (the
|
||||
* caller takes ownership of *out), or -1 on the first invalid option name or
|
||||
* value. Pure/static-analysis friendly: performs no socket calls, so it is
|
||||
* directly unit-testable. */
|
||||
int config_sockopts_parse(const char* spec, SockOptEntry** out, int* out_count);
|
||||
Config *config_create(char *version, char *send_directory,
|
||||
char *receive_directory, bool save_to_disk,
|
||||
bool use_multithreading, bool use_chunk_serialization,
|
||||
bool use_compression, bool use_metadata,
|
||||
int compression_level, bool use_sendfile,
|
||||
unsigned long long chunk_size);
|
||||
void config_delete(Config *config);
|
||||
bool config_send(int file_descriptor, Config *config);
|
||||
Config *config_receive(int file_descriptor);
|
||||
bool is_remote_dest(const char *s);
|
||||
void config_parse_ssh_dest(Config *config);
|
||||
|
||||
#endif
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,203 +0,0 @@
|
||||
#ifndef CREDENTIALS_H
|
||||
#define CREDENTIALS_H
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
|
||||
/* Daemon password authentication (A7 remediation, protocol 2.19.0).
|
||||
*
|
||||
* FastSync authenticates a daemon connection with a SCRAM-SHA-256-style
|
||||
* challenge/response handshake. The daemon stores only a salted PBKDF2
|
||||
* verifier (never the password, and never a value that can be replayed as a
|
||||
* bearer credential): the client proves knowledge of the password against a
|
||||
* per-connection server nonce, and the server proves the same shared secret
|
||||
* back. See credentials.c for the exact derivation.
|
||||
*
|
||||
* Server credential store format (--password-file and --early-input): one line
|
||||
* per entry,
|
||||
* user:$fastsync$1$pbkdf2-sha256$<iters>$<salt_b64>$<stored_key_b64>$<server_key_b64>
|
||||
* with standard base64, a 16-byte salt and 32-byte keys, and iters in
|
||||
* [CREDENTIAL_MIN_ITERS, CREDENTIAL_MAX_ITERS]. Blank lines and lines whose
|
||||
* first non-space character is '#' or ';' are comments. The parser is STRICT:
|
||||
* a malformed line fails the whole load so a typo can never silently change who
|
||||
* may log in. A line holding the legacy (unsalted SHA-256 hex) secret is
|
||||
* hard-rejected with an actionable "legacy" error; there is no auto-upgrade.
|
||||
* Use `fastsync-server --hash-credentials` to generate new-format lines.
|
||||
*
|
||||
* Alongside the store, credentials_load maintains an exact-mode-0600
|
||||
* `<store_path>.dummykey` sidecar holding the store-wide random dummy key. It
|
||||
* is auto-created on first load and MUST be preserved across restarts: it makes
|
||||
* the dummy challenge for an unknown user stable for the life of the store, so
|
||||
* a daemon restart cannot be used as a username-enumeration oracle. A sidecar
|
||||
* that is not an exact-mode-0600 regular file of exactly 32 bytes fails the load
|
||||
* (fail closed); creation forces exact 0600 with fchmod (so a restrictive umask
|
||||
* cannot leave the sidecar unreadable), and only a create/write/fsync/link or
|
||||
* fchmod failure degrades to a transient per-run key with a warning. NOTE: the
|
||||
* sidecar requires EXACT 0600, whereas the store / password files only reject
|
||||
* group/other bits (a deliberate difference).
|
||||
*
|
||||
* Client --password-file format: the FIRST meaningful (non-comment, non-blank)
|
||||
* line is `user:password`, holding the literal password. The client keeps it
|
||||
* only for the duration of the handshake and wipes it at teardown; the file
|
||||
* should be mode 0600 and readable only by its owner. */
|
||||
|
||||
/* Longest accepted credential-file line (excluding the trailing newline). */
|
||||
#define CREDENTIAL_MAX_LINE 4096
|
||||
/* Upper bound on a username in a credential file and on the wire. Kept well
|
||||
* below MAX_STRING_SIZE so a wire username can never exhaust anything. */
|
||||
#define CREDENTIAL_MAX_USER_LEN 256
|
||||
/* Upper bound on a client-file password (before derivation). */
|
||||
#define CREDENTIAL_MAX_PASSWORD_LEN 1024
|
||||
|
||||
/* SCRAM-SHA-256 parameters. Salt and client nonce sizes are fixed by the
|
||||
* shared-auth-message framing; keys are always 32 bytes (SHA-256). */
|
||||
#define CREDENTIAL_SALT_LEN 16
|
||||
#define CREDENTIAL_NONCE_LEN 32
|
||||
#define CREDENTIAL_KEY_LEN 32
|
||||
#define CREDENTIAL_DEFAULT_ITERS 600000u
|
||||
#define CREDENTIAL_MIN_ITERS 100000u
|
||||
#define CREDENTIAL_MAX_ITERS 10000000u
|
||||
/* Buffer size for the full AuthMessage (prefix + three length-prefixed fields).
|
||||
* Worst case: 16 + 4 + 256 + 4 + 32 + 4 + 32. */
|
||||
#define CREDENTIAL_AUTH_MESSAGE_MAX \
|
||||
(16 + 4 + CREDENTIAL_MAX_USER_LEN + 4 + CREDENTIAL_NONCE_LEN + 4 + CREDENTIAL_NONCE_LEN)
|
||||
|
||||
typedef struct CredentialStore CredentialStore;
|
||||
|
||||
/* One resolved verifier. `found` is false for an unknown user or a user not on
|
||||
* a module's auth list; the remaining fields then hold a deterministic dummy
|
||||
* salt (HMAC of the store-wide dummy key over the username), the store-wide
|
||||
* uniform iteration count (default for an empty store) and fixed dummy keys, so
|
||||
* the server can run the same challenge/response math with no enumeration or
|
||||
* timing oracle. */
|
||||
typedef struct {
|
||||
uint8_t salt[CREDENTIAL_SALT_LEN];
|
||||
uint32_t iters;
|
||||
uint8_t stored_key[CREDENTIAL_KEY_LEN];
|
||||
uint8_t server_key[CREDENTIAL_KEY_LEN];
|
||||
bool found;
|
||||
} CredentialVerifier;
|
||||
|
||||
/* Load the daemon credential store.
|
||||
*
|
||||
* password_file and early_input_file are both NULL-or-path, matching the
|
||||
* server's --password-file and --early-input options. A file that cannot be
|
||||
* opened or that fails the strict grammar is a hard error (err filled, NULL
|
||||
* returned) -- the daemon fails CLOSED rather than serving an auth-required
|
||||
* module with a partial store. Both files may be NULL, which yields an empty
|
||||
* store (every auth-required module then refuses connections). Every entry in
|
||||
* the resulting store must agree on the iteration count; entries that disagree
|
||||
* (within one file or across the two layered sources) are rejected. When both
|
||||
* are given, the --early-input file is layered over --password-file: a duplicate
|
||||
* username whose verifier matches is deduplicated; one whose verifier differs
|
||||
* is an error (the two sources disagree), never a silent pick.
|
||||
*
|
||||
* The returned store is heap-owned; free it with credentials_free. */
|
||||
CredentialStore* credentials_load(const char* password_file, const char* early_input_file,
|
||||
char* err, size_t err_size);
|
||||
|
||||
/* Wipe every stored key/salt and free the store. */
|
||||
void credentials_free(CredentialStore* store);
|
||||
|
||||
/* True when `user` is a single bounded token free of whitespace/control bytes
|
||||
* (the rule applied to store users, client-file users and the module list). */
|
||||
bool credentials_username_valid(const char* user);
|
||||
|
||||
/* Standard base64. encode writes NUL-terminated output to out (size out_sz).
|
||||
* decode writes the raw bytes to out (capacity out_sz) and stores the length;
|
||||
* the input must be a well-formed padded base64 string. Both return false on
|
||||
* NULL arguments, a bad character/length, or insufficient output space. */
|
||||
bool credentials_b64_encode(const uint8_t* in, size_t n, char* out, size_t out_sz);
|
||||
bool credentials_b64_decode(const char* in, uint8_t* out, size_t out_sz, size_t* out_len);
|
||||
|
||||
/* Fill out[0..n) from the CSPRNG (RAND_bytes). Returns false on failure. */
|
||||
bool credentials_random_bytes(uint8_t* out, size_t n);
|
||||
|
||||
/* Resolve `user` against the store AND the module's auth-user list. The list
|
||||
* scan is a constant-time full-length comparison with no early break. On a
|
||||
* miss, *out is filled with a dummy verifier (a deterministic per-username salt
|
||||
* derived from the store's dummy key, the store-wide uniform iteration count,
|
||||
* fixed dummy keys, found=false). Returns false on invalid arguments or an
|
||||
* HMAC/crypto primitive failure. */
|
||||
bool credentials_get_verifier(const CredentialStore* store, const char* user,
|
||||
const char* const* module_users, int n, CredentialVerifier* out);
|
||||
|
||||
/* Derive the SCRAM keys from a plaintext password:
|
||||
* K = PBKDF2-HMAC-SHA256(password, salt, iters, 32)
|
||||
* ClientKey = HMAC-SHA256(K, "Client Key"); StoredKey = SHA256(ClientKey)
|
||||
* ServerKey = HMAC-SHA256(K, "Server Key")
|
||||
* Any of client_key/stored_key/server_key may be NULL when not needed.
|
||||
* `iters` must lie in [CREDENTIAL_MIN_ITERS, CREDENTIAL_MAX_ITERS]. */
|
||||
bool credentials_compute_keys(const char* password, const uint8_t salt[CREDENTIAL_SALT_LEN],
|
||||
uint32_t iters, uint8_t client_key[CREDENTIAL_KEY_LEN],
|
||||
uint8_t stored_key[CREDENTIAL_KEY_LEN],
|
||||
uint8_t server_key[CREDENTIAL_KEY_LEN]);
|
||||
|
||||
/* Serialize the shared AuthMessage:
|
||||
* "FastSync-Auth-v1" || be32(len(user)) || user
|
||||
* || be32(32) || server_nonce
|
||||
* || be32(32) || client_nonce
|
||||
* out must hold at least CREDENTIAL_AUTH_MESSAGE_MAX bytes. *out_len receives
|
||||
* the number of bytes written. */
|
||||
bool credentials_build_auth_message(const char* user, const uint8_t* snonce, const uint8_t* cnonce,
|
||||
uint8_t* out, size_t out_sz, size_t* out_len);
|
||||
|
||||
/* Client side: ClientProof = ClientKey XOR HMAC(StoredKey, AuthMessage), and
|
||||
* the expected ServerSignature = HMAC(ServerKey, AuthMessage). */
|
||||
bool credentials_client_proof(const uint8_t client_key[CREDENTIAL_KEY_LEN],
|
||||
const uint8_t stored_key[CREDENTIAL_KEY_LEN],
|
||||
const uint8_t server_key[CREDENTIAL_KEY_LEN], const uint8_t* auth_msg,
|
||||
size_t msg_len, uint8_t proof[CREDENTIAL_KEY_LEN],
|
||||
uint8_t server_sig[CREDENTIAL_KEY_LEN]);
|
||||
|
||||
/* Server side: recompute ClientSig' = HMAC(StoredKey, AuthMessage) and
|
||||
* ClientKey' = proof XOR ClientSig', then accept iff v->found AND
|
||||
* SHA256(ClientKey') equals StoredKey (constant-time over the 32-byte keys).
|
||||
* Always computes server_sig_out = HMAC(ServerKey, AuthMessage). Returns the
|
||||
* accept decision. */
|
||||
bool credentials_verify_response(const CredentialVerifier* v, const char* user,
|
||||
const uint8_t* snonce, const uint8_t* cnonce,
|
||||
const uint8_t proof[CREDENTIAL_KEY_LEN],
|
||||
uint8_t server_sig_out[CREDENTIAL_KEY_LEN]);
|
||||
|
||||
/* Derive a new-format store line for `user`/`password` and write it (without a
|
||||
* trailing newline) into out. A random 16-byte salt is used. On failure err is
|
||||
* filled. Used by --hash-credentials and by tests. */
|
||||
bool credentials_hash_store_line(const char* user, const char* password, uint32_t iters, char* out,
|
||||
size_t out_sz, char* err, size_t err_size);
|
||||
|
||||
/* Read `user:password` lines from `path` (the same no-group/other-bits check as
|
||||
* the other secret files) and write one new-format store line per entry to
|
||||
* `out`.
|
||||
* Blank/comment lines are skipped; a malformed line fails the whole run.
|
||||
* Returns 0 on success, -1 on error (err filled). Used by
|
||||
* `--hash-credentials`. */
|
||||
int credentials_hash_file(const char* path, uint32_t iters, FILE* out, char* err, size_t err_size);
|
||||
|
||||
/* Read the CLIENT-side secret file: the first meaningful line is
|
||||
* `user:password` (the literal password). *user_out and *password_out are
|
||||
* freshly allocated on success (password is plaintext -- the caller derives the
|
||||
* proof and then burns/frees it); both are NULL on error. Returns 0 on
|
||||
* success, -1 on failure (err filled: the path is named, never the credential
|
||||
* itself). Only the line's trailing CR/LF are stripped: the password's bytes
|
||||
* are otherwise preserved exactly, so a password with leading/trailing
|
||||
* whitespace (after the ':') is kept usable. The username is trimmed of
|
||||
* surrounding space/tabs. */
|
||||
int credentials_read_secret_file(const char* path, char** user_out, char** password_out, char* err,
|
||||
size_t err_size);
|
||||
|
||||
/* Constant-time equality over exactly len bytes. */
|
||||
bool credentials_secure_equal(const char* a, const char* b, size_t len);
|
||||
|
||||
/* Overwrite secret[0..len) with zeros (best-effort wipe). */
|
||||
void credentials_burn(char* secret, size_t len);
|
||||
|
||||
/* Number of entries currently in the store (tests/introspection). */
|
||||
int credentials_store_size(const CredentialStore* store);
|
||||
|
||||
/* Whether the store contains an entry for `user` (tests/introspection). */
|
||||
bool credentials_store_has(const CredentialStore* store, const char* user);
|
||||
|
||||
#endif
|
||||
@@ -1,454 +0,0 @@
|
||||
#include "daemon_conf.h"
|
||||
#include "utils.h"
|
||||
#include <ctype.h>
|
||||
#include <errno.h>
|
||||
#include <stdarg.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <strings.h>
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* helpers */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
static void set_error(char* err, size_t err_size, const char* fmt, ...) {
|
||||
if (!err || err_size == 0)
|
||||
return;
|
||||
va_list args;
|
||||
va_start(args, fmt);
|
||||
vsnprintf(err, err_size, fmt, args);
|
||||
va_end(args);
|
||||
}
|
||||
|
||||
/* Trim leading and trailing ASCII space/tab in place; returns the new start. */
|
||||
static char* trim_ws(char* s) {
|
||||
while (*s == ' ' || *s == '\t')
|
||||
s++;
|
||||
size_t len = strlen(s);
|
||||
while (len > 0 && (s[len - 1] == ' ' || s[len - 1] == '\t'))
|
||||
s[--len] = '\0';
|
||||
return s;
|
||||
}
|
||||
|
||||
/* Case-insensitive equality of a parsed key against a canonical key name. */
|
||||
static bool key_equals(const char* key, const char* canonical) {
|
||||
return strcasecmp(key, canonical) == 0;
|
||||
}
|
||||
|
||||
static bool parse_bool_value(const char* value, bool* out) {
|
||||
if (strcasecmp(value, "yes") == 0 || strcasecmp(value, "true") == 0 || strcmp(value, "1") == 0) {
|
||||
*out = true;
|
||||
return true;
|
||||
}
|
||||
if (strcasecmp(value, "no") == 0 || strcasecmp(value, "false") == 0 || strcmp(value, "0") == 0) {
|
||||
*out = false;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool daemon_module_name_valid(const char* name) {
|
||||
if (!name || *name == '\0')
|
||||
return false;
|
||||
size_t len = strlen(name);
|
||||
if (len > DAEMON_MAX_MODULE_NAME)
|
||||
return false;
|
||||
for (size_t i = 0; i < len; i++) {
|
||||
unsigned char c = (unsigned char)name[i];
|
||||
bool alnum = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9');
|
||||
if (!alnum && c != '.' && c != '_' && c != '-')
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
DaemonConf* daemon_conf_create(void) {
|
||||
DaemonConf* conf = calloc(1, sizeof(DaemonConf));
|
||||
if (!conf)
|
||||
return NULL;
|
||||
conf->global.port = DAEMON_CONF_DEFAULT_PORT;
|
||||
return conf;
|
||||
}
|
||||
|
||||
void daemon_conf_free(DaemonConf* conf) {
|
||||
if (!conf)
|
||||
return;
|
||||
free(conf->global.motd_file);
|
||||
free(conf->global.address);
|
||||
for (int i = 0; i < conf->module_count; i++) {
|
||||
DaemonModule* m = &conf->modules[i];
|
||||
free(m->name);
|
||||
free(m->path);
|
||||
for (int j = 0; j < m->auth_user_count; j++)
|
||||
free(m->auth_users[j]);
|
||||
free(m->auth_users);
|
||||
}
|
||||
free(conf->modules);
|
||||
free(conf);
|
||||
}
|
||||
|
||||
const DaemonModule* daemon_conf_find_module(const DaemonConf* conf, const char* name) {
|
||||
if (!conf || !name)
|
||||
return NULL;
|
||||
for (int i = 0; i < conf->module_count; i++) {
|
||||
if (strcmp(conf->modules[i].name, name) == 0)
|
||||
return &conf->modules[i];
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* Replace *slot with a str_dup of value; returns false on allocation failure. */
|
||||
static bool store_string(char** slot, const char* value) {
|
||||
char* dup = str_dup(value);
|
||||
if (!dup)
|
||||
return false;
|
||||
free(*slot);
|
||||
*slot = dup;
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool store_port(int* slot, const char* value, char* err, size_t err_size) {
|
||||
char* end;
|
||||
errno = 0;
|
||||
long p = strtol(value, &end, 10);
|
||||
if (errno != 0 || *end != '\0' || *value == '\0' || p <= 0 || p > 65535) {
|
||||
set_error(err, err_size, "invalid port '%s' (must be 1-65535)", value);
|
||||
return false;
|
||||
}
|
||||
*slot = (int)p;
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Apply a global scalar key/value. Keys are case-insensitive. Returns false
|
||||
* (err filled) on an unknown key or an invalid value. */
|
||||
static bool apply_global_key(DaemonConf* conf, char* key, const char* value, char* err,
|
||||
size_t err_size) {
|
||||
if (key_equals(key, "port"))
|
||||
return store_port(&conf->global.port, value, err, err_size);
|
||||
if (key_equals(key, "motd file")) {
|
||||
if (!store_string(&conf->global.motd_file, value)) {
|
||||
set_error(err, err_size, "out of memory parsing 'motd file'");
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
if (key_equals(key, "address")) {
|
||||
if (!store_string(&conf->global.address, value)) {
|
||||
set_error(err, err_size, "out of memory parsing 'address'");
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
set_error(err, err_size, "unknown global key '%s'", key);
|
||||
return false;
|
||||
}
|
||||
|
||||
/* Apply a module key/value to the currently-open module. Returns false (err
|
||||
* filled) on an unknown module key or an invalid value. */
|
||||
static bool apply_module_key(DaemonModule* module, char* key, char* value, char* err,
|
||||
size_t err_size) {
|
||||
if (key_equals(key, "path")) {
|
||||
if (*value == '\0') {
|
||||
set_error(err, err_size, "module '%s': 'path' must not be empty", module->name);
|
||||
return false;
|
||||
}
|
||||
if (!store_string(&module->path, value)) {
|
||||
set_error(err, err_size, "out of memory parsing 'path' for module '%s'", module->name);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
if (key_equals(key, "read only")) {
|
||||
bool parsed;
|
||||
if (!parse_bool_value(value, &parsed)) {
|
||||
set_error(err, err_size,
|
||||
"module '%s': 'read only' must be yes/no (or true/false/1/0), got '%s'",
|
||||
module->name, value);
|
||||
return false;
|
||||
}
|
||||
module->read_only = parsed;
|
||||
return true;
|
||||
}
|
||||
if (key_equals(key, "client owner")) {
|
||||
bool parsed;
|
||||
if (!parse_bool_value(value, &parsed)) {
|
||||
set_error(err, err_size,
|
||||
"module '%s': 'client owner' must be yes/no (or true/false/1/0), got '%s'",
|
||||
module->name, value);
|
||||
return false;
|
||||
}
|
||||
module->client_owner = parsed;
|
||||
return true;
|
||||
}
|
||||
if (key_equals(key, "auth users")) {
|
||||
char* list = str_dup(value);
|
||||
if (!list) {
|
||||
set_error(err, err_size, "out of memory parsing 'auth users' for module '%s'", module->name);
|
||||
return false;
|
||||
}
|
||||
char* save = NULL;
|
||||
for (char* token = strtok_r(list, ",", &save); token; token = strtok_r(NULL, ",", &save)) {
|
||||
const char* user = trim_ws(token);
|
||||
if (*user == '\0')
|
||||
continue;
|
||||
char** grown =
|
||||
realloc(module->auth_users, (size_t)(module->auth_user_count + 1) * sizeof(char*));
|
||||
if (!grown) {
|
||||
free(list);
|
||||
set_error(err, err_size, "out of memory parsing 'auth users' for module '%s'",
|
||||
module->name);
|
||||
return false;
|
||||
}
|
||||
module->auth_users = grown;
|
||||
char* dup = str_dup(user);
|
||||
if (!dup) {
|
||||
free(list);
|
||||
set_error(err, err_size, "out of memory parsing 'auth users' for module '%s'",
|
||||
module->name);
|
||||
return false;
|
||||
}
|
||||
module->auth_users[module->auth_user_count++] = dup;
|
||||
}
|
||||
free(list);
|
||||
return true;
|
||||
}
|
||||
set_error(err, err_size, "unknown key '%s' in module '%s'", key, module->name);
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool module_open_valid(const DaemonModule* module, char* err, size_t err_size) {
|
||||
if (module->path == NULL) {
|
||||
set_error(err, err_size, "module '%s' has no 'path'", module->name);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Validate a [section] header line body (text between the brackets) and set
|
||||
* *name to the module name. Returns false on a malformed header. */
|
||||
static bool parse_section_name(char* body, const char** name_out, char* err, size_t err_size) {
|
||||
char* name = trim_ws(body);
|
||||
if (!daemon_module_name_valid(name)) {
|
||||
set_error(err, err_size, "invalid module name '%s' (must be 1-%d chars of [A-Za-z0-9._-])",
|
||||
name, DAEMON_MAX_MODULE_NAME);
|
||||
return false;
|
||||
}
|
||||
*name_out = name;
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Open (or switch to) a module section. Closes any previously open module
|
||||
* (validating it has a path) and appends the new one. */
|
||||
static int open_module(DaemonConf* conf, int* current_module, const char* name, char* err,
|
||||
size_t err_size) {
|
||||
if (*current_module >= 0) {
|
||||
if (!module_open_valid(&conf->modules[*current_module], err, err_size))
|
||||
return -1;
|
||||
}
|
||||
if (daemon_conf_find_module(conf, name)) {
|
||||
set_error(err, err_size, "duplicate module '%s'", name);
|
||||
return -1;
|
||||
}
|
||||
DaemonModule* grown =
|
||||
realloc(conf->modules, (size_t)(conf->module_count + 1) * sizeof(DaemonModule));
|
||||
if (!grown) {
|
||||
set_error(err, err_size, "out of memory adding module '%s'", name);
|
||||
return -1;
|
||||
}
|
||||
conf->modules = grown;
|
||||
memset(&conf->modules[conf->module_count], 0, sizeof(DaemonModule));
|
||||
conf->modules[conf->module_count].name = str_dup(name);
|
||||
if (!conf->modules[conf->module_count].name) {
|
||||
set_error(err, err_size, "out of memory adding module '%s'", name);
|
||||
return -1;
|
||||
}
|
||||
conf->module_count++;
|
||||
*current_module = conf->module_count - 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Split a "key = value" line (value pointer returned in *value, pointing into
|
||||
* line). Returns false when there is no '='. */
|
||||
static bool split_key_value(char* line, char** key, char** value) {
|
||||
char* eq = strchr(line, '=');
|
||||
if (!eq)
|
||||
return false;
|
||||
*eq = '\0';
|
||||
*key = trim_ws(line);
|
||||
*value = trim_ws(eq + 1);
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Strip one layer of surrounding double quotes from a trimmed value. A value
|
||||
* that starts with '"' but does not end with '"' is an error. */
|
||||
static bool unquote_value(char* value, char* err, size_t err_size) {
|
||||
size_t len = strlen(value);
|
||||
if (len == 0 || value[0] != '"')
|
||||
return true;
|
||||
if (len < 2 || value[len - 1] != '"') {
|
||||
set_error(err, err_size, "unterminated quoted value");
|
||||
return false;
|
||||
}
|
||||
memmove(value, value + 1, len - 2);
|
||||
value[len - 2] = '\0';
|
||||
return true;
|
||||
}
|
||||
|
||||
DaemonConf* daemon_conf_load(const char* path, char* err, size_t err_size) {
|
||||
if (err && err_size)
|
||||
err[0] = '\0';
|
||||
if (!path) {
|
||||
set_error(err, err_size, "no daemon config path");
|
||||
return NULL;
|
||||
}
|
||||
FILE* fp = fopen(path, "r");
|
||||
if (!fp) {
|
||||
set_error(err, err_size, "cannot open daemon config '%s': %s", path, strerror(errno));
|
||||
return NULL;
|
||||
}
|
||||
|
||||
DaemonConf* conf = daemon_conf_create();
|
||||
if (!conf) {
|
||||
fclose(fp);
|
||||
set_error(err, err_size, "out of memory allocating daemon config");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
int current_module = -1;
|
||||
int line_no = 0;
|
||||
char line[DAEMON_CONF_MAX_LINE + 2];
|
||||
bool ok = true;
|
||||
|
||||
while (ok && fgets(line, sizeof(line), fp)) {
|
||||
line_no++;
|
||||
size_t len = strlen(line);
|
||||
if (len == DAEMON_CONF_MAX_LINE + 1 && line[len - 1] != '\n') {
|
||||
/* The read stopped at the buffer edge without a newline and there is
|
||||
* more file to come: the line exceeds the bound. */
|
||||
if (!feof(fp)) {
|
||||
set_error(err, err_size, "line %d exceeds the %d-byte limit", line_no,
|
||||
DAEMON_CONF_MAX_LINE);
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (len > 0 && line[len - 1] == '\n')
|
||||
line[--len] = '\0';
|
||||
if (len > 0 && line[len - 1] == '\r')
|
||||
line[--len] = '\0';
|
||||
|
||||
char* cursor = line;
|
||||
while (*cursor == ' ' || *cursor == '\t')
|
||||
cursor++;
|
||||
if (*cursor == '\0' || *cursor == '#' || *cursor == ';')
|
||||
continue; /* blank or comment line */
|
||||
|
||||
if (*cursor == '[') {
|
||||
char* close = strchr(cursor, ']');
|
||||
if (!close) {
|
||||
set_error(err, err_size, "line %d: unterminated module header", line_no);
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
*close = '\0';
|
||||
char* trailing = close + 1;
|
||||
const char* rest = trim_ws(trailing);
|
||||
if (*rest != '\0') {
|
||||
set_error(err, err_size, "line %d: unexpected text after module header", line_no);
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
const char* name = NULL;
|
||||
if (!parse_section_name(cursor + 1, &name, err, err_size)) {
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
if (open_module(conf, ¤t_module, name, err, err_size) != 0) {
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
char* key;
|
||||
char* value;
|
||||
if (!split_key_value(cursor, &key, &value)) {
|
||||
set_error(err, err_size, "line %d: expected 'key = value'", line_no);
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
if (*key == '\0') {
|
||||
set_error(err, err_size, "line %d: empty key", line_no);
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
if (!unquote_value(value, err, err_size)) {
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
if (current_module >= 0) {
|
||||
if (!apply_module_key(&conf->modules[current_module], key, value, err, err_size)) {
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
if (!apply_global_key(conf, key, value, err, err_size)) {
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (ok && ferror(fp)) {
|
||||
set_error(err, err_size, "error reading daemon config '%s': %s", path, strerror(errno));
|
||||
ok = false;
|
||||
}
|
||||
fclose(fp);
|
||||
|
||||
if (ok && current_module >= 0 &&
|
||||
!module_open_valid(&conf->modules[current_module], err, err_size)) {
|
||||
ok = false;
|
||||
}
|
||||
if (!ok) {
|
||||
daemon_conf_free(conf);
|
||||
return NULL;
|
||||
}
|
||||
return conf;
|
||||
}
|
||||
|
||||
int daemon_conf_apply_dparam(DaemonConf* conf, const char* assignment, char* err, size_t err_size) {
|
||||
if (err && err_size)
|
||||
err[0] = '\0';
|
||||
if (!conf || !assignment || *assignment == '\0') {
|
||||
set_error(err, err_size, "--dparam requires a KEY=VALUE override");
|
||||
return -1;
|
||||
}
|
||||
char* copy = str_dup(assignment);
|
||||
if (!copy) {
|
||||
set_error(err, err_size, "out of memory parsing --dparam");
|
||||
return -1;
|
||||
}
|
||||
char* eq = strchr(copy, '=');
|
||||
if (!eq) {
|
||||
free(copy);
|
||||
set_error(err, err_size, "--dparam '%s' has no '=' (expected KEY=VALUE)", assignment);
|
||||
return -1;
|
||||
}
|
||||
*eq = '\0';
|
||||
char* key = trim_ws(copy);
|
||||
const char* value = trim_ws(eq + 1);
|
||||
if (*key == '\0') {
|
||||
free(copy);
|
||||
set_error(err, err_size, "--dparam '%s' has an empty key", assignment);
|
||||
return -1;
|
||||
}
|
||||
if (*value == '\0') {
|
||||
free(copy);
|
||||
set_error(err, err_size, "--dparam '%s' has an empty value", assignment);
|
||||
return -1;
|
||||
}
|
||||
bool ok = apply_global_key(conf, key, value, err, err_size);
|
||||
free(copy);
|
||||
return ok ? 0 : -1;
|
||||
}
|
||||
@@ -1,107 +0,0 @@
|
||||
#ifndef DAEMON_CONF_H
|
||||
#define DAEMON_CONF_H
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <stddef.h>
|
||||
|
||||
/* FastSync-native daemon configuration (a FastSync analog of rsyncd.conf).
|
||||
*
|
||||
* This is the config the fastsync-server --daemon listener consumes. It is
|
||||
* line-based with an implicit global section followed by zero or more
|
||||
* [module] sections. The full grammar is documented in RSYNC_COMPAT.md
|
||||
* ("Daemon Mode") and summarized below; the parser lives entirely in
|
||||
* daemon_conf.c so it can be unit tested without any socket code.
|
||||
*
|
||||
* The parser is STRICT: an unknown key, a malformed line, a value that does
|
||||
* not parse, a module without a `path`, or a line longer than
|
||||
* DAEMON_CONF_MAX_LINE all fail the whole load with a clear, line-numbered
|
||||
* error instead of being silently ignored. This keeps a typo from silently
|
||||
* changing what a module serves.
|
||||
*/
|
||||
|
||||
/* A daemon module's configured root is used exactly like the standalone
|
||||
* server's --destination-root: the daemon confines every connection that
|
||||
* selects this module to this path (file_open_secure_parent /
|
||||
* has_path_traversal / path_is_within all keep the existing confinement, just
|
||||
* per-module). There is never any client-chosen root: a module path always
|
||||
* stays confined. A daemon REFUSES every client-chosen ownership / super-user
|
||||
* request by default -- --numeric-ids, --chown, --usermap/--groupmap,
|
||||
* --fake-super, --copy-as and an explicit --super -- because there is no
|
||||
* per-module opt-in unless the operator adds one. An operator opts a single
|
||||
* module in with `client owner = yes` (DaemonModule.client_owner), which allows
|
||||
* that client to choose ownership within that module's root (the standalone/SSH
|
||||
* server honors such requests for its single operator-authorized root). The
|
||||
* operator-level --no-super veto additionally forces super-user activities off
|
||||
* for every daemon connection, even an opted-in module. See server_module_gate
|
||||
* in server.c and RSYNC_COMPAT.md.
|
||||
*
|
||||
* `auth_users` is honored by Wave B daemon authentication: a module that
|
||||
* declares auth users accepts a connection only when the presented username is
|
||||
* on this list AND verifies against the daemon's credential store
|
||||
* (--password-file / --early-input). An auth-required module with no usable
|
||||
* store refuses (fail closed) rather than falling open; see server.c. Auth is
|
||||
* never bypassed by ignoring the list. */
|
||||
typedef struct DaemonModule {
|
||||
char* name; /* module name, as the client requests it */
|
||||
char* path; /* module root (daemon-side authorized root) */
|
||||
bool read_only; /* `read only = yes/no`; default no */
|
||||
bool client_owner; /* `client owner = yes/no`; default no. Per-module opt-in
|
||||
that lets this module's clients choose ownership
|
||||
(--numeric-ids/--chown/--usermap/--groupmap/--fake-super/
|
||||
--copy-as) and request explicit --super super-user
|
||||
activities. Without it the daemon refuses all of them. */
|
||||
char** auth_users; /* `auth users = a,b`; Wave B credential list */
|
||||
int auth_user_count;
|
||||
} DaemonModule;
|
||||
|
||||
/* Global (pre-module) scalar keys. `motd file` is parsed and stored but has
|
||||
* no wire effect yet (MOTD display is Wave C). */
|
||||
typedef struct DaemonConfGlobals {
|
||||
int port; /* `port`, default DAEMON_CONF_DEFAULT_PORT (873) */
|
||||
char* motd_file; /* `motd file`, may be NULL */
|
||||
char* address; /* `address` (optional bind address), may be NULL */
|
||||
} DaemonConfGlobals;
|
||||
|
||||
typedef struct DaemonConf {
|
||||
DaemonConfGlobals global;
|
||||
DaemonModule* modules;
|
||||
int module_count;
|
||||
} DaemonConf;
|
||||
|
||||
#define DAEMON_CONF_DEFAULT_PORT 873
|
||||
/* Longest accepted config line (excluding the trailing newline). Longer lines
|
||||
* are rejected rather than buffered unboundedly. */
|
||||
#define DAEMON_CONF_MAX_LINE 4096
|
||||
/* Upper bound on a module name. Kept far below MAX_STRING_SIZE so a wire
|
||||
* module name can never exhaust anything by being long. */
|
||||
#define DAEMON_MAX_MODULE_NAME 200
|
||||
|
||||
/* Allocate an empty daemon config with defaulted globals (port 873, no
|
||||
* modules, no motd/address). Never fails for an allocation failure; callers
|
||||
* must still NULL-check. */
|
||||
DaemonConf* daemon_conf_create(void);
|
||||
|
||||
/* Parse `path` into a freshly allocated DaemonConf. Returns NULL on any error
|
||||
* and fills `err` (err_size bytes) with a clear, line-numbered message. The
|
||||
* returned object is heap-owned; free it with daemon_conf_free. */
|
||||
DaemonConf* daemon_conf_load(const char* path, char* err, size_t err_size);
|
||||
|
||||
void daemon_conf_free(DaemonConf* conf);
|
||||
|
||||
/* Case-sensitive exact module lookup by name. Returns the module or NULL.
|
||||
* Module names are matched exactly (rsync semantics). */
|
||||
const DaemonModule* daemon_conf_find_module(const DaemonConf* conf, const char* name);
|
||||
|
||||
/* Module-name syntax check: non-empty, at most DAEMON_MAX_MODULE_NAME chars,
|
||||
* and only [A-Za-z0-9._-]. Used by the config parser, the client's
|
||||
* host::module/path destination parser, and (implicitly) by the daemon lookup
|
||||
* (a name that fails this can never match a parsed module). */
|
||||
bool daemon_module_name_valid(const char* name);
|
||||
|
||||
/* Parse one --dparam=KEY=VALUE (or "--dparam KEY=VALUE") override string and
|
||||
* apply it to the global scalars only. Keys are case-insensitive and limited
|
||||
* to the global scalar keys defined by the grammar (port, motd file, address).
|
||||
* Returns 0 on success, -1 on error (err filled). */
|
||||
int daemon_conf_apply_dparam(DaemonConf* conf, const char* assignment, char* err, size_t err_size);
|
||||
|
||||
#endif
|
||||
+9
-17
@@ -1,12 +1,9 @@
|
||||
#include "data.h"
|
||||
#include "log.h"
|
||||
#include "protocol.h"
|
||||
#include <stdlib.h>
|
||||
#include "stdlib.h"
|
||||
|
||||
Data* data_create_empty(size_t data_size) {
|
||||
/* malloc(0) is UB; allocate at least 1 byte but preserve requested size */
|
||||
size_t alloc_size = data_size > 0 ? data_size : 1;
|
||||
void* data = protocol_alloc(alloc_size);
|
||||
Data *data_create_empty(size_t data_size) {
|
||||
void *data = malloc(data_size);
|
||||
if (data == NULL) {
|
||||
log_message(LOG_LEVEL_ERROR, "Could not allocate memory for empty data");
|
||||
return NULL;
|
||||
@@ -14,20 +11,19 @@ Data* data_create_empty(size_t data_size) {
|
||||
return data_create(data, data_size);
|
||||
}
|
||||
|
||||
Data* data_create_reserve(size_t size) {
|
||||
Data* d = protocol_alloc(sizeof(Data));
|
||||
Data *data_create_reserve(size_t size) {
|
||||
Data *d = malloc(sizeof(Data));
|
||||
if (d == NULL) {
|
||||
log_message(LOG_LEVEL_ERROR, "Could not allocate memory for data");
|
||||
return NULL;
|
||||
}
|
||||
d->data = NULL;
|
||||
d->size = size;
|
||||
d->protocol_charge = 0;
|
||||
return d;
|
||||
}
|
||||
|
||||
Data* data_create(void* data, size_t data_size) {
|
||||
Data* new_data = protocol_alloc(sizeof(Data));
|
||||
Data *data_create(void *data, size_t data_size) {
|
||||
Data *new_data = malloc(sizeof(Data));
|
||||
if (new_data == NULL) {
|
||||
log_message(LOG_LEVEL_ERROR, "Could not allocate memory for data");
|
||||
free(data);
|
||||
@@ -35,15 +31,11 @@ Data* data_create(void* data, size_t data_size) {
|
||||
}
|
||||
new_data->data = data;
|
||||
new_data->size = data_size;
|
||||
new_data->protocol_charge = 0;
|
||||
return new_data;
|
||||
}
|
||||
|
||||
void data_destroy(Data* data) {
|
||||
if (data == NULL)
|
||||
return;
|
||||
if (data->protocol_charge != 0)
|
||||
protocol_release_memory(data->protocol_charge);
|
||||
void data_destroy(Data *data) {
|
||||
if (data == NULL) return;
|
||||
free(data->data);
|
||||
free(data);
|
||||
}
|
||||
|
||||
+6
-9
@@ -1,19 +1,16 @@
|
||||
#ifndef DATA_H
|
||||
#define DATA_H
|
||||
|
||||
#include <stdlib.h>
|
||||
#include "stdlib.h"
|
||||
|
||||
typedef struct {
|
||||
void* data;
|
||||
void *data;
|
||||
size_t size;
|
||||
/* Non-zero only for a buffer charged to the protocol connection budget. */
|
||||
size_t protocol_charge;
|
||||
} Data;
|
||||
|
||||
Data* data_create_empty(size_t data_size);
|
||||
Data* data_create_reserve(size_t size);
|
||||
Data* data_create(void* data, size_t data_size);
|
||||
void data_destroy(Data* data);
|
||||
void protocol_release_memory(size_t charge);
|
||||
Data *data_create_empty(size_t data_size);
|
||||
Data *data_create_reserve(size_t size);
|
||||
Data *data_create(void *data, size_t data_size);
|
||||
void data_destroy(Data *data);
|
||||
|
||||
#endif
|
||||
|
||||
@@ -1,338 +0,0 @@
|
||||
#include "delay_updates.h"
|
||||
|
||||
#include "config.h"
|
||||
#include "file.h"
|
||||
#include "log.h"
|
||||
#include "utils.h"
|
||||
#include <dirent.h>
|
||||
#include <errno.h>
|
||||
#include <fcntl.h>
|
||||
#include <libgen.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/file.h>
|
||||
#include <sys/stat.h>
|
||||
#include <unistd.h>
|
||||
|
||||
DelayUpdatesContext* delay_updates_context_create(const char* root_directory) {
|
||||
if (!root_directory)
|
||||
return NULL;
|
||||
DelayUpdatesContext* context = calloc(1, sizeof(DelayUpdatesContext));
|
||||
if (!context)
|
||||
return NULL;
|
||||
context->root_directory = str_dup(root_directory);
|
||||
if (!context->root_directory) {
|
||||
free(context);
|
||||
return NULL;
|
||||
}
|
||||
context->staging_root = path_cat(root_directory, DELAY_UPDATES_STAGING_DIR);
|
||||
if (!context->staging_root) {
|
||||
free(context->root_directory);
|
||||
free(context);
|
||||
return NULL;
|
||||
}
|
||||
context->entries = NULL;
|
||||
context->count = 0;
|
||||
context->capacity = 0;
|
||||
context->prepared = false;
|
||||
context->lock_fd = -1;
|
||||
if (mtx_init(&context->mutex, mtx_plain) != thrd_success) {
|
||||
free(context->staging_root);
|
||||
free(context->root_directory);
|
||||
free(context);
|
||||
return NULL;
|
||||
}
|
||||
return context;
|
||||
}
|
||||
|
||||
void delay_updates_context_destroy(DelayUpdatesContext* context) {
|
||||
if (!context)
|
||||
return;
|
||||
mtx_destroy(&context->mutex);
|
||||
if (context->lock_fd >= 0)
|
||||
close(context->lock_fd);
|
||||
context->lock_fd = -1;
|
||||
free(context->staging_root);
|
||||
free(context->root_directory);
|
||||
for (size_t i = 0; i < context->count; i++) {
|
||||
free(context->entries[i].staged_path);
|
||||
free(context->entries[i].final_path);
|
||||
free(context->entries[i].file_path);
|
||||
}
|
||||
free(context->entries);
|
||||
free(context);
|
||||
}
|
||||
|
||||
bool delay_updates_staging_name_conflict(const char* dir) {
|
||||
if (!dir || !*dir)
|
||||
return false;
|
||||
size_t length = strlen(dir);
|
||||
while (length > 0 && dir[length - 1] == '/')
|
||||
length--;
|
||||
size_t reserved_length = strlen(DELAY_UPDATES_STAGING_DIR);
|
||||
if (length != reserved_length)
|
||||
return false;
|
||||
return strncmp(dir, DELAY_UPDATES_STAGING_DIR, length) == 0;
|
||||
}
|
||||
|
||||
/* Recursively delete every entry inside an open directory (never following
|
||||
symlinks). The directory itself is left in place. Mirrors the fd-relative
|
||||
walk used by the delete code so a symlink planted inside the staging tree
|
||||
can never redirect removal outside of it. */
|
||||
static bool delay_wipe_dir_fd(int dirfd) {
|
||||
int scanfd = dup(dirfd);
|
||||
if (scanfd < 0)
|
||||
return false;
|
||||
DIR* dir = fdopendir(scanfd);
|
||||
if (!dir) {
|
||||
close(scanfd);
|
||||
return false;
|
||||
}
|
||||
bool operation_ok = true;
|
||||
const struct dirent* entry;
|
||||
while ((entry = readdir(dir)) != NULL) {
|
||||
if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
|
||||
continue;
|
||||
struct stat st;
|
||||
if (fstatat(dirfd, entry->d_name, &st, AT_SYMLINK_NOFOLLOW) != 0) {
|
||||
if (errno != ENOENT)
|
||||
operation_ok = false;
|
||||
continue;
|
||||
}
|
||||
if (S_ISDIR(st.st_mode)) {
|
||||
int childfd = openat(dirfd, entry->d_name, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||
bool child_removed = false;
|
||||
if (childfd >= 0) {
|
||||
child_removed = delay_wipe_dir_fd(childfd);
|
||||
close(childfd);
|
||||
} else if (errno != ENOENT) {
|
||||
operation_ok = false;
|
||||
}
|
||||
if (child_removed && unlinkat(dirfd, entry->d_name, AT_REMOVEDIR) != 0 && errno != ENOENT)
|
||||
operation_ok = false;
|
||||
} else {
|
||||
if (unlinkat(dirfd, entry->d_name, 0) != 0 && errno != ENOENT)
|
||||
operation_ok = false;
|
||||
}
|
||||
}
|
||||
closedir(dir);
|
||||
return operation_ok;
|
||||
}
|
||||
|
||||
bool delay_updates_prepare(DelayUpdatesContext* context) {
|
||||
if (!context)
|
||||
return false;
|
||||
if (context->prepared)
|
||||
return true;
|
||||
int fd = file_open_private_dir(context->staging_root);
|
||||
if (fd < 0) {
|
||||
int saved_errno = errno;
|
||||
char* escaped = output_escape(context->staging_root, false);
|
||||
log_message(LOG_LEVEL_ERROR, "could not create --delay-updates staging directory '%s': %s",
|
||||
escaped ? escaped : "<allocation failed>", strerror(saved_errno));
|
||||
free(escaped);
|
||||
return false;
|
||||
}
|
||||
/* Hold an exclusive advisory lock on the staging directory for the whole
|
||||
transfer. The staging directory name is fixed, so two simultaneous
|
||||
delayed transfers to the same destination root would otherwise share it
|
||||
and destroy each other's staged files. The lock makes the second session
|
||||
fail cleanly instead of corrupting the first. The lock is released when
|
||||
the context (and its file descriptor) is destroyed. */
|
||||
if (flock(fd, LOCK_EX | LOCK_NB) != 0) {
|
||||
int saved_errno = errno;
|
||||
close(fd);
|
||||
if (saved_errno == EWOULDBLOCK || saved_errno == EAGAIN) {
|
||||
char* escaped = output_escape(context->staging_root, false);
|
||||
log_message(LOG_LEVEL_ERROR,
|
||||
"another --delay-updates transfer to '%s' is already in progress; refusing to "
|
||||
"share the staging directory",
|
||||
escaped ? escaped : "<allocation failed>");
|
||||
free(escaped);
|
||||
} else {
|
||||
log_message(LOG_LEVEL_ERROR, "could not lock --delay-updates staging directory '%s': %s",
|
||||
context->staging_root, strerror(saved_errno));
|
||||
}
|
||||
return false;
|
||||
}
|
||||
context->lock_fd = fd;
|
||||
/* Only now, with exclusive ownership, wipe leftovers from an interrupted
|
||||
earlier transfer; this can never race with a live session. */
|
||||
bool ok = delay_wipe_dir_fd(fd);
|
||||
if (!ok) {
|
||||
log_message(LOG_LEVEL_ERROR, "could not clear stale --delay-updates staging files under '%s'",
|
||||
context->staging_root);
|
||||
close(context->lock_fd);
|
||||
context->lock_fd = -1;
|
||||
return false;
|
||||
}
|
||||
context->prepared = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool delay_updates_record(DelayUpdatesContext* context, const char* staged_path,
|
||||
const char* final_path, const char* file_path) {
|
||||
if (!context || !staged_path || !final_path || !file_path)
|
||||
return false;
|
||||
char* staged_copy = str_dup(staged_path);
|
||||
char* final_copy = str_dup(final_path);
|
||||
char* file_copy = str_dup(file_path);
|
||||
if (!staged_copy || !final_copy || !file_copy) {
|
||||
free(staged_copy);
|
||||
free(final_copy);
|
||||
free(file_copy);
|
||||
return false;
|
||||
}
|
||||
mtx_lock(&context->mutex);
|
||||
bool ok = true;
|
||||
if (context->count == context->capacity) {
|
||||
size_t new_capacity = context->capacity == 0 ? 64 : context->capacity * 2;
|
||||
if (new_capacity < context->capacity) {
|
||||
ok = false;
|
||||
} else {
|
||||
StagedFileEntry* grown = realloc(context->entries, new_capacity * sizeof(StagedFileEntry));
|
||||
if (!grown) {
|
||||
ok = false;
|
||||
} else {
|
||||
context->entries = grown;
|
||||
context->capacity = new_capacity;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (ok) {
|
||||
context->entries[context->count].staged_path = staged_copy;
|
||||
context->entries[context->count].final_path = final_copy;
|
||||
context->entries[context->count].file_path = file_copy;
|
||||
context->count++;
|
||||
}
|
||||
mtx_unlock(&context->mutex);
|
||||
if (!ok) {
|
||||
free(staged_copy);
|
||||
free(final_copy);
|
||||
free(file_copy);
|
||||
}
|
||||
return ok;
|
||||
}
|
||||
|
||||
/* Move an existing final destination file aside before the staged replacement
|
||||
is installed. Deferred from stage time so the final destination is not
|
||||
modified until publication. Mirrors the immediate-mode backup logic. */
|
||||
static bool delay_publish_backup(const DelayUpdatesContext* context, const Config* config,
|
||||
const StagedFileEntry* entry) {
|
||||
bool backup_enabled = config && config->backup && !config->ignore_existing;
|
||||
if (!backup_enabled)
|
||||
return true;
|
||||
const char* backup_suffix = (config && config->suffix) ? config->suffix : "~";
|
||||
struct stat backup_stat;
|
||||
if (!file_stat_secure(entry->final_path, &backup_stat))
|
||||
return true; /* nothing to back up */
|
||||
|
||||
char* backup_path = NULL;
|
||||
if (config->backup_dir) {
|
||||
char* confined_backup = path_cat(context->root_directory, config->backup_dir);
|
||||
if (!confined_backup)
|
||||
return false;
|
||||
backup_path = path_cat(confined_backup, entry->file_path);
|
||||
free(confined_backup);
|
||||
} else {
|
||||
size_t path_len = strlen(entry->final_path);
|
||||
size_t suffix_len = strlen(backup_suffix);
|
||||
if (path_len > SIZE_MAX - suffix_len - 1)
|
||||
return false;
|
||||
backup_path = malloc(path_len + suffix_len + 1);
|
||||
if (backup_path) {
|
||||
memcpy(backup_path, entry->final_path, path_len);
|
||||
memcpy(backup_path + path_len, backup_suffix, suffix_len + 1);
|
||||
}
|
||||
}
|
||||
if (!backup_path)
|
||||
return false;
|
||||
char* parent_copy = str_dup(backup_path);
|
||||
if (!parent_copy || !file_ensure_directory_secure(dirname(parent_copy))) {
|
||||
free(parent_copy);
|
||||
free(backup_path);
|
||||
return false;
|
||||
}
|
||||
free(parent_copy);
|
||||
bool ok = file_rename_secure(entry->final_path, backup_path);
|
||||
free(backup_path);
|
||||
return ok;
|
||||
}
|
||||
|
||||
static bool delay_publish_entry(DelayUpdatesContext* context, const Config* config,
|
||||
const StagedFileEntry* entry) {
|
||||
if (!delay_publish_backup(context, config, entry))
|
||||
return false;
|
||||
if (!file_rename_secure(entry->staged_path, entry->final_path)) {
|
||||
if (errno == EXDEV) {
|
||||
char* escaped = output_escape(entry->final_path, false);
|
||||
log_message(LOG_LEVEL_ERROR,
|
||||
"staging directory is on a different filesystem than the destination; cannot "
|
||||
"atomically install file (EXDEV): %s",
|
||||
escaped ? escaped : "<allocation failed>");
|
||||
free(escaped);
|
||||
} else {
|
||||
char* escaped = output_escape(entry->final_path, false);
|
||||
log_message(LOG_LEVEL_ERROR, "could not install staged file '%s': %s",
|
||||
escaped ? escaped : "<allocation failed>", strerror(errno));
|
||||
free(escaped);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Remove the staging tree (contents plus the directory itself). Returns true
|
||||
when nothing is left behind (including the case where it never existed). */
|
||||
static bool delay_updates_remove_staging_tree(DelayUpdatesContext* context) {
|
||||
int fd = open(context->staging_root, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||
if (fd < 0)
|
||||
return errno == ENOENT;
|
||||
bool ok = delay_wipe_dir_fd(fd);
|
||||
if (close(fd) != 0)
|
||||
ok = false;
|
||||
if (ok && rmdir(context->staging_root) != 0 && errno != ENOENT)
|
||||
ok = false;
|
||||
return ok;
|
||||
}
|
||||
|
||||
bool delay_updates_publish(DelayUpdatesContext* context, const Config* config) {
|
||||
if (!context)
|
||||
return false;
|
||||
mtx_lock(&context->mutex);
|
||||
bool ok = true;
|
||||
for (size_t i = 0; i < context->count; i++) {
|
||||
if (!delay_publish_entry(context, config, &context->entries[i])) {
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
mtx_unlock(&context->mutex);
|
||||
|
||||
/* Renaming files out of the staging tree leaves the mirrored directories
|
||||
behind, and a mid-publish failure leaves the remaining staged files.
|
||||
Remove whatever is left so a later run starts from a clean staging area
|
||||
and no staged content can linger after a failed publish. If that cleanup
|
||||
fails, tell the operator: a stale staging directory would otherwise
|
||||
silently accumulate and make the next transfer's prepare-wipe fail. */
|
||||
if (!delay_updates_remove_staging_tree(context)) {
|
||||
log_message(LOG_LEVEL_WARNING,
|
||||
"could not fully remove --delay-updates staging directory '%s' after publish; a "
|
||||
"later --delay-updates transfer to this destination will try to clear it",
|
||||
context->staging_root);
|
||||
}
|
||||
return ok;
|
||||
}
|
||||
|
||||
void delay_updates_cleanup(DelayUpdatesContext* context) {
|
||||
if (!context)
|
||||
return;
|
||||
/* Only a context that gained exclusive ownership may touch the shared
|
||||
staging directory. If prepare never succeeded (e.g. lock contention with
|
||||
another live session) the directory belongs to that other session and must
|
||||
be left alone. */
|
||||
if (!context->prepared)
|
||||
return;
|
||||
delay_updates_remove_staging_tree(context);
|
||||
}
|
||||
@@ -1,68 +0,0 @@
|
||||
#ifndef DELAY_UPDATES_H
|
||||
#define DELAY_UPDATES_H
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <stddef.h>
|
||||
#include <threads.h>
|
||||
|
||||
/* Forward-declared in config.h; full type needed by file_save_to_disk. */
|
||||
typedef struct Config Config;
|
||||
|
||||
/* One staged file awaiting publication. */
|
||||
typedef struct {
|
||||
char* staged_path; /* full path inside the staging tree */
|
||||
char* final_path; /* full final destination path */
|
||||
char* file_path; /* the file path as received on the wire */
|
||||
} StagedFileEntry;
|
||||
|
||||
/* Receiver-side --delay-updates staging registry. All successfully written
|
||||
files land under a private staging directory inside the receive root and are
|
||||
atomically renamed into their final destination only at the very end of the
|
||||
transfer. A single PipelineContextReceiver has exactly one writer thread,
|
||||
but the registry is still mutex-protected so the same object can be safely
|
||||
shared with the publish/cleanup phase that runs after the threads join. */
|
||||
typedef struct DelayUpdatesContext {
|
||||
char* root_directory; /* receive root the staging dir lives under */
|
||||
char* staging_root; /* root_directory/<staging dir name> */
|
||||
mtx_t mutex;
|
||||
StagedFileEntry* entries;
|
||||
size_t count;
|
||||
size_t capacity;
|
||||
bool prepared; /* staging dir created, wiped, and exclusively locked */
|
||||
int lock_fd; /* advisory exclusive flock held on the staging dir, or -1 */
|
||||
} DelayUpdatesContext;
|
||||
|
||||
/* Name of the private staging subdirectory created under the receive root. */
|
||||
#define DELAY_UPDATES_STAGING_DIR ".fastsync-stage"
|
||||
|
||||
/* True when `dir` (ignoring a trailing "/") is the reserved staging directory
|
||||
name. Used to reject a --backup-dir that would collide with the internal
|
||||
staging area. */
|
||||
bool delay_updates_staging_name_conflict(const char* dir);
|
||||
|
||||
/* Create an empty staging context rooted below root_directory. Does not touch
|
||||
the filesystem yet. */
|
||||
DelayUpdatesContext* delay_updates_context_create(const char* root_directory);
|
||||
void delay_updates_context_destroy(DelayUpdatesContext* context);
|
||||
|
||||
/* Create the private 0700 staging directory (on first call) and wipe any
|
||||
leftovers from a previously interrupted delayed transfer. Idempotent. */
|
||||
bool delay_updates_prepare(DelayUpdatesContext* context);
|
||||
|
||||
/* Record a fully-written staged file for later publication. Copies all three
|
||||
paths. Returns false on allocation failure. */
|
||||
bool delay_updates_record(DelayUpdatesContext* context, const char* staged_path,
|
||||
const char* final_path, const char* file_path);
|
||||
|
||||
/* Atomically rename every staged file into its final destination. Deferred
|
||||
--backup handling runs immediately before each rename. On any failure the
|
||||
remaining staged files are removed (best effort); already-published files
|
||||
are not rolled back. Afterwards the staging tree is removed so a successful
|
||||
or failed publish leaves no staging leftovers. */
|
||||
bool delay_updates_publish(DelayUpdatesContext* context, const Config* config);
|
||||
|
||||
/* Best-effort removal of every staged file and the staging directory itself.
|
||||
Safe to call when nothing was staged or after a successful publish. */
|
||||
void delay_updates_cleanup(DelayUpdatesContext* context);
|
||||
|
||||
#endif
|
||||
+135
-377
@@ -1,8 +1,5 @@
|
||||
#include "delta.h"
|
||||
#include "log.h"
|
||||
#include "protocol.h"
|
||||
#include <stdint.h>
|
||||
#include <limits.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
@@ -10,12 +7,8 @@
|
||||
#define XXH_IMPLEMENTATION
|
||||
#include <xxhash.h>
|
||||
|
||||
/* Maximum number of blocks/instructions allowed from the wire to prevent OOM */
|
||||
#define MAX_DELTA_BLOCKS (1024U * 1024U) /* 1M signature blocks */
|
||||
#define MAX_DELTA_INSTRUCTIONS (1024U * 1024U) /* 1M delta instructions */
|
||||
|
||||
uint32_t delta_adler32(const void* data, uint32_t len) {
|
||||
const uint8_t* p = (const uint8_t*)data;
|
||||
uint32_t delta_adler32(const void *data, uint32_t len) {
|
||||
const uint8_t *p = (const uint8_t *)data;
|
||||
uint32_t s1 = 1;
|
||||
uint32_t s2 = 0;
|
||||
for (uint32_t i = 0; i < len; i++) {
|
||||
@@ -25,76 +18,51 @@ uint32_t delta_adler32(const void* data, uint32_t len) {
|
||||
return (s2 << 16) | s1;
|
||||
}
|
||||
|
||||
uint32_t delta_xxhash32(const void* data, uint32_t len) {
|
||||
uint32_t delta_xxhash32(const void *data, uint32_t len) {
|
||||
return XXH32(data, len, 0);
|
||||
}
|
||||
|
||||
uint32_t delta_xxhash32_seeded(const void* data, uint32_t len, uint32_t seed) {
|
||||
return XXH32(data, len, seed);
|
||||
}
|
||||
|
||||
uint64_t delta_xxhash64(const void* data, size_t len) {
|
||||
return XXH64(data, len, 0);
|
||||
}
|
||||
|
||||
DeltaSignature* delta_signature_create(const void* old_file_data, uint64_t old_file_size,
|
||||
uint32_t block_size) {
|
||||
return delta_signature_create_seeded(old_file_data, old_file_size, block_size, 0);
|
||||
}
|
||||
|
||||
DeltaSignature* delta_signature_create_seeded(const void* old_file_data, uint64_t old_file_size,
|
||||
uint32_t block_size, uint32_t seed) {
|
||||
DeltaSignature *delta_signature_create(const void *old_file_data,
|
||||
uint64_t old_file_size,
|
||||
uint32_t block_size) {
|
||||
if (old_file_data == NULL || old_file_size == 0 || block_size == 0)
|
||||
return NULL;
|
||||
|
||||
if (old_file_size > DELTA_MAX_FILE_SIZE || block_size > DELTA_BLOCK_SIZE_MAX ||
|
||||
old_file_size > UINT32_MAX * (uint64_t)block_size)
|
||||
return NULL;
|
||||
|
||||
uint32_t block_count = (uint32_t)((old_file_size + block_size - 1) / block_size);
|
||||
|
||||
DeltaSignature* sig = protocol_alloc(sizeof(DeltaSignature));
|
||||
if (!sig)
|
||||
return NULL;
|
||||
DeltaSignature *sig = malloc(sizeof(DeltaSignature));
|
||||
if (!sig) return NULL;
|
||||
|
||||
sig->file_size = old_file_size;
|
||||
sig->block_size = block_size;
|
||||
sig->block_count = block_count;
|
||||
if (block_count == 0) {
|
||||
free(sig);
|
||||
return NULL;
|
||||
}
|
||||
sig->blocks = protocol_alloc((size_t)block_count * sizeof(DeltaBlockSig));
|
||||
sig->blocks = malloc(block_count * sizeof(DeltaBlockSig));
|
||||
if (!sig->blocks) {
|
||||
free(sig);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
const uint8_t* data = (const uint8_t*)old_file_data;
|
||||
const uint8_t *data = (const uint8_t *)old_file_data;
|
||||
for (uint32_t i = 0; i < block_count; i++) {
|
||||
uint64_t offset = (uint64_t)i * block_size;
|
||||
uint32_t len =
|
||||
(uint32_t)((old_file_size - offset < block_size) ? (old_file_size - offset) : block_size);
|
||||
uint32_t len = (uint32_t)((old_file_size - offset < block_size)
|
||||
? (old_file_size - offset)
|
||||
: block_size);
|
||||
sig->blocks[i].adler32 = delta_adler32(data + offset, len);
|
||||
sig->blocks[i].xxhash = delta_xxhash32_seeded(data + offset, len, seed);
|
||||
sig->blocks[i].xxhash = delta_xxhash32(data + offset, len);
|
||||
}
|
||||
|
||||
return sig;
|
||||
}
|
||||
|
||||
Data* delta_signature_serialize(const DeltaSignature* sig) {
|
||||
if (!sig)
|
||||
return NULL;
|
||||
Data *delta_signature_serialize(const DeltaSignature *sig) {
|
||||
if (!sig) return NULL;
|
||||
|
||||
uint64_t block_bytes = (uint64_t)sig->block_count * (sizeof(uint32_t) + sizeof(uint32_t));
|
||||
uint64_t total = sizeof(uint64_t) + sizeof(uint32_t) + sizeof(uint32_t) + block_bytes;
|
||||
if (block_bytes > UINT64_MAX - (sizeof(uint64_t) + sizeof(uint32_t) + sizeof(uint32_t)) ||
|
||||
total > SIZE_MAX)
|
||||
return NULL;
|
||||
uint64_t total = sizeof(uint64_t) + sizeof(uint32_t) + sizeof(uint32_t) +
|
||||
(uint64_t)sig->block_count * (sizeof(uint32_t) + sizeof(uint32_t));
|
||||
|
||||
uint8_t* buf = protocol_alloc((size_t)total);
|
||||
if (!buf)
|
||||
return NULL;
|
||||
uint8_t *buf = malloc((size_t)total);
|
||||
if (!buf) return NULL;
|
||||
|
||||
size_t pos = 0;
|
||||
memcpy(buf + pos, &sig->file_size, sizeof(uint64_t));
|
||||
@@ -114,16 +82,15 @@ Data* delta_signature_serialize(const DeltaSignature* sig) {
|
||||
return data_create(buf, (size_t)total);
|
||||
}
|
||||
|
||||
DeltaSignature* delta_signature_deserialize(const Data* data) {
|
||||
DeltaSignature *delta_signature_deserialize(const Data *data) {
|
||||
if (!data || data->size < sizeof(uint64_t) + sizeof(uint32_t) + sizeof(uint32_t))
|
||||
return NULL;
|
||||
|
||||
const uint8_t* buf = (const uint8_t*)data->data;
|
||||
const uint8_t *buf = (const uint8_t *)data->data;
|
||||
size_t pos = 0;
|
||||
|
||||
DeltaSignature* sig = protocol_alloc(sizeof(DeltaSignature));
|
||||
if (!sig)
|
||||
return NULL;
|
||||
DeltaSignature *sig = malloc(sizeof(DeltaSignature));
|
||||
if (!sig) return NULL;
|
||||
|
||||
memcpy(&sig->file_size, buf + pos, sizeof(uint64_t));
|
||||
pos += sizeof(uint64_t);
|
||||
@@ -132,21 +99,6 @@ DeltaSignature* delta_signature_deserialize(const Data* data) {
|
||||
memcpy(&sig->block_count, buf + pos, sizeof(uint32_t));
|
||||
pos += sizeof(uint32_t);
|
||||
|
||||
// Reject unreasonably large block counts to prevent OOM
|
||||
if (sig->block_count > MAX_DELTA_BLOCKS) {
|
||||
log_message(LOG_LEVEL_ERROR, "Delta signature block count %u exceeds maximum %u",
|
||||
sig->block_count, MAX_DELTA_BLOCKS);
|
||||
free(sig);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (sig->block_size == 0 || sig->block_size > DELTA_BLOCK_SIZE_MAX ||
|
||||
sig->file_size > DELTA_MAX_FILE_SIZE || sig->file_size == 0 ||
|
||||
(sig->file_size + sig->block_size - 1) / sig->block_size != sig->block_count) {
|
||||
free(sig);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
uint64_t expected = sizeof(uint64_t) + sizeof(uint32_t) + sizeof(uint32_t) +
|
||||
(uint64_t)sig->block_count * (sizeof(uint32_t) + sizeof(uint32_t));
|
||||
if (data->size < expected) {
|
||||
@@ -154,12 +106,7 @@ DeltaSignature* delta_signature_deserialize(const Data* data) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
uint64_t blocks_size = (uint64_t)sig->block_count * sizeof(DeltaBlockSig);
|
||||
if (blocks_size > SIZE_MAX) {
|
||||
free(sig);
|
||||
return NULL;
|
||||
}
|
||||
sig->blocks = protocol_alloc((size_t)blocks_size);
|
||||
sig->blocks = malloc(sig->block_count * sizeof(DeltaBlockSig));
|
||||
if (!sig->blocks) {
|
||||
free(sig);
|
||||
return NULL;
|
||||
@@ -175,39 +122,31 @@ DeltaSignature* delta_signature_deserialize(const Data* data) {
|
||||
return sig;
|
||||
}
|
||||
|
||||
void delta_signature_destroy(DeltaSignature* sig) {
|
||||
if (!sig)
|
||||
return;
|
||||
void delta_signature_destroy(DeltaSignature *sig) {
|
||||
if (!sig) return;
|
||||
free(sig->blocks);
|
||||
free(sig);
|
||||
}
|
||||
|
||||
static bool ensure_capacity(DeltaInstruction** instrs, uint32_t* capacity, uint32_t count) {
|
||||
if (count < *capacity)
|
||||
return true;
|
||||
if (*capacity > MAX_DELTA_INSTRUCTIONS / 2)
|
||||
return false;
|
||||
static bool ensure_capacity(DeltaInstruction **instrs, uint32_t *capacity,
|
||||
uint32_t count) {
|
||||
if (count < *capacity) return true;
|
||||
uint32_t new_cap = *capacity * 2;
|
||||
DeltaInstruction* tmp = protocol_realloc(*instrs, (size_t)new_cap * sizeof(DeltaInstruction));
|
||||
if (!tmp)
|
||||
return false;
|
||||
DeltaInstruction *tmp = realloc(*instrs, new_cap * sizeof(DeltaInstruction));
|
||||
if (!tmp) return false;
|
||||
*instrs = tmp;
|
||||
*capacity = new_cap;
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool flush_literal(DeltaInstruction** instrs, uint32_t* capacity, uint32_t* count,
|
||||
const uint8_t* data, uint64_t start, uint64_t end) {
|
||||
if (start >= end)
|
||||
return true;
|
||||
if (end - start > UINT32_MAX || *count >= MAX_DELTA_INSTRUCTIONS)
|
||||
return false;
|
||||
static bool flush_literal(DeltaInstruction **instrs, uint32_t *capacity,
|
||||
uint32_t *count, const uint8_t *data,
|
||||
uint64_t start, uint64_t end) {
|
||||
if (start >= end) return true;
|
||||
uint32_t lit_len = (uint32_t)(end - start);
|
||||
if (!ensure_capacity(instrs, capacity, *count))
|
||||
return false;
|
||||
uint8_t* lit_data = protocol_alloc(lit_len);
|
||||
if (!lit_data)
|
||||
return false;
|
||||
if (!ensure_capacity(instrs, capacity, *count)) return false;
|
||||
uint8_t *lit_data = malloc(lit_len);
|
||||
if (!lit_data) return false;
|
||||
memcpy(lit_data, data + start, lit_len);
|
||||
(*instrs)[*count].type = DELTA_INSTR_LITERAL;
|
||||
(*instrs)[*count].literal.data = lit_data;
|
||||
@@ -216,165 +155,17 @@ static bool flush_literal(DeltaInstruction** instrs, uint32_t* capacity, uint32_
|
||||
return true;
|
||||
}
|
||||
|
||||
static void free_instructions(DeltaInstruction* instrs, uint32_t count) {
|
||||
if (!instrs)
|
||||
return;
|
||||
for (uint32_t i = 0; i < count; i++)
|
||||
if (instrs[i].type == DELTA_INSTR_LITERAL)
|
||||
free(instrs[i].literal.data);
|
||||
free(instrs);
|
||||
}
|
||||
|
||||
/* Sentinel meaning "no signature block" in the lookup index chains. Block
|
||||
* counts are bounded well below UINT32_MAX, so it doubles as a null link. */
|
||||
#define DELTA_NO_BLOCK UINT32_MAX
|
||||
|
||||
/* Avalanche mix for the rolling checksum so blocks do not cluster in the
|
||||
* bucket table when the weak checksum has little entropy (e.g. all-zero or
|
||||
* patterned files). */
|
||||
static uint32_t delta_adler_mix(uint32_t h) {
|
||||
h ^= h >> 16;
|
||||
h *= 0x7feb352dU;
|
||||
h ^= h >> 15;
|
||||
h *= 0x846ca68bU;
|
||||
h ^= h >> 16;
|
||||
return h;
|
||||
}
|
||||
|
||||
/* Smallest power of two >= v. v must be non-zero. */
|
||||
static uint32_t delta_next_pow2(uint32_t v) {
|
||||
v--;
|
||||
v |= v >> 1;
|
||||
v |= v >> 2;
|
||||
v |= v >> 4;
|
||||
v |= v >> 8;
|
||||
v |= v >> 16;
|
||||
return v + 1;
|
||||
}
|
||||
|
||||
/* Build a hash index over sig->blocks keyed by the (mixed) rolling checksum.
|
||||
* All blocks sharing an Adler-32 value land in the same bucket; collisions
|
||||
* are chained through a single contiguous allocation:
|
||||
*
|
||||
* [0, bucket_count) heads (first block per bucket)
|
||||
* [bucket_count, 2*bucket_count) tails (last block per bucket)
|
||||
* [2*bucket_count, ...) per-block chain links
|
||||
*
|
||||
* Blocks are inserted in ascending index order so every bucket chain is
|
||||
* ordered exactly like the historical linear scan. Returns the base pointer
|
||||
* (also the heads array) or NULL when no index could be allocated; callers
|
||||
* then fall back to the linear scan. */
|
||||
static uint32_t* delta_build_index(const DeltaSignature* sig, uint32_t bucket_count) {
|
||||
if (sig->block_count == 0 || bucket_count == 0)
|
||||
Delta *delta_compute(const void *new_file_data, uint64_t new_file_size,
|
||||
const DeltaSignature *sig, uint32_t block_size) {
|
||||
if (!new_file_data || !sig || new_file_size == 0 || block_size == 0)
|
||||
return NULL;
|
||||
|
||||
size_t entries = (size_t)2 * bucket_count + sig->block_count;
|
||||
if (entries > SIZE_MAX / sizeof(uint32_t))
|
||||
return NULL;
|
||||
|
||||
uint32_t* index = protocol_alloc(entries * sizeof(uint32_t));
|
||||
if (!index)
|
||||
return NULL;
|
||||
|
||||
uint32_t* heads = index;
|
||||
uint32_t* tails = index + bucket_count;
|
||||
uint32_t* next = index + 2 * bucket_count;
|
||||
uint32_t mask = bucket_count - 1;
|
||||
|
||||
memset(heads, 0xFF, (size_t)bucket_count * sizeof(uint32_t));
|
||||
memset(tails, 0xFF, (size_t)bucket_count * sizeof(uint32_t));
|
||||
|
||||
for (uint32_t j = 0; j < sig->block_count; j++) {
|
||||
uint32_t b = delta_adler_mix(sig->blocks[j].adler32) & mask;
|
||||
if (heads[b] == DELTA_NO_BLOCK)
|
||||
heads[b] = j;
|
||||
else
|
||||
next[tails[b]] = j;
|
||||
tails[b] = j;
|
||||
next[j] = DELTA_NO_BLOCK;
|
||||
}
|
||||
return index;
|
||||
}
|
||||
|
||||
/* Locate the signature block matching the byte window at new_data[i].
|
||||
*
|
||||
* Mirrors the original per-window behaviour exactly: only a full block_size
|
||||
* window can match, candidates are accepted only when the weak (Adler-32) and
|
||||
* strong (xxHash32) checksums both agree, and the lowest block index wins so
|
||||
* the emitted op stream is byte-identical to the linear scan. When heads is
|
||||
* non-NULL the candidate set is reached through the bucket index (expected
|
||||
* O(1) per window); otherwise an exact linear scan is used. */
|
||||
static uint32_t delta_find_match(const uint8_t* window, uint32_t window_len, uint32_t adler,
|
||||
bool full_window, const DeltaSignature* sig, const uint32_t* heads,
|
||||
const uint32_t* next, uint32_t mask, uint32_t seed) {
|
||||
if (!full_window || sig->block_count == 0)
|
||||
return DELTA_NO_BLOCK;
|
||||
|
||||
if (heads) {
|
||||
uint32_t b = delta_adler_mix(adler) & mask;
|
||||
uint32_t window_xxh = 0;
|
||||
bool have_xxh = false;
|
||||
for (uint32_t j = heads[b]; j != DELTA_NO_BLOCK; j = next[j]) {
|
||||
if (sig->blocks[j].adler32 != adler)
|
||||
continue;
|
||||
if (!have_xxh) {
|
||||
window_xxh = delta_xxhash32_seeded(window, window_len, seed);
|
||||
have_xxh = true;
|
||||
}
|
||||
if (window_xxh == sig->blocks[j].xxhash)
|
||||
return j;
|
||||
}
|
||||
return DELTA_NO_BLOCK;
|
||||
}
|
||||
|
||||
/* Fallback used when the index could not be allocated. */
|
||||
for (uint32_t j = 0; j < sig->block_count; j++) {
|
||||
if (sig->blocks[j].adler32 == adler) {
|
||||
uint32_t window_xxh = delta_xxhash32_seeded(window, window_len, seed);
|
||||
if (window_xxh == sig->blocks[j].xxhash)
|
||||
return j;
|
||||
}
|
||||
}
|
||||
return DELTA_NO_BLOCK;
|
||||
}
|
||||
|
||||
Delta* delta_compute(const void* new_file_data, uint64_t new_file_size, const DeltaSignature* sig,
|
||||
uint32_t block_size) {
|
||||
return delta_compute_seeded(new_file_data, new_file_size, sig, block_size, 0);
|
||||
}
|
||||
|
||||
Delta* delta_compute_seeded(const void* new_file_data, uint64_t new_file_size,
|
||||
const DeltaSignature* sig, uint32_t block_size, uint32_t seed) {
|
||||
if (!new_file_data || !sig || !sig->blocks || new_file_size == 0 || block_size == 0 ||
|
||||
block_size > DELTA_BLOCK_SIZE_MAX || sig->block_size != block_size)
|
||||
return NULL;
|
||||
|
||||
const uint8_t* new_data = (const uint8_t*)new_file_data;
|
||||
const uint8_t *new_data = (const uint8_t *)new_file_data;
|
||||
|
||||
uint32_t capacity = 64;
|
||||
uint32_t count = 0;
|
||||
DeltaInstruction* instrs = protocol_alloc((size_t)capacity * sizeof(DeltaInstruction));
|
||||
if (!instrs)
|
||||
return NULL;
|
||||
|
||||
/* Build a one-time bucket index over the signature blocks keyed by the weak
|
||||
* checksum. This turns the per-byte-window candidate lookup from an
|
||||
* O(block_count) linear scan into an expected O(1) probe, which dominates
|
||||
* the cost for large mostly-matching files (the diff steps one byte at a
|
||||
* time through changed regions). On allocation failure the probe falls back
|
||||
* to the original linear scan, so behaviour is unchanged under memory
|
||||
* pressure. */
|
||||
uint32_t* index = NULL;
|
||||
const uint32_t* chain_next = NULL;
|
||||
uint32_t mask = 0;
|
||||
if (sig->block_count > 0) {
|
||||
uint32_t bucket_count = delta_next_pow2(sig->block_count);
|
||||
index = delta_build_index(sig, bucket_count);
|
||||
if (index) {
|
||||
chain_next = index + 2 * bucket_count;
|
||||
mask = bucket_count - 1;
|
||||
}
|
||||
}
|
||||
DeltaInstruction *instrs = malloc(capacity * sizeof(DeltaInstruction));
|
||||
if (!instrs) return NULL;
|
||||
|
||||
uint64_t literal_start = 0;
|
||||
bool has_literal = false;
|
||||
@@ -385,17 +176,20 @@ Delta* delta_compute_seeded(const void* new_file_data, uint64_t new_file_size,
|
||||
bool rolling_valid = false;
|
||||
|
||||
while (i < new_file_size) {
|
||||
uint32_t window_len =
|
||||
(uint32_t)((new_file_size - i < block_size) ? (new_file_size - i) : block_size);
|
||||
uint32_t window_len = (uint32_t)((new_file_size - i < block_size)
|
||||
? (new_file_size - i)
|
||||
: block_size);
|
||||
bool full_window = (window_len == block_size);
|
||||
|
||||
uint32_t adler;
|
||||
if (rolling_valid && full_window) {
|
||||
uint8_t old_byte = new_data[i - 1];
|
||||
uint8_t new_byte = new_data[i + block_size - 1];
|
||||
s1 = (s1 + DELTA_ADLER32_MODULUS - old_byte + new_byte) % DELTA_ADLER32_MODULUS;
|
||||
s1 = (s1 + DELTA_ADLER32_MODULUS - old_byte + new_byte) %
|
||||
DELTA_ADLER32_MODULUS;
|
||||
s2 = (s2 + DELTA_ADLER32_MODULUS -
|
||||
(uint32_t)((uint64_t)block_size * old_byte % DELTA_ADLER32_MODULUS) + s1 - 1) %
|
||||
(uint32_t)((uint64_t)block_size * old_byte % DELTA_ADLER32_MODULUS) +
|
||||
s1 - 1) %
|
||||
DELTA_ADLER32_MODULUS;
|
||||
adler = (s2 << 16) | s1;
|
||||
} else {
|
||||
@@ -410,32 +204,35 @@ Delta* delta_compute_seeded(const void* new_file_data, uint64_t new_file_size,
|
||||
}
|
||||
|
||||
bool matched = false;
|
||||
uint32_t match_block = delta_find_match(new_data + i, window_len, adler, full_window, sig,
|
||||
index, chain_next, mask, seed);
|
||||
if (match_block != DELTA_NO_BLOCK) {
|
||||
if (has_literal) {
|
||||
if (!flush_literal(&instrs, &capacity, &count, new_data, literal_start, i)) {
|
||||
free_instructions(instrs, count);
|
||||
free(index);
|
||||
return NULL;
|
||||
for (uint32_t j = 0; j < sig->block_count; j++) {
|
||||
if (adler == sig->blocks[j].adler32 && full_window) {
|
||||
uint32_t xxh = delta_xxhash32(new_data + i, window_len);
|
||||
if (xxh == sig->blocks[j].xxhash) {
|
||||
if (has_literal) {
|
||||
if (!flush_literal(&instrs, &capacity, &count, new_data,
|
||||
literal_start, i)) {
|
||||
free(instrs);
|
||||
return NULL;
|
||||
}
|
||||
has_literal = false;
|
||||
}
|
||||
|
||||
if (!ensure_capacity(&instrs, &capacity, count)) {
|
||||
free(instrs);
|
||||
return NULL;
|
||||
}
|
||||
instrs[count].type = DELTA_INSTR_BLOCK_MATCH;
|
||||
instrs[count].match.block_index = j;
|
||||
instrs[count].match.block_offset = 0;
|
||||
instrs[count].match.length = window_len;
|
||||
count++;
|
||||
|
||||
i += window_len;
|
||||
rolling_valid = false;
|
||||
matched = true;
|
||||
break;
|
||||
}
|
||||
has_literal = false;
|
||||
}
|
||||
|
||||
if (!ensure_capacity(&instrs, &capacity, count)) {
|
||||
free_instructions(instrs, count);
|
||||
free(index);
|
||||
return NULL;
|
||||
}
|
||||
instrs[count].type = DELTA_INSTR_BLOCK_MATCH;
|
||||
instrs[count].match.block_index = match_block;
|
||||
instrs[count].match.block_offset = 0;
|
||||
instrs[count].match.length = window_len;
|
||||
count++;
|
||||
|
||||
i += window_len;
|
||||
rolling_valid = false;
|
||||
matched = true;
|
||||
}
|
||||
|
||||
if (!matched) {
|
||||
@@ -447,18 +244,21 @@ Delta* delta_compute_seeded(const void* new_file_data, uint64_t new_file_size,
|
||||
}
|
||||
}
|
||||
|
||||
free(index);
|
||||
|
||||
if (has_literal) {
|
||||
if (!flush_literal(&instrs, &capacity, &count, new_data, literal_start, new_file_size)) {
|
||||
free_instructions(instrs, count);
|
||||
if (!flush_literal(&instrs, &capacity, &count, new_data,
|
||||
literal_start, new_file_size)) {
|
||||
free(instrs);
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
Delta* delta = protocol_alloc(sizeof(Delta));
|
||||
Delta *delta = malloc(sizeof(Delta));
|
||||
if (!delta) {
|
||||
free_instructions(instrs, count);
|
||||
for (uint32_t k = 0; k < count; k++) {
|
||||
if (instrs[k].type == DELTA_INSTR_LITERAL)
|
||||
free(instrs[k].literal.data);
|
||||
}
|
||||
free(instrs);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
@@ -468,43 +268,23 @@ Delta* delta_compute_seeded(const void* new_file_data, uint64_t new_file_size,
|
||||
delta->delta_size = 0;
|
||||
|
||||
for (uint32_t k = 0; k < count; k++) {
|
||||
if (delta->delta_size == UINT64_MAX) {
|
||||
delta_destroy(delta);
|
||||
return NULL;
|
||||
}
|
||||
delta->delta_size += 1;
|
||||
if (instrs[k].type == DELTA_INSTR_BLOCK_MATCH) {
|
||||
if (delta->delta_size > UINT64_MAX - sizeof(uint32_t) * 3) {
|
||||
delta_destroy(delta);
|
||||
return NULL;
|
||||
}
|
||||
delta->delta_size += sizeof(uint32_t) * 3;
|
||||
} else {
|
||||
uint64_t extra = sizeof(uint32_t) + instrs[k].literal.length;
|
||||
if (delta->delta_size > UINT64_MAX - extra) {
|
||||
delta_destroy(delta);
|
||||
return NULL;
|
||||
}
|
||||
delta->delta_size += extra;
|
||||
delta->delta_size += sizeof(uint32_t) + instrs[k].literal.length;
|
||||
}
|
||||
}
|
||||
|
||||
return delta;
|
||||
}
|
||||
|
||||
Data* delta_serialize(const Delta* delta) {
|
||||
if (!delta)
|
||||
return NULL;
|
||||
Data *delta_serialize(const Delta *delta) {
|
||||
if (!delta) return NULL;
|
||||
|
||||
if (delta->instruction_count > 0 && !delta->instructions)
|
||||
return NULL;
|
||||
uint64_t header_size = sizeof(uint64_t) + sizeof(uint32_t);
|
||||
if (delta->delta_size > UINT64_MAX - header_size || header_size + delta->delta_size > SIZE_MAX)
|
||||
return NULL;
|
||||
uint64_t total = header_size + delta->delta_size;
|
||||
uint8_t* buf = protocol_alloc((size_t)total);
|
||||
if (!buf)
|
||||
return NULL;
|
||||
uint64_t total = sizeof(uint64_t) + sizeof(uint32_t) + delta->delta_size;
|
||||
uint8_t *buf = malloc((size_t)total);
|
||||
if (!buf) return NULL;
|
||||
|
||||
size_t pos = 0;
|
||||
memcpy(buf + pos, &delta->new_file_size, sizeof(uint64_t));
|
||||
@@ -527,7 +307,8 @@ Data* delta_serialize(const Delta* delta) {
|
||||
} else {
|
||||
memcpy(buf + pos, &delta->instructions[i].literal.length, sizeof(uint32_t));
|
||||
pos += sizeof(uint32_t);
|
||||
memcpy(buf + pos, delta->instructions[i].literal.data, delta->instructions[i].literal.length);
|
||||
memcpy(buf + pos, delta->instructions[i].literal.data,
|
||||
delta->instructions[i].literal.length);
|
||||
pos += delta->instructions[i].literal.length;
|
||||
}
|
||||
}
|
||||
@@ -535,35 +316,23 @@ Data* delta_serialize(const Delta* delta) {
|
||||
return data_create(buf, (size_t)total);
|
||||
}
|
||||
|
||||
Delta* delta_deserialize(const Data* data) {
|
||||
Delta *delta_deserialize(const Data *data) {
|
||||
if (!data || data->size < sizeof(uint64_t) + sizeof(uint32_t))
|
||||
return NULL;
|
||||
|
||||
const uint8_t* buf = (const uint8_t*)data->data;
|
||||
const uint8_t *buf = (const uint8_t *)data->data;
|
||||
size_t pos = 0;
|
||||
|
||||
Delta* delta = protocol_alloc(sizeof(Delta));
|
||||
if (!delta)
|
||||
return NULL;
|
||||
Delta *delta = malloc(sizeof(Delta));
|
||||
if (!delta) return NULL;
|
||||
|
||||
memcpy(&delta->new_file_size, buf + pos, sizeof(uint64_t));
|
||||
pos += sizeof(uint64_t);
|
||||
memcpy(&delta->instruction_count, buf + pos, sizeof(uint32_t));
|
||||
pos += sizeof(uint32_t);
|
||||
|
||||
// Reject unreasonably large instruction counts to prevent OOM
|
||||
if (delta->instruction_count > MAX_DELTA_INSTRUCTIONS) {
|
||||
log_message(LOG_LEVEL_ERROR, "Delta instruction count %u exceeds maximum %u",
|
||||
delta->instruction_count, MAX_DELTA_INSTRUCTIONS);
|
||||
free(delta);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
delta->instructions =
|
||||
delta->instruction_count == 0
|
||||
? NULL
|
||||
: protocol_alloc((size_t)delta->instruction_count * sizeof(DeltaInstruction));
|
||||
if (delta->instruction_count > 0 && !delta->instructions) {
|
||||
delta->instructions = malloc(delta->instruction_count * sizeof(DeltaInstruction));
|
||||
if (!delta->instructions) {
|
||||
free(delta);
|
||||
return NULL;
|
||||
}
|
||||
@@ -572,7 +341,11 @@ Delta* delta_deserialize(const Data* data) {
|
||||
|
||||
for (uint32_t i = 0; i < delta->instruction_count; i++) {
|
||||
if (pos >= data->size) {
|
||||
free_instructions(delta->instructions, i);
|
||||
for (uint32_t k = 0; k < i; k++) {
|
||||
if (delta->instructions[k].type == DELTA_INSTR_LITERAL)
|
||||
free(delta->instructions[k].literal.data);
|
||||
}
|
||||
free(delta->instructions);
|
||||
free(delta);
|
||||
return NULL;
|
||||
}
|
||||
@@ -584,8 +357,8 @@ Delta* delta_deserialize(const Data* data) {
|
||||
delta->delta_size += 1;
|
||||
|
||||
if (type == DELTA_OP_BLOCK_MATCH) {
|
||||
if (data->size - pos < sizeof(uint32_t) * 3) {
|
||||
free_instructions(delta->instructions, i);
|
||||
if (pos + sizeof(uint32_t) * 3 > data->size) {
|
||||
free(delta->instructions);
|
||||
free(delta);
|
||||
return NULL;
|
||||
}
|
||||
@@ -598,8 +371,8 @@ Delta* delta_deserialize(const Data* data) {
|
||||
pos += sizeof(uint32_t);
|
||||
delta->delta_size += sizeof(uint32_t) * 3;
|
||||
} else if (type == DELTA_OP_LITERAL) {
|
||||
if (data->size - pos < sizeof(uint32_t)) {
|
||||
free_instructions(delta->instructions, i);
|
||||
if (pos + sizeof(uint32_t) > data->size) {
|
||||
free(delta->instructions);
|
||||
free(delta);
|
||||
return NULL;
|
||||
}
|
||||
@@ -608,15 +381,14 @@ Delta* delta_deserialize(const Data* data) {
|
||||
pos += sizeof(uint32_t);
|
||||
|
||||
uint32_t lit_len = delta->instructions[i].literal.length;
|
||||
if (lit_len > data->size - pos) {
|
||||
free_instructions(delta->instructions, i);
|
||||
if (pos + lit_len > data->size) {
|
||||
free(delta->instructions);
|
||||
free(delta);
|
||||
return NULL;
|
||||
}
|
||||
delta->instructions[i].literal.data = protocol_alloc(lit_len ? lit_len : 1);
|
||||
delta->instructions[i].literal.data = malloc(lit_len);
|
||||
if (!delta->instructions[i].literal.data) {
|
||||
log_message(LOG_LEVEL_ERROR, "Failed to allocate %u bytes for literal data", lit_len);
|
||||
free_instructions(delta->instructions, i);
|
||||
free(delta->instructions);
|
||||
free(delta);
|
||||
return NULL;
|
||||
}
|
||||
@@ -624,7 +396,11 @@ Delta* delta_deserialize(const Data* data) {
|
||||
pos += lit_len;
|
||||
delta->delta_size += sizeof(uint32_t) + lit_len;
|
||||
} else {
|
||||
free_instructions(delta->instructions, i);
|
||||
for (uint32_t k = 0; k < i; k++) {
|
||||
if (delta->instructions[k].type == DELTA_INSTR_LITERAL)
|
||||
free(delta->instructions[k].literal.data);
|
||||
}
|
||||
free(delta->instructions);
|
||||
free(delta);
|
||||
return NULL;
|
||||
}
|
||||
@@ -633,49 +409,34 @@ Delta* delta_deserialize(const Data* data) {
|
||||
return delta;
|
||||
}
|
||||
|
||||
void* delta_apply(const void* old_data, uint64_t old_size, const Delta* delta,
|
||||
void *delta_apply(const void *old_data, uint64_t old_size, const Delta *delta,
|
||||
uint32_t block_size) {
|
||||
if (!old_data || !delta || (delta->new_file_size > 0 && delta->instructions == NULL) ||
|
||||
(delta->instruction_count > 0 && block_size == 0) ||
|
||||
delta->new_file_size > DELTA_MAX_FILE_SIZE || delta->new_file_size > SIZE_MAX)
|
||||
return NULL;
|
||||
if (!old_data || !delta) return NULL;
|
||||
|
||||
void* output = protocol_alloc(delta->new_file_size ? (size_t)delta->new_file_size : 1);
|
||||
if (!output)
|
||||
return NULL;
|
||||
void *output = malloc((size_t)delta->new_file_size);
|
||||
if (!output) return NULL;
|
||||
|
||||
uint8_t* out = (uint8_t*)output;
|
||||
const uint8_t* old = (const uint8_t*)old_data;
|
||||
uint8_t *out = (uint8_t *)output;
|
||||
uint8_t *old = (uint8_t *)old_data;
|
||||
uint64_t out_pos = 0;
|
||||
|
||||
for (uint32_t i = 0; i < delta->instruction_count; i++) {
|
||||
if (delta->instructions[i].type == DELTA_INSTR_BLOCK_MATCH) {
|
||||
uint64_t src_offset = (uint64_t)delta->instructions[i].match.block_index * block_size;
|
||||
if (src_offset > UINT64_MAX - delta->instructions[i].match.block_offset) {
|
||||
free(output);
|
||||
return NULL;
|
||||
}
|
||||
uint64_t src_offset = (uint64_t)delta->instructions[i].match.block_index *
|
||||
block_size;
|
||||
src_offset += delta->instructions[i].match.block_offset;
|
||||
uint32_t len = delta->instructions[i].match.length;
|
||||
|
||||
if (src_offset > old_size || (uint64_t)len > old_size - src_offset ||
|
||||
out_pos > delta->new_file_size || (uint64_t)len > delta->new_file_size - out_pos) {
|
||||
if (src_offset + len > old_size) {
|
||||
free(output);
|
||||
return NULL;
|
||||
}
|
||||
memcpy(out + out_pos, old + src_offset, len);
|
||||
out_pos += len;
|
||||
} else if (delta->instructions[i].type == DELTA_INSTR_LITERAL) {
|
||||
} else {
|
||||
uint32_t len = delta->instructions[i].literal.length;
|
||||
if (out_pos > delta->new_file_size || (uint64_t)len > delta->new_file_size - out_pos) {
|
||||
free(output);
|
||||
return NULL;
|
||||
}
|
||||
memcpy(out + out_pos, delta->instructions[i].literal.data, len);
|
||||
out_pos += len;
|
||||
} else {
|
||||
free(output);
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -687,9 +448,8 @@ void* delta_apply(const void* old_data, uint64_t old_size, const Delta* delta,
|
||||
return output;
|
||||
}
|
||||
|
||||
void delta_destroy(Delta* delta) {
|
||||
if (!delta)
|
||||
return;
|
||||
void delta_destroy(Delta *delta) {
|
||||
if (!delta) return;
|
||||
for (uint32_t i = 0; i < delta->instruction_count; i++) {
|
||||
if (delta->instructions[i].type == DELTA_INSTR_LITERAL)
|
||||
free(delta->instructions[i].literal.data);
|
||||
@@ -710,9 +470,8 @@ bool delta_should_attempt(uint64_t old_size, uint64_t new_size, uint64_t max_fil
|
||||
return true;
|
||||
}
|
||||
|
||||
bool delta_is_worthwhile(const Delta* delta, uint64_t new_file_size) {
|
||||
if (!delta || delta->instruction_count == 0 || new_file_size == 0)
|
||||
return false;
|
||||
bool delta_is_worthwhile(const Delta *delta, uint64_t new_file_size) {
|
||||
if (!delta || delta->instruction_count == 0) return false;
|
||||
|
||||
bool has_match = false;
|
||||
for (uint32_t i = 0; i < delta->instruction_count; i++) {
|
||||
@@ -721,8 +480,7 @@ bool delta_is_worthwhile(const Delta* delta, uint64_t new_file_size) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!has_match)
|
||||
return false;
|
||||
if (!has_match) return false;
|
||||
|
||||
double ratio = (double)delta->delta_size / (double)new_file_size;
|
||||
return ratio < DELTA_FALLBACK_RATIO;
|
||||
|
||||
+33
-40
@@ -6,17 +6,17 @@
|
||||
#include <stdint.h>
|
||||
#include <stddef.h>
|
||||
|
||||
#define DELTA_BLOCK_SIZE_DEFAULT 8192U
|
||||
#define DELTA_BLOCK_SIZE_MIN 1024U
|
||||
#define DELTA_BLOCK_SIZE_MAX 65536U
|
||||
#define DELTA_MIN_FILE_SIZE 16384ULL
|
||||
#define DELTA_MAX_FILE_SIZE (256ULL * 1024 * 1024)
|
||||
#define DELTA_MAX_SIZE_RATIO 10.0
|
||||
#define DELTA_FALLBACK_RATIO 0.7
|
||||
#define DELTA_ADLER32_MODULUS 65521U
|
||||
#define DELTA_BLOCK_SIZE_DEFAULT 8192U
|
||||
#define DELTA_BLOCK_SIZE_MIN 1024U
|
||||
#define DELTA_BLOCK_SIZE_MAX 65536U
|
||||
#define DELTA_MIN_FILE_SIZE 16384ULL
|
||||
#define DELTA_MAX_FILE_SIZE (256ULL * 1024 * 1024)
|
||||
#define DELTA_MAX_SIZE_RATIO 10.0
|
||||
#define DELTA_FALLBACK_RATIO 0.7
|
||||
#define DELTA_ADLER32_MODULUS 65521U
|
||||
|
||||
#define DELTA_OP_BLOCK_MATCH 0x01
|
||||
#define DELTA_OP_LITERAL 0x02
|
||||
#define DELTA_OP_BLOCK_MATCH 0x01
|
||||
#define DELTA_OP_LITERAL 0x02
|
||||
|
||||
typedef struct {
|
||||
uint32_t adler32;
|
||||
@@ -27,10 +27,13 @@ typedef struct {
|
||||
uint64_t file_size;
|
||||
uint32_t block_size;
|
||||
uint32_t block_count;
|
||||
DeltaBlockSig* blocks;
|
||||
DeltaBlockSig *blocks;
|
||||
} DeltaSignature;
|
||||
|
||||
typedef enum { DELTA_INSTR_BLOCK_MATCH = 0x01, DELTA_INSTR_LITERAL = 0x02 } DeltaInstrType;
|
||||
typedef enum {
|
||||
DELTA_INSTR_BLOCK_MATCH = 0x01,
|
||||
DELTA_INSTR_LITERAL = 0x02
|
||||
} DeltaInstrType;
|
||||
|
||||
typedef struct {
|
||||
DeltaInstrType type;
|
||||
@@ -41,7 +44,7 @@ typedef struct {
|
||||
uint32_t length;
|
||||
} match;
|
||||
struct {
|
||||
uint8_t* data;
|
||||
uint8_t *data;
|
||||
uint32_t length;
|
||||
} literal;
|
||||
};
|
||||
@@ -50,39 +53,29 @@ typedef struct {
|
||||
typedef struct {
|
||||
uint64_t new_file_size;
|
||||
uint32_t instruction_count;
|
||||
DeltaInstruction* instructions;
|
||||
DeltaInstruction *instructions;
|
||||
uint64_t delta_size;
|
||||
} Delta;
|
||||
|
||||
DeltaSignature* delta_signature_create(const void* old_file_data, uint64_t old_file_size,
|
||||
uint32_t block_size);
|
||||
/* Seeded equivalent of delta_signature_create: the per-block strong (xxHash32)
|
||||
* checksum uses `seed` (the low 32 bits of --checksum-seed). Passing seed 0 is
|
||||
* identical to the unseeded function. */
|
||||
DeltaSignature* delta_signature_create_seeded(const void* old_file_data, uint64_t old_file_size,
|
||||
uint32_t block_size, uint32_t seed);
|
||||
Data* delta_signature_serialize(const DeltaSignature* sig);
|
||||
DeltaSignature* delta_signature_deserialize(const Data* data);
|
||||
void delta_signature_destroy(DeltaSignature* sig);
|
||||
DeltaSignature *delta_signature_create(const void *old_file_data,
|
||||
uint64_t old_file_size,
|
||||
uint32_t block_size);
|
||||
Data *delta_signature_serialize(const DeltaSignature *sig);
|
||||
DeltaSignature *delta_signature_deserialize(const Data *data);
|
||||
void delta_signature_destroy(DeltaSignature *sig);
|
||||
|
||||
Delta* delta_compute(const void* new_file_data, uint64_t new_file_size, const DeltaSignature* sig,
|
||||
uint32_t block_size);
|
||||
/* Seeded equivalent of delta_compute: the per-window strong (xxHash32) check
|
||||
* uses `seed` (the low 32 bits of --checksum-seed). The receiver's signature
|
||||
* must have been built with the same seed for matching. */
|
||||
Delta* delta_compute_seeded(const void* new_file_data, uint64_t new_file_size,
|
||||
const DeltaSignature* sig, uint32_t block_size, uint32_t seed);
|
||||
Data* delta_serialize(const Delta* delta);
|
||||
Delta* delta_deserialize(const Data* data);
|
||||
void* delta_apply(const void* old_data, uint64_t old_size, const Delta* delta, uint32_t block_size);
|
||||
void delta_destroy(Delta* delta);
|
||||
Delta *delta_compute(const void *new_file_data, uint64_t new_file_size,
|
||||
const DeltaSignature *sig, uint32_t block_size);
|
||||
Data *delta_serialize(const Delta *delta);
|
||||
Delta *delta_deserialize(const Data *data);
|
||||
void *delta_apply(const void *old_data, uint64_t old_size, const Delta *delta,
|
||||
uint32_t block_size);
|
||||
void delta_destroy(Delta *delta);
|
||||
|
||||
bool delta_should_attempt(uint64_t old_size, uint64_t new_size, uint64_t max_file_size);
|
||||
bool delta_is_worthwhile(const Delta* delta, uint64_t new_file_size);
|
||||
bool delta_is_worthwhile(const Delta *delta, uint64_t new_file_size);
|
||||
|
||||
uint32_t delta_adler32(const void* data, uint32_t len);
|
||||
uint32_t delta_xxhash32(const void* data, uint32_t len);
|
||||
uint32_t delta_xxhash32_seeded(const void* data, uint32_t len, uint32_t seed);
|
||||
uint64_t delta_xxhash64(const void* data, size_t len);
|
||||
uint32_t delta_adler32(const void *data, uint32_t len);
|
||||
uint32_t delta_xxhash32(const void *data, uint32_t len);
|
||||
|
||||
#endif
|
||||
|
||||
+390
-1220
File diff suppressed because it is too large
Load Diff
+27
-134
@@ -1,144 +1,37 @@
|
||||
#ifndef FILE_H
|
||||
#define FILE_H
|
||||
|
||||
#include "file_send.h"
|
||||
#include "file_receive.h"
|
||||
#include "file_types.h"
|
||||
#include "checksum.h"
|
||||
#include "config.h"
|
||||
#include "data.h"
|
||||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
#include <sys/stat.h>
|
||||
|
||||
/* File/FileMetadata lifecycle, local disk helpers, and secure filesystem
|
||||
primitives shared by the send/receive pipelines. */
|
||||
typedef struct {
|
||||
mode_t mode;
|
||||
uid_t uid;
|
||||
gid_t gid;
|
||||
time_t mtime_sec;
|
||||
long mtime_nsec;
|
||||
} FileMetadata;
|
||||
|
||||
File* file_create(const char* path);
|
||||
void file_destroy(void* item);
|
||||
bool file_load_data(File* file);
|
||||
/* Compute the whole-file content digest of `file` with the negotiated
|
||||
* --checksum-choice algorithm and --checksum-seed. Writes the digest into
|
||||
* `out` (capacity `out_capacity`) and its length into `*out_len`. Returns
|
||||
* false on read/allocation failure or when the digest would not fit. */
|
||||
bool file_checksum(File* file, ChecksumAlgo algo, uint64_t seed, uint8_t* out, size_t out_capacity,
|
||||
size_t* out_len);
|
||||
size_t file_content_to_buffer(File* file);
|
||||
FileMetadata* file_metadata_create(const char* path, const struct stat* stats, bool capture_atime,
|
||||
bool capture_crtime);
|
||||
void file_metadata_destroy(void* metadata);
|
||||
/* --open-noatime process-wide sender policy; see file.c. */
|
||||
void file_set_open_noatime(bool enable);
|
||||
bool file_get_open_noatime(void);
|
||||
/* Open `path` read-only for transfer, honouring --open-noatime when set. */
|
||||
int file_open_for_read(const char* path);
|
||||
bool file_write_to_disk(const char* path, const void* data, unsigned long long data_size,
|
||||
bool inplace, bool sparse);
|
||||
typedef struct {
|
||||
char *path;
|
||||
Data *data;
|
||||
FileMetadata *metadata;
|
||||
} File;
|
||||
|
||||
/* Symlink trust-boundary helpers (Phase 4, symlink wave). --munge-links
|
||||
* sender-side marker: every transmitted symlink target is prefixed with this
|
||||
* while the flag is on; the receiver strips it to restore the real target. */
|
||||
#define SYMLINK_MUNGE_PREFIX "#SYMLINK/"
|
||||
|
||||
char* file_symlink_munge(const char* target);
|
||||
/* True when a lexical target is relative and contains no ".." component, so it
|
||||
* can never escape the receive root once created beneath it. */
|
||||
bool file_symlink_target_contained(const char* target);
|
||||
/* Strip a leading SYMLINK_MUNGE_PREFIX from `target` (mutable, in place);
|
||||
* returns true when a marker was removed. */
|
||||
bool file_symlink_unmunge(char* target);
|
||||
/* Create a symlink at `path` -> `target`, confined below the authorized root
|
||||
* (O_NOFOLLOW parent walk, symlinkat; the target is never followed). Returns
|
||||
* false when a directory already occupies `path`. */
|
||||
bool file_symlink_at_secure(const char* path, const char* target);
|
||||
/* --keep-dirlinks (-K) receiver process-wide policy: allow an in-root existing
|
||||
* symlink-to-directory to be followed as a directory. */
|
||||
void file_set_keep_dirlinks(bool enable);
|
||||
bool file_get_keep_dirlinks(void);
|
||||
|
||||
/* --trust-sender receiver process-wide policy (Phase 5). When set, the
|
||||
* receiver trusts that the sender already produced a clean file list and skips
|
||||
* its own redundant up-front re-validation of incoming paths (the empty/".."
|
||||
* rejection and the escaping-symlink-target containment). The low-level
|
||||
* fd-relative confinement primitives below are deliberately NOT disabled by
|
||||
* this flag, so a hostile sender still cannot escape the authorized root. */
|
||||
void file_set_trust_sender(bool enable);
|
||||
bool file_get_trust_sender(void);
|
||||
|
||||
/* A configured fd without a canonical identity deliberately rejects paths. */
|
||||
bool file_set_authorized_root(int fd, const char* canonical_path);
|
||||
|
||||
/* Secure path/filesystem primitives (symlink-safe, O_NOFOLLOW, root-confined). */
|
||||
bool file_path_exists_secure(const char* path);
|
||||
bool file_stat_secure(const char* path, struct stat* st);
|
||||
bool file_destination_is_newer_secure(const char* path, const FileMetadata* metadata);
|
||||
int file_open_secure_parent(const char* path, char** leaf_out, bool create_dirs);
|
||||
bool file_ensure_directory_secure(const char* path);
|
||||
bool file_directory_exists_secure(const char* path);
|
||||
bool file_rename_secure(const char* old_path, const char* new_path);
|
||||
/* Remove the whole directory tree at `path` (confined, symlink-safe). Used by
|
||||
--force to clear a non-empty destination directory that blocks an incoming
|
||||
regular file. See the .c for the exact success semantics. */
|
||||
bool file_remove_tree_secure(const char* path);
|
||||
/* Open a private 0700 directory (creating it on demand) that must live below
|
||||
the authorized root. Used for the --temp-dir scratch directory and the
|
||||
--delay-updates staging directory. */
|
||||
int file_open_private_dir(const char* dir_path);
|
||||
|
||||
/* The file_to_disk_secure* variants write a temporary copy in the destination
|
||||
directory and atomically rename it over `path`. temp_dir is an absolute,
|
||||
root-confined scratch directory (already validated by the caller): when it
|
||||
is non-NULL the temporary copy is instead created there (with a name unique
|
||||
across the whole scratch directory) and atomically renamed into the
|
||||
destination directory once fully written and fsynced. A rename across
|
||||
filesystems (EXDEV) fails the write with an error; the file is never
|
||||
silently copied into place. Pass NULL for the historical same-directory
|
||||
behavior. --inplace writes never use temp_dir. */
|
||||
bool file_to_disk_secure(const char* path, const void* data, unsigned long long data_size,
|
||||
bool inplace, bool sparse, bool preallocate, const FileMetadata* metadata,
|
||||
bool preserve_executability, const char* temp_dir);
|
||||
bool file_to_disk_secure_with_fsync(const char* path, const void* data,
|
||||
unsigned long long data_size, bool inplace, bool sparse,
|
||||
bool preallocate, const FileMetadata* metadata,
|
||||
bool preserve_executability, bool use_fsync,
|
||||
const char* temp_dir);
|
||||
/* With update enabled, an existing newer destination is left untouched. The
|
||||
check is descriptor-based for inplace writes; atomic replacement still has
|
||||
an unavoidable final rename race without filesystem locking. */
|
||||
bool file_to_disk_secure_update(const char* path, const void* data, unsigned long long data_size,
|
||||
bool inplace, bool sparse, bool preallocate,
|
||||
const FileMetadata* metadata, bool preserve_executability,
|
||||
const char* temp_dir);
|
||||
bool file_to_disk_secure_no_replace(const char* path, const void* data,
|
||||
unsigned long long data_size, bool sparse, bool preallocate,
|
||||
const FileMetadata* metadata, bool preserve_executability,
|
||||
const char* temp_dir);
|
||||
/* Receiver write-path variant that also applies per-file xattrs (-X/-A) and the
|
||||
* --fake-super stat xattr fd-relative before the final rename. `update` /
|
||||
* `no_replace` / `use_fsync` mirror the plain wrappers above; `keep_partial`
|
||||
* enables --partial best-effort retention of a failed write's temp. */
|
||||
bool file_to_disk_secure_attrs(const char* path, const void* data, unsigned long long data_size,
|
||||
bool inplace, bool sparse, bool preallocate,
|
||||
const FileMetadata* metadata, bool preserve_executability,
|
||||
bool update, bool no_replace, bool use_fsync,
|
||||
const FileXattrList* xattrs, bool fake_super, bool keep_partial,
|
||||
const char* temp_dir);
|
||||
/* Atomic --link-dest install: replace `path` with a hard link to `basis_path`
|
||||
(via a temp name + rename); fall back to a byte-identical local copy from
|
||||
`data` when the link is impossible (EXDEV/EPERM/unsupported filesystem).
|
||||
`metadata` is applied only on the copy fallback. `preallocate` applies to
|
||||
that copy fallback only (a hard-linked file shares the basis inode and is
|
||||
never re-allocated). */
|
||||
bool file_to_disk_secure_link(const char* path, const char* basis_path, const void* data,
|
||||
unsigned long long data_size, bool preallocate,
|
||||
const FileMetadata* metadata, bool preserve_executability,
|
||||
bool use_fsync, const char* temp_dir);
|
||||
/* Like file_to_disk_secure_link, but the byte-copy fallback also applies the
|
||||
* per-file xattrs (-X/-A) and --fake-super stat xattr (fd-relative). On a
|
||||
* successful hard link no attributes are applied (the shared inode already
|
||||
* carries the basis's). */
|
||||
bool file_to_disk_secure_link_attrs(const char* path, const char* basis_path, const void* data,
|
||||
unsigned long long data_size, bool preallocate,
|
||||
const FileMetadata* metadata, bool preserve_executability,
|
||||
bool use_fsync, const FileXattrList* xattrs, bool fake_super,
|
||||
const char* temp_dir);
|
||||
File *file_create(const char *path);
|
||||
void file_destroy(void *item);
|
||||
bool file_load_data(File *file);
|
||||
File *file_receive(Config *config, int file_descriptor);
|
||||
bool file_send_single_calls(File *file, int file_descriptor, bool use_metadata, int compression_level, bool send_path);
|
||||
bool file_send_sendfile(File *file, int file_descriptor, bool use_metadata, int compression_level, bool send_path);
|
||||
size_t file_content_to_buffer(File *file);
|
||||
FileMetadata *file_metadata_create(struct stat *stats);
|
||||
void file_metadata_destroy(void *metadata);
|
||||
bool to_disk(const char *path, const void *data, unsigned long long data_size);
|
||||
bool file_save_to_disk(const char *root_directory, File *file);
|
||||
File *receive_incremental_check(int fd, Config *config, bool *skipped);
|
||||
int receive_manifest(int fd, Config *config, int *next_status);
|
||||
|
||||
#endif
|
||||
|
||||
@@ -1,191 +0,0 @@
|
||||
#include "file_list.h"
|
||||
#include "log.h"
|
||||
#include "utils.h"
|
||||
#include <errno.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
typedef struct {
|
||||
char** items;
|
||||
int count;
|
||||
int capacity;
|
||||
} StringList;
|
||||
|
||||
static void string_list_destroy(StringList* list) {
|
||||
if (!list)
|
||||
return;
|
||||
for (int i = 0; i < list->count; i++)
|
||||
free(list->items[i]);
|
||||
free(list->items);
|
||||
}
|
||||
|
||||
static bool string_list_add(StringList* list, const char* text) {
|
||||
if (list->count == list->capacity) {
|
||||
int new_cap = list->capacity > 0 ? list->capacity * 2 : 16;
|
||||
char** grown = realloc(list->items, (size_t)new_cap * sizeof(char*));
|
||||
if (!grown)
|
||||
return false;
|
||||
list->items = grown;
|
||||
list->capacity = new_cap;
|
||||
}
|
||||
list->items[list->count] = str_dup(text);
|
||||
if (!list->items[list->count])
|
||||
return false;
|
||||
list->count++;
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Validate and normalize one entry. Returns:
|
||||
* 1 -> added to `out`
|
||||
* 0 -> blank entry, skip
|
||||
* -1 -> invalid (message set in `err`)
|
||||
* `strip_line_endings` trims a trailing CR/LF (line mode only); NUL mode keeps
|
||||
* the entry bytes verbatim so names ending in CR/LF survive. */
|
||||
static int normalize_entry(const char* raw, size_t len, bool strip_line_endings, StringList* out,
|
||||
char* err, size_t err_size) {
|
||||
if (strip_line_endings) {
|
||||
while (len > 0 && (raw[len - 1] == '\n' || raw[len - 1] == '\r'))
|
||||
len--;
|
||||
}
|
||||
if (len == 0)
|
||||
return 0;
|
||||
if (raw[0] == '/') {
|
||||
snprintf(err, err_size, "absolute path entries are not allowed: '%.*s'", (int)len, raw);
|
||||
return -1;
|
||||
}
|
||||
/* Reject NUL bytes inside a token defensively (NUL-delimited mode splits on
|
||||
* them, so this only guards against embedded garbage). */
|
||||
char* dup = malloc(len + 1);
|
||||
if (!dup) {
|
||||
snprintf(err, err_size, "memory allocation failed");
|
||||
return -1;
|
||||
}
|
||||
memcpy(dup, raw, len);
|
||||
dup[len] = '\0';
|
||||
|
||||
/* Rebuild the path token-by-token: skip '.' and empty segments, reject '..'. */
|
||||
size_t out_len = 0;
|
||||
for (const char* part = dup;;) {
|
||||
const char* slash = strchr(part, '/');
|
||||
size_t part_len = slash ? (size_t)(slash - part) : strlen(part);
|
||||
if (part_len == 1 && part[0] == '.') {
|
||||
/* skip "." segment */
|
||||
} else if (part_len == 2 && part[0] == '.' && part[1] == '.') {
|
||||
snprintf(err, err_size, "path traversal entry is not allowed: '%s'", dup);
|
||||
free(dup);
|
||||
return -1;
|
||||
} else if (part_len > 0) {
|
||||
if (out_len > 0)
|
||||
dup[out_len++] = '/';
|
||||
memmove(dup + out_len, part, part_len);
|
||||
out_len += part_len;
|
||||
}
|
||||
if (!slash)
|
||||
break;
|
||||
part = slash + 1;
|
||||
}
|
||||
dup[out_len] = '\0';
|
||||
|
||||
int result;
|
||||
if (out_len == 0) {
|
||||
/* "." / "./" lists the source root: the whole tree is transferred. */
|
||||
result = string_list_add(out, "") ? 1 : -1;
|
||||
if (result < 0)
|
||||
snprintf(err, err_size, "memory allocation failed");
|
||||
} else {
|
||||
result = string_list_add(out, dup) ? 1 : -1;
|
||||
if (result < 0)
|
||||
snprintf(err, err_size, "memory allocation failed");
|
||||
}
|
||||
free(dup);
|
||||
return result;
|
||||
}
|
||||
|
||||
static FileListSet* string_list_to_set(StringList* raw, char* err, size_t err_size) {
|
||||
FileListSet* set = malloc(sizeof(FileListSet));
|
||||
if (!set) {
|
||||
snprintf(err, err_size, "memory allocation failed");
|
||||
return NULL;
|
||||
}
|
||||
set->count = raw->count;
|
||||
set->entries = raw->items;
|
||||
raw->items = NULL;
|
||||
raw->count = 0;
|
||||
return set;
|
||||
}
|
||||
|
||||
FileListSet* file_list_load(const char* path, bool null_separated, char* err, size_t err_size) {
|
||||
if (err && err_size > 0)
|
||||
err[0] = '\0';
|
||||
if (!path || !*path) {
|
||||
snprintf(err, err_size, "no file given");
|
||||
return NULL;
|
||||
}
|
||||
FILE* fp = fopen(path, "r");
|
||||
if (!fp) {
|
||||
char* escaped = output_escape(path, false);
|
||||
snprintf(err, err_size, "could not open '%s': %s", escaped ? escaped : path, strerror(errno));
|
||||
free(escaped);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
StringList raw = {0};
|
||||
char* line = NULL;
|
||||
size_t line_cap = 0;
|
||||
ssize_t n;
|
||||
bool ok = true;
|
||||
char delim = null_separated ? '\0' : '\n';
|
||||
while (ok && (n = getdelim(&line, &line_cap, delim, fp)) != -1) {
|
||||
int r = normalize_entry(line, (size_t)n, !null_separated, &raw, err, err_size);
|
||||
if (r < 0) {
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
free(line);
|
||||
fclose(fp);
|
||||
if (!ok) {
|
||||
string_list_destroy(&raw);
|
||||
return NULL;
|
||||
}
|
||||
FileListSet* set = string_list_to_set(&raw, err, err_size);
|
||||
if (!set)
|
||||
string_list_destroy(&raw);
|
||||
return set;
|
||||
}
|
||||
|
||||
void file_list_destroy(FileListSet* set) {
|
||||
if (!set)
|
||||
return;
|
||||
for (int i = 0; i < set->count; i++)
|
||||
free(set->entries[i]);
|
||||
free(set->entries);
|
||||
free(set);
|
||||
}
|
||||
|
||||
static bool path_has_prefix(const char* path, const char* prefix) {
|
||||
size_t plen = strlen(prefix);
|
||||
if (strncmp(path, prefix, plen) != 0)
|
||||
return false;
|
||||
return path[plen] == '/' || path[plen] == '\0';
|
||||
}
|
||||
|
||||
bool file_list_affects(const FileListSet* set, const char* rel) {
|
||||
if (!set)
|
||||
return true;
|
||||
if (!rel)
|
||||
return false;
|
||||
for (int i = 0; i < set->count; i++) {
|
||||
const char* entry = set->entries[i];
|
||||
if (entry[0] == '\0')
|
||||
return true; /* whole tree listed */
|
||||
if (strcmp(rel, entry) == 0)
|
||||
return true; /* the entry itself is listed */
|
||||
if (path_has_prefix(rel, entry))
|
||||
return true; /* rel lives under a listed directory */
|
||||
if (path_has_prefix(entry, rel))
|
||||
return true; /* rel is an ancestor directory of a listed entry */
|
||||
}
|
||||
return false;
|
||||
}
|
||||
@@ -1,35 +0,0 @@
|
||||
#ifndef FILE_LIST_H
|
||||
#define FILE_LIST_H
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <stddef.h>
|
||||
|
||||
/* --files-from allow-set. The file lists source paths RELATIVE to the source
|
||||
* root. A listed regular file is transferred; a listed directory transfers its
|
||||
* whole subtree (FastSync's recursion is always on). Blank lines are ignored.
|
||||
*
|
||||
* Entries are normalized: leading "./" and duplicate "/" are removed, an entry
|
||||
* of "." means the whole tree, absolute entries and ".." traversal are
|
||||
* rejected at parse time. The set is immutable and shared read-only across
|
||||
* scanner worker threads.
|
||||
*/
|
||||
|
||||
typedef struct {
|
||||
char** entries; /* normalized rel paths; "" means the whole tree */
|
||||
int count;
|
||||
} FileListSet;
|
||||
|
||||
/* Load and validate a --files-from file. When `null_separated` (-0/--from0)
|
||||
* entries are delimited by NUL instead of newlines. Returns NULL with a message
|
||||
* in `err` on open/validation failure. An empty file yields an empty set
|
||||
* (nothing is transferred). */
|
||||
FileListSet* file_list_load(const char* path, bool null_separated, char* err, size_t err_size);
|
||||
void file_list_destroy(FileListSet* set);
|
||||
|
||||
/* True when `rel` (path relative to the source root, "" == root) is a listed
|
||||
* entry, lives under a listed directory, or is an ancestor directory of a
|
||||
* listed entry. Used to prune scanning: directories are descended only when
|
||||
* this returns true, files are transferred only when it returns true. */
|
||||
bool file_list_affects(const FileListSet* set, const char* rel);
|
||||
|
||||
#endif
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,97 +0,0 @@
|
||||
#ifndef FILE_RECEIVE_H
|
||||
#define FILE_RECEIVE_H
|
||||
|
||||
#include "config.h"
|
||||
#include "file_types.h"
|
||||
#include <stdbool.h>
|
||||
|
||||
/* Server-side file receive/save path. */
|
||||
|
||||
File* file_receive(const Config* config, int file_descriptor);
|
||||
File* file_receive_directory(int file_descriptor, const Config* config);
|
||||
File* file_receive_dir_time(int file_descriptor, const Config* config);
|
||||
File* file_receive_hardlink(int file_descriptor);
|
||||
File* file_receive_symlink(int file_descriptor, const Config* config);
|
||||
File* file_receive_special(int file_descriptor);
|
||||
bool file_special_rdev_valid(int32_t major, int32_t minor, mode_t mode);
|
||||
File* receive_incremental_check(int fd, const Config* config, bool* skipped);
|
||||
|
||||
/* P7 Wave D directory-time accumulator. The receiver collects the metadata of
|
||||
* every directory it creates/receives (STATUS_MKDIR with metadata and/or the
|
||||
* trailing STATUS_DIR_TIMES frame(s)) and applies the times only at the END of the
|
||||
* transfer, after all children have been written and after the delete /
|
||||
* --delay-updates phases have committed (writing or removing a child bumps the
|
||||
* parent's mtime). -O/--omit-dir-times skips the application entirely. The
|
||||
* list owns deep copies of the paths and metadata; freed on every path. */
|
||||
typedef struct {
|
||||
char** paths; /* owned, destination-relative wire paths */
|
||||
FileMetadata* entries; /* owned, parallel to paths */
|
||||
size_t count;
|
||||
size_t capacity;
|
||||
} DirTimeList;
|
||||
|
||||
void dir_time_list_init(DirTimeList* list);
|
||||
void dir_time_list_free(DirTimeList* list);
|
||||
/* Deep-copy one directory's path + metadata into the list. Returns false on
|
||||
* allocation failure (the caller fails the transfer). */
|
||||
bool dir_time_list_add(DirTimeList* list, const char* wire_path, const FileMetadata* metadata);
|
||||
/* Apply every accumulated directory's mtime (and atime when captured) beneath
|
||||
* `root_directory`, confined fd-relative. Best-effort per entry: an absent
|
||||
* directory (an empty/pruned source dir that was deliberately not created) or a
|
||||
* non-directory at the path is skipped QUIETLY, an unreachable one with a
|
||||
* warning, and never fatal. */
|
||||
void dir_time_list_apply(const DirTimeList* list, const char* root_directory);
|
||||
|
||||
/* A received delete-manifest frame: the keep-set (`keeps`, destination-relative
|
||||
paths the sender transferred/keeps) plus `protected`, destination-relative
|
||||
prefixes the sender asks the receiver never to delete (paths excluded on the
|
||||
source, protected at any depth). When --delete-excluded is given the sender
|
||||
transmits an empty protected list so excluded destination mirrors are treated
|
||||
as ordinary extras. With --delete-missing-args a third section (`missing`)
|
||||
carries the destination mirrors of explicitly-listed source entries that do
|
||||
not exist: each is an exact deletion request, independent of the ordinary
|
||||
extras walk (never blocked by the protected prefixes) and processed when the
|
||||
manifest is committed. */
|
||||
typedef struct DeleteManifest {
|
||||
ArrayList* keeps;
|
||||
ArrayList* protected;
|
||||
ArrayList* missing;
|
||||
} DeleteManifest;
|
||||
|
||||
void delete_manifest_free(DeleteManifest* manifest);
|
||||
/* Read a delete-manifest frame: keep count + keeps, then protected count +
|
||||
protected prefixes, then missing count + missing paths (self-delimiting; the
|
||||
leading STATUS_MANIFEST code has been consumed). Returns an owned
|
||||
DeleteManifest, or NULL after signalling STATUS_ERROR on a malformed frame. */
|
||||
DeleteManifest* receive_manifest_entries(int fd);
|
||||
/* Remove destination entries under config->receive_root_directory that are not
|
||||
in `manifest` (bounded, all-or-nothing walk; staging-dir, basis-dir and
|
||||
protected-prefix skips). `--max-delete` and `--force` are honored here. The
|
||||
caller decides WHEN to run it based on the negotiated delete timing. Returns
|
||||
false (and the transfer fails) when the deletion cannot be committed. */
|
||||
bool manifest_delete_extras(const Config* config, DeleteManifest* manifest);
|
||||
/* --delete-missing-args exact-path deletions: remove each destination mirror
|
||||
in `manifest->missing` (never blocked by the protected prefixes, staging dir
|
||||
and basis dirs excluded). A regular file/symlink is unlinked; an empty
|
||||
directory is removed; a NON-empty directory is removed recursively only when
|
||||
--delete or --force is in effect, otherwise it is left with a warning (rsync
|
||||
parity). A missing path is a no-op. Returns false only on a genuine
|
||||
confinement or I/O error (the run then fails); tolerated per-path cases are
|
||||
reported and skipped. */
|
||||
bool manifest_delete_missing_args(const Config* config, DeleteManifest* manifest);
|
||||
/* Run every deletion family the manifest carries: the --delete-missing-args
|
||||
exact-path deletions first (user requests are not blocked by exclusion
|
||||
protection), then the ordinary extras walk when --delete is active. Returns
|
||||
true when nothing to do or everything committed. */
|
||||
bool manifest_delete_all(const Config* config, DeleteManifest* manifest);
|
||||
|
||||
/* Outcome of a single file_save_to_disk operation. The receiver needs to
|
||||
distinguish "written" from "skipped" so --remove-source-files can be told
|
||||
which sources were actually stored. */
|
||||
typedef enum { FILE_SAVE_ERROR = 0, FILE_SAVE_WRITTEN = 1, FILE_SAVE_SKIPPED = 2 } FileSaveResult;
|
||||
|
||||
FileSaveResult file_save_to_disk_full(const char* root_directory, const File* file,
|
||||
const Config* config);
|
||||
bool file_save_to_disk(const char* root_directory, const File* file, const Config* config);
|
||||
|
||||
#endif
|
||||
@@ -1,181 +0,0 @@
|
||||
#include <errno.h>
|
||||
#include <fcntl.h>
|
||||
#include <limits.h>
|
||||
#include <poll.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/sendfile.h>
|
||||
#include <sys/stat.h>
|
||||
#include <time.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include "charset.h"
|
||||
#include "compression.h"
|
||||
#include "data.h"
|
||||
#include "file.h"
|
||||
#include "log.h"
|
||||
#include "metadata.h"
|
||||
#include "protocol.h"
|
||||
#include "xattr.h"
|
||||
|
||||
/* Transmit a device/special node (--devices / --specials) as a STATUS_SPECIAL
|
||||
* frame: the destination path, the metadata frame (whose mode's S_IFMT bits
|
||||
* carry the node kind) and the device rdev major/minor. The receiver validates
|
||||
* the kind and rdev and recreates the node (privilege-gating the mknod). */
|
||||
bool file_send_special(const File* file, int file_descriptor, bool use_metadata) {
|
||||
if (!file || !file_wire_path(file))
|
||||
return false;
|
||||
if (!send_status(file_descriptor, STATUS_SPECIAL))
|
||||
return false;
|
||||
if (!send_wire_str(file_descriptor, file_wire_path(file)))
|
||||
return false;
|
||||
if (use_metadata && !metadata_send(file_descriptor, file->metadata))
|
||||
return false;
|
||||
int32_t major = file->rdev_major;
|
||||
int32_t minor = file->rdev_minor;
|
||||
return send_n_data(file_descriptor, &major, sizeof(major)) &&
|
||||
send_n_data(file_descriptor, &minor, sizeof(minor));
|
||||
}
|
||||
|
||||
bool file_send_single_calls(File* file, int file_descriptor, bool use_metadata,
|
||||
int compression_level, bool send_path) {
|
||||
return file_send_single_calls_with_skip(file, file_descriptor, use_metadata, compression_level,
|
||||
send_path, NULL, -1, 0, false);
|
||||
}
|
||||
|
||||
bool file_send_single_calls_with_skip(File* file, int file_descriptor, bool use_metadata,
|
||||
int compression_level, bool send_path,
|
||||
char* const* skip_suffixes, int skip_count,
|
||||
int compression_threads, bool send_xattrs) {
|
||||
if (!file || !file->path || !file->data || (file->data->size != 0 && !file->data->data))
|
||||
return false;
|
||||
const Data* data_to_send = file->data;
|
||||
Data* compressed_data = NULL;
|
||||
if (compression_level > 0 &&
|
||||
!compression_should_skip_with_suffixes(file->path, skip_suffixes, skip_count)) {
|
||||
compressed_data =
|
||||
data_compress_with_threads(file->data, compression_level, compression_threads);
|
||||
if (compressed_data == NULL) {
|
||||
log_message(LOG_LEVEL_ERROR, "Failed to compress file data");
|
||||
return false;
|
||||
}
|
||||
data_to_send = compressed_data;
|
||||
}
|
||||
if (send_path && !send_wire_str(file_descriptor, file_wire_path(file))) {
|
||||
data_destroy(compressed_data);
|
||||
return false;
|
||||
}
|
||||
if (use_metadata && !metadata_send(file_descriptor, file->metadata)) {
|
||||
data_destroy(compressed_data);
|
||||
return false;
|
||||
}
|
||||
if (send_xattrs && !xattr_send(file_descriptor, file ? file->xattrs : NULL)) {
|
||||
data_destroy(compressed_data);
|
||||
return false;
|
||||
}
|
||||
if (!send_data(file_descriptor, data_to_send)) {
|
||||
data_destroy(compressed_data);
|
||||
return false;
|
||||
}
|
||||
data_destroy(compressed_data);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool file_send_sendfile(File* file, int file_descriptor, bool use_metadata, int compression_level,
|
||||
bool send_path) {
|
||||
return file_send_sendfile_with_skip(file, file_descriptor, use_metadata, compression_level,
|
||||
send_path, NULL, -1, 0, false);
|
||||
}
|
||||
|
||||
bool file_send_sendfile_with_skip(File* file, int file_descriptor, bool use_metadata,
|
||||
int compression_level, bool send_path, char* const* skip_suffixes,
|
||||
int skip_count, int compression_threads, bool send_xattrs) {
|
||||
if (!file || !file->path || !file->data)
|
||||
return false;
|
||||
if (compression_level > 0)
|
||||
return file_send_single_calls_with_skip(file, file_descriptor, use_metadata, compression_level,
|
||||
send_path, skip_suffixes, skip_count,
|
||||
compression_threads, send_xattrs);
|
||||
|
||||
if (send_path && !send_wire_str(file_descriptor, file_wire_path(file)))
|
||||
return false;
|
||||
if (use_metadata && !metadata_send(file_descriptor, file->metadata))
|
||||
return false;
|
||||
if (send_xattrs && !xattr_send(file_descriptor, file ? file->xattrs : NULL))
|
||||
return false;
|
||||
|
||||
int fd = file_open_for_read(file->path);
|
||||
if (fd == -1) {
|
||||
log_perror("Could not open file for sendfile");
|
||||
return false;
|
||||
}
|
||||
|
||||
unsigned long long file_size = file->data->size;
|
||||
struct stat source_stat;
|
||||
if (fstat(fd, &source_stat) != 0 || !S_ISREG(source_stat.st_mode) ||
|
||||
(unsigned long long)source_stat.st_size < file_size) {
|
||||
close(fd);
|
||||
return false;
|
||||
}
|
||||
if (!send_n_data(file_descriptor, &file_size, sizeof(unsigned long long))) {
|
||||
close(fd);
|
||||
return false;
|
||||
}
|
||||
|
||||
/* sendfile cannot encrypt TLS records. Keep the framing identical but
|
||||
route encrypted transfers through the deadline-aware IO layer. */
|
||||
if (io_get_ssl() != NULL) {
|
||||
unsigned char buffer[64 * 1024];
|
||||
unsigned long long remaining = file_size;
|
||||
bool ok = true;
|
||||
while (remaining > 0) {
|
||||
size_t want = remaining > sizeof(buffer) ? sizeof(buffer) : (size_t)remaining;
|
||||
ssize_t got = read(fd, buffer, want);
|
||||
if (got <= 0 || !send_n_data(file_descriptor, buffer, (size_t)got)) {
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
remaining -= (unsigned long long)got;
|
||||
}
|
||||
close(fd);
|
||||
return ok;
|
||||
}
|
||||
|
||||
off_t offset = 0;
|
||||
struct timespec deadline;
|
||||
clock_gettime(CLOCK_MONOTONIC, &deadline);
|
||||
deadline.tv_sec += 60;
|
||||
while ((unsigned long long)offset < file_size) {
|
||||
struct timespec now;
|
||||
clock_gettime(CLOCK_MONOTONIC, &now);
|
||||
long long remaining = (long long)(deadline.tv_sec - now.tv_sec) * 1000LL +
|
||||
(deadline.tv_nsec - now.tv_nsec) / 1000000LL;
|
||||
if (remaining <= 0) {
|
||||
close(fd);
|
||||
return false;
|
||||
}
|
||||
struct pollfd pfd = {.fd = file_descriptor, .events = POLLOUT};
|
||||
int timeout = remaining > INT_MAX ? INT_MAX : (int)remaining;
|
||||
int polled = poll(&pfd, 1, timeout);
|
||||
if (polled <= 0 || (pfd.revents & (POLLERR | POLLHUP | POLLNVAL))) {
|
||||
close(fd);
|
||||
return false;
|
||||
}
|
||||
ssize_t sent = sendfile(file_descriptor, fd, &offset, file_size - offset);
|
||||
if (sent == -1) {
|
||||
if (errno == EAGAIN || errno == EINTR)
|
||||
continue;
|
||||
log_perror("sendfile failed");
|
||||
close(fd);
|
||||
return false;
|
||||
}
|
||||
if (sent == 0) {
|
||||
close(fd);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
close(fd);
|
||||
return true;
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
#ifndef FILE_SEND_H
|
||||
#define FILE_SEND_H
|
||||
|
||||
#include "file_types.h"
|
||||
#include <stdbool.h>
|
||||
|
||||
/* Client-side file send path. */
|
||||
|
||||
bool file_send_special(const File* file, int file_descriptor, bool use_metadata);
|
||||
bool file_send_single_calls(File* file, int file_descriptor, bool use_metadata,
|
||||
int compression_level, bool send_path);
|
||||
bool file_send_single_calls_with_skip(File* file, int file_descriptor, bool use_metadata,
|
||||
int compression_level, bool send_path,
|
||||
char* const* skip_suffixes, int skip_count,
|
||||
int compression_threads, bool send_xattrs);
|
||||
bool file_send_sendfile(File* file, int file_descriptor, bool use_metadata, int compression_level,
|
||||
bool send_path);
|
||||
bool file_send_sendfile_with_skip(File* file, int file_descriptor, bool use_metadata,
|
||||
int compression_level, bool send_path, char* const* skip_suffixes,
|
||||
int skip_count, int compression_threads, bool send_xattrs);
|
||||
|
||||
#endif
|
||||
@@ -1,242 +0,0 @@
|
||||
#include <errno.h>
|
||||
#include <fcntl.h>
|
||||
#include <libgen.h>
|
||||
#include <limits.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include "file_store.h"
|
||||
#include "metadata.h"
|
||||
#include "utils.h"
|
||||
|
||||
static int authorized_root_fd = -1;
|
||||
static char* authorized_root_path;
|
||||
|
||||
static bool path_is_within_root(const char* root, const char* path) {
|
||||
size_t root_length = strlen(root);
|
||||
return strncmp(root, path, root_length) == 0 &&
|
||||
(path[root_length] == '\0' || path[root_length] == '/');
|
||||
}
|
||||
|
||||
bool file_store_set_authorized_root(int fd, const char* canonical_path) {
|
||||
char* new_path = canonical_path ? str_dup(canonical_path) : NULL;
|
||||
if (canonical_path && !new_path) {
|
||||
authorized_root_fd = -1;
|
||||
free(authorized_root_path);
|
||||
authorized_root_path = NULL;
|
||||
return false;
|
||||
}
|
||||
free(authorized_root_path);
|
||||
authorized_root_path = new_path;
|
||||
authorized_root_fd = fd;
|
||||
return true;
|
||||
}
|
||||
|
||||
int file_store_open_secure_parent(const char* path, char** leaf_out) {
|
||||
char* copy = str_dup(path);
|
||||
if (!copy)
|
||||
return -1;
|
||||
char* parent = dirname(copy);
|
||||
const char* slash = strrchr(path, '/');
|
||||
char* leaf = str_dup(slash ? slash + 1 : path);
|
||||
if (!leaf) {
|
||||
free(copy);
|
||||
return -1;
|
||||
}
|
||||
int fd;
|
||||
if (authorized_root_fd >= 0) {
|
||||
if (!authorized_root_path || path[0] != '/' ||
|
||||
!path_is_within_root(authorized_root_path, path)) {
|
||||
free(copy);
|
||||
free(leaf);
|
||||
return -1;
|
||||
}
|
||||
fd = dup(authorized_root_fd);
|
||||
if (fd < 0) {
|
||||
free(copy);
|
||||
free(leaf);
|
||||
return -1;
|
||||
}
|
||||
size_t root_length = strlen(authorized_root_path);
|
||||
char* relative = str_dup(path + root_length);
|
||||
if (!relative) {
|
||||
free(copy);
|
||||
free(leaf);
|
||||
close(fd);
|
||||
return -1;
|
||||
}
|
||||
free(copy);
|
||||
copy = relative;
|
||||
parent = dirname(copy);
|
||||
} else {
|
||||
fd = (parent[0] == '/') ? open("/", O_RDONLY | O_DIRECTORY | O_CLOEXEC)
|
||||
: open(".", O_RDONLY | O_DIRECTORY | O_CLOEXEC);
|
||||
}
|
||||
if (fd < 0) {
|
||||
free(copy);
|
||||
free(leaf);
|
||||
return -1;
|
||||
}
|
||||
char* save = NULL;
|
||||
char* component = strtok_r(parent, "/", &save);
|
||||
while (component) {
|
||||
if (strcmp(component, "..") == 0) {
|
||||
close(fd);
|
||||
free(copy);
|
||||
free(leaf);
|
||||
return -1;
|
||||
}
|
||||
if (strcmp(component, ".") != 0) {
|
||||
int next = openat(fd, component, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||
if (next < 0 && errno == ENOENT) {
|
||||
if (mkdirat(fd, component, 0755) == 0 || errno == EEXIST)
|
||||
next = openat(fd, component, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||
}
|
||||
if (next < 0) {
|
||||
close(fd);
|
||||
free(copy);
|
||||
free(leaf);
|
||||
return -1;
|
||||
}
|
||||
close(fd);
|
||||
fd = next;
|
||||
}
|
||||
component = strtok_r(NULL, "/", &save);
|
||||
}
|
||||
free(copy);
|
||||
*leaf_out = leaf;
|
||||
return fd;
|
||||
}
|
||||
|
||||
bool file_store_rename_secure(const char* old_path, const char* new_path) {
|
||||
char *old_leaf = NULL, *new_leaf = NULL;
|
||||
int old_parent = file_store_open_secure_parent(old_path, &old_leaf);
|
||||
int new_parent = file_store_open_secure_parent(new_path, &new_leaf);
|
||||
bool ok = old_parent >= 0 && new_parent >= 0 &&
|
||||
renameat(old_parent, old_leaf, new_parent, new_leaf) == 0;
|
||||
if (old_parent >= 0)
|
||||
close(old_parent);
|
||||
if (new_parent >= 0)
|
||||
close(new_parent);
|
||||
free(old_leaf);
|
||||
free(new_leaf);
|
||||
return ok;
|
||||
}
|
||||
|
||||
static bool write_all(int fd, const void* data, unsigned long long size) {
|
||||
const unsigned char* p = data;
|
||||
unsigned long long done = 0;
|
||||
while (done < size) {
|
||||
ssize_t n = write(fd, p + done, (size_t)(size - done));
|
||||
if (n < 0 && errno == EINTR)
|
||||
continue;
|
||||
if (n <= 0)
|
||||
return false;
|
||||
done += (unsigned long long)n;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/* A run of NUL bytes at least this long is emitted as a hole (lseek) rather
|
||||
* than written, so the resulting file is genuinely sparse on the filesystem. */
|
||||
#define SPARSE_HOLE_MIN 4096U
|
||||
|
||||
/* Sparse-aware writer (--sparse/-S). Walks `data`; any all-zero run of at
|
||||
* least SPARSE_HOLE_MIN bytes is skipped with lseek(SEEK_CUR) so the block is
|
||||
* never allocated (a real hole on the destination); every other byte is written
|
||||
* normally. The file is pre-sized with ftruncate by the callers before this
|
||||
* runs, so holes are guaranteed and the offset bookkeeping stays correct
|
||||
* (each lseek advances the fd offset exactly as a write of that many bytes
|
||||
* would). After the final run, ftruncate(size) guarantees the logical size is
|
||||
* exactly `size` even when the tail was a hole. The full file image is in
|
||||
* memory, so no wire change is needed. Returns false on I/O error. */
|
||||
bool file_store_write_sparse(int fd, const unsigned char* data, unsigned long long size) {
|
||||
unsigned long long i = 0;
|
||||
while (i < size) {
|
||||
if (data[i] == 0) {
|
||||
unsigned long long run_start = i;
|
||||
while (i < size && data[i] == 0)
|
||||
i++;
|
||||
unsigned long long run_len = i - run_start;
|
||||
if (run_len >= SPARSE_HOLE_MIN) {
|
||||
if (lseek(fd, (off_t)run_len, SEEK_CUR) < 0)
|
||||
return false;
|
||||
} else if (!write_all(fd, data + run_start, run_len)) {
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
unsigned long long run_start = i;
|
||||
while (i < size && data[i] != 0)
|
||||
i++;
|
||||
if (!write_all(fd, data + run_start, i - run_start))
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return ftruncate(fd, (off_t)size) == 0;
|
||||
}
|
||||
|
||||
bool file_store_write_secure(const char* path, const void* data, unsigned long long data_size,
|
||||
bool inplace, bool sparse, const FileMetadata* metadata,
|
||||
bool preserve_executability) {
|
||||
char* leaf = NULL;
|
||||
int dirfd = file_store_open_secure_parent(path, &leaf);
|
||||
if (dirfd < 0)
|
||||
return false;
|
||||
int fd = -1;
|
||||
bool ok = false;
|
||||
if (inplace) {
|
||||
fd = openat(dirfd, leaf, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC | O_NOFOLLOW, 0644);
|
||||
if (fd >= 0) {
|
||||
if (sparse && data_size > 0) {
|
||||
if (ftruncate(fd, (off_t)data_size) == 0)
|
||||
ok = file_store_write_sparse(fd, data, data_size);
|
||||
} else {
|
||||
ok = write_all(fd, data, data_size);
|
||||
}
|
||||
if (ok && metadata)
|
||||
ok = file_restore_metadata_fd(fd, metadata, preserve_executability);
|
||||
}
|
||||
} else {
|
||||
int tmp_size = snprintf(NULL, 0, ".%s.tmp.%ld.%u", leaf, (long)getpid(), 99U);
|
||||
if (tmp_size < 0) {
|
||||
close(dirfd);
|
||||
free(leaf);
|
||||
return false;
|
||||
}
|
||||
char* tmp = malloc((size_t)tmp_size + 1);
|
||||
if (!tmp) {
|
||||
close(dirfd);
|
||||
free(leaf);
|
||||
return false;
|
||||
}
|
||||
for (unsigned int i = 0; i < 100 && !ok; ++i) {
|
||||
snprintf(tmp, (size_t)tmp_size + 1, ".%s.tmp.%ld.%u", leaf, (long)getpid(), i);
|
||||
fd = openat(dirfd, tmp, O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC | O_NOFOLLOW, 0600);
|
||||
if (fd < 0)
|
||||
continue;
|
||||
if (sparse && data_size > 0)
|
||||
ok = ftruncate(fd, (off_t)data_size) == 0;
|
||||
if (ok || (!sparse || data_size == 0))
|
||||
ok = (sparse && data_size > 0)
|
||||
? file_store_write_sparse(fd, (const unsigned char*)data, data_size)
|
||||
: write_all(fd, data, data_size);
|
||||
if (ok && metadata)
|
||||
ok = file_restore_metadata_fd(fd, metadata, preserve_executability);
|
||||
if (close(fd) != 0)
|
||||
ok = false;
|
||||
fd = -1;
|
||||
if (ok && renameat(dirfd, tmp, dirfd, leaf) != 0)
|
||||
ok = false;
|
||||
if (!ok)
|
||||
unlinkat(dirfd, tmp, 0);
|
||||
}
|
||||
free(tmp);
|
||||
}
|
||||
if (fd >= 0)
|
||||
close(fd);
|
||||
close(dirfd);
|
||||
free(leaf);
|
||||
return ok;
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
#ifndef FILE_STORE_H
|
||||
#define FILE_STORE_H
|
||||
|
||||
#include "file.h"
|
||||
#include <stdbool.h>
|
||||
|
||||
bool file_store_set_authorized_root(int fd, const char* canonical_path);
|
||||
int file_store_open_secure_parent(const char* path, char** leaf_out);
|
||||
bool file_store_rename_secure(const char* old_path, const char* new_path);
|
||||
bool file_store_write_secure(const char* path, const void* data, unsigned long long data_size,
|
||||
bool inplace, bool sparse, const FileMetadata* metadata,
|
||||
bool preserve_executability);
|
||||
/* Sparse-aware write (--sparse/-S): every all-zero run of at least
|
||||
* SPARSE_HOLE_MIN bytes is skipped with lseek(SEEK_CUR) so it becomes a real
|
||||
* hole; every other byte is written. The caller pre-sizes the file with
|
||||
* ftruncate; this function also ftruncate()s to `size` at the end so a trailing
|
||||
* hole keeps the exact logical length. Shared by the file_store and file write
|
||||
* paths. Returns false on write/lseek/ftruncate error. */
|
||||
bool file_store_write_sparse(int fd, const unsigned char* data, unsigned long long size);
|
||||
|
||||
#endif
|
||||
@@ -1,97 +0,0 @@
|
||||
#ifndef FILE_TYPES_H
|
||||
#define FILE_TYPES_H
|
||||
|
||||
#include "data.h"
|
||||
#include "xattr.h"
|
||||
#include <stdbool.h>
|
||||
#include <sys/stat.h>
|
||||
|
||||
typedef enum { FILE_TYPE_REGULAR, FILE_TYPE_SYMLINK, FILE_TYPE_DIR } FileType;
|
||||
|
||||
typedef struct {
|
||||
mode_t mode;
|
||||
uid_t uid;
|
||||
gid_t gid;
|
||||
time_t mtime_sec;
|
||||
long mtime_nsec;
|
||||
/* Optional access time (-U/--atimes) and creation/birth time (-N/--crtimes),
|
||||
* appended for protocol 2.12.0. The SENDER sets the corresponding *_valid
|
||||
* flag only when the preserve option is active (and, for crtime, only when
|
||||
* the source platform exposed a birth time via statx STATX_BTIME). The wire
|
||||
* always carries the fields and the flags; a false flag tells the receiver to
|
||||
* ignore the value. */
|
||||
bool atime_valid;
|
||||
time_t atime_sec;
|
||||
long atime_nsec;
|
||||
bool crtime_valid;
|
||||
time_t crtime_sec;
|
||||
long crtime_nsec;
|
||||
} FileMetadata;
|
||||
|
||||
typedef struct {
|
||||
char* path;
|
||||
/* Sender-side override for the path transmitted on the wire (and used for
|
||||
* the delete manifest / change output). NULL means "use `path`". With
|
||||
* -R + --files-from this holds the entry's bare relative destination path,
|
||||
* while `path` stays the absolute local source path the client reads from.
|
||||
* Never populated on the receiver. */
|
||||
char* send_path;
|
||||
Data* data;
|
||||
FileMetadata* metadata;
|
||||
bool skip;
|
||||
/* True when this entry is an explicit directory entry (--dirs mode): the
|
||||
* receiver creates the directory instead of writing a regular file. */
|
||||
bool is_dir;
|
||||
/* Receiver-only (P7 Wave D): this is a STATUS_DIR_TIMES entry. It carries a
|
||||
* traversed source directory's metadata for DEFERRED application, but must
|
||||
* NEVER create the directory: the scanner captures every traversed directory
|
||||
* (including empty ones whose parents no child write created), so creation
|
||||
* would resurrect the empty dirs that FastSync deliberately never transfers.
|
||||
* file_save_to_disk_full short-circuits such an entry as FILE_SAVE_SKIPPED,
|
||||
* and the sink still accumulates the metadata into its DirTimeList. */
|
||||
bool dir_time_only;
|
||||
/* Receiver-only, --link-dest: when set, install the destination entry as a
|
||||
* hard link to this absolute (root-confined) path instead of writing
|
||||
* `data`. The matching code has already verified the link target's content
|
||||
* equals the incoming file, and `data` is kept as the cross-filesystem
|
||||
* fallback (a local copy) if the hard link cannot be created. */
|
||||
char* basis_link;
|
||||
/* --hard-links (-H), sender + receiver wire state. link_group is a run-local
|
||||
* id shared by every member of one source inode (0 = not part of a group).
|
||||
* The FIRST member (link_first == true) carries its data on the wire and is
|
||||
* written normally; every sibling (link_first == false) carries NO data and
|
||||
* hardlink_target holds the first member's wire path so the receiver can link
|
||||
* to (or copy from) the already-installed first member. */
|
||||
int link_group;
|
||||
bool link_first;
|
||||
char* hardlink_target;
|
||||
/* Symlink-type entry (-l/--links, or -k/--copy-dirlinks' keep-as-symlink
|
||||
* branch). When true, `symlink_target` holds the (sender-munged, if
|
||||
* --munge-links) target string that is carried on the wire; the receiver
|
||||
* creates a symlink to (an unmunged) target instead of writing regular-file
|
||||
* data. `data` is empty for a symlink entry. Sender + receiver state. */
|
||||
bool is_symlink;
|
||||
char* symlink_target;
|
||||
/* Phase 4 special/devices: when `is_special` is true this entry is a device
|
||||
* or special node to be RECREATED on the destination (mknod/mkfifo) rather
|
||||
* than written from `data`. The concrete node kind is derived from the
|
||||
* metadata mode's S_IFMT bits (receiver-validated), and rdev_major/minor
|
||||
* carry the device major/minor numbers for char/block devices. CROSSES the
|
||||
* wire (protocol 2.13.0). */
|
||||
bool is_special;
|
||||
int32_t rdev_major;
|
||||
int32_t rdev_minor;
|
||||
/* Phase-4 xattrs (-X/--xattrs, -A/--acls). Sender: captured from the source
|
||||
* file when use_xattrs is set; transmitted in the per-file metadata frame.
|
||||
* Receiver: parsed off the wire, attached here, and applied fd-relative on
|
||||
* the written file. NULL/0 == the file carries no xattrs. */
|
||||
FileXattrList* xattrs;
|
||||
} File;
|
||||
|
||||
/* The path that should be sent on the wire and used for the receiver-side
|
||||
* destination layout (see send_path). */
|
||||
static inline const char* file_wire_path(const File* file) {
|
||||
return file && file->send_path ? file->send_path : (file ? file->path : NULL);
|
||||
}
|
||||
|
||||
#endif
|
||||
@@ -1,427 +0,0 @@
|
||||
#include "filter.h"
|
||||
#include "log.h"
|
||||
#include "utils.h"
|
||||
#include <errno.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
/* ---- Single rule parsing ---- */
|
||||
|
||||
static bool rule_text_is_unsupported_word(const char* p, size_t len) {
|
||||
static const char* const words[] = {"merge", "dir-merge", "hide", "show",
|
||||
"protect", "risk", "clear"};
|
||||
for (size_t i = 0; i < sizeof(words) / sizeof(words[0]); i++) {
|
||||
size_t wl = strlen(words[i]);
|
||||
if (len == wl && strncmp(p, words[i], wl) == 0)
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/* rsync include/exclude rule modifiers we do NOT implement. A rule whose +/- is
|
||||
* immediately followed by one of these is rejected instead of being silently
|
||||
* parsed as a literal pattern. */
|
||||
static bool is_unsupported_rule_modifier(char c) {
|
||||
return c == '!' || c == 'C' || c == 's' || c == 'r' || c == 'p' || c == 'x';
|
||||
}
|
||||
|
||||
FilterRule* filter_rule_parse(const char* line, char* err, size_t err_size) {
|
||||
if (err && err_size > 0)
|
||||
err[0] = '\0';
|
||||
if (!line)
|
||||
return NULL;
|
||||
char* text = str_dup(line);
|
||||
if (!text) {
|
||||
if (err)
|
||||
snprintf(err, err_size, "memory allocation failed");
|
||||
return NULL;
|
||||
}
|
||||
size_t len = strlen(text);
|
||||
while (len > 0 && (text[len - 1] == '\n' || text[len - 1] == '\r'))
|
||||
text[--len] = '\0';
|
||||
|
||||
const char* p = text;
|
||||
while (*p == ' ' || *p == '\t')
|
||||
p++;
|
||||
if (*p == '\0') {
|
||||
snprintf(err, err_size, "empty filter rule");
|
||||
free(text);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
FilterAction action = FILTER_ACTION_EXCLUDE;
|
||||
if (*p == '+' || *p == '-') {
|
||||
action = *p == '+' ? FILTER_ACTION_INCLUDE : FILTER_ACTION_EXCLUDE;
|
||||
p++;
|
||||
/* rsync attaches rule modifiers directly to the +/- (e.g. "-s foo"). Only
|
||||
* the '/' anchor modifier is supported; anything else is a clear error
|
||||
* rather than a silently-ignored literal. */
|
||||
if (*p != ' ' && *p != '\t' && *p != '\0' && is_unsupported_rule_modifier(*p)) {
|
||||
snprintf(err, err_size,
|
||||
"filter rule modifier '%c' is not supported (only the '/' anchor after +/- "
|
||||
"is implemented; put a space between +/- and the pattern)",
|
||||
*p);
|
||||
free(text);
|
||||
return NULL;
|
||||
}
|
||||
while (*p == ' ' || *p == '\t')
|
||||
p++;
|
||||
} else {
|
||||
/* ':' (dir-merge) and '.' (merge) are rsync filter-rule shorthands. At the
|
||||
* start of a rule they mean "merge this file", so reject them instead of
|
||||
* silently turning them into inert exclude patterns. */
|
||||
if (*p == ':' || *p == '.' || *p == '!') {
|
||||
snprintf(err, err_size,
|
||||
"filter rule starting with '%c' is not supported (merge/dir-merge/list-clear "
|
||||
"shorthands are not implemented; use +/- include/exclude rules)",
|
||||
*p);
|
||||
free(text);
|
||||
return NULL;
|
||||
}
|
||||
const char* sp = p;
|
||||
while (*sp != '\0' && *sp != ' ' && *sp != '\t')
|
||||
sp++;
|
||||
size_t word_len = (size_t)(sp - p);
|
||||
if (rule_text_is_unsupported_word(p, word_len)) {
|
||||
snprintf(err, err_size,
|
||||
"'%.*s' filter directives are not supported (only +/- include/exclude rules "
|
||||
"with an optional '/' anchor and trailing '/' dir marker)",
|
||||
(int)word_len, p);
|
||||
free(text);
|
||||
return NULL;
|
||||
}
|
||||
if (word_len == strlen("include") && strncmp(p, "include", word_len) == 0) {
|
||||
action = FILTER_ACTION_INCLUDE;
|
||||
p = sp;
|
||||
} else if (word_len == strlen("exclude") && strncmp(p, "exclude", word_len) == 0) {
|
||||
action = FILTER_ACTION_EXCLUDE;
|
||||
p = sp;
|
||||
}
|
||||
while (*p == ' ' || *p == '\t')
|
||||
p++;
|
||||
}
|
||||
|
||||
if (*p == '\0') {
|
||||
snprintf(err, err_size, "filter rule has no pattern");
|
||||
free(text);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* A pattern beginning with '/' is anchored (either as "-/foo" or "- /foo"). */
|
||||
bool anchored = false;
|
||||
if (*p == '/') {
|
||||
anchored = true;
|
||||
p++;
|
||||
while (*p == ' ' || *p == '\t')
|
||||
p++;
|
||||
}
|
||||
if (*p == '\0') {
|
||||
snprintf(err, err_size, "filter rule has no pattern after '/' anchor");
|
||||
free(text);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* Pattern runs to the end of the rule; a single trailing '/' marks dir-only. */
|
||||
size_t pat_len = strlen(p);
|
||||
bool dir_only = false;
|
||||
if (pat_len > 1 && p[pat_len - 1] == '/') {
|
||||
dir_only = true;
|
||||
pat_len--;
|
||||
} else if (pat_len == 1 && p[0] == '/') {
|
||||
/* "//" anchored with nothing after: meaningless. */
|
||||
snprintf(err, err_size, "filter rule has no pattern");
|
||||
free(text);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
FilterRule* rule = calloc(1, sizeof(FilterRule));
|
||||
if (!rule) {
|
||||
snprintf(err, err_size, "memory allocation failed");
|
||||
free(text);
|
||||
return NULL;
|
||||
}
|
||||
rule->pattern = malloc(pat_len + 1);
|
||||
if (!rule->pattern) {
|
||||
free(rule);
|
||||
snprintf(err, err_size, "memory allocation failed");
|
||||
free(text);
|
||||
return NULL;
|
||||
}
|
||||
memcpy(rule->pattern, p, pat_len);
|
||||
rule->pattern[pat_len] = '\0';
|
||||
rule->action = action;
|
||||
rule->anchored = anchored;
|
||||
rule->dir_only = dir_only;
|
||||
rule->owner = NULL;
|
||||
free(text);
|
||||
return rule;
|
||||
}
|
||||
|
||||
void filter_rule_free(FilterRule* rule) {
|
||||
if (!rule)
|
||||
return;
|
||||
free(rule->pattern);
|
||||
free(rule->owner);
|
||||
free(rule);
|
||||
}
|
||||
|
||||
/* ---- Ordered rule lists ---- */
|
||||
|
||||
FilterRuleList* filter_rule_list_create(void) {
|
||||
return calloc(1, sizeof(FilterRuleList));
|
||||
}
|
||||
|
||||
bool filter_rule_list_add(FilterRuleList* list, FilterRule* rule) {
|
||||
if (!list || !rule)
|
||||
return false;
|
||||
if (list->count == list->capacity) {
|
||||
int new_cap = list->capacity > 0 ? list->capacity * 2 : 8;
|
||||
FilterRule** grown = realloc(list->items, (size_t)new_cap * sizeof(FilterRule*));
|
||||
if (!grown)
|
||||
return false;
|
||||
list->items = grown;
|
||||
list->capacity = new_cap;
|
||||
}
|
||||
list->items[list->count++] = rule;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool filter_rule_list_parse_append(FilterRuleList* list, const char* line, char* err,
|
||||
size_t err_size) {
|
||||
FilterRule* rule = filter_rule_parse(line, err, err_size);
|
||||
if (!rule)
|
||||
return false;
|
||||
if (!filter_rule_list_add(list, rule)) {
|
||||
filter_rule_free(rule);
|
||||
snprintf(err, err_size, "memory allocation failed");
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void filter_rule_list_free(FilterRuleList* list) {
|
||||
if (!list)
|
||||
return;
|
||||
for (int i = 0; i < list->count; i++)
|
||||
filter_rule_free(list->items[i]);
|
||||
free(list->items);
|
||||
free(list);
|
||||
}
|
||||
|
||||
static bool set_rule_owner(FilterRule* rule, const char* owner) {
|
||||
char* dup = str_dup(owner ? owner : "");
|
||||
if (!dup)
|
||||
return false;
|
||||
free(rule->owner);
|
||||
rule->owner = dup;
|
||||
return true;
|
||||
}
|
||||
|
||||
/* ---- CVS default excludes (-C) ---- */
|
||||
|
||||
typedef struct {
|
||||
const char* pattern;
|
||||
bool dir_only;
|
||||
} CvsDefaultRule;
|
||||
|
||||
static const CvsDefaultRule CVS_DEFAULTS[] = {
|
||||
{"RCS", false}, {"SCCS", false}, {"CVS", false}, {"CVS.adm", false},
|
||||
{"RCSLOG", false}, {"cvslog.*", false}, {"tags", false}, {"TAGS", false},
|
||||
{".make.state", false}, {".nse_depinfo", false}, {"*~", false}, {"#*", false},
|
||||
{".#*", false}, {",*", false}, {"_$*", false}, {"*$", false},
|
||||
{"*.old", false}, {"*.bak", false}, {"*.BAK", false}, {"*.orig", false},
|
||||
{"*.rej", false}, {".del-*", false}, {"*.a", false}, {"*.olb", false},
|
||||
{"*.o", false}, {"*.obj", false}, {"*.so", false}, {"*.exe", false},
|
||||
{"*.Z", false}, {"*.elc", false}, {"*.ln", false}, {"core", false},
|
||||
{".svn/", true}, {".git/", true}, {".hg/", true}, {".bzr/", true},
|
||||
};
|
||||
|
||||
static bool cvs_rule_list_append(FilterRuleList* list) {
|
||||
for (size_t i = 0; i < sizeof(CVS_DEFAULTS) / sizeof(CVS_DEFAULTS[0]); i++) {
|
||||
FilterRule* rule = calloc(1, sizeof(FilterRule));
|
||||
if (!rule)
|
||||
return false;
|
||||
rule->action = FILTER_ACTION_EXCLUDE;
|
||||
rule->dir_only = CVS_DEFAULTS[i].dir_only;
|
||||
size_t plen = strlen(CVS_DEFAULTS[i].pattern);
|
||||
if (rule->dir_only && plen > 0 && CVS_DEFAULTS[i].pattern[plen - 1] == '/')
|
||||
plen--; /* keep the cleaned pattern, matching filter_rule_parse */
|
||||
rule->pattern = malloc(plen + 1);
|
||||
if (!rule->pattern) {
|
||||
free(rule);
|
||||
return false;
|
||||
}
|
||||
memcpy(rule->pattern, CVS_DEFAULTS[i].pattern, plen);
|
||||
rule->pattern[plen] = '\0';
|
||||
if (!set_rule_owner(rule, "")) {
|
||||
filter_rule_free(rule);
|
||||
return false;
|
||||
}
|
||||
if (!filter_rule_list_add(list, rule)) {
|
||||
filter_rule_free(rule);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
FilterRuleList* filter_base_build(const char* const* rule_texts, int rule_count, bool cvs_exclude,
|
||||
char* err, size_t err_size) {
|
||||
if (err && err_size > 0)
|
||||
err[0] = '\0';
|
||||
FilterRuleList* list = filter_rule_list_create();
|
||||
if (!list) {
|
||||
snprintf(err, err_size, "memory allocation failed");
|
||||
return NULL;
|
||||
}
|
||||
for (int i = 0; i < rule_count; i++) {
|
||||
if (!rule_texts || !rule_texts[i])
|
||||
continue;
|
||||
FilterRule* rule = filter_rule_parse(rule_texts[i], err, err_size);
|
||||
if (!rule) {
|
||||
filter_rule_list_free(list);
|
||||
return NULL;
|
||||
}
|
||||
if (!set_rule_owner(rule, "")) {
|
||||
filter_rule_free(rule);
|
||||
filter_rule_list_free(list);
|
||||
snprintf(err, err_size, "memory allocation failed");
|
||||
return NULL;
|
||||
}
|
||||
if (!filter_rule_list_add(list, rule)) {
|
||||
filter_rule_free(rule);
|
||||
filter_rule_list_free(list);
|
||||
snprintf(err, err_size, "memory allocation failed");
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
if (cvs_exclude && !cvs_rule_list_append(list)) {
|
||||
filter_rule_list_free(list);
|
||||
snprintf(err, err_size, "memory allocation failed");
|
||||
return NULL;
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
/* ---- Per-directory .rsync-filter files ---- */
|
||||
|
||||
FilterRuleList* filter_file_read(const char* dir_path, const char* owner_rel, bool* exists,
|
||||
char* err, size_t err_size) {
|
||||
if (err && err_size > 0)
|
||||
err[0] = '\0';
|
||||
if (exists)
|
||||
*exists = false;
|
||||
char* filter_path = path_cat(dir_path, ".rsync-filter");
|
||||
if (!filter_path) {
|
||||
snprintf(err, err_size, "memory allocation failed");
|
||||
return NULL;
|
||||
}
|
||||
FILE* fp = fopen(filter_path, "r");
|
||||
free(filter_path);
|
||||
if (!fp) {
|
||||
if (errno == ENOENT || errno == ENOTDIR)
|
||||
return filter_rule_list_create();
|
||||
log_message(LOG_LEVEL_WARNING, "Could not read .rsync-filter in %s: %s", dir_path,
|
||||
strerror(errno));
|
||||
return filter_rule_list_create();
|
||||
}
|
||||
if (exists)
|
||||
*exists = true;
|
||||
FilterRuleList* list = filter_rule_list_create();
|
||||
if (!list) {
|
||||
fclose(fp);
|
||||
snprintf(err, err_size, "memory allocation failed");
|
||||
return NULL;
|
||||
}
|
||||
char* line = NULL;
|
||||
size_t line_cap = 0;
|
||||
ssize_t n;
|
||||
bool ok = true;
|
||||
while ((n = getline(&line, &line_cap, fp)) != -1) {
|
||||
const char* p = line;
|
||||
while (*p == ' ' || *p == '\t')
|
||||
p++;
|
||||
if (*p == '\0' || *p == '\n' || *p == '\r' || *p == '#')
|
||||
continue;
|
||||
FilterRule* rule = filter_rule_parse(p, err, err_size);
|
||||
if (!rule) {
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
if (!set_rule_owner(rule, owner_rel)) {
|
||||
filter_rule_free(rule);
|
||||
snprintf(err, err_size, "memory allocation failed");
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
if (!filter_rule_list_add(list, rule)) {
|
||||
filter_rule_free(rule);
|
||||
snprintf(err, err_size, "memory allocation failed");
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
free(line);
|
||||
fclose(fp);
|
||||
if (!ok) {
|
||||
filter_rule_list_free(list);
|
||||
return NULL;
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
/* ---- Rule matching ---- */
|
||||
|
||||
/* Match a pattern that contains '/' (non-anchored) against the end of the
|
||||
* relative path, starting at any path-component boundary. */
|
||||
static bool glob_suffix_match(const char* pattern, const char* str) {
|
||||
if (glob_match(pattern, str))
|
||||
return true;
|
||||
for (const char* slash = strchr(str, '/'); slash; slash = strchr(slash + 1, '/')) {
|
||||
if (glob_match(pattern, slash + 1))
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static FilterAction rule_matches(const FilterRule* rule, const char* rel_path, const char* leaf,
|
||||
bool is_dir) {
|
||||
if (!rule || !rule->pattern)
|
||||
return FILTER_ACTION_NONE;
|
||||
if (rule->dir_only && !is_dir)
|
||||
return FILTER_ACTION_NONE;
|
||||
/* A rule applies only to entries below its owner directory. */
|
||||
const char* rel2 = rel_path;
|
||||
if (rule->owner && rule->owner[0] != '\0') {
|
||||
size_t owner_len = strlen(rule->owner);
|
||||
if (strncmp(rule->owner, rel_path, owner_len) != 0)
|
||||
return FILTER_ACTION_NONE;
|
||||
if (rel_path[owner_len] != '/')
|
||||
return FILTER_ACTION_NONE;
|
||||
rel2 = rel_path + owner_len + 1;
|
||||
}
|
||||
if (rel2[0] == '\0')
|
||||
return FILTER_ACTION_NONE;
|
||||
bool matched;
|
||||
if (rule->anchored) {
|
||||
matched = glob_match(rule->pattern, rel2);
|
||||
} else if (strchr(rule->pattern, '/') != NULL) {
|
||||
matched = glob_suffix_match(rule->pattern, rel2);
|
||||
} else {
|
||||
matched = glob_match(rule->pattern, leaf);
|
||||
}
|
||||
return matched ? rule->action : FILTER_ACTION_NONE;
|
||||
}
|
||||
|
||||
FilterAction filter_rules_apply(const FilterRuleList* list, const char* rel_path, const char* leaf,
|
||||
bool is_dir) {
|
||||
if (!list)
|
||||
return FILTER_ACTION_NONE;
|
||||
for (int i = 0; i < list->count; i++) {
|
||||
FilterAction action = rule_matches(list->items[i], rel_path, leaf, is_dir);
|
||||
if (action != FILTER_ACTION_NONE)
|
||||
return action;
|
||||
}
|
||||
return FILTER_ACTION_NONE;
|
||||
}
|
||||
@@ -1,83 +0,0 @@
|
||||
#ifndef FILTER_H
|
||||
#define FILTER_H
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <stddef.h>
|
||||
|
||||
/* rsync-style filter rule engine (client-side file selection).
|
||||
*
|
||||
* Supported rule syntax (documented subset):
|
||||
* [+|-] [anchored '/' prefix] pattern [trailing '/' for dir-only]
|
||||
*
|
||||
* "+ PATTERN" include rule (first match wins)
|
||||
* "- PATTERN" exclude rule
|
||||
* "PATTERN" implicit exclude rule (rsync default)
|
||||
* "include PATTERN" / "exclude PATTERN" word forms
|
||||
* leading '/' after the +/- anchors the pattern to its owner directory
|
||||
* (the transfer root for command-line/-C rules, the directory that
|
||||
* contains a .rsync-filter file for per-directory rules)
|
||||
* a trailing '/' makes the rule match directories only
|
||||
*
|
||||
* Rejected explicitly (no silent no-ops): the rsync merge/dir-merge/list-clear
|
||||
* shorthands written as a rule that starts with ':' or '.' or '!', the
|
||||
* merge/dir-merge/hide/show/protect/risk/clear words, and every include/exclude
|
||||
* rule modifier other than '/' (! C s r p x). The pattern must be separated
|
||||
* from +/- by a space (or a single '/' anchor), exactly like rsync's
|
||||
* "-s foo"/"-p ..." modifier syntax is refused.
|
||||
*/
|
||||
|
||||
typedef enum {
|
||||
FILTER_ACTION_NONE = 0, /* no rule matched */
|
||||
FILTER_ACTION_EXCLUDE = -1,
|
||||
FILTER_ACTION_INCLUDE = 1
|
||||
} FilterAction;
|
||||
|
||||
typedef struct {
|
||||
FilterAction action;
|
||||
bool anchored; /* pattern anchored to the rule's owner directory */
|
||||
bool dir_only; /* pattern had a trailing '/': matches directories only */
|
||||
char* owner; /* owning directory rel path ("" == transfer root) */
|
||||
char* pattern; /* cleaned glob pattern (no leading '/', no trailing '/') */
|
||||
} FilterRule;
|
||||
|
||||
typedef struct {
|
||||
FilterRule** items; /* owned array of rule pointers */
|
||||
int count;
|
||||
int capacity;
|
||||
} FilterRuleList;
|
||||
|
||||
/* Parse a single filter-rule line (no trailing newline required). Returns an
|
||||
* owned rule, or NULL on unsupported/invalid syntax with a message in `err`. */
|
||||
FilterRule* filter_rule_parse(const char* line, char* err, size_t err_size);
|
||||
void filter_rule_free(FilterRule* rule);
|
||||
|
||||
FilterRuleList* filter_rule_list_create(void);
|
||||
/* Append a fully-parsed rule (takes ownership). Returns false on OOM. */
|
||||
bool filter_rule_list_add(FilterRuleList* list, FilterRule* rule);
|
||||
/* Parse `line` and append it. Returns false and fills `err` on bad syntax. */
|
||||
bool filter_rule_list_parse_append(FilterRuleList* list, const char* line, char* err,
|
||||
size_t err_size);
|
||||
void filter_rule_list_free(FilterRuleList* list);
|
||||
|
||||
/* Build the command-line filter set: `rule_texts` (--filter=RULE in the order
|
||||
* given, 0..rule_count) followed by the -C CVS default excludes when
|
||||
* cvs_exclude is true. All rules are owned by "" (the transfer root).
|
||||
* Returns NULL on unsupported rule text (message in `err`). */
|
||||
FilterRuleList* filter_base_build(const char* const* rule_texts, int rule_count, bool cvs_exclude,
|
||||
char* err, size_t err_size);
|
||||
|
||||
/* Read "<dir_path>/.rsync-filter" and return its rules, each owned by
|
||||
* `owner_rel`. A missing file yields an empty list with *exists=false; an
|
||||
* unreadable file is treated as missing. Returns NULL only on parse or
|
||||
* allocation failure (message in `err`). */
|
||||
FilterRuleList* filter_file_read(const char* dir_path, const char* owner_rel, bool* exists,
|
||||
char* err, size_t err_size);
|
||||
|
||||
/* Evaluate an entry against one ordered rule list. Returns FILTER_ACTION_NONE
|
||||
* when no rule matched, otherwise the first matching rule's action.
|
||||
* `rel_path` is the entry's path relative to the transfer root ("" == root),
|
||||
* `leaf` its final name, `is_dir` whether it is a directory. */
|
||||
FilterAction filter_rules_apply(const FilterRuleList* list, const char* rel_path, const char* leaf,
|
||||
bool is_dir);
|
||||
|
||||
#endif
|
||||
@@ -1,127 +0,0 @@
|
||||
#include "hardlink.h"
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "log.h"
|
||||
#include "utils.h"
|
||||
|
||||
/* ---- Sender-side detection table ---- */
|
||||
|
||||
HardLinkTable* hardlink_table_create(void) {
|
||||
HardLinkTable* table = calloc(1, sizeof(HardLinkTable));
|
||||
if (!table)
|
||||
return NULL;
|
||||
if (mtx_init(&table->mutex, mtx_plain) != thrd_success) {
|
||||
free(table);
|
||||
return NULL;
|
||||
}
|
||||
table->next_gid = 1;
|
||||
return table;
|
||||
}
|
||||
|
||||
static void hardlink_item_destroy(HardLinkItem* item) {
|
||||
if (!item)
|
||||
return;
|
||||
free(item->first_path);
|
||||
item->first_path = NULL;
|
||||
}
|
||||
|
||||
void hardlink_table_destroy(HardLinkTable* table) {
|
||||
if (!table)
|
||||
return;
|
||||
for (size_t i = 0; i < table->count; i++)
|
||||
hardlink_item_destroy(&table->items[i]);
|
||||
free(table->items);
|
||||
table->items = NULL;
|
||||
table->count = 0;
|
||||
table->capacity = 0;
|
||||
mtx_destroy(&table->mutex);
|
||||
free(table);
|
||||
}
|
||||
|
||||
static HardLinkItem* hardlink_table_find_locked(HardLinkTable* table, dev_t dev, ino_t ino) {
|
||||
for (size_t i = 0; i < table->count; i++) {
|
||||
if (table->items[i].dev == dev && table->items[i].ino == ino)
|
||||
return &table->items[i];
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static bool hardlink_table_add_locked(HardLinkTable* table, dev_t dev, ino_t ino, const char* path,
|
||||
int gid, HardLinkItem** out) {
|
||||
if (table->count == table->capacity) {
|
||||
size_t new_capacity = table->capacity == 0 ? 8 : table->capacity * 2;
|
||||
if (new_capacity < table->capacity)
|
||||
return false;
|
||||
HardLinkItem* grown = realloc(table->items, new_capacity * sizeof(HardLinkItem));
|
||||
if (!grown)
|
||||
return false;
|
||||
table->items = grown;
|
||||
table->capacity = new_capacity;
|
||||
}
|
||||
HardLinkItem* item = &table->items[table->count];
|
||||
char* dup = str_dup(path);
|
||||
if (!dup)
|
||||
return false;
|
||||
memset(item, 0, sizeof(*item));
|
||||
item->dev = dev;
|
||||
item->ino = ino;
|
||||
item->gid = gid;
|
||||
item->first_path = dup;
|
||||
table->count++;
|
||||
*out = item;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool hardlink_table_assign(HardLinkTable* table, const char* wire_path, dev_t dev, ino_t ino,
|
||||
int* gid, bool* is_first, char** first_path_out) {
|
||||
if (!table || !wire_path || !gid || !is_first || !first_path_out)
|
||||
return false;
|
||||
if (mtx_lock(&table->mutex) != thrd_success)
|
||||
return false;
|
||||
bool ok = true;
|
||||
const HardLinkItem* item = hardlink_table_find_locked(table, dev, ino);
|
||||
int next_gid;
|
||||
if (item) {
|
||||
*is_first = false;
|
||||
char* dup = str_dup(item->first_path);
|
||||
if (!dup) {
|
||||
ok = false;
|
||||
} else {
|
||||
*gid = item->gid;
|
||||
*first_path_out = dup;
|
||||
}
|
||||
next_gid = -1;
|
||||
} else {
|
||||
if (table->next_gid <= 0) {
|
||||
ok = false;
|
||||
next_gid = -1;
|
||||
} else {
|
||||
next_gid = table->next_gid;
|
||||
HardLinkItem* created = NULL;
|
||||
if (!hardlink_table_add_locked(table, dev, ino, wire_path, next_gid, &created)) {
|
||||
ok = false;
|
||||
} else {
|
||||
char* dup = str_dup(wire_path);
|
||||
if (!dup) {
|
||||
hardlink_item_destroy(created);
|
||||
table->count--;
|
||||
ok = false;
|
||||
} else {
|
||||
*is_first = true;
|
||||
*gid = next_gid;
|
||||
*first_path_out = dup;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (ok && next_gid > 0)
|
||||
table->next_gid++;
|
||||
mtx_unlock(&table->mutex);
|
||||
if (!ok) {
|
||||
log_message(LOG_LEVEL_ERROR, "memory allocation failed while detecting hard links");
|
||||
}
|
||||
return ok;
|
||||
}
|
||||
@@ -1,66 +0,0 @@
|
||||
#ifndef HARDLINK_H
|
||||
#define HARDLINK_H
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <stddef.h>
|
||||
#include <sys/types.h>
|
||||
#include <threads.h>
|
||||
|
||||
/*
|
||||
* --hard-links / -H support.
|
||||
*
|
||||
* Sender side: a HardLinkTable detects regular files on the source that share
|
||||
* an (st_dev, st_ino) identity (a `cp -al`-style hard-linked tree) and assigns
|
||||
* each distinct inode a stable, run-local link-group id. The first member
|
||||
* encountered carries the file data; every later member is marked as a sibling
|
||||
* (no data payload) that the receiver creates as a hard link to the first
|
||||
* member's destination file. Grouping is scoped by st_dev so inode reuse
|
||||
* across different filesystems is never conflated. The table is mutex-guarded
|
||||
* so the parallel (multi-threaded) scanner COULD share one instance across its
|
||||
* worker threads; the first-thread-to-call designates the data-carrying member,
|
||||
* which is safe because a hard-link group's members are byte-identical. (In
|
||||
* practice the sender forces the sequential scanner whenever -H is on; the
|
||||
* mutex guards the shared table for any path that supplies one.)
|
||||
*
|
||||
* ORDERING (why there is no receiver-side handshake): the receiver stores every
|
||||
* file - including a hard-link group's first member - through a SINGLE writer
|
||||
* thread draining a single FIFO queue driven by a single receive thread, so
|
||||
* wire order == write order and every sibling is processed AFTER its group's
|
||||
* first member. The sender additionally forces the sequential scanner with -H
|
||||
* so the first-member frame always precedes its siblings on the wire. Sibling
|
||||
* install therefore needs no present/wait registry: it hard-links to the first
|
||||
* member (or copies it) knowing that path is already installed - or that, if
|
||||
* the first member was skipped (already up to date), its destination still
|
||||
* exists. This guarantee is REQUIRED; do not introduce a concurrent
|
||||
* multi-writer receiver for -H without re-adding an ordering mechanism.
|
||||
*/
|
||||
|
||||
typedef struct HardLinkItem {
|
||||
dev_t dev;
|
||||
ino_t ino;
|
||||
int gid;
|
||||
char* first_path; /* wire path of the group's data-carrying first member */
|
||||
} HardLinkItem;
|
||||
|
||||
typedef struct HardLinkTable {
|
||||
mtx_t mutex;
|
||||
HardLinkItem* items;
|
||||
size_t count;
|
||||
size_t capacity;
|
||||
int next_gid;
|
||||
} HardLinkTable;
|
||||
|
||||
HardLinkTable* hardlink_table_create(void);
|
||||
void hardlink_table_destroy(HardLinkTable* table);
|
||||
|
||||
/* Assign a link-group id to the regular file at `wire_path` with (dev, ino).
|
||||
* On the first encounter the file becomes the group's first (data-carrying)
|
||||
* member (*is_first = true) and a fresh gid is allocated. On a later member
|
||||
* *is_first = false and *first_path_out is set to a malloc'd copy of the first
|
||||
* member's wire path (the caller stores it and owns it; on the first member
|
||||
* path the returned *first_path_out is a malloc'd copy of its own wire path).
|
||||
* Returns false on allocation failure (transfer should abort). */
|
||||
bool hardlink_table_assign(HardLinkTable* table, const char* wire_path, dev_t dev, ino_t ino,
|
||||
int* gid, bool* is_first, char** first_path_out);
|
||||
|
||||
#endif
|
||||
@@ -1,748 +0,0 @@
|
||||
#include "identity.h"
|
||||
#include "log.h"
|
||||
#include "utils.h"
|
||||
#include <errno.h>
|
||||
#include <fcntl.h>
|
||||
#include <grp.h>
|
||||
#include <limits.h>
|
||||
#include <pwd.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/stat.h>
|
||||
#include <unistd.h>
|
||||
|
||||
/* The active identity snapshot lives in a per-process global. The TCP server
|
||||
* forks one child process per connection, so a connection never shares this
|
||||
* with another; within a connection the multithreaded receiver reads it without
|
||||
* mutation. This is what lets the fd-relative metadata path consult the
|
||||
* negotiated policy without threading a Config through every write helper. */
|
||||
typedef struct {
|
||||
bool numeric_ids;
|
||||
bool chown_uid_set;
|
||||
int32_t chown_uid;
|
||||
bool chown_gid_set;
|
||||
int32_t chown_gid;
|
||||
IdentityMap* usermap;
|
||||
int usermap_count;
|
||||
IdentityMap* groupmap;
|
||||
int groupmap_count;
|
||||
/* --super / --no-super tri-state (SUPER_MODE_AUTO when unset). Snapshotted
|
||||
* per connection so privilege_super_permitted() can gate super-user
|
||||
* activities without a Config argument. */
|
||||
int super_mode;
|
||||
/* --copy-as=USER[:GROUP]: snapshotted so the ownership resolver can force the
|
||||
* target ids without a Config argument. */
|
||||
bool copy_as_set;
|
||||
int32_t copy_as_uid;
|
||||
int32_t copy_as_gid;
|
||||
bool set;
|
||||
} IdentityActive;
|
||||
|
||||
static IdentityActive g_identity;
|
||||
|
||||
static void identity_active_reset(void) {
|
||||
free(g_identity.usermap);
|
||||
free(g_identity.groupmap);
|
||||
g_identity.usermap = NULL;
|
||||
g_identity.groupmap = NULL;
|
||||
g_identity.usermap_count = 0;
|
||||
g_identity.groupmap_count = 0;
|
||||
g_identity.numeric_ids = false;
|
||||
g_identity.chown_uid_set = false;
|
||||
g_identity.chown_uid = 0;
|
||||
g_identity.chown_gid_set = false;
|
||||
g_identity.chown_gid = 0;
|
||||
g_identity.super_mode = SUPER_MODE_AUTO;
|
||||
g_identity.copy_as_set = false;
|
||||
g_identity.copy_as_uid = 0;
|
||||
g_identity.copy_as_gid = 0;
|
||||
g_identity.set = false;
|
||||
}
|
||||
|
||||
void identity_clear_active(void) {
|
||||
identity_active_reset();
|
||||
}
|
||||
|
||||
bool identity_set_active(const Config* config) {
|
||||
identity_active_reset();
|
||||
if (!config)
|
||||
return true;
|
||||
g_identity.numeric_ids = config->numeric_ids;
|
||||
g_identity.chown_uid_set = config->chown_uid_set;
|
||||
g_identity.chown_uid = config->chown_uid;
|
||||
g_identity.chown_gid_set = config->chown_gid_set;
|
||||
g_identity.chown_gid = config->chown_gid;
|
||||
g_identity.super_mode = config->super_mode;
|
||||
g_identity.copy_as_set = config->copy_as_set;
|
||||
g_identity.copy_as_uid = config->copy_as_uid;
|
||||
g_identity.copy_as_gid = config->copy_as_gid;
|
||||
if (config->usermap_count > 0) {
|
||||
g_identity.usermap = calloc((size_t)config->usermap_count, sizeof(IdentityMap));
|
||||
if (!g_identity.usermap)
|
||||
goto alloc_failed;
|
||||
memcpy(g_identity.usermap, config->usermap,
|
||||
(size_t)config->usermap_count * sizeof(IdentityMap));
|
||||
g_identity.usermap_count = config->usermap_count;
|
||||
}
|
||||
if (config->groupmap_count > 0) {
|
||||
g_identity.groupmap = calloc((size_t)config->groupmap_count, sizeof(IdentityMap));
|
||||
if (!g_identity.groupmap)
|
||||
goto alloc_failed;
|
||||
memcpy(g_identity.groupmap, config->groupmap,
|
||||
(size_t)config->groupmap_count * sizeof(IdentityMap));
|
||||
g_identity.groupmap_count = config->groupmap_count;
|
||||
}
|
||||
g_identity.set = true;
|
||||
/* A root receiver would honor any client-supplied ownership request (a
|
||||
--usermap/--groupmap/--chown/--copy-as, or raw ids under --numeric-ids).
|
||||
Surface that prominently; a privileged daemon applying arbitrary client
|
||||
ownership is a deliberate, opt-in choice the operator should be aware of. */
|
||||
if (geteuid() == 0)
|
||||
log_message(LOG_LEVEL_WARNING,
|
||||
"identity mapping active and running as root: client-supplied "
|
||||
"ownership (usermap/groupmap/chown/numeric-ids) will be honored; "
|
||||
"run the daemon as an unprivileged user unless intended");
|
||||
/* --super explicitly requests super-user activities, but FastSync never
|
||||
elevates privileges: when the receiver is not already root the kernel will
|
||||
refuse those confined attempts and each is skipped per entry. Warn exactly
|
||||
once at activation time (never abort) so the operator knows the flag cannot
|
||||
succeed on this host. */
|
||||
if (g_identity.super_mode == SUPER_MODE_ON && geteuid() != 0)
|
||||
log_message(LOG_LEVEL_WARNING,
|
||||
"--super requested but the receiver is not privileged; super-user "
|
||||
"activities (ownership, device nodes) will be attempted but refused "
|
||||
"by the kernel and skipped per entry");
|
||||
return true;
|
||||
|
||||
alloc_failed:
|
||||
/* Never proceed with a partial (count-left-zero) map: that would silently
|
||||
apply the WRONG ownership policy. Fail closed and let the caller refuse
|
||||
the connection. */
|
||||
log_message(LOG_LEVEL_ERROR, "memory allocation failed while activating identity policy");
|
||||
identity_active_reset();
|
||||
return false;
|
||||
}
|
||||
|
||||
bool privilege_super_permitted(void) {
|
||||
return privilege_super_mode_permitted(g_identity.super_mode);
|
||||
}
|
||||
|
||||
bool privilege_super_mode_permitted(int mode) {
|
||||
/* AUTO and ON both attempt the confined operation; OFF forbids it even for a
|
||||
* root receiver. AUTO is the historical FastSync behavior (always attempt
|
||||
* and let the kernel refuse an unprivileged call, which the caller skips), so
|
||||
* it must stay permissive or a group-only chown that a non-root receiver is
|
||||
* allowed to make would regress. */
|
||||
return mode != SUPER_MODE_OFF;
|
||||
}
|
||||
|
||||
bool identity_active_enabled(void) {
|
||||
/* numeric_ids is included: this set only gates identity_apply_ownership,
|
||||
which runs only when metadata is present (a -M/--preserve transfer). A
|
||||
standalone --numeric-ids (no ownership-affecting flag) carries no
|
||||
metadata, never reaches identity_apply_ownership, and therefore correctly
|
||||
stays inert; combined with -M it activates raw-id application. --super /
|
||||
--no-super does NOT enable ownership: it only permits or forbids the
|
||||
already-requested super-user activities, so a --super with no explicit
|
||||
identity flag must never silently apply client-chosen ownership. */
|
||||
return g_identity.set &&
|
||||
(g_identity.numeric_ids || g_identity.chown_uid_set || g_identity.chown_gid_set ||
|
||||
g_identity.usermap_count > 0 || g_identity.groupmap_count > 0 || g_identity.copy_as_set);
|
||||
}
|
||||
|
||||
bool identity_ownership_requested(const Config* config) {
|
||||
if (!config)
|
||||
return false;
|
||||
/* Every value that makes the receiver act on a client-chosen owner, plus an
|
||||
* explicit --super (super-user device-node activities). Pure config, so the
|
||||
* daemon gate can evaluate it before identity_set_active(). */
|
||||
return config->numeric_ids || config->chown_uid_set || config->chown_gid_set ||
|
||||
config->usermap_count > 0 || config->groupmap_count > 0 || config->copy_as_set ||
|
||||
config->fake_super || config->super_mode == SUPER_MODE_ON;
|
||||
}
|
||||
|
||||
bool identity_copy_as_active(void) {
|
||||
return g_identity.set && g_identity.copy_as_set;
|
||||
}
|
||||
|
||||
bool identity_copy_as_refused(const Config* config) {
|
||||
if (!config || !config->copy_as_set)
|
||||
return false;
|
||||
/* The safe-subset --copy-as needs a privileged (root) receiver, and an
|
||||
* operator/--no-super veto forbids the ownership change even for root. This
|
||||
* is deliberately a pure function of the config and the current effective uid
|
||||
* (never the active snapshot) because the server evaluates it at the
|
||||
* pre-STATUS_OK config gate, before identity_set_active() has run. */
|
||||
return geteuid() != 0 || config->super_mode == SUPER_MODE_OFF;
|
||||
}
|
||||
|
||||
bool identity_wire_valid(const Config* config) {
|
||||
if (!config)
|
||||
return false;
|
||||
if (config->usermap_count < 0 || config->usermap_count > MAX_IDENTITY_MAP ||
|
||||
config->groupmap_count < 0 || config->groupmap_count > MAX_IDENTITY_MAP)
|
||||
return false;
|
||||
if (config->chown_uid_set && config->chown_uid < IDENTITY_MATCH_ANY)
|
||||
return false;
|
||||
if (config->chown_gid_set && config->chown_gid < IDENTITY_MATCH_ANY)
|
||||
return false;
|
||||
for (int i = 0; i < config->usermap_count; i++) {
|
||||
if (config->usermap[i].from < IDENTITY_MATCH_ANY || config->usermap[i].to < IDENTITY_CURRENT)
|
||||
return false;
|
||||
}
|
||||
for (int i = 0; i < config->groupmap_count; i++) {
|
||||
if (config->groupmap[i].from < IDENTITY_MATCH_ANY || config->groupmap[i].to < IDENTITY_CURRENT)
|
||||
return false;
|
||||
}
|
||||
/* Defense-in-depth: a --copy-as block must never carry a negative (sentinel)
|
||||
* id into the ownership path. receive_copy_as_options already rejects them,
|
||||
* but identity_wire_valid is the shared validation used by both the receiver
|
||||
* and unit tests, so re-assert it here. */
|
||||
if (config->copy_as_set && (config->copy_as_uid < 0 || config->copy_as_gid < 0))
|
||||
return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
/* ---- CLI-time name/number resolution ---- */
|
||||
|
||||
/* Parse a single FROM/TO token into an int32 id. Returns 0 on success, -1 on a
|
||||
* malformed or unresolvable token. When is_group, name lookups use the group
|
||||
* database; otherwise the user database. A `*` token returns IDENTITY_MATCH_ANY
|
||||
* / IDENTITY_CURRENT (the same -1 value, disambiguated by the caller's
|
||||
* position). An `@`-prefixed or bare-decimal token is a numeric id. */
|
||||
static int identity_resolve_token(const char* token, bool is_group, int32_t* out) {
|
||||
if (!token || *token == '\0')
|
||||
return -1;
|
||||
if (strcmp(token, "*") == 0) {
|
||||
*out = IDENTITY_MATCH_ANY;
|
||||
return 0;
|
||||
}
|
||||
const char* num = (token[0] == '@') ? token + 1 : token;
|
||||
if (*num != '\0') {
|
||||
bool all_digits = true;
|
||||
for (const char* p = num; *p; p++)
|
||||
if (*p < '0' || *p > '9')
|
||||
all_digits = false;
|
||||
if (all_digits) {
|
||||
char* endptr = NULL;
|
||||
errno = 0;
|
||||
long val = strtol(num, &endptr, 10);
|
||||
if (errno == 0 && endptr && *endptr == '\0' && val >= 0 && val <= INT32_MAX) {
|
||||
*out = (int32_t)val;
|
||||
return 0;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
/* A name (or a name-like numeric that failed strict numeric parse). */
|
||||
if (is_group) {
|
||||
struct group* gr = getgrnam(token);
|
||||
if (!gr)
|
||||
return -1;
|
||||
*out = (int32_t)gr->gr_gid;
|
||||
return 0;
|
||||
}
|
||||
struct passwd* pw = getpwnam(token);
|
||||
if (!pw)
|
||||
return -1;
|
||||
*out = (int32_t)pw->pw_uid;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int identity_append_rule(IdentityMap** map, int* count, int32_t from, int32_t to) {
|
||||
if (*count >= MAX_IDENTITY_MAP)
|
||||
return -1;
|
||||
IdentityMap* grown = realloc(*map, (size_t)(*count + 1) * sizeof(IdentityMap));
|
||||
if (!grown)
|
||||
return -1;
|
||||
*map = grown;
|
||||
(*map)[*count].from = from;
|
||||
(*map)[*count].to = to;
|
||||
(*count)++;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int identity_parse_map(Config* config, const char* value, bool is_group) {
|
||||
if (!config || !value || *value == '\0') {
|
||||
log_message(LOG_LEVEL_ERROR, "%smap requires a value", is_group ? "--group" : "--user");
|
||||
return -1;
|
||||
}
|
||||
char* list = str_dup(value);
|
||||
if (!list)
|
||||
return -1;
|
||||
const char* optname = is_group ? "--groupmap" : "--usermap";
|
||||
char* saveptr = NULL;
|
||||
for (char* rule = strtok_r(list, ",", &saveptr); rule; rule = strtok_r(NULL, ",", &saveptr)) {
|
||||
char* colon = strchr(rule, ':');
|
||||
if (!colon || colon == rule) {
|
||||
/* Log before freeing: `rule` points into the str_dup'd list. */
|
||||
log_message(LOG_LEVEL_ERROR, "%s rules must be FROM:TO (got '%s')", optname, rule);
|
||||
free(list);
|
||||
return -1;
|
||||
}
|
||||
*colon = '\0';
|
||||
char* from_token = rule;
|
||||
char* to_token = colon + 1;
|
||||
if (*to_token == '\0') {
|
||||
free(list);
|
||||
log_message(LOG_LEVEL_ERROR, "%s rule 'FROM:' is missing the TO value (got '%s')", optname,
|
||||
value);
|
||||
return -1;
|
||||
}
|
||||
int32_t from_id, to_id;
|
||||
if (identity_resolve_token(from_token, is_group, &from_id) != 0 ||
|
||||
identity_resolve_token(to_token, is_group, &to_id) != 0) {
|
||||
free(list);
|
||||
log_message(LOG_LEVEL_ERROR,
|
||||
"%s could not resolve '%s' (name must exist on the source; use "
|
||||
"@N for a numeric id)",
|
||||
optname, value);
|
||||
return -1;
|
||||
}
|
||||
if (identity_append_rule(is_group ? &config->groupmap : &config->usermap,
|
||||
is_group ? &config->groupmap_count : &config->usermap_count, from_id,
|
||||
to_id) != 0) {
|
||||
free(list);
|
||||
log_message(LOG_LEVEL_ERROR, "%s has too many rules (max %d)", optname, MAX_IDENTITY_MAP);
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
free(list);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Split --chown=USER:GROUP on the first UNESCAPED colon, honoring backslash
|
||||
* escapes (a `\:` is a literal colon inside a name; a lone backslash before any
|
||||
* other character is kept verbatim). Both sides are returned as malloc'd
|
||||
* strings (the absent side is NULL). */
|
||||
static int identity_split_chown(const char* value, char** puser, char** pgroup) {
|
||||
size_t len = strlen(value);
|
||||
char* user = malloc(len + 1);
|
||||
char* group = malloc(len + 1);
|
||||
if (!user || !group) {
|
||||
free(user);
|
||||
free(group);
|
||||
return -1;
|
||||
}
|
||||
const char* p = value;
|
||||
size_t ui = 0;
|
||||
bool split_seen = false;
|
||||
size_t gi = 0;
|
||||
while (*p) {
|
||||
if (*p == '\\' && p[1] == ':') {
|
||||
/* an escaped colon: a literal ':' in the current side's name */
|
||||
if (split_seen)
|
||||
group[gi++] = ':';
|
||||
else
|
||||
user[ui++] = ':';
|
||||
p += 2;
|
||||
continue;
|
||||
}
|
||||
if (*p == ':') {
|
||||
split_seen = true;
|
||||
p++;
|
||||
continue;
|
||||
}
|
||||
if (split_seen)
|
||||
group[gi++] = *p;
|
||||
else
|
||||
user[ui++] = *p;
|
||||
p++;
|
||||
}
|
||||
user[ui] = '\0';
|
||||
group[gi] = '\0';
|
||||
char* u = str_dup(user);
|
||||
char* g = str_dup(group);
|
||||
free(user);
|
||||
free(group);
|
||||
if (!u || !g) {
|
||||
free(u);
|
||||
free(g);
|
||||
return -1;
|
||||
}
|
||||
*puser = u;
|
||||
*pgroup = g;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int identity_parse_chown(Config* config, const char* value) {
|
||||
if (!config || !value || *value == '\0') {
|
||||
log_message(LOG_LEVEL_ERROR, "--chown requires a value (USER:GROUP, USER, or :GROUP)");
|
||||
return -1;
|
||||
}
|
||||
/* Reject more than one UNESCAPED colon (a name or group may not contain an
|
||||
* unescaped ':' in the spec). The scan is escape-aware: a `\:` is a literal
|
||||
* colon inside a name, not a field separator. */
|
||||
int colons = 0;
|
||||
bool saw_colon = false;
|
||||
const char* p = value;
|
||||
while (*p) {
|
||||
if (*p == '\\' && p[1] == ':') {
|
||||
p += 2;
|
||||
continue;
|
||||
}
|
||||
if (*p == ':') {
|
||||
colons++;
|
||||
saw_colon = true;
|
||||
}
|
||||
p++;
|
||||
}
|
||||
if (colons > 1) {
|
||||
log_message(LOG_LEVEL_ERROR, "--chown must have at most one ':' (got '%s')", value);
|
||||
return -1;
|
||||
}
|
||||
|
||||
char *user = NULL, *group = NULL;
|
||||
if (identity_split_chown(value, &user, &group) != 0) {
|
||||
log_message(LOG_LEVEL_ERROR, "memory allocation failed for --chown");
|
||||
return -1;
|
||||
}
|
||||
int ret = 0;
|
||||
if (!saw_colon) {
|
||||
/* --chown=USER: owner only. */
|
||||
if (*user == '\0') {
|
||||
log_message(LOG_LEVEL_ERROR, "--chown requires a user or group (got '%s')", value);
|
||||
ret = -1;
|
||||
} else if (identity_resolve_token(user, false, &config->chown_uid) != 0) {
|
||||
log_message(LOG_LEVEL_ERROR,
|
||||
"--chown could not resolve user '%s' (use a name that exists "
|
||||
"on the source, '*', or @N)",
|
||||
value);
|
||||
ret = -1;
|
||||
} else {
|
||||
config->chown_uid_set = true;
|
||||
}
|
||||
} else {
|
||||
/* --chown=USER:GROUP, --chown=:GROUP, --chown=USER: */
|
||||
if (*user != '\0') {
|
||||
if (identity_resolve_token(user, false, &config->chown_uid) != 0) {
|
||||
log_message(LOG_LEVEL_ERROR, "--chown could not resolve user '%s'", value);
|
||||
ret = -1;
|
||||
goto done;
|
||||
}
|
||||
config->chown_uid_set = true;
|
||||
}
|
||||
if (*group != '\0') {
|
||||
if (identity_resolve_token(group, true, &config->chown_gid) != 0) {
|
||||
log_message(LOG_LEVEL_ERROR, "--chown could not resolve group '%s'", value);
|
||||
ret = -1;
|
||||
goto done;
|
||||
}
|
||||
config->chown_gid_set = true;
|
||||
}
|
||||
if (!*user && !*group) {
|
||||
log_message(LOG_LEVEL_ERROR, "--chown must set a user, a group, or both (got '%s')", value);
|
||||
ret = -1;
|
||||
}
|
||||
}
|
||||
done:
|
||||
free(user);
|
||||
free(group);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/* uid_t/gid_t are unsigned and may hold a value wider than the signed int32 the
|
||||
* wire (and the identity policy) uses. Reject such an id instead of truncating
|
||||
* it to an out-of-range (possibly negative sentinel) value. */
|
||||
static bool identity_id_fits_int32(unsigned long id) {
|
||||
return id <= (unsigned long)INT32_MAX;
|
||||
}
|
||||
|
||||
/* Resolve one --copy-as id token. A '*' token means the caller's current
|
||||
* effective uid (user) or gid (group). Returns 0 on success. On failure sets
|
||||
* *overflow when a '*' id was wider than int32 so the caller can log the
|
||||
* specific message; otherwise the token was simply unresolvable. */
|
||||
static int identity_resolve_copy_as_id(const char* token, bool is_group, int32_t* out,
|
||||
bool* overflow) {
|
||||
*overflow = false;
|
||||
if (strcmp(token, "*") == 0) {
|
||||
unsigned long current = is_group ? (unsigned long)getegid() : (unsigned long)geteuid();
|
||||
if (!identity_id_fits_int32(current)) {
|
||||
*overflow = true;
|
||||
return -1;
|
||||
}
|
||||
*out = (int32_t)current;
|
||||
return 0;
|
||||
}
|
||||
return identity_resolve_token(token, is_group, out);
|
||||
}
|
||||
|
||||
int identity_parse_copy_as(Config* config, const char* value) {
|
||||
if (!config || !value || *value == '\0') {
|
||||
log_message(LOG_LEVEL_ERROR, "--copy-as requires USER[:GROUP]");
|
||||
return -1;
|
||||
}
|
||||
/* --copy-as=USER[:GROUP] is the whole grammar: at most one field separator.
|
||||
* (Unlike --chown there is no escaped-colon form; a name containing ':' is
|
||||
* simply not expressible, and the extra colon is a clear parse error.) */
|
||||
int colons = 0;
|
||||
for (const char* p = value; *p; p++)
|
||||
if (*p == ':')
|
||||
colons++;
|
||||
if (colons > 1) {
|
||||
char* escaped = output_escape(value, false);
|
||||
log_message(LOG_LEVEL_ERROR, "--copy-as must be USER[:GROUP] (got '%s')",
|
||||
escaped ? escaped : "<allocation failed>");
|
||||
free(escaped);
|
||||
return -1;
|
||||
}
|
||||
|
||||
char* spec = str_dup(value);
|
||||
if (!spec) {
|
||||
log_message(LOG_LEVEL_ERROR, "memory allocation failed for --copy-as");
|
||||
return -1;
|
||||
}
|
||||
const char* user_token = spec;
|
||||
const char* group_token = NULL;
|
||||
char* colon = strchr(spec, ':');
|
||||
if (colon) {
|
||||
*colon = '\0';
|
||||
group_token = colon + 1;
|
||||
}
|
||||
|
||||
/* The spec is untrusted user input echoed back in error paths: escape it once
|
||||
* (8-bit-safe) so a control byte cannot forge a log line. */
|
||||
char* escaped_spec = output_escape(value, false);
|
||||
const char* shown = escaped_spec ? escaped_spec : "<allocation failed>";
|
||||
int ret = -1;
|
||||
|
||||
if (*user_token == '\0') {
|
||||
log_message(LOG_LEVEL_ERROR, "--copy-as is missing the user (got '%s')", shown);
|
||||
goto done;
|
||||
}
|
||||
bool overflow = false;
|
||||
int32_t uid;
|
||||
if (identity_resolve_copy_as_id(user_token, false, &uid, &overflow) != 0) {
|
||||
if (overflow)
|
||||
log_message(LOG_LEVEL_ERROR, "--copy-as: current user id %lu exceeds INT32_MAX",
|
||||
(unsigned long)geteuid());
|
||||
else
|
||||
log_message(LOG_LEVEL_ERROR,
|
||||
"--copy-as could not resolve user (use a name that exists on the "
|
||||
"source, '*', or @N): %s",
|
||||
shown);
|
||||
goto done;
|
||||
}
|
||||
|
||||
int32_t gid;
|
||||
if (group_token) {
|
||||
if (*group_token == '\0') {
|
||||
log_message(LOG_LEVEL_ERROR, "--copy-as group is empty (got '%s')", shown);
|
||||
goto done;
|
||||
}
|
||||
if (identity_resolve_copy_as_id(group_token, true, &gid, &overflow) != 0) {
|
||||
if (overflow)
|
||||
log_message(LOG_LEVEL_ERROR, "--copy-as: current group id %lu exceeds INT32_MAX",
|
||||
(unsigned long)getegid());
|
||||
else
|
||||
log_message(LOG_LEVEL_ERROR, "--copy-as could not resolve group (got '%s')", shown);
|
||||
goto done;
|
||||
}
|
||||
} else {
|
||||
/* Group omitted: use the user's primary gid. A numeric id with no local
|
||||
* passwd entry has no primary gid to look up, so fall back to gid == uid
|
||||
* (the rsync-style numeric convention; documented divergence). */
|
||||
struct passwd* pw = getpwuid((uid_t)uid);
|
||||
if (pw) {
|
||||
if (!identity_id_fits_int32((unsigned long)pw->pw_gid)) {
|
||||
log_message(LOG_LEVEL_ERROR,
|
||||
"--copy-as: primary group id %lu for the requested user exceeds INT32_MAX",
|
||||
(unsigned long)pw->pw_gid);
|
||||
goto done;
|
||||
}
|
||||
gid = (int32_t)pw->pw_gid;
|
||||
} else {
|
||||
gid = uid;
|
||||
}
|
||||
}
|
||||
/* The group-default and gid==uid fallbacks must never store a negative
|
||||
* (sentinel) value; the explicit numeric path is already capped by
|
||||
* identity_resolve_token. */
|
||||
if (uid < 0 || gid < 0) {
|
||||
log_message(LOG_LEVEL_ERROR, "--copy-as resolved id does not fit in int32 (got '%s')", shown);
|
||||
goto done;
|
||||
}
|
||||
|
||||
config->copy_as_set = true;
|
||||
config->copy_as_uid = uid;
|
||||
config->copy_as_gid = gid;
|
||||
/* Ownership application needs the metadata path (the source uid/gid must be
|
||||
* transmitted); imply it exactly like --chown/--usermap/--groupmap. */
|
||||
config->use_metadata = true;
|
||||
ret = 0;
|
||||
|
||||
done:
|
||||
free(escaped_spec);
|
||||
free(spec);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/* ---- Receiver-side ownership application ---- */
|
||||
|
||||
static bool identity_map_lookup(const IdentityMap* map, int count, int32_t source_id,
|
||||
int32_t* out_to) {
|
||||
for (int i = 0; i < count; i++) {
|
||||
if (map[i].from == IDENTITY_MATCH_ANY || map[i].from == source_id) {
|
||||
*out_to = map[i].to;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/* Resolve the target ownership from the negotiated policy against the entry's
|
||||
* current stat. Shared by the fd (regular file) and no-follow (symlink) apply
|
||||
* paths. Returns false when no side is to be changed. */
|
||||
static bool identity_resolve_targets(const struct stat* st, int32_t source_uid, int32_t source_gid,
|
||||
uid_t* out_uid, gid_t* out_gid) {
|
||||
bool set_uid = false;
|
||||
bool set_gid = false;
|
||||
uid_t uid = 0;
|
||||
gid_t gid = 0;
|
||||
|
||||
/* --copy-as (P7 Wave E) has the highest priority: it forces BOTH the owner
|
||||
* and group of every written entry to the requested ids, beating usermap /
|
||||
* groupmap / --chown / --numeric-ids and the best-effort name lookup. Only
|
||||
* skip when the entry already carries exactly those ids. */
|
||||
if (g_identity.copy_as_set) {
|
||||
uid = (uid_t)g_identity.copy_as_uid;
|
||||
gid = (gid_t)g_identity.copy_as_gid;
|
||||
if (st->st_uid == uid && st->st_gid == gid)
|
||||
return false;
|
||||
*out_uid = uid;
|
||||
*out_gid = gid;
|
||||
return true;
|
||||
}
|
||||
|
||||
int32_t target;
|
||||
if (identity_map_lookup(g_identity.usermap, g_identity.usermap_count, source_uid, &target)) {
|
||||
uid = target == IDENTITY_CURRENT ? geteuid() : (uid_t)target;
|
||||
set_uid = true;
|
||||
} else if (g_identity.chown_uid_set) {
|
||||
uid = g_identity.chown_uid == IDENTITY_CURRENT ? geteuid() : (uid_t)g_identity.chown_uid;
|
||||
set_uid = true;
|
||||
} else if (g_identity.numeric_ids) {
|
||||
uid = (uid_t)source_uid;
|
||||
set_uid = true;
|
||||
} else {
|
||||
/* Best-effort name mapping against the receiver's own database: if the
|
||||
* transmitted (numeric) id resolves to a name present on this machine,
|
||||
* re-resolve it. On a shared-account host this is the identity operation;
|
||||
* when the id has no name here, the user side is left alone. */
|
||||
struct passwd* pw = getpwuid((uid_t)source_uid);
|
||||
if (pw) {
|
||||
const struct passwd* mapped = getpwnam(pw->pw_name);
|
||||
if (mapped) {
|
||||
uid = mapped->pw_uid;
|
||||
set_uid = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (identity_map_lookup(g_identity.groupmap, g_identity.groupmap_count, source_gid, &target)) {
|
||||
gid = target == IDENTITY_CURRENT ? getegid() : (gid_t)target;
|
||||
set_gid = true;
|
||||
} else if (g_identity.chown_gid_set) {
|
||||
gid = g_identity.chown_gid == IDENTITY_CURRENT ? getegid() : (gid_t)g_identity.chown_gid;
|
||||
set_gid = true;
|
||||
} else if (g_identity.numeric_ids) {
|
||||
gid = (gid_t)source_gid;
|
||||
set_gid = true;
|
||||
} else {
|
||||
struct group* gr = getgrgid((gid_t)source_gid);
|
||||
if (gr) {
|
||||
const struct group* mapped = getgrnam(gr->gr_name);
|
||||
if (mapped) {
|
||||
gid = mapped->gr_gid;
|
||||
set_gid = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!set_uid && !set_gid)
|
||||
return false;
|
||||
/* An unset side keeps the file's current id so the other side can change. */
|
||||
if (!set_uid)
|
||||
uid = st->st_uid;
|
||||
if (!set_gid)
|
||||
gid = st->st_gid;
|
||||
/* Only change ownership when the target differs (avoid needless syscalls and
|
||||
* any chance of clearing setuid/setgid on an already-correct entry). */
|
||||
if (st->st_uid == uid && st->st_gid == gid)
|
||||
return false;
|
||||
*out_uid = uid;
|
||||
*out_gid = gid;
|
||||
return true;
|
||||
}
|
||||
|
||||
static void identity_log_chown_failure(const char* what, uid_t uid, gid_t gid) {
|
||||
/* EPERM/EACCES are expected when the receiver is not privileged (e.g. the CI
|
||||
* `nobody` user): warn and continue, never abort the transfer. Any other
|
||||
* error (EIO/EROFS/ENOSPC/...) is a real failure and must not be silently
|
||||
* downgraded to a warning.
|
||||
*
|
||||
* --copy-as is different: the whole point of the flag is that the target
|
||||
* ownership is REQUIRED (the pre-flight gate already refused an unprivileged
|
||||
* receiver). If the chown still fails with EPERM/EACCES (a capability-
|
||||
* restricted root, root-squash, or a read-only mount) the run would be
|
||||
* silently producing the WRONG ownership, so surface it at ERROR. The
|
||||
* caller (identity_apply_ownership*) then reports the ENTRY as failed rather
|
||||
* than as written, which becomes a FILE_SAVE_ERROR and fails the transfer
|
||||
* (fail-fast) instead of reporting overall success with the wrong owner. */
|
||||
if (errno == EPERM || errno == EACCES) {
|
||||
if (identity_copy_as_active())
|
||||
log_message(LOG_LEVEL_ERROR,
|
||||
"could not apply --copy-as ownership on %s (uid=%ld gid=%ld): %s; "
|
||||
"entry was written with the wrong owner",
|
||||
what, (long)uid, (long)gid, strerror(errno));
|
||||
else
|
||||
log_message(LOG_LEVEL_WARNING,
|
||||
"could not apply ownership (uid=%ld gid=%ld): %s; leaving as-is", (long)uid,
|
||||
(long)gid, strerror(errno));
|
||||
} else {
|
||||
log_message(LOG_LEVEL_ERROR, "failed to apply ownership on %s (uid=%ld gid=%ld): %s", what,
|
||||
(long)uid, (long)gid, strerror(errno));
|
||||
}
|
||||
}
|
||||
|
||||
bool identity_apply_ownership(int fd, int32_t source_uid, int32_t source_gid) {
|
||||
/* Ownership application is OFF unless the client requested an identity flag.
|
||||
* This is the controlled gate: a default (or plain -M) transfer never changes
|
||||
* ownership, byte-for-byte preserving FastSync's existing behavior. --no-super
|
||||
* additionally forbids it even when the receiver is root. */
|
||||
if (!identity_active_enabled() || !privilege_super_permitted() || fd < 0)
|
||||
return true;
|
||||
struct stat st;
|
||||
if (fstat(fd, &st) != 0)
|
||||
return !identity_copy_as_active();
|
||||
uid_t uid;
|
||||
gid_t gid;
|
||||
if (!identity_resolve_targets(&st, source_uid, source_gid, &uid, &gid))
|
||||
return true;
|
||||
if (fchown(fd, uid, gid) != 0) {
|
||||
identity_log_chown_failure("file", uid, gid);
|
||||
/* A required --copy-as ownership that did not land is a per-entry failure;
|
||||
* every other policy stays best-effort (rsync parity). */
|
||||
return !identity_copy_as_active();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool identity_apply_ownership_link(int parent_fd, const char* leaf, int32_t source_uid,
|
||||
int32_t source_gid) {
|
||||
if (!identity_active_enabled() || !privilege_super_permitted() || parent_fd < 0 || !leaf)
|
||||
return true;
|
||||
struct stat st;
|
||||
if (fstatat(parent_fd, leaf, &st, AT_SYMLINK_NOFOLLOW) != 0)
|
||||
return !identity_copy_as_active();
|
||||
uid_t uid;
|
||||
gid_t gid;
|
||||
if (!identity_resolve_targets(&st, source_uid, source_gid, &uid, &gid))
|
||||
return true;
|
||||
if (fchownat(parent_fd, leaf, uid, gid, AT_SYMLINK_NOFOLLOW) != 0) {
|
||||
identity_log_chown_failure("no-follow entry", uid, gid);
|
||||
return !identity_copy_as_active();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
@@ -1,133 +0,0 @@
|
||||
#ifndef IDENTITY_H
|
||||
#define IDENTITY_H
|
||||
|
||||
#include "config.h"
|
||||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
#include <sys/types.h>
|
||||
|
||||
/*
|
||||
* Identity mapping: --numeric-ids / --usermap / --groupmap / --chown / --copy-as.
|
||||
*
|
||||
* FastSync transmits uid/gid numerically (int32 on the wire) and, by design,
|
||||
* NEVER applies client-supplied ownership unless a user explicitly opts in with
|
||||
* an identity flag below. This module is the controlled, opt-in,
|
||||
* privilege-gated path for applying ownership on the receiver: the wire config
|
||||
* is snapshotted once per connection via identity_set_active() and applied
|
||||
* through an fd-relative fchown() in the receiver's metadata-restore path.
|
||||
*
|
||||
* Because only numeric ids cross the wire, name-based values are resolved to
|
||||
* numbers at CLI parse time using the CLIENT (sender) machine's databases. On
|
||||
* a shared-account source/destination this reproduces rsync's semantics; a
|
||||
* genuinely different destination database is a documented divergence (see
|
||||
* RSYNC_COMPAT.md).
|
||||
*/
|
||||
|
||||
/* Parse one --usermap= / --groupmap= value (comma-separated FROM:TO rules,
|
||||
* first match wins) into config->usermap / config->groupmap. is_group selects
|
||||
* the group tables and name databases. Returns 0 on success, -1 on a
|
||||
* malformed spec or an unresolvable name (never a silent no-op). */
|
||||
int identity_parse_map(Config* config, const char* value, bool is_group);
|
||||
|
||||
/* Parse --chown=USER:GROUP. Supports USER:GROUP, USER (owner only), :GROUP
|
||||
* (group only), '*' (current/root as appropriate) and numeric ids. Returns 0
|
||||
* on success, -1 on a malformed spec / unresolvable name. */
|
||||
int identity_parse_chown(Config* config, const char* value);
|
||||
|
||||
/* Parse --copy-as=USER[:GROUP] (P7 Wave E). USER is resolved with the same
|
||||
* user-database rules as --chown (a name, @N/bare N numeric id, or '*' meaning
|
||||
* the client's current euid); when ':GROUP' is present the group is resolved
|
||||
* with the group database ('*' meaning the client's egid). When the group is
|
||||
* omitted, the user's primary gid is used (getpwuid(uid)->pw_gid); if the
|
||||
* resolved user is a numeric id with no local passwd entry, gid falls back to
|
||||
* uid. On success sets copy_as_set/copy_as_uid/copy_as_gid and forces
|
||||
* metadata transmission (ownership application needs the metadata path).
|
||||
* Returns 0 on success, -1 on a malformed / empty / unresolvable spec (never a
|
||||
* silent no-op). */
|
||||
int identity_parse_copy_as(Config* config, const char* value);
|
||||
|
||||
/* True when a --copy-as request is active but the receiver is not permitted to
|
||||
* perform the privileged ownership application it needs. This is the up-front
|
||||
* refusal predicate: the server rejects the whole transfer at the config
|
||||
* handshake rather than silently ignoring the requested ownership. It is a
|
||||
* pure function of the config mode and the current effective uid (it does NOT
|
||||
* read the active snapshot, so it is valid at the pre-STATUS_OK gate, before
|
||||
* identity_set_active() has run). `super_mode` is the EFFECTIVE mode after any
|
||||
* server-side policy veto. */
|
||||
bool identity_copy_as_refused(const Config* config);
|
||||
|
||||
/* True when the CURRENT per-connection snapshot has a --copy-as active (i.e.
|
||||
* identity_set_active() has run against a config with copy_as_set). The
|
||||
* --fake-super owner replay consults this so a copy-as run never lets the
|
||||
* recorded source owner overwrite the forced target owner. Reads the active
|
||||
* snapshot, so call identity_set_active() first (the receiver does, before any
|
||||
* write). */
|
||||
bool identity_copy_as_active(void);
|
||||
|
||||
/* Receiver-side snapshot of the negotiated identity config. The server calls
|
||||
* identity_set_active() once per connection (before any file write) using the
|
||||
* config received over the wire; the snapshot is a deep copy so the caller may
|
||||
* free its Config immediately. identity_clear_active() releases it.
|
||||
*
|
||||
* Returns true on success. On an allocation failure while deep-copying a
|
||||
* requested usermap/groupmap it logs a LOG_LEVEL_ERROR, leaves the snapshot
|
||||
* cleared (never a partial/wrong policy) and returns false; the caller must
|
||||
* refuse the connection. */
|
||||
bool identity_set_active(const Config* config);
|
||||
void identity_clear_active(void);
|
||||
|
||||
/* True when any ownership-affecting identity option is present in the active
|
||||
* snapshot. Ownership stays OFF ("do not apply") for every transfer that
|
||||
* requests none of them, preserving FastSync's existing behavior. --super /
|
||||
* --no-super alone does NOT enable ownership; an explicit identity flag
|
||||
* (--numeric-ids / --chown / --usermap / --groupmap / --copy-as) is required. */
|
||||
bool identity_active_enabled(void);
|
||||
|
||||
/* Pure, config-only predicate: true when the client requested ANY
|
||||
* client-chosen ownership or super-user activity (--numeric-ids, --chown,
|
||||
* --usermap/--groupmap, --copy-as, --fake-super, or an explicit --super). Used
|
||||
* by the daemon module gate to decide whether a module's per-module opt-in is
|
||||
* required; it never reads the per-connection snapshot. */
|
||||
bool identity_ownership_requested(const Config* config);
|
||||
|
||||
/* Apply the negotiated ownership to an already-written file descriptor.
|
||||
* source_uid/source_gid are the transmitted numeric ids. Resolution order:
|
||||
* --copy-as (highest priority, forces both ids), then a matching
|
||||
* usermap/groupmap rule, then --chown, then --numeric-ids (raw), then a
|
||||
* best-effort name lookup on the receiver's own databases (skipped when the
|
||||
* transmitted id has no name on this system). Only calls fchown() when the
|
||||
* result differs from the current value.
|
||||
*
|
||||
* Returns false ONLY when an active --copy-as ownership application failed: its
|
||||
* forced ownership is REQUIRED, so the caller must treat the entry as failed
|
||||
* rather than reporting success with the wrong owner. For every other identity
|
||||
* policy an fchown EPERM/EACCES is logged and ignored and true is returned
|
||||
* (rsync parity: the transfer must not abort). A no-op when no identity policy
|
||||
* is active returns true. */
|
||||
bool identity_apply_ownership(int fd, int32_t source_uid, int32_t source_gid);
|
||||
|
||||
/* P7 Wave D: the no-follow (symlink) counterpart. Resolves the same
|
||||
* usermap/groupmap/chown/numeric-ids/copy-as policy but applies it with
|
||||
* fchownat(..., AT_SYMLINK_NOFOLLOW) so a symlink's own ownership is changed
|
||||
* without ever dereferencing it. A no-op unless an identity flag is active.
|
||||
* The return value follows identity_apply_ownership(): false only when an
|
||||
* active --copy-as application failed. */
|
||||
bool identity_apply_ownership_link(int parent_fd, const char* leaf, int32_t source_uid,
|
||||
int32_t source_gid);
|
||||
|
||||
/* Receiver-side wire validation of the resolved identity fields. */
|
||||
bool identity_wire_valid(const Config* config);
|
||||
|
||||
/* P7 Wave E receiver-side permission gate for super-user activities (ownership
|
||||
* application and char/block device-node creation). `privilege_super_permitted`
|
||||
* consults the per-connection snapshot (call identity_set_active() first);
|
||||
* `privilege_super_mode_permitted` is the pure mode predicate and is what
|
||||
* callers holding a Config use (the config-frame gate, file_receive). Both
|
||||
* return false only for SUPER_MODE_OFF; SUPER_MODE_ON and SUPER_MODE_AUTO (the
|
||||
* default) permit a confined attempt, matching FastSync's historical
|
||||
* best-effort behavior where an unprivileged attempt is refused by the kernel
|
||||
* and skipped. Neither EVER elevates privileges. */
|
||||
bool privilege_super_permitted(void);
|
||||
bool privilege_super_mode_permitted(int mode);
|
||||
|
||||
#endif
|
||||
+8
-124
@@ -1,144 +1,28 @@
|
||||
#include "log.h"
|
||||
#include <errno.h>
|
||||
#include <stdbool.h>
|
||||
#include <stdarg.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include <time.h>
|
||||
|
||||
static const char* log_level_strings[] = {"DEBUG", "INFO", "WARN", "ERROR"};
|
||||
static const char *log_level_strings[] = {"DEBUG", "INFO", "WARN", "ERROR"};
|
||||
static LogLevel current_log_level = LOG_LEVEL_WARNING;
|
||||
static uint32_t current_debug_flags = 0;
|
||||
static uint32_t info_flags = 0;
|
||||
static bool info_flags_explicit = false;
|
||||
static FILE* log_fp = NULL;
|
||||
static _Thread_local bool eight_bit_output;
|
||||
static LogStderrMode stderr_mode = LOG_STDERR_ERRORS;
|
||||
|
||||
void set_log_level(LogLevel level) {
|
||||
current_log_level = level;
|
||||
}
|
||||
|
||||
void set_log_debug_flags(uint32_t flags) {
|
||||
current_debug_flags = flags;
|
||||
}
|
||||
|
||||
uint32_t get_log_debug_flags(void) {
|
||||
return current_debug_flags;
|
||||
}
|
||||
|
||||
bool log_debug_enabled(LogDebugFlag flag) {
|
||||
return current_log_level <= LOG_LEVEL_DEBUG && (current_debug_flags & flag) != 0;
|
||||
}
|
||||
|
||||
void set_log_info_flags(uint32_t flags) {
|
||||
info_flags = flags;
|
||||
info_flags_explicit = true;
|
||||
}
|
||||
|
||||
uint32_t get_log_info_flags(void) {
|
||||
return info_flags;
|
||||
}
|
||||
|
||||
void log_set_file(FILE* fp) {
|
||||
log_fp = fp;
|
||||
}
|
||||
|
||||
void log_set_8_bit_output(bool enabled) {
|
||||
eight_bit_output = enabled;
|
||||
}
|
||||
|
||||
bool log_get_8_bit_output(void) {
|
||||
return eight_bit_output;
|
||||
}
|
||||
|
||||
void log_set_stderr_mode(LogStderrMode mode) {
|
||||
stderr_mode = mode;
|
||||
}
|
||||
|
||||
LogStderrMode log_get_stderr_mode(void) {
|
||||
return stderr_mode;
|
||||
}
|
||||
|
||||
static inline void write_message(FILE* dest_io, LogLevel log_level, struct tm t, const char* format,
|
||||
va_list args) {
|
||||
fprintf(dest_io, "%04d-%02d-%02d %02d:%02d:%02d [%s]: ", t.tm_year + 1900, t.tm_mon + 1,
|
||||
t.tm_mday, t.tm_hour, t.tm_min, t.tm_sec, log_level_strings[log_level]);
|
||||
|
||||
vfprintf(dest_io, format, args);
|
||||
fprintf(dest_io, "\n");
|
||||
}
|
||||
|
||||
void log_message(LogLevel log_level, const char* format, ...) {
|
||||
void log_message(LogLevel log_level, char *format, ...) {
|
||||
if (log_level < current_log_level)
|
||||
return;
|
||||
if (log_level < 0 || log_level >= (int)(sizeof(log_level_strings) / sizeof(log_level_strings[0])))
|
||||
return;
|
||||
time_t now = time(NULL);
|
||||
struct tm t;
|
||||
if (!localtime_r(&now, &t))
|
||||
return;
|
||||
struct tm *t = localtime(&now);
|
||||
|
||||
FILE* dest_io = stdout;
|
||||
if (stderr_mode == LOG_STDERR_ALL || log_level == LOG_LEVEL_ERROR) {
|
||||
dest_io = stderr;
|
||||
}
|
||||
fprintf(stderr, "%04d-%02d-%02d %02d:%02d:%02d [%s]: ", t->tm_year + 1900,
|
||||
t->tm_mon + 1, t->tm_mday, t->tm_hour, t->tm_min, t->tm_sec,
|
||||
log_level_strings[log_level]);
|
||||
|
||||
va_list args;
|
||||
va_start(args, format);
|
||||
write_message(dest_io, log_level, t, format, args);
|
||||
vfprintf(stderr, format, args);
|
||||
va_end(args);
|
||||
|
||||
if (log_fp) {
|
||||
va_start(args, format);
|
||||
write_message(log_fp, log_level, t, format, args);
|
||||
va_end(args);
|
||||
}
|
||||
}
|
||||
|
||||
void log_debug_message(LogDebugFlag flag, const char* format, ...) {
|
||||
if (current_log_level > LOG_LEVEL_DEBUG || !(current_debug_flags & flag))
|
||||
return;
|
||||
|
||||
time_t now = time(NULL);
|
||||
struct tm t;
|
||||
if (!localtime_r(&now, &t))
|
||||
return;
|
||||
|
||||
va_list args;
|
||||
va_start(args, format);
|
||||
write_message(stdout, LOG_LEVEL_DEBUG, t, format, args);
|
||||
va_end(args);
|
||||
|
||||
if (log_fp) {
|
||||
va_start(args, format);
|
||||
write_message(log_fp, LOG_LEVEL_DEBUG, t, format, args);
|
||||
va_end(args);
|
||||
}
|
||||
}
|
||||
|
||||
void log_info_message(LogInfoFlag flag, const char* format, ...) {
|
||||
if ((info_flags_explicit && (info_flags & flag) == 0) ||
|
||||
(!info_flags_explicit && current_log_level > LOG_LEVEL_DEBUG))
|
||||
return;
|
||||
|
||||
time_t now = time(NULL);
|
||||
struct tm t;
|
||||
if (!localtime_r(&now, &t))
|
||||
return;
|
||||
|
||||
va_list args;
|
||||
va_start(args, format);
|
||||
write_message(stdout, LOG_LEVEL_INFO, t, format, args);
|
||||
va_end(args);
|
||||
|
||||
if (log_fp) {
|
||||
va_start(args, format);
|
||||
write_message(log_fp, LOG_LEVEL_INFO, t, format, args);
|
||||
va_end(args);
|
||||
}
|
||||
}
|
||||
|
||||
void log_perror(const char* context) {
|
||||
log_message(LOG_LEVEL_ERROR, "%s: %s", context, strerror(errno));
|
||||
fprintf(stderr, "\n");
|
||||
}
|
||||
|
||||
+6
-38
@@ -1,46 +1,14 @@
|
||||
#ifndef LOG_H
|
||||
#define LOG_H
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
|
||||
typedef enum { LOG_LEVEL_DEBUG, LOG_LEVEL_INFO, LOG_LEVEL_WARNING, LOG_LEVEL_ERROR } LogLevel;
|
||||
typedef enum { LOG_STDERR_ERRORS, LOG_STDERR_ALL } LogStderrMode;
|
||||
|
||||
typedef enum {
|
||||
LOG_DEBUG_IO = 1u << 0,
|
||||
LOG_DEBUG_PROTO = 1u << 1,
|
||||
LOG_DEBUG_PACK = 1u << 2,
|
||||
LOG_DEBUG_UTIL = 1u << 3,
|
||||
LOG_DEBUG_ALL = (1u << 4) - 1,
|
||||
} LogDebugFlag;
|
||||
LOG_LEVEL_DEBUG,
|
||||
LOG_LEVEL_INFO,
|
||||
LOG_LEVEL_WARNING,
|
||||
LOG_LEVEL_ERROR
|
||||
} LogLevel;
|
||||
|
||||
typedef enum {
|
||||
LOG_INFO_COPY = 1u << 0,
|
||||
LOG_INFO_MISC = 1u << 1,
|
||||
LOG_INFO_SKIP = 1u << 2,
|
||||
LOG_INFO_STATS = 1u << 3,
|
||||
LOG_INFO_ALL = LOG_INFO_COPY | LOG_INFO_MISC | LOG_INFO_SKIP | LOG_INFO_STATS,
|
||||
} LogInfoFlag;
|
||||
|
||||
void log_message(LogLevel log_level, const char* message, ...);
|
||||
void log_perror(const char* context);
|
||||
void log_message(LogLevel log_level, char *message, ...);
|
||||
void set_log_level(LogLevel level);
|
||||
void set_log_debug_flags(uint32_t flags);
|
||||
uint32_t get_log_debug_flags(void);
|
||||
/* True when a log_debug_message() call with the same flag would actually emit:
|
||||
* the debug log level is enabled AND the flag is selected. Hot paths use this
|
||||
* to skip expensive message formatting/escaping when the line is filtered. */
|
||||
bool log_debug_enabled(LogDebugFlag flag);
|
||||
void log_debug_message(LogDebugFlag flag, const char* message, ...);
|
||||
void set_log_info_flags(uint32_t flags);
|
||||
uint32_t get_log_info_flags(void);
|
||||
void log_info_message(LogInfoFlag flag, const char* message, ...);
|
||||
void log_set_file(FILE* fp);
|
||||
void log_set_8_bit_output(bool enabled);
|
||||
bool log_get_8_bit_output(void);
|
||||
void log_set_stderr_mode(LogStderrMode mode);
|
||||
LogStderrMode log_get_stderr_mode(void);
|
||||
|
||||
#endif
|
||||
|
||||
+49
-402
@@ -1,443 +1,90 @@
|
||||
#include "metadata.h"
|
||||
#include "file.h"
|
||||
#include "identity.h"
|
||||
#include "log.h"
|
||||
#include "protocol.h"
|
||||
#include "utils.h"
|
||||
#include <errno.h>
|
||||
#include <fcntl.h>
|
||||
#include <stdint.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/stat.h>
|
||||
#include <time.h>
|
||||
#include <unistd.h>
|
||||
|
||||
/*
|
||||
* Wire format serialization (protocol version 2.0.0+):
|
||||
* All metadata fields are serialized as fixed-width integers (int32_t / int64_t)
|
||||
* to ensure cross-platform binary compatiblity. See metadata.h for the
|
||||
* exact wire layout.
|
||||
*
|
||||
* Compile-time assertions verify that the native platform types fit within
|
||||
* the chosen fixed-width representations.
|
||||
*/
|
||||
typedef char static_assert_mode_t_fits[(sizeof(mode_t) <= sizeof(int32_t)) ? 1 : -1];
|
||||
typedef char static_assert_uid_t_fits[(sizeof(uid_t) <= sizeof(int32_t)) ? 1 : -1];
|
||||
typedef char static_assert_gid_t_fits[(sizeof(gid_t) <= sizeof(int32_t)) ? 1 : -1];
|
||||
|
||||
bool metadata_mtime_matches(time_t left_sec, long left_nsec, time_t right_sec, long right_nsec,
|
||||
int modify_window) {
|
||||
int64_t left = (int64_t)left_sec;
|
||||
int64_t right = (int64_t)right_sec;
|
||||
int64_t seconds;
|
||||
int64_t nanoseconds;
|
||||
|
||||
if (left > right || (left == right && left_nsec >= right_nsec)) {
|
||||
seconds = left - right;
|
||||
nanoseconds = (int64_t)left_nsec - (int64_t)right_nsec;
|
||||
} else {
|
||||
seconds = right - left;
|
||||
nanoseconds = (int64_t)right_nsec - (int64_t)left_nsec;
|
||||
}
|
||||
if (nanoseconds < 0) {
|
||||
seconds--;
|
||||
nanoseconds += 1000000000LL;
|
||||
}
|
||||
if (modify_window == 0)
|
||||
return left == right;
|
||||
return seconds < modify_window || (seconds == modify_window && nanoseconds == 0);
|
||||
}
|
||||
|
||||
void metadata_to_buf(char** buf, const FileMetadata* m) {
|
||||
int32_t present = (m != NULL) ? 1 : 0;
|
||||
memcpy(*buf, &present, sizeof(present));
|
||||
*buf += sizeof(present);
|
||||
void metadata_to_buf(char **buf, FileMetadata *m) {
|
||||
int present = (m != NULL) ? 1 : 0;
|
||||
memcpy(*buf, &present, sizeof(int));
|
||||
*buf += sizeof(int);
|
||||
if (m == NULL)
|
||||
return;
|
||||
int32_t mode = (int32_t)m->mode;
|
||||
memcpy(*buf, &mode, sizeof(mode));
|
||||
*buf += sizeof(mode);
|
||||
int32_t uid = (int32_t)m->uid;
|
||||
memcpy(*buf, &uid, sizeof(uid));
|
||||
*buf += sizeof(uid);
|
||||
int32_t gid = (int32_t)m->gid;
|
||||
memcpy(*buf, &gid, sizeof(gid));
|
||||
*buf += sizeof(gid);
|
||||
int64_t mtime_sec = (int64_t)m->mtime_sec;
|
||||
memcpy(*buf, &mtime_sec, sizeof(mtime_sec));
|
||||
*buf += sizeof(mtime_sec);
|
||||
int64_t mtime_nsec = (int64_t)m->mtime_nsec;
|
||||
memcpy(*buf, &mtime_nsec, sizeof(mtime_nsec));
|
||||
*buf += sizeof(mtime_nsec);
|
||||
int32_t atime_valid = m->atime_valid ? 1 : 0;
|
||||
memcpy(*buf, &atime_valid, sizeof(atime_valid));
|
||||
*buf += sizeof(atime_valid);
|
||||
int64_t atime_sec = (int64_t)m->atime_sec;
|
||||
memcpy(*buf, &atime_sec, sizeof(atime_sec));
|
||||
*buf += sizeof(atime_sec);
|
||||
int64_t atime_nsec = (int64_t)m->atime_nsec;
|
||||
memcpy(*buf, &atime_nsec, sizeof(atime_nsec));
|
||||
*buf += sizeof(atime_nsec);
|
||||
int32_t crtime_valid = m->crtime_valid ? 1 : 0;
|
||||
memcpy(*buf, &crtime_valid, sizeof(crtime_valid));
|
||||
*buf += sizeof(crtime_valid);
|
||||
int64_t crtime_sec = (int64_t)m->crtime_sec;
|
||||
memcpy(*buf, &crtime_sec, sizeof(crtime_sec));
|
||||
*buf += sizeof(crtime_sec);
|
||||
int64_t crtime_nsec = (int64_t)m->crtime_nsec;
|
||||
memcpy(*buf, &crtime_nsec, sizeof(crtime_nsec));
|
||||
*buf += sizeof(crtime_nsec);
|
||||
memcpy(*buf, &m->mode, sizeof(mode_t)); *buf += sizeof(mode_t);
|
||||
memcpy(*buf, &m->uid, sizeof(uid_t)); *buf += sizeof(uid_t);
|
||||
memcpy(*buf, &m->gid, sizeof(gid_t)); *buf += sizeof(gid_t);
|
||||
memcpy(*buf, &m->mtime_sec, sizeof(time_t)); *buf += sizeof(time_t);
|
||||
memcpy(*buf, &m->mtime_nsec, sizeof(long)); *buf += sizeof(long);
|
||||
}
|
||||
|
||||
FileMetadata* metadata_from_buf(char** buf) {
|
||||
int32_t present;
|
||||
memcpy(&present, *buf, sizeof(present));
|
||||
*buf += sizeof(present);
|
||||
if (present != 0 && present != 1)
|
||||
return NULL;
|
||||
FileMetadata *metadata_from_buf(char **buf) {
|
||||
int present;
|
||||
memcpy(&present, *buf, sizeof(int));
|
||||
*buf += sizeof(int);
|
||||
if (!present)
|
||||
return NULL;
|
||||
FileMetadata* m = protocol_alloc(sizeof(FileMetadata));
|
||||
if (m == NULL)
|
||||
return NULL;
|
||||
int32_t mode;
|
||||
memcpy(&mode, *buf, sizeof(mode));
|
||||
*buf += sizeof(mode);
|
||||
m->mode = (mode_t)mode;
|
||||
int32_t uid;
|
||||
memcpy(&uid, *buf, sizeof(uid));
|
||||
*buf += sizeof(uid);
|
||||
m->uid = (uid_t)uid;
|
||||
int32_t gid;
|
||||
memcpy(&gid, *buf, sizeof(gid));
|
||||
*buf += sizeof(gid);
|
||||
m->gid = (gid_t)gid;
|
||||
int64_t mtime_sec;
|
||||
memcpy(&mtime_sec, *buf, sizeof(mtime_sec));
|
||||
*buf += sizeof(mtime_sec);
|
||||
m->mtime_sec = (time_t)mtime_sec;
|
||||
int64_t mtime_nsec;
|
||||
memcpy(&mtime_nsec, *buf, sizeof(mtime_nsec));
|
||||
*buf += sizeof(mtime_nsec);
|
||||
m->mtime_nsec = (long)mtime_nsec;
|
||||
int32_t atime_valid;
|
||||
memcpy(&atime_valid, *buf, sizeof(atime_valid));
|
||||
*buf += sizeof(atime_valid);
|
||||
int64_t atime_sec;
|
||||
memcpy(&atime_sec, *buf, sizeof(atime_sec));
|
||||
*buf += sizeof(atime_sec);
|
||||
int64_t atime_nsec;
|
||||
memcpy(&atime_nsec, *buf, sizeof(atime_nsec));
|
||||
*buf += sizeof(atime_nsec);
|
||||
int32_t crtime_valid;
|
||||
memcpy(&crtime_valid, *buf, sizeof(crtime_valid));
|
||||
*buf += sizeof(crtime_valid);
|
||||
int64_t crtime_sec;
|
||||
memcpy(&crtime_sec, *buf, sizeof(crtime_sec));
|
||||
*buf += sizeof(crtime_sec);
|
||||
int64_t crtime_nsec;
|
||||
memcpy(&crtime_nsec, *buf, sizeof(crtime_nsec));
|
||||
*buf += sizeof(crtime_nsec);
|
||||
m->atime_valid = atime_valid != 0;
|
||||
m->atime_sec = (time_t)atime_sec;
|
||||
m->atime_nsec = (long)atime_nsec;
|
||||
m->crtime_valid = crtime_valid != 0;
|
||||
m->crtime_sec = (time_t)crtime_sec;
|
||||
m->crtime_nsec = (long)crtime_nsec;
|
||||
if (present != 1 || mtime_nsec < 0 || mtime_nsec >= 1000000000LL || mode < 0 || uid < 0 ||
|
||||
gid < 0 || atime_valid < 0 || atime_valid > 1 || crtime_valid < 0 || crtime_valid > 1 ||
|
||||
(atime_valid && (atime_nsec < 0 || atime_nsec >= 1000000000LL)) ||
|
||||
(crtime_valid && (crtime_nsec < 0 || crtime_nsec >= 1000000000LL))) {
|
||||
free(m);
|
||||
return NULL;
|
||||
}
|
||||
FileMetadata *m = malloc(sizeof(FileMetadata));
|
||||
memcpy(&m->mode, *buf, sizeof(mode_t)); *buf += sizeof(mode_t);
|
||||
memcpy(&m->uid, *buf, sizeof(uid_t)); *buf += sizeof(uid_t);
|
||||
memcpy(&m->gid, *buf, sizeof(gid_t)); *buf += sizeof(gid_t);
|
||||
memcpy(&m->mtime_sec, *buf, sizeof(time_t)); *buf += sizeof(time_t);
|
||||
memcpy(&m->mtime_nsec, *buf, sizeof(long)); *buf += sizeof(long);
|
||||
return m;
|
||||
}
|
||||
|
||||
bool metadata_send(int file_descriptor, const FileMetadata* m) {
|
||||
bool metadata_send(int file_descriptor, FileMetadata *m) {
|
||||
if (m == NULL) {
|
||||
int32_t zero = 0;
|
||||
return send_n_data(file_descriptor, &zero, sizeof(zero));
|
||||
int zero = 0;
|
||||
return send_n_data(file_descriptor, &zero, sizeof(int));
|
||||
}
|
||||
int32_t present = 1;
|
||||
int32_t mode = (int32_t)m->mode;
|
||||
int32_t uid = (int32_t)m->uid;
|
||||
int32_t gid = (int32_t)m->gid;
|
||||
int64_t mtime_sec = (int64_t)m->mtime_sec;
|
||||
int64_t mtime_nsec = (int64_t)m->mtime_nsec;
|
||||
int32_t atime_valid = m->atime_valid ? 1 : 0;
|
||||
int64_t atime_sec = (int64_t)m->atime_sec;
|
||||
int64_t atime_nsec = (int64_t)m->atime_nsec;
|
||||
int32_t crtime_valid = m->crtime_valid ? 1 : 0;
|
||||
int64_t crtime_sec = (int64_t)m->crtime_sec;
|
||||
int64_t crtime_nsec = (int64_t)m->crtime_nsec;
|
||||
return send_n_data(file_descriptor, &present, sizeof(present)) &&
|
||||
send_n_data(file_descriptor, &mode, sizeof(mode)) &&
|
||||
send_n_data(file_descriptor, &uid, sizeof(uid)) &&
|
||||
send_n_data(file_descriptor, &gid, sizeof(gid)) &&
|
||||
send_n_data(file_descriptor, &mtime_sec, sizeof(mtime_sec)) &&
|
||||
send_n_data(file_descriptor, &mtime_nsec, sizeof(mtime_nsec)) &&
|
||||
send_n_data(file_descriptor, &atime_valid, sizeof(atime_valid)) &&
|
||||
send_n_data(file_descriptor, &atime_sec, sizeof(atime_sec)) &&
|
||||
send_n_data(file_descriptor, &atime_nsec, sizeof(atime_nsec)) &&
|
||||
send_n_data(file_descriptor, &crtime_valid, sizeof(crtime_valid)) &&
|
||||
send_n_data(file_descriptor, &crtime_sec, sizeof(crtime_sec)) &&
|
||||
send_n_data(file_descriptor, &crtime_nsec, sizeof(crtime_nsec));
|
||||
int present = 1;
|
||||
return send_n_data(file_descriptor, &present, sizeof(int)) &&
|
||||
send_n_data(file_descriptor, &m->mode, sizeof(mode_t)) &&
|
||||
send_n_data(file_descriptor, &m->uid, sizeof(uid_t)) &&
|
||||
send_n_data(file_descriptor, &m->gid, sizeof(gid_t)) &&
|
||||
send_n_data(file_descriptor, &m->mtime_sec, sizeof(time_t)) &&
|
||||
send_n_data(file_descriptor, &m->mtime_nsec, sizeof(long));
|
||||
}
|
||||
|
||||
FileMetadata* metadata_receive(int file_descriptor, int* ok) {
|
||||
int32_t present;
|
||||
if (!receive_n_data(file_descriptor, &present, sizeof(present))) {
|
||||
if (ok)
|
||||
*ok = 0;
|
||||
FileMetadata *metadata_receive(int file_descriptor, int *ok) {
|
||||
int present;
|
||||
if (!receive_n_data(file_descriptor, &present, sizeof(int))) {
|
||||
if (ok) *ok = 0;
|
||||
return NULL;
|
||||
}
|
||||
if (present == 0) {
|
||||
if (ok)
|
||||
*ok = 1;
|
||||
if (!present) {
|
||||
if (ok) *ok = 1;
|
||||
return NULL;
|
||||
}
|
||||
if (present != 1) {
|
||||
if (ok)
|
||||
*ok = 0;
|
||||
return NULL;
|
||||
}
|
||||
FileMetadata* m = protocol_alloc(sizeof(FileMetadata));
|
||||
if (m == NULL) {
|
||||
if (ok)
|
||||
*ok = 0;
|
||||
return NULL;
|
||||
}
|
||||
int32_t mode;
|
||||
if (!receive_n_data(file_descriptor, &mode, sizeof(mode))) {
|
||||
FileMetadata *m = malloc(sizeof(FileMetadata));
|
||||
if (m == NULL) { if (ok) *ok = 0; return NULL; }
|
||||
if (!receive_n_data(file_descriptor, &m->mode, sizeof(mode_t)) ||
|
||||
!receive_n_data(file_descriptor, &m->uid, sizeof(uid_t)) ||
|
||||
!receive_n_data(file_descriptor, &m->gid, sizeof(gid_t)) ||
|
||||
!receive_n_data(file_descriptor, &m->mtime_sec, sizeof(time_t)) ||
|
||||
!receive_n_data(file_descriptor, &m->mtime_nsec, sizeof(long))) {
|
||||
free(m);
|
||||
if (ok)
|
||||
*ok = 0;
|
||||
if (ok) *ok = 0;
|
||||
return NULL;
|
||||
}
|
||||
m->mode = (mode_t)mode;
|
||||
int32_t uid;
|
||||
if (!receive_n_data(file_descriptor, &uid, sizeof(uid))) {
|
||||
free(m);
|
||||
if (ok)
|
||||
*ok = 0;
|
||||
return NULL;
|
||||
}
|
||||
m->uid = (uid_t)uid;
|
||||
int32_t gid;
|
||||
if (!receive_n_data(file_descriptor, &gid, sizeof(gid))) {
|
||||
free(m);
|
||||
if (ok)
|
||||
*ok = 0;
|
||||
return NULL;
|
||||
}
|
||||
m->gid = (gid_t)gid;
|
||||
int64_t mtime_sec;
|
||||
if (!receive_n_data(file_descriptor, &mtime_sec, sizeof(mtime_sec))) {
|
||||
free(m);
|
||||
if (ok)
|
||||
*ok = 0;
|
||||
return NULL;
|
||||
}
|
||||
m->mtime_sec = (time_t)mtime_sec;
|
||||
int64_t mtime_nsec;
|
||||
if (!receive_n_data(file_descriptor, &mtime_nsec, sizeof(mtime_nsec))) {
|
||||
free(m);
|
||||
if (ok)
|
||||
*ok = 0;
|
||||
return NULL;
|
||||
}
|
||||
m->mtime_nsec = (long)mtime_nsec;
|
||||
int32_t atime_valid;
|
||||
if (!receive_n_data(file_descriptor, &atime_valid, sizeof(atime_valid))) {
|
||||
free(m);
|
||||
if (ok)
|
||||
*ok = 0;
|
||||
return NULL;
|
||||
}
|
||||
int64_t atime_sec;
|
||||
if (!receive_n_data(file_descriptor, &atime_sec, sizeof(atime_sec))) {
|
||||
free(m);
|
||||
if (ok)
|
||||
*ok = 0;
|
||||
return NULL;
|
||||
}
|
||||
int64_t atime_nsec;
|
||||
if (!receive_n_data(file_descriptor, &atime_nsec, sizeof(atime_nsec))) {
|
||||
free(m);
|
||||
if (ok)
|
||||
*ok = 0;
|
||||
return NULL;
|
||||
}
|
||||
int32_t crtime_valid;
|
||||
if (!receive_n_data(file_descriptor, &crtime_valid, sizeof(crtime_valid))) {
|
||||
free(m);
|
||||
if (ok)
|
||||
*ok = 0;
|
||||
return NULL;
|
||||
}
|
||||
int64_t crtime_sec;
|
||||
if (!receive_n_data(file_descriptor, &crtime_sec, sizeof(crtime_sec))) {
|
||||
free(m);
|
||||
if (ok)
|
||||
*ok = 0;
|
||||
return NULL;
|
||||
}
|
||||
int64_t crtime_nsec;
|
||||
if (!receive_n_data(file_descriptor, &crtime_nsec, sizeof(crtime_nsec))) {
|
||||
free(m);
|
||||
if (ok)
|
||||
*ok = 0;
|
||||
return NULL;
|
||||
}
|
||||
m->atime_valid = atime_valid != 0;
|
||||
m->atime_sec = (time_t)atime_sec;
|
||||
m->atime_nsec = (long)atime_nsec;
|
||||
m->crtime_valid = crtime_valid != 0;
|
||||
m->crtime_sec = (time_t)crtime_sec;
|
||||
m->crtime_nsec = (long)crtime_nsec;
|
||||
if (mtime_nsec < 0 || mtime_nsec >= 1000000000LL || mode < 0 || uid < 0 || gid < 0 ||
|
||||
atime_valid < 0 || atime_valid > 1 || crtime_valid < 0 || crtime_valid > 1 ||
|
||||
(atime_valid && (atime_nsec < 0 || atime_nsec >= 1000000000LL)) ||
|
||||
(crtime_valid && (crtime_nsec < 0 || crtime_nsec >= 1000000000LL))) {
|
||||
free(m);
|
||||
if (ok)
|
||||
*ok = 0;
|
||||
return NULL;
|
||||
}
|
||||
if (ok)
|
||||
*ok = 1;
|
||||
if (ok) *ok = 1;
|
||||
return m;
|
||||
}
|
||||
|
||||
static mode_t metadata_mode(const FileMetadata* metadata, mode_t current_mode,
|
||||
bool preserve_executability) {
|
||||
const mode_t execute_bits = S_IXUSR | S_IXGRP | S_IXOTH;
|
||||
if (preserve_executability)
|
||||
return (current_mode & 0777 & ~execute_bits) | (metadata->mode & execute_bits);
|
||||
return metadata->mode & 0777 & ~(S_IWGRP | S_IWOTH);
|
||||
}
|
||||
|
||||
void file_restore_metadata(const char* path, const FileMetadata* metadata,
|
||||
bool preserve_executability) {
|
||||
void file_restore_metadata(const char *path, FileMetadata *metadata) {
|
||||
if (metadata == NULL)
|
||||
return;
|
||||
struct stat current;
|
||||
mode_t current_mode = stat(path, ¤t) == 0 ? current.st_mode : 0;
|
||||
mode_t safe_mode = metadata_mode(metadata, current_mode, preserve_executability);
|
||||
if (chmod(path, safe_mode) != 0) {
|
||||
char* escaped_path = output_escape(path, log_get_8_bit_output());
|
||||
log_message(LOG_LEVEL_WARNING, "Failed to chmod %s: %s",
|
||||
escaped_path ? escaped_path : "<allocation failed>", strerror(errno));
|
||||
free(escaped_path);
|
||||
}
|
||||
/* Never apply client-supplied ownership. The descriptor API below is the
|
||||
receiver write path; retain this legacy API only for compatibility. */
|
||||
chmod(path, metadata->mode & 07777);
|
||||
int chown_ret = chown(path, metadata->uid, metadata->gid);
|
||||
(void)chown_ret;
|
||||
struct timespec times[2];
|
||||
times[0].tv_sec = 0;
|
||||
times[0].tv_nsec = UTIME_OMIT;
|
||||
times[1].tv_sec = metadata->mtime_sec;
|
||||
times[1].tv_nsec = metadata->mtime_nsec;
|
||||
if (metadata->atime_valid) {
|
||||
times[0].tv_sec = metadata->atime_sec;
|
||||
times[0].tv_nsec = metadata->atime_nsec;
|
||||
}
|
||||
if (metadata->crtime_valid) {
|
||||
log_message(LOG_LEVEL_DEBUG,
|
||||
"crtime (birth time) %lld.%09ld transmitted for %s but not applied: no portable "
|
||||
"setter exists",
|
||||
(long long)metadata->crtime_sec, metadata->crtime_nsec, path);
|
||||
}
|
||||
if (utimensat(AT_FDCWD, path, times, 0) != 0) {
|
||||
char* escaped_path = output_escape(path, log_get_8_bit_output());
|
||||
log_message(LOG_LEVEL_WARNING, "Failed to set timestamps on %s: %s",
|
||||
escaped_path ? escaped_path : "<allocation failed>", strerror(errno));
|
||||
free(escaped_path);
|
||||
}
|
||||
}
|
||||
|
||||
bool file_restore_symlink_metadata(const char* path, const FileMetadata* metadata,
|
||||
bool omit_link_times) {
|
||||
if (path == NULL || metadata == NULL)
|
||||
return !identity_copy_as_active();
|
||||
char* leaf = NULL;
|
||||
int parent_fd = file_open_secure_parent(path, &leaf, false);
|
||||
if (parent_fd < 0)
|
||||
return !identity_copy_as_active();
|
||||
/* Ownership (only when the identity policy is active) via lchown semantics:
|
||||
fchownat with AT_SYMLINK_NOFOLLOW never dereferences the link. A failed
|
||||
REQUIRED --copy-as ownership marks the entry failed; every other policy is
|
||||
best-effort. */
|
||||
bool owned = identity_apply_ownership_link(parent_fd, leaf, (int32_t)metadata->uid,
|
||||
(int32_t)metadata->gid);
|
||||
/* Symlink mode: not settable on Linux (fchmodat AT_SYMLINK_NOFOLLOW returns
|
||||
EOPNOTSUPP/ENOTSUP); attempt it for platforms that support it and quietly
|
||||
ignore the unsupported case so the transfer never fails over it. */
|
||||
mode_t link_mode = metadata->mode & 0777;
|
||||
if (fchmodat(parent_fd, leaf, link_mode, AT_SYMLINK_NOFOLLOW) != 0 && errno != EOPNOTSUPP &&
|
||||
errno != ENOTSUP && errno != ENOSYS) {
|
||||
log_message(LOG_LEVEL_DEBUG, "Could not set symlink mode on %s: %s", path, strerror(errno));
|
||||
}
|
||||
if (!omit_link_times) {
|
||||
struct timespec times[2] = {{.tv_sec = 0, .tv_nsec = UTIME_OMIT},
|
||||
{.tv_sec = metadata->mtime_sec, .tv_nsec = metadata->mtime_nsec}};
|
||||
if (metadata->atime_valid) {
|
||||
times[0].tv_sec = metadata->atime_sec;
|
||||
times[0].tv_nsec = metadata->atime_nsec;
|
||||
}
|
||||
if (utimensat(parent_fd, leaf, times, AT_SYMLINK_NOFOLLOW) != 0) {
|
||||
char* escaped_path = output_escape(path, log_get_8_bit_output());
|
||||
log_message(LOG_LEVEL_WARNING, "Failed to set symlink timestamps on %s: %s",
|
||||
escaped_path ? escaped_path : "<allocation failed>", strerror(errno));
|
||||
free(escaped_path);
|
||||
}
|
||||
}
|
||||
close(parent_fd);
|
||||
free(leaf);
|
||||
return owned;
|
||||
}
|
||||
|
||||
bool file_restore_metadata_fd(int fd, const FileMetadata* metadata, bool preserve_executability) {
|
||||
if (fd < 0 || metadata == NULL)
|
||||
return metadata == NULL;
|
||||
bool ok = true;
|
||||
struct stat current;
|
||||
if (fstat(fd, ¤t) != 0)
|
||||
return false;
|
||||
mode_t safe_mode = metadata_mode(metadata, current.st_mode, preserve_executability);
|
||||
if (fchmod(fd, safe_mode) != 0)
|
||||
ok = false;
|
||||
/* Client uid/gid values are deliberately not authoritative UNLESS the client
|
||||
explicitly opted in with an identity flag (--numeric-ids / --usermap /
|
||||
--groupmap / --chown). identity_apply_ownership is the controlled,
|
||||
privilege-gated path: it consults the negotiated policy, resolves the
|
||||
target ids, and applies them via an fd-relative fchown() that is confined
|
||||
to the just-written file (EPERM/EACCES are logged, never fatal) -- EXCEPT
|
||||
for an active --copy-as, whose forced ownership is REQUIRED: a failure
|
||||
marks this entry as failed instead of reporting a wrong-owner write as
|
||||
success. With no identity flag set it is a no-op, so a default or plain -M
|
||||
transfer keeps FastSync's existing behavior of never applying client
|
||||
ownership. */
|
||||
if (!identity_apply_ownership(fd, (int32_t)metadata->uid, (int32_t)metadata->gid))
|
||||
ok = false;
|
||||
struct timespec times[2] = {{.tv_sec = 0, .tv_nsec = UTIME_OMIT},
|
||||
{.tv_sec = metadata->mtime_sec, .tv_nsec = metadata->mtime_nsec}};
|
||||
if (metadata->atime_valid) {
|
||||
times[0].tv_sec = metadata->atime_sec;
|
||||
times[0].tv_nsec = metadata->atime_nsec;
|
||||
}
|
||||
/* --crtimes captures and transmits the source birth time, but there is no
|
||||
* portable way to set a birth time (utimensat can only set atime/mtime), so
|
||||
* the receiver deliberately does NOT apply it. This is explicit, honest
|
||||
* unsupported-attribute handling: log a debug note and continue — never fail
|
||||
* the transfer and never pretend the crtime was applied. */
|
||||
if (metadata->crtime_valid) {
|
||||
log_message(LOG_LEVEL_DEBUG,
|
||||
"crtime (birth time) %lld.%09ld transmitted but not applied: no portable setter",
|
||||
(long long)metadata->crtime_sec, metadata->crtime_nsec);
|
||||
}
|
||||
if (futimens(fd, times) != 0)
|
||||
ok = false;
|
||||
return ok;
|
||||
utimensat(AT_FDCWD, path, times, 0);
|
||||
}
|
||||
|
||||
+6
-47
@@ -3,55 +3,14 @@
|
||||
|
||||
#include "file.h"
|
||||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
#include <sys/stat.h>
|
||||
#include <time.h>
|
||||
|
||||
/*
|
||||
* Wire format (introduced in protocol version 2.0.0):
|
||||
* int32_t present
|
||||
* int32_t mode (was mode_t, platform-dependent)
|
||||
* int32_t uid (was uid_t, platform-dependent)
|
||||
* int32_t gid (was gid_t, platform-dependent)
|
||||
* int64_t mtime_sec (was time_t, platform-dependent)
|
||||
* int64_t mtime_nsec (was long, platform-dependent)
|
||||
* int32_t atime_valid (-U/--atimes; protocol 2.12.0)
|
||||
* int64_t atime_sec
|
||||
* int64_t atime_nsec
|
||||
* int32_t crtime_valid (-N/--crtimes; protocol 2.12.0)
|
||||
* int64_t crtime_sec
|
||||
* int64_t crtime_nsec
|
||||
*
|
||||
* Prior to 2.0.0 the wire format used the raw platform-dependent types,
|
||||
* which broke compatiblity across different systems. All fields are now
|
||||
* serialized as fixed-width integers.
|
||||
*/
|
||||
#define FILE_METADATA_WIRE_SIZE (sizeof(mode_t) + sizeof(uid_t) + sizeof(gid_t) + sizeof(time_t) + sizeof(long))
|
||||
|
||||
/* Size of metadata fields on wire, excluding the int32_t `present` field that
|
||||
* is always sent first. The total wire size for present metadata is
|
||||
* sizeof(int32_t) + FILE_METADATA_WIRE_SIZE (68 bytes on most platforms). */
|
||||
#define FILE_METADATA_WIRE_SIZE (sizeof(int32_t) * 5 + sizeof(int64_t) * 6)
|
||||
|
||||
void metadata_to_buf(char** buf, const FileMetadata* m);
|
||||
FileMetadata* metadata_from_buf(char** buf);
|
||||
bool metadata_send(int file_descriptor, const FileMetadata* m);
|
||||
FileMetadata* metadata_receive(int file_descriptor, int* ok);
|
||||
void file_restore_metadata(const char* path, const FileMetadata* metadata,
|
||||
bool preserve_executability);
|
||||
bool file_restore_metadata_fd(int fd, const FileMetadata* metadata, bool preserve_executability);
|
||||
/* P7 Wave D: apply a SYMLINK's own metadata using no-follow primitives only
|
||||
* (utimensat/lchown/fchmodat with AT_SYMLINK_NOFOLLOW), confined fd-relative
|
||||
* under the authorized root. `omit_link_times` (-J/--omit-link-times)
|
||||
* suppresses the timestamps; the link's mode/ownership are still attempted
|
||||
* (ownership stays gated by the identity policy and by default is not applied).
|
||||
* A null metadata or an unfollowable parent is a harmless no-op. Returns false
|
||||
* only when a REQUIRED --copy-as ownership application failed, so the caller can
|
||||
* report the entry as failed instead of claiming a wrong-owner success. */
|
||||
bool file_restore_symlink_metadata(const char* path, const FileMetadata* metadata,
|
||||
bool omit_link_times);
|
||||
|
||||
/* Compare timestamps using rsync's whole-second modification window. */
|
||||
bool metadata_mtime_matches(time_t left_sec, long left_nsec, time_t right_sec, long right_nsec,
|
||||
int modify_window);
|
||||
void metadata_to_buf(char **buf, FileMetadata *m);
|
||||
FileMetadata *metadata_from_buf(char **buf);
|
||||
bool metadata_send(int file_descriptor, FileMetadata *m);
|
||||
FileMetadata *metadata_receive(int file_descriptor, int *ok);
|
||||
void file_restore_metadata(const char *path, FileMetadata *metadata);
|
||||
|
||||
#endif
|
||||
|
||||
@@ -1,76 +0,0 @@
|
||||
#include "motd.h"
|
||||
#include "protocol.h"
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
char* motd_read_file(const char* path) {
|
||||
if (!path || path[0] == '\0')
|
||||
return NULL;
|
||||
FILE* fp = fopen(path, "rb");
|
||||
if (!fp)
|
||||
return NULL;
|
||||
char* buffer = malloc(MOTD_MAX_BYTES + 1);
|
||||
if (!buffer) {
|
||||
fclose(fp);
|
||||
return NULL;
|
||||
}
|
||||
/* fread stops at the bound; a larger file is truncated rather than read
|
||||
* unbounded. ferror distinguishes a truncated read from an I/O failure. */
|
||||
size_t total = fread(buffer, 1, MOTD_MAX_BYTES, fp);
|
||||
if (ferror(fp)) {
|
||||
free(buffer);
|
||||
fclose(fp);
|
||||
return NULL;
|
||||
}
|
||||
fclose(fp);
|
||||
buffer[total] = '\0';
|
||||
return buffer;
|
||||
}
|
||||
|
||||
char* motd_render(const char* motd, bool eight_bit_output) {
|
||||
if (!motd)
|
||||
return NULL;
|
||||
size_t length = strlen(motd);
|
||||
if (length > (SIZE_MAX - 1) / 5)
|
||||
return NULL;
|
||||
char* rendered = malloc(length * 5 + 1);
|
||||
if (!rendered)
|
||||
return NULL;
|
||||
size_t out = 0;
|
||||
for (size_t i = 0; i < length; i++) {
|
||||
unsigned char byte = (unsigned char)motd[i];
|
||||
if (byte == '\n' || byte == '\t') {
|
||||
rendered[out++] = (char)byte;
|
||||
} else if ((byte >= 32 && byte <= 126) || (eight_bit_output && byte >= 128)) {
|
||||
rendered[out++] = (char)byte;
|
||||
} else {
|
||||
rendered[out++] = '\\';
|
||||
rendered[out++] = '#';
|
||||
rendered[out++] = (char)('0' + ((byte >> 6) & 7));
|
||||
rendered[out++] = (char)('0' + ((byte >> 3) & 7));
|
||||
rendered[out++] = (char)('0' + (byte & 7));
|
||||
}
|
||||
}
|
||||
rendered[out] = '\0';
|
||||
return rendered;
|
||||
}
|
||||
|
||||
bool motd_send(int file_descriptor, const char* motd) {
|
||||
return send_str(file_descriptor, motd ? motd : "");
|
||||
}
|
||||
|
||||
char* motd_receive(int file_descriptor) {
|
||||
char* motd = receive_str(file_descriptor);
|
||||
if (!motd)
|
||||
return NULL;
|
||||
/* Guard against a hostile/oversized peer: receive_str already bounded the
|
||||
* frame at MAX_STRING_SIZE and consumed it, so discarding an over-bound
|
||||
* body here keeps the stream framed while refusing to display it. */
|
||||
if (strlen(motd) > MOTD_MAX_BYTES) {
|
||||
free(motd);
|
||||
return NULL;
|
||||
}
|
||||
return motd;
|
||||
}
|
||||
@@ -1,54 +0,0 @@
|
||||
#ifndef MOTD_H
|
||||
#define MOTD_H
|
||||
|
||||
#include <stdbool.h>
|
||||
|
||||
/* Daemon Message-Of-The-Day (Wave C).
|
||||
*
|
||||
* The daemon listener (fastsync-server --daemon) may advertise a `motd file`
|
||||
* configured in its globals. When a client connects with a host::module/path
|
||||
* destination and the module gate accepts the connection, the server sends the
|
||||
* MOTD as a single string frame BEFORE any transfer data (rsync sends its MOTD
|
||||
* as the first thing from the server at the start of a daemon connection).
|
||||
* The client reads that frame right after the config/status handshake and
|
||||
* displays it on stdout unless --no-motd was given.
|
||||
*
|
||||
* The MOTD is ordinary display text, never a secret, so it uses the normal
|
||||
* (non-redacted) string primitive. The exchange is strictly server->client
|
||||
* and happens on the daemon listener path only; the --stdio SSH path has no
|
||||
* MOTD.
|
||||
*
|
||||
* No PROTOCOL_VERSION bump is involved: the frame is sent and read
|
||||
* symmetrically by every 2.15.0 daemon build (the strict same-version
|
||||
* handshake rejects any other version before the frame), so it cannot
|
||||
* desynchronize a peer. */
|
||||
|
||||
/* Upper bound on the MOTD bytes the server will read from disk and put on the
|
||||
* wire. Kept far below MAX_STRING_SIZE (64 KB) so a huge/hostile motd file
|
||||
* can never produce an unbounded frame or allocation. */
|
||||
#define MOTD_MAX_BYTES 4096
|
||||
|
||||
/* Read a daemon MOTD file, bounded to MOTD_MAX_BYTES. Returns a malloc'd
|
||||
* NUL-terminated copy of the file content (bytes beyond the bound are
|
||||
* truncated) or NULL when path is NULL/empty, the file cannot be opened or
|
||||
* read, or allocation fails. An absent or unreadable motd file is NOT an
|
||||
* error: the caller simply sends an empty MOTD frame and continues. */
|
||||
char* motd_read_file(const char* path);
|
||||
|
||||
/* Render MOTD text for terminal display. Newlines and tabs are preserved so
|
||||
* a multi-line motd still reads naturally, while every other non-printable /
|
||||
* control byte (ESC included) is escaped with FastSync's `\NNN` octal
|
||||
* convention, so a hostile server cannot inject terminal escape sequences
|
||||
* through the MOTD. eight_bit_output keeps bytes >= 0x80 verbatim (matching
|
||||
* --8-bit-output). Returns a malloc'd string or NULL on allocation failure. */
|
||||
char* motd_render(const char* motd, bool eight_bit_output);
|
||||
|
||||
/* Send/receive the MOTD string frame. These wrap the normal string
|
||||
* primitive: the MOTD is not a credential, so no redaction is used. The
|
||||
* receiver additionally rejects an over-bound frame (> MOTD_MAX_BYTES) as a
|
||||
* hostile input guard; the frame itself is always fully consumed first, so the
|
||||
* stream stays framed. */
|
||||
bool motd_send(int file_descriptor, const char* motd);
|
||||
char* motd_receive(int file_descriptor);
|
||||
|
||||
#endif
|
||||
+80
-286
@@ -1,6 +1,4 @@
|
||||
#include "multiprocessing.h"
|
||||
#include "receiver.h"
|
||||
|
||||
#include "array_list.h"
|
||||
#include "chunk.h"
|
||||
#include "config.h"
|
||||
@@ -15,101 +13,34 @@
|
||||
#include <string.h>
|
||||
#include <threads.h>
|
||||
|
||||
PipelineContextSender* pipeline_context_sender_create(Config* config, Queue* queue_scanner,
|
||||
Queue* queue_loader) {
|
||||
PipelineContextSender* context = malloc(sizeof(PipelineContextSender));
|
||||
if (context == NULL)
|
||||
return NULL;
|
||||
PipelineContextSender *pipeline_context_sender_create(Config *config,
|
||||
Queue *queue_scanner,
|
||||
Queue *queue_loader) {
|
||||
PipelineContextSender *context = malloc(sizeof(PipelineContextSender));
|
||||
if (context == NULL) return NULL;
|
||||
context->config = config;
|
||||
context->queue_scanner = queue_scanner;
|
||||
context->queue_loader = queue_loader;
|
||||
context->scanner_done = false;
|
||||
context->loader_done = false;
|
||||
context->manifest = NULL;
|
||||
context->excluded_paths = NULL;
|
||||
context->missing_args = NULL;
|
||||
context->scan_had_io_error = false;
|
||||
context->remove_source_files = NULL;
|
||||
context->early_delete = false;
|
||||
context->scan_stopped_early = false;
|
||||
context->total_files = 0;
|
||||
context->progress_bytes = 0;
|
||||
context->total_bytes = 0;
|
||||
context->sender_done = false;
|
||||
atomic_init(&context->cancelled, false);
|
||||
protocol_session_init(&context->allocation_session, -1, -1);
|
||||
protocol_session_set_max_alloc(&context->allocation_session, config->max_alloc);
|
||||
context->dir_entries = NULL;
|
||||
context->dir_entries_mutex_init = false;
|
||||
int init = 0;
|
||||
if (config->use_metadata) {
|
||||
context->dir_entries = array_list_create(file_destroy);
|
||||
if (!context->dir_entries)
|
||||
goto fail;
|
||||
if (mtx_init(&context->mutex_scanner, mtx_plain) != thrd_success ||
|
||||
cnd_init(&context->condition_not_full_scanner) != thrd_success ||
|
||||
cnd_init(&context->condition_not_empty_scanner) != thrd_success ||
|
||||
mtx_init(&context->mutex_loader, mtx_plain) != thrd_success ||
|
||||
cnd_init(&context->condition_not_full_loader) != thrd_success ||
|
||||
cnd_init(&context->condition_not_empty_loader) != thrd_success) {
|
||||
perror("Error initializing synchronization objects");
|
||||
free(context);
|
||||
return NULL;
|
||||
}
|
||||
if (mtx_init(&context->mutex_scanner, mtx_plain) != thrd_success)
|
||||
goto fail;
|
||||
init++;
|
||||
if (cnd_init(&context->condition_not_full_scanner) != thrd_success)
|
||||
goto fail;
|
||||
init++;
|
||||
if (cnd_init(&context->condition_not_empty_scanner) != thrd_success)
|
||||
goto fail;
|
||||
init++;
|
||||
if (mtx_init(&context->mutex_loader, mtx_plain) != thrd_success)
|
||||
goto fail;
|
||||
init++;
|
||||
if (cnd_init(&context->condition_not_full_loader) != thrd_success)
|
||||
goto fail;
|
||||
init++;
|
||||
if (cnd_init(&context->condition_not_empty_loader) != thrd_success)
|
||||
goto fail;
|
||||
init++;
|
||||
if (mtx_init(&context->mutex_progress, mtx_plain) != thrd_success)
|
||||
goto fail;
|
||||
// cppcheck-suppress unreadVariable
|
||||
init++;
|
||||
if (mtx_init(&context->dir_entries_mutex, mtx_plain) != thrd_success)
|
||||
goto fail;
|
||||
context->dir_entries_mutex_init = true;
|
||||
return context;
|
||||
|
||||
fail:
|
||||
log_perror("Error initializing synchronization objects");
|
||||
if (context->dir_entries_mutex_init)
|
||||
mtx_destroy(&context->dir_entries_mutex);
|
||||
if (context->dir_entries)
|
||||
array_list_delete(context->dir_entries);
|
||||
if (init >= 6)
|
||||
cnd_destroy(&context->condition_not_empty_loader);
|
||||
if (init >= 5)
|
||||
cnd_destroy(&context->condition_not_full_loader);
|
||||
if (init >= 4)
|
||||
mtx_destroy(&context->mutex_loader);
|
||||
if (init >= 3)
|
||||
cnd_destroy(&context->condition_not_empty_scanner);
|
||||
if (init >= 2)
|
||||
cnd_destroy(&context->condition_not_full_scanner);
|
||||
if (init >= 1)
|
||||
mtx_destroy(&context->mutex_scanner);
|
||||
free(context);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
void pipeline_context_sender_destroy(PipelineContextSender* context) {
|
||||
void pipeline_context_sender_destroy(PipelineContextSender *context) {
|
||||
if (context->manifest) {
|
||||
array_list_delete(context->manifest);
|
||||
}
|
||||
if (context->excluded_paths)
|
||||
array_list_delete(context->excluded_paths);
|
||||
if (context->missing_args)
|
||||
array_list_delete(context->missing_args);
|
||||
if (context->remove_source_files)
|
||||
array_list_delete(context->remove_source_files);
|
||||
if (context->dir_entries)
|
||||
array_list_delete(context->dir_entries);
|
||||
if (context->dir_entries_mutex_init)
|
||||
mtx_destroy(&context->dir_entries_mutex);
|
||||
config_delete(context->config);
|
||||
queue_destroy(context->queue_scanner);
|
||||
queue_destroy(context->queue_loader);
|
||||
@@ -119,251 +50,114 @@ void pipeline_context_sender_destroy(PipelineContextSender* context) {
|
||||
mtx_destroy(&context->mutex_loader);
|
||||
cnd_destroy(&context->condition_not_full_loader);
|
||||
cnd_destroy(&context->condition_not_empty_loader);
|
||||
mtx_destroy(&context->mutex_progress);
|
||||
free(context);
|
||||
}
|
||||
|
||||
PipelineContextReceiver* pipeline_context_receiver_create(Config* config, Queue* queue,
|
||||
int file_descriptor, SSL* ssl) {
|
||||
PipelineContextReceiver* context = malloc(sizeof(PipelineContextReceiver));
|
||||
if (context == NULL)
|
||||
return NULL;
|
||||
PipelineContextReceiver *pipeline_context_receiver_create(Config *config,
|
||||
Queue *queue,
|
||||
int file_descriptor) {
|
||||
PipelineContextReceiver *context = malloc(sizeof(PipelineContextReceiver));
|
||||
if (context == NULL) return NULL;
|
||||
context->config = config;
|
||||
context->queue = queue;
|
||||
context->file_descriptor = file_descriptor;
|
||||
context->ssl = ssl;
|
||||
context->outcomes.entries = NULL;
|
||||
context->outcomes.count = 0;
|
||||
context->outcomes.capacity = 0;
|
||||
dir_time_list_init(&context->dir_times);
|
||||
protocol_session_init(&context->session, file_descriptor, file_descriptor);
|
||||
protocol_session_set_ssl(&context->session, ssl);
|
||||
context->receiver_done = false;
|
||||
context->queued_bytes = 0;
|
||||
context->max_queue_bytes = 0;
|
||||
context->deferred_manifest = NULL;
|
||||
atomic_init(&context->cancelled, false);
|
||||
int init = 0;
|
||||
if (mtx_init(&context->mutex, mtx_plain) != thrd_success)
|
||||
goto fail;
|
||||
init++;
|
||||
if (cnd_init(&context->condition_not_full) != thrd_success)
|
||||
goto fail;
|
||||
init++;
|
||||
if (cnd_init(&context->condition_not_empty) != thrd_success)
|
||||
goto fail;
|
||||
// cppcheck-suppress unreadVariable
|
||||
init++;
|
||||
if (mtx_init(&context->mutex, mtx_plain) != thrd_success ||
|
||||
cnd_init(&context->condition_not_full) != thrd_success ||
|
||||
cnd_init(&context->condition_not_empty) != thrd_success) {
|
||||
perror("Error initializing synchronization objects");
|
||||
free(context);
|
||||
return NULL;
|
||||
}
|
||||
return context;
|
||||
|
||||
fail:
|
||||
log_perror("Error initializing synchronization objects");
|
||||
if (init >= 3)
|
||||
cnd_destroy(&context->condition_not_empty);
|
||||
if (init >= 2)
|
||||
cnd_destroy(&context->condition_not_full);
|
||||
if (init >= 1)
|
||||
mtx_destroy(&context->mutex);
|
||||
free(context);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
void pipeline_context_receiver_destroy(PipelineContextReceiver* context) {
|
||||
void pipeline_context_receiver_destroy(PipelineContextReceiver *context) {
|
||||
config_delete(context->config);
|
||||
if (context->deferred_manifest)
|
||||
delete_manifest_free(context->deferred_manifest);
|
||||
queue_destroy(context->queue);
|
||||
receiver_outcomes_destroy(&context->outcomes);
|
||||
dir_time_list_free(&context->dir_times);
|
||||
mtx_destroy(&context->mutex);
|
||||
cnd_destroy(&context->condition_not_full);
|
||||
cnd_destroy(&context->condition_not_empty);
|
||||
free(context);
|
||||
}
|
||||
|
||||
void pipeline_context_receiver_set_queue_byte_limit(PipelineContextReceiver* context,
|
||||
size_t max_bytes) {
|
||||
if (context == NULL)
|
||||
return;
|
||||
mtx_lock(&context->mutex);
|
||||
context->max_queue_bytes = max_bytes;
|
||||
context->queued_bytes = 0;
|
||||
cnd_broadcast(&context->condition_not_full);
|
||||
mtx_unlock(&context->mutex);
|
||||
}
|
||||
static void receive_chunk_enqueue(int file_descriptor,
|
||||
PipelineContextReceiver *context) {
|
||||
Chunk *chunk = receive_chunk_data(file_descriptor, context->config);
|
||||
if (chunk == NULL) return;
|
||||
|
||||
void pipeline_context_receiver_note_bytes_released(PipelineContextReceiver* context,
|
||||
size_t released_bytes) {
|
||||
if (context == NULL || context->max_queue_bytes == 0 || released_bytes == 0)
|
||||
return;
|
||||
mtx_lock(&context->mutex);
|
||||
if (released_bytes >= context->queued_bytes)
|
||||
context->queued_bytes = 0;
|
||||
else
|
||||
context->queued_bytes -= released_bytes;
|
||||
cnd_signal(&context->condition_not_full);
|
||||
mtx_unlock(&context->mutex);
|
||||
}
|
||||
|
||||
bool pipeline_context_receiver_enqueue_file(PipelineContextReceiver* context, File* file) {
|
||||
if (context == NULL || file == NULL)
|
||||
return false;
|
||||
size_t file_bytes = file->data ? file->data->size : 0;
|
||||
mtx_lock(&context->mutex);
|
||||
while (!atomic_load(&context->cancelled)) {
|
||||
bool blocked_by_count = queue_is_full(context->queue);
|
||||
bool blocked_by_budget = false;
|
||||
if (context->max_queue_bytes > 0) {
|
||||
size_t budget = context->max_queue_bytes;
|
||||
size_t used = context->queued_bytes;
|
||||
if (used >= budget) {
|
||||
blocked_by_budget = true;
|
||||
} else if (file_bytes > budget - used) {
|
||||
/* A single payload larger than the whole budget (not possible with
|
||||
the per-file receive cap) is only admitted to an empty pipeline so
|
||||
the wait can never deadlock. */
|
||||
blocked_by_budget = used != 0;
|
||||
}
|
||||
}
|
||||
if (!blocked_by_count && !blocked_by_budget)
|
||||
break;
|
||||
cnd_wait(&context->condition_not_full, &context->mutex);
|
||||
for (int i = 0; i < chunk->element_count; i++) {
|
||||
File *file = chunk->items[i];
|
||||
chunk->items[i] = NULL;
|
||||
queue_enqueue_multithreaded(context->queue, file, &context->mutex,
|
||||
&context->condition_not_empty,
|
||||
&context->condition_not_full);
|
||||
}
|
||||
if (atomic_load(&context->cancelled)) {
|
||||
mtx_unlock(&context->mutex);
|
||||
file_destroy(file);
|
||||
return false;
|
||||
}
|
||||
if (!queue_enqueue(context->queue, file)) {
|
||||
mtx_unlock(&context->mutex);
|
||||
file_destroy(file);
|
||||
return false;
|
||||
}
|
||||
context->queued_bytes += file_bytes;
|
||||
cnd_signal(&context->condition_not_empty);
|
||||
mtx_unlock(&context->mutex);
|
||||
return true;
|
||||
chunk_destroy(chunk);
|
||||
}
|
||||
|
||||
static bool receiver_enqueue_file(File* file, void* context_pointer) {
|
||||
PipelineContextReceiver* context = (PipelineContextReceiver*)context_pointer;
|
||||
return pipeline_context_receiver_enqueue_file(context, file);
|
||||
}
|
||||
|
||||
static void receiver_thread_fail(PipelineContextReceiver* context) {
|
||||
mtx_lock(&context->mutex);
|
||||
atomic_store(&context->cancelled, true);
|
||||
context->receiver_done = true;
|
||||
cnd_broadcast(&context->condition_not_empty);
|
||||
cnd_broadcast(&context->condition_not_full);
|
||||
mtx_unlock(&context->mutex);
|
||||
}
|
||||
|
||||
int receive_thread(void* pipeline_context) {
|
||||
PipelineContextReceiver* context = (PipelineContextReceiver*)pipeline_context;
|
||||
protocol_session_bind(&context->session);
|
||||
int receive_thread(void *pipeline_context) {
|
||||
PipelineContextReceiver *context =
|
||||
(PipelineContextReceiver *)pipeline_context;
|
||||
mtx_lock(&context->mutex);
|
||||
int file_descriptor = context->file_descriptor;
|
||||
const Config* config = context->config;
|
||||
Config *config = context->config;
|
||||
mtx_unlock(&context->mutex);
|
||||
|
||||
ReceiverSink sink = {receiver_enqueue_file, context, false, false, NULL};
|
||||
if (receiver_process_pending((Config*)config, file_descriptor, &sink,
|
||||
&context->deferred_manifest) != 0) {
|
||||
receiver_thread_fail(context);
|
||||
protocol_session_unbind();
|
||||
return thrd_error;
|
||||
Status status;
|
||||
if (!receive_status(file_descriptor, &status)) return thrd_error;
|
||||
while (status == STATUS_NEXT || status == STATUS_CHUNK || status == STATUS_CHECK) {
|
||||
if (status == STATUS_CHECK) {
|
||||
bool skipped;
|
||||
File *file = receive_incremental_check(file_descriptor, config, &skipped);
|
||||
if (!skipped) {
|
||||
if (file == NULL) return thrd_error;
|
||||
queue_enqueue_multithreaded(context->queue, file, &context->mutex,
|
||||
&context->condition_not_empty,
|
||||
&context->condition_not_full);
|
||||
}
|
||||
} else if (status == STATUS_CHUNK) {
|
||||
receive_chunk_enqueue(file_descriptor, context);
|
||||
} else {
|
||||
File *file = file_receive(config, file_descriptor);
|
||||
if (file) {
|
||||
queue_enqueue_multithreaded(context->queue, file, &context->mutex,
|
||||
&context->condition_not_empty,
|
||||
&context->condition_not_full);
|
||||
} else {
|
||||
log_message(LOG_LEVEL_ERROR, "Failed to receive file");
|
||||
}
|
||||
}
|
||||
if (!receive_status(file_descriptor, &status)) return thrd_error;
|
||||
}
|
||||
if (status == STATUS_MANIFEST) {
|
||||
if (receive_manifest(file_descriptor, config, &status) != 0) return thrd_error;
|
||||
}
|
||||
mtx_lock(&context->mutex);
|
||||
context->receiver_done = true;
|
||||
cnd_signal(&context->condition_not_empty);
|
||||
mtx_unlock(&context->mutex);
|
||||
protocol_session_unbind();
|
||||
return thrd_success;
|
||||
}
|
||||
|
||||
int write_thread(void* pipeline_context) {
|
||||
PipelineContextReceiver* context = (PipelineContextReceiver*)pipeline_context;
|
||||
protocol_session_bind(&context->session);
|
||||
int write_thread(void *pipeline_context) {
|
||||
PipelineContextReceiver *context =
|
||||
(PipelineContextReceiver *)pipeline_context;
|
||||
mtx_lock(&context->mutex);
|
||||
bool save_to_disk = context->config->save_to_disk;
|
||||
char* root_directory = str_dup(context->config->receive_root_directory);
|
||||
char *root_directory = str_dup(context->config->receive_root_directory);
|
||||
mtx_unlock(&context->mutex);
|
||||
if (save_to_disk && !root_directory) {
|
||||
mtx_lock(&context->mutex);
|
||||
atomic_store(&context->cancelled, true);
|
||||
context->receiver_done = true;
|
||||
cnd_broadcast(&context->condition_not_full);
|
||||
cnd_broadcast(&context->condition_not_empty);
|
||||
mtx_unlock(&context->mutex);
|
||||
protocol_session_unbind();
|
||||
return thrd_error;
|
||||
}
|
||||
|
||||
while (true) {
|
||||
File* file =
|
||||
queue_dequeue_multithreaded(context->queue, &context->mutex, &context->condition_not_empty,
|
||||
&context->condition_not_full, &context->receiver_done);
|
||||
File *file = queue_dequeue_multithreaded(
|
||||
context->queue, &context->mutex, &context->condition_not_empty,
|
||||
&context->condition_not_full, &context->receiver_done);
|
||||
if (file == NULL) {
|
||||
free(root_directory);
|
||||
protocol_session_unbind();
|
||||
return thrd_success;
|
||||
}
|
||||
size_t file_bytes = file->data ? file->data->size : 0;
|
||||
FileSaveResult result = FILE_SAVE_SKIPPED;
|
||||
if (save_to_disk) {
|
||||
result = file_save_to_disk_full(root_directory, file, context->config);
|
||||
if (result == FILE_SAVE_ERROR) {
|
||||
file_destroy(file);
|
||||
pipeline_context_receiver_note_bytes_released(context, file_bytes);
|
||||
mtx_lock(&context->mutex);
|
||||
atomic_store(&context->cancelled, true);
|
||||
context->receiver_done = true;
|
||||
cnd_broadcast(&context->condition_not_full);
|
||||
cnd_broadcast(&context->condition_not_empty);
|
||||
mtx_unlock(&context->mutex);
|
||||
free(root_directory);
|
||||
protocol_session_unbind();
|
||||
return thrd_error;
|
||||
}
|
||||
}
|
||||
/* P7 Wave D: a directory's times are never applied inline (a later child
|
||||
write would clobber them); accumulate the metadata here and let the
|
||||
caller apply it once every writer has drained. */
|
||||
if (result != FILE_SAVE_ERROR && file->is_dir && file->metadata &&
|
||||
context->config->use_metadata && !context->config->omit_dir_times &&
|
||||
!dir_time_list_add(&context->dir_times, file->path, file->metadata)) {
|
||||
file_destroy(file);
|
||||
pipeline_context_receiver_note_bytes_released(context, file_bytes);
|
||||
mtx_lock(&context->mutex);
|
||||
atomic_store(&context->cancelled, true);
|
||||
context->receiver_done = true;
|
||||
cnd_broadcast(&context->condition_not_full);
|
||||
cnd_broadcast(&context->condition_not_empty);
|
||||
mtx_unlock(&context->mutex);
|
||||
free(root_directory);
|
||||
protocol_session_unbind();
|
||||
return thrd_error;
|
||||
}
|
||||
/* Record the per-file outcome so a --remove-source-files sender learns
|
||||
which sources were actually written versus skipped on the receiver.
|
||||
Explicit directory entries and recreated device/special nodes have no
|
||||
source and are never acknowledged (mirrors receiver.c). */
|
||||
if (context->config->remove_source_files && !file->is_dir && !file->is_special && !file->skip &&
|
||||
!receiver_outcomes_append(&context->outcomes, (unsigned char)result)) {
|
||||
file_destroy(file);
|
||||
pipeline_context_receiver_note_bytes_released(context, file_bytes);
|
||||
mtx_lock(&context->mutex);
|
||||
atomic_store(&context->cancelled, true);
|
||||
context->receiver_done = true;
|
||||
cnd_broadcast(&context->condition_not_full);
|
||||
cnd_broadcast(&context->condition_not_empty);
|
||||
mtx_unlock(&context->mutex);
|
||||
free(root_directory);
|
||||
protocol_session_unbind();
|
||||
return thrd_error;
|
||||
}
|
||||
if (save_to_disk)
|
||||
file_save_to_disk(root_directory, file);
|
||||
file_destroy(file);
|
||||
pipeline_context_receiver_note_bytes_released(context, file_bytes);
|
||||
}
|
||||
}
|
||||
|
||||
+16
-103
@@ -2,132 +2,45 @@
|
||||
#define MULTIPROCESSING_H
|
||||
|
||||
#include <threads.h>
|
||||
#include <stdatomic.h>
|
||||
|
||||
#include "array_list.h"
|
||||
#include "config.h"
|
||||
#include "file.h"
|
||||
#include "protocol.h"
|
||||
#include "queue.h"
|
||||
#include "receiver.h"
|
||||
#include "stop_condition.h"
|
||||
#include <openssl/ssl.h>
|
||||
|
||||
typedef struct {
|
||||
Config* config;
|
||||
Queue* queue_scanner;
|
||||
Config *config;
|
||||
Queue *queue_scanner;
|
||||
mtx_t mutex_scanner;
|
||||
cnd_t condition_not_full_scanner;
|
||||
cnd_t condition_not_empty_scanner;
|
||||
bool scanner_done;
|
||||
Queue* queue_loader;
|
||||
Queue *queue_loader;
|
||||
mtx_t mutex_loader;
|
||||
cnd_t condition_not_full_loader;
|
||||
cnd_t condition_not_empty_loader;
|
||||
bool loader_done;
|
||||
ArrayList* manifest;
|
||||
/* Protected prefixes (paths the source scan excluded by user rules) sent
|
||||
with the keep-set manifest so --delete leaves them alone unless
|
||||
--delete-excluded is set. NULL when not collecting. Populated by the
|
||||
scanner thread (parallel workers append under mutex_scanner via the
|
||||
scanner's exclusion sink) or, in the early modes, by the path-only pre-scan
|
||||
on the calling thread before the pipeline starts. */
|
||||
ArrayList* excluded_paths;
|
||||
/* --delete-missing-args: the destination-relative mirrors of the --files-from
|
||||
entries that are missing under the source. Computed by the preflight on
|
||||
the calling thread before the pipeline starts; the sender thread transmits
|
||||
them in the manifest frame's third section and the receiver deletes each as
|
||||
an explicit request. */
|
||||
ArrayList* missing_args;
|
||||
/* A source I/O error (unreadable directory) was recorded during the scan.
|
||||
Set by the pre-scan (before the threads start) or by the scanner thread
|
||||
under mutex_scanner; the caller turns it into a non-zero exit when
|
||||
--ignore-errors kept the run going. */
|
||||
bool scan_had_io_error;
|
||||
ArrayList* remove_source_files;
|
||||
/* True when --delete-before/--delete-during require the keep-set manifest to
|
||||
be transmitted before any file data: context->manifest is then prebuilt by
|
||||
a path-only pre-scan on the calling thread and the pipeline scanner must
|
||||
not append to it. Set once before the worker threads start. */
|
||||
bool early_delete;
|
||||
mtx_t mutex_progress;
|
||||
int total_files;
|
||||
unsigned long long progress_bytes;
|
||||
unsigned long long total_bytes;
|
||||
bool sender_done;
|
||||
atomic_bool cancelled;
|
||||
ProtocolSession allocation_session;
|
||||
/* Phase 6: client-only sender stop deadline, computed once before the worker
|
||||
* threads start and shared read-only by the scanner and the sender thread. */
|
||||
StopCondition stop_condition;
|
||||
/* Phase 6: set when the scanner/sender reached the stop deadline before the
|
||||
* scan (and thus the keep-set manifest) completed naturally. When true the
|
||||
* completion tail must NOT transmit the partial manifest, or the receiver
|
||||
* would delete unscanned source mirrors. Written by the sender thread
|
||||
* before it reads the manifest, so no additional synchronization is needed
|
||||
* to suppress the manifest. */
|
||||
bool scan_stopped_early;
|
||||
/* P7 Wave D: captured source directory times, filled by the scanner thread
|
||||
* (and its parallel workers, guarded by dir_entries_mutex) and drained by the
|
||||
* sender thread in trailing STATUS_DIR_TIMES frame(s). Owned by the
|
||||
* context; NULL for non-metadata transfers. */
|
||||
ArrayList* dir_entries;
|
||||
mtx_t dir_entries_mutex;
|
||||
bool dir_entries_mutex_init;
|
||||
ArrayList *manifest;
|
||||
} PipelineContextSender;
|
||||
|
||||
typedef struct PipelineContextReceiver {
|
||||
Queue* queue;
|
||||
Config* config;
|
||||
Queue *queue;
|
||||
Config *config;
|
||||
int file_descriptor;
|
||||
SSL* ssl;
|
||||
ProtocolSession session;
|
||||
ReceiverOutcomes outcomes;
|
||||
mtx_t mutex;
|
||||
cnd_t condition_not_full;
|
||||
cnd_t condition_not_empty;
|
||||
bool receiver_done;
|
||||
atomic_bool cancelled;
|
||||
/* Aggregate payload bytes that have been received but not yet released by
|
||||
the disk writer (queued or in the writer's hand). Guarded by `mutex`.
|
||||
When `max_queue_bytes` is non-zero the receiver blocks before enqueuing
|
||||
once this total would exceed it, so decompressed/copied file payloads
|
||||
buffered ahead of a slow disk writer respect the per-connection memory
|
||||
budget instead of growing without bound. */
|
||||
size_t queued_bytes;
|
||||
size_t max_queue_bytes;
|
||||
/* Keep-set manifest for the commit-style (late) deletion
|
||||
(--delete/--delete-after/--delete-delay). receive_thread parses the whole
|
||||
protocol stream but hands the manifest here instead of deleting while the
|
||||
disk writer may still be draining; the caller (server.c) commits the
|
||||
deletion after both threads have joined, so no extra is removed unless the
|
||||
transfer truly succeeded. NULL in the early delete modes (which delete at
|
||||
the manifest). */
|
||||
DeleteManifest* deferred_manifest;
|
||||
/* P7 Wave D: directory metadata collected by write_thread from received
|
||||
directory entries. Only write_thread mutates it (before it joins); the
|
||||
caller (server.c) applies it after the delete/delay-updates phase. */
|
||||
DirTimeList dir_times;
|
||||
} PipelineContextReceiver;
|
||||
|
||||
PipelineContextSender* pipeline_context_sender_create(Config* config, Queue* queue_scanner,
|
||||
Queue* queue_loader);
|
||||
void pipeline_context_sender_destroy(PipelineContextSender* context);
|
||||
PipelineContextReceiver* pipeline_context_receiver_create(Config* config, Queue* queue_receiver,
|
||||
int file_descriptor, SSL* ssl);
|
||||
void pipeline_context_receiver_destroy(PipelineContextReceiver* context);
|
||||
/* Bound the bytes buffered ahead of the disk writer (see max_queue_bytes). */
|
||||
void pipeline_context_receiver_set_queue_byte_limit(PipelineContextReceiver* context,
|
||||
size_t max_bytes);
|
||||
/* Blocking enqueue used by the receive pipeline sink. Blocks while the queue
|
||||
is full by element count or when adding `file` would push queued_bytes over
|
||||
the configured byte limit; waits until the disk writer releases bytes.
|
||||
Takes ownership of `file` on success and destroys it on failure/cancel. */
|
||||
bool pipeline_context_receiver_enqueue_file(PipelineContextReceiver* context, File* file);
|
||||
/* Account for `released_bytes` of payload memory that has been freed by the
|
||||
disk writer, unblocking a receiver that is waiting on the byte limit. */
|
||||
void pipeline_context_receiver_note_bytes_released(PipelineContextReceiver* context,
|
||||
size_t released_bytes);
|
||||
int receive_thread(void* pipeline_context);
|
||||
int write_thread(void* pipeline_context);
|
||||
PipelineContextSender *pipeline_context_sender_create(Config *config,
|
||||
Queue *queue_scanner,
|
||||
Queue *queue_loader);
|
||||
void pipeline_context_sender_destroy(PipelineContextSender *context);
|
||||
PipelineContextReceiver *pipeline_context_receiver_create(Config *config,
|
||||
Queue *queue_receiver,
|
||||
int file_descriptor);
|
||||
void pipeline_context_receiver_destroy(PipelineContextReceiver *context);
|
||||
int receive_thread(void *pipeline_context);
|
||||
int write_thread(void *pipeline_context);
|
||||
#endif
|
||||
|
||||
+81
-511
@@ -1,377 +1,119 @@
|
||||
#include "protocol.h"
|
||||
#include "log.h"
|
||||
#include "utils.h"
|
||||
#include <errno.h>
|
||||
#include <limits.h>
|
||||
#include <openssl/ssl.h>
|
||||
#include <poll.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <threads.h>
|
||||
#include <time.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#define RECEIVE_TIMEOUT_SEC 60 /* 60 second per-message timeout */
|
||||
#define SEND_TIMEOUT_SEC 60
|
||||
|
||||
static __thread int io_read_fd = -1;
|
||||
static __thread int io_write_fd = -1;
|
||||
static __thread SSL* io_ssl;
|
||||
static __thread ProtocolSession* bound_session;
|
||||
static __thread ProtocolSession legacy_io_session = {
|
||||
.read_fd = -1, .write_fd = -1, .max_alloc = DEFAULT_MAX_ALLOC};
|
||||
static SSL *io_ssl = NULL;
|
||||
|
||||
static unsigned long long io_bwlimit = 0;
|
||||
static mtx_t bw_mutex;
|
||||
static once_flag bw_mutex_once = ONCE_FLAG_INIT;
|
||||
static long long bw_tokens = 0;
|
||||
static struct timespec bw_last_refill = {0, 0};
|
||||
|
||||
static unsigned long long global_bwlimit(void);
|
||||
|
||||
static bool protocol_reserve_memory(ProtocolSession* session, size_t charge) {
|
||||
unsigned long long allocated = atomic_load(&session->total_allocated_bytes);
|
||||
while (true) {
|
||||
if (allocated > MAX_CONNECTION_MEMORY ||
|
||||
(unsigned long long)charge > MAX_CONNECTION_MEMORY - allocated)
|
||||
return false;
|
||||
if (atomic_compare_exchange_weak(&session->total_allocated_bytes, &allocated,
|
||||
allocated + (unsigned long long)charge))
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
static void protocol_release_memory_for_session(ProtocolSession* session, size_t charge) {
|
||||
unsigned long long allocated = atomic_load(&session->total_allocated_bytes);
|
||||
while (true) {
|
||||
unsigned long long remaining = (unsigned long long)charge >= allocated ? 0 : allocated - charge;
|
||||
if (atomic_compare_exchange_weak(&session->total_allocated_bytes, &allocated, remaining))
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
void protocol_release_memory(size_t charge) {
|
||||
ProtocolSession* session = bound_session ? bound_session : &legacy_io_session;
|
||||
protocol_release_memory_for_session(session, charge);
|
||||
}
|
||||
void io_set_fds(int read_fd, int write_fd) {
|
||||
bound_session = NULL;
|
||||
io_read_fd = read_fd;
|
||||
io_write_fd = write_fd;
|
||||
/* A descriptor switch starts a new transport; never reuse a TLS object
|
||||
belonging to a previous connection or test pipe. */
|
||||
io_ssl = NULL;
|
||||
legacy_io_session.read_fd = read_fd;
|
||||
legacy_io_session.write_fd = write_fd;
|
||||
legacy_io_session.ssl = NULL;
|
||||
legacy_io_session.eight_bit_output = false;
|
||||
atomic_store(&legacy_io_session.total_allocated_bytes, 0);
|
||||
legacy_io_session.max_alloc = DEFAULT_MAX_ALLOC;
|
||||
protocol_session_set_bwlimit(&legacy_io_session, global_bwlimit());
|
||||
}
|
||||
|
||||
void protocol_session_init(ProtocolSession* session, int read_fd, int write_fd) {
|
||||
if (!session)
|
||||
return;
|
||||
memset(session, 0, sizeof(*session));
|
||||
session->read_fd = read_fd;
|
||||
session->write_fd = write_fd;
|
||||
session->max_alloc = DEFAULT_MAX_ALLOC;
|
||||
atomic_init(&session->total_allocated_bytes, 0);
|
||||
protocol_session_set_bwlimit(session, global_bwlimit());
|
||||
}
|
||||
|
||||
void protocol_session_set_max_alloc(ProtocolSession* session, unsigned long long max_alloc) {
|
||||
if (!session)
|
||||
session = bound_session ? bound_session : &legacy_io_session;
|
||||
session->max_alloc = max_alloc;
|
||||
}
|
||||
|
||||
static bool allocation_allowed(const ProtocolSession* session, size_t size) {
|
||||
return (unsigned long long)size <= session->max_alloc;
|
||||
}
|
||||
|
||||
static void* protocol_alloc_for_session(const ProtocolSession* session, size_t size) {
|
||||
if (!allocation_allowed(session, size))
|
||||
return NULL;
|
||||
return malloc(size);
|
||||
}
|
||||
|
||||
static void* protocol_realloc_for_session(const ProtocolSession* session, void* ptr, size_t size) {
|
||||
if (!allocation_allowed(session, size))
|
||||
return NULL;
|
||||
return realloc(ptr, size);
|
||||
}
|
||||
|
||||
void* protocol_alloc(size_t size) {
|
||||
const ProtocolSession* session = bound_session ? bound_session : &legacy_io_session;
|
||||
return protocol_alloc_for_session(session, size);
|
||||
}
|
||||
|
||||
void* protocol_realloc(void* ptr, size_t size) {
|
||||
const ProtocolSession* session = bound_session ? bound_session : &legacy_io_session;
|
||||
return protocol_realloc_for_session(session, ptr, size);
|
||||
}
|
||||
|
||||
void protocol_session_bind(ProtocolSession* session) {
|
||||
bound_session = session;
|
||||
log_set_8_bit_output(session && session->eight_bit_output);
|
||||
}
|
||||
|
||||
void protocol_session_unbind(void) {
|
||||
bound_session = NULL;
|
||||
}
|
||||
|
||||
void protocol_session_set_ssl(ProtocolSession* session, SSL* ssl) {
|
||||
if (session)
|
||||
session->ssl = ssl;
|
||||
}
|
||||
|
||||
static void bw_mutex_init(void) {
|
||||
mtx_init(&bw_mutex, mtx_plain);
|
||||
}
|
||||
|
||||
static unsigned long long global_bwlimit(void) {
|
||||
unsigned long long limit;
|
||||
call_once(&bw_mutex_once, bw_mutex_init);
|
||||
mtx_lock(&bw_mutex);
|
||||
limit = io_bwlimit;
|
||||
mtx_unlock(&bw_mutex);
|
||||
return limit;
|
||||
}
|
||||
|
||||
void io_set_bwlimit(unsigned long long bytes_per_sec) {
|
||||
call_once(&bw_mutex_once, bw_mutex_init);
|
||||
mtx_lock(&bw_mutex);
|
||||
io_bwlimit =
|
||||
bytes_per_sec > (unsigned long long)LLONG_MAX ? (unsigned long long)LLONG_MAX : bytes_per_sec;
|
||||
mtx_unlock(&bw_mutex);
|
||||
io_bwlimit = bytes_per_sec;
|
||||
bw_tokens = (long long)io_bwlimit;
|
||||
clock_gettime(CLOCK_MONOTONIC, &bw_last_refill);
|
||||
}
|
||||
|
||||
void protocol_session_set_bwlimit(ProtocolSession* session, unsigned long long bytes_per_sec) {
|
||||
if (!session)
|
||||
return;
|
||||
session->bwlimit =
|
||||
bytes_per_sec > (unsigned long long)LLONG_MAX ? (unsigned long long)LLONG_MAX : bytes_per_sec;
|
||||
session->bw_tokens = (long long)session->bwlimit;
|
||||
struct timespec now;
|
||||
clock_gettime(CLOCK_MONOTONIC, &now);
|
||||
session->bw_last_refill_sec = now.tv_sec;
|
||||
session->bw_last_refill_nsec = now.tv_nsec;
|
||||
}
|
||||
|
||||
void protocol_session_set_8_bit_output(ProtocolSession* session, bool enabled) {
|
||||
if (!session)
|
||||
return;
|
||||
session->eight_bit_output = enabled;
|
||||
if (session == bound_session)
|
||||
log_set_8_bit_output(enabled);
|
||||
}
|
||||
|
||||
void protocol_set_8_bit_output(bool enabled) {
|
||||
ProtocolSession* session = bound_session ? bound_session : &legacy_io_session;
|
||||
protocol_session_set_8_bit_output(session, enabled);
|
||||
}
|
||||
|
||||
static void bw_throttle_session(ProtocolSession* session, size_t bytes_written) {
|
||||
if (session->bwlimit == 0)
|
||||
return;
|
||||
static void bw_throttle(size_t bytes_written) {
|
||||
if (io_bwlimit == 0) return;
|
||||
|
||||
struct timespec now;
|
||||
clock_gettime(CLOCK_MONOTONIC, &now);
|
||||
|
||||
long long elapsed_ns = (now.tv_sec - session->bw_last_refill_sec) * 1000000000LL +
|
||||
(now.tv_nsec - session->bw_last_refill_nsec);
|
||||
session->bw_last_refill_sec = now.tv_sec;
|
||||
session->bw_last_refill_nsec = now.tv_nsec;
|
||||
long long elapsed_ns = (now.tv_sec - bw_last_refill.tv_sec) * 1000000000LL +
|
||||
(now.tv_nsec - bw_last_refill.tv_nsec);
|
||||
bw_last_refill = now;
|
||||
|
||||
long long tokens_to_add = (long long)((double)session->bwlimit * elapsed_ns / 1000000000.0);
|
||||
session->bw_tokens += tokens_to_add;
|
||||
if (session->bw_tokens > (long long)session->bwlimit)
|
||||
session->bw_tokens = (long long)session->bwlimit;
|
||||
long long tokens_to_add = (long long)((double)io_bwlimit * elapsed_ns / 1000000000.0);
|
||||
bw_tokens += tokens_to_add;
|
||||
if (bw_tokens > (long long)io_bwlimit)
|
||||
bw_tokens = (long long)io_bwlimit;
|
||||
|
||||
session->bw_tokens -= bytes_written;
|
||||
bw_tokens -= (long long)bytes_written;
|
||||
|
||||
if (session->bw_tokens < 0) {
|
||||
long long deficit_us =
|
||||
(long long)((double)(-session->bw_tokens) / session->bwlimit * 1000000.0);
|
||||
if (deficit_us >= 1000)
|
||||
poll(NULL, 0, (int)(deficit_us / 1000));
|
||||
else
|
||||
usleep((useconds_t)deficit_us);
|
||||
session->bw_tokens = 0;
|
||||
session->bw_last_refill_sec = now.tv_sec;
|
||||
session->bw_last_refill_nsec = now.tv_nsec;
|
||||
if (bw_tokens < 0) {
|
||||
long long deficit_ns = (long long)((double)(-bw_tokens) / io_bwlimit * 1000000000.0);
|
||||
struct timespec sleep_time, remaining;
|
||||
sleep_time.tv_sec = deficit_ns / 1000000000LL;
|
||||
sleep_time.tv_nsec = deficit_ns % 1000000000LL;
|
||||
while (nanosleep(&sleep_time, &remaining) < 0 && errno == EINTR)
|
||||
sleep_time = remaining;
|
||||
bw_tokens = 0;
|
||||
clock_gettime(CLOCK_MONOTONIC, &bw_last_refill);
|
||||
}
|
||||
}
|
||||
|
||||
void io_set_ssl(SSL* ssl) {
|
||||
bound_session = NULL;
|
||||
void io_set_ssl(SSL *ssl) {
|
||||
io_ssl = ssl;
|
||||
}
|
||||
|
||||
SSL* io_get_ssl(void) {
|
||||
return io_ssl;
|
||||
static int io_fd(int dir_fd, int file_descriptor) {
|
||||
return (dir_fd != -1) ? dir_fd : file_descriptor;
|
||||
}
|
||||
|
||||
static ProtocolSession* legacy_session(int read_fd, int write_fd) {
|
||||
if (bound_session)
|
||||
return bound_session;
|
||||
int target_read_fd = io_read_fd != -1 ? io_read_fd : read_fd;
|
||||
int target_write_fd = io_write_fd != -1 ? io_write_fd : write_fd;
|
||||
if (legacy_io_session.read_fd != target_read_fd ||
|
||||
legacy_io_session.write_fd != target_write_fd) {
|
||||
legacy_io_session.read_fd = target_read_fd;
|
||||
legacy_io_session.write_fd = target_write_fd;
|
||||
atomic_store(&legacy_io_session.total_allocated_bytes, 0);
|
||||
legacy_io_session.max_alloc = DEFAULT_MAX_ALLOC;
|
||||
protocol_session_set_bwlimit(&legacy_io_session, global_bwlimit());
|
||||
} else if (legacy_io_session.bwlimit != global_bwlimit()) {
|
||||
protocol_session_set_bwlimit(&legacy_io_session, global_bwlimit());
|
||||
}
|
||||
legacy_io_session.ssl = io_ssl;
|
||||
return &legacy_io_session;
|
||||
}
|
||||
|
||||
bool send_n_data(int file_descriptor, const void* data, size_t data_size) {
|
||||
return protocol_send_n_data(legacy_session(-1, file_descriptor), data, data_size);
|
||||
}
|
||||
|
||||
bool receive_n_data(int file_descriptor, void* data, size_t data_size) {
|
||||
return protocol_receive_n_data(legacy_session(file_descriptor, -1), data, data_size);
|
||||
}
|
||||
|
||||
static int deadline_remaining_ms(const struct timespec* deadline) {
|
||||
struct timespec now;
|
||||
clock_gettime(CLOCK_MONOTONIC, &now);
|
||||
long long ns =
|
||||
(long long)(deadline->tv_sec - now.tv_sec) * 1000000000LL + deadline->tv_nsec - now.tv_nsec;
|
||||
if (ns <= 0)
|
||||
return 0;
|
||||
long long ms = (ns + 999999) / 1000000;
|
||||
return ms > INT_MAX ? INT_MAX : (int)ms;
|
||||
}
|
||||
|
||||
bool protocol_send_n_data(ProtocolSession* session, const void* data, size_t data_size) {
|
||||
if (!data && data_size != 0)
|
||||
return false;
|
||||
log_debug_message(LOG_DEBUG_IO, " Sending n Data: %zu", data_size);
|
||||
if (!session)
|
||||
return false;
|
||||
int fd = session->write_fd;
|
||||
struct timespec deadline;
|
||||
clock_gettime(CLOCK_MONOTONIC, &deadline);
|
||||
deadline.tv_sec += SEND_TIMEOUT_SEC;
|
||||
short wait_events = POLLOUT;
|
||||
bool send_n_data(int file_descriptor, void *data, size_t data_size) {
|
||||
log_message(LOG_LEVEL_DEBUG, " Sending n Data: %zu", data_size);
|
||||
int fd = io_fd(io_write_fd, file_descriptor);
|
||||
ssize_t total_bytes_send = 0;
|
||||
while ((size_t)total_bytes_send < data_size) {
|
||||
while (total_bytes_send < data_size) {
|
||||
size_t chunk = data_size - total_bytes_send;
|
||||
if (session->bwlimit > 0 && chunk > 65536)
|
||||
if (io_bwlimit > 0 && chunk > 65536)
|
||||
chunk = 65536;
|
||||
struct pollfd pfd = {.fd = fd, .events = wait_events};
|
||||
int poll_result = poll(&pfd, 1, deadline_remaining_ms(&deadline));
|
||||
if (poll_result == 0 || (poll_result < 0 && errno != EINTR)) {
|
||||
log_message(LOG_LEVEL_ERROR, "Send timeout or poll failure");
|
||||
return false;
|
||||
}
|
||||
if (poll_result < 0)
|
||||
continue;
|
||||
if (pfd.revents & (POLLERR | POLLNVAL))
|
||||
return false;
|
||||
ssize_t bytes_send;
|
||||
if (session->ssl)
|
||||
bytes_send = SSL_write(session->ssl, (const char*)data + total_bytes_send, chunk);
|
||||
if (io_ssl)
|
||||
bytes_send = SSL_write(io_ssl, (char *)data + total_bytes_send, chunk);
|
||||
else
|
||||
bytes_send = write(fd, (const char*)data + total_bytes_send, chunk);
|
||||
bytes_send = write(fd, (char *)data + total_bytes_send, chunk);
|
||||
if (bytes_send <= 0) {
|
||||
if (session->ssl) {
|
||||
int ssl_err = SSL_get_error(session->ssl, (int)bytes_send);
|
||||
if (ssl_err == SSL_ERROR_WANT_WRITE || ssl_err == SSL_ERROR_WANT_READ) {
|
||||
wait_events = ssl_err == SSL_ERROR_WANT_WRITE ? POLLOUT : POLLIN;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
log_message(LOG_LEVEL_ERROR, "Could not send data");
|
||||
return false;
|
||||
}
|
||||
bw_throttle_session(session, (size_t)bytes_send);
|
||||
bw_throttle((size_t)bytes_send);
|
||||
total_bytes_send += bytes_send;
|
||||
if (session->ssl)
|
||||
wait_events = POLLOUT;
|
||||
}
|
||||
log_debug_message(LOG_DEBUG_IO, " Send n Data: %zu", total_bytes_send);
|
||||
log_message(LOG_LEVEL_DEBUG, " Send n Data: %zu", total_bytes_send);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool protocol_receive_n_data_timed(ProtocolSession* session, void* data, size_t data_size,
|
||||
int timeout_sec);
|
||||
|
||||
bool protocol_receive_n_data(ProtocolSession* session, void* data, size_t data_size) {
|
||||
return protocol_receive_n_data_timed(session, data, data_size, RECEIVE_TIMEOUT_SEC);
|
||||
}
|
||||
|
||||
bool protocol_receive_n_data_timed(ProtocolSession* session, void* data, size_t data_size,
|
||||
int timeout_sec) {
|
||||
log_debug_message(LOG_DEBUG_IO, " Receiving n Data: %zu", data_size);
|
||||
if (!session)
|
||||
return false;
|
||||
int fd = session->read_fd;
|
||||
if (timeout_sec <= 0)
|
||||
timeout_sec = RECEIVE_TIMEOUT_SEC;
|
||||
|
||||
struct timespec deadline;
|
||||
clock_gettime(CLOCK_MONOTONIC, &deadline);
|
||||
deadline.tv_sec += timeout_sec;
|
||||
|
||||
bool receive_n_data(int file_descriptor, void *data, size_t data_size) {
|
||||
log_message(LOG_LEVEL_DEBUG, " Receiving n Data: %zu", data_size);
|
||||
int fd = io_fd(io_read_fd, file_descriptor);
|
||||
size_t total_bytes_received = 0;
|
||||
short wait_events = POLLIN;
|
||||
while (total_bytes_received < data_size) {
|
||||
if (!session->ssl || SSL_pending(session->ssl) == 0) {
|
||||
struct pollfd pfd = {.fd = fd, .events = wait_events};
|
||||
int poll_result = poll(&pfd, 1, deadline_remaining_ms(&deadline));
|
||||
if (poll_result == 0) {
|
||||
log_message(LOG_LEVEL_ERROR, "Receive timeout after %ds", timeout_sec);
|
||||
return false;
|
||||
}
|
||||
if (poll_result < 0) {
|
||||
if (errno == EINTR)
|
||||
continue;
|
||||
return false;
|
||||
}
|
||||
/* POLLHUP may accompany the final readable bytes on pipes/sockets. */
|
||||
if (pfd.revents & (POLLERR | POLLNVAL))
|
||||
return false;
|
||||
}
|
||||
|
||||
ssize_t bytes_received;
|
||||
if (session->ssl)
|
||||
bytes_received = SSL_read(session->ssl, (char*)data + total_bytes_received,
|
||||
if (io_ssl)
|
||||
bytes_received = SSL_read(io_ssl, (char *)data + total_bytes_received,
|
||||
data_size - total_bytes_received);
|
||||
else
|
||||
bytes_received =
|
||||
read(fd, (char*)data + total_bytes_received, data_size - total_bytes_received);
|
||||
bytes_received = read(fd, (char *)data + total_bytes_received,
|
||||
data_size - total_bytes_received);
|
||||
if (bytes_received <= 0) {
|
||||
if (session->ssl) {
|
||||
int ssl_err = SSL_get_error(session->ssl, (int)bytes_received);
|
||||
if (ssl_err == SSL_ERROR_WANT_WRITE || ssl_err == SSL_ERROR_WANT_READ) {
|
||||
wait_events = ssl_err == SSL_ERROR_WANT_WRITE ? POLLOUT : POLLIN;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if (bytes_received == 0)
|
||||
log_message(LOG_LEVEL_ERROR, "Connection closed while receiving data");
|
||||
else
|
||||
log_message(LOG_LEVEL_ERROR, "Could not receive bytes");
|
||||
return false;
|
||||
}
|
||||
total_bytes_received += (size_t)bytes_received;
|
||||
if (session->ssl)
|
||||
wait_events = POLLIN;
|
||||
total_bytes_received += bytes_received;
|
||||
}
|
||||
log_debug_message(LOG_DEBUG_IO, " Received n Data: %zu", total_bytes_received);
|
||||
log_message(LOG_LEVEL_DEBUG, " Received n Data: %zu", total_bytes_received);
|
||||
return true;
|
||||
}
|
||||
|
||||
static const char* status_to_string(Status status) {
|
||||
static const char *status_to_string(Status status) {
|
||||
switch (status) {
|
||||
case STATUS_OK:
|
||||
return "OK";
|
||||
@@ -389,249 +131,77 @@ static const char* status_to_string(Status status) {
|
||||
return "DELTA_SIGNATURE";
|
||||
case STATUS_DELTA_DATA:
|
||||
return "DELTA_DATA";
|
||||
case STATUS_KEEPALIVE:
|
||||
return "KEEPALIVE";
|
||||
case STATUS_ABORT:
|
||||
return "ABORT";
|
||||
case STATUS_CHECK_BATCH:
|
||||
return "CHECK_BATCH";
|
||||
case STATUS_MKDIR:
|
||||
return "MKDIR";
|
||||
case STATUS_APPEND:
|
||||
return "APPEND";
|
||||
case STATUS_APPEND_SIG:
|
||||
return "APPEND_SIG";
|
||||
case STATUS_APPEND_OK:
|
||||
return "APPEND_OK";
|
||||
case STATUS_APPEND_DATA:
|
||||
return "APPEND_DATA";
|
||||
case STATUS_HARDLINK:
|
||||
return "HARDLINK";
|
||||
case STATUS_SYMLINK:
|
||||
return "SYMLINK";
|
||||
case STATUS_SPECIAL:
|
||||
return "SPECIAL";
|
||||
case STATUS_DIR_TIMES:
|
||||
return "DIR_TIMES";
|
||||
case STATUS_AUTH_CHALLENGE:
|
||||
return "AUTH_CHALLENGE";
|
||||
case STATUS_AUTH_RESPONSE:
|
||||
return "AUTH_RESPONSE";
|
||||
case STATUS_AUTH_OK:
|
||||
return "AUTH_OK";
|
||||
case STATUS_AUTH_FAILED:
|
||||
return "AUTH_FAILED";
|
||||
default:
|
||||
return "UNKNOWN";
|
||||
}
|
||||
}
|
||||
|
||||
/* Shared string send/receive implementation. `redact` selects whether the
|
||||
* payload body is written to the LOG_DEBUG_PROTO debug log: daemon auth material
|
||||
* (the username and the proof/signature fields) sets it so a --verbose log never
|
||||
* captures a replayable credential, while every other string keeps its normal
|
||||
* debug trace. */
|
||||
static bool protocol_send_str_impl(ProtocolSession* session, const char* data, bool redact) {
|
||||
if (data == NULL)
|
||||
return false;
|
||||
bool send_str(int file_descriptor, char *data) {
|
||||
size_t size = strlen(data);
|
||||
if (!protocol_send_n_data(session, &size, sizeof(size_t)))
|
||||
return false;
|
||||
if (!protocol_send_n_data(session, data, size))
|
||||
return false;
|
||||
if (redact) {
|
||||
log_debug_message(LOG_DEBUG_PROTO, "Send String: <redacted>");
|
||||
} else if (log_debug_enabled(LOG_DEBUG_PROTO)) {
|
||||
char* escaped_data = output_escape(data, log_get_8_bit_output());
|
||||
log_debug_message(LOG_DEBUG_PROTO, "Send String: %s",
|
||||
escaped_data ? escaped_data : "<allocation failed>");
|
||||
free(escaped_data);
|
||||
}
|
||||
if (!send_n_data(file_descriptor, &size, sizeof(size_t))) return false;
|
||||
if (!send_n_data(file_descriptor, data, size)) return false;
|
||||
log_message(LOG_LEVEL_DEBUG, "Send String: %s", data);
|
||||
return true;
|
||||
}
|
||||
|
||||
static char* protocol_receive_str_impl(ProtocolSession* session, bool redact) {
|
||||
char *receive_str(int file_descriptor) {
|
||||
size_t size;
|
||||
if (!protocol_receive_n_data(session, &size, sizeof(size_t)))
|
||||
return NULL;
|
||||
if (size > MAX_STRING_SIZE || size > SIZE_MAX - 1) {
|
||||
log_message(LOG_LEVEL_ERROR, "String size %zu exceeds maximum %llu", size,
|
||||
(unsigned long long)MAX_STRING_SIZE);
|
||||
return NULL;
|
||||
}
|
||||
char* data = (char*)protocol_alloc_for_session(session, size + 1);
|
||||
if (data == NULL)
|
||||
return NULL;
|
||||
if (!protocol_receive_n_data(session, data, size)) {
|
||||
if (!receive_n_data(file_descriptor, &size, sizeof(size_t))) return NULL;
|
||||
char *data = (char *)malloc(size + 1);
|
||||
if (data == NULL) return NULL;
|
||||
if (!receive_n_data(file_descriptor, data, size)) {
|
||||
free(data);
|
||||
return NULL;
|
||||
}
|
||||
if (memchr(data, '\0', size) != NULL) {
|
||||
free(data);
|
||||
log_message(LOG_LEVEL_ERROR, "Received string contains an embedded NUL");
|
||||
return NULL;
|
||||
}
|
||||
data[size] = '\0';
|
||||
if (redact) {
|
||||
log_debug_message(LOG_DEBUG_PROTO, "Received String: <redacted>");
|
||||
} else if (log_debug_enabled(LOG_DEBUG_PROTO)) {
|
||||
char* escaped_data = output_escape(data, log_get_8_bit_output());
|
||||
log_debug_message(LOG_DEBUG_PROTO, "Received String: %s",
|
||||
escaped_data ? escaped_data : "<allocation failed>");
|
||||
free(escaped_data);
|
||||
}
|
||||
log_message(LOG_LEVEL_DEBUG, "Received String: %s", data);
|
||||
return data;
|
||||
}
|
||||
|
||||
bool protocol_send_str(ProtocolSession* session, const char* data) {
|
||||
return protocol_send_str_impl(session, data, false);
|
||||
}
|
||||
|
||||
bool protocol_send_str_redacted(ProtocolSession* session, const char* data) {
|
||||
return protocol_send_str_impl(session, data, true);
|
||||
}
|
||||
|
||||
char* protocol_receive_str(ProtocolSession* session) {
|
||||
return protocol_receive_str_impl(session, false);
|
||||
}
|
||||
|
||||
char* protocol_receive_str_redacted(ProtocolSession* session) {
|
||||
return protocol_receive_str_impl(session, true);
|
||||
}
|
||||
|
||||
bool protocol_send_data(ProtocolSession* session, const Data* data) {
|
||||
if (!data || (!data->data && data->size != 0))
|
||||
return false;
|
||||
if (!session)
|
||||
return false;
|
||||
bool send_data(int file_descriptor, Data *data) {
|
||||
unsigned long long data_size = data->size;
|
||||
if (!protocol_send_n_data(session, &data_size, sizeof(unsigned long long)))
|
||||
if (!send_n_data(file_descriptor, &data_size, sizeof(unsigned long long)))
|
||||
return false;
|
||||
if (!protocol_send_n_data(session, data->data, data_size))
|
||||
if (!send_n_data(file_descriptor, data->data, data_size))
|
||||
return false;
|
||||
log_debug_message(LOG_DEBUG_PROTO, "Send %lld data", data_size);
|
||||
log_message(LOG_LEVEL_DEBUG, "Send %lld data", data_size);
|
||||
return true;
|
||||
}
|
||||
|
||||
Data* protocol_receive_data_limited(ProtocolSession* session, unsigned long long maximum_size) {
|
||||
if (!session)
|
||||
return NULL;
|
||||
Data *receive_data(int file_descriptor) {
|
||||
unsigned long long size = 0;
|
||||
if (!protocol_receive_n_data(session, &size, sizeof(unsigned long long)))
|
||||
if (!receive_n_data(file_descriptor, &size, sizeof(unsigned long long)))
|
||||
return NULL;
|
||||
if (size > MAX_DATA_PAYLOAD_SIZE || size > maximum_size) {
|
||||
log_message(LOG_LEVEL_ERROR, "Data size %llu exceeds maximum %llu", size,
|
||||
(unsigned long long)MAX_DATA_PAYLOAD_SIZE);
|
||||
return NULL;
|
||||
}
|
||||
if (size > SIZE_MAX)
|
||||
return NULL;
|
||||
size_t allocation_size = size == 0 ? 1 : (size_t)size;
|
||||
if (!protocol_reserve_memory(session, allocation_size)) {
|
||||
log_message(LOG_LEVEL_ERROR, "Per-connection memory limit exceeded (%llu + %llu > %llu)",
|
||||
(unsigned long long)atomic_load(&session->total_allocated_bytes), size,
|
||||
(unsigned long long)MAX_CONNECTION_MEMORY);
|
||||
return NULL;
|
||||
}
|
||||
void* data = protocol_alloc_for_session(session, allocation_size);
|
||||
if (data == NULL) {
|
||||
protocol_release_memory_for_session(session, allocation_size);
|
||||
return NULL;
|
||||
}
|
||||
if (!protocol_receive_n_data(session, data, (size_t)size)) {
|
||||
void *data = malloc((size_t)size);
|
||||
if (data == NULL) return NULL;
|
||||
if (!receive_n_data(file_descriptor, data, (size_t)size)) {
|
||||
free(data);
|
||||
protocol_release_memory_for_session(session, allocation_size);
|
||||
return NULL;
|
||||
}
|
||||
log_debug_message(LOG_DEBUG_PROTO, "Received %lld data", size);
|
||||
Data* result = data_create(data, (size_t)size);
|
||||
if (!result) {
|
||||
protocol_release_memory_for_session(session, allocation_size);
|
||||
return NULL;
|
||||
}
|
||||
result->protocol_charge = allocation_size;
|
||||
return result;
|
||||
log_message(LOG_LEVEL_DEBUG, "Received %lld data", size);
|
||||
return data_create(data, (size_t)size);
|
||||
}
|
||||
|
||||
Data* protocol_receive_data(ProtocolSession* session) {
|
||||
return protocol_receive_data_limited(session, MAX_DATA_PAYLOAD_SIZE);
|
||||
}
|
||||
|
||||
bool protocol_send_int(ProtocolSession* session, int data) {
|
||||
if (!protocol_send_n_data(session, &data, sizeof(int)))
|
||||
return false;
|
||||
log_debug_message(LOG_DEBUG_PROTO, "Send Int: %d", data);
|
||||
bool send_int(int file_descriptor, int data) {
|
||||
if (!send_n_data(file_descriptor, &data, sizeof(int))) return false;
|
||||
log_message(LOG_LEVEL_DEBUG, "Send Int: %d", data);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool protocol_receive_int(ProtocolSession* session, int* data) {
|
||||
if (!protocol_receive_n_data(session, data, sizeof(int)))
|
||||
return false;
|
||||
log_debug_message(LOG_DEBUG_PROTO, "Received Int: %d", *data);
|
||||
bool receive_int(int file_descriptor, int *data) {
|
||||
if (!receive_n_data(file_descriptor, data, sizeof(int))) return false;
|
||||
log_message(LOG_LEVEL_DEBUG, "Received Int: %d", *data);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool protocol_send_status(ProtocolSession* session, Status status) {
|
||||
if (!protocol_send_n_data(session, &status, sizeof(Status)))
|
||||
return false;
|
||||
log_debug_message(LOG_DEBUG_PROTO, "Send Status: %s", status_to_string(status));
|
||||
bool send_status(int file_descriptor, Status status) {
|
||||
if (!send_n_data(file_descriptor, &status, sizeof(Status))) return false;
|
||||
log_message(LOG_LEVEL_DEBUG, "Send Status: %s", status_to_string(status));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool protocol_receive_status(ProtocolSession* session, Status* status) {
|
||||
if (!protocol_receive_n_data(session, status, sizeof(Status)))
|
||||
return false;
|
||||
log_debug_message(LOG_DEBUG_PROTO, "Received Status: %s", status_to_string(*status));
|
||||
bool receive_status(int file_descriptor, Status *status) {
|
||||
if (!receive_n_data(file_descriptor, status, sizeof(Status))) return false;
|
||||
log_message(LOG_LEVEL_DEBUG, "Received Status: %s", status_to_string(*status));
|
||||
return true;
|
||||
}
|
||||
|
||||
/* protocol_receive_status with an explicit per-message deadline (seconds).
|
||||
Used where a single reply may legitimately take far longer than the default
|
||||
60 s receive window - e.g. the sender waiting for the early-delete ACK after
|
||||
the receiver committed a large (up to MAX_SERVER_DELETE_COUNT) deletion. */
|
||||
bool protocol_receive_status_timed(ProtocolSession* session, Status* status, int timeout_sec) {
|
||||
if (!protocol_receive_n_data_timed(session, status, sizeof(Status), timeout_sec))
|
||||
return false;
|
||||
log_debug_message(LOG_DEBUG_PROTO, "Received Status: %s", status_to_string(*status));
|
||||
return true;
|
||||
}
|
||||
|
||||
bool send_str(int fd, const char* data) {
|
||||
return protocol_send_str(legacy_session(-1, fd), data);
|
||||
}
|
||||
char* receive_str(int fd) {
|
||||
return protocol_receive_str(legacy_session(fd, -1));
|
||||
}
|
||||
/* Redacted variants: identical framing, but the string body is never written to
|
||||
the debug protocol log. Used for daemon auth material (username, proof,
|
||||
signature). */
|
||||
bool send_str_redacted(int fd, const char* data) {
|
||||
return protocol_send_str_redacted(legacy_session(-1, fd), data);
|
||||
}
|
||||
char* receive_str_redacted(int fd) {
|
||||
return protocol_receive_str_redacted(legacy_session(fd, -1));
|
||||
}
|
||||
bool send_data(int fd, const Data* data) {
|
||||
return protocol_send_data(legacy_session(-1, fd), data);
|
||||
}
|
||||
Data* receive_data(int fd) {
|
||||
return protocol_receive_data_limited(legacy_session(fd, -1), MAX_DATA_PAYLOAD_SIZE);
|
||||
}
|
||||
Data* receive_data_limited(int fd, unsigned long long maximum_size) {
|
||||
return protocol_receive_data_limited(legacy_session(fd, -1), maximum_size);
|
||||
}
|
||||
bool send_int(int fd, int data) {
|
||||
return protocol_send_int(legacy_session(-1, fd), data);
|
||||
}
|
||||
bool receive_int(int fd, int* data) {
|
||||
return protocol_receive_int(legacy_session(fd, -1), data);
|
||||
}
|
||||
bool send_status(int fd, Status status) {
|
||||
return protocol_send_status(legacy_session(-1, fd), status);
|
||||
}
|
||||
bool receive_status(int fd, Status* status) {
|
||||
return protocol_receive_status(legacy_session(fd, -1), status);
|
||||
}
|
||||
bool receive_status_timed(int fd, Status* status, int timeout_sec) {
|
||||
return protocol_receive_status_timed(legacy_session(fd, -1), status, timeout_sec);
|
||||
}
|
||||
|
||||
+11
-169
@@ -4,184 +4,26 @@
|
||||
#include "data.h"
|
||||
#include <stdbool.h>
|
||||
#include <stddef.h>
|
||||
#include <stdatomic.h>
|
||||
|
||||
/* Maximum allowed string size for receive_str (64 KB) */
|
||||
#define MAX_STRING_SIZE (64 * 1024)
|
||||
|
||||
/* Maximum uncompressed file payload accepted by the receiver's whole-file
|
||||
* paths. A single whole file is charged against the per-connection memory
|
||||
* reservation (MAX_CONNECTION_MEMORY) and against the server allocation
|
||||
* ceiling (MAX_SERVER_ALLOC), so this mirrors those 256 MB bounds rather than
|
||||
* the older 64 MB chunk-era cap. Chunk-serialized payloads keep their own
|
||||
* 64 MB cap (MAX_CHUNK_SIZE). */
|
||||
#define MAX_RECEIVE_WHOLE_FILE_SIZE (256ULL * 1024 * 1024)
|
||||
|
||||
/* Maximum allowed data payload size for receive_data (whole-file bound) */
|
||||
#define MAX_DATA_PAYLOAD_SIZE MAX_RECEIVE_WHOLE_FILE_SIZE
|
||||
|
||||
/* Maximum chunk size (64 MB) — prevents unbounded allocation from the wire */
|
||||
#define MAX_CHUNK_SIZE (64ULL * 1024 * 1024)
|
||||
#define MAX_MANIFEST_ENTRIES (1024 * 1024)
|
||||
/* Aggregate bytes retained by one received deletion manifest. */
|
||||
#define MAX_MANIFEST_BYTES (16ULL * 1024 * 1024)
|
||||
#define DEFAULT_MAX_ALLOC (1ULL * 1024 * 1024 * 1024)
|
||||
/* Server policy ceiling for a client-provided allocation limit. */
|
||||
#define MAX_SERVER_ALLOC (256ULL * 1024 * 1024)
|
||||
/* Bounded cumulative per-connection receive budget. In-flight wire buffers,
|
||||
decompression buffers and queued (not yet written) file payloads for a
|
||||
connection must stay within this ceiling. */
|
||||
#define MAX_CONNECTION_MEMORY (256ULL * 1024 * 1024)
|
||||
|
||||
typedef struct ssl_st SSL;
|
||||
|
||||
/*
|
||||
* Explicit owner of protocol I/O. A session does not own the descriptors or
|
||||
* SSL object; it only describes the transport used by a transfer. This makes
|
||||
* it safe to pass the transport to a worker without relying on inherited
|
||||
* thread-local state.
|
||||
*/
|
||||
typedef struct ProtocolSession {
|
||||
int read_fd;
|
||||
int write_fd;
|
||||
SSL* ssl;
|
||||
unsigned long long bwlimit;
|
||||
long long bw_tokens;
|
||||
long long bw_last_refill_sec;
|
||||
long bw_last_refill_nsec;
|
||||
atomic_ullong total_allocated_bytes;
|
||||
bool eight_bit_output;
|
||||
unsigned long long max_alloc;
|
||||
} ProtocolSession;
|
||||
|
||||
typedef int Status;
|
||||
enum NET_STATUS {
|
||||
STATUS_OK,
|
||||
STATUS_ERROR,
|
||||
STATUS_FINISHED,
|
||||
STATUS_NEXT,
|
||||
STATUS_CHUNK,
|
||||
STATUS_MANIFEST,
|
||||
STATUS_CHECK,
|
||||
STATUS_DELTA_SIGNATURE,
|
||||
STATUS_DELTA_DATA,
|
||||
STATUS_KEEPALIVE,
|
||||
STATUS_ABORT,
|
||||
STATUS_CHECK_BATCH,
|
||||
/* An explicit directory entry (--dirs): the sender transmits only the path;
|
||||
* the receiver creates the directory below the receive root. */
|
||||
STATUS_MKDIR,
|
||||
/* --append / --append-verify tail resume. STATUS_APPEND is sent by the
|
||||
* receiver after a per-file STATUS_CHECK when the existing destination file
|
||||
* is SHORTER than the source and an append mode is negotiated: its payload is
|
||||
* the resume offset (the number of prefix bytes already present), after which
|
||||
* the sender answers either directly with STATUS_APPEND_DATA (plain --append,
|
||||
* prefix not verified) or, for --append-verify, first with STATUS_APPEND_SIG
|
||||
* carrying the xxHash64 of the source prefix; the receiver then replies
|
||||
* STATUS_APPEND_OK (prefix matched -> sender transmits the tail) or
|
||||
* STATUS_NEXT (prefix mismatch -> sender falls back to a full transfer).
|
||||
* STATUS_APPEND_DATA carries the tail bytes (compressed data frame). */
|
||||
STATUS_APPEND,
|
||||
STATUS_APPEND_SIG,
|
||||
STATUS_APPEND_OK,
|
||||
STATUS_APPEND_DATA,
|
||||
/* --hard-links/-H: a sibling (later member) of a source hard-link group.
|
||||
* The sender transmits only the path, the run-local link-group id, and the
|
||||
* first (data-carrying) member's destination-relative wire path; the receiver
|
||||
* creates this entry as a hard link to the first member's installed file
|
||||
* (falling back to a byte-identical copy if link() fails). Protocol 2.12.0. */
|
||||
STATUS_HARDLINK,
|
||||
/* A symlink-type entry (-l/--links, -k/--copy-dirlinks' keep-as-symlink
|
||||
* branch). The sender transmits the destination path, the (sender-munged,
|
||||
* if --munge-links) symlink target, and optional metadata; the receiver
|
||||
* creates a symlink to the unmunged target beneath the receive root (see
|
||||
* file_receive_symlink). Protocol 2.13.0. */
|
||||
STATUS_SYMLINK,
|
||||
/* --devices / --specials (-D): a device or special node the sender wants
|
||||
* recreated (not written from content). Payload: destination path, the
|
||||
* metadata frame (whose mode's S_IFMT bits carry the node kind), and two
|
||||
* int32 rdev major/minor fields. The receiver validates the kind and rdev,
|
||||
* confines the node below the receive root, and recreates it (mknod/mkfifo),
|
||||
* privilege-gating the mknod. Protocol 2.13.0. */
|
||||
STATUS_SPECIAL,
|
||||
/* Directory-time superstructure (P7 Wave D, protocol 2.17.0): one or more
|
||||
* trailing frames sent after all file data (and after the optional delete
|
||||
* manifest) carrying the source directories' captured metadata so the
|
||||
* receiver can apply directory mtimes/atimes AFTER all of a directory's
|
||||
* children have been written. Payload per frame: an int count, then count
|
||||
* repetitions of (wire path string, metadata frame); an entry count larger
|
||||
* than MAX_MANIFEST_ENTRIES is split across repeated frames. The receiver
|
||||
* defers the actual utimensat until its own delete/publish phase has
|
||||
* committed, then skips the whole set when -O/--omit-dir-times is set. */
|
||||
STATUS_DIR_TIMES,
|
||||
/* Daemon SCRAM-SHA-256 authentication (A7 remediation, protocol 2.19.0).
|
||||
* STATUS_AUTH_CHALLENGE: the server requires auth and is about to send the
|
||||
* iteration count, the base64 salt and the base64 server nonce.
|
||||
* STATUS_AUTH_RESPONSE: the client's reply, followed by the base64 client
|
||||
* nonce and the base64 ClientProof. STATUS_AUTH_OK: the client proof
|
||||
* verified, followed by the base64 ServerSignature. STATUS_AUTH_FAILED:
|
||||
* a single generic refusal (unknown user, off-list user, wrong proof,
|
||||
* missing/malformed credentials) after which the server closes without
|
||||
* writing any data. */
|
||||
STATUS_AUTH_CHALLENGE,
|
||||
STATUS_AUTH_RESPONSE,
|
||||
STATUS_AUTH_OK,
|
||||
STATUS_AUTH_FAILED
|
||||
};
|
||||
enum NET_STATUS { STATUS_OK, STATUS_ERROR, STATUS_FINISHED, STATUS_NEXT, STATUS_CHUNK, STATUS_MANIFEST, STATUS_CHECK, STATUS_DELTA_SIGNATURE, STATUS_DELTA_DATA };
|
||||
|
||||
void io_set_fds(int read_fd, int write_fd);
|
||||
void io_set_bwlimit(unsigned long long bytes_per_sec);
|
||||
void io_set_ssl(SSL* ssl);
|
||||
SSL* io_get_ssl(void);
|
||||
typedef struct ssl_st SSL;
|
||||
void io_set_ssl(SSL *ssl);
|
||||
bool send_n_data(int file_descriptor, void *data, size_t data_size);
|
||||
bool receive_n_data(int file_descriptor, void *data, size_t data_size);
|
||||
|
||||
void protocol_session_init(ProtocolSession* session, int read_fd, int write_fd);
|
||||
/* Transitional bridge for helpers whose signatures still carry only an fd. */
|
||||
void protocol_session_bind(ProtocolSession* session);
|
||||
void protocol_session_unbind(void);
|
||||
void protocol_session_set_ssl(ProtocolSession* session, SSL* ssl);
|
||||
void protocol_session_set_bwlimit(ProtocolSession* session, unsigned long long bytes_per_sec);
|
||||
void protocol_session_set_max_alloc(ProtocolSession* session, unsigned long long max_alloc);
|
||||
void* protocol_alloc(size_t size);
|
||||
void* protocol_realloc(void* ptr, size_t size);
|
||||
void protocol_session_set_8_bit_output(ProtocolSession* session, bool enabled);
|
||||
void protocol_set_8_bit_output(bool enabled);
|
||||
bool protocol_send_n_data(ProtocolSession* session, const void* data, size_t data_size);
|
||||
bool protocol_receive_n_data(ProtocolSession* session, void* data, size_t data_size);
|
||||
bool protocol_send_str(ProtocolSession* session, const char* data);
|
||||
char* protocol_receive_str(ProtocolSession* session);
|
||||
/* Redacted string variants: identical wire framing to protocol_send_str /
|
||||
* protocol_receive_str, but the payload body is replaced by `<redacted>` in the
|
||||
* LOG_DEBUG_PROTO debug log. Used for daemon auth material (the username and
|
||||
* the proof/signature fields) so a --verbose log can never capture a credential
|
||||
* that could be replayed. */
|
||||
bool protocol_send_str_redacted(ProtocolSession* session, const char* data);
|
||||
char* protocol_receive_str_redacted(ProtocolSession* session);
|
||||
bool protocol_send_data(ProtocolSession* session, const Data* data);
|
||||
Data* protocol_receive_data(ProtocolSession* session);
|
||||
Data* protocol_receive_data_limited(ProtocolSession* session, unsigned long long maximum_size);
|
||||
bool protocol_send_int(ProtocolSession* session, int data);
|
||||
bool protocol_receive_int(ProtocolSession* session, int* data);
|
||||
bool protocol_send_status(ProtocolSession* session, Status status);
|
||||
bool protocol_receive_status(ProtocolSession* session, Status* status);
|
||||
bool send_n_data(int file_descriptor, const void* data, size_t data_size);
|
||||
bool receive_n_data(int file_descriptor, void* data, size_t data_size);
|
||||
|
||||
bool send_str(int file_descriptor, const char* data);
|
||||
char* receive_str(int file_descriptor);
|
||||
/* Redacted fd-level string variants (see protocol_send_str_redacted). */
|
||||
bool send_str_redacted(int file_descriptor, const char* data);
|
||||
char* receive_str_redacted(int file_descriptor);
|
||||
bool send_data(int file_descriptor, const Data* data);
|
||||
Data* receive_data(int file_descriptor);
|
||||
Data* receive_data_limited(int file_descriptor, unsigned long long maximum_size);
|
||||
bool send_str(int file_descriptor, char *data);
|
||||
char *receive_str(int file_descriptor);
|
||||
bool send_data(int file_descriptor, Data *data);
|
||||
Data *receive_data(int file_descriptor);
|
||||
bool send_int(int file_descriptor, int data);
|
||||
bool receive_int(int file_descriptor, int* data);
|
||||
bool receive_int(int file_descriptor, int *data);
|
||||
bool send_status(int file_descriptor, Status status);
|
||||
bool receive_status(int file_descriptor, Status* status);
|
||||
/* receive_status with an explicit per-message deadline in seconds, instead of
|
||||
the default RECEIVE_TIMEOUT_SEC. A reply that may legitimately take longer
|
||||
(e.g. the early-delete ACK after a large receiver-side deletion) must use
|
||||
this so the sender does not abort after the deletion already committed. */
|
||||
bool receive_status_timed(int file_descriptor, Status* status, int timeout_sec);
|
||||
bool receive_status(int file_descriptor, Status *status);
|
||||
|
||||
#endif
|
||||
|
||||
+132
-155
@@ -1,155 +1,132 @@
|
||||
#include "log.h"
|
||||
#include <stdbool.h>
|
||||
#include <limits.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <threads.h>
|
||||
|
||||
#include "queue.h"
|
||||
|
||||
Queue* queue_create(int capacity, void (*destroyer)(void* item)) {
|
||||
if (capacity <= 0)
|
||||
return NULL;
|
||||
|
||||
Queue* queue = (Queue*)malloc(sizeof(Queue));
|
||||
if (queue == NULL) {
|
||||
log_perror("ERROR: Could not allocate memory for queue structure");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
queue->items = malloc(capacity * sizeof(void*));
|
||||
if (queue->items == NULL) {
|
||||
free(queue);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
for (int i = 0; i < capacity; ++i) {
|
||||
queue->items[i] = NULL;
|
||||
}
|
||||
|
||||
queue->capacity = capacity;
|
||||
queue->front = 0;
|
||||
queue->rear = 0;
|
||||
queue->size = 0;
|
||||
queue->item_destroyer = destroyer;
|
||||
|
||||
return queue;
|
||||
}
|
||||
|
||||
void queue_destroy(Queue* queue) {
|
||||
if (queue == NULL)
|
||||
return;
|
||||
|
||||
if (queue->item_destroyer != NULL) {
|
||||
for (int i = 0; i < queue->size; ++i) {
|
||||
int index = (queue->front + i) % queue->capacity;
|
||||
queue->item_destroyer(queue->items[index]);
|
||||
}
|
||||
}
|
||||
free(queue->items);
|
||||
free(queue);
|
||||
}
|
||||
|
||||
bool queue_is_empty(const Queue* queue) {
|
||||
if (queue == NULL)
|
||||
return true;
|
||||
return queue->size == 0;
|
||||
}
|
||||
|
||||
bool queue_is_full(const Queue* queue) {
|
||||
if (queue == NULL)
|
||||
return false;
|
||||
return queue->size == queue->capacity;
|
||||
}
|
||||
|
||||
static bool queue_double_capacity(Queue* queue) {
|
||||
if (queue == NULL)
|
||||
return false;
|
||||
if (queue->capacity > INT_MAX / 2)
|
||||
return false;
|
||||
int new_capacity = queue->capacity * 2;
|
||||
if (new_capacity <= 1)
|
||||
new_capacity = 100;
|
||||
void** new_items = malloc(new_capacity * sizeof(void*));
|
||||
if (new_items == NULL) {
|
||||
log_perror("ERROR: Could not allocate memory for doubling capacity of queue.");
|
||||
return false;
|
||||
}
|
||||
for (int i = 0; i < queue->size; i++)
|
||||
new_items[i] = queue->items[(i + queue->front) % queue->capacity];
|
||||
free(queue->items);
|
||||
queue->items = new_items;
|
||||
queue->front = 0;
|
||||
queue->rear = queue->size;
|
||||
queue->capacity = new_capacity;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool queue_enqueue(Queue* queue, void* item) {
|
||||
if (queue == NULL || item == NULL)
|
||||
return false;
|
||||
if (queue_is_full(queue)) {
|
||||
if (!queue_double_capacity(queue))
|
||||
return false;
|
||||
}
|
||||
queue->items[queue->rear] = item;
|
||||
queue->rear = (queue->rear + 1) % queue->capacity;
|
||||
queue->size++;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool queue_enqueue_multithreaded(Queue* queue, void* item, mtx_t* mutex, cnd_t* condition_not_empty,
|
||||
cnd_t* condition_not_full) {
|
||||
mtx_lock(mutex);
|
||||
while (queue_is_full(queue))
|
||||
cnd_wait(condition_not_full, mutex);
|
||||
bool ok = queue_enqueue(queue, item);
|
||||
cnd_signal(condition_not_empty);
|
||||
mtx_unlock(mutex);
|
||||
return ok;
|
||||
}
|
||||
|
||||
bool queue_enqueue_multithreaded_cancel(Queue* queue, void* item, mtx_t* mutex,
|
||||
cnd_t* condition_not_empty, cnd_t* condition_not_full,
|
||||
const atomic_bool* cancelled) {
|
||||
mtx_lock(mutex);
|
||||
while (queue_is_full(queue) && (cancelled == NULL || !atomic_load(cancelled)))
|
||||
cnd_wait(condition_not_full, mutex);
|
||||
if (cancelled != NULL && atomic_load(cancelled)) {
|
||||
mtx_unlock(mutex);
|
||||
return false;
|
||||
}
|
||||
bool ok = queue_enqueue(queue, item);
|
||||
cnd_signal(condition_not_empty);
|
||||
mtx_unlock(mutex);
|
||||
return ok;
|
||||
}
|
||||
|
||||
void* queue_dequeue(Queue* queue) {
|
||||
if (queue == NULL || queue_is_empty(queue)) {
|
||||
log_perror("ERROR: Could not dequeue from null or empty queue.");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
void* item = queue->items[queue->front];
|
||||
queue->items[queue->front] = NULL;
|
||||
queue->front = (queue->front + 1) % queue->capacity;
|
||||
queue->size--;
|
||||
return item;
|
||||
}
|
||||
|
||||
void* queue_dequeue_multithreaded(Queue* queue, mtx_t* mutex, cnd_t* condition_not_empty,
|
||||
cnd_t* condition_not_full, const bool* other_thread_done) {
|
||||
mtx_lock(mutex);
|
||||
while (queue_is_empty(queue) && !*other_thread_done)
|
||||
cnd_wait(condition_not_empty, mutex);
|
||||
if (queue_is_empty(queue) && *other_thread_done) {
|
||||
mtx_unlock(mutex);
|
||||
return NULL;
|
||||
}
|
||||
void* item = queue_dequeue(queue);
|
||||
cnd_signal(condition_not_full);
|
||||
mtx_unlock(mutex);
|
||||
return item;
|
||||
}
|
||||
#include <stdbool.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <threads.h>
|
||||
|
||||
#include "queue.h"
|
||||
|
||||
Queue *queue_create(int capacity, void (*destroyer)(void *item)) {
|
||||
Queue *queue = (Queue *)malloc(sizeof(Queue));
|
||||
if (queue == NULL) {
|
||||
perror("ERROR: Could not allocate memory for queue structure");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
queue->items = malloc(capacity * sizeof(void *));
|
||||
if (queue->items == NULL) {
|
||||
free(queue);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
for (int i = 0; i < capacity; ++i) {
|
||||
queue->items[i] = NULL;
|
||||
}
|
||||
|
||||
queue->capacity = capacity;
|
||||
queue->front = 0;
|
||||
queue->rear = 0;
|
||||
queue->size = 0;
|
||||
queue->item_destroyer = destroyer;
|
||||
|
||||
return queue;
|
||||
}
|
||||
|
||||
void queue_destroy(Queue *queue) {
|
||||
if (queue == NULL)
|
||||
return;
|
||||
|
||||
if (queue->item_destroyer != NULL) {
|
||||
for (int i = 0; i < queue->size; ++i) {
|
||||
int index = (queue->front + i) % queue->capacity;
|
||||
queue->item_destroyer(queue->items[index]);
|
||||
}
|
||||
}
|
||||
free(queue->items);
|
||||
free(queue);
|
||||
}
|
||||
|
||||
bool queue_is_empty(Queue *queue) {
|
||||
if (queue == NULL)
|
||||
return true;
|
||||
return queue->size == 0;
|
||||
}
|
||||
|
||||
bool queue_is_full(Queue *queue) {
|
||||
if (queue == NULL)
|
||||
return false;
|
||||
return queue->size == queue->capacity;
|
||||
}
|
||||
|
||||
static bool queue_double_capacity(Queue *queue) {
|
||||
if (queue == NULL) return false;
|
||||
unsigned int new_capacity = queue->capacity * 2;
|
||||
if (new_capacity <= 1)
|
||||
new_capacity = 100;
|
||||
void **new_items = malloc(new_capacity * sizeof(void *));
|
||||
if (new_items == NULL) {
|
||||
perror("ERROR: Could not allocate memory for doubling capacity of queue.");
|
||||
return false;
|
||||
}
|
||||
for (int i = 0; i < queue->size; i++)
|
||||
new_items[i] = queue->items[(i + queue->front) % queue->capacity];
|
||||
free(queue->items);
|
||||
queue->items = new_items;
|
||||
queue->front = 0;
|
||||
queue->rear = queue->size;
|
||||
queue->capacity = new_capacity;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool queue_enqueue(Queue *queue, void *item) {
|
||||
if (queue == NULL || item == NULL) return false;
|
||||
if (queue_is_full(queue)) {
|
||||
if (!queue_double_capacity(queue)) return false;
|
||||
}
|
||||
queue->items[queue->rear] = item;
|
||||
queue->rear = (queue->rear + 1) % queue->capacity;
|
||||
queue->size++;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool queue_enqueue_multithreaded(Queue *queue, void *item, mtx_t *mutex,
|
||||
cnd_t *condition_not_empty,
|
||||
cnd_t *condition_not_full) {
|
||||
mtx_lock(mutex);
|
||||
while (queue_is_full(queue))
|
||||
cnd_wait(condition_not_full, mutex);
|
||||
bool ok = queue_enqueue(queue, item);
|
||||
cnd_signal(condition_not_empty);
|
||||
mtx_unlock(mutex);
|
||||
return ok;
|
||||
}
|
||||
|
||||
void *queue_dequeue(Queue *queue) {
|
||||
if (queue == NULL || queue_is_empty(queue)) {
|
||||
perror("ERROR: Could not dequeue from null or empty queue.");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
void *item = queue->items[queue->front];
|
||||
queue->items[queue->front] = NULL;
|
||||
queue->front = (queue->front + 1) % queue->capacity;
|
||||
queue->size--;
|
||||
return item;
|
||||
}
|
||||
|
||||
void *queue_dequeue_multithreaded(Queue *queue, mtx_t *mutex,
|
||||
cnd_t *condition_not_empty,
|
||||
cnd_t *condition_not_full,
|
||||
bool *other_thread_done) {
|
||||
mtx_lock(mutex);
|
||||
while (queue_is_empty(queue) && !*other_thread_done)
|
||||
cnd_wait(condition_not_empty, mutex);
|
||||
if (queue_is_empty(queue) && *other_thread_done) {
|
||||
mtx_unlock(mutex);
|
||||
return NULL;
|
||||
}
|
||||
void *item = queue_dequeue(queue);
|
||||
cnd_signal(condition_not_full);
|
||||
mtx_unlock(mutex);
|
||||
return item;
|
||||
}
|
||||
|
||||
+30
-31
@@ -1,31 +1,30 @@
|
||||
#ifndef QUEUE_H
|
||||
#define QUEUE_H
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <stdatomic.h>
|
||||
#include <threads.h>
|
||||
|
||||
typedef struct Queue {
|
||||
void** items;
|
||||
int front;
|
||||
int rear;
|
||||
int size;
|
||||
int capacity;
|
||||
void (*item_destroyer)(void* item);
|
||||
} Queue;
|
||||
|
||||
Queue* queue_create(int capacity, void (*destroyer)(void* item));
|
||||
void queue_destroy(Queue* queue);
|
||||
bool queue_is_empty(const Queue* queue);
|
||||
bool queue_is_full(const Queue* queue);
|
||||
bool queue_enqueue(Queue* queue, void* item);
|
||||
bool queue_enqueue_multithreaded(Queue* queue, void* item, mtx_t* mutex, cnd_t* condition_not_empty,
|
||||
cnd_t* condition_not_full);
|
||||
bool queue_enqueue_multithreaded_cancel(Queue* queue, void* item, mtx_t* mutex,
|
||||
cnd_t* condition_not_empty, cnd_t* condition_not_full,
|
||||
const atomic_bool* cancelled);
|
||||
void* queue_dequeue(Queue* queue);
|
||||
void* queue_dequeue_multithreaded(Queue* queue, mtx_t* mutex, cnd_t* condition_not_empty,
|
||||
cnd_t* condition_not_full, const bool* other_thread_done);
|
||||
|
||||
#endif
|
||||
#ifndef QUEUE_H
|
||||
#define QUEUE_H
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <threads.h>
|
||||
|
||||
typedef struct Queue {
|
||||
void **items;
|
||||
int front;
|
||||
int rear;
|
||||
int size;
|
||||
int capacity;
|
||||
void (*item_destroyer)(void *item);
|
||||
} Queue;
|
||||
|
||||
Queue *queue_create(int capacity, void (*destroyer)(void *item));
|
||||
void queue_destroy(Queue *queue);
|
||||
bool queue_is_empty(Queue *queue);
|
||||
bool queue_is_full(Queue *queue);
|
||||
bool queue_enqueue(Queue *queue, void *item);
|
||||
bool queue_enqueue_multithreaded(Queue *queue, void *item, mtx_t *mutex,
|
||||
cnd_t *condition_not_empty,
|
||||
cnd_t *condition_not_full);
|
||||
void *queue_dequeue(Queue *queue);
|
||||
void *queue_dequeue_multithreaded(Queue *queue, mtx_t *mutex,
|
||||
cnd_t *condition_not_empty,
|
||||
cnd_t *condition_not_full,
|
||||
bool *other_thread_done);
|
||||
|
||||
#endif
|
||||
|
||||
@@ -1,157 +0,0 @@
|
||||
#include "stop_condition.h"
|
||||
#include <errno.h>
|
||||
#include <limits.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
/* Parse a strictly positive decimal integer: only ASCII digits, no leading
|
||||
* whitespace, sign or trailing garbage. */
|
||||
static bool parse_positive_minutes(const char* value, long* out) {
|
||||
if (!value || *value == '\0')
|
||||
return false;
|
||||
if (*value < '0' || *value > '9')
|
||||
return false;
|
||||
long v = 0;
|
||||
for (const char* p = value; *p != '\0'; p++) {
|
||||
if (*p < '0' || *p > '9')
|
||||
return false;
|
||||
int digit = *p - '0';
|
||||
if (v > (LONG_MAX - digit) / 10)
|
||||
return false;
|
||||
v = v * 10 + digit;
|
||||
}
|
||||
if (v <= 0 || v > INT_MAX)
|
||||
return false;
|
||||
*out = v;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool stop_parse_after_minutes(const char* value, int* out_minutes) {
|
||||
if (!out_minutes)
|
||||
return false;
|
||||
long minutes = 0;
|
||||
if (!parse_positive_minutes(value, &minutes))
|
||||
return false;
|
||||
*out_minutes = (int)minutes;
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Two consecutive ASCII digits -> 0..99. */
|
||||
static bool parse_two_digits(const char* s, int* out) {
|
||||
if (s[0] < '0' || s[0] > '9' || s[1] < '0' || s[1] > '9')
|
||||
return false;
|
||||
*out = (s[0] - '0') * 10 + (s[1] - '0');
|
||||
return true;
|
||||
}
|
||||
|
||||
bool stop_parse_at_time(const char* value, time_t now, time_t* out_deadline) {
|
||||
if (!value || !out_deadline)
|
||||
return false;
|
||||
|
||||
/* now+N[smhd]: N whole units from the current wall clock. */
|
||||
if (strncmp(value, "now+", 4) == 0) {
|
||||
const char* p = value + 4;
|
||||
/* The count must be a bare non-negative digit run: reject leading
|
||||
whitespace ('now+ 5s') and a leading sign ('now++5s'). */
|
||||
if (*p < '0' || *p > '9')
|
||||
return false;
|
||||
errno = 0;
|
||||
char* end = NULL;
|
||||
long amount = strtol(p, &end, 10);
|
||||
if (errno != 0 || end == p || amount < 0)
|
||||
return false;
|
||||
long unit_seconds;
|
||||
switch (*end) {
|
||||
case 's':
|
||||
unit_seconds = 1;
|
||||
break;
|
||||
case 'm':
|
||||
unit_seconds = 60;
|
||||
break;
|
||||
case 'h':
|
||||
unit_seconds = 3600;
|
||||
break;
|
||||
case 'd':
|
||||
unit_seconds = 86400;
|
||||
break;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
if (end[1] != '\0')
|
||||
return false;
|
||||
if (amount > LONG_MAX / unit_seconds)
|
||||
return false;
|
||||
long long delta = (long long)amount * unit_seconds;
|
||||
/* Guard against signed overflow of now + delta. */
|
||||
if ((long long)now > 0 && delta > (long long)LLONG_MAX - (long long)now)
|
||||
return false;
|
||||
if ((long long)now < 0 && delta < (long long)LLONG_MIN - (long long)now)
|
||||
return false;
|
||||
*out_deadline = now + (time_t)delta;
|
||||
return true;
|
||||
}
|
||||
|
||||
/* HH:MM or HH:MM:SS on the current local day. */
|
||||
size_t len = strlen(value);
|
||||
if (len != 5 && len != 8)
|
||||
return false;
|
||||
if (value[2] != ':' || (len == 8 && value[5] != ':'))
|
||||
return false;
|
||||
int hh, mm, ss = 0;
|
||||
if (!parse_two_digits(value, &hh) || !parse_two_digits(value + 3, &mm))
|
||||
return false;
|
||||
if (len == 8 && !parse_two_digits(value + 6, &ss))
|
||||
return false;
|
||||
if (hh > 23 || mm > 59 || ss > 59)
|
||||
return false;
|
||||
|
||||
struct tm today;
|
||||
if (!localtime_r(&now, &today))
|
||||
return false;
|
||||
today.tm_hour = hh;
|
||||
today.tm_min = mm;
|
||||
today.tm_sec = ss;
|
||||
today.tm_isdst = -1;
|
||||
time_t deadline = mktime(&today);
|
||||
if (deadline == (time_t)-1)
|
||||
return false;
|
||||
*out_deadline = deadline;
|
||||
return true;
|
||||
}
|
||||
|
||||
StopCondition stop_condition_make(bool has_after, int after_minutes, bool has_at, time_t at_time,
|
||||
struct timespec now_mono) {
|
||||
StopCondition condition;
|
||||
condition.has_monotonic = false;
|
||||
condition.monotonic_deadline.tv_sec = 0;
|
||||
condition.monotonic_deadline.tv_nsec = 0;
|
||||
condition.has_wall = false;
|
||||
condition.wall_deadline = 0;
|
||||
if (has_after && after_minutes > 0) {
|
||||
condition.has_monotonic = true;
|
||||
condition.monotonic_deadline.tv_sec = now_mono.tv_sec + (time_t)after_minutes * 60;
|
||||
condition.monotonic_deadline.tv_nsec = now_mono.tv_nsec;
|
||||
}
|
||||
if (has_at) {
|
||||
condition.has_wall = true;
|
||||
condition.wall_deadline = at_time;
|
||||
}
|
||||
return condition;
|
||||
}
|
||||
|
||||
bool stop_condition_reached(const StopCondition* condition) {
|
||||
if (!condition)
|
||||
return false;
|
||||
if (condition->has_wall && time(NULL) >= condition->wall_deadline)
|
||||
return true;
|
||||
if (condition->has_monotonic) {
|
||||
struct timespec now;
|
||||
if (clock_gettime(CLOCK_MONOTONIC, &now) != 0)
|
||||
return false;
|
||||
if (now.tv_sec > condition->monotonic_deadline.tv_sec ||
|
||||
(now.tv_sec == condition->monotonic_deadline.tv_sec &&
|
||||
now.tv_nsec >= condition->monotonic_deadline.tv_nsec))
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user