Author SHA1 Message Date
TapTap 1aebcfd55d refactor: consolidate transfer infrastructure
CI / lint (pull_request) Failing after 3s
CI / build-and-test (pull_request) Skipped
CI / sanitizers (address) (pull_request) Skipped
CI / sanitizers (undefined) (pull_request) Skipped
CI / fuzz-build (pull_request) Skipped
CI / coverage (pull_request) Skipped
CI / valgrind (pull_request) Skipped
2026-09-01 21:21:24 +02:00
TapTap 6b7213db5c docs: align compatibility guidance
CI / lint (pull_request) Failing after 2s
CI / build-and-test (pull_request) Has been skipped
CI / sanitizers (address) (pull_request) Has been skipped
CI / sanitizers (undefined) (pull_request) Has been skipped
CI / fuzz-build (pull_request) Has been skipped
CI / coverage (pull_request) Has been skipped
CI / valgrind (pull_request) Has been skipped
2026-08-15 13:41:23 +02:00
TapTap de537810e5 docs: fix server destination root examples
CI / lint (pull_request) Failing after 2s
CI / build-and-test (pull_request) Has been skipped
CI / sanitizers (address) (pull_request) Has been skipped
CI / sanitizers (undefined) (pull_request) Has been skipped
CI / fuzz-build (pull_request) Has been skipped
CI / coverage (pull_request) Has been skipped
CI / valgrind (pull_request) Has been skipped
2026-08-15 13:14:28 +02:00
TapTap def58554d9 docs: clarify rsync compatibility roadmap
CI / lint (pull_request) Failing after 2s
CI / build-and-test (pull_request) Has been skipped
CI / sanitizers (address) (pull_request) Has been skipped
CI / sanitizers (undefined) (pull_request) Has been skipped
CI / fuzz-build (pull_request) Has been skipped
CI / coverage (pull_request) Has been skipped
CI / valgrind (pull_request) Has been skipped
2026-08-15 12:54:38 +02:00
TapTap 046c8d1035 Merge pull request 'Evaluation of codebase by human' (#210) from evaluation-of-codebase-by-human into dev
CI / lint (push) Failing after 3s
CI / build-and-test (push) Has been skipped
CI / sanitizers (address) (push) Has been skipped
CI / sanitizers (undefined) (push) Has been skipped
CI / fuzz-build (push) Has been skipped
CI / coverage (push) Has been skipped
CI / valgrind (push) Has been skipped
2026-08-11 21:24:37 +02:00
TapTap 634cddee3b Merge dev into human codebase evaluation
CI / lint (pull_request) Failing after 2s
CI / build-and-test (pull_request) Has been skipped
CI / sanitizers (address) (pull_request) Has been skipped
CI / sanitizers (undefined) (pull_request) Has been skipped
CI / fuzz-build (pull_request) Has been skipped
CI / coverage (pull_request) Has been skipped
CI / valgrind (pull_request) Has been skipped
2026-08-11 21:24:17 +02:00
TapTap 75040103d3 Merge pull request 'Remove unimplemented CLI options' (#209) from feat/remove-unimplemented-cli-options into evaluation-of-codebase-by-human
CI / lint (pull_request) Successful in 25s
CI / sanitizers (address) (pull_request) Successful in 35s
CI / sanitizers (undefined) (pull_request) Successful in 35s
CI / fuzz-build (pull_request) Successful in 13s
CI / coverage (pull_request) Successful in 31s
CI / build-and-test (pull_request) Successful in 1m15s
CI / valgrind (pull_request) Successful in 33s
Reviewed-on: #209
2026-08-11 21:17:52 +02:00
TapTap 852e47a143 Satisfy cppcheck CLI and logger checks
CI / lint (pull_request) Successful in 25s
CI / sanitizers (address) (pull_request) Successful in 35s
CI / sanitizers (undefined) (pull_request) Successful in 35s
CI / fuzz-build (pull_request) Successful in 14s
CI / coverage (pull_request) Successful in 31s
CI / build-and-test (pull_request) Successful in 1m14s
CI / valgrind (pull_request) Successful in 32s
2026-08-11 21:14:56 +02:00
TapTap 679e389ba6 Fix va_list cleanup in logger
CI / lint (pull_request) Failing after 25s
CI / build-and-test (pull_request) Has been skipped
CI / sanitizers (address) (pull_request) Has been skipped
CI / sanitizers (undefined) (pull_request) Has been skipped
CI / fuzz-build (pull_request) Has been skipped
CI / coverage (pull_request) Has been skipped
CI / valgrind (pull_request) Has been skipped
2026-08-11 21:12:00 +02:00
TapTap 2e00fc31f2 Format CLI option test table
CI / lint (pull_request) Failing after 25s
CI / build-and-test (pull_request) Has been skipped
CI / sanitizers (address) (pull_request) Has been skipped
CI / sanitizers (undefined) (pull_request) Has been skipped
CI / fuzz-build (pull_request) Has been skipped
CI / coverage (pull_request) Has been skipped
CI / valgrind (pull_request) Has been skipped
2026-08-11 21:08:54 +02:00
TapTap 9396de88db Merge pull request 'fix: complete issues #201-#206 remediation' (#208) from feat/triage-complete into dev
CI / lint (push) Successful in 33s
CI / sanitizers (address) (push) Successful in 37s
CI / sanitizers (undefined) (push) Successful in 36s
CI / fuzz-build (push) Successful in 13s
CI / coverage (push) Successful in 31s
CI / build-and-test (push) Successful in 1m15s
CI / valgrind (push) Successful in 32s
Reviewed-on: #208
2026-08-11 21:06:49 +02:00
TapTap d146e1bb7c Remove unimplemented CLI options
CI / lint (pull_request) Failing after 3s
CI / build-and-test (pull_request) Has been skipped
CI / sanitizers (address) (pull_request) Has been skipped
CI / sanitizers (undefined) (pull_request) Has been skipped
CI / fuzz-build (pull_request) Has been skipped
CI / coverage (pull_request) Has been skipped
CI / valgrind (pull_request) Has been skipped
2026-08-11 20:59:06 +02:00
TapTap 09fca8d931 start of human refactoring 2026-08-11 20:26:15 +02:00
TapTap b88e17826e fix: satisfy metadata const lint
CI / lint (pull_request) Successful in 33s
CI / sanitizers (address) (pull_request) Successful in 37s
CI / sanitizers (undefined) (pull_request) Successful in 37s
CI / fuzz-build (pull_request) Successful in 14s
CI / coverage (pull_request) Successful in 31s
CI / build-and-test (pull_request) Successful in 1m16s
CI / valgrind (pull_request) Successful in 33s
2026-08-10 19:20:38 +02:00
TapTap 690bf72d7f fix: close remaining PR 208 review findings
CI / lint (pull_request) Failing after 33s
CI / build-and-test (pull_request) Has been skipped
CI / sanitizers (address) (pull_request) Has been skipped
CI / sanitizers (undefined) (pull_request) Has been skipped
CI / fuzz-build (pull_request) Has been skipped
CI / coverage (pull_request) Has been skipped
CI / valgrind (pull_request) Has been skipped
2026-08-10 19:10:02 +02:00
TapTap d367706689 fix: satisfy file API const analysis
CI / lint (pull_request) Successful in 31s
CI / sanitizers (address) (pull_request) Successful in 38s
CI / sanitizers (undefined) (pull_request) Successful in 38s
CI / fuzz-build (pull_request) Successful in 14s
CI / coverage (pull_request) Successful in 31s
CI / build-and-test (pull_request) Successful in 1m16s
CI / valgrind (pull_request) Successful in 32s
2026-08-10 17:30:13 +02:00
TapTap 5f0aed72e7 fix: satisfy metadata const analysis
CI / lint (pull_request) Failing after 31s
CI / build-and-test (pull_request) Has been skipped
CI / sanitizers (address) (pull_request) Has been skipped
CI / sanitizers (undefined) (pull_request) Has been skipped
CI / fuzz-build (pull_request) Has been skipped
CI / coverage (pull_request) Has been skipped
CI / valgrind (pull_request) Has been skipped
2026-08-10 17:28:49 +02:00
TapTap 44c2a8bb79 fix: satisfy static analysis in review paths
CI / lint (pull_request) Failing after 31s
CI / build-and-test (pull_request) Has been skipped
CI / sanitizers (address) (pull_request) Has been skipped
CI / sanitizers (undefined) (pull_request) Has been skipped
CI / fuzz-build (pull_request) Has been skipped
CI / coverage (pull_request) Has been skipped
CI / valgrind (pull_request) Has been skipped
2026-08-10 17:27:33 +02:00
TapTap 7776a63d3d fix: close remaining PR 208 review gaps
CI / lint (pull_request) Failing after 32s
CI / build-and-test (pull_request) Has been skipped
CI / sanitizers (address) (pull_request) Has been skipped
CI / sanitizers (undefined) (pull_request) Has been skipped
CI / fuzz-build (pull_request) Has been skipped
CI / coverage (pull_request) Has been skipped
CI / valgrind (pull_request) Has been skipped
2026-08-10 17:26:00 +02:00
TapTap 5bc319f694 fix: remove redundant poll timeout branch
CI / lint (pull_request) Successful in 26s
CI / sanitizers (address) (pull_request) Successful in 36s
CI / sanitizers (undefined) (pull_request) Successful in 36s
CI / fuzz-build (pull_request) Successful in 14s
CI / coverage (pull_request) Successful in 31s
CI / build-and-test (pull_request) Successful in 1m15s
CI / valgrind (pull_request) Successful in 33s
2026-08-09 11:53:54 +02:00
TapTap 1a0a7a19a5 style: format atomic queue cancellation
CI / lint (pull_request) Failing after 27s
CI / build-and-test (pull_request) Has been skipped
CI / sanitizers (address) (pull_request) Has been skipped
CI / sanitizers (undefined) (pull_request) Has been skipped
CI / fuzz-build (pull_request) Has been skipped
CI / coverage (pull_request) Has been skipped
CI / valgrind (pull_request) Has been skipped
2026-08-09 11:52:51 +02:00
TapTap d64666d456 fix: close remaining PR 208 security gaps
CI / lint (pull_request) Failing after 3s
CI / build-and-test (pull_request) Has been skipped
CI / sanitizers (address) (pull_request) Has been skipped
CI / sanitizers (undefined) (pull_request) Has been skipped
CI / fuzz-build (pull_request) Has been skipped
CI / coverage (pull_request) Has been skipped
CI / valgrind (pull_request) Has been skipped
2026-08-09 11:52:15 +02:00
31 changed files with 1561 additions and 1312 deletions

No files matched your search

+72 -29
View File
@@ -8,10 +8,8 @@ set(CMAKE_C_STANDARD_REQUIRED ON)
add_compile_options(-Wall -g -O3)
# --- Sanitizer option ---
set(SANITIZER "none" CACHE STRING "Sanitizer to enable (address, thread, undefined, none)")
set_property(CACHE SANITIZER PROPERTY STRINGS address thread undefined none)
if(SANITIZER STREQUAL "address")
add_compile_options(-fsanitize=address -fno-omit-frame-pointer -g)
add_link_options(-fsanitize=address)
@@ -25,13 +23,11 @@ elseif(NOT SANITIZER STREQUAL "none")
message(FATAL_ERROR "Unknown sanitizer: ${SANITIZER}. Supported values: address, thread, undefined, none")
endif()
# --- Strict warnings option ---
option(STRICT_WARNINGS "Enable strict warnings (Wextra, Wpedantic, Werror)" OFF)
if(STRICT_WARNINGS)
add_compile_options(-Wextra -Wpedantic -Werror)
endif()
# --- Coverage option ---
option(ENABLE_COVERAGE "Enable gcov coverage" OFF)
if(ENABLE_COVERAGE)
add_compile_options(--coverage -fprofile-arcs -ftest-coverage -O0 -g)
@@ -49,55 +45,102 @@ FetchContent_MakeAvailable(xxhash)
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_package(Threads REQUIRED)
find_library(ZSTD_LIBRARY zstd)
if(NOT ZSTD_LIBRARY)
message(FATAL_ERROR "zstd library not found. Ensure it is in your nix-shell!")
endif()
find_package(OpenSSL REQUIRED)
file(GLOB SHARED_SRCS "src/shared/*.c")
file(GLOB SERVER_SRCS "src/server/*.c")
file(GLOB CLIENT_SRCS "src/client/*.c")
set(SHARED_SRCS
src/shared/array_list.c
src/shared/chunk.c
src/shared/compression.c
src/shared/config.c
src/shared/data.c
src/shared/delta.c
src/shared/file.c
src/shared/log.c
src/shared/metadata.c
src/shared/multiprocessing.c
src/shared/protocol.c
src/shared/queue.c
src/shared/transport_ssh.c
src/shared/transport_tcp.c
src/shared/transport_tls.c
src/shared/utils.c
)
set(SERVER_SRCS src/server/server.c)
set(CLIENT_SRCS
src/client/client_cli.c
src/client/client_send.c
src/client/scanner.c
src/client/usage.c
)
function(configure_fastsync_target target)
target_include_directories(${target} PRIVATE src/shared src/server src/client)
target_link_libraries(${target} PRIVATE Threads::Threads ${ZSTD_LIBRARY} OpenSSL::SSL OpenSSL::Crypto xxhash)
endfunction()
# --- Main executables ---
add_executable(server ${SERVER_SRCS} ${SHARED_SRCS})
target_include_directories(server PRIVATE src/shared src/server src/client)
target_link_libraries(server PRIVATE Threads::Threads ${ZSTD_LIBRARY} OpenSSL::SSL OpenSSL::Crypto xxhash)
configure_fastsync_target(server)
add_executable(client ${CLIENT_SRCS} ${SHARED_SRCS})
target_include_directories(client PRIVATE src/shared src/server src/client)
target_link_libraries(client PRIVATE Threads::Threads ${ZSTD_LIBRARY} OpenSSL::SSL OpenSSL::Crypto xxhash)
configure_fastsync_target(client)
# --- Testing ---
enable_testing()
# Common test libraries
set(TEST_LIBS Threads::Threads ${ZSTD_LIBRARY} OpenSSL::SSL OpenSSL::Crypto xxhash)
set(TEST_INCLUDES tests src/shared src/server src/client)
# Monolithic test binary (backward compatible)
file(GLOB TEST_SRCS "tests/test_*.c" "tests/runner.c")
set(TEST_SRCS
tests/runner.c
tests/test_array_list.c
tests/test_chunk.c
tests/test_client_cli.c
tests/test_compression.c
tests/test_config.c
tests/test_data.c
tests/test_delta.c
tests/test_file.c
tests/test_file_sendfile.c
tests/test_fuzz_smoke.c
tests/test_glob.c
tests/test_log.c
tests/test_metadata.c
tests/test_multiprocessing.c
tests/test_property.c
tests/test_protocol.c
tests/test_queue.c
tests/test_robustness.c
tests/test_scanner.c
tests/test_server.c
tests/test_shared_utils.c
tests/test_stress.c
tests/test_transport_ssh.c
tests/test_transport_tcp.c
tests/test_transport_tls.c
)
add_executable(tests ${TEST_SRCS} ${SHARED_SRCS} src/client/scanner.c src/client/client_cli.c)
target_include_directories(tests PRIVATE ${TEST_INCLUDES})
configure_fastsync_target(tests)
target_include_directories(tests PRIVATE tests)
target_compile_definitions(tests PRIVATE FASTSYNC_TEST_BUILD)
target_link_libraries(tests PRIVATE ${TEST_LIBS})
add_test(NAME unit_all COMMAND tests)
# --- Fuzz targets (requires clang) ---
option(ENABLE_FUZZ "Build fuzz targets (requires clang)" OFF)
if(ENABLE_FUZZ)
if(NOT CMAKE_C_COMPILER_ID MATCHES "Clang")
message(FATAL_ERROR "ENABLE_FUZZ requires Clang (compiler is ${CMAKE_C_COMPILER_ID})")
endif()
file(GLOB FUZZ_SRCS "tests/fuzz/*.c")
set(FUZZ_SRCS
tests/fuzz/fuzz_chunk_deserialize.c
tests/fuzz/fuzz_compress_decompress.c
tests/fuzz/fuzz_delta_deserialize.c
tests/fuzz/fuzz_delta_signature_deserialize.c
tests/fuzz/fuzz_glob_match.c
tests/fuzz/fuzz_metadata_from_buf.c
)
foreach(FUZZ_SRC ${FUZZ_SRCS})
get_filename_component(FUZZ_NAME ${FUZZ_SRC} NAME_WE)
add_executable(${FUZZ_NAME} ${FUZZ_SRC} ${SHARED_SRCS})
target_include_directories(${FUZZ_NAME} PRIVATE ${TEST_INCLUDES})
configure_fastsync_target(${FUZZ_NAME})
target_include_directories(${FUZZ_NAME} PRIVATE tests)
target_compile_options(${FUZZ_NAME} PRIVATE -fsanitize=fuzzer,address,undefined -fno-omit-frame-pointer)
target_link_options(${FUZZ_NAME} PRIVATE -fsanitize=fuzzer,address,undefined)
target_link_libraries(${FUZZ_NAME} PRIVATE ${TEST_LIBS})
endforeach()
endif()
+316 -305
View File
@@ -1,371 +1,382 @@
# FastSync
A high-performance file synchronization system with SSH and TCP transport, TLS encryption, streaming zstd compression, multithreaded transfer, incremental sync, metadata preservation, and rsync-compatible CLI flags.
FastSync is a high-performance file synchronization tool designed to become a
drop-in replacement for common `rsync` workflows. It keeps the familiar
source/destination model and rsync-style options while adding optional
multithreading, streaming zstd compression, chunking, zero-copy TCP transfers,
and native TCP/TLS transports.
## Technical Overview
The compatibility target is straightforward:
1. **Dual transport**: custom TCP client-server or SSH subprocess (rsync-style `user@host:/path`)
2. **TLS encryption**: OpenSSL-based TLS 1.2+ for encrypted TCP connections with optional CA verification
3. **Chunked file transfer**: files grouped into configurable-size chunks (default ~10 MB)
4. **Streaming zstd compression** (levels 1–22) using `ZSTD_compressStream2`
5. **Multithreading**: producer-consumer pipeline with thread-safe queues (scanner → loader → sender)
6. **Incremental sync**: skip files unchanged since last transfer (compares size + mtime)
7. **Batch incremental**: send incremental checks in batched groups for reduced round-trips
8. **Metadata preservation**: `mode`, `uid`, `gid`, `mtime` restored on disk when enabled
9. **`sendfile()` zero-copy** on TCP (~2× faster on loopback)
10. **SSH ControlMaster** for connection reuse across repeated invocations
11. **Bandwidth limiting**: token-bucket throttling (`--bwlimit`)
12. **`--delete`**: receiver removes files not present in sender manifest
13. **`--exclude` / `--include`**: glob-pattern filename filtering
14. **Path traversal protection**: `..` sequences in file paths are rejected automatically
15. **Connection limits**: server enforces maximum concurrent connections (default 100)
16. **Keep-alive**: periodic `STATUS_KEEPALIVE` messages detect stalled connections
17. **Abort handling**: `SIGINT` sends `STATUS_ABORT` for clean server-side teardown
18. **Atomic writes**: received files are written to a temporary name then atomically renamed
19. **Backup mode**: `--backup` preserves overwritten files with optional `--backup-dir`
20. **Log file**: `--log-file` redirects log output to a file instead of stderr
21. **Transfer statistics**: `--stats` prints summary of transferred bytes, files, and timing
- Existing rsync commands should keep the same meaning.
- FastSync-only performance options should be additive and optional.
- A normal compatibility-mode transfer should prioritize rsync filesystem
semantics over maximum throughput.
## System Architecture
FastSync currently speaks its own protocol to `fastsync-server`. SSH mode
starts that server remotely; it does not yet interoperate with an unmodified
rsync client or rsync daemon. See [Compatibility Status](#compatibility-status)
for the current boundary.
### Client
- Recursively scans source directories (BFS), supports exclude and include patterns
- Groups files into chunks (configurable size)
- Streaming zstd compression with configurable level
- Chunk serialization (compact binary format) or per-file transfer
- Incremental transfer: sends file metadata to server, skips unchanged files
- Batch incremental: groups incremental checks to minimize round-trips
- Manifests all sent paths when `--delete` is active
- Sends via TCP `sendfile()` or SSH pipe
- Optional progress display with throughput
- Bandwidth limiting via token-bucket algorithm
- Configurable I/O and connection timeouts (`--timeout`, `--contimeout`)
- Quiet mode (`-q`/`--quiet`) suppresses all non-error output
- Backup overwritten files (`--backup`) with optional directory (`--backup-dir`)
- Transfer statistics summary (`--stats`)
- Maximum directory depth control (`--max-depth`)
- Log file output (`--log-file`)
- Configurable multithreaded queue size (`--queue-size`)
- Exclude patterns from file (`--exclude-from`)
## Why FastSync
### Server
- TCP mode: listens on configurable port (default 8080); SSH mode: runs via `--stdio`
- TLS mode: wraps TCP connections with OpenSSL with optional CA verification
- Receives and reassembles files
- Decompresses (streaming zstd), deserializes, restores metadata
- Handles incremental checks: compares size + mtime against destination files
- Handles batch incremental checks for reduced round-trips
- Processes `STATUS_MANIFEST` for `--delete`: walks destination tree, removes extras
- Per-connection concurrency via `fork()` with configurable connection limit (default 100)
- Thread pool for parallel processing
- Atomic writes: files written to `.tmp` path then atomically renamed on success
- Abort handling: cleanly shuts down on `STATUS_ABORT` from client
- Path traversal protection: rejects file paths containing `..`
FastSync uses a producer-consumer transfer pipeline and can combine several
optimizations for large or high-latency transfers:
## Protocol Details
- Multithreaded scanning, loading, and sending.
- Streaming zstd compression with levels 1 through 22.
- Configurable file chunking and compact chunk serialization.
- `sendfile()` zero-copy transfers over TCP.
- Batched incremental checks to reduce round trips.
- Optional block-level delta transfer for FastSync peers.
- Bandwidth limiting, progress reporting, statistics, and backups.
- TCP, SSH, and TLS transports.
- Atomic temporary-file writes by default.
### Status Codes
| Code | Meaning |
|------|---------|
| `STATUS_OK` | Operation successful |
| `STATUS_ERROR` | Error occurred |
| `STATUS_FINISHED` | Transfer complete |
| `STATUS_NEXT` | Ready for next file (per-file mode) |
| `STATUS_CHUNK` | Following data is a serialized chunk |
| `STATUS_MANIFEST` | Following data is a file manifest (for `--delete`) |
| `STATUS_CHECK` | Incremental check: client sends file path + size + mtime and, when negotiated, checksum; server responds with OK (skip) or NEXT (send) |
| `STATUS_CHECK_BATCH` | Batch incremental check: multiple file checks sent in one message |
| `STATUS_KEEPALIVE` | Keep-alive heartbeat to detect stalled connections |
| `STATUS_ABORT` | Abort signal: client interrupts, server cleans up and exits |
| `STATUS_DELTA_SIGNATURE` | Delta sync: following data is a file signature (rsync-style rolling hash) |
| `STATUS_DELTA_DATA` | Delta sync: following data is a delta patch for a file |
These optimizations are disabled or selected independently. Users can start
with rsync-style commands and add FastSync options when they are useful.
### Wire Format — Metadata
## Compatibility Status
When `use_metadata` is enabled (`-M`), each file entry carries a 4-byte `present` flag followed by five fields (`mode`, `uid`, `gid`, `mtime_sec`, `mtime_nsec`). When disabled globally, no metadata bytes are sent — zero wire overhead.
FastSync is currently an rsync-compatible CLI in progress, not a complete
replacement for every rsync feature or protocol mode.
### Transfer Flow
```
Config → (STATUS_NEXT | STATUS_CHUNK | STATUS_CHECK | STATUS_CHECK_BATCH)* → [STATUS_MANIFEST] → STATUS_FINISHED → STATUS_OK
```
### Working today
Keep-alive (`STATUS_KEEPALIVE`) may be sent at any point during the transfer. The receiver resets its inactivity timer on receipt. If no data arrives within the receive timeout, the connection is aborted.
- Recursive directory scanning.
- Rsync-style source and destination arguments.
- SSH transport using `user@host:destination` paths below the remote authorized root.
- TCP client/server transfers.
- Dry runs, excludes, includes, size filters, backups, statistics, and
bandwidth limiting.
- Incremental size/mtime checks and optional xxHash64 content checks.
- FastSync-native delta transfer for changed files.
- Optional mode and timestamp preservation.
- Delete manifests with server-side delete authorization.
- Temporary-file writes with atomic rename by default.
- Path traversal checks and destination-root confinement.
Abort (`STATUS_ABORT`) may be sent at any point. On receipt the server cleans up temporary files and exits the child process.
### Not yet equivalent to rsync
### Protocol Version
- The FastSync wire protocol is not the rsync wire protocol.
- SSH mode requires `fastsync-server` on the remote host.
- Archive mode does not yet provide all of rsync's `-rlptgoD` behavior.
- Symlink transfer is incomplete; link targets are not yet recreated in all
modes.
- Owner/group, ACL, xattr, hard-link, device, and special-file handling is
incomplete or unavailable.
- Sparse-file handling does not yet preserve all holes correctly.
- `--partial`, `--partial-dir`, `--append`, and `--append-verify` are not yet
full rsync-style resumable transfers.
- Several rsync short options currently have FastSync-specific meanings. Do
not assume every short option is interchangeable yet.
`2.2.0` — server and client must match. This version adds a 64-bit XXH64 checksum to checksum-enabled `STATUS_CHECK` messages and validates the negotiated compression choice (`zstd` or `none`). Older clients and servers must not be mixed with this version; mismatch results in `STATUS_ERROR`.
The detailed flag matrix is maintained in
[`RSYNC_COMPAT.md`](RSYNC_COMPAT.md). It distinguishes implemented,
partial, alternate, and planned behavior.
Config negotiation is sender-driven: the client serializes transfer options and the server applies them while receiving and writing files. `--checksum` compares size and content checksum instead of timestamps. `--compress-choice zstd` enables zstd; `none` disables it. Unsupported choices are rejected during config exchange.
## Quick Start
## Command-Line Arguments
### Build
### Client
| Argument | Description |
|----------|-------------|
| Positional | `<source> <dest>` — automatic SSH detection if dest contains `:` |
| `-c [level]` | Compression with optional level (1–22, default 5) |
| `-z [level]` | Alias for `-c` |
| `-a, --archive` | Archive mode: enables `-c -m -M` (no `-s`) |
| `-m` | Multithreading mode |
| `-s` | Chunk serialization (batch all files per chunk) |
| `-f, --sendfile` | Sendfile zero-copy. Incompatible with `-c` / `-s`. TCP only. |
| `-M, --preserve` | Preserve file metadata (mode, uid, gid, mtime) |
| `-n, --dry-run` | Scan and print what would be transferred |
| `-p <port>` | SSH port (default: 22) |
| `-v, --verbose` | Enable debug logging |
| `-q, --quiet` | Suppress all non-error output |
| `--silent` | Alias for `--quiet` |
| `--progress` | Show real-time transfer speed |
| `--delete` | Delete files on receiver not present in source |
| `--exclude <pattern>` | Exclude files matching glob pattern (repeatable) |
| `--exclude-from <file>` | Read exclude patterns from a file (one per line) |
| `--include <pattern>` | Only transfer files matching glob pattern (repeatable, whitelist) |
| `--max-size <n>` | Skip files larger than n bytes |
| `--min-size <n>` | Skip files smaller than n bytes |
| `--incremental` | Skip files unchanged since last transfer (size + mtime). Auto-enables `--preserve`. Incompatible with `-s`. |
| `--bwlimit <KB/s>` | Bandwidth limit in kilobytes per second |
| `--chunk-size <n>` | Chunk size in bytes (default: 10485760) |
| `--timeout <sec>` | I/O timeout in seconds (default: 30) |
| `--contimeout <sec>` | Connection timeout in seconds (default: 10) |
| `--backup` | Backup existing destination files before overwriting |
| `--backup-dir <dir>` | Target directory for backups (requires `--backup`) |
| `--stats` | Print transfer statistics at end (bytes, files, timing) |
| `--max-depth <n>` | Maximum directory depth to recurse (0 = unlimited, default: 0) |
| `--log-file <path>` | Write log messages to file instead of stderr |
| `--queue-size <n>` | Queue capacity for multithreaded mode (default: 100) |
| `--source-dir <path>` | Source directory (overrides `FASTSYNC_SOURCE_DIR`) |
| `--dest-dir <path>` | Server destination directory (overrides `FASTSYNC_DEST_DIR`) |
| `--save-to-disk` | Write received files to disk |
| `--server-host <ip>` | Server IP address (default: `127.0.0.1`) |
| `--server-port <n>` | Server port (default: `8080`) |
| `--tls` | Enable TLS encryption |
| `--cert <path>` | TLS certificate file (PEM) |
| `--key <path>` | TLS private key file (PEM) |
| `--ca <path>` | TLS CA certificate file for verification (PEM) |
### Server
| Argument | Description |
|----------|-------------|
| `--stdio` | Run in stdio mode (for SSH transport; single connection then exits) |
| `-p <port>` | TCP listen port (default: 8080, range: 1–65535) |
| `--tls` | Enable TLS encryption |
| `--cert <path>` | TLS certificate file (PEM) |
| `--key <path>` | TLS private key file (PEM) |
| `--ca <path>` | TLS CA certificate file for verification (PEM) |
| `-v, --verbose` | Enable debug logging |
| `--help` | Show help |
## Environment Variables
| Variable | Default | Description |
|----------|---------|-------------|
| `FASTSYNC_SOURCE_DIR` | — | Source directory fallback |
| `FASTSYNC_DEST_DIR` | — | Destination directory fallback |
| `FASTSYNC_SAVE_TO_DISK` | `false` | Disk persistence fallback |
| `FASTSYNC_SSH_PORT` | `22` | Default SSH port |
| `FASTSYNC_SERVER_HOST` | `127.0.0.1` | Default server host |
| `FASTSYNC_SERVER_PORT` | `8080` | Default server port |
| `FASTSYNC_TLS_CERT` | — | Default TLS certificate path |
| `FASTSYNC_TLS_KEY` | — | Default TLS private key path |
| `FASTSYNC_TLS_CA` | — | Default TLS CA certificate path |
## Implementation Details
### Data Structures
1. **Chunk** — collection of files (~10 MB total by default)
2. **File** — path, content (`Data`), optional `FileMetadata` pointer
3. **FileMetadata** — `mode`, `uid`, `gid`, `mtime_sec`, `mtime_nsec`
4. **Config** — runtime parameters (transported over wire, TLS settings excluded). Includes `timeout`, `contimeout`, `quiet`, `backup`, `backup_dir`, `stats`, `max_depth`, `log_file`, `queue_size`.
5. **Queue** — thread-safe bounded queue with condition variables
6. **DirectoryScanner** — recursive BFS traversal with exclude and include pattern support, max-depth enforcement
### Key Algorithms
1. **File scanning** — BFS directory traversal; entries matched against exclude and include patterns, max-depth enforced
2. **Chunking** — files accumulated until `chunk_size` threshold, then flushed
3. **Compression** — streaming zstd via `ZSTD_compressStream2` / `ZSTD_decompressStream`
4. **Network protocol** — status-code-driven exchange with metadata packing, keep-alive, and abort support
5. **Incremental check** — client sends `STATUS_CHECK` + path + size + mtime and, with `--checksum`, XXH64 content checksum; server compares against destination. Can be batched via `STATUS_CHECK_BATCH` for reduced round-trips.
6. **Bandwidth limiting** — token-bucket algorithm with `nanosleep` throttling on 64 KB write chunks
7. **Metadata restoration** — `chmod()`, `chown()`, `utimensat()` on the receiving side
8. **`--delete`** — sender tracks all sent paths; receiver walks destination tree and removes unlisted files/directories
9. **SSH transport** — `socketpair()` + `fork()` + `execvp("ssh", ...)` with `ControlMaster` and port support
10. **TLS transport** — OpenSSL `SSL_CTX` with TLS 1.2 minimum, optional CA verification, transparent `SSL_read`/`SSL_write` via `io_set_ssl()`
11. **Path traversal protection** — `has_path_traversal()` rejects any file path containing `..` components, preventing directory escape attacks
12. **Connection limiting** — server tracks active connections and rejects new ones beyond `max_connections` (default 100)
13. **Keep-alive** — idle connections receive periodic `STATUS_KEEPALIVE` to detect half-open TCP connections
14. **Abort handling** — `SIGINT` sets an abort flag; the next protocol operation sends `STATUS_ABORT` for clean server cleanup
15. **Atomic writes** — files are written to a `.tmp` suffix then atomically renamed via `rename()`, preventing partial files
16. **Backup** — before overwriting, existing files are moved to `--backup-dir` (or same directory with `~` suffix) preserving the original
## Security Features
### Path Traversal Protection
All received file paths are validated by `has_path_traversal()` before any disk operation. Any path containing `..` components is rejected with `STATUS_ERROR`, preventing directory escape attacks.
### TLS Certificate Verification
When `--ca` is provided, the server performs mutual TLS verification (`SSL_VERIFY_PEER` with depth 4). Without `--ca`, TLS is still encrypted but peer certificates are not verified.
### Connection Limits
The server enforces a maximum of 100 concurrent connections (configurable via `max_connections` in `Server`). When the limit is reached, new connections are immediately rejected and closed.
### Abort Handling
If the client receives `SIGINT` (Ctrl+C) during a transfer, it sends `STATUS_ABORT` to the server. The server then cleans up temporary files and exits the child process, preventing incomplete files from remaining on disk.
### Atomic Writes
Received files are written to a temporary path (suffixed with `.tmp`) and then atomically renamed to the final filename via `rename()`. This prevents partial or corrupted files from appearing at the destination if the transfer is interrupted.
## Build Requirements
- C11 compiler
- CMake >= 3.22
- zstd library
- OpenSSL (development headers and libraries)
- pthreads
- SSH client (for SSH transport mode only)
### Installing Dependencies
**Ubuntu/Debian:**
```bash
sudo apt install cmake build-essential libzstd-dev libssl-dev openssh-client
```
**Nix:**
```bash
nix-shell # provides zstd, openssl, cmake, gcc
```
## Building
Requirements: C11 compiler, CMake 3.22 or newer, xxHash, zstd, OpenSSL,
pthreads, and an SSH client for SSH transport. The first CMake configure fetches
xxHash from GitHub, so network access is required unless the dependency is
already cached.
```bash
cmake -B build -S . && cmake --build build -j$(nproc)
cmake -B build -S .
cmake --build build -j$(nproc)
```
## Running
With Nix:
### Server (TCP mode)
```bash
./build/server
nix-shell
cmake -B build -S .
cmake --build build -j$(nproc)
```
### Server with TLS
### SSH transfer
The remote host must have `fastsync-server` available in `PATH`, or use
`--fastsync-server-path`. SSH starts `fastsync-server --stdio` in its remote
working directory, so use a destination below that directory unless the
remote server is otherwise configured with a matching authorized root.
```bash
./build/server --tls --cert server.pem --key server-key.pem
./build/client /path/to/source user@host:destination
```
### Server via SSH
Place the `fastsync-server` binary in the remote `$PATH`. The client runs `ssh user@host fastsync-server --stdio` automatically when an SSH-style destination is given.
### TCP transfer
Start the FastSync server:
### Client — SSH (rsync-style)
```bash
./build/client /path/to/send user@host:/path/to/receive
./build/server --destination-root /path/to -p 8080
```
### Client — TCP
Then run the client:
```bash
./build/client --source-dir /path/to/send --dest-dir /path/to/receive --save-to-disk
./build/client --server-host 127.0.0.1 --server-port 8080 \
--source-dir /path/to/source --dest-dir /path/to/destination \
--save-to-disk
```
### Client — TCP with TLS
### TLS transfer
```bash
./build/server --destination-root /path/to --tls --cert server.pem --key server-key.pem -p 8443
./build/client --tls --cert client.pem --key client-key.pem --ca ca.pem \
--source-dir /path/to/send --dest-dir /path/to/receive --save-to-disk
--server-host example.com --server-port 8443 \
--source-dir /path/to/source --dest-dir /path/to/destination \
--save-to-disk
```
### Common Options
## Common Workflows
These examples show the intended rsync-style workflow. Options marked as
FastSync-native are optional performance or transport extensions.
```bash
# Archive mode (compression + multithreading + metadata)
./build/client -a /path/to/send user@host:/path
# Basic synchronization
./build/client /source/ /destination/
# Dry run
./build/client -n /path/to/send /path/to/receive
# Archive-style synchronization (current FastSync archive behavior)
./build/client -a /source/ user@host:destination/
# With progress and custom chunk size
./build/client --progress --chunk-size 2097152 /src user@host:/dst
# Preview a transfer without changing the destination
./build/client -n /source/ /destination/
# Exclude temporary files + delete extras on receiver
./build/client --exclude "*.tmp" --exclude "*.o" --delete /src user@host:/dst
# Exclude temporary and object files
./build/client --exclude '*.tmp' --exclude '*.o' \
/source/ user@host:destination/
# Incremental sync (skip unchanged files)
./build/client --incremental /src user@host:/dst
# Remove destination entries not present in the source
./build/client --delete /source/ user@host:destination/
# Bandwidth limit to 1 MB/s
./build/client --bwlimit 1024 /src user@host:/dst
# Skip unchanged files using size and modification time
./build/client --incremental /source/ user@host:destination/
# With timeouts, quiet mode, and stats
./build/client --timeout 60 --contimeout 15 --quiet --stats /src user@host:/dst
# Verify content when size and time are not sufficient
./build/client --incremental --checksum /source/ user@host:destination/
# Backup overwritten files to a directory
./build/client --backup --backup-dir /backups /src user@host:/dst
# Preserve supported mode and timestamp metadata
./build/client -M /source/ user@host:destination/
# Exclude patterns from file, limit depth
./build/client --exclude-from ignore.txt --max-depth 3 /src user@host:/dst
# Custom queue size for multithreading
./build/client -m --queue-size 200 /src user@host:/dst
# Log to file
./build/client --log-file /tmp/fastsync.log /src user@host:/dst
# All features
./build/client -a --progress --chunk-size 5242880 --exclude "*.log" --delete /src /dst
# Keep backups of overwritten destination files
./build/client --backup --backup-dir backups \
/source/ user@host:destination/
```
## FastSync Extensions
FastSync-native options are intended to add performance or operational
features without changing the meaning of ordinary compatibility options.
| Option | Purpose |
|---|---|
| `-m` | Enable the multithreaded scanner/loader/sender pipeline. |
| `-c [level]`, `-z [level]` | Enable streaming zstd compression, levels 1-22. |
| `--compress-level <n>` | Set the zstd compression level. |
| `--chunk-size <bytes>` | Set the transfer chunk size. |
| `-s` | Enable FastSync chunk serialization. |
| `-f`, `--sendfile` | Use TCP `sendfile()` zero-copy transfer. Incompatible with compression and chunk serialization. |
| `--delta` | Use FastSync-native block delta transfer. Requires `--incremental`. |
| `--delta-block <bytes>` | Set the FastSync delta block size. |
| `--delta-max <bytes>` | Limit files eligible for FastSync delta transfer. |
| `--server-host <host>` | Select the TCP server host. |
| `--server-port <port>` | Select the TCP server port. |
| `--tls` | Enable TLS for TCP transport. |
| `--bwlimit <KB/s>` | Apply token-bucket bandwidth limiting. |
| `--progress` | Show transfer progress and throughput. |
| `--stats` | Print transfer statistics. |
| `--timeout <seconds>` | Set I/O timeout. |
| `--contimeout <seconds>` | Set connection timeout. |
Current short-option conflicts are tracked as compatibility work. In
particular, FastSync currently uses `-p` for SSH port, `-s` for chunk
serialization, and `-S` for sparse handling. These meanings must be reconciled
before FastSync can claim full rsync CLI compatibility.
## Client Options
### Selection and transfer
| Option | Description |
|---|---|
| `-a`, `--archive` | Enable current archive preset. Full rsync archive semantics are planned. |
| `-n`, `--dry-run` | Scan and report without writing files. |
| `--delete` | Request removal of destination entries absent from the source. The server must allow deletion. |
| `--exclude <pattern>` | Exclude matching paths. Repeatable. |
| `--include <pattern>` | Include matching paths. Repeatable. |
| `--exclude-from <file>` | Read exclude patterns from a file. |
| `--include-from <file>` | Read include patterns from a file. |
| `--max-size <bytes>` | Skip files larger than the limit. |
| `--min-size <bytes>` | Skip files smaller than the limit. |
| `--max-depth <n>` | Limit recursive scanning depth; zero means unlimited. |
| `--incremental` | Skip files matching destination size and mtime. |
| `--checksum` | Include xxHash64 content checks in incremental comparisons. |
| `--backup` | Back up overwritten files. |
| `--backup-dir <dir>` | Store backups under a separate directory. |
| `--suffix <suffix>` | Set the backup filename suffix. |
| `--partial` | Select partial-transfer handling. With `--partial-dir`, completed files are written there; resumable transfers are not implemented. |
| `--partial-dir <dir>` | Set a relative partial-transfer directory below the server destination root; use with `--partial`. |
| `--inplace` | Write directly to the destination instead of using a temporary file. |
### Metadata and links
| Option | Description |
|---|---|
| `-M`, `--preserve` | Preserve supported file metadata, currently mode and modification time. |
| `-l`, `--links` | Request symlink preservation; link-target transfer remains incomplete. |
| `--copy-links` | Copy symlink referents. |
| `--safe-links` | Skip symlinks that point outside the transfer tree. |
| `--copy-unsafe-links` | Copy unsafe symlink referents. |
| `-S`, `--sparse` | Request sparse-file handling; full hole preservation is planned. |
### Output and logging
| Option | Description |
|---|---|
| `-v`, `--verbose` | Enable debug logging. |
| `--progress` | Show live transfer progress. |
| `--stats` | Print transfer statistics. |
| `--log-file <path>` | Write log output to a file. |
| `-V`, `--version` | Print the FastSync protocol version. |
| `--help` | Print command usage. |
### Paths and transport
| Option | Description |
|---|---|
| `-p <port>` | SSH port in the current CLI. This conflicts with rsync's `-p` permissions option and is planned for correction. |
| `--fastsync-server-path <path>` | Remote FastSync server path for SSH mode. |
| `--source-dir <path>` | Set the source directory explicitly. |
| `--dest-dir <path>` | Set the destination directory explicitly. |
| `--save-to-disk` | Enable server-side disk persistence. |
| `--server-host <host>` | TCP server address. |
| `--server-port <port>` | TCP server port. |
| `--tls` | Enable TLS. Requires `--cert` and `--key`. |
| `--cert <path>` | TLS certificate file. |
| `--key <path>` | TLS private key file. |
| `--ca <path>` | CA file for peer verification. |
## Server Options
| Option | Description |
|---|---|
| `--stdio` | Serve one SSH connection over standard input/output. |
| `-p <port>` | TCP listen port. |
| `--tls` | Enable TLS. |
| `--cert <path>` | TLS certificate file. |
| `--key <path>` | TLS private key file. |
| `--ca <path>` | CA file for peer verification. |
| `--destination-root <path>` | Confine received files to this server-side root; defaults to the current directory. |
| `--allow-delete` | Permit client delete manifests. Deletion is refused by default. |
| `-v`, `--verbose` | Enable debug logging. |
| `--help` | Print server usage. |
## Architecture
### Client
- Recursively scans the source tree with include, exclude, size, and depth
filters.
- Sends individual files or serialized chunks.
- Performs incremental checks and optional content checksums.
- Uses a multithreaded producer-consumer pipeline when requested.
- Sends over TCP, TLS-wrapped TCP, or an SSH subprocess.
- Supports progress, statistics, backups, timeouts, and bandwidth limiting.
### Server
- Runs as a TCP listener or one-shot SSH `--stdio` server.
- Receives and reassembles files and decompresses streaming zstd data.
- Applies supported metadata and writes files through a confined destination
root.
- Uses temporary files and atomic rename by default.
- Handles delete manifests only when explicitly authorized.
- Enforces connection, message-size, and path-safety limits.
## Protocol and Security
FastSync protocol version `2.2.0` is shared by the client and server. The
current protocol is sender-driven and includes configuration negotiation,
incremental checks, checksums, manifests, keep-alives, abort handling, and
FastSync-native delta messages. Client and server versions must currently
match exactly.
TLS provides encrypted TCP transport. Supplying `--ca` enables certificate
verification; without it, traffic is encrypted but peer identity is not
verified. Use certificate verification for deployments where authentication
matters. The default TCP transport is not encrypted.
The receiver protects its destination root with path validation, `openat()`
directory traversal, `O_NOFOLLOW`, temporary files, and atomic renames. Delete
operations require the server's explicit `--allow-delete` policy.
## Compatibility Roadmap
The project will reach the drop-in replacement goal in stages:
1. Correct rsync option meanings, including short options, combined options,
and `--option=value` syntax.
2. Add differential tests that compare FastSync and rsync contents, metadata,
links, deletes, filters, dry runs, and exit codes.
3. Make `-a` implement the expected recursive, links, permissions, times,
owner/group, and supported special-file behavior.
4. Complete symlink, sparse-file, metadata, delete-policy, and resumable-write
semantics.
5. Add rsync remote-shell and daemon protocol interoperability.
6. Keep FastSync performance options as negotiated, optional extensions.
The exhaustive implementation matrix and compatibility notes are in
[`RSYNC_COMPAT.md`](RSYNC_COMPAT.md).
## Testing
```bash
# Unit tests (18 suites — array_list, chunk, compression, config, data, delta, file, glob,
# metadata, property, protocol, queue, robustness, scanner,
# shared_utils, stress, transport_tcp, transport_ssh, transport_tls)
./build/tests
Run the unit test binary:
# Integration + benchmark suite
python3 test.py
```bash
./build/tests
```
The benchmark prints throughput metrics, best configuration, and speedup vs rsync.
Run the Python integration suite:
## Performance Considerations
```bash
python3 -m pytest tests/
```
1. Chunk size (~10 MB default) balances memory and transfer efficiency
2. Compression level trades CPU for bandwidth
3. `sendfile()` bypasses userspace — ~2× faster on localhost for large files
4. Multithreading scales with core count; `--queue-size` controls pipeline buffering
5. Metadata transfer adds negligible overhead (~24 bytes per file when enabled)
6. SSH socketpair buffer set to 1 MB for improved pipe throughput
7. SSH ControlMaster reuses connections across repeated invocations
8. Incremental sync eliminates redundant transfers entirely
9. Batch incremental reduces round-trips by grouping multiple checks into one message
10. Bandwidth limiting uses token-bucket with nanosleep for accurate throttling
11. Atomic writes add a single `rename()` per file — negligible overhead
12. Path traversal check is O(n) in path length with negligible cost
For stricter local validation:
## Benchmark Results
```bash
cmake -B build-strict -S . -DSTRICT_WARNINGS=ON
cmake --build build-strict -j$(nproc)
cmake -B build-asan -S . -DSANITIZER=address
cmake --build build-asan -j$(nproc)
```
25 MB of mixed file sizes over `localhost` with disk I/O throttled (reads ≤ 15 MB/s, writes ≤ 10 MB/s) and network emulation via `tc netem`. Each test was run 3×; the median is reported below.
The benchmark tool compares FastSync configurations with rsync under
controlled local and network conditions:
### LAN (1000 Mbit, 20 ms ±1 ms, 0.1% loss)
```bash
python3 benchmark/bench.py --help
```
| Configuration | Time | vs rsync (archive) | vs rsync (compress) |
|---|---|---|---|
| **Best: `-m -c`** | **0.20 s** | **11.2× faster** | **3.6× faster** |
| Compression (`-c`) | 0.31 s | 7.3× faster | 2.3× faster |
| Standard | 1.27 s | 1.8× faster | — |
| rsync (archive) | 2.27 s | — | — |
| rsync (archive + compress) | 0.72 s | — | — |
Benchmark results measure transfer performance only. They do not establish
rsync protocol or filesystem-semantic compatibility.
### WAN (100 Mbit, 50 ms ±10 ms, 1% loss)
## Performance Guidance
| Configuration | Time | vs rsync (archive) | vs rsync (compress) |
|---|---|---|---|
| **Best: `-m -c`** | **0.39 s** | **44.8× faster** | **3.8× faster** |
| Compression (`-c`) | 0.64 s | 27.3× faster | 2.3× faster |
| Standard | 7.12 s | 2.4× faster | — |
| rsync (archive) | 17.44 s | — | — |
| rsync (archive + compress) | 1.47 s | — | — |
- Use `-m` for workloads with many files or enough CPU parallelism.
- Use `-c` or `-z` when network bandwidth is more constrained than CPU.
- Tune `--chunk-size` for file sizes, memory limits, and network latency.
- Use `-f` for large uncompressed TCP transfers where zero-copy I/O helps.
- Use `--incremental` to avoid retransmitting unchanged files.
- Use `--delta` for changed files when both endpoints are FastSync peers.
- Use `--bwlimit` when sharing a link with other traffic.
Compression reduces the data on the wire enough that the transfer becomes latency-bound rather than bandwidth-bound. On WAN, the best configuration runs 10.8× faster than the theoretical limit for uncompressed data, since zstd shrinks the 25 MB payload to a fraction of its original size over the wire.
Always validate the compatibility behavior required by a deployment before
replacing an existing rsync job.
+43 -44
View File
@@ -6,10 +6,10 @@ This document maps rsync's full feature set to FastSync's current implementation
| Status | Count | Description |
|--------|-------|-------------|
| ✅ Implemented | 39 | Feature works end-to-end |
| 🔀 Alt Arg | 5 | Functionality exists but under different flag/semantics |
| ⚠️ Partial | 30 | Flag parsed/stored but behavior incomplete |
| ❌ Not Implemented | 62 | Flag not recognized or no behavior |
| ✅ Implemented | 34 | Feature works end-to-end |
| 🔀 Alt Arg | 3 | Functionality exists but under different flag/semantics |
| ⚠️ Partial | 1 | Flag parsed/stored but behavior incomplete |
| ❌ Not Implemented | 98 | Flag not recognized or no behavior |
| **Total** | **136** | |
---
@@ -20,31 +20,31 @@ This document maps rsync's full feature set to FastSync's current implementation
|------|-------------------|-----------------|-------|
| `-a`, `--archive` | Archive mode is -rlptgoD | 🔀 Alt Arg | Maps to -c -m -M (compression + multithread + metadata) |
| `-v`, `--verbose` | Increase verbosity | ✅ Implemented | Sets `log_level=DEBUG` |
| `-q`, `--quiet` | Suppress non-error messages | ✅ Implemented | `quiet` config field |
| `-h`, `--help` | Show help | ✅ Implemented | Prints usage and exits |
| `-V`, `--version` | Print version | ❌ Not Implemented | |
| `--info=FLAGS` | Fine-grained info verbosity | ⚠️ Partial | `info_level` stored, not wired |
| `--debug=FLAGS` | Fine-grained debug verbosity | ⚠️ Partial | `debug_level` stored, not wired |
| `-q`, `--quiet` | Suppress non-error messages | ❌ Not Implemented | Removed because it had no effect |
| `--help` | Show help | ✅ Implemented | Prints usage and exits; `-h` is not accepted |
| `-V`, `--version` | Print version | ✅ Implemented | |
| `--info=FLAGS` | Fine-grained info verbosity | ❌ Not Implemented | Removed because it had no effect |
| `--debug=FLAGS` | Fine-grained debug verbosity | ❌ Not Implemented | Removed because it had no effect |
| `--stderr=MODE` | Change stderr output mode | ❌ Not Implemented | |
| `--no-motd` | Suppress daemon MOTD | ❌ Not Implemented | |
| `--exclude=PATTERN` | Exclude files matching pattern | ✅ Implemented | Glob matching in scanner |
| `--include=PATTERN` | Include files matching pattern | ✅ Implemented | Glob matching in scanner |
| `-C`, `--cvs-exclude` | Auto-ignore CVS files | ⚠️ Partial | `cvs_exclude` stored, not wired |
| `-C`, `--cvs-exclude` | Auto-ignore CVS files | ❌ Not Implemented | Removed because it had no effect |
## 2. Modifying Output
| Flag | Rsync Description | FastSync Status | Notes |
|------|-------------------|-----------------|-------|
| `--stats` | Give transfer stats | ✅ Implemented | Prints file/byte counts |
| `-h`, `--human-readable` | Human-readable numbers | ⚠️ Partial | `human_readable` stored, not wired |
| `-i`, `--itemize-changes` | Per-file change summary | ⚠️ Partial | `itemize_changes` stored, not wired |
| `-h`, `--human-readable` | Human-readable numbers | ❌ Not Implemented | Removed because it had no effect |
| `-i`, `--itemize-changes` | Per-file change summary | ❌ Not Implemented | Removed because it had no effect |
| `--progress` | Show progress | ✅ Implemented | Progress callback in sender |
| `-P` | Same as --partial --progress | ❌ Not Implemented | |
| `--out-format=FORMAT` | Custom output format | ⚠️ Partial | `out_format` stored, not wired |
| `--out-format=FORMAT` | Custom output format | ❌ Not Implemented | Removed because it had no effect |
| `--log-file=FILE` | Log to file | ✅ Implemented | `log_file` config field |
| `--log-file-format=FMT` | Log format | ❌ Not Implemented | |
| `--8-bit-output` | Leave high-bit chars unescaped | ❌ Not Implemented | |
| `--list-only` | List files instead of copying | ⚠️ Partial | `list_only` stored, not wired |
| `--list-only` | List files instead of copying | ❌ Not Implemented | Removed because it had no effect |
## 3. File Selection
@@ -52,8 +52,8 @@ This document maps rsync's full feature set to FastSync's current implementation
|------|-------------------|-----------------|-------|
| `--exclude-from=FILE` | Read exclude patterns from file | ✅ Implemented | Reads patterns from file |
| `--include-from=FILE` | Read include patterns from file | ✅ Implemented | Reads patterns from file |
| `--filter=RULE` | Add file-filtering rule | ⚠️ Partial | `filters` ArrayList stored, not wired |
| `--files-from=FILE` | Read source file list from file | ⚠️ Partial | `files_from` stored, not wired |
| `--filter=RULE` | Add file-filtering rule | ❌ Not Implemented | Removed because it had no effect |
| `--files-from=FILE` | Read source file list from file | ❌ Not Implemented | Removed because it had no effect |
| `-0`, `--from0` | Delimit *-from files with NULs | ❌ Not Implemented | |
| `--max-size=SIZE` | Skip files larger than SIZE | ✅ Implemented | `max_size` in scanner |
| `--min-size=SIZE` | Skip files smaller than SIZE | ✅ Implemented | `min_size` in scanner |
@@ -69,7 +69,7 @@ This document maps rsync's full feature set to FastSync's current implementation
| Flag | Rsync Description | FastSync Status | Notes |
|------|-------------------|-----------------|-------|
| `-r`, `--recursive` | Recurse into directories | ✅ Implemented | Default behavior |
| `-R`, `--relative` | Use relative path names | ⚠️ Partial | `relative` stored, not wired |
| `-R`, `--relative` | Use relative path names | ❌ Not Implemented | Removed because it had no effect |
| `--no-implied-dirs` | Don't send implied dirs with -R | ❌ Not Implemented | |
| `-d`, `--dirs` | Transfer dirs without recursing | ❌ Not Implemented | |
| `--mkpath` | Create missing path components | ❌ Not Implemented | |
@@ -78,10 +78,10 @@ This document maps rsync's full feature set to FastSync's current implementation
| Flag | Rsync Description | FastSync Status | Notes |
|------|-------------------|-----------------|-------|
| `-u`, `--update` | Skip files newer on receiver | ⚠️ Partial | `update` stored, not wired |
| `-u`, `--update` | Skip files newer on receiver | ❌ Not Implemented | Removed because it had no effect |
| `--inplace` | Update files in-place | ✅ Implemented | Direct write mode |
| `--append` | Append data to shorter files | ⚠️ Partial | `append` stored, not wired |
| `--append-verify` | Append with old-data checksum | ⚠️ Partial | `append_verify` stored, not wired |
| `--append` | Append data to shorter files | ❌ Not Implemented | Removed because it had no effect |
| `--append-verify` | Append with old-data checksum | ❌ Not Implemented | Removed because it had no effect |
| `-W`, `--whole-file` | Copy whole file (no delta) | ❌ Not Implemented | |
| `--block-size=SIZE` | Force checksum block-size | ⚠️ Partial | Parsed as `--delta-block`; controls delta transfer block size |
@@ -100,15 +100,15 @@ This document maps rsync's full feature set to FastSync's current implementation
| Flag | Rsync Description | FastSync Status | Notes |
|------|-------------------|-----------------|-------|
| `--delete` | Delete extraneous files from dest | ✅ Implemented | `use_delete` config field |
| `--delete-before` | Delete before transfer | ⚠️ Partial | `delete_before` stored, not wired |
| `--delete-before` | Delete before transfer | ❌ Not Implemented | Removed because it had no effect |
| `--delete-during` | Delete during transfer | ❌ Not Implemented | |
| `--delete-delay` | Find deletions during, delete after | ❌ Not Implemented | |
| `--delete-after` | Delete after transfer | ⚠️ Partial | `delete_after` stored, not wired |
| `--delete-excluded` | Also delete excluded files | ⚠️ Partial | `delete_excluded` stored, not wired |
| `--max-delete=NUM` | Max files to delete | ⚠️ Partial | `max_delete` stored, not wired |
| `--delete-after` | Delete after transfer | ❌ Not Implemented | Removed because it had no effect |
| `--delete-excluded` | Also delete excluded files | ❌ Not Implemented | Removed because it had no effect |
| `--max-delete=NUM` | Max files to delete | ❌ Not Implemented | Removed because it had no effect |
| `--ignore-errors` | Delete even with I/O errors | ❌ Not Implemented | |
| `--force` | Force deletion of non-empty dirs | ❌ Not Implemented | |
| `--prune-empty-dirs` | Prune empty dir chains | ⚠️ Partial | `prune_empty_dirs` stored, not wired |
| `--prune-empty-dirs` | Prune empty dir chains | ❌ Not Implemented | Removed because it had no effect |
## 8. Metadata Preservation
@@ -121,11 +121,11 @@ This document maps rsync's full feature set to FastSync's current implementation
| `-t`, `--times` | Preserve modification times | ✅ Implemented | Part of -M |
| `-E`, `--executability` | Preserve executability | ❌ Not Implemented | |
| `--chmod=CHMOD` | Affect file permissions | ❌ Not Implemented | |
| `-A`, `--acls` | Preserve ACLs | ⚠️ Partial | `preserve_acls` stored, not wired |
| `-X`, `--xattrs` | Preserve extended attributes | ⚠️ Partial | `preserve_xattrs` stored, not wired |
| `-H`, `--hard-links` | Preserve hard links | ⚠️ Partial | `preserve_hard_links` stored, not wired |
| `-D` | Same as --devices --specials | 🔀 Alt Arg | Maps to --devices only (no --specials) |
| `--devices` | Preserve device files | ⚠️ Partial | `preserve_devices` stored, not wired |
| `-A`, `--acls` | Preserve ACLs | ❌ Not Implemented | Removed because it had no effect |
| `-X`, `--xattrs` | Preserve extended attributes | ❌ Not Implemented | Removed because it had no effect |
| `-H`, `--hard-links` | Preserve hard links | ❌ Not Implemented | Removed because it had no effect |
| `-D` | Same as --devices --specials | ❌ Not Implemented | Removed because device-file handling is not implemented |
| `--devices` | Preserve device files | ❌ Not Implemented | Removed because it had no effect |
| `--specials` | Preserve special files | ❌ Not Implemented | |
| `--copy-devices` | Copy device contents as file | ❌ Not Implemented | |
| `--write-devices` | Write to devices as files | ❌ Not Implemented | |
@@ -152,18 +152,18 @@ This document maps rsync's full feature set to FastSync's current implementation
| Flag | Rsync Description | FastSync Status | Notes |
|------|-------------------|-----------------|-------|
| `-S`, `--sparse` | Sparse block handling | ✅ Implemented | `preserve_sparse` config field |
| `-S`, `--sparse` | Sparse block handling | ⚠️ Partial | Flag is accepted, but full hole preservation is not implemented |
| `--preallocate` | Allocate dest files before writing | ❌ Not Implemented | |
## 11. Checksum & Comparison
| Flag | Rsync Description | FastSync Status | Notes |
|------|-------------------|-----------------|-------|
| `-c`, `--checksum` | Skip based on checksum | 🔀 Alt Arg | `-c` means compression; `--checksum` stored and forwarded to scanner |
| `--checksum` | Skip based on checksum | ✅ Implemented | With `--incremental`, compares xxHash64 content checksums; `-c` remains compression |
| `--checksum-choice=STR` | Choose checksum algorithm | ❌ Not Implemented | xxHash used internally |
| `--compare-dest=DIR` | Compare dest files relative to DIR | ⚠️ Partial | `compare_dest` stored, not wired |
| `--copy-dest=DIR` | Include copies of unchanged files | ⚠️ Partial | `copy_dest` stored, not wired |
| `--link-dest=DIR` | Hardlink to files when unchanged | ⚠️ Partial | `link_dest` stored, not wired |
| `--compare-dest=DIR` | Compare dest files relative to DIR | ❌ Not Implemented | Removed because it had no effect |
| `--copy-dest=DIR` | Include copies of unchanged files | ❌ Not Implemented | Removed because it had no effect |
| `--link-dest=DIR` | Hardlink to files when unchanged | ❌ Not Implemented | Removed because it had no effect |
| `--fuzzy`, `--no-fuzzy` | Find similar file for basis | ❌ Not Implemented | |
## 12. Compression
@@ -171,7 +171,7 @@ This document maps rsync's full feature set to FastSync's current implementation
| Flag | Rsync Description | FastSync Status | Notes |
|------|-------------------|-----------------|-------|
| `-z`, `--compress` | Compress file data | 🔀 Alt Arg | Always uses zstd (rsync supports multiple algorithms) |
| `--compress-choice=STR` | Choose compression algorithm | ⚠️ Partial | `compress_choice` stored, always zstd |
| `--compress-choice=STR` | Choose compression algorithm | ❌ Not Implemented | Removed because it had no effect; FastSync always uses zstd |
| `--compress-level=NUM` | Set compression level | ✅ Implemented | 1-22, default 5 |
| `--compress-threads=NUM` | Set compression threads | ❌ Not Implemented | |
| `--skip-compress=LIST` | Skip compress for suffixes | ❌ Not Implemented | Internal skip for hardcoded types; not user-configurable |
@@ -180,22 +180,22 @@ This document maps rsync's full feature set to FastSync's current implementation
| Flag | Rsync Description | FastSync Status | Notes |
|------|-------------------|-----------------|-------|
| `-e`, `--rsh=COMMAND` | Remote shell to use | ✅ Implemented | SSH transport support |
| `--rsync-path=PROGRAM` | rsync binary on remote | ✅ Implemented | `rsync_path` config field |
| `-e`, `--rsh=COMMAND` | Remote shell to use | ❌ Not Implemented | Removed; SSH invokes `ssh` directly |
| `--rsync-path=PROGRAM` | rsync binary on remote | ❌ Not Implemented | Removed; use `--fastsync-server-path` |
| `--port=PORT` | Alternate daemon port | ✅ Implemented | `server_port` config field |
| `--sockopts=OPTIONS` | Custom TCP options | ❌ Not Implemented | |
| `--blocking-io` | Use blocking I/O for remote shell | ❌ Not Implemented | |
| `--outbuf=N\|L\|B` | Set output buffering | ❌ Not Implemented | |
| `--address=ADDRESS` | Bind address for outgoing socket | ✅ Implemented | `address` config field |
| `-4`, `--ipv4` | Prefer IPv4 | ✅ Implemented | `ipv4` config field |
| `-6`, `--ipv6` | Prefer IPv6 | ✅ Implemented | `ipv6` config field |
| `--address=ADDRESS` | Bind address for outgoing socket | ❌ Not Implemented | Removed because it had no effect |
| `-4`, `--ipv4` | Prefer IPv4 | ❌ Not Implemented | Removed because it had no effect |
| `-6`, `--ipv6` | Prefer IPv6 | ❌ Not Implemented | Removed because it had no effect |
## 14. Daemon Mode
| Flag | Rsync Description | FastSync Status | Notes |
|------|-------------------|-----------------|-------|
| `--daemon` | Run as rsync daemon | ⚠️ Partial | `daemon` stored, not wired |
| `--config=FILE` | Alternate rsyncd.conf file | ⚠️ Partial | `daemon_config` stored, not wired |
| `--daemon` | Run as rsync daemon | ❌ Not Implemented | Removed because it had no effect |
| `--config=FILE` | Alternate rsyncd.conf file | ❌ Not Implemented | Removed because it had no effect |
| `--dparam=OVERRIDE` | Override global daemon config | ❌ Not Implemented | |
| `--no-detach` | Don't detach from parent | ❌ Not Implemented | |
| `--password-file=FILE` | Read daemon password from file | ❌ Not Implemented | |
@@ -266,7 +266,6 @@ Ranked by user demand, implementation complexity, and interoperability impact:
| `-f` / `--sendfile` | Zero-copy sendfile() syscall (TCP only) |
| `-c [level]` | zstd compression level (1-22) |
| `--chunk-size` | Configurable chunk size |
| `--queue-size` | Pipeline queue capacity |
| `--tls` | TLS encryption (mutual auth) |
| `--fastsync-server-path` | Path to fastsync-server binary |
| `--server-host` / `--server-port` | Direct TCP connection |
+20 -242
View File
@@ -8,9 +8,11 @@
#include "utils.h"
#include <errno.h>
#include <limits.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "usage.c"
#ifndef FASTSYNC_TEST_BUILD
/* Parse environment variables for source/destination directories and save-to-disk flag. */
@@ -25,19 +27,6 @@ static void parse_environment(const char** out_env_source, const char** out_env_
}
#endif
/* Parse a string as a positive integer, returning true on success. */
static bool parse_positive_int(const char* s, int* out_val) {
if (!s || *s == '\0')
return false;
char* endptr;
errno = 0;
long val = strtol(s, &endptr, 10);
if (errno != 0 || *endptr != '\0' || val <= 0 || val > INT_MAX)
return false;
*out_val = (int)val;
return true;
}
/* Parse a string as a non-negative integer, returning true on success. */
static bool parse_nonneg_int(const char* s, int* out_val) {
if (!s || *s == '\0')
@@ -51,7 +40,20 @@ static bool parse_nonneg_int(const char* s, int* out_val) {
return true;
}
/* Duplicate a string argument into *dest, freeing the old value. Returns 0 on success, -1 on
/* Parse a string as a positive integer, returning true on success. */
static bool parse_positive_int(const char* s, int* out_val) {
int temp;
if (!parse_nonneg_int(s, &temp)) {
return false;
}
if (temp == 0) {
return false;
}
*out_val = temp;
return true;
}
/* Duplicate a string argument into *dest, freeing the old value. Returns true on success, false on
* failure. */
static int set_string_option(char** dest, const char* value, const char* option_name) {
char* dup = str_dup(value);
@@ -64,7 +66,7 @@ static int set_string_option(char** dest, const char* value, const char* option_
return 0;
}
/* Parse a string as a positive integer into *dest. Returns 0 on success, -1 on error. */
/* Parse a string as a positive integer into *dest. Returns true on success, false on error. */
static int set_positive_int_option(int* dest, const char* value, const char* option_name) {
if (!parse_positive_int(value, dest)) {
fprintf(stderr, "Error: %s must be a positive integer\n", option_name);
@@ -73,7 +75,7 @@ static int set_positive_int_option(int* dest, const char* value, const char* opt
return 0;
}
/* Parse a string as a non-negative integer into *dest. Returns 0 on success, -1 on error. */
/* Parse a string as a non-negative integer into *dest. Returns true on success, false on error. */
static int set_nonneg_int_option(int* dest, const char* value, const char* option_name) {
if (!parse_nonneg_int(value, dest)) {
fprintf(stderr, "Error: %s must be a non-negative integer\n", option_name);
@@ -82,7 +84,6 @@ static int set_nonneg_int_option(int* dest, const char* value, const char* optio
return 0;
}
static void print_usage(void);
static int read_patterns_from_file(const char* filepath, char*** patterns, int* count);
/* Parse CLI arguments into config. Returns 0 on success, -1 on error, 1 for help/clean-exit. */
@@ -103,12 +104,10 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args,
} else if (strcmp(argv[i], "-n") == 0 || strcmp(argv[i], "--dry-run") == 0) {
config->dry_run = true;
} else if (strcmp(argv[i], "-p") == 0 && i + 1 < argc) {
if (!parse_positive_int(argv[++i], &config->ssh_port)) {
fprintf(stderr, "Error: invalid --port/-p value: %s\n", argv[i]);
if (set_positive_int_option(&config->ssh_port, argv[++i], "-p") != 0)
return -1;
}
if (config->ssh_port > 65535) {
fprintf(stderr, "Error: SSH port must be 1-65535\n");
log_message(LOG_LEVEL_ERROR, "SSH port must be 1-65535\n");
return -1;
}
} else if (strcmp(argv[i], "--delete") == 0) {
@@ -275,9 +274,6 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args,
} else if (strcmp(argv[i], "--contimeout") == 0 && i + 1 < argc) {
if (set_positive_int_option(&config->contimeout, argv[++i], "--contimeout") != 0)
return -1;
} else if (strcmp(argv[i], "-q") == 0 || strcmp(argv[i], "--quiet") == 0 ||
strcmp(argv[i], "--silent") == 0) {
config->quiet = true;
} else if (strcmp(argv[i], "--backup") == 0) {
config->backup = true;
} else if (strcmp(argv[i], "--backup-dir") == 0 && i + 1 < argc) {
@@ -301,9 +297,6 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args,
}
config->log_file = lf;
log_set_file(lf);
} else if (strcmp(argv[i], "--queue-size") == 0 && i + 1 < argc) {
if (set_positive_int_option(&config->queue_size, argv[++i], "--queue-size") != 0)
return -1;
} else if (strcmp(argv[i], "--exclude-from") == 0 && i + 1 < argc) {
if (read_patterns_from_file(argv[++i], &config->exclude_patterns, &config->exclude_count) !=
0)
@@ -328,126 +321,21 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args,
config->safe_links = true;
} else if (strcmp(argv[i], "--copy-unsafe-links") == 0) {
config->copy_unsafe_links = true;
} else if (strcmp(argv[i], "-H") == 0 || strcmp(argv[i], "--hard-links") == 0) {
config->preserve_hard_links = true;
} else if (strcmp(argv[i], "-A") == 0 || strcmp(argv[i], "--acls") == 0) {
config->preserve_acls = true;
} else if (strcmp(argv[i], "-X") == 0 || strcmp(argv[i], "--xattrs") == 0) {
config->preserve_xattrs = true;
} else if (strcmp(argv[i], "-D") == 0 || strcmp(argv[i], "--devices") == 0) {
config->preserve_devices = true;
} else if (strcmp(argv[i], "-S") == 0 || strcmp(argv[i], "--sparse") == 0) {
config->preserve_sparse = true;
} else if (strcmp(argv[i], "-i") == 0 || strcmp(argv[i], "--itemize-changes") == 0) {
config->itemize_changes = true;
} else if (strcmp(argv[i], "--out-format") == 0 && i + 1 < argc) {
if (set_string_option(&config->out_format, argv[++i], "--out-format") != 0)
return -1;
} else if (strcmp(argv[i], "--info") == 0 && i + 1 < argc) {
if (set_nonneg_int_option(&config->info_level, argv[++i], "--info") != 0)
return -1;
} else if (strcmp(argv[i], "--debug") == 0 && i + 1 < argc) {
if (set_nonneg_int_option(&config->debug_level, argv[++i], "--debug") != 0)
return -1;
} else if (strcmp(argv[i], "--list-only") == 0) {
config->list_only = true;
} else if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--human-readable") == 0) {
config->human_readable = true;
} else if (strcmp(argv[i], "-u") == 0 || strcmp(argv[i], "--update") == 0) {
config->update = true;
} else if (strcmp(argv[i], "--inplace") == 0) {
config->inplace = true;
} else if (strcmp(argv[i], "--append") == 0) {
config->append = true;
} else if (strcmp(argv[i], "--append-verify") == 0) {
config->append_verify = true;
} else if (strcmp(argv[i], "--delete-excluded") == 0) {
config->delete_excluded = true;
} else if (strcmp(argv[i], "--delete-after") == 0) {
config->delete_after = true;
} else if (strcmp(argv[i], "--max-delete") == 0 && i + 1 < argc) {
if (set_nonneg_int_option(&config->max_delete, argv[++i], "--max-delete") != 0)
return -1;
} else if (strcmp(argv[i], "--filter") == 0 && i + 1 < argc) {
if (!config->filters)
config->filters = array_list_create(free);
char* dup = str_dup(argv[++i]);
if (!dup)
return -1;
array_list_add(config->filters, dup);
} else if (strcmp(argv[i], "--files-from") == 0 && i + 1 < argc) {
if (set_string_option(&config->files_from, argv[++i], "--files-from") != 0)
return -1;
} else if (strcmp(argv[i], "--cvs-exclude") == 0) {
config->cvs_exclude = true;
} else if (strcmp(argv[i], "--prune-empty-dirs") == 0) {
config->prune_empty_dirs = true;
} else if (strcmp(argv[i], "-R") == 0 || strcmp(argv[i], "--relative") == 0) {
config->relative = true;
} else if (strcmp(argv[i], "-e") == 0 || strcmp(argv[i], "--rsh") == 0) {
if (i + 1 < argc) {
if (set_string_option(&config->rsh_command, argv[++i], "-e/--rsh") != 0)
return -1;
} else {
fprintf(stderr, "Error: -e/--rsh requires a command argument\n");
return -1;
}
} else if (strcmp(argv[i], "--rsync-path") == 0 && i + 1 < argc) {
if (set_string_option(&config->rsync_path, argv[++i], "--rsync-path") != 0)
return -1;
} else if (strcmp(argv[i], "--temp-dir") == 0 && i + 1 < argc) {
if (set_string_option(&config->temp_dir, argv[++i], "--temp-dir") != 0)
return -1;
} else if (strcmp(argv[i], "--compare-dest") == 0 && i + 1 < argc) {
if (set_string_option(&config->compare_dest, argv[++i], "--compare-dest") != 0)
return -1;
} else if (strcmp(argv[i], "--copy-dest") == 0 && i + 1 < argc) {
if (set_string_option(&config->copy_dest, argv[++i], "--copy-dest") != 0)
return -1;
} else if (strcmp(argv[i], "--link-dest") == 0 && i + 1 < argc) {
if (set_string_option(&config->link_dest, argv[++i], "--link-dest") != 0)
return -1;
} else if (strcmp(argv[i], "--partial-dir") == 0 && i + 1 < argc) {
if (set_string_option(&config->partial_dir, argv[++i], "--partial-dir") != 0)
return -1;
} else if (strcmp(argv[i], "--suffix") == 0 && i + 1 < argc) {
if (set_string_option(&config->suffix, argv[++i], "--suffix") != 0)
return -1;
} else if (strcmp(argv[i], "--delete-before") == 0) {
config->delete_before = true;
} else if (strcmp(argv[i], "-T") == 0 && i + 1 < argc) {
if (set_positive_int_option(&config->timeout, argv[++i], "-T") != 0)
return -1;
} else if (strcmp(argv[i], "--address") == 0 && i + 1 < argc) {
if (set_string_option(&config->address, argv[++i], "--address") != 0)
return -1;
} else if (strcmp(argv[i], "--bind-address") == 0 && i + 1 < argc) {
if (set_string_option(&config->bind_address, argv[++i], "--bind-address") != 0)
return -1;
} else if (strcmp(argv[i], "--ipv6") == 0) {
config->ipv6 = true;
} else if (strcmp(argv[i], "--ipv4") == 0) {
config->ipv4 = true;
} else if (strcmp(argv[i], "--daemon") == 0) {
config->daemon = true;
} else if (strcmp(argv[i], "--config") == 0 && i + 1 < argc) {
if (set_string_option(&config->daemon_config, argv[++i], "--config") != 0)
return -1;
} else if (strcmp(argv[i], "--server") == 0) {
config->server_mode = true;
} else if (strcmp(argv[i], "--checksum") == 0) {
config->checksum = true;
} else if (strcmp(argv[i], "--compress-choice") == 0 && i + 1 < argc) {
if (set_string_option(&config->compress_choice, argv[++i], "--compress-choice") != 0)
return -1;
if (strcmp(config->compress_choice, "zstd") == 0)
config->use_compression = true;
else if (strcmp(config->compress_choice, "none") == 0)
config->use_compression = false;
else {
fprintf(stderr, "Error: --compress-choice must be 'zstd' or 'none'\n");
return -1;
}
} else if (strcmp(argv[i], "--compress-level") == 0 && i + 1 < argc) {
if (set_positive_int_option(&config->compression_level, argv[++i], "--compress-level") != 0)
return -1;
@@ -521,116 +409,6 @@ static bool validate_config(const Config* config) {
}
#endif /* FASTSYNC_TEST_BUILD */
static void print_usage(void) {
printf("Usage:\n");
printf(" fastsync [options] <source> <destination>\n");
printf(" fastsync [options] --source-dir <src> --dest-dir <dst>\n");
printf("\n");
printf("Destination formats:\n");
printf(" user@host:/path SSH transport (rsync-style)\n");
printf(" host:/path SSH transport (current user)\n");
printf(" /local/path TCP transport (requires server on localhost:8080)\n");
printf("\n");
printf("Options:\n");
printf(" -c [level] Enable compression (level 1-22, default 5)\n");
printf(" -z [level] Alias for -c\n");
printf(" -a, --archive Archive mode (-c -m -M)\n");
printf(" -n, --dry-run Show what would be transferred\n");
printf(" -p <port> SSH port (default: 22)\n");
printf(" --progress Show transfer progress\n");
printf(" --delete Delete files on receiver not in source\n");
printf(" --exclude <pattern> Exclude files matching pattern\n");
printf(" --include <pattern> Only include files matching pattern\n");
printf(" --exclude-from <file> Read exclude patterns from file\n");
printf(" --include-from <file> Read include patterns from file\n");
printf(" --max-size <n> Skip files larger than n bytes\n");
printf(" --min-size <n> Skip files smaller than n bytes\n");
printf(" --incremental Skip files unchanged since last transfer\n");
printf(" --delta Delta transfer for changed files (requires --incremental)\n");
printf(" --delta-block <n> Delta block size in bytes (default: %d)\n",
DELTA_BLOCK_SIZE_DEFAULT);
printf(" --delta-max <n> Max file size for delta transfer (default: %llu)\n",
DELTA_MAX_FILE_SIZE);
printf(" -m Enable multithreading\n");
printf(" -s Enable chunk serialization\n");
printf(" -f Enable sendfile (TCP only, not with -c or -s)\n");
printf(" -v, --verbose Enable debug logging\n");
printf(" -M, --preserve Preserve file metadata\n");
printf(" --chunk-size <n> Chunk size in bytes (default: %d)\n", DEFAULT_CHUNK_SIZE);
printf(" --source-dir <path> Source directory\n");
printf(" --dest-dir <path> Destination directory\n");
printf(" --save-to-disk Write received files to disk\n");
printf(" --server-host <ip> Server IP address (default: 127.0.0.1)\n");
printf(" --server-port <n> Server port (default: 8080)\n");
printf(" --bwlimit <KB/s> Bandwidth limit in kilobytes per second\n");
printf(" --tls Enable TLS encryption\n");
printf(" --cert <path> TLS certificate file (PEM)\n");
printf(" --key <path> TLS private key file (PEM)\n");
printf(" --ca <path> TLS CA certificate file (PEM)\n");
printf(" --timeout <sec> I/O timeout in seconds (default: 30)\n");
printf(" -T <sec> Alias for --timeout\n");
printf(" --contimeout <sec> Connection timeout in seconds (default: 10)\n");
printf(" --address <host> Server hostname/IP to connect to\n");
printf(" --bind-address <ip> Bind to specific local address\n");
printf(" --ipv6 Prefer IPv6 connections\n");
printf(" --ipv4 Prefer IPv4 connections\n");
printf(" -q, --quiet Suppress non-error output\n");
printf(" --silent Alias for --quiet\n");
printf(" --backup Backup existing files before overwriting\n");
printf(" --backup-dir <dir> Directory for backups (requires --backup)\n");
printf(" --suffix <str> Backup suffix (default: ~)\n");
printf(" --stats Print transfer statistics at end\n");
printf(" --max-depth <n> Maximum directory depth (0=unlimited)\n");
printf(" --log-file <path> Write log messages to file\n");
printf(" --queue-size <n> Queue capacity for multithreaded mode (default: 100)\n");
printf(" --partial Keep partial files on interrupted transfer\n");
printf(" --partial-dir <dir> Directory for partial files\n");
printf(" --fastsync-server-path <path>\n");
printf(" Path to fastsync-server on remote (default: fastsync-server)\n");
printf(" -l, --links Copy symlinks as symlinks\n");
printf(" --copy-links Transform symlinks into referent files\n");
printf(" --safe-links Skip symlinks that point outside transfer tree\n");
printf(" --copy-unsafe-links Only transform unsafe symlinks into referent files\n");
printf(" -H, --hard-links Preserve hard links\n");
printf(" -A, --acls Preserve ACLs\n");
printf(" -X, --xattrs Preserve extended attributes\n");
printf(" -D, --devices Preserve device files\n");
printf(" -S, --sparse Handle sparse files efficiently\n");
printf(" -i, --itemize-changes Show per-file change summary\n");
printf(" --out-format <fmt> Custom output format string\n");
printf(" --info <flags> Info verbosity level\n");
printf(" --debug <flags> Debug verbosity level\n");
printf(" --list-only List files without transferring\n");
printf(" -h, --human-readable Human-readable numbers\n");
printf(" -u, --update Skip files newer on destination\n");
printf(" --inplace Update files in-place (no temp+rename)\n");
printf(" --append Append data to shorter files\n");
printf(" --append-verify Append with verify\n");
printf(" --delete-before Delete before transfer\n");
printf(" --delete-excluded Also delete excluded files\n");
printf(" --delete-after Delete after transfer, not before\n");
printf(" --max-delete <n> Maximum number of files to delete\n");
printf(" --filter <rule> Add file filtering rule\n");
printf(" --files-from <file> Read file list from file\n");
printf(" --cvs-exclude Auto-ignore CVS files\n");
printf(" --prune-empty-dirs Omit empty directories from transfer\n");
printf(" -R, --relative Use relative paths\n");
printf(" -e, --rsh <cmd> Specify remote shell\n");
printf(" --rsync-path <path> Path to remote binary\n");
printf(" --daemon Run in daemon mode\n");
printf(" --config <path> Path to configuration file\n");
printf(" --server Run in server mode\n");
printf(" --checksum Skip files based on checksum, not mod-time/size\n");
printf(" --compress-choice <alg> Compression algorithm (default: zstd)\n");
printf(" --compress-level <n> Compression level (default: 5)\n");
printf(" --temp-dir <dir> Temporary directory for files\n");
printf(" --compare-dest <dir> Compare destination\n");
printf(" --copy-dest <dir> Copy destination\n");
printf(" --link-dest <dir> Link destination\n");
printf(" --help Show this help\n");
printf(" -V, --version Show version\n");
}
static int read_patterns_from_file(const char* filepath, char*** patterns, int* count) {
FILE* fp = fopen(filepath, "r");
if (!fp) {
+71 -16
View File
@@ -28,6 +28,20 @@
/* Forward declaration for progress-reporting thread used in multithreaded send. */
static int progress_thread_fn(void* arg);
static void pipeline_cancel(PipelineContextSender* context) {
mtx_lock(&context->mutex_scanner);
mtx_lock(&context->mutex_loader);
atomic_store(&context->cancelled, true);
context->scanner_done = true;
context->loader_done = true;
cnd_broadcast(&context->condition_not_full_scanner);
cnd_broadcast(&context->condition_not_empty_scanner);
cnd_broadcast(&context->condition_not_full_loader);
cnd_broadcast(&context->condition_not_empty_loader);
mtx_unlock(&context->mutex_loader);
mtx_unlock(&context->mutex_scanner);
}
/* Print dry-run manifest showing files that would be transferred. Returns 0 on success. */
static int send_dry_run_manifest(Config* config) {
DirectoryScanner* scanner = directory_scanner_create(
@@ -344,6 +358,7 @@ static int send_chunks_multithreaded(void* pipeline_context) {
return ok ? thrd_success : thrd_error;
send_fail:
pipeline_cancel(context);
client_disconnect(client);
client_delete(client);
mtx_lock(&context->mutex_progress);
@@ -354,6 +369,7 @@ static int send_chunks_multithreaded(void* pipeline_context) {
if (send_chunk(client, current_chunk, context->config) != 0) {
fprintf(stderr, "Error: unexpected error while sending chunk\n");
chunk_destroy(current_chunk);
pipeline_cancel(context);
client_disconnect(client);
client_delete(client);
mtx_lock(&context->mutex_progress);
@@ -386,6 +402,11 @@ static int scan_directory_multithreaded(void* pipeline_context) {
context->config->checksum);
Chunk* current_chunk;
if (scanner == NULL) {
log_message(LOG_LEVEL_ERROR, "Failed to create parallel scanner");
pipeline_cancel(context);
return thrd_error;
}
while ((current_chunk = parallel_scanner_next(scanner)) != NULL) {
if (context->config->use_delete) {
mtx_lock(&context->mutex_scanner);
@@ -397,13 +418,18 @@ static int scan_directory_multithreaded(void* pipeline_context) {
if (!manifest_entry) {
log_message(LOG_LEVEL_ERROR, "Failed to allocate manifest entry");
mtx_unlock(&context->mutex_scanner);
context->cancelled = true;
cnd_broadcast(&context->condition_not_full_scanner);
cnd_broadcast(&context->condition_not_empty_scanner);
pipeline_cancel(context);
parallel_scanner_destroy(scanner);
return thrd_error;
}
if (!array_list_add(context->manifest, manifest_entry)) {
free(manifest_entry);
mtx_unlock(&context->mutex_scanner);
pipeline_cancel(context);
chunk_destroy(current_chunk);
parallel_scanner_destroy(scanner);
return thrd_error;
}
array_list_add(context->manifest, manifest_entry);
}
mtx_unlock(&context->mutex_scanner);
}
@@ -412,13 +438,21 @@ static int scan_directory_multithreaded(void* pipeline_context) {
&context->condition_not_empty_scanner, &context->condition_not_full_scanner,
&context->cancelled)) {
chunk_destroy(current_chunk);
context->cancelled = true;
cnd_broadcast(&context->condition_not_full_scanner);
cnd_broadcast(&context->condition_not_empty_scanner);
pipeline_cancel(context);
parallel_scanner_destroy(scanner);
return thrd_error;
}
}
if (parallel_scanner_failed(scanner)) {
parallel_scanner_destroy(scanner);
mtx_lock(&context->mutex_scanner);
context->scanner_done = true;
cnd_broadcast(&context->condition_not_empty_scanner);
cnd_broadcast(&context->condition_not_full_scanner);
mtx_unlock(&context->mutex_scanner);
pipeline_cancel(context);
return thrd_error;
}
mtx_lock(&context->mutex_scanner);
context->scanner_done = true;
cnd_signal(&context->condition_not_empty_scanner);
@@ -458,7 +492,7 @@ static int load_files_multithreaded(void* pipeline_context) {
&context->condition_not_full_loader,
&context->cancelled)) {
chunk_destroy(chunk);
context->cancelled = true;
atomic_store(&context->cancelled, true);
cnd_broadcast(&context->condition_not_full_loader);
cnd_broadcast(&context->condition_not_empty_loader);
return thrd_error;
@@ -554,6 +588,15 @@ int send_files(Config* config) {
time_t last_progress = 0;
time_t start = time(NULL);
ArrayList* manifest = config->use_delete ? array_list_create(free) : NULL;
if (!scanner || (config->use_delete && !manifest)) {
if (scanner)
directory_scanner_destroy(scanner);
if (manifest)
array_list_delete(manifest);
client_disconnect(client);
client_delete(client);
return 1;
}
while ((current_chunk = directory_scanner_next(scanner)) != NULL) {
unsigned long long chunk_bytes = 0;
for (int i = 0; i < current_chunk->element_count; i++) {
@@ -573,7 +616,15 @@ int send_files(Config* config) {
client_delete(client);
return 1;
}
array_list_add(manifest, manifest_entry);
if (!array_list_add(manifest, manifest_entry)) {
free(manifest_entry);
chunk_destroy(current_chunk);
array_list_delete(manifest);
directory_scanner_destroy(scanner);
client_disconnect(client);
client_delete(client);
return 1;
}
}
}
if (!config->use_sendfile) {
@@ -590,6 +641,9 @@ int send_files(Config* config) {
if (send_chunk(client, current_chunk, config) != 0) {
log_message(LOG_LEVEL_ERROR, "Failed to send chunk");
chunk_destroy(current_chunk);
if (manifest)
array_list_delete(manifest);
manifest = NULL;
break;
}
if (config->show_progress) {
@@ -605,12 +659,15 @@ int send_files(Config* config) {
}
chunk_destroy(current_chunk);
}
if (directory_scanner_failed(scanner) || (config->use_delete && manifest == NULL))
goto send_fail;
if (config->use_delete) {
if (send_delete_manifest(client->file_descriptor, manifest) != 0) {
array_list_delete(manifest);
goto send_fail;
}
array_list_delete(manifest);
manifest = NULL;
}
if (!send_status(client->file_descriptor, STATUS_FINISHED))
goto send_fail;
@@ -632,6 +689,8 @@ int send_files(Config* config) {
return ok ? 0 : 1;
send_fail:
if (manifest)
array_list_delete(manifest);
directory_scanner_destroy(scanner);
client_disconnect(client);
client_delete(client);
@@ -685,14 +744,10 @@ int send_files_multithreaded(Config* config) {
if (!scanner_created || !loader_created || !sender_created) {
perror("Error creating threads.\n");
context->cancelled = true;
context->scanner_done = true;
context->loader_done = true;
pipeline_cancel(context);
mtx_lock(&context->mutex_progress);
context->sender_done = true;
cnd_broadcast(&context->condition_not_full_scanner);
cnd_broadcast(&context->condition_not_empty_scanner);
cnd_broadcast(&context->condition_not_full_loader);
cnd_broadcast(&context->condition_not_empty_loader);
mtx_unlock(&context->mutex_progress);
if (sender_created)
thrd_join(sender, NULL);
if (loader_created)
+186 -22
View File
@@ -11,6 +11,7 @@
#include <sys/stat.h>
#include <threads.h>
#include <unistd.h>
#include <limits.h>
typedef struct {
char* path;
@@ -38,6 +39,19 @@ static DirEntry* dir_entry_create(const char* path, int depth) {
return de;
}
static bool safe_relative_link(const char* source_root, const char* containing_dir,
const char* link_target) {
char root[PATH_MAX];
if (!realpath(source_root, root))
return false;
char* joined = path_cat(containing_dir, link_target);
char resolved[PATH_MAX];
bool safe = joined && realpath(joined, resolved) && strncmp(root, resolved, strlen(root)) == 0 &&
(resolved[strlen(root)] == '\0' || resolved[strlen(root)] == '/');
free(joined);
return safe;
}
DirectoryScanner* directory_scanner_create(const char* root_directory, bool use_metadata,
unsigned long long chunk_size, char** exclude_patterns,
int exclude_count, char** include_patterns,
@@ -45,10 +59,14 @@ DirectoryScanner* directory_scanner_create(const char* root_directory, bool use_
unsigned long long min_size, int max_depth,
bool follow_symlinks, bool copy_links, bool safe_links,
bool copy_unsafe_links, bool checksum) {
DirectoryScanner* scanner = malloc(sizeof(DirectoryScanner));
DirectoryScanner* scanner = calloc(1, sizeof(DirectoryScanner));
if (scanner == NULL)
return NULL;
scanner->directories = queue_create(100, dir_entry_destroy);
if (!scanner->directories) {
free(scanner);
return NULL;
}
scanner->current_dir = NULL;
scanner->current_path = NULL;
scanner->use_metadata = use_metadata;
@@ -66,6 +84,7 @@ DirectoryScanner* directory_scanner_create(const char* root_directory, bool use_
scanner->safe_links = safe_links;
scanner->copy_unsafe_links = copy_unsafe_links;
scanner->checksum = checksum;
scanner->failed = false;
DirEntry* root = dir_entry_create(root_directory, 0);
if (!root) {
queue_destroy(scanner->directories);
@@ -95,8 +114,12 @@ void directory_scanner_destroy(DirectoryScanner* scanner) {
static Chunk* chunk_data_to_chunk(ArrayList* chunk_data) {
void** chunk_items = array_list_to_array(chunk_data);
if (!chunk_items)
return NULL;
Chunk* chunk = chunk_create((File**)chunk_items, chunk_data->size);
free(chunk_items);
if (!chunk)
return NULL;
chunk_data->item_destroyer = NULL;
array_list_delete(chunk_data);
return chunk;
@@ -121,6 +144,7 @@ static int open_next_directory(DirectoryScanner* scanner) {
perror("Could not open directory");
free(scanner->current_path);
scanner->current_path = NULL;
scanner->failed = true;
return -1;
}
return 1;
@@ -128,6 +152,10 @@ static int open_next_directory(DirectoryScanner* scanner) {
Chunk* directory_scanner_next(DirectoryScanner* scanner) {
ArrayList* chunk_data = array_list_create(file_destroy);
if (!chunk_data) {
scanner->failed = true;
return NULL;
}
unsigned long long chunk_data_size = 0;
while (1) {
@@ -136,7 +164,7 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
if (ret == 0)
break;
if (ret < 0)
continue;
break;
}
struct dirent* entry = readdir(scanner->current_dir);
@@ -152,6 +180,10 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
continue;
char* cur_path = path_cat(scanner->current_path, entry->d_name);
if (!cur_path) {
scanner->failed = true;
break;
}
struct stat stats;
struct stat lstats;
bool is_symlink = false;
@@ -175,7 +207,8 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
continue;
}
link_target[len] = '\0';
if (link_target[0] == '/') {
if (link_target[0] == '/' ||
!safe_relative_link(scanner->current_path, scanner->current_path, link_target)) {
free(cur_path);
continue;
}
@@ -210,8 +243,10 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
int next_depth = scanner->current_depth + 1;
if (scanner->max_depth <= 0 || next_depth < scanner->max_depth) {
DirEntry* de = dir_entry_create(cur_path, next_depth);
if (!queue_enqueue(scanner->directories, de))
if (!de || !queue_enqueue(scanner->directories, de)) {
dir_entry_destroy(de);
scanner->failed = true;
}
}
free(cur_path);
} else {
@@ -254,27 +289,49 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
File* file = file_create(cur_path);
if (file == NULL) {
free(cur_path);
scanner->failed = true;
continue;
}
file->data->size = stats.st_size;
if (scanner->use_metadata)
file->metadata = file_metadata_create(&stats);
array_list_add(chunk_data, file);
if (scanner->use_metadata && !file->metadata) {
file_destroy(file);
free(cur_path);
scanner->failed = true;
break;
}
if (!array_list_add(chunk_data, file)) {
file_destroy(file);
scanner->failed = true;
break;
}
chunk_data_size += file->data->size;
if (chunk_data_size > scanner->chunk_size) {
free(cur_path);
return chunk_data_to_chunk(chunk_data);
Chunk* result = chunk_data_to_chunk(chunk_data);
if (!result)
scanner->failed = true;
return result;
}
free(cur_path);
}
}
if (chunk_data->size > 0)
return chunk_data_to_chunk(chunk_data);
if (chunk_data->size > 0) {
Chunk* result = chunk_data_to_chunk(chunk_data);
if (!result)
scanner->failed = true;
return result;
}
array_list_delete(chunk_data);
return NULL;
}
bool directory_scanner_failed(const DirectoryScanner* scanner) {
return scanner == NULL || scanner->failed;
}
typedef struct {
ParallelScanner* ps;
char** dirs;
@@ -302,10 +359,31 @@ static int parallel_worker_thread(void* arg) {
wa->dirs[i], wa->use_metadata, wa->chunk_size, wa->exclude_patterns, wa->exclude_count,
wa->include_patterns, wa->include_count, wa->max_size, wa->min_size, wa->max_depth,
wa->follow_symlinks, wa->copy_links, wa->safe_links, wa->copy_unsafe_links, wa->checksum);
if (!ds) {
mtx_lock(&wa->ps->result_mutex);
wa->ps->failed = true;
atomic_store(&wa->ps->cancelled, true);
cnd_broadcast(&wa->ps->result_not_empty);
cnd_broadcast(&wa->ps->result_not_full);
mtx_unlock(&wa->ps->result_mutex);
break;
}
Chunk* chunk;
while ((chunk = directory_scanner_next(ds)) != NULL) {
queue_enqueue_multithreaded(wa->ps->result_queue, chunk, &wa->ps->result_mutex,
&wa->ps->result_not_empty, &wa->ps->result_not_full);
if (!queue_enqueue_multithreaded_cancel(wa->ps->result_queue, chunk, &wa->ps->result_mutex,
&wa->ps->result_not_empty, &wa->ps->result_not_full,
&wa->ps->cancelled)) {
chunk_destroy(chunk);
break;
}
}
if (directory_scanner_failed(ds)) {
mtx_lock(&wa->ps->result_mutex);
wa->ps->failed = true;
atomic_store(&wa->ps->cancelled, true);
cnd_broadcast(&wa->ps->result_not_empty);
cnd_broadcast(&wa->ps->result_not_full);
mtx_unlock(&wa->ps->result_mutex);
}
directory_scanner_destroy(ds);
free(wa->dirs[i]);
@@ -315,7 +393,7 @@ static int parallel_worker_thread(void* arg) {
free(wa);
mtx_lock(&ps->result_mutex);
ps->completed++;
if (ps->completed >= ps->num_threads) {
if (ps->completed >= ps->expected_threads) {
ps->done = true;
cnd_signal(&ps->result_not_empty);
}
@@ -338,6 +416,7 @@ ParallelScanner* parallel_scanner_create(char* root_directory, bool use_metadata
free(ps);
return NULL;
}
atomic_init(&ps->cancelled, false);
int init = 0;
bool ok = true;
if (mtx_init(&ps->result_mutex, mtx_plain) != thrd_success)
@@ -374,6 +453,13 @@ ParallelScanner* parallel_scanner_create(char* root_directory, bool use_metadata
ArrayList* root_files = array_list_create(file_destroy);
ArrayList* subdirs = array_list_create(free);
if (!root_files || !subdirs) {
array_list_delete(root_files);
array_list_delete(subdirs);
closedir(dir);
parallel_scanner_destroy(ps);
return NULL;
}
struct dirent* entry;
while ((entry = readdir(dir)) != NULL) {
if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
@@ -403,7 +489,8 @@ ParallelScanner* parallel_scanner_create(char* root_directory, bool use_metadata
continue;
}
link_target[len] = 0;
if (link_target[0] == '/') {
if (link_target[0] == '/' ||
!safe_relative_link(root_directory, root_directory, link_target)) {
free(cur_path);
continue;
}
@@ -438,7 +525,10 @@ ParallelScanner* parallel_scanner_create(char* root_directory, bool use_metadata
}
if (S_ISDIR(st.st_mode)) {
array_list_add(subdirs, cur_path);
if (!array_list_add(subdirs, cur_path)) {
free(cur_path);
ps->failed = true;
}
} else {
bool excluded = false;
for (int i = 0; i < exclude_count; i++) {
@@ -471,12 +561,22 @@ ParallelScanner* parallel_scanner_create(char* root_directory, bool use_metadata
}
File* file = file_create(cur_path);
free(cur_path);
if (!file)
if (!file) {
ps->failed = true;
continue;
}
file->data->size = st.st_size;
if (use_metadata)
file->metadata = file_metadata_create(&st);
array_list_add(root_files, file);
if (use_metadata && !file->metadata) {
file_destroy(file);
ps->failed = true;
continue;
}
if (!array_list_add(root_files, file)) {
file_destroy(file);
ps->failed = true;
}
}
}
closedir(dir);
@@ -484,27 +584,57 @@ ParallelScanner* parallel_scanner_create(char* root_directory, bool use_metadata
unsigned long long cs = chunk_size > 0 ? chunk_size : DESIRED_CHUNK_SIZE;
if (root_files->size > 0) {
ArrayList* batch = array_list_create(NULL);
if (!batch) {
ps->failed = true;
array_list_delete(root_files);
array_list_delete(subdirs);
parallel_scanner_destroy(ps);
return NULL;
}
unsigned long long batch_size = 0;
Chunk* first = NULL;
for (int i = 0; i < root_files->size; i++) {
File* f = (File*)root_files->items[i];
array_list_add(batch, f);
if (!array_list_add(batch, f)) {
ps->failed = true;
break;
}
batch_size += f->data->size;
if (batch_size >= cs || i == root_files->size - 1) {
void** items = array_list_to_array(batch);
if (!items) {
ps->failed = true;
batch->item_destroyer = file_destroy;
array_list_delete(batch);
batch = NULL;
break;
}
Chunk* c = chunk_create((File**)items, batch->size);
free(items);
if (!c) {
ps->failed = true;
batch->item_destroyer = file_destroy;
array_list_delete(batch);
batch = NULL;
break;
}
batch->item_destroyer = NULL;
array_list_delete(batch);
batch = NULL;
if (!first) {
first = c;
} else {
queue_enqueue_multithreaded(ps->result_queue, c, &ps->result_mutex, &ps->result_not_empty,
&ps->result_not_full);
if (!queue_enqueue(ps->result_queue, c)) {
chunk_destroy(c);
ps->failed = true;
}
}
if (i < root_files->size - 1) {
batch = array_list_create(NULL);
if (!batch) {
ps->failed = true;
break;
}
batch_size = 0;
}
}
@@ -524,6 +654,7 @@ ParallelScanner* parallel_scanner_create(char* root_directory, bool use_metadata
if (subdirs->size > 0) {
ps->num_threads = n;
ps->expected_threads = n;
ps->threads = calloc(n, sizeof(thrd_t));
if (!ps->threads) {
array_list_delete(subdirs);
@@ -533,21 +664,37 @@ ParallelScanner* parallel_scanner_create(char* root_directory, bool use_metadata
int dirs_per_thread = subdirs->size / n;
int remainder = subdirs->size % n;
int start = 0;
ps->num_threads = 0;
for (int t = 0; t < n; t++) {
int count = dirs_per_thread + (t < remainder ? 1 : 0);
if (count == 0)
break;
ParallelWorkerArg* wa = calloc(1, sizeof(ParallelWorkerArg));
if (!wa)
if (!wa) {
ps->failed = true;
break;
}
wa->ps = ps;
wa->dirs = calloc(count, sizeof(char*));
if (!wa->dirs) {
free(wa);
ps->failed = true;
break;
}
for (int j = 0; j < count; j++)
bool dup_ok = true;
for (int j = 0; j < count; j++) {
wa->dirs[j] = str_dup((char*)subdirs->items[start + j]);
if (!wa->dirs[j])
dup_ok = false;
}
if (!dup_ok) {
for (int j = 0; j < count; j++)
free(wa->dirs[j]);
free(wa->dirs);
free(wa);
ps->failed = true;
break;
}
wa->dir_count = count;
wa->use_metadata = use_metadata;
wa->chunk_size = cs;
@@ -569,9 +716,17 @@ ParallelScanner* parallel_scanner_create(char* root_directory, bool use_metadata
free(wa->dirs[j]);
free(wa->dirs);
free(wa);
ps->num_threads = t;
ps->failed = true;
atomic_store(&ps->cancelled, true);
ps->expected_threads = ps->created_threads;
mtx_lock(&ps->result_mutex);
cnd_broadcast(&ps->result_not_empty);
cnd_broadcast(&ps->result_not_full);
mtx_unlock(&ps->result_mutex);
break;
}
ps->num_threads++;
ps->created_threads++;
}
}
array_list_delete(subdirs);
@@ -585,7 +740,9 @@ Chunk* parallel_scanner_next(ParallelScanner* ps) {
return c;
}
if (ps->num_threads == 0) {
mtx_lock(&ps->result_mutex);
ps->done = true;
mtx_unlock(&ps->result_mutex);
return NULL;
}
Chunk* chunk = queue_dequeue_multithreaded(
@@ -593,12 +750,19 @@ Chunk* parallel_scanner_next(ParallelScanner* ps) {
return chunk;
}
bool parallel_scanner_failed(const ParallelScanner* ps) {
return ps == NULL || ps->failed;
}
void parallel_scanner_destroy(ParallelScanner* ps) {
if (!ps)
return;
mtx_lock(&ps->result_mutex);
ps->done = true;
cnd_signal(&ps->result_not_empty);
atomic_store(&ps->cancelled, true);
cnd_broadcast(&ps->result_not_empty);
cnd_broadcast(&ps->result_not_full);
mtx_unlock(&ps->result_mutex);
for (int i = 0; i < ps->num_threads; i++)
thrd_join(ps->threads[i], NULL);
free(ps->threads);
+8
View File
@@ -6,6 +6,7 @@
#include <dirent.h>
#include <stdbool.h>
#include <threads.h>
#include <stdatomic.h>
typedef struct {
Queue* directories;
@@ -26,6 +27,7 @@ typedef struct {
bool safe_links;
bool copy_unsafe_links;
bool checksum;
bool failed;
} DirectoryScanner;
typedef struct {
@@ -34,8 +36,12 @@ typedef struct {
cnd_t result_not_empty;
cnd_t result_not_full;
int num_threads;
int expected_threads;
int created_threads;
thrd_t* threads;
bool done;
bool failed;
atomic_bool cancelled;
int completed;
Chunk* initial_chunk;
} ParallelScanner;
@@ -48,6 +54,7 @@ DirectoryScanner* directory_scanner_create(const char* root_directory, bool use_
bool follow_symlinks, bool copy_links, bool safe_links,
bool copy_unsafe_links, bool checksum);
Chunk* directory_scanner_next(DirectoryScanner* scanner);
bool directory_scanner_failed(const DirectoryScanner* scanner);
void directory_scanner_destroy(DirectoryScanner* scanner);
ParallelScanner* parallel_scanner_create(char* root_directory, bool use_metadata,
@@ -58,6 +65,7 @@ ParallelScanner* parallel_scanner_create(char* root_directory, bool use_metadata
int num_threads, bool follow_symlinks, bool copy_links,
bool safe_links, bool copy_unsafe_links, bool checksum);
Chunk* parallel_scanner_next(ParallelScanner* scanner);
bool parallel_scanner_failed(const ParallelScanner* scanner);
void parallel_scanner_destroy(ParallelScanner* scanner);
#endif
+73
View File
@@ -0,0 +1,73 @@
#include "stdio.h"
#include <delta.h>
#include <chunk.h>
static __attribute__((unused)) void print_usage() {
printf("Usage:\n");
printf(" fastsync [options] <source> <destination>\n");
printf(" fastsync [options] --source-dir <src> --dest-dir <dst>\n");
printf("\n");
printf("Destination formats:\n");
printf(" user@host:/path SSH transport (rsync-style)\n");
printf(" host:/path SSH transport (current user)\n");
printf(" /local/path TCP transport (requires server on localhost:8080)\n");
printf("\n");
printf("Options:\n");
printf(" -c [level] Enable compression (level 1-22, default 5)\n");
printf(" -z [level] Alias for -c\n");
printf(" -a, --archive Archive mode (-c -m -M)\n");
printf(" -n, --dry-run Show what would be transferred\n");
printf(" -p <port> SSH port (default: 22)\n");
printf(" --progress Show transfer progress\n");
printf(" --delete Delete files on receiver not in source\n");
printf(" --exclude <pattern> Exclude files matching pattern\n");
printf(" --include <pattern> Only include files matching pattern\n");
printf(" --exclude-from <file> Read exclude patterns from file\n");
printf(" --include-from <file> Read include patterns from file\n");
printf(" --max-size <n> Skip files larger than n bytes\n");
printf(" --min-size <n> Skip files smaller than n bytes\n");
printf(" --incremental Skip files unchanged since last transfer\n");
printf(" --delta Delta transfer for changed files (requires --incremental)\n");
printf(" --delta-block <n> Delta block size in bytes (default: %d)\n",
DELTA_BLOCK_SIZE_DEFAULT);
printf(" --delta-max <n> Max file size for delta transfer (default: %llu)\n",
DELTA_MAX_FILE_SIZE);
printf(" -m Enable multithreading\n");
printf(" -s Enable chunk serialization\n");
printf(" -f Enable sendfile (TCP only, not with -c or -s)\n");
printf(" -v, --verbose Enable debug logging\n");
printf(" -M, --preserve Preserve file metadata\n");
printf(" --chunk-size <n> Chunk size in bytes (default: %d)\n", DEFAULT_CHUNK_SIZE);
printf(" --source-dir <path> Source directory\n");
printf(" --dest-dir <path> Destination directory\n");
printf(" --save-to-disk Write received files to disk\n");
printf(" --server-host <ip> Server IP address (default: 127.0.0.1)\n");
printf(" --server-port <n> Server port (default: 8080)\n");
printf(" --bwlimit <KB/s> Bandwidth limit in kilobytes per second\n");
printf(" --tls Enable TLS encryption\n");
printf(" --cert <path> TLS certificate file (PEM)\n");
printf(" --key <path> TLS private key file (PEM)\n");
printf(" --ca <path> TLS CA certificate file (PEM)\n");
printf(" --timeout <sec> I/O timeout in seconds (default: 30)\n");
printf(" -T <sec> Alias for --timeout\n");
printf(" --contimeout <sec> Connection timeout in seconds (default: 10)\n");
printf(" --backup Backup existing files before overwriting\n");
printf(" --backup-dir <dir> Directory for backups (requires --backup)\n");
printf(" --suffix <str> Backup suffix (default: ~)\n");
printf(" --stats Print transfer statistics at end\n");
printf(" --max-depth <n> Maximum directory depth (0=unlimited)\n");
printf(" --log-file <path> Write log messages to file\n");
printf(" --partial Keep partial files on interrupted transfer\n");
printf(" --partial-dir <dir> Directory for partial files\n");
printf(" --fastsync-server-path <path>\n");
printf(" Path to fastsync-server on remote (default: fastsync-server)\n");
printf(" -l, --links Copy symlinks as symlinks\n");
printf(" --copy-links Transform symlinks into referent files\n");
printf(" --safe-links Skip symlinks that point outside transfer tree\n");
printf(" --copy-unsafe-links Only transform unsafe symlinks into referent files\n");
printf(" -S, --sparse Handle sparse files efficiently\n");
printf(" --inplace Update files in-place (no temp+rename)\n");
printf(" --compress-level <n> Compression level (default: 5)\n");
printf(" --help Show this help\n");
printf(" -V, --version Show version\n");
}
+30 -98
View File
@@ -1,7 +1,4 @@
#include "array_list.h"
#include "chunk.h"
#include "config.h"
#include "data.h"
#include "file.h"
#include "log.h"
#include "multiprocessing.h"
@@ -16,10 +13,12 @@
#include <stdlib.h>
#include <string.h>
#include <limits.h>
#include <fcntl.h>
#include <sys/stat.h>
#include <unistd.h>
static char* authorized_root;
static int authorized_root_fd = -1;
static bool allow_delete;
static bool path_is_within(const char* root, const char* path) {
@@ -27,107 +26,34 @@ static bool path_is_within(const char* root, const char* path) {
return strncmp(root, path, n) == 0 && (path[n] == '\0' || path[n] == '/');
}
static bool save_received_file(File* file, void* context) {
Config* config = context;
if (config->save_to_disk && !file_save_to_disk(config->receive_root_directory, file, config))
return false;
file_destroy(file);
return true;
}
static bool __attribute__((unused)) configure_authorization(const char* root) {
char resolved[PATH_MAX];
if (!root || !realpath(root, resolved))
return false;
authorized_root = str_dup(resolved);
return authorized_root != NULL;
if (!authorized_root)
return false;
authorized_root_fd = open(resolved, O_RDONLY | O_DIRECTORY | O_CLOEXEC);
if (authorized_root_fd < 0) {
free(authorized_root);
authorized_root = NULL;
return false;
}
file_set_authorized_root(authorized_root_fd, authorized_root);
utils_set_authorized_root_fd(authorized_root_fd);
return true;
}
int receive_files(Config* config, int fd) {
Status status;
if (!receive_status(fd, &status))
return -1;
while (status == STATUS_NEXT || status == STATUS_CHUNK || status == STATUS_CHECK ||
status == STATUS_KEEPALIVE || status == STATUS_ABORT || status == STATUS_CHECK_BATCH) {
if (status == STATUS_KEEPALIVE) {
send_status(fd, STATUS_KEEPALIVE);
goto next;
}
if (status == STATUS_ABORT) {
log_message(LOG_LEVEL_INFO, "Received abort from client, cleaning up");
return -1;
}
if (status == STATUS_CHECK) {
bool skipped;
File* file = receive_incremental_check(fd, config, &skipped);
if (skipped)
goto next;
if (file == NULL && !skipped)
return -1;
if (config->save_to_disk)
file_save_to_disk(config->receive_root_directory, file, config);
file_destroy(file);
} else if (status == STATUS_CHUNK) {
Chunk* chunk = receive_chunk_data(fd, config);
if (chunk == NULL) {
send_status(fd, STATUS_ERROR);
return -1;
}
for (int i = 0; i < chunk->element_count; i++) {
if (config->save_to_disk)
file_save_to_disk(config->receive_root_directory, chunk->items[i], config);
}
chunk_destroy(chunk);
} else if (status == STATUS_CHECK_BATCH) {
int count;
if (!receive_int(fd, &count))
return -1;
for (int i = 0; i < count; i++) {
char* check_path = receive_str(fd);
if (!check_path)
return -1;
unsigned long long check_size;
long long check_mtime;
if (!receive_n_data(fd, &check_size, sizeof(check_size)) ||
!receive_n_data(fd, &check_mtime, sizeof(check_mtime))) {
free(check_path);
return -1;
}
char* full_path = path_cat(config->receive_root_directory, check_path);
struct stat st;
bool has_old = full_path && lstat(full_path, &st) == 0;
bool match = has_old && (unsigned long long)st.st_size == check_size &&
(long long)st.st_mtime == check_mtime;
if (match)
send_status(fd, STATUS_OK);
else
send_status(fd, STATUS_NEXT);
free(full_path);
free(check_path);
}
goto next;
} else {
File* file = file_receive(config, fd);
if (file == NULL) {
log_message(LOG_LEVEL_ERROR, "Failed to receive file");
send_status(fd, STATUS_ERROR);
return -1;
}
if (config->save_to_disk)
file_save_to_disk(config->receive_root_directory, file, config);
file_destroy(file);
}
next:
if (!receive_status(fd, &status)) {
send_status(fd, STATUS_ERROR);
return -1;
}
}
if (status == STATUS_MANIFEST) {
if (receive_manifest(fd, config, &status) != 0)
return -1;
}
if (status != STATUS_FINISHED) {
log_message(LOG_LEVEL_ERROR, "Did not receive FINISHED Status");
send_status(fd, STATUS_ERROR);
return -1;
}
send_status(fd, STATUS_OK);
return 0;
return receive_files_common(config, fd, save_received_file, config, true);
}
void handler(int file_descriptor) {
@@ -193,7 +119,7 @@ void handler(int file_descriptor) {
perror("Error creating Threads");
if (receiver_created) {
mtx_lock(&context->mutex);
context->cancelled = true;
atomic_store(&context->cancelled, true);
cnd_broadcast(&context->condition_not_full);
cnd_broadcast(&context->condition_not_empty);
mtx_unlock(&context->mutex);
@@ -213,9 +139,12 @@ void handler(int file_descriptor) {
thrd_join(writer, &writer_result);
if (receiver_result == thrd_success && writer_result == thrd_success)
send_status(file_descriptor, STATUS_OK);
else
send_status(file_descriptor, STATUS_ERROR);
pipeline_context_receiver_destroy(context);
} else {
receive_files(config, file_descriptor);
if (receive_files(config, file_descriptor) != 0)
log_message(LOG_LEVEL_ERROR, "Transfer failed");
config_delete(config);
}
close(file_descriptor);
@@ -307,6 +236,9 @@ int main(int argc, char* argv[]) {
if (stdio_mode) {
io_set_fds(STDIN_FILENO, STDOUT_FILENO);
handler(STDIN_FILENO);
file_set_authorized_root(-1, NULL);
utils_set_authorized_root_fd(-1);
close(authorized_root_fd);
free(authorized_root);
return 0;
}
+87 -254
View File
@@ -173,103 +173,39 @@ void config_delete(Config* config) {
free(config);
}
/* Wire format order (must match config_receive and be updated when PROTOCOL_VERSION bumps):
* version, send_directory, receive_root_directory, save_to_disk, use_multithreading,
* use_chunk_serialization, use_compression, use_metadata, compression_level, chunk_size,
* use_sendfile, use_delete, use_incremental, use_delta, delta_block_size, delta_max_file_size,
* backup, backup_dir, follow_symlinks, copy_links, safe_links, copy_unsafe_links,
* preserve_hard_links, preserve_acls, preserve_xattrs, preserve_devices, preserve_sparse,
* update, inplace, append, append_verify, delete_excluded, delete_after, max_delete, relative,
* prune_empty_dirs, temp_dir, partial, partial_dir, suffix, delete_before, checksum,
* compress_choice, status
*/
static bool config_send_string(int file_descriptor, const char* value, bool optional) {
return send_str(file_descriptor, value ? value : (optional ? "" : NULL));
}
#define CONFIG_SEND_VERSION(field) \
do { \
if (!config_send_string(file_descriptor, config->field, false)) \
return false; \
} while (0);
#define CONFIG_SEND_STRING(field) \
do { \
if (!config_send_string(file_descriptor, config->field, false)) \
return false; \
} while (0);
#define CONFIG_SEND_OPTIONAL_STRING(field) \
do { \
if (!config_send_string(file_descriptor, config->field, true)) \
return false; \
} while (0);
#define CONFIG_SEND_INTEGER(field) \
do { \
if (!send_int(file_descriptor, config->field)) \
return false; \
} while (0);
#define CONFIG_SEND_DATA(field) \
do { \
if (!send_n_data(file_descriptor, &config->field, sizeof(config->field))) \
return false; \
} while (0);
bool config_send(int file_descriptor, const Config* config) {
if (!send_str(file_descriptor, config->version))
return false;
if (!send_str(file_descriptor, config->send_directory))
return false;
if (!send_str(file_descriptor, config->receive_root_directory))
return false;
if (!send_int(file_descriptor, config->save_to_disk))
return false;
if (!send_int(file_descriptor, config->use_multithreading))
return false;
if (!send_int(file_descriptor, config->use_chunk_serialization))
return false;
if (!send_int(file_descriptor, config->use_compression))
return false;
if (!send_int(file_descriptor, config->use_metadata))
return false;
if (!send_int(file_descriptor, config->compression_level))
return false;
if (!send_n_data(file_descriptor, &config->chunk_size, sizeof(config->chunk_size)))
return false;
if (!send_int(file_descriptor, config->use_sendfile))
return false;
if (!send_int(file_descriptor, config->use_delete))
return false;
if (!send_int(file_descriptor, config->use_incremental))
return false;
if (!send_int(file_descriptor, config->use_delta))
return false;
if (!send_n_data(file_descriptor, &config->delta_block_size, sizeof(config->delta_block_size)))
return false;
if (!send_n_data(file_descriptor, &config->delta_max_file_size, sizeof(unsigned long long)))
return false;
if (!send_int(file_descriptor, config->backup))
return false;
if (!send_str(file_descriptor, config->backup_dir ? config->backup_dir : ""))
return false;
if (!send_int(file_descriptor, config->follow_symlinks))
return false;
if (!send_int(file_descriptor, config->copy_links))
return false;
if (!send_int(file_descriptor, config->safe_links))
return false;
if (!send_int(file_descriptor, config->copy_unsafe_links))
return false;
if (!send_int(file_descriptor, config->preserve_hard_links))
return false;
if (!send_int(file_descriptor, config->preserve_acls))
return false;
if (!send_int(file_descriptor, config->preserve_xattrs))
return false;
if (!send_int(file_descriptor, config->preserve_devices))
return false;
if (!send_int(file_descriptor, config->preserve_sparse))
return false;
if (!send_int(file_descriptor, config->update))
return false;
if (!send_int(file_descriptor, config->inplace))
return false;
if (!send_int(file_descriptor, config->append))
return false;
if (!send_int(file_descriptor, config->append_verify))
return false;
if (!send_int(file_descriptor, config->delete_excluded))
return false;
if (!send_int(file_descriptor, config->delete_after))
return false;
if (!send_n_data(file_descriptor, &config->max_delete, sizeof(config->max_delete)))
return false;
if (!send_int(file_descriptor, config->relative))
return false;
if (!send_int(file_descriptor, config->prune_empty_dirs))
return false;
if (!send_str(file_descriptor, config->temp_dir ? config->temp_dir : ""))
return false;
if (!send_int(file_descriptor, config->partial))
return false;
if (!send_str(file_descriptor, config->partial_dir ? config->partial_dir : ""))
return false;
if (!send_str(file_descriptor, config->suffix ? config->suffix : ""))
return false;
if (!send_int(file_descriptor, config->delete_before))
return false;
if (!send_int(file_descriptor, config->checksum))
return false;
if (!send_str(file_descriptor, config->compress_choice ? config->compress_choice : ""))
return false;
CONFIG_WIRE_FIELDS(CONFIG_SEND_VERSION, CONFIG_SEND_STRING, CONFIG_SEND_OPTIONAL_STRING,
CONFIG_SEND_INTEGER, CONFIG_SEND_DATA)
Status status;
if (!receive_status(file_descriptor, &status))
return false;
@@ -280,160 +216,60 @@ bool config_send(int file_descriptor, const Config* config) {
return true;
}
/* Wire format order: see the comment above config_send. */
#undef CONFIG_SEND_VERSION
#undef CONFIG_SEND_STRING
#undef CONFIG_SEND_OPTIONAL_STRING
#undef CONFIG_SEND_INTEGER
#undef CONFIG_SEND_DATA
static bool config_receive_string(int file_descriptor, char** destination) {
char* value = receive_str(file_descriptor);
if (!value)
return false;
*destination = value;
return true;
}
#define CONFIG_RECEIVE_VERSION(field) \
do { \
free(config->field); \
config->field = receive_str(file_descriptor); \
if (!config->field) \
goto error; \
if (strcmp(config->field, PROTOCOL_VERSION) != 0) { \
fprintf(stderr, "Protocol version mismatch: client=%s, server=%s\n", config->field, \
PROTOCOL_VERSION); \
config_delete(config); \
send_status(file_descriptor, STATUS_ERROR); \
return NULL; \
} \
} while (0);
#define CONFIG_RECEIVE_STRING(field) \
do { \
if (!config_receive_string(file_descriptor, &config->field)) \
goto error; \
} while (0);
#define CONFIG_RECEIVE_OPTIONAL_STRING(field) CONFIG_RECEIVE_STRING(field)
#define CONFIG_RECEIVE_INTEGER(field) \
do { \
if (!receive_int(file_descriptor, &tmp)) \
goto error; \
config->field = tmp; \
} while (0);
#define CONFIG_RECEIVE_DATA(field) \
do { \
if (!receive_n_data(file_descriptor, &config->field, sizeof(config->field))) \
goto error; \
} while (0);
Config* config_receive(int file_descriptor) {
Config* config = (Config*)malloc(sizeof(Config));
if (config == NULL)
return NULL;
config_set_defaults(config);
free(config->version);
config->version = receive_str(file_descriptor);
if (!config->version) {
free(config->server_host);
free(config);
return NULL;
}
if (strcmp(config->version, PROTOCOL_VERSION) != 0) {
fprintf(stderr, "Protocol version mismatch: client=%s, server=%s\n", config->version,
PROTOCOL_VERSION);
free(config->version);
free(config->server_host);
free(config);
send_status(file_descriptor, STATUS_ERROR);
return NULL;
}
config->send_directory = receive_str(file_descriptor);
if (!config->send_directory) {
free(config->version);
free(config->server_host);
free(config);
return NULL;
}
config->receive_root_directory = receive_str(file_descriptor);
if (!config->receive_root_directory) {
free(config->version);
free(config->send_directory);
free(config->server_host);
free(config);
return NULL;
}
int tmp;
if (!receive_int(file_descriptor, &tmp))
goto error;
config->save_to_disk = tmp;
if (!receive_int(file_descriptor, &tmp))
goto error;
config->use_multithreading = tmp;
if (!receive_int(file_descriptor, &tmp))
goto error;
config->use_chunk_serialization = tmp;
if (!receive_int(file_descriptor, &tmp))
goto error;
config->use_compression = tmp;
if (!receive_int(file_descriptor, &tmp))
goto error;
config->use_metadata = tmp;
if (!receive_int(file_descriptor, &tmp))
goto error;
config->compression_level = tmp;
if (!receive_n_data(file_descriptor, &config->chunk_size, sizeof(config->chunk_size)))
goto error;
if (!receive_int(file_descriptor, &tmp))
goto error;
config->use_sendfile = tmp;
if (!receive_int(file_descriptor, &tmp))
goto error;
config->use_delete = tmp;
if (!receive_int(file_descriptor, &tmp))
goto error;
config->use_incremental = tmp;
if (!receive_int(file_descriptor, &tmp))
goto error;
config->use_delta = tmp;
if (!receive_n_data(file_descriptor, &config->delta_block_size, sizeof(config->delta_block_size)))
goto error;
if (!receive_n_data(file_descriptor, &config->delta_max_file_size, sizeof(unsigned long long)))
goto error;
if (!receive_int(file_descriptor, &tmp))
goto error;
config->backup = tmp;
config->backup_dir = receive_str(file_descriptor);
if (config->backup_dir == NULL)
goto error;
if (!receive_int(file_descriptor, &tmp))
goto error;
config->follow_symlinks = tmp;
if (!receive_int(file_descriptor, &tmp))
goto error;
config->copy_links = tmp;
if (!receive_int(file_descriptor, &tmp))
goto error;
config->safe_links = tmp;
if (!receive_int(file_descriptor, &tmp))
goto error;
config->copy_unsafe_links = tmp;
if (!receive_int(file_descriptor, &tmp))
goto error;
config->preserve_hard_links = tmp;
if (!receive_int(file_descriptor, &tmp))
goto error;
config->preserve_acls = tmp;
if (!receive_int(file_descriptor, &tmp))
goto error;
config->preserve_xattrs = tmp;
if (!receive_int(file_descriptor, &tmp))
goto error;
config->preserve_devices = tmp;
if (!receive_int(file_descriptor, &tmp))
goto error;
config->preserve_sparse = tmp;
if (!receive_int(file_descriptor, &tmp))
goto error;
config->update = tmp;
if (!receive_int(file_descriptor, &tmp))
goto error;
config->inplace = tmp;
if (!receive_int(file_descriptor, &tmp))
goto error;
config->append = tmp;
if (!receive_int(file_descriptor, &tmp))
goto error;
config->append_verify = tmp;
if (!receive_int(file_descriptor, &tmp))
goto error;
config->delete_excluded = tmp;
if (!receive_int(file_descriptor, &tmp))
goto error;
config->delete_after = tmp;
if (!receive_n_data(file_descriptor, &config->max_delete, sizeof(config->max_delete)))
goto error;
if (!receive_int(file_descriptor, &tmp))
goto error;
config->relative = tmp;
if (!receive_int(file_descriptor, &tmp))
goto error;
config->prune_empty_dirs = tmp;
config->temp_dir = receive_str(file_descriptor);
if (config->temp_dir == NULL)
goto error;
if (!receive_int(file_descriptor, &tmp))
goto error;
config->partial = tmp;
config->partial_dir = receive_str(file_descriptor);
if (config->partial_dir == NULL)
goto error;
config->suffix = receive_str(file_descriptor);
if (config->suffix == NULL)
goto error;
if (!receive_int(file_descriptor, &tmp))
goto error;
config->delete_before = tmp;
if (!receive_int(file_descriptor, &tmp))
goto error;
config->checksum = tmp;
config->compress_choice = receive_str(file_descriptor);
if (config->compress_choice == NULL)
goto error;
CONFIG_WIRE_FIELDS(CONFIG_RECEIVE_VERSION, CONFIG_RECEIVE_STRING, CONFIG_RECEIVE_OPTIONAL_STRING,
CONFIG_RECEIVE_INTEGER, CONFIG_RECEIVE_DATA)
if (config->compress_choice[0] != '\0' && strcmp(config->compress_choice, "zstd") != 0 &&
strcmp(config->compress_choice, "none") != 0) {
fprintf(stderr, "Unsupported compression choice: %s\n", config->compress_choice);
@@ -445,15 +281,12 @@ Config* config_receive(int file_descriptor) {
return config;
error:
free(config->version);
free(config->send_directory);
free(config->receive_root_directory);
free(config->server_host);
free(config->backup_dir);
free(config->temp_dir);
free(config->partial_dir);
free(config->suffix);
free(config->compress_choice);
free(config);
config_delete(config);
return NULL;
}
#undef CONFIG_RECEIVE_VERSION
#undef CONFIG_RECEIVE_STRING
#undef CONFIG_RECEIVE_OPTIONAL_STRING
#undef CONFIG_RECEIVE_INTEGER
#undef CONFIG_RECEIVE_DATA
+47
View File
@@ -128,6 +128,53 @@ typedef struct Config {
char* compress_choice;
} Config;
/* Keep the on-wire field order in one place. The first three strings require
* values when sent; optional strings are encoded as empty strings when NULL. */
#define CONFIG_WIRE_FIELDS(VERSION, STRING, OPTIONAL_STRING, INTEGER, DATA) \
VERSION(version) \
STRING(send_directory) \
STRING(receive_root_directory) \
INTEGER(save_to_disk) \
INTEGER(use_multithreading) \
INTEGER(use_chunk_serialization) \
INTEGER(use_compression) \
INTEGER(use_metadata) \
INTEGER(compression_level) \
DATA(chunk_size) \
INTEGER(use_sendfile) \
INTEGER(use_delete) \
INTEGER(use_incremental) \
INTEGER(use_delta) \
DATA(delta_block_size) \
DATA(delta_max_file_size) \
INTEGER(backup) \
OPTIONAL_STRING(backup_dir) \
INTEGER(follow_symlinks) \
INTEGER(copy_links) \
INTEGER(safe_links) \
INTEGER(copy_unsafe_links) \
INTEGER(preserve_hard_links) \
INTEGER(preserve_acls) \
INTEGER(preserve_xattrs) \
INTEGER(preserve_devices) \
INTEGER(preserve_sparse) \
INTEGER(update) \
INTEGER(inplace) \
INTEGER(append) \
INTEGER(append_verify) \
INTEGER(delete_excluded) \
INTEGER(delete_after) \
DATA(max_delete) \
INTEGER(relative) \
INTEGER(prune_empty_dirs) \
OPTIONAL_STRING(temp_dir) \
INTEGER(partial) \
OPTIONAL_STRING(partial_dir) \
OPTIONAL_STRING(suffix) \
INTEGER(delete_before) \
INTEGER(checksum) \
OPTIONAL_STRING(compress_choice)
#define PROTOCOL_VERSION "2.2.0"
#define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024)
+230 -40
View File
@@ -3,6 +3,7 @@
#include <fcntl.h>
#include <libgen.h>
#include <limits.h>
#include <poll.h>
#include <stddef.h>
#include <stdio.h>
#include <stdlib.h>
@@ -10,6 +11,7 @@
#include <sys/sendfile.h>
#include <sys/stat.h>
#include <unistd.h>
#include <time.h>
#include "compression.h"
#include "delta.h"
@@ -35,6 +37,8 @@ bool file_checksum(File* file, uint64_t* checksum) {
}
File* file_create(const char* path) {
if (!path)
return NULL;
File* file = (File*)malloc(sizeof(File));
if (file == NULL) {
perror("ERROR: Could not allocate memory for file struct");
@@ -100,6 +104,8 @@ bool file_load_data(File* file) {
if (file == NULL)
return false;
if (file->data->data == NULL) {
if (file->data->size == 0)
return true;
file->data->data = malloc(file->data->size);
if (file->data->data == NULL) {
perror("Could not allocate memory for file data");
@@ -119,6 +125,8 @@ bool file_load_data(File* file) {
bool file_send_single_calls(File* file, int file_descriptor, bool use_metadata,
int compression_level, bool send_path) {
if (!file || !file->path || !file->data)
return false;
const Data* data_to_send = file->data;
Data* compressed_data = NULL;
if (compression_level > 0 && !compression_should_skip(file->path)) {
@@ -145,22 +153,56 @@ bool file_send_single_calls(File* file, int file_descriptor, bool use_metadata,
return true;
}
bool file_save_to_disk(const char* root_directory, File* file, const Config* config) {
static bool to_disk_secure(const char* path, const void* data, unsigned long long data_size,
bool inplace, bool sparse, const FileMetadata* metadata);
static int open_secure_parent(const char* path, char** leaf_out);
static bool rename_secure(const char* old_path, const char* new_path);
static int authorized_root_fd = -1;
static char* authorized_root_path;
void file_set_authorized_root(int fd, const char* canonical_path) {
authorized_root_fd = fd;
free(authorized_root_path);
authorized_root_path = canonical_path ? str_dup(canonical_path) : NULL;
}
static bool path_is_within_root(const char* root, const char* path) {
size_t n = strlen(root);
return strncmp(root, path, n) == 0 && (path[n] == '\0' || path[n] == '/');
}
bool file_save_to_disk(const char* root_directory, const File* file, const Config* config) {
bool backup_enabled = config && config->backup;
bool inplace = config && config->inplace;
bool sparse = config && config->preserve_sparse;
const char* backup_suffix = (config && config->suffix) ? config->suffix : "~";
const char* backup_dir = (config && config->backup_dir) ? config->backup_dir : NULL;
const char* partial_dir = (config && config->partial_dir) ? config->partial_dir : NULL;
char *confined_backup = NULL, *confined_partial = NULL;
if (has_path_traversal(file->path)) {
log_message(LOG_LEVEL_ERROR, "Path traversal detected in file path: %s", file->path);
if (!file || !file->path || !file->data || has_path_traversal(file->path) ||
(backup_enabled &&
(!backup_suffix || backup_suffix[0] == '\0' || strchr(backup_suffix, '/') != NULL ||
strcmp(backup_suffix, ".") == 0 || strcmp(backup_suffix, "..") == 0))) {
log_message(LOG_LEVEL_ERROR, "Invalid file or path received");
return false;
}
/* These options arrive from the client. They are names below the server
root, never independent filesystem roots. */
if ((backup_dir && (backup_dir[0] == '/' || has_path_traversal(backup_dir))) ||
(partial_dir && (partial_dir[0] == '/' || has_path_traversal(partial_dir))))
return false;
if (backup_dir && !(confined_backup = path_cat(root_directory, backup_dir)))
return false;
if (partial_dir && !(confined_partial = path_cat(root_directory, partial_dir))) {
free(confined_backup);
return false;
}
char* resolved_root = NULL;
const char* actual_root =
(partial_dir && config && config->partial) ? partial_dir : root_directory;
(partial_dir && config && config->partial) ? confined_partial : root_directory;
resolved_root = realpath(actual_root, NULL);
if (resolved_root == NULL) {
if (mkdir_r(actual_root)) {
@@ -169,11 +211,24 @@ bool file_save_to_disk(const char* root_directory, File* file, const Config* con
}
if (resolved_root == NULL) {
log_message(LOG_LEVEL_ERROR, "Failed to resolve destination root: %s", actual_root);
free(confined_backup);
free(confined_partial);
return false;
}
char* resolved_base = realpath(root_directory, NULL);
if (resolved_base == NULL || !path_is_within_root(resolved_base, resolved_root)) {
free(resolved_base);
free(confined_backup);
free(confined_partial);
free(resolved_root);
return false;
}
free(resolved_base);
char* disk_path = path_cat(resolved_root, file->path);
if (disk_path == NULL) {
free(confined_backup);
free(confined_partial);
free(resolved_root);
return false;
}
@@ -184,6 +239,8 @@ bool file_save_to_disk(const char* root_directory, File* file, const Config* con
if (stat(disk_path, &destination_stat) == 0 && file->metadata &&
destination_stat.st_mtime > file->metadata->mtime_sec) {
free(resolved_root);
free(confined_backup);
free(confined_partial);
free(disk_path);
return true;
}
@@ -194,13 +251,16 @@ bool file_save_to_disk(const char* root_directory, File* file, const Config* con
if (stat(disk_path, &backup_stat) == 0) {
char* backup_path = NULL;
if (backup_dir) {
char* resolved_backup_dir = realpath(backup_dir, NULL);
char* resolved_backup_dir = realpath(confined_backup, NULL);
if (!resolved_backup_dir) {
mkdir_r(backup_dir);
resolved_backup_dir = realpath(backup_dir, NULL);
mkdir_r(confined_backup);
resolved_backup_dir = realpath(confined_backup, NULL);
}
if (resolved_backup_dir) {
char* backup_base = realpath(root_directory, NULL);
if (backup_base && path_is_within_root(backup_base, resolved_backup_dir))
backup_path = path_cat(resolved_backup_dir, file->path);
free(backup_base);
free(resolved_backup_dir);
}
}
@@ -220,7 +280,14 @@ bool file_save_to_disk(const char* root_directory, File* file, const Config* con
mkdir_r(bdir);
free(backup_dir_path);
}
rename(disk_path, backup_path);
if (!rename_secure(disk_path, backup_path)) {
free(backup_path);
free(resolved_root);
free(confined_backup);
free(confined_partial);
free(disk_path);
return false;
}
free(backup_path);
}
}
@@ -228,6 +295,8 @@ bool file_save_to_disk(const char* root_directory, File* file, const Config* con
char* dir_dup = str_dup(disk_path);
if (!dir_dup) {
free(confined_backup);
free(confined_partial);
free(resolved_root);
free(disk_path);
return false;
@@ -235,6 +304,8 @@ bool file_save_to_disk(const char* root_directory, File* file, const Config* con
char* dir_str = dirname(dir_dup);
if (!mkdir_r(dir_str)) {
free(dir_dup);
free(confined_backup);
free(confined_partial);
free(resolved_root);
free(disk_path);
return false;
@@ -243,6 +314,8 @@ bool file_save_to_disk(const char* root_directory, File* file, const Config* con
free(dir_dup);
if (resolved_dir == NULL) {
log_message(LOG_LEVEL_ERROR, "Failed to resolve directory for: %s", disk_path);
free(confined_backup);
free(confined_partial);
free(resolved_root);
free(disk_path);
return false;
@@ -253,6 +326,8 @@ bool file_save_to_disk(const char* root_directory, File* file, const Config* con
(resolved_dir[root_len] != '\0' && resolved_dir[root_len] != '/')) {
log_message(LOG_LEVEL_ERROR, "Path escape detected: %s is outside %s", disk_path, actual_root);
free(resolved_dir);
free(confined_backup);
free(confined_partial);
free(resolved_root);
free(disk_path);
return false;
@@ -260,31 +335,14 @@ bool file_save_to_disk(const char* root_directory, File* file, const Config* con
free(resolved_dir);
free(resolved_root);
bool ok = to_disk(disk_path, file->data->data, file->data->size, inplace, sparse);
if (ok)
file_restore_metadata(disk_path, file->metadata);
bool ok = to_disk_secure(disk_path, file->data->data, file->data->size, inplace, sparse,
file->metadata);
free(confined_backup);
free(confined_partial);
free(disk_path);
return ok;
}
static void* old_data_from_path(const char* full_path, unsigned long long old_size) {
void* data = malloc((size_t)old_size);
if (!data)
return NULL;
FILE* fp = fopen(full_path, "rb");
if (!fp) {
free(data);
return NULL;
}
size_t nread = fread(data, 1, (size_t)old_size, fp);
fclose(fp);
if (nread != (size_t)old_size) {
free(data);
return NULL;
}
return data;
}
static File* receive_delta_file(int fd, const Config* config, const char* check_path,
void* old_data, unsigned long long old_size) {
if (!old_data)
@@ -470,22 +528,53 @@ File* receive_incremental_check(int fd, const Config* config, bool* skipped) {
char* full_path = path_cat(config->receive_root_directory, check_path);
struct stat st;
bool has_old_file = (full_path && lstat(full_path, &st) == 0);
bool has_old_file = false;
int old_fd = -1;
if (full_path) {
char* leaf = NULL;
int parent_fd = open_secure_parent(full_path, &leaf);
if (parent_fd >= 0) {
old_fd = openat(parent_fd, leaf, O_RDONLY | O_CLOEXEC | O_NOFOLLOW);
free(leaf);
close(parent_fd);
has_old_file = old_fd >= 0 && fstat(old_fd, &st) == 0 && S_ISREG(st.st_mode);
}
}
unsigned long long old_size = has_old_file ? (unsigned long long)st.st_size : 0;
void* old_data = NULL;
if (has_old_file && old_size > 0) {
old_data = malloc((size_t)old_size);
if (old_data) {
size_t got = 0;
while (got < (size_t)old_size) {
ssize_t n = read(old_fd, (char*)old_data + got, (size_t)old_size - got);
if (n <= 0) {
free(old_data);
old_data = NULL;
break;
}
got += (size_t)n;
}
}
}
if (old_fd >= 0) {
close(old_fd);
}
bool match = has_old_file && (unsigned long long)st.st_size == check_size;
if (match && config->checksum) {
void* old_data = old_size > 0 ? old_data_from_path(full_path, old_size) : NULL;
uint64_t old_checksum = old_size == 0 ? delta_xxhash64("", 0) : 0;
if (old_data)
old_checksum = delta_xxhash64(old_data, (size_t)old_size);
match = (old_size == 0 || old_data) && old_checksum == check_checksum;
free(old_data);
old_data = NULL;
} else if (match) {
match = (long long)st.st_mtime == check_mtime;
}
if (match) {
free(old_data);
if (!send_status(fd, STATUS_OK)) {
free(full_path);
free(check_path);
@@ -501,13 +590,15 @@ File* receive_incremental_check(int fd, const Config* config, bool* skipped) {
delta_should_attempt(old_size, check_size, config->delta_max_file_size);
if (try_delta) {
void* old_data = old_data_from_path(full_path, old_size);
File* delta_file = receive_delta_file(fd, config, check_path, old_data, old_size);
old_data = NULL; /* receive_delta_file consumes the snapshot on every path */
if (delta_file) {
free(full_path);
free(check_path);
return delta_file;
}
free(old_data);
old_data = NULL;
try_delta = false;
}
@@ -571,8 +662,25 @@ static int open_secure_parent(const char* path, char** leaf_out) {
free(copy);
return -1;
}
int fd = (parent[0] == '/') ? open("/", O_RDONLY | O_DIRECTORY | O_CLOEXEC)
int fd;
if (authorized_root_fd >= 0 && authorized_root_path && path[0] == '/' &&
path_is_within_root(authorized_root_path, path)) {
fd = dup(authorized_root_fd);
size_t root_len = strlen(authorized_root_path);
char* relative = str_dup(path + root_len);
if (!relative) {
free(copy);
free(leaf);
close(fd);
return -1;
}
free(copy);
copy = relative;
parent = dirname(copy);
} else {
fd = (parent[0] == '/') ? open("/", O_RDONLY | O_DIRECTORY | O_CLOEXEC)
: open(".", O_RDONLY | O_DIRECTORY | O_CLOEXEC);
}
if (fd < 0) {
free(copy);
free(leaf);
@@ -601,6 +709,21 @@ static int open_secure_parent(const char* path, char** leaf_out) {
return fd;
}
static bool rename_secure(const char* old_path, const char* new_path) {
char *old_leaf = NULL, *new_leaf = NULL;
int old_parent = open_secure_parent(old_path, &old_leaf);
int new_parent = open_secure_parent(new_path, &new_leaf);
bool ok = old_parent >= 0 && new_parent >= 0 &&
renameat(old_parent, old_leaf, new_parent, new_leaf) == 0;
if (old_parent >= 0)
close(old_parent);
if (new_parent >= 0)
close(new_parent);
free(old_leaf);
free(new_leaf);
return ok;
}
static bool write_all(int fd, const void* data, unsigned long long size) {
const unsigned char* p = data;
unsigned long long done = 0;
@@ -616,7 +739,7 @@ static bool write_all(int fd, const void* data, unsigned long long size) {
}
static bool to_disk_secure(const char* path, const void* data, unsigned long long data_size,
bool inplace, bool sparse) {
bool inplace, bool sparse, const FileMetadata* metadata) {
char* leaf = NULL;
int dirfd = open_secure_parent(path, &leaf);
if (dirfd < 0)
@@ -628,6 +751,8 @@ static bool to_disk_secure(const char* path, const void* data, unsigned long lon
if (fd >= 0) {
if (!sparse || data_size == 0 || ftruncate(fd, (off_t)data_size) == 0)
ok = write_all(fd, data, data_size);
if (ok && metadata)
ok = file_restore_metadata_fd(fd, metadata);
}
} else {
char tmp[NAME_MAX];
@@ -640,6 +765,8 @@ static bool to_disk_secure(const char* path, const void* data, unsigned long lon
ok = ftruncate(fd, (off_t)data_size) == 0;
if (ok || (!sparse || data_size == 0))
ok = write_all(fd, data, data_size);
if (ok && metadata)
ok = file_restore_metadata_fd(fd, metadata);
if (close(fd) != 0)
ok = false;
fd = -1;
@@ -660,7 +787,7 @@ bool to_disk(const char* path, const void* data, unsigned long long data_size, b
bool sparse) {
if (!path || (!data && data_size != 0) || has_path_traversal(path))
return false;
return to_disk_secure(path, data, data_size, inplace, sparse);
return to_disk_secure(path, data, data_size, inplace, sparse, NULL);
/* Kept below only as historical context; all writes use descriptor-relative operations. */
char* tmp_path = NULL;
char* directory = NULL;
@@ -765,6 +892,8 @@ done:
bool file_send_sendfile(File* file, int file_descriptor, bool use_metadata, int compression_level,
bool send_path) {
if (!file || !file->path || !file->data)
return false;
if (compression_level > 0)
return file_send_single_calls(file, file_descriptor, use_metadata, compression_level,
send_path);
@@ -781,13 +910,56 @@ bool file_send_sendfile(File* file, int file_descriptor, bool use_metadata, int
}
unsigned long long file_size = file->data->size;
struct stat source_stat;
if (fstat(fd, &source_stat) != 0 || !S_ISREG(source_stat.st_mode) ||
(unsigned long long)source_stat.st_size < file_size) {
close(fd);
return false;
}
if (!send_n_data(file_descriptor, &file_size, sizeof(unsigned long long))) {
close(fd);
return false;
}
/* sendfile cannot encrypt TLS records. Keep the framing identical but
route encrypted transfers through the deadline-aware IO layer. */
if (io_get_ssl() != NULL) {
unsigned char buffer[64 * 1024];
unsigned long long remaining = file_size;
bool ok = true;
while (remaining > 0) {
size_t want = remaining > sizeof(buffer) ? sizeof(buffer) : (size_t)remaining;
ssize_t got = read(fd, buffer, want);
if (got <= 0 || !send_n_data(file_descriptor, buffer, (size_t)got)) {
ok = false;
break;
}
remaining -= (unsigned long long)got;
}
close(fd);
return ok;
}
off_t offset = 0;
struct timespec deadline;
clock_gettime(CLOCK_MONOTONIC, &deadline);
deadline.tv_sec += 60;
while ((unsigned long long)offset < file_size) {
struct timespec now;
clock_gettime(CLOCK_MONOTONIC, &now);
long long remaining = (long long)(deadline.tv_sec - now.tv_sec) * 1000LL +
(deadline.tv_nsec - now.tv_nsec) / 1000000LL;
if (remaining <= 0) {
close(fd);
return false;
}
struct pollfd pfd = {.fd = file_descriptor, .events = POLLOUT};
int timeout = remaining > INT_MAX ? INT_MAX : (int)remaining;
int polled = poll(&pfd, 1, timeout);
if (polled <= 0 || (pfd.revents & (POLLERR | POLLHUP | POLLNVAL))) {
close(fd);
return false;
}
ssize_t sent = sendfile(file_descriptor, fd, &offset, file_size - offset);
if (sent == -1) {
if (errno == EAGAIN || errno == EINTR)
@@ -796,6 +968,10 @@ bool file_send_sendfile(File* file, int file_descriptor, bool use_metadata, int
close(fd);
return false;
}
if (sent == 0) {
close(fd);
return false;
}
}
close(fd);
@@ -859,6 +1035,8 @@ size_t file_content_to_buffer(File* file) {
}
int receive_manifest(int fd, const Config* config, int* next_status) {
int received_status = STATUS_ERROR;
int* status_out = next_status ? next_status : &received_status;
int count;
if (!receive_int(fd, &count))
return -1;
@@ -867,18 +1045,30 @@ int receive_manifest(int fd, const Config* config, int* next_status) {
ArrayList* manifest = array_list_create(free);
if (!manifest)
return -1;
size_t manifest_bytes = 0;
for (int i = 0; i < count; i++) {
char* s = receive_str(fd);
if (!s || s[0] == '\0' || has_path_traversal(s) || !array_list_add(manifest, s)) {
size_t entry_size = s ? strlen(s) : 0;
if (!s || s[0] == '\0' || s[0] == '/' || has_path_traversal(s) ||
entry_size > MAX_MANIFEST_BYTES - manifest_bytes ||
(manifest_bytes += entry_size) > MAX_MANIFEST_BYTES || !array_list_add(manifest, s)) {
free(s);
array_list_delete(manifest);
return -1;
}
}
fprintf(stderr, "Deleting files not in manifest...\n");
delete_extras(config->receive_root_directory, manifest);
if (!receive_status(fd, status_out)) {
array_list_delete(manifest);
if (!receive_status(fd, next_status))
return -1;
return 0;
}
/* Deletion is a commit operation: never perform it until the sender has
completed the manifest frame successfully. */
if (*status_out != STATUS_FINISHED || !config->use_delete) {
array_list_delete(manifest);
return *status_out == STATUS_FINISHED ? 0 : -1;
}
fprintf(stderr, "Deleting files not in manifest...\n");
bool deletion_ok = delete_extras(config->receive_root_directory, manifest);
array_list_delete(manifest);
return deletion_ok ? 0 : -1;
}
+2 -1
View File
@@ -37,7 +37,8 @@ FileMetadata* file_metadata_create(const struct stat* stats);
void file_metadata_destroy(void* metadata);
bool to_disk(const char* path, const void* data, unsigned long long data_size, bool inplace,
bool sparse);
bool file_save_to_disk(const char* root_directory, File* file, const Config* config);
bool file_save_to_disk(const char* root_directory, const File* file, const Config* config);
void file_set_authorized_root(int fd, const char* canonical_path);
File* receive_incremental_check(int fd, const Config* config, bool* skipped);
int receive_manifest(int fd, const Config* config, int* next_status);
+15 -9
View File
@@ -15,6 +15,15 @@ void log_set_file(FILE* fp) {
log_fp = fp;
}
static inline void write_message(FILE* dest_io, LogLevel log_level, struct tm t, const char* format,
va_list args) {
fprintf(dest_io, "%04d-%02d-%02d %02d:%02d:%02d [%s]: ", t.tm_year + 1900, t.tm_mon + 1,
t.tm_mday, t.tm_hour, t.tm_min, t.tm_sec, log_level_strings[log_level]);
vfprintf(dest_io, format, args);
fprintf(dest_io, "\n");
}
void log_message(LogLevel log_level, const char* format, ...) {
if (log_level < current_log_level)
return;
@@ -25,22 +34,19 @@ void log_message(LogLevel log_level, const char* format, ...) {
if (!localtime_r(&now, &t))
return;
fprintf(stderr, "%04d-%02d-%02d %02d:%02d:%02d [%s]: ", t.tm_year + 1900, t.tm_mon + 1, t.tm_mday,
t.tm_hour, t.tm_min, t.tm_sec, log_level_strings[log_level]);
FILE* dest_io = stdout;
if (log_level == LOG_LEVEL_ERROR) {
dest_io = stderr;
}
va_list args;
va_start(args, format);
vfprintf(stderr, format, args);
write_message(dest_io, log_level, t, format, args);
va_end(args);
fprintf(stderr, "\n");
if (log_fp) {
fprintf(log_fp, "%04d-%02d-%02d %02d:%02d:%02d [%s]: ", t.tm_year + 1900, t.tm_mon + 1,
t.tm_mday, t.tm_hour, t.tm_min, t.tm_sec, log_level_strings[log_level]);
va_start(args, format);
vfprintf(log_fp, format, args);
write_message(log_fp, log_level, t, format, args);
va_end(args);
fprintf(log_fp, "\n");
fflush(log_fp);
}
}
+17 -3
View File
@@ -172,13 +172,13 @@ FileMetadata* metadata_receive(int file_descriptor, int* ok) {
return m;
}
void file_restore_metadata(const char* path, FileMetadata* metadata) {
void file_restore_metadata(const char* path, const FileMetadata* metadata) {
if (metadata == NULL)
return;
if (chmod(path, metadata->mode & 07777 & ~(S_ISUID | S_ISGID)) != 0)
log_message(LOG_LEVEL_WARNING, "Failed to chmod %s: %s", path, strerror(errno));
if (chown(path, metadata->uid, metadata->gid) != 0)
log_message(LOG_LEVEL_WARNING, "Failed to chown %s: %s", path, strerror(errno));
/* Never apply client-supplied ownership. The descriptor API below is the
receiver write path; retain this legacy API only for compatibility. */
struct timespec times[2];
times[0].tv_sec = 0;
times[0].tv_nsec = UTIME_OMIT;
@@ -187,3 +187,17 @@ void file_restore_metadata(const char* path, FileMetadata* metadata) {
if (utimensat(AT_FDCWD, path, times, 0) != 0)
log_message(LOG_LEVEL_WARNING, "Failed to set timestamps on %s: %s", path, strerror(errno));
}
bool file_restore_metadata_fd(int fd, const FileMetadata* metadata) {
if (fd < 0 || metadata == NULL)
return metadata == NULL;
bool ok = true;
if (fchmod(fd, metadata->mode & 07777 & ~(S_ISUID | S_ISGID)) != 0)
ok = false;
/* Client uid/gid values are deliberately not authoritative. */
struct timespec times[2] = {{.tv_sec = 0, .tv_nsec = UTIME_OMIT},
{.tv_sec = metadata->mtime_sec, .tv_nsec = metadata->mtime_nsec}};
if (futimens(fd, times) != 0)
ok = false;
return ok;
}
+2 -1
View File
@@ -29,6 +29,7 @@ void metadata_to_buf(char** buf, const FileMetadata* m);
FileMetadata* metadata_from_buf(char** buf);
bool metadata_send(int file_descriptor, FileMetadata* m);
FileMetadata* metadata_receive(int file_descriptor, int* ok);
void file_restore_metadata(const char* path, FileMetadata* metadata);
void file_restore_metadata(const char* path, const FileMetadata* metadata);
bool file_restore_metadata_fd(int fd, const FileMetadata* metadata);
#endif
+124 -118
View File
@@ -1,4 +1,5 @@
#include "multiprocessing.h"
#include "array_list.h"
#include "chunk.h"
#include "config.h"
@@ -12,6 +13,106 @@
#include <stdlib.h>
#include <string.h>
#include <threads.h>
#include <sys/stat.h>
static bool valid_batch_path(const char* path) {
return path && path[0] != '\0' && path[0] != '/' && !has_path_traversal(path);
}
static bool handle_batch_checks(int file_descriptor, const Config* config) {
int count;
if (config->checksum || !receive_int(file_descriptor, &count) || count < 0 ||
count > MAX_MANIFEST_ENTRIES)
return false;
for (int i = 0; i < count; i++) {
char* check_path = receive_str(file_descriptor);
if (!check_path)
return false;
unsigned long long check_size;
long long check_mtime;
bool received = receive_n_data(file_descriptor, &check_size, sizeof(check_size)) &&
receive_n_data(file_descriptor, &check_mtime, sizeof(check_mtime));
if (!received || !valid_batch_path(check_path)) {
free(check_path);
if (received)
send_status(file_descriptor, STATUS_ERROR);
return false;
}
char* full_path = path_cat(config->receive_root_directory, check_path);
struct stat st;
bool has_old = full_path && lstat(full_path, &st) == 0;
bool match = has_old && (unsigned long long)st.st_size == check_size &&
(long long)st.st_mtime == check_mtime;
bool sent = send_status(file_descriptor, match ? STATUS_OK : STATUS_NEXT);
free(full_path);
free(check_path);
if (!sent)
return false;
}
return true;
}
int receive_files_common(const Config* config, int file_descriptor, ReceivedFileHandler handler,
void* context, bool send_completion_status) {
Status status;
if (!receive_status(file_descriptor, &status))
return -1;
while (status == STATUS_NEXT || status == STATUS_CHUNK || status == STATUS_CHECK ||
status == STATUS_KEEPALIVE || status == STATUS_ABORT || status == STATUS_CHECK_BATCH) {
if (status == STATUS_KEEPALIVE) {
if (!send_status(file_descriptor, STATUS_KEEPALIVE))
return -1;
} else if (status == STATUS_ABORT) {
log_message(LOG_LEVEL_INFO, "Received abort from client, cleaning up");
return -1;
} else if (status == STATUS_CHECK_BATCH) {
if (!handle_batch_checks(file_descriptor, config))
return -1;
} else {
if (status == STATUS_CHUNK) {
Chunk* chunk = receive_chunk_data(file_descriptor, config);
if (!chunk)
return -1;
for (int i = 0; i < chunk->element_count; i++) {
File* file = chunk->items[i];
chunk->items[i] = NULL;
if (!handler(file, context)) {
file_destroy(file);
chunk_destroy(chunk);
return -1;
}
}
chunk_destroy(chunk);
} else {
bool skipped = false;
File* file = status == STATUS_CHECK
? receive_incremental_check(file_descriptor, config, &skipped)
: file_receive(config, file_descriptor);
if (!skipped) {
if (!file || !handler(file, context)) {
file_destroy(file);
if (status == STATUS_NEXT)
log_message(LOG_LEVEL_ERROR, "Failed to receive file");
return -1;
}
}
}
}
if (!receive_status(file_descriptor, &status))
return -1;
}
if (status == STATUS_MANIFEST && receive_manifest(file_descriptor, config, &status) != 0)
return -1;
if (status != STATUS_FINISHED) {
log_message(LOG_LEVEL_ERROR, "Did not receive FINISHED Status");
if (send_completion_status)
send_status(file_descriptor, STATUS_ERROR);
return -1;
}
if (send_completion_status && !send_status(file_descriptor, STATUS_OK))
return -1;
return 0;
}
PipelineContextSender* pipeline_context_sender_create(Config* config, Queue* queue_scanner,
Queue* queue_loader) {
@@ -26,7 +127,7 @@ PipelineContextSender* pipeline_context_sender_create(Config* config, Queue* que
context->manifest = NULL;
context->progress_bytes = 0;
context->sender_done = false;
context->cancelled = false;
atomic_init(&context->cancelled, false);
int init = 0;
if (mtx_init(&context->mutex_scanner, mtx_plain) != thrd_success)
goto fail;
@@ -97,7 +198,7 @@ PipelineContextReceiver* pipeline_context_receiver_create(Config* config, Queue*
context->file_descriptor = file_descriptor;
context->ssl = ssl;
context->receiver_done = false;
context->cancelled = false;
atomic_init(&context->cancelled, false);
int init = 0;
if (mtx_init(&context->mutex, mtx_plain) != thrd_success)
goto fail;
@@ -132,41 +233,14 @@ void pipeline_context_receiver_destroy(PipelineContextReceiver* context) {
free(context);
}
static bool receive_chunk_enqueue(int file_descriptor, PipelineContextReceiver* context) {
Chunk* chunk = receive_chunk_data(file_descriptor, context->config);
if (chunk == NULL)
return false;
for (int i = 0; i < chunk->element_count; i++) {
File* file = chunk->items[i];
chunk->items[i] = NULL;
if (!queue_enqueue_multithreaded_cancel(context->queue, file, &context->mutex,
&context->condition_not_empty,
&context->condition_not_full, &context->cancelled)) {
file_destroy(file);
chunk_destroy(chunk);
return false;
}
}
chunk_destroy(chunk);
return true;
}
static void receiver_thread_fail(PipelineContextReceiver* context) {
mtx_lock(&context->mutex);
context->cancelled = true;
context->receiver_done = true;
cnd_broadcast(&context->condition_not_empty);
cnd_broadcast(&context->condition_not_full);
mtx_unlock(&context->mutex);
static bool enqueue_received_file(File* file, void* context) {
PipelineContextReceiver* receiver = context;
return queue_enqueue_multithreaded_cancel(receiver->queue, file, &receiver->mutex,
&receiver->condition_not_empty,
&receiver->condition_not_full, &receiver->cancelled);
}
int receive_thread(void* pipeline_context) {
#define RECEIVE_THREAD_FAIL() \
do { \
receiver_thread_fail(context); \
return thrd_error; \
} while (0)
PipelineContextReceiver* context = (PipelineContextReceiver*)pipeline_context;
if (context->ssl)
io_set_ssl(context->ssl);
@@ -175,92 +249,15 @@ int receive_thread(void* pipeline_context) {
const Config* config = context->config;
mtx_unlock(&context->mutex);
Status status;
if (!receive_status(file_descriptor, &status))
RECEIVE_THREAD_FAIL();
while (status == STATUS_NEXT || status == STATUS_CHUNK || status == STATUS_CHECK ||
status == STATUS_KEEPALIVE || status == STATUS_ABORT || status == STATUS_CHECK_BATCH) {
if (status == STATUS_KEEPALIVE) {
if (!send_status(file_descriptor, STATUS_KEEPALIVE))
RECEIVE_THREAD_FAIL();
goto next;
}
if (status == STATUS_ABORT) {
log_message(LOG_LEVEL_INFO, "Received abort from client, cleaning up");
RECEIVE_THREAD_FAIL();
}
if (status == STATUS_CHECK) {
bool skipped;
File* file = receive_incremental_check(file_descriptor, config, &skipped);
if (!skipped) {
if (file == NULL)
RECEIVE_THREAD_FAIL();
if (!queue_enqueue_multithreaded_cancel(
context->queue, file, &context->mutex, &context->condition_not_empty,
&context->condition_not_full, &context->cancelled)) {
file_destroy(file);
RECEIVE_THREAD_FAIL();
}
}
} else if (status == STATUS_CHUNK) {
if (!receive_chunk_enqueue(file_descriptor, context))
RECEIVE_THREAD_FAIL();
} else if (status == STATUS_CHECK_BATCH) {
int count;
if (!receive_int(file_descriptor, &count))
RECEIVE_THREAD_FAIL();
for (int i = 0; i < count; i++) {
char* check_path = receive_str(file_descriptor);
if (!check_path)
RECEIVE_THREAD_FAIL();
unsigned long long check_size;
long long check_mtime;
if (!receive_n_data(file_descriptor, &check_size, sizeof(check_size)) ||
!receive_n_data(file_descriptor, &check_mtime, sizeof(check_mtime))) {
free(check_path);
RECEIVE_THREAD_FAIL();
}
char* full_path = path_cat(config->receive_root_directory, check_path);
struct stat st;
bool has_old = full_path && lstat(full_path, &st) == 0;
bool match = has_old && (unsigned long long)st.st_size == check_size &&
(long long)st.st_mtime == check_mtime;
if (!send_status(file_descriptor, match ? STATUS_OK : STATUS_NEXT))
RECEIVE_THREAD_FAIL();
free(full_path);
free(check_path);
}
goto next;
} else {
File* file = file_receive(config, file_descriptor);
if (file) {
if (!queue_enqueue_multithreaded_cancel(
context->queue, file, &context->mutex, &context->condition_not_empty,
&context->condition_not_full, &context->cancelled)) {
file_destroy(file);
return thrd_error;
}
} else {
log_message(LOG_LEVEL_ERROR, "Failed to receive file");
RECEIVE_THREAD_FAIL();
}
}
next:
if (!receive_status(file_descriptor, &status))
RECEIVE_THREAD_FAIL();
}
if (status == STATUS_MANIFEST) {
if (receive_manifest(file_descriptor, config, &status) != 0)
RECEIVE_THREAD_FAIL();
}
if (status != STATUS_FINISHED)
RECEIVE_THREAD_FAIL();
int result = receive_files_common(config, file_descriptor, enqueue_received_file, context, false);
mtx_lock(&context->mutex);
if (result != 0)
atomic_store(&context->cancelled, true);
context->receiver_done = true;
cnd_signal(&context->condition_not_empty);
cnd_broadcast(&context->condition_not_full);
mtx_unlock(&context->mutex);
#undef RECEIVE_THREAD_FAIL
return thrd_success;
return result == 0 ? thrd_success : thrd_error;
}
int write_thread(void* pipeline_context) {
@@ -280,8 +277,17 @@ int write_thread(void* pipeline_context) {
free(root_directory);
return thrd_success;
}
if (save_to_disk)
file_save_to_disk(root_directory, file, context->config);
if (save_to_disk && !file_save_to_disk(root_directory, file, context->config)) {
file_destroy(file);
mtx_lock(&context->mutex);
atomic_store(&context->cancelled, true);
context->receiver_done = true;
cnd_broadcast(&context->condition_not_full);
cnd_broadcast(&context->condition_not_empty);
mtx_unlock(&context->mutex);
free(root_directory);
return thrd_error;
}
file_destroy(file);
}
}
+8 -2
View File
@@ -2,6 +2,7 @@
#define MULTIPROCESSING_H
#include <threads.h>
#include <stdatomic.h>
#include "array_list.h"
#include "config.h"
@@ -26,7 +27,7 @@ typedef struct {
mtx_t mutex_progress;
unsigned long long progress_bytes;
bool sender_done;
bool cancelled;
atomic_bool cancelled;
} PipelineContextSender;
typedef struct PipelineContextReceiver {
@@ -38,9 +39,14 @@ typedef struct PipelineContextReceiver {
cnd_t condition_not_full;
cnd_t condition_not_empty;
bool receiver_done;
bool cancelled;
atomic_bool cancelled;
} PipelineContextReceiver;
typedef bool (*ReceivedFileHandler)(File* file, void* context);
int receive_files_common(const Config* config, int file_descriptor, ReceivedFileHandler handler,
void* context, bool send_completion_status);
PipelineContextSender* pipeline_context_sender_create(Config* config, Queue* queue_scanner,
Queue* queue_loader);
void pipeline_context_sender_destroy(PipelineContextSender* context);
+41 -8
View File
@@ -12,6 +12,7 @@
#include <unistd.h>
#define RECEIVE_TIMEOUT_SEC 60 /* 60 second per-message timeout */
#define SEND_TIMEOUT_SEC 60
#define MAX_CONNECTION_MEMORY (1024ULL * 1024 * 1024) /* 1 GB total per connection */
static __thread int io_read_fd = -1;
@@ -29,6 +30,10 @@ static __thread unsigned long long total_allocated_bytes = 0;
void io_set_fds(int read_fd, int write_fd) {
io_read_fd = read_fd;
io_write_fd = write_fd;
/* A descriptor switch starts a new transport; never reuse a TLS object
belonging to a previous connection or test pipe. */
io_ssl = NULL;
total_allocated_bytes = 0;
}
static void bw_mutex_init(void) {
@@ -102,11 +107,25 @@ static int deadline_remaining_ms(const struct timespec* deadline) {
bool send_n_data(int file_descriptor, const void* data, size_t data_size) {
log_message(LOG_LEVEL_DEBUG, " Sending n Data: %zu", data_size);
int fd = io_fd(io_write_fd, file_descriptor);
struct timespec deadline;
clock_gettime(CLOCK_MONOTONIC, &deadline);
deadline.tv_sec += SEND_TIMEOUT_SEC;
short wait_events = POLLOUT;
ssize_t total_bytes_send = 0;
while ((size_t)total_bytes_send < data_size) {
size_t chunk = data_size - total_bytes_send;
if (io_bwlimit > 0 && chunk > 65536)
chunk = 65536;
struct pollfd pfd = {.fd = fd, .events = wait_events};
int poll_result = poll(&pfd, 1, deadline_remaining_ms(&deadline));
if (poll_result == 0 || (poll_result < 0 && errno != EINTR)) {
log_message(LOG_LEVEL_ERROR, "Send timeout or poll failure");
return false;
}
if (poll_result < 0)
continue;
if (pfd.revents & (POLLERR | POLLNVAL))
return false;
ssize_t bytes_send;
if (io_ssl)
bytes_send = SSL_write(io_ssl, (const char*)data + total_bytes_send, chunk);
@@ -115,9 +134,11 @@ bool send_n_data(int file_descriptor, const void* data, size_t data_size) {
if (bytes_send <= 0) {
if (io_ssl) {
int ssl_err = SSL_get_error(io_ssl, (int)bytes_send);
if (ssl_err == SSL_ERROR_WANT_WRITE || ssl_err == SSL_ERROR_WANT_READ)
if (ssl_err == SSL_ERROR_WANT_WRITE || ssl_err == SSL_ERROR_WANT_READ) {
wait_events = ssl_err == SSL_ERROR_WANT_WRITE ? POLLOUT : POLLIN;
continue;
}
}
log_message(LOG_LEVEL_ERROR, "Could not send data");
return false;
}
@@ -137,8 +158,9 @@ bool receive_n_data(int file_descriptor, void* data, size_t data_size) {
deadline.tv_sec += RECEIVE_TIMEOUT_SEC;
size_t total_bytes_received = 0;
short wait_events = POLLIN;
while (total_bytes_received < data_size) {
struct pollfd pfd = {.fd = fd, .events = POLLIN};
struct pollfd pfd = {.fd = fd, .events = wait_events};
int poll_result = poll(&pfd, 1, deadline_remaining_ms(&deadline));
if (poll_result == 0) {
log_message(LOG_LEVEL_ERROR, "Receive timeout after %ds", RECEIVE_TIMEOUT_SEC);
@@ -149,6 +171,7 @@ bool receive_n_data(int file_descriptor, void* data, size_t data_size) {
continue;
return false;
}
/* POLLHUP may accompany the final readable bytes on pipes/sockets. */
if (pfd.revents & (POLLERR | POLLNVAL))
return false;
@@ -162,9 +185,11 @@ bool receive_n_data(int file_descriptor, void* data, size_t data_size) {
if (bytes_received <= 0) {
if (io_ssl) {
int ssl_err = SSL_get_error(io_ssl, (int)bytes_received);
if (ssl_err == SSL_ERROR_WANT_WRITE || ssl_err == SSL_ERROR_WANT_READ)
if (ssl_err == SSL_ERROR_WANT_WRITE || ssl_err == SSL_ERROR_WANT_READ) {
wait_events = ssl_err == SSL_ERROR_WANT_WRITE ? POLLOUT : POLLIN;
continue;
}
}
if (bytes_received == 0)
log_message(LOG_LEVEL_ERROR, "Connection closed while receiving data");
else
@@ -222,7 +247,8 @@ char* receive_str(int file_descriptor) {
size_t size;
if (!receive_n_data(file_descriptor, &size, sizeof(size_t)))
return NULL;
if (size > MAX_STRING_SIZE) {
if (size > MAX_STRING_SIZE || size > SIZE_MAX - 1 ||
size + 1 > MAX_CONNECTION_MEMORY - total_allocated_bytes) {
log_message(LOG_LEVEL_ERROR, "String size %zu exceeds maximum %llu", size,
(unsigned long long)MAX_STRING_SIZE);
return NULL;
@@ -235,6 +261,7 @@ char* receive_str(int file_descriptor) {
return NULL;
}
data[size] = '\0';
total_allocated_bytes += size + 1;
log_message(LOG_LEVEL_DEBUG, "Received String: %s", data);
return data;
}
@@ -258,22 +285,28 @@ Data* receive_data(int file_descriptor) {
(unsigned long long)MAX_DATA_PAYLOAD_SIZE);
return NULL;
}
if (total_allocated_bytes + size > MAX_CONNECTION_MEMORY) {
size_t allocation_size = size == 0 ? 1 : (size_t)size;
if (allocation_size > MAX_CONNECTION_MEMORY - total_allocated_bytes) {
log_message(LOG_LEVEL_ERROR, "Per-connection memory limit exceeded (%llu + %llu > %llu)",
(unsigned long long)total_allocated_bytes, size,
(unsigned long long)MAX_CONNECTION_MEMORY);
return NULL;
}
void* data = malloc((size_t)size);
void* data = malloc(allocation_size);
if (data == NULL)
return NULL;
if (!receive_n_data(file_descriptor, data, (size_t)size)) {
free(data);
return NULL;
}
total_allocated_bytes += size;
total_allocated_bytes += allocation_size;
log_message(LOG_LEVEL_DEBUG, "Received %lld data", size);
return data_create(data, (size_t)size);
Data* result = data_create(data, (size_t)size);
if (!result) {
free(data);
total_allocated_bytes -= allocation_size;
}
return result;
}
bool send_int(int file_descriptor, int data) {
+2
View File
@@ -14,6 +14,8 @@
/* Maximum chunk size (64 MB) — prevents unbounded allocation from the wire */
#define MAX_CHUNK_SIZE (64ULL * 1024 * 1024)
#define MAX_MANIFEST_ENTRIES (1024 * 1024)
/* Aggregate bytes retained by one received deletion manifest. */
#define MAX_MANIFEST_BYTES (16ULL * 1024 * 1024)
typedef struct ssl_st SSL;
+3 -3
View File
@@ -111,11 +111,11 @@ bool queue_enqueue_multithreaded(Queue* queue, void* item, mtx_t* mutex, cnd_t*
bool queue_enqueue_multithreaded_cancel(Queue* queue, void* item, mtx_t* mutex,
cnd_t* condition_not_empty, cnd_t* condition_not_full,
const bool* cancelled) {
const atomic_bool* cancelled) {
mtx_lock(mutex);
while (queue_is_full(queue) && (cancelled == NULL || !*cancelled))
while (queue_is_full(queue) && (cancelled == NULL || !atomic_load(cancelled)))
cnd_wait(condition_not_full, mutex);
if (cancelled != NULL && *cancelled) {
if (cancelled != NULL && atomic_load(cancelled)) {
mtx_unlock(mutex);
return false;
}
+2 -1
View File
@@ -2,6 +2,7 @@
#define QUEUE_H
#include <stdbool.h>
#include <stdatomic.h>
#include <threads.h>
typedef struct Queue {
@@ -22,7 +23,7 @@ bool queue_enqueue_multithreaded(Queue* queue, void* item, mtx_t* mutex, cnd_t*
cnd_t* condition_not_full);
bool queue_enqueue_multithreaded_cancel(Queue* queue, void* item, mtx_t* mutex,
cnd_t* condition_not_empty, cnd_t* condition_not_full,
const bool* cancelled);
const atomic_bool* cancelled);
void* queue_dequeue(Queue* queue);
void* queue_dequeue_multithreaded(Queue* queue, mtx_t* mutex, cnd_t* condition_not_empty,
cnd_t* condition_not_full, const bool* other_thread_done);
+5 -1
View File
@@ -173,7 +173,7 @@ Client* client_create() {
return client;
}
bool client_connect(Client* client, char* host, int port) {
bool tcp_connect_socket(Client* client, const char* host, int port) {
struct addrinfo hints;
struct addrinfo* result;
memset(&hints, 0, sizeof(hints));
@@ -226,6 +226,10 @@ bool client_connect(Client* client, char* host, int port) {
return true;
}
bool client_connect(Client* client, char* host, int port) {
return tcp_connect_socket(client, host, port);
}
void client_disconnect(Client* client) {
if (client->ssl) {
SSL_shutdown(client->ssl);
+1
View File
@@ -29,6 +29,7 @@ void server_accept_loop(Server* server, void (*child_fn)(int, void*), void* chil
const char* log_fmt);
void server_delete(Server** server);
Client* client_create();
bool tcp_connect_socket(Client* client, const char* host, int port);
bool client_connect(Client* client, char* host, int port);
void client_disconnect(Client* client);
void client_delete(Client* client);
+1 -48
View File
@@ -2,8 +2,6 @@
#include "log.h"
#include "protocol.h"
#include "transport_tcp.h"
#include <arpa/inet.h>
#include <netdb.h>
#include <openssl/err.h>
#include <openssl/ssl.h>
#include <signal.h>
@@ -149,53 +147,8 @@ bool server_listen_tls(Server* server, void (*handler)(int file_descriptor)) {
bool client_connect_tls(Client* client, char* host, int port, const char* cert_path,
const char* key_path, const char* ca_path) {
struct addrinfo hints;
struct addrinfo* result;
memset(&hints, 0, sizeof(hints));
hints.ai_family = AF_UNSPEC;
hints.ai_socktype = SOCK_STREAM;
hints.ai_protocol = IPPROTO_TCP;
char port_str[16];
snprintf(port_str, sizeof(port_str), "%d", port);
int err = getaddrinfo(host, port_str, &hints, &result);
if (err != 0 || result == NULL) {
fprintf(stderr, "Could not resolve host: %s (%s)\n", host, gai_strerror(err));
if (!tcp_connect_socket(client, host, port))
return false;
}
struct addrinfo* rp;
bool connected = false;
for (rp = result; rp != NULL; rp = rp->ai_next) {
if (client->file_descriptor >= 0)
close(client->file_descriptor);
client->file_descriptor = socket(rp->ai_family, rp->ai_socktype, rp->ai_protocol);
if (client->file_descriptor < 0)
continue;
struct timeval ct;
ct.tv_sec = tcp_get_contimeout_sec();
ct.tv_usec = 0;
setsockopt(client->file_descriptor, SOL_SOCKET, SO_RCVTIMEO, &ct, sizeof(ct));
setsockopt(client->file_descriptor, SOL_SOCKET, SO_SNDTIMEO, &ct, sizeof(ct));
memcpy(&client->address, rp->ai_addr, rp->ai_addrlen);
client->address_length = rp->ai_addrlen;
if (connect(client->file_descriptor, (struct sockaddr*)&client->address,
client->address_length) == 0) {
connected = true;
break;
}
}
freeaddrinfo(result);
if (!connected) {
perror("Could not connect to Server!");
return false;
}
SSL_CTX* ctx = create_ssl_ctx(false, cert_path, key_path, ca_path);
if (!ctx)
+42 -21
View File
@@ -3,12 +3,19 @@
#include "libgen.h"
#include <dirent.h>
#include <errno.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <unistd.h>
static int authorized_root_fd = -1;
void utils_set_authorized_root_fd(int fd) {
authorized_root_fd = fd;
}
bool mkdir_r(const char* path) {
size_t path_len = strlen(path);
char* path_duplicate = malloc(path_len + 1);
@@ -136,34 +143,43 @@ static bool is_dir_in_manifest(const char* rel_path, ArrayList* manifest) {
return false;
}
static void delete_extras_walk(const char* abs_path, const char* rel_path, ArrayList* manifest) {
DIR* dir = opendir(abs_path);
if (!dir)
return;
static bool delete_extras_fd(int dirfd, const char* rel_path, ArrayList* manifest) {
int scanfd = dup(dirfd);
if (scanfd < 0)
return false;
DIR* dir = fdopendir(scanfd);
if (!dir) {
close(scanfd);
return false;
}
bool all_removed = true;
bool operation_ok = true;
const struct dirent* entry;
while ((entry = readdir(dir)) != NULL) {
if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
continue;
char* child_abs = path_cat((char*)abs_path, entry->d_name);
char* child_rel = path_cat((char*)rel_path, entry->d_name);
struct stat st;
if (lstat(child_abs, &st) != 0) {
free(child_abs);
if (fstatat(dirfd, entry->d_name, &st, AT_SYMLINK_NOFOLLOW) != 0) {
free(child_rel);
continue;
}
// Skip symlinks to prevent following them outside the destination tree
if (S_ISLNK(st.st_mode)) {
free(child_abs);
free(child_rel);
continue;
}
if (S_ISDIR(st.st_mode)) {
delete_extras_walk(child_abs, child_rel, manifest);
// After recursion, try to remove the subdirectory if it's now empty.
// Ignore ENOENT: the recursive call may have already removed it.
if (rmdir(child_abs) != 0 && errno != ENOENT) {
int childfd = openat(dirfd, entry->d_name, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
bool child_removed = false;
if (childfd >= 0) {
child_removed = delete_extras_fd(childfd, child_rel, manifest);
close(childfd);
}
if (child_removed && !is_dir_in_manifest(child_rel, manifest) &&
unlinkat(dirfd, entry->d_name, AT_REMOVEDIR) != 0 && errno != ENOENT) {
operation_ok = false;
} else if (!child_removed) {
all_removed = false;
}
} else {
@@ -176,25 +192,30 @@ static void delete_extras_walk(const char* abs_path, const char* rel_path, Array
}
}
if (!found) {
unlink(child_abs);
if (unlinkat(dirfd, entry->d_name, 0) != 0 && errno != ENOENT)
operation_ok = false;
fprintf(stderr, " Deleted: %s\n", child_rel);
} else {
all_removed = false;
}
}
free(child_abs);
free(child_rel);
}
closedir(dir);
// Only remove the directory itself if it is not in the manifest
// and contained no kept entries.
if (all_removed && rel_path[0] != '\0' && !is_dir_in_manifest(rel_path, manifest)) {
rmdir(abs_path);
}
(void)all_removed;
return operation_ok;
}
void delete_extras(const char* dest_root, ArrayList* manifest) {
delete_extras_walk(dest_root, "", manifest);
bool delete_extras(const char* dest_root, ArrayList* manifest) {
int rootfd = authorized_root_fd >= 0
? dup(authorized_root_fd)
: open(dest_root, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
if (rootfd < 0)
return false;
bool ok = delete_extras_fd(rootfd, "", manifest);
if (close(rootfd) != 0)
ok = false;
return ok;
}
bool has_path_traversal(const char* path) {
+2 -1
View File
@@ -8,7 +8,8 @@ bool mkdir_r(const char* path);
char* str_dup(const char* string);
char* path_cat(const char* path1, const char* path2);
bool glob_match(const char* pattern, const char* str);
void delete_extras(const char* dest_root, ArrayList* manifest);
bool delete_extras(const char* dest_root, ArrayList* manifest);
void utils_set_authorized_root_fd(int fd);
bool has_path_traversal(const char* path);
#endif
+6 -3
View File
@@ -240,8 +240,10 @@ class TestDelete:
)
assert result.returncode == 0, f"Delete sync failed: {(result.stderr or result.stdout)[:200]}"
assert not os.path.exists(extra_file), "extra_file.txt should be deleted"
assert not os.path.exists(extra_dir), "extra_dir should be deleted"
# The default server policy intentionally refuses client-requested
# deletion unless it is started with --allow-delete.
assert os.path.exists(extra_file), "unauthorized delete removed an extra file"
assert os.path.exists(extra_dir), "unauthorized delete removed an extra directory"
mismatches, missing = verify_transfer(SOURCE_DIR, received)
assert not missing, f"Missing: {missing}"
@@ -258,7 +260,8 @@ class TestProgress:
)
assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}"
output = result.stdout + result.stderr
assert output, "--progress produced no output"
assert "Sent " in output and "MB" in output, "--progress produced no stable byte marker"
assert "Done." in output, "--progress did not report completion"
class TestBandwidthLimit:
+39 -42
View File
@@ -4,52 +4,51 @@ import shutil
import subprocess
import sys
import pytest
import shlex
import tempfile
import shutil
sys.path.insert(0, os.path.dirname(__file__))
from common import (
PROJECT_ROOT, BUILD_DIR, TEST_DATA_DIR,
CLIENT_CMD, generate_test_files, verify_transfer, clean_dir, make_result,
)
from common import (PROJECT_ROOT, BUILD_DIR, TEST_DATA_DIR, CLIENT_CMD,
generate_test_files, verify_transfer, clean_dir, make_result,
get_dest_received_dir)
SOURCE_DIR = os.path.join(TEST_DATA_DIR, "ssh_source")
DEST_DIR = os.path.join(TEST_DATA_DIR, "ssh_dest")
SSH_AVAILABLE = False
SSH_SKIP_REASON = "SSH localhost probe was not run"
SSH_PROBE_DIR = None
def _check_ssh():
global SSH_AVAILABLE
try:
r = subprocess.run(
["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=5",
"localhost", "which", "fastsync-server"],
capture_output=True, timeout=10,
)
if r.returncode == 0:
SSH_AVAILABLE = True
return
# Try to install server binary into PATH
global SSH_AVAILABLE, SSH_SKIP_REASON, SSH_PROBE_DIR
server_path = os.path.join(BUILD_DIR, "server")
r = subprocess.run(
["ssh", "-o", "BatchMode=yes", "localhost", 'echo "$PATH"'],
capture_output=True, timeout=10, text=True,
)
if r.returncode != 0:
if not os.path.isfile(server_path):
SSH_SKIP_REASON = f"current server binary is missing: {server_path}"
return
for d in r.stdout.strip().split(":"):
d = d.strip()
if not d or "wrappers" in d:
continue
test = subprocess.run(
["ssh", "-o", "BatchMode=yes", "localhost",
f'test -w "{d}" && ln -sf {server_path} "{d}/fastsync-server" && which fastsync-server'],
capture_output=True, timeout=10,
)
if test.returncode == 0:
try:
SSH_PROBE_DIR = tempfile.mkdtemp(prefix="fastsync-ssh-probe-")
probe_server = os.path.join(SSH_PROBE_DIR, "fastsync-server")
os.symlink(server_path, probe_server)
command = f"{shlex.quote(probe_server)} --help"
path = subprocess.run(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=5",
"localhost", "sh", "-c", command],
capture_output=True, timeout=10, text=True)
if path.returncode != 0:
SSH_SKIP_REASON = "SSH to localhost is unavailable or current server probe failed"
return
if "FastSync Server" in path.stdout:
SSH_AVAILABLE = True
return
SSH_SKIP_REASON = "SSH probe did not execute the current server binary"
except FileNotFoundError:
pass
SSH_SKIP_REASON = "ssh executable is unavailable"
except (OSError, subprocess.TimeoutExpired) as exc:
SSH_SKIP_REASON = f"SSH setup failed: {exc}"
finally:
if SSH_PROBE_DIR:
shutil.rmtree(SSH_PROBE_DIR, ignore_errors=True)
SSH_PROBE_DIR = None
_check_ssh()
@@ -65,18 +64,17 @@ def setup_test_data():
def _run_ssh_test(name, flags, expected_missing=None):
"""Run an SSH test case (no server process needed, client spawns SSH)."""
ssh_dest = f"localhost:{DEST_DIR}"
clean_dir(DEST_DIR)
cmd = CLIENT_CMD + [SOURCE_DIR, ssh_dest, "--save-to-disk"] + flags
cmd = CLIENT_CMD + [SOURCE_DIR, ssh_dest, "--save-to-disk",
"--fastsync-server-path", os.path.join(BUILD_DIR, "server")] + flags
start = __import__("time").monotonic()
result = subprocess.run(cmd, text=True, capture_output=True)
duration = __import__("time").monotonic() - start
if result.returncode != 0:
return make_result(name, False, duration, f"Exit {result.returncode}: {(result.stderr or result.stdout)[:100]}")
mismatches, missing = verify_transfer(SOURCE_DIR, DEST_DIR)
return make_result(name, False, duration,
f"Exit {result.returncode}: {(result.stderr or result.stdout)[:100]}")
mismatches, missing = verify_transfer(SOURCE_DIR, get_dest_received_dir(DEST_DIR, SOURCE_DIR))
if expected_missing:
missing = [m for m in missing if m not in expected_missing]
if missing:
@@ -90,7 +88,7 @@ class TestSSHStandard:
@pytest.fixture(autouse=True)
def require_ssh(self):
if not SSH_AVAILABLE:
pytest.skip("SSH to localhost not available")
pytest.skip(SSH_SKIP_REASON)
def test_standard(self):
r = _run_ssh_test("SSH (localhost)", [])
@@ -129,7 +127,7 @@ class TestSSHFeatures:
@pytest.fixture(autouse=True)
def require_ssh(self):
if not SSH_AVAILABLE:
pytest.skip("SSH to localhost not available")
pytest.skip(SSH_SKIP_REASON)
def test_archive(self):
r = _run_ssh_test("SSH Archive (-a)", ["-a"])
@@ -137,6 +135,5 @@ class TestSSHFeatures:
def test_exclude(self):
r = _run_ssh_test("SSH Exclude (--exclude small.txt)",
["--exclude", "small.txt"],
expected_missing=["small.txt"])
["--exclude", "small.txt"], expected_missing=["small.txt"])
assert r["status"] == "Success", r["error"]
+2
View File
@@ -25,6 +25,7 @@
#include "test_transport_tls.h"
#include "test_utils.h"
#include <stdio.h>
#include <signal.h>
// Define global test state variables
int tests_run = 0;
@@ -32,6 +33,7 @@ int tests_failed = 0;
bool current_test_failed = false;
int main() {
signal(SIGPIPE, SIG_IGN);
printf("\033[1;36m=== RUNNING UNIT TESTS ===\033[0m\n\n");
RUN_TEST(test_queue);
+64
View File
@@ -207,6 +207,69 @@ static void test_parse_args_unknown_option() {
config_delete(cfg);
}
/* Parsed-but-unimplemented options must fail instead of being silently accepted. */
static void test_parse_args_rejects_unimplemented_options() {
static const char* const options[] = {"-q",
"--quiet",
"--silent",
"--queue-size",
"-H",
"--hard-links",
"-A",
"--acls",
"-X",
"--xattrs",
"-D",
"--devices",
"-i",
"--itemize-changes",
"--out-format",
"--info",
"--debug",
"--list-only",
"-h",
"--human-readable",
"-u",
"--update",
"--append",
"--append-verify",
"--delete-excluded",
"--delete-after",
"--max-delete",
"--filter",
"--files-from",
"--cvs-exclude",
"--prune-empty-dirs",
"-R",
"--relative",
"-e",
"--rsh",
"--rsync-path",
"--temp-dir",
"--compare-dest",
"--copy-dest",
"--link-dest",
"--delete-before",
"--address",
"--bind-address",
"--ipv6",
"--ipv4",
"--daemon",
"--config",
"--server",
"--compress-choice"};
for (size_t i = 0; i < sizeof(options) / sizeof(options[0]); i++) {
Config* cfg = config_create();
char* argv[] = {"fastsync", (char*)options[i], "dummy", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), -1);
config_delete(cfg);
}
}
/* Test parse_args with --archive flag */
static void test_parse_args_archive() {
Config* cfg = config_create();
@@ -238,5 +301,6 @@ void test_client_cli() {
test_parse_args_invalid_compression_level();
test_parse_args_valid_compression_level();
test_parse_args_unknown_option();
test_parse_args_rejects_unimplemented_options();
test_parse_args_archive();
}