diff --git a/README.md b/README.md index 2691485..3eeabcb 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -#FastSync +# FastSync FastSync is a high-performance file synchronization tool designed to become a drop-in replacement for common `rsync` workflows. It keeps the familiar @@ -7,7 +7,7 @@ multithreading, streaming zstd compression, chunking, zero-copy TCP transfers, and native TCP/TLS transports. The release version is FastSync's client/server protocol version (printed by -`fastsync --version`); client and server must match. See +`./build/client --version`); client and server must match. See [CHANGELOG.md](CHANGELOG.md) for the history. The compatibility target is straightforward: @@ -51,8 +51,9 @@ replacement for every rsync feature or protocol mode. - Rsync-style source and destination arguments. - SSH transport using `user@host:destination` paths below the remote authorized root. - TCP client/server transfers. -- Dry runs, excludes, includes, size filters, backups, statistics, and - bandwidth limiting. +- Dry runs (server-contacting since protocol 2.21.0 for server-routed targets), + excludes, includes, size filters, backups, statistics, and bandwidth + limiting. - Incremental size/mtime checks and optional xxHash64 content checks. - FastSync-native delta transfer for changed files. - Optional mode and timestamp preservation. @@ -64,11 +65,21 @@ replacement for every rsync feature or protocol mode. - The FastSync wire protocol is not the rsync wire protocol. - SSH mode requires `fastsync-server` on the remote host. -- Archive mode does not yet provide all of rsync's `-rlptgoD` behavior. -- Symlink transfer is incomplete; link targets are not yet recreated in all - modes. -- Owner/group, ACL, xattr, and hard-link handling is incomplete or - unavailable. +- Archive mode covers rsync's `-rlptD` behavior — links, permissions, times, + devices, and special files — and does not imply compression or multithreading + (see [Client](#client)). Owner/group (`-o`/`-g`) are NOT implied; identity is + applied only through the opt-in identity flags (`--chown`/`--usermap`/ + `--groupmap`/`--numeric-ids`/`--copy-as`). +- Symlink transfer recreates only relative, `..`-free link targets + (`-l`/`--links`); an absolute target or any target containing a `..` component + is dropped rather than created, even if it would resolve within the receive + root. This containment check is skipped under `--trust-sender`. +- Hard links (`-H`/`--hard-links`), extended attributes (`-X`/`--xattrs`), and + POSIX ACLs (`-A`/`--acls`) are preserved; owner/group is applied only through + the opt-in identity flags (`--chown`/`--usermap`/`--groupmap`/`--numeric-ids`/ + `--copy-as`) and only when the receiver has permission. See + [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md) for the exact semantics and documented + divergences. - Device and special-file preservation is implemented with documented divergences: recreated device nodes require `CAP_MKNOD` on the receiver (a non-root receiver skips the entry), and sockets cannot be recreated (FIFOs @@ -80,9 +91,8 @@ replacement for every rsync feature or protocol mode. the write atomic (temp + rename). With `--partial`, a failed/interrupted write now retains the already-written temp at the destination path (best-effort) so a later `--append`/`--append-verify` run can resume it. -- `--dirs` is not implemented. Its compatibility aliases `--old-dirs` and - `--old-d` are recognized but rejected explicitly rather than silently using - FastSync's recursive directory behavior. +- `-d`/`--dirs` and its aliases `--old-dirs`/`--old-d` transfer the named + directory entries without recursing into their contents. - Short-option names are now rsync-parity (Phase 7 Wave A): FastSync's former collisions were renamed (`-j`/`--threads`, `--preserve`, `--sendfile`, `--chunk-serialization`, `--timeout`, `--ssh-port`), so `-m`, `-M`, `-f`, @@ -96,7 +106,12 @@ partial, alternate, and planned behavior. ### Build -`compile_commands.json` is a symlink to `build/compile_commands.json` and is used by clangd/editor tooling; its target is generated by the build, so it dangles until the first build. +```bash +cmake -B build -S . +cmake --build build -j$(nproc) +``` + +This produces `./build/client` and `./build/server`. `compile_commands.json` is a symlink to `build/compile_commands.json` and is used by clangd/editor tooling; its target is generated by the build, so it dangles until the first build. ### Client @@ -105,53 +120,101 @@ partial, alternate, and planned behavior. | Positional | ` ` — automatic SSH detection if dest contains `:` | | `-c, --checksum` | Verify content by checksum instead of size+mtime | | `-z, --compress [level]` | Enable streaming zstd compression (level 1–22, default 5) | -| `-a, --archive` | rsync archive mode (`-rlptgoD`): links, metadata, devices and specials (not compression/multithreading) | +| `-a, --archive` | rsync archive mode (`-rlptD`): links, metadata, devices and specials; owner/group (`-o`/`-g`) are not implied and stay opt-in via the identity flags (not compression/multithreading) | | `-j, --threads[=N]` | Multithreading mode; `N` (1–256) sets the parallel scanner worker count, bare `-j`/`--threads` uses the default | | `-m` | rsync `--prune-empty-dirs` (short form now rsync-parity) | +| `-d, --dirs` | Transfer the named directory entries without recursing into their contents; aliases `--old-dirs`/`--old-d` | +| `-R, --relative` | With `--files-from`, preserve each listed entry's relative path below the destination root | | `--chunk-serialization` | Chunk serialization (batch all files per chunk; long form only) | | `-s` | rsync `--secluded-args` compatibility no-op (remote SSH argv is already injection-safe) | | `--sendfile` | Sendfile zero-copy. Incompatible with compression / chunk serialization. TCP only. Long form only. | -| `--preserve` | Preserve supported file metadata (mode and mtime; ownership and atime are unsupported) | -| `-n, --dry-run` | Scan and print what would be transferred | +| `--preallocate` | Allocate destination file space up front (fail-fast on a full disk) | +| `--append` | Resume a shorter destination by appending only its tail (prefix not verified; requires `--incremental`) | +| `--append-verify` | Like `--append`, but verifies the retained prefix checksum first (falls back to a full transfer on mismatch) | +| `-W, --whole-file` | Transfer changed files without delta processing | +| `-I, --ignore-times` | Transfer files even when size and mtime match | +| `--size-only` | Skip incremental files matching in size, ignoring mtime | +| `--preserve` | Preserve file metadata (mode and mtime; add `-U`/`--atimes` for atime, or an identity flag for owner/group; `-N`/`--crtimes` captures birth time but cannot apply it) | +| `-U, --atimes` | Preserve access times. Captured with the metadata payload; does not enable ownership. | +| `-N, --crtimes` | Capture birth time; cannot be applied (documented divergence) | | `-p, --perms` | Preserve permission bits (part of the metadata bundle) | -| `--ssh-port ` | SSH port (default: 22) | -| `-v, --verbose` | Enable debug logging | -| `-q, --quiet` | Suppress non-error output | -| `--progress` | Show real-time transfer speed | -| `-P` | Enables partial-transfer mode + progress output; interrupted writes retain the already-written temp for resumption | +| `-E, --executability` | Preserve executable permission bits | +| `-X, --xattrs` | Preserve user `user.*` extended attributes | +| `-A, --acls` | Preserve POSIX ACLs | +| `--chmod ` | Modify transferred permissions (rsync syntax) | +| `--chown=USER:GROUP` | Override the ownership of transferred files | +| `--usermap=MAP` | Map usernames when applying ownership | +| `--groupmap=MAP` | Map group names when applying ownership | +| `--numeric-ids` | Apply source numeric uid/gid directly instead of mapping by name | +| `--copy-as=USER[:GROUP]` | Force every written entry to USER[:GROUP] (requires a privileged receiver) | +| `--fake-super` | Record/replay effective metadata via a reserved `user.fastsync.stat` xattr | +| `--super` | Permit the receiver to attempt confined super-user activities (device nodes) | +| `-D` | Preserve device and special files (implies `--devices --specials`) | +| `--devices` | Recreate device nodes on the destination (privileged; skipped without `CAP_MKNOD`) | +| `--specials` | Recreate special files (FIFOs); sockets cannot be recreated | +| `--remove-source-files` | Remove regular source files after a successful transfer | +| `--exclude ` | Exclude files matching glob pattern (repeatable) | +| `--exclude-from ` | Read exclude patterns from a file (one per line) | +| `--include ` | Only transfer files matching glob pattern (repeatable, whitelist) | +| `--include-from ` | Read include patterns from a file | +| `--files-from ` | Read the source file list from FILE (paths relative to the source root) | +| `--max-size ` | Skip files larger than n bytes | +| `--min-size ` | Skip files smaller than n bytes | +| `--max-alloc ` | Maximum single allocation (binary units: B, K, M, G, T, P, E; default 1G) | +| `-u, --update` | Skip files newer than the source on the receiver | +| `--incremental` | Skip files unchanged since last transfer (size + mtime). Auto-enables `--preserve`. Incompatible with `--chunk-serialization`. | +| `--existing` | Skip files not already present at the destination; update existing files normally. | +| `--compare-dest ` | Extra comparison basis: unchanged files are not transferred (requires/implies `--incremental`) | +| `--copy-dest ` | Like `--compare-dest`, but copies the unchanged file from DIR into the destination | +| `--link-dest ` | Like `--copy-dest`, but hard-links the unchanged file from DIR (repeatable; earlier DIRs win) | | `--delete` | Delete files on receiver not present in source (default timing: delete-after, i.e. only after the whole transfer succeeded) | | `--delete-before` | Delete extras before the transfer starts (implies `--delete`) | | `--delete-during`, `--del` | Delete extras once the keep-set is known, before data is applied (implies `--delete`) | | `--delete-delay` | Delete extras only after a successful transfer (implies `--delete`) | | `--delete-after` | Explicit delete-after timing (implies `--delete`) | -| `--exclude ` | Exclude files matching glob pattern (repeatable) | -| `--exclude-from ` | Read exclude patterns from a file (one per line) | -| `--include ` | Only transfer files matching glob pattern (repeatable, whitelist) | -| `--max-size ` | Skip files larger than n bytes | -| `--min-size ` | Skip files smaller than n bytes | -| `--max-alloc ` | Maximum single allocation (binary units: B, K, M, G, T, P, E; default 1G) | -| `--incremental` | Skip files unchanged since last transfer (size + mtime). Auto-enables `--preserve`. Incompatible with `--chunk-serialization`. | -| `--existing` | Skip files not already present at the destination; update existing files normally. | -| `--bwlimit ` | Bandwidth limit in kilobytes per second | -| `--chunk-size ` | Chunk size in bytes (default: 10485760) | -| `--timeout ` | Positive I/O timeout in seconds, applied to both the socket (`SO_RCVTIMEO`/`SO_SNDTIMEO`, built-in default 30 s) and the per-message protocol poll deadline (built-in default 60 s). Omit the option to keep both built-ins; `0` is rejected. The server side keeps the built-in 60 s protocol window (the value is not sent on the wire). | -| `--contimeout ` | Connection timeout in seconds (default: 10) | -| `--backup` | Backup existing destination files before overwriting | -| `--backup-dir ` | Target directory for backups (requires `--backup`) | +| `--delay-updates` | Put updated files into place only at the end of the transfer | +| `-n, --dry-run` | Report what would be transferred without mutating the destination. Since protocol 2.21.0 a server-routed target contacts the receiver and reports would-transfer based on receiver state; a plain local destination keeps the client-side scan. Never mutates or deletes. | +| `-v, --verbose` | Enable debug logging | +| `-q, --quiet` | Suppress non-error output | +| `--progress` | Show real-time transfer speed | +| `-P` | Enables partial-transfer mode + progress output; interrupted writes retain the already-written temp for resumption | | `--stats` | Print transfer statistics at end (bytes, files, timing) | +| `-i, --itemize-changes` | Print an rsync-style per-file change line | +| `--out-format=FORMAT` | Output format for changed files (`%f %n %l %b %M %%`) | +| `--list-only` | List source files instead of transferring | +| `--fsync` | Fsync every written file before publication | | `-h, --human-readable` | Format transfer byte sizes with binary units | | `--max-depth ` | Maximum directory depth to recurse (0 = unlimited, default: 0) | | `--log-file ` | Write log messages to file instead of stderr | +| `--write-batch=FILE` | Run the normal live transfer and also emit a self-contained batch file of the source tree | +| `--only-write-batch=FILE` | Emit the batch file only (no destination, no server) | +| `--read-batch=FILE` | Apply a batch file to the destination (no source, no server) | | `--source-dir ` | Source directory (overrides `FASTSYNC_SOURCE_DIR`) | | `--dest-dir ` | Server destination directory (overrides `FASTSYNC_DEST_DIR`) | | `--save-to-disk` | Write received files to disk | | `--server-host ` | Server IP address (default: `127.0.0.1`) | | `--server-port ` | Server port (default: `8080`) | +| `--ssh-port ` | SSH port (default: 22) | +| `-e, --rsh ` | Remote shell to launch for the SSH transport (default: `ssh`; may include arguments, e.g. `-e "ssh -p 2222"`) | +| `-M, --remote-option=OPT` | Append OPT to the remote server invocation over SSH (repeatable) | +| `--address ` | Bind the outgoing client socket to this source address | +| `-4, --ipv4` | Force IPv4 for destination resolution | +| `-6, --ipv6` | Force IPv6 for destination resolution | +| `--sockopts=OPTS` | Comma-separated OPT=VAL socket options applied before connect (`TCP_NODELAY`, `SO_KEEPALIVE`, `SO_RCVBUF`, `SO_SNDBUF`, `SO_REUSEADDR`) | +| `--bwlimit ` | Bandwidth limit in kilobytes per second | +| `--chunk-size ` | Chunk size in bytes (default: 10485760) | +| `--timeout ` | Positive I/O timeout in seconds, applied to both the socket (`SO_RCVTIMEO`/`SO_SNDTIMEO`, built-in default 30 s) and the per-message protocol poll deadline (built-in default 60 s). Omit the option to keep both built-ins; `0` is rejected. The server side keeps the built-in 60 s protocol window (the value is not sent on the wire). | +| `--contimeout ` | Connection timeout in seconds (default: 10) | +| `--stop-after=MINS` | Stop the transfer after MINS minutes (a positive integer); whatever was already transferred is kept | +| `--stop-at=TIME` | Stop at an absolute time (`HH:MM`, `HH:MM:SS`, or `now+N[smhd]`); an early stop skips the late `--delete` keep-set | +| `--backup` | Backup existing destination files before overwriting | +| `--backup-dir ` | Target directory for backups (requires `--backup`) | | `--tls` | Enable TLS encryption | | `--cert ` | TLS certificate file (PEM) | | `--key ` | TLS private key file (PEM) | | `--ca ` | TLS CA certificate file for verification (PEM) | -| `--client-cn ` | TLS client certificate common name; mandatory with `--tls` (a TLS connection always verifies the client CN) | + +The exhaustive rsync flag matrix is in [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md). **Per-message vs. connection timeouts.** `--timeout` bounds each individual protocol send/receive (the `poll()` deadline), so a peer that stops mid-frame is dropped. It @@ -190,60 +253,64 @@ transfer is never aborted. | `FASTSYNC_SOURCE_DIR` | — | Source directory fallback | | `FASTSYNC_DEST_DIR` | — | Destination directory fallback | | `FASTSYNC_SAVE_TO_DISK` | `false` | Disk persistence fallback | -| `FASTSYNC_SSH_PORT` | `22` | Default SSH port | -| `FASTSYNC_SERVER_HOST` | `127.0.0.1` | Default server host | -| `FASTSYNC_SERVER_PORT` | `8080` | Default server port | -| `FASTSYNC_TLS_CERT` | — | Default TLS certificate path | -| `FASTSYNC_TLS_KEY` | — | Default TLS private key path | -| `FASTSYNC_TLS_CA` | — | Default TLS CA certificate path | ## Implementation Details ### Data Structures -1. **Chunk** — collection of files (~10 MB total by default) -2. **File** — path, content (`Data`), optional `FileMetadata` pointer -3. **FileMetadata** — `mode`, `uid`, `gid`, `mtime_sec`, `mtime_nsec`; -uid / gid are advisory wire fields and are never applied by the receiver; -atime is unsupported -4. **Config** — runtime parameters (transported over wire, TLS settings excluded). Includes `timeout`, `contimeout`, `quiet`, `backup`, `backup_dir`, `stats`, `max_depth`, `log_file`. -5. **Queue** — thread-safe bounded queue with condition variables -6. **DirectoryScanner** — recursive BFS traversal with exclude and include pattern support, max-depth enforcement + +1. **Chunk** — collection of files (~10 MB total by default). +2. **File** — path, content (`Data`), optional `FileMetadata` pointer. +3. **FileMetadata** — `mode`, `uid`, `gid`, `mtime_sec`, `mtime_nsec` (plus + atime/crtime fields). `uid`/`gid` are applied only through the opt-in + identity path; atime is preserved with `-U`/`--atimes`; crtime is captured + but cannot be set on the destination. +4. **Config** — runtime parameters. Most cross the wire (TLS settings + excluded); `backup` and `backup_dir` are in the serialized wire table, while + `timeout`, `contimeout`, `quiet`, `stats`, `max_depth`, and `log_file` are + client-only. +5. **Queue** — thread-safe bounded queue with condition variables. +6. **DirectoryScanner** — recursive BFS traversal with exclude and include + pattern support, max-depth enforcement. ### Key Algorithms -1. **File scanning** — BFS directory traversal; -entries matched against exclude and include patterns, - max - depth enforced 2. * *Chunking ** — files accumulated until `chunk_size` threshold, - then flushed 3. * - *Compression ** — streaming zstd - via `ZSTD_compressStream2` / `ZSTD_decompressStream` 4. * - *Network protocol ** — status - - code - driven exchange with metadata packing, - keep - alive, - and abort support 5. * *Incremental check ** — client sends `STATUS_CHECK` + path + size + - mtime and, - with `--checksum`, XXH64 content checksum; server compares against destination. Can be batched via `STATUS_CHECK_BATCH` for reduced round-trips. -6. **Bandwidth limiting** — token-bucket algorithm with `nanosleep` throttling on 64 KB write chunks -7. **Metadata restoration** — `chmod()`, `chown()`, `utimensat()` on the receiving side -8. **`--delete`** — sender tracks all sent paths; -receiver walks destination tree and removes unlisted files / directories 9. * - *SSH transport * - * — `socketpair()` + `fork()` + `execvp("ssh", - ...)` with `ControlMaster` and port support - 10. * - *TLS transport ** — OpenSSL `SSL_CTX` with TLS - 1.2 minimum, - mutual CA verification, - transparent `SSL_read`/`SSL_write` via `io_set_ssl()` 11. * - *Path traversal protection ** — `has_path_traversal()` rejects any file path - containing `..` components, - preventing directory escape attacks 12. * - *Connection limiting ** — server tracks active connections and rejects - new ones beyond `max_connections` (default 100)13. * - *Keep - - alive ** — idle connections receive periodic `STATUS_KEEPALIVE` to detect half - - open TCP connections 14. * *Abort handling ** — `SIGINT` sets an abort flag; the next protocol operation sends `STATUS_ABORT` for clean server cleanup -15. **Atomic writes** — files are written to a `.tmp` suffix then atomically renamed via `rename()`, preventing partial files -16. **Backup** — before overwriting, existing files are moved to `--backup-dir` (or same directory with `~` suffix) preserving the original + +1. **File scanning** — BFS directory traversal; entries matched against exclude + and include patterns, with max-depth enforced. +2. **Chunking** — files accumulated until the `chunk_size` threshold (default + 10 MiB) is reached, then flushed. +3. **Compression** — streaming zstd via `ZSTD_compressStream2()` / + `ZSTD_decompressStream()`. +4. **Network protocol** — status-code-driven exchange with metadata packing, + keep-alive, and abort support. +5. **Incremental check** — the client sends `STATUS_CHECK` + path + size + + mtime and, with `--checksum`, a whole-file content checksum (xxHash64 by + default, or md5 via `--checksum-choice=md5`/`--cc`, seeded by + `--checksum-seed`); the server compares against the destination. Can be + batched via `STATUS_CHECK_BATCH` for reduced round-trips. +6. **Bandwidth limiting** — token-bucket algorithm with sleep throttling on + 64 KiB write chunks. +7. **Metadata restoration** — mode via `chmod()`/`fchmod()`, times via + `utimensat()`/`futimens()`, and ownership only with an identity flag via + fd-relative `fchown()`/`fchownat()`. +8. **`--delete`** — the sender tracks all sent paths; the receiver walks the + destination tree and removes unlisted files and directories. +9. **SSH transport** — `socketpair()` + `fork()` + `execvp("ssh", ...)` with + `ControlMaster` and port support. +10. **TLS transport** — OpenSSL `SSL_CTX` with TLS 1.2 minimum, mutual CA + verification, and transparent `SSL_read()`/`SSL_write()` via + `io_set_ssl()`. +11. **Path traversal protection** — `has_path_traversal()` rejects any file + path containing `..` components, preventing directory escape attacks. +12. **Connection limiting** — the server tracks active connections and rejects + new ones beyond `max_connections` (default 100). +13. **Keep-alive** — idle connections receive periodic `STATUS_KEEPALIVE` to + detect half-open TCP connections. +14. **Abort handling** — `SIGINT` sets an abort flag; the next protocol + operation sends `STATUS_ABORT` for clean server cleanup. +15. **Atomic writes** — files are written to a `.tmp` suffix then atomically + renamed via `rename()`, preventing partial files. +16. **Backup** — before overwriting, existing files are moved to `--backup-dir` + (or the same directory with a `~` suffix), preserving the original. ## Security Features @@ -300,7 +367,8 @@ cmake --build build -j$(nproc) ### SSH transfer -The remote host must have `fastsync-server` available in `PATH`, or use +The remote host must have `fastsync-server` available in `PATH` (install or +copy the built `./build/server` there as `fastsync-server`), or use `--fastsync-server-path`. SSH starts `fastsync-server --stdio` in its remote working directory, so use a destination below that directory unless the remote server is otherwise configured with a matching authorized root. @@ -341,8 +409,13 @@ Plain TCP requires the explicit `--allow-unauthenticated` server option. Use TLS authenticated network connections. ### TLS transfer + +Server TLS requires `--cert`, `--key`, `--ca`, and `--client-cn`; the client +requires `--cert`, `--key`, and `--ca`. + ```bash -./build/server --destination-root /path/to --tls --cert server.pem --key server-key.pem -p 8443 +./build/server --destination-root /path/to --tls --cert server.pem --key server-key.pem \ + --ca ca.pem --client-cn client -p 8443 ./build/client --tls --cert client.pem --key client-key.pem --ca ca.pem \ --server-host example.com --server-port 8443 \ --source-dir /path/to/source --dest-dir /path/to/destination \ @@ -355,32 +428,32 @@ These examples show the intended rsync-style workflow. Options marked as FastSync-native are optional performance or transport extensions. ```bash -#Basic synchronization +# Basic synchronization ./build/client /source/ /destination/ -#Archive - style synchronization(current FastSync archive behavior) +# Archive-style synchronization (current FastSync archive behavior) ./build/client -a /source/ user@host:destination/ -#Preview a transfer without changing the destination +# Preview a transfer without changing the destination ./build/client -n /source/ /destination/ -#Exclude temporary and object files +# Exclude temporary and object files ./build/client --exclude '*.tmp' --exclude '*.o' \ /source/ user@host:destination/ -#Remove destination entries not present in the source +# Remove destination entries not present in the source ./build/client --delete /source/ user@host:destination/ -#Skip unchanged files using size and modification time +# Skip unchanged files using size and modification time ./build/client --incremental /source/ user@host:destination/ -#Verify content when size and time are not sufficient +# Verify content when size and time are not sufficient ./build/client --incremental --checksum /source/ user@host:destination/ -#Preserve supported mode and timestamp metadata +# Preserve supported mode and timestamp metadata ./build/client --preserve /source/ user@host:destination/ -#Keep backups of overwritten destination files +# Keep backups of overwritten destination files ./build/client --backup --backup-dir backups \ /source/ user@host:destination/ ``` @@ -421,7 +494,8 @@ is `--remote-option`, `-f` is `--filter`, `-s` is `--secluded-args`, `-p` is long-form-only or new shorts: multithreading is `-j`/`--threads`, metadata is `--preserve`, sendfile is `--sendfile`, chunk serialization is `--chunk-serialization`, timeout is `--timeout`, and SSH port is `--ssh-port`. -`-a`/`--archive` is now real rsync archive (`-rlptgoD`). +`-a`/`--archive` is now rsync archive `-rlptD`; owner/group (`-o`/`-g`) are not +implied and remain opt-in via the identity flags. `--secluded-args` (and its short form `-s`) is accepted as a compatibility no-op. It does not change FastSync's transport or protocol behavior, because @@ -433,8 +507,25 @@ remote SSH argv is already built injection-safe. | Option | Description | |---|---| -| `-a`, `--archive` | rsync archive mode (`-rlptgoD`): links, metadata, devices and specials. | -| `-n`, `--dry-run` | Scan and report without writing files. | +| `-a`, `--archive` | rsync archive mode (`-rlptD`): links, metadata, devices and specials; owner/group (`-o`/`-g`) are not implied (opt in via the identity flags). | +| `-n`, `--dry-run` | Report what would be transferred without mutating the destination. Since protocol 2.21.0 a server-routed target contacts the receiver and reports would-transfer based on receiver state; a plain local destination keeps the client-side scan. Never mutates or deletes. | +| `--remove-source-files` | Remove regular source files after a successful transfer. | +| `--incremental` | Skip files matching destination size and mtime. Auto-enables `--preserve`. Incompatible with `--chunk-serialization`. | +| `--checksum` | Include xxHash64 content checks in incremental comparisons. | +| `--size-only` | Skip incremental files matching in size, ignoring mtime. | +| `-I, --ignore-times` | Transfer files even when size and mtime match. | +| `-u, --update` | Skip files newer than the source on the receiver. | +| `-W, --whole-file` | Transfer changed files without delta processing. | +| `-d, --dirs` | Transfer the named directory entries without recursing into their contents (aliases `--old-dirs`/`--old-d`). | +| `-R, --relative` | With `--files-from`, preserve each listed entry's relative path below the destination root. | +| `--files-from ` | Read the source file list from FILE (paths relative to the source root). | +| `--delay-updates` | Put updated files into place only at the end of the transfer. | +| `--compare-dest ` | Extra comparison basis: unchanged files are not transferred (requires/implies `--incremental`). | +| `--copy-dest ` | Like `--compare-dest`, but copies the unchanged file from DIR into the destination. | +| `--link-dest ` | Like `--copy-dest`, but hard-links the unchanged file from DIR (repeatable; earlier DIRs win). | +| `--preallocate` | Allocate destination file space up front (fail-fast on a full disk). | +| `--append` | Resume a shorter destination by appending only its tail (prefix not verified; requires `--incremental`). | +| `--append-verify` | Like `--append`, but verifies the retained prefix checksum first (falls back to a full transfer on mismatch). | | `--delete` | Request removal of destination entries absent from the source. The server must allow deletion. Default timing is delete-after: extras are removed only after the whole transfer succeeded. | | `--delete-before` | Delete extras before the transfer starts (implies `--delete`). | | `--delete-during`, `--del` | Delete extras once the keep-set manifest is known, before data is applied (implies `--delete`; early mode, same engine behaviour as `--delete-before`). | @@ -446,29 +537,49 @@ remote SSH argv is already built injection-safe. | `--include-from ` | Read include patterns from a file. | | `--max-size ` | Skip files larger than the limit. | | `--min-size ` | Skip files smaller than the limit. | -| `--max-depth ` | Limit recursive scanning depth; -zero means unlimited.| | `--incremental` | Skip files matching destination size and mtime.| - | `--checksum` | Include xxHash64 content checks in incremental comparisons.| | `--backup` | - Back up overwritten files.| | `--backup - dir` | Store backups under a separate directory.| -| `--suffix` | Set the backup filename suffix.| | `--partial` | - Select partial - transfer handling. On failed/interrupted writes the - already-written temp file is retained (best-effort) for resumption.| - With `--partial --partial-dir `, completed files are written under the - partial directory and installed atomically. | | `--partial - dir` | - Set a relative partial - transfer directory below the server destination root. - Use with `--partial`. | +| `--max-alloc ` | Maximum single allocation (binary units; default 1G). | +| `--max-depth ` | Limit recursive scanning depth; zero means unlimited. | +| `--backup` | Back up overwritten files. | +| `--backup-dir ` | Store backups under a separate directory (requires `--backup`). | +| `--suffix ` | Set the backup filename suffix (default: `~`). | +| `--partial` | Select partial-transfer handling. On failed/interrupted writes the already-written temp file is retained (best-effort) for resumption. With `--partial --partial-dir `, completed files are written under the partial directory and installed atomically. | +| `--partial-dir ` | Set a relative partial-transfer directory below the server destination root. Use with `--partial`. | | `--inplace` | Write directly to the destination instead of using a temporary file. | +| `--fsync` | Fsync every written file before publication. | +| `--write-batch=FILE` | Run the normal live transfer and also emit a self-contained batch file of the source tree. | +| `--only-write-batch=FILE` | Emit the batch file only (no destination, no server). | +| `--read-batch=FILE` | Apply a batch file to the destination (no source, no server). | +| `--stop-after=MINS` | Stop the transfer after MINS minutes; whatever was already transferred is kept. | +| `--stop-at=TIME` | Stop at an absolute time (`HH:MM`, `HH:MM:SS`, or `now+N[smhd]`). An early stop skips the late `--delete` keep-set. | ### Metadata and links | Option | Description | |---|---| -| `--preserve` | Preserve supported file metadata, currently mode and modification time (long form only). | -| `-l`, `--links` | Request symlink preservation; -link-target transfer remains incomplete. | +| `--preserve` | Preserve mode and mtime (long form only). Add `-U`/`--atimes` for atime, or an identity flag (`--chown`/`--usermap`/`--groupmap`/`--numeric-ids`/`--copy-as`) for owner/group. | +| `-U`, `--atimes` | Preserve access times. Captured with the metadata payload; does not enable ownership. | +| `-N`, `--crtimes` | Capture birth time and transmit it; it cannot be applied because no portable filesystem call can set a birth time (documented divergence). | +| `-p`, `--perms` | Preserve permission bits (part of the metadata bundle). | +| `-E`, `--executability` | Preserve executable permission bits. | +| `-X`, `--xattrs` | Preserve user `user.*` extended attributes. | +| `-A`, `--acls` | Preserve POSIX ACLs. | +| `--chmod ` | Modify transferred permissions (rsync syntax). | +| `--chown=USER:GROUP` | Override the ownership of transferred files (`USER:GROUP`, `USER`, or `:GROUP`). | +| `--usermap=MAP` | Map usernames when applying ownership (comma-separated `FROM:TO` rules). | +| `--groupmap=MAP` | Map group names when applying ownership (same syntax as `--usermap`). | +| `--numeric-ids` | Apply the source numeric uid/gid directly instead of mapping by name. | +| `--copy-as=USER[:GROUP]` | Force every written entry to USER[:GROUP]; requires a privileged receiver. | +| `--fake-super` | Record/replay effective metadata via a reserved `user.fastsync.stat` xattr. | +| `--super` | Permit the receiver to attempt confined super-user activities (device nodes). | +| `--no-super` | Forbid those super-user activities even when the receiver is root. | +| `-l`, `--links` | Copy symlinks as symlinks; the target is transmitted and recreated under the receive root. | | `--copy-links` | Copy symlink referents. | | `--safe-links` | Skip symlinks that point outside the transfer tree. | | `--copy-unsafe-links` | Copy unsafe symlink referents. | +| `-H`, `--hard-links` | Preserve hard-link relationships across the transfer. | +| `-D` | Preserve device and special files (implies `--devices --specials`). | +| `--devices` | Recreate device nodes on the destination (privileged; skipped without `CAP_MKNOD`). | +| `--specials` | Recreate special files (FIFOs); sockets cannot be recreated. | | `-S`, `--sparse` | Sparse-file handling: receiver preserves holes (zero runs are written as holes; no wire change). | ### Output and logging @@ -476,8 +587,12 @@ link-target transfer remains incomplete. | | Option | Description | |---|---| | `-v`, `--verbose` | Enable debug logging. | +| `-q`, `--quiet` | Suppress non-error output. | | `--progress` | Show live transfer progress. | | `--stats` | Print transfer statistics. | +| `-i`, `--itemize-changes` | Print an rsync-style per-file change line. | +| `--out-format=FORMAT` | Output format for changed files (`%f %n %l %b %M %%`). | +| `--list-only` | List source files instead of transferring. | | `--log-file ` | Write log output to a file. | | `-V`, `--version` | Print the FastSync protocol version. | | `--help` | Print command usage. | @@ -487,31 +602,52 @@ link-target transfer remains incomplete. | | Option | Description | |---|---| | `--ssh-port ` | SSH port for the SSH transport (default: 22). Note the short `-p` is now rsync's `--perms`. | +| `-e`, `--rsh ` | Remote shell to launch for the SSH transport (default: `ssh`; may include arguments). | | `--fastsync-server-path ` | Remote FastSync server path for SSH mode. | +| `-M`, `--remote-option=OPT` | Append OPT to the remote server invocation over SSH (repeatable). | | `--source-dir ` | Set the source directory explicitly. | | `--dest-dir ` | Set the destination directory explicitly. | | `--save-to-disk` | Enable server-side disk persistence. | | `--server-host ` | TCP server address. | | `--server-port ` | TCP server port. `--port ` / `--port=` is an alias. | -| `--tls` | Enable TLS. Requires `--cert` and `--key`. | +| `--address ` | Bind the outgoing client socket to this source address. | +| `-4`, `--ipv4` | Force IPv4 for destination resolution. | +| `-6`, `--ipv6` | Force IPv6 for destination resolution. | +| `--sockopts=OPTS` | Comma-separated OPT=VAL socket options applied before connect. | +| `--tls` | Enable TLS. Requires `--cert`, `--key`, and `--ca`. | | `--cert ` | TLS certificate file. | | `--key ` | TLS private key file. | -| `--ca ` | CA file for peer verification. | +| `--ca ` | CA file for peer verification (always required with `--tls`). | ## Server Options | Option | Description | |---|---| | `--stdio` | Serve one SSH connection over standard input/output. | -| `-p ` | TCP listen port. | +| `--daemon` | Run as a persistent daemon listener using a module config file; the daemon default port is 873 (unlike `-p`, which defaults to 8080). | +| `--config=FILE` | Daemon config file (default: `~/.config/fastsync/fastsyncd.conf`, else `/etc/fastsyncd.conf`). Requires `--daemon`. | +| `--dparam=KEY=VALUE` | Override one global config key on the command line. Requires `--daemon`. | +| `--no-detach` | Stay in the foreground (default detaches to the background when running `--daemon`). | +| `-p ` | TCP listen port (default: 8080, range: 1–65535). | | `--tls` | Enable TLS. | -| `--cert ` | TLS certificate file. | -| `--key ` | TLS private key file. | -| `--ca ` | CA file for peer verification. | -| `--destination-root ` | Confine received files to this server-side root; -defaults to the current directory. | +| `--cert ` | TLS certificate file (PEM). | +| `--key ` | TLS private key file (PEM). | +| `--ca ` | CA file for peer verification (PEM). | +| `--client-cn ` | TLS client certificate CN; mandatory with `--tls` (the server verifies the client CN). | +| `--destination-root ` | Confine received files to this server-side root; defaults to the current directory. | +| `--address ` | Bind the listening socket to this address. | +| `-4`, `--ipv4` | Bind an IPv4 socket (default). | +| `-6`, `--ipv6` | Bind an IPv6 socket. | | `--allow-delete` | Permit client delete manifests. Deletion is refused by default. This also gates `--force` (which can recursively replace/remove a destination directory tree). | | `--allow-super` | Standalone TCP listener only: keep super-user activities enabled for a **root** receiver. Without it a root standalone server forces `SUPER_MODE_OFF`, so client `--devices`/`--write-devices`/`--super` and client-chosen ownership requests are skipped/refused. Rejected with `--stdio` (the SSH remote argv is client-composed; use a forced command if the default must hold). No effect when not root. Daemon modules opt in per module with `client owner = yes`. | +| `--trust-sender` | Trust the remote sender's file list: skip the receiver's up-front path-traversal and escaping-symlink-target containment re-validation (fewer checks, faster, potentially unsafe; off by default). | +| `--no-super` | Operator veto: never attempt super-user activities (ownership, device nodes) even as root, and refuse any client `--copy-as`/`--super` request. | +| `--allow-unauthenticated` | Permit plaintext/anonymous network clients; an auth-required module still accepts only opted-in loopback plaintext. | +| `--iconv=LOCAL[,REMOTE]` | Declare this server's LOCAL charset for file-name conversion. | +| `--password-file=FILE` | Credential store for modules that declare `auth users`. Requires `--daemon`. | +| `--early-input=FILE` | Second credential store layered over `--password-file`. Requires `--daemon`. | +| `--hash-credentials ` | Read ``'s `user:password` lines and print PBKDF2 credential-store lines to stdout, then exit. Cannot be combined with `--daemon` or `--stdio`. | +| `--iterations N` | PBKDF2 iteration count for `--hash-credentials` (default 600000, range 100000–10000000). Requires `--hash-credentials`. | | `-v`, `--verbose` | Enable debug logging. | | `--help` | Print server usage. | @@ -540,8 +676,9 @@ and `address`, the global section accepts: - `hosts allow` / `hosts deny` — comma- and/or whitespace-separated host access patterns. -A `[module]` may also set `max connections` (0 = unlimited; enforced per module -across all connection children) and its own `hosts allow`/`hosts deny`. +A `[module]` requires `path`, and may also set `read only`, `client owner`, +`auth users`, `max connections` (0 = unlimited; enforced per module across all +connection children), and its own `hosts allow`/`hosts deny`. The per-host cap and the shared auth lockout identify a source by its numeric peer IP. **Loopback peers (127.0.0.0/8, IPv6 `::1`) are exempt**: every local @@ -647,10 +784,9 @@ mandates `--client-cn`, so a TLS connection to an auth-required module always has its client CN verified (`--client-cn` matches the certificate's CN only, not a subjectAltName, which is acceptable for a private CA). -TLS provides encrypted TCP transport. Supplying `--ca` enables certificate -verification; without it, traffic is encrypted but peer identity is not -verified. Use certificate verification for deployments where authentication -matters. The default TCP transport is not encrypted. +TLS provides encrypted TCP transport. Both the client and the server require +`--ca` together with `--tls`, so peer certificates are always verified +(`SSL_VERIFY_PEER`, depth 4). The default TCP transport is not encrypted. The receiver protects its destination root with path validation, `openat()` directory traversal, `O_NOFOLLOW`, temporary files, and atomic renames. Delete @@ -664,10 +800,12 @@ The project will reach the drop-in replacement goal in stages: and `--option=value` syntax. 2. Add differential tests that compare FastSync and rsync contents, metadata, links, deletes, filters, dry runs, and exit codes. -3. Make `-a` implement the expected recursive, links, permissions, times, - owner/group, and supported special-file behavior. -4. Complete symlink, sparse-file, metadata, delete-policy, and resumable-write - semantics. +3. `-a` now implements the expected recursive, links, permissions, times, and + supported device/special-file behavior; owner/group (`-o`/`-g`) stay opt-in + via the identity flags, and remaining work is the documented + device/special-file divergences. +4. Symlink, sparse-file, metadata, delete-policy, and resumable-write semantics + are implemented; remaining work is the documented edge cases. 5. Add rsync remote-shell and daemon protocol interoperability. 6. Keep FastSync performance options as negotiated, optional extensions. @@ -709,10 +847,13 @@ rsync protocol or filesystem-semantic compatibility. ## Performance Guidance -- Use `-m` for workloads with many files or enough CPU parallelism. -- Use `-c` or `-z` when network bandwidth is more constrained than CPU. +- Use `-j`/`--threads` for workloads with many files or enough CPU parallelism + (`-m` is `--prune-empty-dirs`). +- Use `-z` when network bandwidth is more constrained than CPU (`-c` is + `--checksum`, not a bandwidth option). - Tune `--chunk-size` for file sizes, memory limits, and network latency. -- Use `-f` for large uncompressed TCP transfers where zero-copy I/O helps. +- Use `--sendfile` for large uncompressed TCP transfers where zero-copy I/O + helps (`-f` is `--filter`). - Use `--incremental` to avoid retransmitting unchanged files. - Use `--delta` for changed files when both endpoints are FastSync peers. - Use `--bwlimit` when sharing a link with other traffic. diff --git a/RSYNC_COMPAT.md b/RSYNC_COMPAT.md index 1e28cc5..e9663d9 100644 --- a/RSYNC_COMPAT.md +++ b/RSYNC_COMPAT.md @@ -20,7 +20,7 @@ This document maps rsync's full feature set to FastSync's current implementation | Flag | Rsync Description | FastSync Status | Notes | |------|-------------------|-----------------|-------| -| `-a`, `--archive` | Archive mode is -rlptgoD | ✅ Implemented | Phase 7 Wave A: real rsync archive. `-a`/`--archive` now implies `--links` + metadata (perms/times/group/owner as FastSync's broad bundle) + `--devices` + `--specials`. FastSync is always recursive, so no `-r` is needed. It no longer implies compression or multithreading (those moved to `-z`/`-j`). The short-option namespace is now rsync-parity (see the Phase 7 note) | +| `-a`, `--archive` | Archive mode is -rlptgoD (rsync includes owner/group) | ✅ Implemented | Phase 7 Wave A: real rsync archive. `-a`/`--archive` now implies `--links` + metadata (perms/times) + `--devices` + `--specials`, i.e. **`-rlptD`**. Owner/group (`-o`/`-g`) are **NOT** implied; they require an explicit identity flag (`--numeric-ids`/`--usermap`/`--groupmap`/`--chown`/`--copy-as`). FastSync is always recursive, so no `-r` is needed. It no longer implies compression or multithreading (those moved to `-z`/`-j`). The short-option namespace is now rsync-parity (see the Phase 7 note) | | `-v`, `--verbose` | Increase verbosity | ✅ Implemented | Sets `log_level=DEBUG` | | `-q`, `--quiet` | Suppress non-error messages | ✅ Implemented | Suppresses client output while preserving errors | | `--help` | Show help | ✅ Implemented | Prints usage and exits; `-h` is not accepted | @@ -238,11 +238,11 @@ why plain `--append` works on the normal atomic path, not only with `--inplace`. | Flag | Rsync Description | FastSync Status | Notes | |------|-------------------|-----------------|-------| -| `-M`, `--preserve` | Preserve file metadata | ✅ Implemented | Mode, uid, gid, mtime | +| `-M`, `--preserve` | Preserve file metadata | ✅ Implemented | Carries mode, uid, gid, and mtime on the wire; ownership is applied only via an explicit identity flag (`--numeric-ids`/`--usermap`/`--groupmap`/`--chown`/`--copy-as`) | | `-p`, `--perms` | Preserve permissions | ✅ Implemented | Phase 7 Wave A: `-p`/`--perms` now preserve permission bits, folded into FastSync's broad metadata bundle (`--preserve`); the SSH port moved to `--ssh-port`. rsync-parity short form | -| `-o`, `--owner` | Preserve owner | ✅ Implemented | Part of -M | -| `-g`, `--group` | Preserve group | ✅ Implemented | Part of -M | -| `-t`, `--times` | Preserve modification times | ✅ Implemented | Part of -M | +| `-o`, `--owner` | Preserve owner | ✅ Implemented | Not parsed as a separate flag; owner application is provided by the identity flags only (`--numeric-ids`/`--usermap`/`--groupmap`/`--chown`/`--copy-as`) | +| `-g`, `--group` | Preserve group | ✅ Implemented | Not parsed as a separate flag; group application is provided by the identity flags only (`--numeric-ids`/`--usermap`/`--groupmap`/`--chown`/`--copy-as`) | +| `-t`, `--times` | Preserve modification times | ✅ Implemented | Not parsed separately; modification-time preservation is provided by `--preserve`/`-a` | | `-E`, `--executability` | Preserve executability | ✅ Implemented | Preserves executable permission bits (implies metadata preservation) | | `--chmod=CHMOD` | Affect file permissions | ✅ Implemented | Supports numeric and symbolic `ugo` `rwx` changes; retains receiver safety masking | | `-A`, `--acls` | Preserve ACLs | ✅ Implemented | Implemented on Linux via the POSIX-ACL xattr representation: the sender captures the `system.posix_acl_access` / `system.posix_acl_default` xattrs into the same bounded whitelisted set as `-X`, transmits them per-file, and the receiver re-applies them fd-relative. Setting an ACL the receiver is not permitted to set (non-root on a file it does not own, unsupported filesystem) is logged and skipped, never fatal. libacl is **not** required. Only the `system.posix_acl_*` namespaces plus `user.*` are ever applied; privileged namespaces are never applied (see the Phase-4 xattr/ACL notes below). Implies metadata transmission | @@ -806,7 +806,7 @@ These are the hardest compatibility items because they require durable formats o These are the last compatibility items and the closing phase toward rsync flag parity. Per the project decision: every rsync flag (short **and** long) that is *possible* gets real rsync-parity behavior; anything physically impossible becomes an explicit **Impossible/Divergence** status (accepted for CLI compatibility, safely inert, with coverage tests proving that); and the two privilege flags (`--super`, `--copy-as`) adopt the deliberately-scoped **safe-subset + clear-refusal** model rather than blind elevation. The remaining `⚠️ Partial`, `🔄 Compatibility No-op`, `🔀 Alt Arg`, and `❌ Not Implemented` rows in the Summary are this phase's scope. All Wave A renames are **client-side only** (the wire config fields `use_compression`/`use_metadata`/`use_sendfile`/`use_chunk_serialization` are unchanged), so they require **no `PROTOCOL_VERSION` bump**. -**Wave A — CLI namespace parity (rename colliding FastSync short flags) — ✅ implemented.** This freed the short letters rsync needs and made the three `🔀 Alt Arg` rows real. `-c`→`--checksum`, `-m`→`--prune-empty-dirs`, `-M`→`--remote-option`, `-f`→`--filter`, `-s`→`--secluded-args`, `-p`→`--perms`, `-T`→`--temp-dir`, `-a`/`--archive`→real `-rlptgoD`. FastSync's own flags moved to long-form-only or new shorts: `-j`/`--threads` (multithreading), `--preserve` (metadata), `--sendfile`, `--chunk-serialization`, `--timeout`, `--ssh-port`. The server's independent little CLI keeps `-p` as its port. All client-side, no wire change, no `PROTOCOL_VERSION` bump. Unit tests 37/37, full integration 400 passed, cppcheck and clang-format clean. Known Wave-A limitation: `--no-perms`/`--no-compress`-style negation of the newly-aliased shorts is not wired into the negatable set (only the long-form `--preserve`/`--compress`/`--no-links` negations exist); `--archive --no-perms` is consequently not supported yet — a minor deviation from rsync, acceptable for Wave A. +**Wave A — CLI namespace parity (rename colliding FastSync short flags) — ✅ implemented.** This freed the short letters rsync needs and made the three `🔀 Alt Arg` rows real. `-c`→`--checksum`, `-m`→`--prune-empty-dirs`, `-M`→`--remote-option`, `-f`→`--filter`, `-s`→`--secluded-args`, `-p`→`--perms`, `-T`→`--temp-dir`, `-a`/`--archive`→real `-rlptD`. FastSync's own flags moved to long-form-only or new shorts: `-j`/`--threads` (multithreading), `--preserve` (metadata), `--sendfile`, `--chunk-serialization`, `--timeout`, `--ssh-port`. The server's independent little CLI keeps `-p` as its port. All client-side, no wire change, no `PROTOCOL_VERSION` bump. Unit tests 37/37, full integration 400 passed, cppcheck and clang-format clean. Known Wave-A limitation: `--no-perms`/`--no-compress`-style negation of the newly-aliased shorts is not wired into the negatable set (only the long-form `--preserve`/`--compress`/`--no-links` negations exist); `--archive --no-perms` is consequently not supported yet — a minor deviation from rsync, acceptable for Wave A. | FastSync flag today | rsync wants that name | Proposed rename | |---------------------|----------------------|-----------------| @@ -817,7 +817,7 @@ These are the last compatibility items and the closing phase toward rsync flag p | `-s` / `--chunk-serialization` | `-s` = `--secluded-args`/`--protect-args` | → `--chunk-serialization` (long-only) | | `-p` (SSH port) | `-p` = `--perms` | → `--port` (long-only; `--server-port` already exists) | | `-T` / `--timeout` | `-T` = `--temp-dir` | → `--timeout` (long-only) | -| `-a` / `--archive` (= `-c -m -M`) | `-a` = `-rlptgoD` | → becomes **real rsync `-a`** after the renames | +| `-a` / `--archive` (= `-c -m -M`) | `-a` = `-rlptD` | → becomes **real rsync `-a`** after the renames | **Wave B — Output & filesystem completion (✅ implemented).** `-S`/`--sparse` (`⚠️→✅`): real hole preservation — a sparse-aware writer (`write_all_sparse`) skips all-zero runs ≥ 4096 bytes with `lseek(SEEK_CUR)` and `ftruncate`s the final size, wired into both the atomic temp+rename store and `--inplace` receiver-side with **no wire change** (the full file image is already in memory; the ftruncate presize is kept). `-P` (`⚠️→✅`): interrupted-write retention — on a save failure after data reached the temp fd, `--partial` now renames the already-written temp to the destination path (best-effort; falls through to the normal unlink on failure, never retains when `--partial` is off) so a later `--append`/`--append-verify` run can resume. `--block-size=SIZE` (`⚠️→✅`): promoted after verification — `--block-size` is now an alias for `--delta-block`, both set `config->delta_block_size`, which the delta engine already honored end-to-end (`delta_signature_create_seeded` + `delta_apply`); out-of-range values keep the default. `--fake-super` (`⚠️→✅`): added `fake_super_restore_fd` to parse and re-apply the recorded `user.fastsync.stat` record fd-relative (fchown best-effort/non-root skipped, fchmod, futimens); a save under `--fake-super` now re-applies the recorded attrs instead of only recording them, with the recording format unchanged. `--stderr=client` (`⚠️→⛔ Impossible/Divergence`): FastSync has no rsync client-message channel, and `client` is rejected at CLI parse — the rejection is the documented behavior (unit-tested). `-N`/`--crtimes` (`⚠️→⛔ Impossible/Divergence`): birth-times cannot be set by any portable fs call (`utimensat` sets only atime/mtime); capture/transmit stays, setting is impossible, the flag is accepted and safely inert. Review-hardening (post-eval): fake-super replay applies the mode through the same sanitization as the normal metadata path (group/other write bits are never granted); `--sparse` takes precedence over `--preallocate` (posix_fallocate skipped so holes survive); `--partial` retention is disabled for `--no_replace` (ignore/existing) and only marks a write-attempt after the actual write begins; `--block-size=SIZE`/`--delta-block=SIZE` inline forms are accepted. diff --git a/src/client/client_cli.c b/src/client/client_cli.c index b8637c3..d604262 100644 --- a/src/client/client_cli.c +++ b/src/client/client_cli.c @@ -1206,17 +1206,20 @@ static bool cli_handle_meta_flags(CliParseCtx* ctx) { return true; } if (opt_is(arg, "-a", "--archive")) { - /* Real rsync archive (-rlptgoD). FastSync is always recursive and always + /* FastSync archive mode (-rlptD). FastSync is always recursive and always * preserves hard-link/other transfer semantics per its own flags, so -a - * implies links, full metadata (perms/times/group/owner as FastSync's - * broad bundle), devices and specials. Compression and multithreading - * are NOT implied (they are no longer part of archive mode). */ + * implies links, metadata (perms/times), devices and specials. Owner/group + * are NOT implied; they require an explicit identity flag + * (--numeric-ids/--usermap/--groupmap/--chown/--copy-as). Compression and + * multithreading are NOT implied either (they are no longer part of + * archive mode). */ config->follow_symlinks = true; config->use_metadata = true; config->preserve_devices = true; config->preserve_specials = true; log_info_message(LOG_INFO_MISC, - "Enabled archive mode (-rlptgoD: links, metadata, devices, specials)"); + "Enabled archive mode (-rlptD: links, metadata, devices, specials; " + "owner/group opt-in)"); return true; } if (opt_is(arg, "-p", "--perms")) { diff --git a/src/client/usage.c b/src/client/usage.c index 0045d26..78c070e 100644 --- a/src/client/usage.c +++ b/src/client/usage.c @@ -20,8 +20,9 @@ void print_usage(void) { printf("Options:\n"); printf(" -c, --checksum Verify content by checksum instead of size+mtime\n"); printf(" -z, --compress [level] Enable compression (level 1-22, default 5)\n"); - printf(" -a, --archive rsync archive mode (-rlptgoD): links, metadata,\n"); - printf(" devices and specials (not compression/multithreading)\n"); + printf(" -a, --archive rsync archive mode (-rlptD): links, perms, times,\n"); + printf(" devices and specials; owner/group are not implied;\n"); + printf(" not compression/multithreading\n"); printf(" -n, --dry-run Show what would be transferred\n"); printf(" --remove-source-files Remove regular source files after successful transfer\n"); printf(" -p, --perms Preserve permission bits (part of the metadata bundle)\n"); @@ -162,6 +163,9 @@ void print_usage(void) { printf(" none suppresses info even with --verbose\n"); printf(" --preserve Preserve file metadata (long form only)\n"); printf(" -E, --executability Preserve executable permission bits\n"); + printf(" -U, --atimes Preserve access times\n"); + printf(" -N, --crtimes Capture birth time; cannot be applied (documented\n"); + printf(" divergence)\n"); printf(" -X, --xattrs Preserve user extended attributes (user.* only;\n"); printf(" privileged security.*/trusted.* namespaces are\n"); printf(" never captured or applied)\n"); diff --git a/tests/integration/test_readme_consistency.py b/tests/integration/test_readme_consistency.py new file mode 100644 index 0000000..2456506 --- /dev/null +++ b/tests/integration/test_readme_consistency.py @@ -0,0 +1,262 @@ +"""Guard the README against drifting from the real CLI. + +This test parses README.md and checks it against the actual sources of truth +instead of against a hand-maintained copy: + + * client ``--help`` output -> ``src/client/usage.c`` (``print_usage``) + * server ``--help`` output -> ``src/server/server.c`` (``print_server_usage``) + * ``FASTSYNC_*`` env vars -> ``getenv("...")`` call sites under ``src/`` + +It is deliberately offline and read-only: no server is started, no transfer is +performed. Each binary is invoked at most once per test session and the result +is cached. +""" + +import functools +import os +import re +import subprocess +import sys + +import pytest + +sys.path.insert(0, os.path.dirname(__file__)) +from common import BUILD_DIR, PROJECT_ROOT + +pytestmark = pytest.mark.ci + +README_PATH = os.path.join(PROJECT_ROOT, "README.md") +SRC_DIR = os.path.join(PROJECT_ROOT, "src") + +# --------------------------------------------------------------------------- +# Markdown helpers +# --------------------------------------------------------------------------- + +_HEADING_RE = re.compile(r"^(#+)\s+(.*?)\s*$") +_BACKTICK_RE = re.compile(r"`([^`]*)`") +# A documented option may carry an argument annotation that is not part of the +# option name itself: ``--out=FILE``, ``--exclude ``, ``--threads[=N]``, +# ``--copy-as=USER[:GROUP]``. Cut the name loose from the first such marker. +_OPTION_SUFFIX_RE = re.compile(r"[=\s<\[(].*$") +_OPTION_TOKEN_RE = re.compile(r"^--?[A-Za-z][A-Za-z0-9-]*$") + + +def _readme_lines(): + with open(README_PATH, encoding="utf-8") as fh: + return fh.read().splitlines() + + +def _heading_level(line): + match = _HEADING_RE.match(line) + return len(match.group(1)) if match else 0 + + +def _section(lines, heading): + """Return ``(lineno, line)`` pairs under the first exact ``heading``. + + The section runs until the next heading of the same or higher level, so a + ``##`` section includes its ``###`` subsections. Line numbers are 1-based + to match what a reader sees in an editor. + """ + target_level = _heading_level(heading) + for index, line in enumerate(lines): + if line.strip() != heading: + continue + start = index + 1 + for end in range(start, len(lines)): + level = _heading_level(lines[end]) + if level and level <= target_level: + return [(n + 1, lines[n]) for n in range(start, end)] + return [(n + 1, lines[n]) for n in range(start, len(lines))] + raise AssertionError( + f"README heading not found (has the README been restructured?): {heading!r}" + ) + + +def _first_column_spans(section_lines): + """Backticked spans from the first column of every markdown table row.""" + spans = [] + for lineno, line in section_lines: + stripped = line.strip() + if not stripped.startswith("|"): + continue + cells = stripped.split("|") + if len(cells) < 2: + continue + first = cells[1] + if set(first.strip()) <= set("-: "): + continue # header separator row, e.g. |---|---| + for match in _BACKTICK_RE.finditer(first): + spans.append((match.group(1), lineno)) + return spans + + +def _documented_option_tokens(section_lines): + """``(token, lineno, raw_cell)`` for each CLI option in a section's tables.""" + found = [] + for raw, lineno in _first_column_spans(section_lines): + for piece in re.split(r"[,\s]+", raw): + name = _OPTION_SUFFIX_RE.sub("", piece).strip() + if _OPTION_TOKEN_RE.match(name): + found.append((name, lineno, raw)) + return found + + +# --------------------------------------------------------------------------- +# Sources of truth +# --------------------------------------------------------------------------- + +_GETENV_RE = re.compile(r'getenv\s*\(\s*"([^"]+)"\s*\)') +_FASTSYNC_ENV_RE = re.compile(r"FASTSYNC_[A-Z0-9_]+") + + +def _getenv_names(): + """Every string literal passed to ``getenv()`` anywhere under ``src/``.""" + names = set() + for root, _dirs, files in os.walk(SRC_DIR): + for filename in files: + if not filename.endswith((".c", ".h")): + continue + path = os.path.join(root, filename) + with open(path, encoding="utf-8", errors="replace") as fh: + names.update(_GETENV_RE.findall(fh.read())) + return names + + +@functools.lru_cache(maxsize=None) +def _help_stdout(binary_name): + """Cached `` --help`` stdout; skipped (not failed) if unbuilt.""" + binary = os.path.join(BUILD_DIR, binary_name) + if not (os.path.isfile(binary) and os.access(binary, os.X_OK)): + pytest.skip( + f"{binary} is not built; run " + "`cmake -B build -S . && cmake --build build` first" + ) + try: + result = subprocess.run( + [binary, "--help"], capture_output=True, text=True, timeout=30 + ) + except OSError as exc: + pytest.skip(f"could not execute {binary}: {exc}") + assert result.returncode == 0, ( + f"{binary} --help exited {result.returncode}: " + f"{(result.stderr or result.stdout).strip()[:200]}" + ) + return result.stdout + + +def _mentions_option(help_text, token): + """True when ``token`` appears as a standalone option in ``help_text``. + + A plain substring test would let a removed token hide behind a longer one + (``--del`` inside ``--delete``); requiring a non-word boundary on both sides + keeps every documented token individually accountable. + """ + return ( + re.search(r"(?