Compare commits
106
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1b67f5ffcf | ||
|
|
7f5547e900 | ||
|
|
06e83f2402 | ||
|
|
f7c6c91e13 | ||
|
|
39805e4eee | ||
|
|
9ef78ef48b | ||
|
|
68e7ed57d0 | ||
|
|
11c591c5aa | ||
|
|
1d61a1426e | ||
|
|
beb681c2dc | ||
|
|
3704a6adaa | ||
|
|
14c064a093 | ||
|
|
2234879b2f | ||
|
|
98120fc722 | ||
|
|
5fed0888aa | ||
|
|
9f8b58893b | ||
|
|
48dfd5dfa0 | ||
|
|
8384a1997b | ||
|
|
a1981a78ad | ||
|
|
b5646a0b9e | ||
|
|
c6a341bd1e | ||
|
|
c1e9509f16 | ||
|
|
8689778233 | ||
|
|
cf0d10fccb | ||
|
|
90112a7585 | ||
|
|
3304541337 | ||
|
|
6a95efbe41 | ||
|
|
3b013bf9d2 | ||
|
|
4f21498ee6 | ||
|
|
315e572d18 | ||
|
|
55172ff6de | ||
|
|
c650f9a3d0 | ||
|
|
192aff8c46 | ||
|
|
6d10116d50 | ||
|
|
e42df1bde3 | ||
|
|
9fc1e72972 | ||
|
|
71f1529222 | ||
|
|
2adfa5a03b | ||
|
|
47d1cbdae8 | ||
|
|
b59139589d | ||
|
|
49e4af275f | ||
|
|
8e1bc9bb9c | ||
|
|
421a94773f | ||
|
|
ead4651d12 | ||
|
|
638d953b1d | ||
|
|
04cea295b5 | ||
|
|
820afd334a | ||
|
|
2f553a2a43 | ||
|
|
f990e86309 | ||
|
|
0d306940e1 | ||
|
|
552c19d3fe | ||
|
|
14c5da98d8 | ||
|
|
aab45873ac | ||
|
|
605f79a450 | ||
|
|
066c7ed1af | ||
|
|
97a628c3fd | ||
|
|
9f23b23893 | ||
|
|
2ace6416a7 | ||
|
|
e491e811e3 | ||
|
|
ec02ee6dde | ||
|
|
98cbf3495d | ||
|
|
245a34fa11 | ||
|
|
68aaf89da7 | ||
|
|
e37d5b9438 | ||
|
|
5b8cdf457b | ||
|
|
1ebe261e05 | ||
|
|
05313f00ea | ||
|
|
4152bf379e | ||
|
|
f8291d895d | ||
|
|
ba236d5b2e | ||
|
|
ec6160a8b9 | ||
|
|
302c3cb664 | ||
|
|
caabd2e314 | ||
|
|
21e6b1c1bc | ||
|
|
5350290f52 | ||
|
|
5b997d5d25 | ||
|
|
fb96c21d93 | ||
|
|
17b5b31845 | ||
|
|
48a36b0038 | ||
|
|
ce64fcd4b1 | ||
|
|
4982e15969 | ||
|
|
d69f5db652 | ||
|
|
f443b2986d | ||
|
|
2a08a7ba5c | ||
|
|
36a373ff5d | ||
|
|
3b85658024 | ||
|
|
5e67a721a4 | ||
|
|
0b5b1a8643 | ||
|
|
2afb1d9649 | ||
|
|
230401c629 | ||
|
|
c91db3266a | ||
|
|
44ce154e37 | ||
|
|
cd27886dde | ||
|
|
a0aff000ad | ||
|
|
50f9ebb55b | ||
|
|
393a440c78 | ||
|
|
2ff2688ef1 | ||
|
|
5c7d82b79c | ||
|
|
4872d425a3 | ||
|
|
60eb7f11fe | ||
|
|
eb4e9fba1f | ||
|
|
8ab5026224 | ||
|
|
dbacc4f0e5 | ||
|
|
1517b3fc35 | ||
|
|
5c055960b8 | ||
|
|
405e5b3b00 |
No files matched your search
+1
-1
@@ -82,7 +82,7 @@ set(TEST_INCLUDES tests src/shared src/server src/client)
|
|||||||
|
|
||||||
# Monolithic test binary (backward compatible)
|
# Monolithic test binary (backward compatible)
|
||||||
file(GLOB TEST_SRCS "tests/test_*.c" "tests/runner.c")
|
file(GLOB TEST_SRCS "tests/test_*.c" "tests/runner.c")
|
||||||
add_executable(tests ${TEST_SRCS} ${SHARED_SRCS} ${FILE_STORE_SRCS} ${SERVER_RECEIVER_SRCS} src/client/scanner.c src/client/client_cli.c src/client/client_validation.c src/client/usage.c)
|
add_executable(tests ${TEST_SRCS} ${SHARED_SRCS} ${FILE_STORE_SRCS} ${SERVER_RECEIVER_SRCS} src/client/scanner.c src/client/change_list.c src/client/client_cli.c src/client/client_validation.c src/client/usage.c)
|
||||||
target_include_directories(tests PRIVATE ${TEST_INCLUDES})
|
target_include_directories(tests PRIVATE ${TEST_INCLUDES})
|
||||||
target_compile_definitions(tests PRIVATE FASTSYNC_TEST_BUILD)
|
target_compile_definitions(tests PRIVATE FASTSYNC_TEST_BUILD)
|
||||||
target_link_libraries(tests PRIVATE ${TEST_LIBS})
|
target_link_libraries(tests PRIVATE ${TEST_LIBS})
|
||||||
|
|||||||
@@ -66,8 +66,12 @@ replacement for every rsync feature or protocol mode.
|
|||||||
- Owner/group, ACL, xattr, hard-link, device, and special-file handling is
|
- Owner/group, ACL, xattr, hard-link, device, and special-file handling is
|
||||||
incomplete or unavailable.
|
incomplete or unavailable.
|
||||||
- Sparse-file handling does not yet preserve all holes correctly.
|
- Sparse-file handling does not yet preserve all holes correctly.
|
||||||
- `--partial`, `--partial-dir`, `--append`, and `--append-verify` are not yet
|
- `--partial`, `--partial-dir`, `-P`, `--append`, and `--append-verify` are not
|
||||||
full rsync-style resumable transfers.
|
yet full rsync-style resumable transfers. Interrupted files are not retained
|
||||||
|
for resumption.
|
||||||
|
- `--dirs` is not implemented. Its compatibility aliases `--old-dirs` and
|
||||||
|
`--old-d` are recognized but rejected explicitly rather than silently using
|
||||||
|
FastSync's recursive directory behavior.
|
||||||
- Several rsync short options currently have FastSync-specific meanings. Do
|
- Several rsync short options currently have FastSync-specific meanings. Do
|
||||||
not assume every short option is interchangeable yet.
|
not assume every short option is interchangeable yet.
|
||||||
|
|
||||||
@@ -89,19 +93,24 @@ partial, alternate, and planned behavior.
|
|||||||
| `-a, --archive` | Archive mode: enables `-c -m -M` (no `-s`) |
|
| `-a, --archive` | Archive mode: enables `-c -m -M` (no `-s`) |
|
||||||
| `-m` | Multithreading mode |
|
| `-m` | Multithreading mode |
|
||||||
| `-s` | Chunk serialization (batch all files per chunk) |
|
| `-s` | Chunk serialization (batch all files per chunk) |
|
||||||
|
| `--secluded-args` | Accepted as an rsync compatibility option with no effect; `-s` remains chunk serialization. |
|
||||||
| `-f, --sendfile` | Sendfile zero-copy. Incompatible with `-c` / `-s`. TCP only. |
|
| `-f, --sendfile` | Sendfile zero-copy. Incompatible with `-c` / `-s`. TCP only. |
|
||||||
| `-M, --preserve` | Preserve supported file metadata (mode and mtime; ownership and atime are unsupported) |
|
| `-M, --preserve` | Preserve supported file metadata (mode and mtime; ownership and atime are unsupported) |
|
||||||
| `-n, --dry-run` | Scan and print what would be transferred |
|
| `-n, --dry-run` | Scan and print what would be transferred |
|
||||||
| `-p <port>` | SSH port (default: 22) |
|
| `-p <port>` | SSH port (default: 22) |
|
||||||
| `-v, --verbose` | Enable debug logging |
|
| `-v, --verbose` | Enable debug logging |
|
||||||
|
| `-q, --quiet` | Suppress non-error output |
|
||||||
| `--progress` | Show real-time transfer speed |
|
| `--progress` | Show real-time transfer speed |
|
||||||
|
| `-P` | Enables partial-transfer mode and progress output (partial retention is incomplete) |
|
||||||
| `--delete` | Delete files on receiver not present in source |
|
| `--delete` | Delete files on receiver not present in source |
|
||||||
| `--exclude <pattern>` | Exclude files matching glob pattern (repeatable) |
|
| `--exclude <pattern>` | Exclude files matching glob pattern (repeatable) |
|
||||||
| `--exclude-from <file>` | Read exclude patterns from a file (one per line) |
|
| `--exclude-from <file>` | Read exclude patterns from a file (one per line) |
|
||||||
| `--include <pattern>` | Only transfer files matching glob pattern (repeatable, whitelist) |
|
| `--include <pattern>` | Only transfer files matching glob pattern (repeatable, whitelist) |
|
||||||
| `--max-size <n>` | Skip files larger than n bytes |
|
| `--max-size <n>` | Skip files larger than n bytes |
|
||||||
| `--min-size <n>` | Skip files smaller than n bytes |
|
| `--min-size <n>` | Skip files smaller than n bytes |
|
||||||
|
| `--max-alloc <SIZE>` | Maximum single allocation (binary units: B, K, M, G, T, P, E; default 1G) |
|
||||||
| `--incremental` | Skip files unchanged since last transfer (size + mtime). Auto-enables `--preserve`. Incompatible with `-s`. |
|
| `--incremental` | Skip files unchanged since last transfer (size + mtime). Auto-enables `--preserve`. Incompatible with `-s`. |
|
||||||
|
| `--existing` | Skip files not already present at the destination; update existing files normally. |
|
||||||
| `--bwlimit <KB/s>` | Bandwidth limit in kilobytes per second |
|
| `--bwlimit <KB/s>` | Bandwidth limit in kilobytes per second |
|
||||||
| `--chunk-size <n>` | Chunk size in bytes (default: 10485760) |
|
| `--chunk-size <n>` | Chunk size in bytes (default: 10485760) |
|
||||||
| `--timeout <sec>` | I/O timeout in seconds (default: 30) |
|
| `--timeout <sec>` | I/O timeout in seconds (default: 30) |
|
||||||
@@ -109,6 +118,7 @@ partial, alternate, and planned behavior.
|
|||||||
| `--backup` | Backup existing destination files before overwriting |
|
| `--backup` | Backup existing destination files before overwriting |
|
||||||
| `--backup-dir <dir>` | Target directory for backups (requires `--backup`) |
|
| `--backup-dir <dir>` | Target directory for backups (requires `--backup`) |
|
||||||
| `--stats` | Print transfer statistics at end (bytes, files, timing) |
|
| `--stats` | Print transfer statistics at end (bytes, files, timing) |
|
||||||
|
| `-h, --human-readable` | Format transfer byte sizes with binary units |
|
||||||
| `--max-depth <n>` | Maximum directory depth to recurse (0 = unlimited, default: 0) |
|
| `--max-depth <n>` | Maximum directory depth to recurse (0 = unlimited, default: 0) |
|
||||||
| `--log-file <path>` | Write log messages to file instead of stderr |
|
| `--log-file <path>` | Write log messages to file instead of stderr |
|
||||||
| `--source-dir <path>` | Source directory (overrides `FASTSYNC_SOURCE_DIR`) |
|
| `--source-dir <path>` | Source directory (overrides `FASTSYNC_SOURCE_DIR`) |
|
||||||
@@ -339,6 +349,10 @@ features without changing the meaning of ordinary compatibility options.
|
|||||||
| `-m` | Enable the multithreaded scanner/loader/sender pipeline. |
|
| `-m` | Enable the multithreaded scanner/loader/sender pipeline. |
|
||||||
| `-c [level]`, `-z [level]` | Enable streaming zstd compression, levels 1-22. |
|
| `-c [level]`, `-z [level]` | Enable streaming zstd compression, levels 1-22. |
|
||||||
| `--compress-level <n>` | Set the zstd compression level. |
|
| `--compress-level <n>` | Set the zstd compression level. |
|
||||||
|
| `--zc <alg>` | Alias for `--compress-choice`. FastSync supports `zstd` and `none`. |
|
||||||
|
| `--zl <n>` | Alias for `--compress-level`. |
|
||||||
|
| `--skip-compress <list>` | Skip compression for comma-separated suffixes; incompatible with `-s`. |
|
||||||
|
| `--compress-threads <n>` | Use `n` zstd compression workers. Requires compression and a zstd build with threaded support; the setting affects sender CPU work only. |
|
||||||
| `--chunk-size <bytes>` | Set the transfer chunk size. |
|
| `--chunk-size <bytes>` | Set the transfer chunk size. |
|
||||||
| `-s` | Enable FastSync chunk serialization. |
|
| `-s` | Enable FastSync chunk serialization. |
|
||||||
| `-f`, `--sendfile` | Use TCP `sendfile()` zero-copy transfer. Incompatible with compression and chunk serialization. |
|
| `-f`, `--sendfile` | Use TCP `sendfile()` zero-copy transfer. Incompatible with compression and chunk serialization. |
|
||||||
@@ -359,6 +373,11 @@ particular, FastSync currently uses `-p` for SSH port, `-s` for chunk
|
|||||||
serialization, and `-S` for sparse handling. These meanings must be reconciled
|
serialization, and `-S` for sparse handling. These meanings must be reconciled
|
||||||
before FastSync can claim full rsync CLI compatibility.
|
before FastSync can claim full rsync CLI compatibility.
|
||||||
|
|
||||||
|
`--secluded-args` is accepted as a long-form compatibility no-op. It does not
|
||||||
|
change FastSync's transport or protocol behavior. The rsync short form `-s` is
|
||||||
|
intentionally not aliased because it remains FastSync's chunk-serialization
|
||||||
|
option.
|
||||||
|
|
||||||
## Client Options
|
## Client Options
|
||||||
|
|
||||||
### Selection and transfer
|
### Selection and transfer
|
||||||
@@ -465,11 +484,12 @@ defaults to the current directory. |
|
|||||||
|
|
||||||
## Protocol and Security
|
## Protocol and Security
|
||||||
|
|
||||||
FastSync protocol version `2.2.0` is shared by the client and server. The
|
FastSync protocol version `2.5.0` is shared by the client and server. The
|
||||||
current protocol is sender-driven and includes configuration negotiation,
|
current protocol is sender-driven and includes configuration negotiation,
|
||||||
incremental checks, checksums, manifests, keep-alives, abort handling, and
|
including the maximum allocation limit, incremental checks, checksums,
|
||||||
FastSync-native delta messages. Client and server versions must currently
|
manifests, keep-alives, abort handling, per-file remove-source results, and
|
||||||
match exactly.
|
FastSync-native delta messages.
|
||||||
|
Client and server versions must currently match exactly.
|
||||||
|
|
||||||
TLS provides encrypted TCP transport. Supplying `--ca` enables certificate
|
TLS provides encrypted TCP transport. Supplying `--ca` enables certificate
|
||||||
verification; without it, traffic is encrypted but peer identity is not
|
verification; without it, traffic is encrypted but peer identity is not
|
||||||
|
|||||||
+155
-36
@@ -6,11 +6,12 @@ This document maps rsync's full feature set to FastSync's current implementation
|
|||||||
|
|
||||||
| Status | Count | Description |
|
| Status | Count | Description |
|
||||||
|--------|-------|-------------|
|
|--------|-------|-------------|
|
||||||
| ✅ Implemented | 34 | Feature works end-to-end |
|
| ✅ Implemented | 54 | Feature works end-to-end |
|
||||||
| 🔀 Alt Arg | 3 | Functionality exists but under different flag/semantics |
|
| 🔀 Alt Arg | 3 | Functionality exists but under different flag/semantics |
|
||||||
| ⚠️ Partial | 1 | Flag parsed/stored but behavior incomplete |
|
| ⚠️ Partial | 5 | Flag parsed/stored but behavior incomplete |
|
||||||
| ❌ Not Implemented | 98 | Flag not recognized or no behavior |
|
| 🔄 Compatibility No-op | 1 | Flag is accepted for CLI compatibility but has no effect |
|
||||||
| **Total** | **136** | |
|
| ❌ Not Implemented | 84 | Flag not recognized or no behavior |
|
||||||
|
| **Total** | **147** | |
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -20,12 +21,12 @@ This document maps rsync's full feature set to FastSync's current implementation
|
|||||||
|------|-------------------|-----------------|-------|
|
|------|-------------------|-----------------|-------|
|
||||||
| `-a`, `--archive` | Archive mode is -rlptgoD | 🔀 Alt Arg | Maps to -c -m -M (compression + multithread + metadata) |
|
| `-a`, `--archive` | Archive mode is -rlptgoD | 🔀 Alt Arg | Maps to -c -m -M (compression + multithread + metadata) |
|
||||||
| `-v`, `--verbose` | Increase verbosity | ✅ Implemented | Sets `log_level=DEBUG` |
|
| `-v`, `--verbose` | Increase verbosity | ✅ Implemented | Sets `log_level=DEBUG` |
|
||||||
| `-q`, `--quiet` | Suppress non-error messages | ❌ Not Implemented | Removed because it had no effect |
|
| `-q`, `--quiet` | Suppress non-error messages | ✅ Implemented | Suppresses client output while preserving errors |
|
||||||
| `--help` | Show help | ✅ Implemented | Prints usage and exits; `-h` is not accepted |
|
| `--help` | Show help | ✅ Implemented | Prints usage and exits; `-h` is not accepted |
|
||||||
| `-V`, `--version` | Print version | ✅ Implemented | |
|
| `-V`, `--version` | Print version | ✅ Implemented | |
|
||||||
| `--info=FLAGS` | Fine-grained info verbosity | ❌ Not Implemented | Removed because it had no effect |
|
| `--info=FLAGS` | Fine-grained info verbosity | ✅ Implemented | Supports `copy`, `misc`, `skip`, `stats`, `all`, and `none`; explicit flags override `--verbose`, and `none` suppresses info output; unsupported names are rejected |
|
||||||
| `--debug=FLAGS` | Fine-grained debug verbosity | ❌ Not Implemented | Removed because it had no effect |
|
| `--debug=FLAGS` | Fine-grained debug verbosity | ✅ Implemented | `io`, `proto`, `pack`, and `util` are supported; `--debug=help` lists flags; other rsync categories are rejected |
|
||||||
| `--stderr=MODE` | Change stderr output mode | ❌ Not Implemented | |
|
| `--stderr=MODE` | Change stderr output mode | ⚠️ Partial | `errors` (default) and `all` are supported; `client` is rejected because FastSync has no rsync message channel |
|
||||||
| `--no-motd` | Suppress daemon MOTD | ❌ Not Implemented | |
|
| `--no-motd` | Suppress daemon MOTD | ❌ Not Implemented | |
|
||||||
| `--exclude=PATTERN` | Exclude files matching pattern | ✅ Implemented | Glob matching in scanner |
|
| `--exclude=PATTERN` | Exclude files matching pattern | ✅ Implemented | Glob matching in scanner |
|
||||||
| `--include=PATTERN` | Include files matching pattern | ✅ Implemented | Glob matching in scanner |
|
| `--include=PATTERN` | Include files matching pattern | ✅ Implemented | Glob matching in scanner |
|
||||||
@@ -36,15 +37,15 @@ This document maps rsync's full feature set to FastSync's current implementation
|
|||||||
| Flag | Rsync Description | FastSync Status | Notes |
|
| Flag | Rsync Description | FastSync Status | Notes |
|
||||||
|------|-------------------|-----------------|-------|
|
|------|-------------------|-----------------|-------|
|
||||||
| `--stats` | Give transfer stats | ✅ Implemented | Prints file/byte counts |
|
| `--stats` | Give transfer stats | ✅ Implemented | Prints file/byte counts |
|
||||||
| `-h`, `--human-readable` | Human-readable numbers | ❌ Not Implemented | Removed because it had no effect |
|
| `-h`, `--human-readable` | Human-readable numbers | ✅ Implemented | Formats transfer byte sizes using binary units |
|
||||||
| `-i`, `--itemize-changes` | Per-file change summary | ❌ Not Implemented | Removed because it had no effect |
|
| `-i`, `--itemize-changes` | Per-file change summary | ✅ Implemented | Prints rsync-style `>f+++++++++` lines to stdout only for files actually sent (also under `-m`); unchanged files print nothing, matching single-`-i` behavior |
|
||||||
| `--progress` | Show progress | ✅ Implemented | Progress callback in sender |
|
| `--progress` | Show progress | ✅ Implemented | Progress callback in sender |
|
||||||
| `-P` | Same as --partial --progress | ❌ Not Implemented | |
|
| `-P` | Same as --partial --progress | ⚠️ Partial | Parses and enables progress, but interrupted files are not retained for resumable transfers |
|
||||||
| `--out-format=FORMAT` | Custom output format | ❌ Not Implemented | Removed because it had no effect |
|
| `--out-format=FORMAT` | Custom output format | ✅ Implemented | Per-transfer template on stdout; tokens `%f` `%n` `%l` `%b` `%M` `%%` (`%b` is the source length, always `== %l`; post-compression/delta wire bytes are not counted); unknown escapes preserved |
|
||||||
| `--log-file=FILE` | Log to file | ✅ Implemented | `log_file` config field |
|
| `--log-file=FILE` | Log to file | ✅ Implemented | `log_file` config field |
|
||||||
| `--log-file-format=FMT` | Log format | ❌ Not Implemented | |
|
| `--log-file-format=FMT` | Log format | ✅ Implemented | Requires `--log-file`; writes one template line per transferred file using the same token set as `--out-format` (including `%b` `==` source length) |
|
||||||
| `--8-bit-output` | Leave high-bit chars unescaped | ❌ Not Implemented | |
|
| `--8-bit-output`, `-8` | Leave high-bit chars unescaped | ✅ Implemented | Applies to displayed paths and protocol debug output |
|
||||||
| `--list-only` | List files instead of copying | ❌ Not Implemented | Removed because it had no effect |
|
| `--list-only` | List files instead of copying | ✅ Implemented | `ls -l`-style listing of files that would be transferred; scans the source only, contacts no server, writes nothing; also works with `-n` |
|
||||||
|
|
||||||
## 3. File Selection
|
## 3. File Selection
|
||||||
|
|
||||||
@@ -58,11 +59,13 @@ This document maps rsync's full feature set to FastSync's current implementation
|
|||||||
| `--max-size=SIZE` | Skip files larger than SIZE | ✅ Implemented | `max_size` in scanner |
|
| `--max-size=SIZE` | Skip files larger than SIZE | ✅ Implemented | `max_size` in scanner |
|
||||||
| `--min-size=SIZE` | Skip files smaller than SIZE | ✅ Implemented | `min_size` in scanner |
|
| `--min-size=SIZE` | Skip files smaller than SIZE | ✅ Implemented | `min_size` in scanner |
|
||||||
| `-I`, `--ignore-times` | Don't skip files matching size+time | ❌ Not Implemented | |
|
| `-I`, `--ignore-times` | Don't skip files matching size+time | ❌ Not Implemented | |
|
||||||
| `--size-only` | Skip based on size only | ❌ Not Implemented | |
|
| `--size-only` | Skip based on size only | ✅ Implemented | With `--incremental`, ignores mtime |
|
||||||
| `-@`, `--modify-window=NUM` | Mod-time comparison accuracy | ❌ Not Implemented | |
|
| `-@`, `--modify-window=NUM` | Mod-time comparison accuracy | ✅ Implemented | Whole-second tolerance with nanosecond-aware comparisons |
|
||||||
| `--existing` | Skip creating new files on receiver | ❌ Not Implemented | |
|
| `--existing` | Skip creating new files on receiver | ✅ Implemented | Existing destination files continue through normal update handling |
|
||||||
| `--ignore-existing` | Skip updating existing files | ❌ Not Implemented | |
|
| `--ignore-existing` | Skip updating existing files | ❌ Not Implemented | |
|
||||||
| `--remove-source-files` | Sender removes synced files | ❌ Not Implemented | |
|
| `--remove-source-files` | Sender removes regular files after confirmed transfer | ✅ Implemented | |
|
||||||
|
| `-x`, `--one-file-system` | Do not cross filesystem boundaries | ❌ Not Implemented | |
|
||||||
|
| `-F` | Add the default `.rsync-filter` rules | ❌ Not Implemented | |
|
||||||
|
|
||||||
## 4. Directory Options
|
## 4. Directory Options
|
||||||
|
|
||||||
@@ -71,7 +74,7 @@ This document maps rsync's full feature set to FastSync's current implementation
|
|||||||
| `-r`, `--recursive` | Recurse into directories | ✅ Implemented | Default behavior |
|
| `-r`, `--recursive` | Recurse into directories | ✅ Implemented | Default behavior |
|
||||||
| `-R`, `--relative` | Use relative path names | ❌ Not Implemented | Removed because it had no effect |
|
| `-R`, `--relative` | Use relative path names | ❌ Not Implemented | Removed because it had no effect |
|
||||||
| `--no-implied-dirs` | Don't send implied dirs with -R | ❌ Not Implemented | |
|
| `--no-implied-dirs` | Don't send implied dirs with -R | ❌ Not Implemented | |
|
||||||
| `-d`, `--dirs` | Transfer dirs without recursing | ❌ Not Implemented | |
|
| `-d`, `--dirs`, `--old-dirs`, `--old-d` | Transfer dirs without recursing | ❌ Not Implemented | The aliases are recognized and rejected explicitly; they depend on the unimplemented `--dirs` behavior |
|
||||||
| `--mkpath` | Create missing path components | ❌ Not Implemented | |
|
| `--mkpath` | Create missing path components | ❌ Not Implemented | |
|
||||||
|
|
||||||
## 5. Transfer Modifications
|
## 5. Transfer Modifications
|
||||||
@@ -94,6 +97,7 @@ This document maps rsync's full feature set to FastSync's current implementation
|
|||||||
| `--backup-dir=DIR` | Backup directory hierarchy | ✅ Implemented | `backup_dir` config field |
|
| `--backup-dir=DIR` | Backup directory hierarchy | ✅ Implemented | `backup_dir` config field |
|
||||||
| `--suffix=SUFFIX` | Backup suffix (default ~) | ✅ Implemented | `suffix` config field |
|
| `--suffix=SUFFIX` | Backup suffix (default ~) | ✅ Implemented | `suffix` config field |
|
||||||
| `--delay-updates` | Put updated files in place at end | ❌ Not Implemented | |
|
| `--delay-updates` | Put updated files in place at end | ❌ Not Implemented | |
|
||||||
|
| `-T`, `--temp-dir=DIR` | Create temporary files in DIR | ❌ Not Implemented | `-T` is FastSync's timeout alias |
|
||||||
|
|
||||||
## 7. Deletion
|
## 7. Deletion
|
||||||
|
|
||||||
@@ -101,7 +105,7 @@ This document maps rsync's full feature set to FastSync's current implementation
|
|||||||
|------|-------------------|-----------------|-------|
|
|------|-------------------|-----------------|-------|
|
||||||
| `--delete` | Delete extraneous files from dest | ✅ Implemented | `use_delete` config field |
|
| `--delete` | Delete extraneous files from dest | ✅ Implemented | `use_delete` config field |
|
||||||
| `--delete-before` | Delete before transfer | ❌ Not Implemented | Removed because it had no effect |
|
| `--delete-before` | Delete before transfer | ❌ Not Implemented | Removed because it had no effect |
|
||||||
| `--delete-during` | Delete during transfer | ❌ Not Implemented | |
|
| `--del`, `--delete-during` | Delete during transfer | ❌ Not Implemented | Both flags are recognized but rejected; delete timing is not implemented |
|
||||||
| `--delete-delay` | Find deletions during, delete after | ❌ Not Implemented | |
|
| `--delete-delay` | Find deletions during, delete after | ❌ Not Implemented | |
|
||||||
| `--delete-after` | Delete after transfer | ❌ Not Implemented | Removed because it had no effect |
|
| `--delete-after` | Delete after transfer | ❌ Not Implemented | Removed because it had no effect |
|
||||||
| `--delete-excluded` | Also delete excluded files | ❌ Not Implemented | Removed because it had no effect |
|
| `--delete-excluded` | Also delete excluded files | ❌ Not Implemented | Removed because it had no effect |
|
||||||
@@ -119,7 +123,7 @@ This document maps rsync's full feature set to FastSync's current implementation
|
|||||||
| `-o`, `--owner` | Preserve owner | ✅ Implemented | Part of -M |
|
| `-o`, `--owner` | Preserve owner | ✅ Implemented | Part of -M |
|
||||||
| `-g`, `--group` | Preserve group | ✅ Implemented | Part of -M |
|
| `-g`, `--group` | Preserve group | ✅ Implemented | Part of -M |
|
||||||
| `-t`, `--times` | Preserve modification times | ✅ Implemented | Part of -M |
|
| `-t`, `--times` | Preserve modification times | ✅ Implemented | Part of -M |
|
||||||
| `-E`, `--executability` | Preserve executability | ❌ Not Implemented | |
|
| `-E`, `--executability` | Preserve executability | ✅ Implemented | Preserves executable permission bits (implies metadata preservation) |
|
||||||
| `--chmod=CHMOD` | Affect file permissions | ✅ Implemented | Supports numeric and symbolic `ugo` `rwx` changes; retains receiver safety masking |
|
| `--chmod=CHMOD` | Affect file permissions | ✅ Implemented | Supports numeric and symbolic `ugo` `rwx` changes; retains receiver safety masking |
|
||||||
| `-A`, `--acls` | Preserve ACLs | ❌ Not Implemented | Removed because it had no effect |
|
| `-A`, `--acls` | Preserve ACLs | ❌ Not Implemented | Removed because it had no effect |
|
||||||
| `-X`, `--xattrs` | Preserve extended attributes | ❌ Not Implemented | Removed because it had no effect |
|
| `-X`, `--xattrs` | Preserve extended attributes | ❌ Not Implemented | Removed because it had no effect |
|
||||||
@@ -135,6 +139,12 @@ This document maps rsync's full feature set to FastSync's current implementation
|
|||||||
| `-J`, `--omit-link-times` | Omit symlinks from --times | ❌ Not Implemented | |
|
| `-J`, `--omit-link-times` | Omit symlinks from --times | ❌ Not Implemented | |
|
||||||
| `--super` | Receiver attempts super-user activities | ❌ Not Implemented | |
|
| `--super` | Receiver attempts super-user activities | ❌ Not Implemented | |
|
||||||
| `--fake-super` | Store/recover privileged attrs via xattrs | ❌ Not Implemented | |
|
| `--fake-super` | Store/recover privileged attrs via xattrs | ❌ Not Implemented | |
|
||||||
|
| `--open-noatime` | Avoid changing access time when opening files | ❌ Not Implemented | |
|
||||||
|
| `--numeric-ids` | Do not map uid/gid by name | ❌ Not Implemented | |
|
||||||
|
| `--usermap=STRING` | Map usernames | ❌ Not Implemented | |
|
||||||
|
| `--groupmap=STRING` | Map group names | ❌ Not Implemented | |
|
||||||
|
| `--chown=USER:GROUP` | Map owner and group | ❌ Not Implemented | |
|
||||||
|
| `--copy-as=USER[:GROUP]` | Perform the copy as another user/group | ❌ Not Implemented | |
|
||||||
|
|
||||||
## 9. Symlink Handling
|
## 9. Symlink Handling
|
||||||
|
|
||||||
@@ -171,10 +181,10 @@ This document maps rsync's full feature set to FastSync's current implementation
|
|||||||
| Flag | Rsync Description | FastSync Status | Notes |
|
| Flag | Rsync Description | FastSync Status | Notes |
|
||||||
|------|-------------------|-----------------|-------|
|
|------|-------------------|-----------------|-------|
|
||||||
| `-z`, `--compress` | Compress file data | 🔀 Alt Arg | Always uses zstd (rsync supports multiple algorithms) |
|
| `-z`, `--compress` | Compress file data | 🔀 Alt Arg | Always uses zstd (rsync supports multiple algorithms) |
|
||||||
| `--compress-choice=STR` | Choose compression algorithm | ❌ Not Implemented | Removed because it had no effect; FastSync always uses zstd |
|
| `--compress-choice=STR`, `--zc=STR` | Choose compression algorithm | ✅ Implemented | FastSync supports `zstd` and `none` |
|
||||||
| `--compress-level=NUM` | Set compression level | ✅ Implemented | 1-22, default 5 |
|
| `--compress-level=NUM`, `--zl=NUM` | Set compression level | ✅ Implemented | 1-22, default 5 |
|
||||||
| `--compress-threads=NUM` | Set compression threads | ❌ Not Implemented | |
|
| `--compress-threads=NUM` | Set compression threads | ❌ Not Implemented | |
|
||||||
| `--skip-compress=LIST` | Skip compress for suffixes | ❌ Not Implemented | Internal skip for hardcoded types; not user-configurable |
|
| `--skip-compress=LIST` | Skip compress for suffixes | ✅ Implemented | Comma-separated, case-insensitive suffix list; empty list skips none; incompatible with FastSync chunk serialization (`-s`) |
|
||||||
|
|
||||||
## 13. Connectivity
|
## 13. Connectivity
|
||||||
|
|
||||||
@@ -189,6 +199,7 @@ This document maps rsync's full feature set to FastSync's current implementation
|
|||||||
| `--address=ADDRESS` | Bind address for outgoing socket | ❌ Not Implemented | Removed because it had no effect |
|
| `--address=ADDRESS` | Bind address for outgoing socket | ❌ Not Implemented | Removed because it had no effect |
|
||||||
| `-4`, `--ipv4` | Prefer IPv4 | ❌ Not Implemented | Removed because it had no effect |
|
| `-4`, `--ipv4` | Prefer IPv4 | ❌ Not Implemented | Removed because it had no effect |
|
||||||
| `-6`, `--ipv6` | Prefer IPv6 | ❌ Not Implemented | Removed because it had no effect |
|
| `-6`, `--ipv6` | Prefer IPv6 | ❌ Not Implemented | Removed because it had no effect |
|
||||||
|
| `--remote-option=OPT`, `-M` | Send an option only to the remote side | ❌ Not Implemented | `-M` is FastSync's metadata-preservation flag |
|
||||||
|
|
||||||
## 14. Daemon Mode
|
## 14. Daemon Mode
|
||||||
|
|
||||||
@@ -210,8 +221,9 @@ This document maps rsync's full feature set to FastSync's current implementation
|
|||||||
| Protocol version check | Verify compatible versions | ✅ Implemented | `config_receive()` |
|
| Protocol version check | Verify compatible versions | ✅ Implemented | `config_receive()` |
|
||||||
| Max data/string/chunk sizes | Prevent OOM attacks | ✅ Implemented | Per-message limits |
|
| Max data/string/chunk sizes | Prevent OOM attacks | ✅ Implemented | Per-message limits |
|
||||||
| Per-connection memory limit | 1GB per connection | ✅ Implemented | `MAX_CONNECTION_MEMORY` |
|
| Per-connection memory limit | 1GB per connection | ✅ Implemented | `MAX_CONNECTION_MEMORY` |
|
||||||
|
| `--max-alloc=SIZE` | Limit a single memory allocation | ✅ Implemented | Caps the largest single allocation; binary units, default 1G |
|
||||||
| `--trust-sender` | Trust remote sender's file list | ❌ Not Implemented | |
|
| `--trust-sender` | Trust remote sender's file list | ❌ Not Implemented | |
|
||||||
| `--old-args` | Disable modern arg protection | ❌ Not Implemented | |
|
| `--old-args` | Disable modern arg protection | ✅ Implemented | SSH-only legacy mode; restores raw remote command construction and permits shell interpretation of the configured server path |
|
||||||
| `--ignore-missing-args` | Ignore missing source args | ❌ Not Implemented | |
|
| `--ignore-missing-args` | Ignore missing source args | ❌ Not Implemented | |
|
||||||
| `--delete-missing-args` | Delete missing source args | ❌ Not Implemented | |
|
| `--delete-missing-args` | Delete missing source args | ❌ Not Implemented | |
|
||||||
|
|
||||||
@@ -229,12 +241,118 @@ This document maps rsync's full feature set to FastSync's current implementation
|
|||||||
|------|-------------------|-----------------|-------|
|
|------|-------------------|-----------------|-------|
|
||||||
| `--stop-after=MINS` | Stop after N minutes | ❌ Not Implemented | |
|
| `--stop-after=MINS` | Stop after N minutes | ❌ Not Implemented | |
|
||||||
| `--stop-at=TIME` | Stop at specified time | ❌ Not Implemented | |
|
| `--stop-at=TIME` | Stop at specified time | ❌ Not Implemented | |
|
||||||
| `--fsync` | Fsync every written file | ❌ Not Implemented | |
|
| `--fsync` | Fsync every written file before publication | ✅ Implemented | |
|
||||||
| `--protocol=NUM` | Force older protocol version | ❌ Not Implemented | |
|
| `--protocol=NUM` | Force older protocol version | ❌ Not Implemented | |
|
||||||
| `--iconv=CONVERT_SPEC` | Charset conversion | ❌ Not Implemented | |
|
| `--iconv=CONVERT_SPEC` | Charset conversion | ❌ Not Implemented | |
|
||||||
| `--checksum-seed=NUM` | Set checksum seed | ❌ Not Implemented | |
|
| `--checksum-seed=NUM` | Set checksum seed | ❌ Not Implemented | |
|
||||||
| `-s`, `--secluded-args` | Use protocol to send args | ❌ Not Implemented | |
|
| `--secluded-args` | Use protocol to send args | 🔄 Compatibility No-op | Accepted for CLI compatibility; it does not change FastSync transport or protocol behavior. `-s` remains chunk serialization. |
|
||||||
| `--no-OPTION` | Turn off implied option | ❌ Not Implemented | |
|
| `--no-OPTION` | Turn off implied option | ✅ Supported | Supported boolean FastSync options and archive-implied options; unsafe or value-taking options are rejected. |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Implementation Difficulty Plan
|
||||||
|
|
||||||
|
The estimates below cover the currently unimplemented features in this document. They assume one engineer familiar with the codebase, include implementation and focused tests, and exclude production rollout time. A feature should not be marked implemented until its behavior is tested in both local and SSH/TCP paths where applicable.
|
||||||
|
|
||||||
|
> **Note:** This plan is a superset snapshot written while several of the listed features were still outstanding. The Summary matrix above is the authoritative record of what is already shipped (for example quiet/info/debug output, `--existing`, `--remove-source-files`, `-h`, and `--size-only` are now implemented on `dev`). Treat the phases as sequencing guidance for the work that remains unimplemented.
|
||||||
|
|
||||||
|
| Effort | Typical duration | Meaning |
|
||||||
|
|--------|------------------|---------|
|
||||||
|
| XS | 0.5-1 day | CLI alias or a local formatting/validation change |
|
||||||
|
| S | 1-3 days | Isolated behavior with little or no protocol change |
|
||||||
|
| M | 3-7 days | Cross-cutting client, server, or scanner behavior |
|
||||||
|
| L | 1-3 weeks | Protocol, filesystem, privilege, or compatibility work |
|
||||||
|
| XL | 3+ weeks | New transfer mode, daemon subsystem, or broad interoperability effort |
|
||||||
|
|
||||||
|
### Phase 1: Low-Risk CLI and Local Behavior
|
||||||
|
|
||||||
|
These are the best first changes because they require limited wire-format work and can be tested with existing transfer fixtures.
|
||||||
|
|
||||||
|
| Features | Effort | Implementation plan |
|
||||||
|
|----------|--------|--------------------|
|
||||||
|
| `--quiet`, `-q`; `--human-readable`, `-h`; `--8-bit-output`, `-8`; `--stderr=MODE`; `--info=FLAGS`; `--debug=FLAGS` | S | Extend logging and output formatting without changing transferred data. |
|
||||||
|
| `--no-OPTION`; `--old-args`; `--secluded-args`, `-s` | M | Add option implication/negation and safely serialize or protect remote arguments. `-s` currently has FastSync-specific semantics and needs a compatibility decision. |
|
||||||
|
| `-P`; `--del`; `--old-dirs`, `--old-d`; `--cc`; `--zc`; `--zl` | XS | Add aliases and composed behaviors after the underlying options exist. |
|
||||||
|
| `--whole-file`, `-W`; `--ignore-times`, `-I`; `--size-only`; `--modify-window`, `-@`; `--update`, `-u` | S | Extend the existing incremental comparison decision. |
|
||||||
|
| `--existing`; `--ignore-existing`; `--remove-source-files` | S | Add scanner/receiver eligibility checks and remove successfully synchronized source files. |
|
||||||
|
| `--executability`, `-E`; `--chmod=CHMOD` | M | Apply permission transformations safely while preserving current metadata behavior. |
|
||||||
|
| `--skip-compress=LIST`; `--compress-threads=NUM` | S | Make compression selection configurable and validate the thread setting against zstd behavior. |
|
||||||
|
| `--max-alloc=SIZE`; `--fsync` | S | Reuse existing allocation limits and add an explicit durability step after file writes. |
|
||||||
|
|
||||||
|
### Phase 2: Filesystem Selection and Update Semantics
|
||||||
|
|
||||||
|
These features are moderate because they affect traversal, temporary files, manifests, or the receiver's update policy.
|
||||||
|
|
||||||
|
| Features | Effort | Implementation plan |
|
||||||
|
|----------|--------|--------------------|
|
||||||
|
| `--one-file-system`, `-x` | M | Track the source device during scanner traversal and skip mount-point crossings. |
|
||||||
|
| `--relative`, `-R`; `--no-implied-dirs`; `--dirs`, `-d`; `--mkpath` | M | Extend path-list construction and destination directory creation while preserving traversal safety. |
|
||||||
|
| `--temp-dir`, `-T` | M | Separate temporary-file placement from FastSync's timeout alias and define collision, permissions, and cleanup rules. |
|
||||||
|
| `--delay-updates` | L | Stage all successful updates and publish them at completion, including crash and cancellation cleanup. |
|
||||||
|
| `--files-from=FILE`; `--from0`, `-0`; `--filter=RULE`, `-f`; `-F`; `--cvs-exclude`, `-C` | L | Build a complete filter/parser layer and integrate it with scanner pruning, manifests, and delete behavior. `-f` conflicts with FastSync sendfile mode. |
|
||||||
|
| `--list-only`; `--itemize-changes`, `-i`; `--out-format=FORMAT`; `--log-file-format=FMT` | M | Add a structured change-event model so output modes share one source of truth. |
|
||||||
|
|
||||||
|
### Phase 3: Deletion, Comparison, and Delta Compatibility
|
||||||
|
|
||||||
|
These features require careful interaction with manifests, incremental checks, backups, and the existing delta protocol.
|
||||||
|
|
||||||
|
| Features | Effort | Implementation plan |
|
||||||
|
|----------|--------|--------------------|
|
||||||
|
| `--delete-during`; `--delete-before`; `--delete-after`; `--delete-delay`; `--del` | L | Add deletion timing to the transfer state machine and ensure failures cannot remove files unexpectedly. |
|
||||||
|
| `--delete-excluded`; `--max-delete=NUM`; `--ignore-errors`; `--force`; `--prune-empty-dirs`, `-m` | M | Extend delete walks with policy limits, error handling, empty-directory pruning, and the `-m` short-flag conflict. |
|
||||||
|
| `--ignore-missing-args`; `--delete-missing-args` | M | Distinguish missing source arguments from traversal errors and apply explicit deletion policy. |
|
||||||
|
| `--compare-dest=DIR`; `--copy-dest=DIR`; `--link-dest=DIR` | L | Add alternate basis roots and hard-link handling, including metadata and cross-filesystem failures. |
|
||||||
|
| `--fuzzy`, `-y`; `--no-fuzzy` | L | Index candidate files and select a safe similar basis without making transfer time unbounded. |
|
||||||
|
| `--append`; `--append-verify` | M | Negotiate file length and verify the retained prefix before resuming. |
|
||||||
|
| `--checksum-choice=STR`, `--cc`; `--checksum-seed=NUM` | M | Negotiate checksum algorithms/seeds and preserve compatibility with existing xxHash checks. |
|
||||||
|
|
||||||
|
### Phase 4: Metadata, Links, and Devices
|
||||||
|
|
||||||
|
These features are platform-sensitive and need Linux permission, ACL, xattr, and special-file integration tests.
|
||||||
|
|
||||||
|
| Features | Effort | Implementation plan |
|
||||||
|
|----------|--------|--------------------|
|
||||||
|
| `--numeric-ids`; `--usermap=STRING`; `--groupmap=STRING`; `--chown=USER:GROUP` | L | Define identity mapping, privilege failures, and wire representation before applying ownership. |
|
||||||
|
| `--open-noatime`; `--atimes`, `-U`; `--crtimes`, `-N`; `--omit-dir-times`, `-O`; `--omit-link-times`, `-J` | L | Extend metadata capture/apply with platform capability checks and explicit unsupported-attribute handling. |
|
||||||
|
| `--acls`, `-A`; `--xattrs`, `-X`; `--fake-super` | XL | Add portable serialization, size limits, privilege behavior, and security tests for ACL/xattr data. |
|
||||||
|
| `--hard-links`, `-H` | L | Preserve inode relationships across the file list and coordinate hard-link creation order. |
|
||||||
|
| `--munge-links`; `--copy-dirlinks`, `-k`; `--keep-dirlinks`, `-K` | L | Define symlink trust boundaries and receiver-side directory/link collision behavior. |
|
||||||
|
| `--devices`; `--specials`; `-D`; `--copy-devices`; `--write-devices` | XL | Add privileged special-file handling with strict type, path, and authorization checks. |
|
||||||
|
| `--super`; `--copy-as=USER[:GROUP]` | XL | Requires a deliberate privilege model, identity switching, and refusal paths; do not implement by blindly elevating the process. |
|
||||||
|
| `--preallocate` | S | Use platform allocation APIs before writes and fall back cleanly when unsupported. |
|
||||||
|
|
||||||
|
### Phase 5: Connectivity and Daemon Compatibility
|
||||||
|
|
||||||
|
These options affect process startup, authentication, sockets, and remote execution. They should follow the filesystem and protocol work rather than being added as parser-only flags.
|
||||||
|
|
||||||
|
| Features | Effort | Implementation plan |
|
||||||
|
|----------|--------|--------------------|
|
||||||
|
| `--rsh=COMMAND`, `-e`; `--rsync-path=PROGRAM`; `--blocking-io`; `--outbuf=N\|L\|B` | M | Generalize SSH command construction and subprocess I/O while retaining argument escaping and timeout guarantees. |
|
||||||
|
| `--address=ADDRESS`; `--ipv4`, `-4`; `--ipv6`, `-6`; `--sockopts=OPTIONS`; `--port=PORT` daemon semantics | M | Add explicit socket-family/bind configuration and validate it independently for TCP client and daemon modes. |
|
||||||
|
| `--remote-option=OPT`, `-M`; `--trust-sender` | L | Add authenticated remote-option/config negotiation and reject unsafe sender-controlled values. `-M` conflicts with FastSync metadata mode. |
|
||||||
|
| `--daemon`; `--config=FILE`; `--dparam=OVERRIDE`; `--no-detach`; `--password-file=FILE`; `--early-input=FILE`; `--no-motd` | XL | Implement a real daemon lifecycle, module configuration, authentication, privilege separation, and process management. |
|
||||||
|
|
||||||
|
### Phase 6: Batch, Encoding, and Protocol Interoperability
|
||||||
|
|
||||||
|
These are the hardest compatibility items because they require durable formats or behavior that must interoperate with rsync itself.
|
||||||
|
|
||||||
|
| Features | Effort | Implementation plan |
|
||||||
|
|----------|--------|--------------------|
|
||||||
|
| `--write-batch=FILE`; `--only-write-batch=FILE`; `--read-batch=FILE` | XL | Specify a versioned batch format, persist all required metadata, and test replay, corruption, and partial application. |
|
||||||
|
| `--protocol=NUM` | XL | Add protocol-version negotiation and compatibility branches without weakening current validation. |
|
||||||
|
| `--iconv=CONVERT_SPEC` | L | Convert filenames at the protocol boundary with invalid-sequence and normalization tests. |
|
||||||
|
| `--stop-after=MINS`; `--stop-at=TIME` | M | Add deadline propagation, interruptible I/O, and safe checkpoint/cleanup behavior. |
|
||||||
|
| `--early-input=FILE`; `--password-file=FILE` | M | Securely read startup credentials/input with permission checks and no secret disclosure in logs. |
|
||||||
|
|
||||||
|
### Recommended Delivery Order
|
||||||
|
|
||||||
|
1. Resolve short-option conflicts (`-m`, `-M`, `-T`, `-f`, `-s`) and define the compatibility contract.
|
||||||
|
2. Implement Phase 1 comparison, update, output, and alias features with unit and integration coverage.
|
||||||
|
3. Implement Phase 2 traversal/filtering and Phase 3 deletion semantics.
|
||||||
|
4. Implement metadata and link features that are safe on the supported platforms.
|
||||||
|
5. Treat daemon mode, special files, batch mode, and protocol-version compatibility as separate projects.
|
||||||
|
|
||||||
|
The existing priority list below is a feature shortlist, not an implementation schedule; this plan supersedes it for effort and sequencing.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -247,13 +365,14 @@ Ranked by user demand, implementation complexity, and interoperability impact:
|
|||||||
| 1 | `--whole-file` / `-W` | Low | High — users expect opt-out of delta |
|
| 1 | `--whole-file` / `-W` | Low | High — users expect opt-out of delta |
|
||||||
| 2 | `--ignore-times` / `-I` | Low | Medium — useful for forcing re-transfer |
|
| 2 | `--ignore-times` / `-I` | Low | Medium — useful for forcing re-transfer |
|
||||||
| 3 | `--size-only` | Low | Medium — common migration scenario |
|
| 3 | `--size-only` | Low | Medium — common migration scenario |
|
||||||
| 4 | `--existing` / `--ignore-existing` | Low | Medium — common sync patterns |
|
| 4 | `--ignore-existing` | Low | Medium — common sync patterns |
|
||||||
| 5 | `--remove-source-files` | Low | High — common for moves/backup |
|
| 5 | `--existing` | Low | Medium — common sync patterns |
|
||||||
| 6 | `--delete-during` | Medium | High — performance improvement |
|
| 6 | `--remove-source-files` | Low | High — common for moves/backup |
|
||||||
| 7 | `--delay-updates` | Medium | High — atomic updates |
|
| 7 | `--delete-during` | Medium | High — performance improvement |
|
||||||
| 8 | `--chmod` | Low | Medium — permission flexibility |
|
| 8 | `--delay-updates` | Medium | High — atomic updates |
|
||||||
| 9 | `--executability` / `-E` | Low | Low — simple flag |
|
| 9 | `--chmod` | Low | Medium — permission flexibility |
|
||||||
| 10 | `--skip-compress` | Low | Medium — performance tuning |
|
| 10 | `--executability` / `-E` | Low | Low — simple flag |
|
||||||
|
| 11 | `--skip-compress` | Low | Medium — performance tuning |
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,310 @@
|
|||||||
|
#include "change_list.h"
|
||||||
|
#include "utils.h"
|
||||||
|
#include <limits.h>
|
||||||
|
#include <stdint.h>
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
|
#include <sys/stat.h>
|
||||||
|
#include <time.h>
|
||||||
|
|
||||||
|
/* Itemize code emitted for a transferred regular file.
|
||||||
|
*
|
||||||
|
* Layout (rsync-compatible 11-char item): `>f` marks a regular file that was
|
||||||
|
* transferred to the remote host; the trailing nine markers are, in order,
|
||||||
|
* c(hecksum) s(ize) t(ime) p(erms) o(wner) g(roup) u(ser/acl) a(ttrs) x(attrs).
|
||||||
|
* Every marker is `+` (FastSync does not compare each attribute on the
|
||||||
|
* receiving side, so a sent file is reported as fully updated). Files that
|
||||||
|
* are already up to date print no line at all, matching rsync's single -i
|
||||||
|
* which only itemizes changes.
|
||||||
|
*
|
||||||
|
* Because the scanner only yields regular-file transfer candidates, `>d`
|
||||||
|
* (directory) lines are never produced; directories are not transferred as
|
||||||
|
* items by FastSync. */
|
||||||
|
#define ITEMIZE_SENT_FILE ">f+++++++++"
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
char* data;
|
||||||
|
size_t length;
|
||||||
|
size_t capacity;
|
||||||
|
} StrBuf;
|
||||||
|
|
||||||
|
static void strbuf_free(StrBuf* buf) {
|
||||||
|
if (buf == NULL)
|
||||||
|
return;
|
||||||
|
free(buf->data);
|
||||||
|
buf->data = NULL;
|
||||||
|
buf->length = 0;
|
||||||
|
buf->capacity = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool strbuf_reserve(StrBuf* buf, size_t extra) {
|
||||||
|
if (buf->length > SIZE_MAX - extra - 1)
|
||||||
|
return false;
|
||||||
|
size_t need = buf->length + extra + 1;
|
||||||
|
if (need <= buf->capacity)
|
||||||
|
return true;
|
||||||
|
size_t capacity = buf->capacity > 0 ? buf->capacity : 32;
|
||||||
|
while (capacity < need) {
|
||||||
|
if (capacity > SIZE_MAX / 2) {
|
||||||
|
capacity = need;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
capacity *= 2;
|
||||||
|
}
|
||||||
|
char* grown = realloc(buf->data, capacity);
|
||||||
|
if (!grown)
|
||||||
|
return false;
|
||||||
|
buf->data = grown;
|
||||||
|
buf->capacity = capacity;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool strbuf_append_char(StrBuf* buf, char c) {
|
||||||
|
if (!strbuf_reserve(buf, 1))
|
||||||
|
return false;
|
||||||
|
buf->data[buf->length++] = c;
|
||||||
|
buf->data[buf->length] = '\0';
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool strbuf_append(StrBuf* buf, const char* text) {
|
||||||
|
if (text == NULL)
|
||||||
|
return true;
|
||||||
|
size_t length = strlen(text);
|
||||||
|
if (!strbuf_reserve(buf, length))
|
||||||
|
return false;
|
||||||
|
memcpy(buf->data + buf->length, text, length);
|
||||||
|
buf->length += length;
|
||||||
|
buf->data[buf->length] = '\0';
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool strbuf_append_ull(StrBuf* buf, unsigned long long value) {
|
||||||
|
char digits[32];
|
||||||
|
int written = snprintf(digits, sizeof(digits), "%llu", value);
|
||||||
|
if (written < 0 || (size_t)written >= sizeof(digits))
|
||||||
|
return false;
|
||||||
|
return strbuf_append(buf, digits);
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool strbuf_append_longlong(StrBuf* buf, long long value) {
|
||||||
|
char digits[32];
|
||||||
|
int written = snprintf(digits, sizeof(digits), "%lld", value);
|
||||||
|
if (written < 0 || (size_t)written >= sizeof(digits))
|
||||||
|
return false;
|
||||||
|
return strbuf_append(buf, digits);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool change_list_enabled(const Config* config) {
|
||||||
|
return config != NULL && (config->itemize_changes || config->out_format != NULL ||
|
||||||
|
(config->log_file != NULL && config->log_file_format != NULL));
|
||||||
|
}
|
||||||
|
|
||||||
|
char* change_render_itemize(const ChangeEvent* event) {
|
||||||
|
if (event == NULL || event->decision != CHANGE_SENT)
|
||||||
|
return str_dup("");
|
||||||
|
const char* code = event->is_directory ? ">d+++++++++" : ITEMIZE_SENT_FILE;
|
||||||
|
StrBuf line = {0};
|
||||||
|
bool ok = strbuf_append(&line, code) && strbuf_append(&line, " ") &&
|
||||||
|
strbuf_append(&line, event->path != NULL ? event->path : "");
|
||||||
|
if (!ok) {
|
||||||
|
strbuf_free(&line);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
return line.data;
|
||||||
|
}
|
||||||
|
|
||||||
|
static const char* leaf_name(const char* path) {
|
||||||
|
if (path == NULL)
|
||||||
|
return "";
|
||||||
|
const char* slash = strrchr(path, '/');
|
||||||
|
return slash != NULL && slash[1] != '\0' ? slash + 1 : path;
|
||||||
|
}
|
||||||
|
|
||||||
|
char* change_render_format(const char* format, const ChangeEvent* event) {
|
||||||
|
if (format == NULL)
|
||||||
|
return NULL;
|
||||||
|
StrBuf line = {0};
|
||||||
|
bool ok = true;
|
||||||
|
for (const char* p = format; *p != '\0' && ok;) {
|
||||||
|
if (*p != '%') {
|
||||||
|
ok = strbuf_append_char(&line, *p);
|
||||||
|
p++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
char token = p[1];
|
||||||
|
if (token == '\0') {
|
||||||
|
ok = strbuf_append_char(&line, '%');
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
switch (token) {
|
||||||
|
case '%':
|
||||||
|
ok = strbuf_append_char(&line, '%');
|
||||||
|
break;
|
||||||
|
case 'f':
|
||||||
|
ok = strbuf_append(&line, event->path != NULL ? event->path : "");
|
||||||
|
break;
|
||||||
|
case 'n':
|
||||||
|
ok = strbuf_append(&line, leaf_name(event->path));
|
||||||
|
break;
|
||||||
|
case 'l':
|
||||||
|
ok = strbuf_append_ull(&line, event->size);
|
||||||
|
break;
|
||||||
|
case 'b':
|
||||||
|
ok = strbuf_append_ull(&line, event->bytes_sent);
|
||||||
|
break;
|
||||||
|
case 'M':
|
||||||
|
ok = strbuf_append_longlong(&line, (long long)event->mtime_sec);
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
/* Unknown escape sequences are preserved verbatim. */
|
||||||
|
ok = strbuf_append_char(&line, '%') && strbuf_append_char(&line, token);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
p += 2;
|
||||||
|
}
|
||||||
|
if (!ok) {
|
||||||
|
strbuf_free(&line);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
if (line.data == NULL) {
|
||||||
|
line.data = str_dup("");
|
||||||
|
if (!line.data)
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
return line.data;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Format a mode as an `ls -l` permission string, e.g. `-rw-r--r--`. */
|
||||||
|
static void mode_to_ls_string(mode_t mode, char out[11]) {
|
||||||
|
out[0] = S_ISDIR(mode) ? 'd'
|
||||||
|
: S_ISLNK(mode) ? 'l'
|
||||||
|
: S_ISCHR(mode) ? 'c'
|
||||||
|
: S_ISBLK(mode) ? 'b'
|
||||||
|
: S_ISFIFO(mode) ? 'p'
|
||||||
|
: S_ISSOCK(mode) ? 's'
|
||||||
|
: '-';
|
||||||
|
mode_t bits = mode & 07777;
|
||||||
|
out[1] = (bits & S_IRUSR) ? 'r' : '-';
|
||||||
|
out[2] = (bits & S_IWUSR) ? 'w' : '-';
|
||||||
|
out[3] = (bits & S_IXUSR) ? (bits & S_ISUID ? 's' : 'x') : (bits & S_ISUID ? 'S' : '-');
|
||||||
|
out[4] = (bits & S_IRGRP) ? 'r' : '-';
|
||||||
|
out[5] = (bits & S_IWGRP) ? 'w' : '-';
|
||||||
|
out[6] = (bits & S_IXGRP) ? (bits & S_ISGID ? 's' : 'x') : (bits & S_ISGID ? 'S' : '-');
|
||||||
|
out[7] = (bits & S_IROTH) ? 'r' : '-';
|
||||||
|
out[8] = (bits & S_IWOTH) ? 'w' : '-';
|
||||||
|
out[9] = (bits & S_IXOTH) ? (bits & S_ISVTX ? 't' : 'x') : (bits & S_ISVTX ? 'T' : '-');
|
||||||
|
out[10] = '\0';
|
||||||
|
}
|
||||||
|
|
||||||
|
char* change_render_list_line(mode_t mode, unsigned long long size, time_t mtime,
|
||||||
|
const char* path) {
|
||||||
|
char permission[11];
|
||||||
|
mode_to_ls_string(mode, permission);
|
||||||
|
char date[32];
|
||||||
|
struct tm broken_down;
|
||||||
|
if (localtime_r(&mtime, &broken_down) != NULL) {
|
||||||
|
if (strftime(date, sizeof(date), "%Y/%m/%d %H:%M:%S", &broken_down) == 0)
|
||||||
|
snprintf(date, sizeof(date), "?");
|
||||||
|
} else {
|
||||||
|
snprintf(date, sizeof(date), "?");
|
||||||
|
}
|
||||||
|
StrBuf line = {0};
|
||||||
|
char size_field[32];
|
||||||
|
int written = snprintf(size_field, sizeof(size_field), "%llu", size);
|
||||||
|
if (written < 0 || (size_t)written >= sizeof(size_field)) {
|
||||||
|
strbuf_free(&line);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
bool ok = strbuf_append(&line, permission) && strbuf_append_char(&line, ' ') &&
|
||||||
|
strbuf_append(&line, size_field) && strbuf_append_char(&line, ' ') &&
|
||||||
|
strbuf_append(&line, date) && strbuf_append_char(&line, ' ') &&
|
||||||
|
strbuf_append(&line, path != NULL ? path : "");
|
||||||
|
if (!ok) {
|
||||||
|
strbuf_free(&line);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
return line.data;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void print_escaped_line(FILE* stream, const char* line, bool eight_bit_output) {
|
||||||
|
char* escaped = output_escape(line, eight_bit_output);
|
||||||
|
if (escaped != NULL) {
|
||||||
|
fprintf(stream, "%s\n", escaped);
|
||||||
|
free(escaped);
|
||||||
|
} else {
|
||||||
|
fprintf(stream, "%s\n", line);
|
||||||
|
}
|
||||||
|
fflush(stream);
|
||||||
|
}
|
||||||
|
|
||||||
|
void change_emit(const Config* config, const ChangeEvent* event) {
|
||||||
|
if (event == NULL || !change_list_enabled(config))
|
||||||
|
return;
|
||||||
|
if (event->decision == CHANGE_UP_TO_DATE)
|
||||||
|
return;
|
||||||
|
bool to_stdout = config->itemize_changes || config->out_format != NULL;
|
||||||
|
bool to_log = config->log_file != NULL && config->log_file_format != NULL;
|
||||||
|
if (to_stdout) {
|
||||||
|
char* line = config->out_format != NULL ? change_render_format(config->out_format, event)
|
||||||
|
: change_render_itemize(event);
|
||||||
|
if (line != NULL) {
|
||||||
|
print_escaped_line(stdout, line, config->eight_bit_output);
|
||||||
|
free(line);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (to_log) {
|
||||||
|
char* line = change_render_format(config->log_file_format, event);
|
||||||
|
if (line != NULL) {
|
||||||
|
print_escaped_line(config->log_file, line, config->eight_bit_output);
|
||||||
|
free(line);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool format_uses_mtime(const char* format) {
|
||||||
|
if (format == NULL)
|
||||||
|
return false;
|
||||||
|
/* Mirror change_render_format's tokenizer: "%%" is a literal percent (so
|
||||||
|
* "%%M" does NOT expand %M) and unknown "%X" escapes consume both chars.
|
||||||
|
* This keeps the optional stat() fallback below in step with the renderer. */
|
||||||
|
for (const char* p = format; *p != '\0';) {
|
||||||
|
if (*p != '%') {
|
||||||
|
p++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
char token = p[1];
|
||||||
|
if (token == '\0')
|
||||||
|
break;
|
||||||
|
if (token == 'M')
|
||||||
|
return true;
|
||||||
|
p += 2;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
void change_emit_file_sent(const Config* config, const File* file) {
|
||||||
|
if (file == NULL || !change_list_enabled(config))
|
||||||
|
return;
|
||||||
|
ChangeEvent event;
|
||||||
|
memset(&event, 0, sizeof(event));
|
||||||
|
event.path = file->path;
|
||||||
|
event.decision = CHANGE_SENT;
|
||||||
|
event.is_directory = false;
|
||||||
|
event.size = file->data != NULL ? file->data->size : 0;
|
||||||
|
/* FastSync has no wire-byte counter yet, so %b reports the source length
|
||||||
|
* that had to be delivered (always equal to %l); the actual bytes written
|
||||||
|
* to the socket (compressed/delta) are not measured. */
|
||||||
|
event.bytes_sent = event.size;
|
||||||
|
if (file->metadata != NULL) {
|
||||||
|
event.mtime_sec = file->metadata->mtime_sec;
|
||||||
|
} else if (format_uses_mtime(config->out_format) || format_uses_mtime(config->log_file_format)) {
|
||||||
|
/* Best-effort fallback for %M when no metadata was captured (no -M): the
|
||||||
|
* path is stat()ed just to fill the field, and any failure leaves 0. */
|
||||||
|
struct stat st;
|
||||||
|
if (file->path != NULL && stat(file->path, &st) == 0)
|
||||||
|
event.mtime_sec = st.st_mtime;
|
||||||
|
}
|
||||||
|
change_emit(config, &event);
|
||||||
|
}
|
||||||
@@ -0,0 +1,75 @@
|
|||||||
|
#ifndef CHANGE_LIST_H
|
||||||
|
#define CHANGE_LIST_H
|
||||||
|
|
||||||
|
#include "config.h"
|
||||||
|
#include "file_types.h"
|
||||||
|
#include <stdbool.h>
|
||||||
|
#include <sys/stat.h>
|
||||||
|
#include <time.h>
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Shared per-file change-event / output model (rsync --itemize-changes,
|
||||||
|
* --out-format, --log-file-format, and --list-only all render from here).
|
||||||
|
*
|
||||||
|
* FastSync is a push-style tool: the client sends files from the source tree
|
||||||
|
* to a server that writes them under the destination root. Events are
|
||||||
|
* emitted by whichever code path decides a file's fate (the single-threaded
|
||||||
|
* send loop and the `-m` sender thread both call the same per-file sender), so
|
||||||
|
* all change events are emitted by exactly one thread and itemize/out-format
|
||||||
|
* lines never interleave with each other. They may still interleave with
|
||||||
|
* legacy log messages (log.c) that share the same stdout/log-file stream.
|
||||||
|
*/
|
||||||
|
|
||||||
|
typedef enum {
|
||||||
|
CHANGE_SENT, /* file data (full or delta) was transmitted */
|
||||||
|
CHANGE_UP_TO_DATE, /* receiver already had an identical file; skipped */
|
||||||
|
} ChangeDecision;
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
const char* path; /* full source path */
|
||||||
|
ChangeDecision decision;
|
||||||
|
bool is_directory;
|
||||||
|
unsigned long long size; /* source file length in bytes */
|
||||||
|
/* The number of bytes reported for a sent file. FastSync has no wire-byte
|
||||||
|
* counter, so this is always the source length (== size / %l); actual
|
||||||
|
* post-compression/delta bytes on the wire are not counted. */
|
||||||
|
unsigned long long bytes_sent;
|
||||||
|
time_t mtime_sec; /* 0 when unknown */
|
||||||
|
} ChangeEvent;
|
||||||
|
|
||||||
|
/* True when any output mode is active and per-file events matter. */
|
||||||
|
bool change_list_enabled(const Config* config);
|
||||||
|
|
||||||
|
/* Render the rsync-style itemize line for a transferred file:
|
||||||
|
* `>f+++++++++ <path>`
|
||||||
|
* The 11-char code is `>f` (regular file transferred to the remote host)
|
||||||
|
* followed by c/s/t/p/o/g/u/a/x markers that are all `+` (value will be set
|
||||||
|
* / differs) because FastSync does not separately compare checksums, size,
|
||||||
|
* mtime, perms, owner, group, uid, acl, or xattr on the receiving side, so a
|
||||||
|
* sent file is reported as fully updated. Up-to-date files print no line
|
||||||
|
* (rsync single `-i` only shows changes). Caller frees the result. */
|
||||||
|
char* change_render_itemize(const ChangeEvent* event);
|
||||||
|
|
||||||
|
/* Expand an --out-format/--log-file-format template. Tokens:
|
||||||
|
* %f full source path %b "bytes sent" == the source length (%l);
|
||||||
|
* %n leaf (base) name actual post-compression/delta wire bytes
|
||||||
|
* %l file length in bytes are not counted
|
||||||
|
* %M mtime in whole seconds %% a literal percent sign
|
||||||
|
* Unknown %X sequences are preserved verbatim. Caller frees the result. */
|
||||||
|
char* change_render_format(const char* format, const ChangeEvent* event);
|
||||||
|
|
||||||
|
/* Render one --list-only long-listing entry:
|
||||||
|
* `-rw-r--r-- 12 2026/09/06 10:00:00 <path>`
|
||||||
|
* (ls -l style columns; mtime in the local time zone). Caller frees it. */
|
||||||
|
char* change_render_list_line(mode_t mode, unsigned long long size, time_t mtime, const char* path);
|
||||||
|
|
||||||
|
/* Emit an event to every active destination:
|
||||||
|
* stdout: --itemize-changes line, or the --out-format expansion when set;
|
||||||
|
* log file: the --log-file-format expansion (requires --log-file).
|
||||||
|
* CHANGE_UP_TO_DATE events produce no output. */
|
||||||
|
void change_emit(const Config* config, const ChangeEvent* event);
|
||||||
|
|
||||||
|
/* Build and emit a CHANGE_SENT event for a file the client just sent. */
|
||||||
|
void change_emit_file_sent(const Config* config, const File* file);
|
||||||
|
|
||||||
|
#endif
|
||||||
+602
-63
@@ -1,6 +1,7 @@
|
|||||||
#include "client_send.h"
|
#include "client_send.h"
|
||||||
#include "client_validation.h"
|
#include "client_validation.h"
|
||||||
#include "chmod.h"
|
#include "chmod.h"
|
||||||
|
#include "compression.h"
|
||||||
#include "config.h"
|
#include "config.h"
|
||||||
#include "delta.h"
|
#include "delta.h"
|
||||||
#include "log.h"
|
#include "log.h"
|
||||||
@@ -11,6 +12,7 @@
|
|||||||
#include "utils.h"
|
#include "utils.h"
|
||||||
#include <errno.h>
|
#include <errno.h>
|
||||||
#include <limits.h>
|
#include <limits.h>
|
||||||
|
#include <signal.h>
|
||||||
#include <stdbool.h>
|
#include <stdbool.h>
|
||||||
#include <stddef.h>
|
#include <stddef.h>
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
@@ -56,7 +58,7 @@ static bool parse_positive_int(const char* s, int* out_val) {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Duplicate a string argument into *dest, freeing the old value. Returns true on success, false on
|
/* Duplicate a string argument into *dest, freeing the old value. Returns 0 on success, -1 on
|
||||||
* failure. */
|
* failure. */
|
||||||
static int set_string_option(char** dest, const char* value, const char* option_name) {
|
static int set_string_option(char** dest, const char* value, const char* option_name) {
|
||||||
char* dup = str_dup(value);
|
char* dup = str_dup(value);
|
||||||
@@ -69,7 +71,7 @@ static int set_string_option(char** dest, const char* value, const char* option_
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Parse a string as a positive integer into *dest. Returns true on success, false on error. */
|
/* Parse a string as a positive integer into *dest. Returns 0 on success, -1 on error. */
|
||||||
static int set_positive_int_option(int* dest, const char* value, const char* option_name) {
|
static int set_positive_int_option(int* dest, const char* value, const char* option_name) {
|
||||||
if (!parse_positive_int(value, dest)) {
|
if (!parse_positive_int(value, dest)) {
|
||||||
log_message(LOG_LEVEL_ERROR, "%s must be a positive integer", option_name);
|
log_message(LOG_LEVEL_ERROR, "%s must be a positive integer", option_name);
|
||||||
@@ -78,7 +80,30 @@ static int set_positive_int_option(int* dest, const char* value, const char* opt
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Parse a string as a non-negative integer into *dest. Returns true on success, false on error. */
|
/* Set and validate the compression algorithm selected by the client. */
|
||||||
|
static int set_compression_choice(Config* config, const char* value) {
|
||||||
|
if (strcmp(value, "zstd") != 0 && strcmp(value, "none") != 0) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "--compress-choice must be zstd or none");
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
if (set_string_option(&config->compress_choice, value, "--compress-choice") != 0)
|
||||||
|
return -1;
|
||||||
|
config->use_compression = strcmp(value, "zstd") == 0;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
static int set_compression_threads_option(int* dest, const char* value) {
|
||||||
|
if (set_positive_int_option(dest, value, "--compress-threads") != 0)
|
||||||
|
return -1;
|
||||||
|
if (*dest > COMPRESSION_MAX_THREADS) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "--compress-threads must be between 1 and %d",
|
||||||
|
COMPRESSION_MAX_THREADS);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Parse a string as a non-negative integer into *dest. Returns 0 on success, -1 on error. */
|
||||||
static int set_nonneg_int_option(int* dest, const char* value, const char* option_name) {
|
static int set_nonneg_int_option(int* dest, const char* value, const char* option_name) {
|
||||||
if (!parse_nonneg_int(value, dest)) {
|
if (!parse_nonneg_int(value, dest)) {
|
||||||
log_message(LOG_LEVEL_ERROR, "%s must be a non-negative integer", option_name);
|
log_message(LOG_LEVEL_ERROR, "%s must be a non-negative integer", option_name);
|
||||||
@@ -87,8 +112,119 @@ static int set_nonneg_int_option(int* dest, const char* value, const char* optio
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static int set_stderr_mode(const char* value) {
|
||||||
|
if (strcmp(value, "errors") == 0 || strcmp(value, "e") == 0)
|
||||||
|
log_set_stderr_mode(LOG_STDERR_ERRORS);
|
||||||
|
else if (strcmp(value, "all") == 0 || strcmp(value, "a") == 0)
|
||||||
|
log_set_stderr_mode(LOG_STDERR_ALL);
|
||||||
|
else if (strcmp(value, "client") == 0 || strcmp(value, "c") == 0) {
|
||||||
|
log_message(LOG_LEVEL_ERROR,
|
||||||
|
"--stderr=client is not supported: FastSync has no client message channel");
|
||||||
|
return -1;
|
||||||
|
} else {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "--stderr must be errors or all");
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
static int read_patterns_from_file(const char* filepath, char*** patterns, int* count);
|
static int read_patterns_from_file(const char* filepath, char*** patterns, int* count);
|
||||||
|
|
||||||
|
static int parse_debug_flags(const char* value, Config* config) {
|
||||||
|
if (!value || value[0] == '\0' || value[0] == ',' || value[strlen(value) - 1] == ',' ||
|
||||||
|
strstr(value, ",,")) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "--debug requires at least one flag");
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
char* flags = str_dup(value);
|
||||||
|
if (!flags) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "memory allocation failed for --debug");
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
uint32_t parsed = (uint32_t)config->debug_level;
|
||||||
|
char* saveptr = NULL;
|
||||||
|
for (char* token = strtok_r(flags, ",", &saveptr); token != NULL;
|
||||||
|
token = strtok_r(NULL, ",", &saveptr)) {
|
||||||
|
uint32_t flag = 0;
|
||||||
|
if (strcmp(token, "help") == 0) {
|
||||||
|
print_debug_usage();
|
||||||
|
free(flags);
|
||||||
|
return 1;
|
||||||
|
} else if (strcmp(token, "all") == 0) {
|
||||||
|
parsed = LOG_DEBUG_ALL;
|
||||||
|
continue;
|
||||||
|
} else if (strcmp(token, "none") == 0) {
|
||||||
|
parsed = 0;
|
||||||
|
continue;
|
||||||
|
} else if (strcmp(token, "io") == 0) {
|
||||||
|
flag = LOG_DEBUG_IO;
|
||||||
|
} else if (strcmp(token, "proto") == 0) {
|
||||||
|
flag = LOG_DEBUG_PROTO;
|
||||||
|
} else if (strcmp(token, "pack") == 0) {
|
||||||
|
flag = LOG_DEBUG_PACK;
|
||||||
|
} else if (strcmp(token, "util") == 0) {
|
||||||
|
flag = LOG_DEBUG_UTIL;
|
||||||
|
} else {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "unsupported --debug flag: %s", token);
|
||||||
|
free(flags);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
parsed |= flag;
|
||||||
|
}
|
||||||
|
free(flags);
|
||||||
|
config->debug_level = (int)parsed;
|
||||||
|
set_log_debug_flags(parsed);
|
||||||
|
set_log_level(LOG_LEVEL_DEBUG);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
static int parse_info_flags(const char* value, Config* config) {
|
||||||
|
if (!value || value[0] == '\0' || value[0] == ',' || value[strlen(value) - 1] == ',' ||
|
||||||
|
strstr(value, ",,")) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "--info requires at least one flag");
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
char* flags = str_dup(value);
|
||||||
|
if (!flags) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "memory allocation failed for --info");
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
uint32_t parsed = (uint32_t)config->info_level;
|
||||||
|
char* saveptr = NULL;
|
||||||
|
for (char* token = strtok_r(flags, ",", &saveptr); token != NULL;
|
||||||
|
token = strtok_r(NULL, ",", &saveptr)) {
|
||||||
|
uint32_t flag = 0;
|
||||||
|
if (strcmp(token, "all") == 0) {
|
||||||
|
parsed = LOG_INFO_ALL;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (strcmp(token, "none") == 0) {
|
||||||
|
parsed = 0;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (strcmp(token, "copy") == 0)
|
||||||
|
flag = LOG_INFO_COPY;
|
||||||
|
else if (strcmp(token, "misc") == 0)
|
||||||
|
flag = LOG_INFO_MISC;
|
||||||
|
else if (strcmp(token, "skip") == 0)
|
||||||
|
flag = LOG_INFO_SKIP;
|
||||||
|
else if (strcmp(token, "stats") == 0)
|
||||||
|
flag = LOG_INFO_STATS;
|
||||||
|
else {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "unsupported --info flag: %s", token);
|
||||||
|
free(flags);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
parsed |= flag;
|
||||||
|
}
|
||||||
|
free(flags);
|
||||||
|
config->info_level = (int)parsed;
|
||||||
|
set_log_info_flags(parsed);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
/* Parse a string as an unsigned long long. Returns 0 on success, -1 on error. */
|
/* Parse a string as an unsigned long long. Returns 0 on success, -1 on error. */
|
||||||
static int parse_ull_arg(const char* val, unsigned long long* out, const char* optname) {
|
static int parse_ull_arg(const char* val, unsigned long long* out, const char* optname) {
|
||||||
char* end;
|
char* end;
|
||||||
@@ -102,6 +238,63 @@ static int parse_ull_arg(const char* val, unsigned long long* out, const char* o
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Parse a byte count with an optional single-letter binary suffix (K/M/G/T/P/E).
|
||||||
|
* When allow_zero is false, a bare 0 is rejected (size limits use true, since 0
|
||||||
|
* means "no limit"). Returns 0 on success, -1 on error. */
|
||||||
|
static int parse_size_arg_allow_zero(const char* value, unsigned long long* out, bool allow_zero) {
|
||||||
|
if (!value || *value < '0' || *value > '9')
|
||||||
|
return -1;
|
||||||
|
char* end;
|
||||||
|
errno = 0;
|
||||||
|
unsigned long long number = strtoull(value, &end, 10);
|
||||||
|
if (errno != 0 || end == value)
|
||||||
|
return -1;
|
||||||
|
unsigned long long multiplier = 1;
|
||||||
|
if (*end != '\0') {
|
||||||
|
if (end[1] != '\0')
|
||||||
|
return -1;
|
||||||
|
switch (*end) {
|
||||||
|
case 'b':
|
||||||
|
case 'B':
|
||||||
|
break;
|
||||||
|
case 'k':
|
||||||
|
case 'K':
|
||||||
|
multiplier = 1024ULL;
|
||||||
|
break;
|
||||||
|
case 'm':
|
||||||
|
case 'M':
|
||||||
|
multiplier = 1024ULL * 1024;
|
||||||
|
break;
|
||||||
|
case 'g':
|
||||||
|
case 'G':
|
||||||
|
multiplier = 1024ULL * 1024 * 1024;
|
||||||
|
break;
|
||||||
|
case 't':
|
||||||
|
case 'T':
|
||||||
|
multiplier = 1024ULL * 1024 * 1024 * 1024;
|
||||||
|
break;
|
||||||
|
case 'p':
|
||||||
|
case 'P':
|
||||||
|
multiplier = 1024ULL * 1024 * 1024 * 1024 * 1024;
|
||||||
|
break;
|
||||||
|
case 'e':
|
||||||
|
case 'E':
|
||||||
|
multiplier = 1024ULL * 1024 * 1024 * 1024 * 1024 * 1024;
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ((!allow_zero && number == 0) || number > ULLONG_MAX / multiplier)
|
||||||
|
return -1;
|
||||||
|
*out = number * multiplier;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
static int parse_size_arg(const char* value, unsigned long long* out) {
|
||||||
|
return parse_size_arg_allow_zero(value, out, false);
|
||||||
|
}
|
||||||
|
|
||||||
/* Append a duplicated pattern to a growable pattern array. Returns 0 on success, -1 on error. */
|
/* Append a duplicated pattern to a growable pattern array. Returns 0 on success, -1 on error. */
|
||||||
static int config_add_pattern(char*** patterns, int* count, const char* value,
|
static int config_add_pattern(char*** patterns, int* count, const char* value,
|
||||||
const char* optname) {
|
const char* optname) {
|
||||||
@@ -120,41 +313,93 @@ static int config_add_pattern(char*** patterns, int* count, const char* value,
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static int parse_skip_compress(Config* config, const char* value) {
|
||||||
|
char* list = str_dup(value);
|
||||||
|
if (!list)
|
||||||
|
return -1;
|
||||||
|
config->skip_compress_set = true;
|
||||||
|
for (char* token = strtok(list, ","); token; token = strtok(NULL, ",")) {
|
||||||
|
while (*token == ' ' || *token == '\t')
|
||||||
|
token++;
|
||||||
|
size_t len = strlen(token);
|
||||||
|
while (len > 0 && (token[len - 1] == ' ' || token[len - 1] == '\t'))
|
||||||
|
token[--len] = '\0';
|
||||||
|
if (len == 0)
|
||||||
|
continue;
|
||||||
|
if (config_add_pattern(&config->skip_compress_suffixes, &config->skip_compress_count, token,
|
||||||
|
"--skip-compress") != 0) {
|
||||||
|
free(list);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
free(list);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
typedef enum {
|
typedef enum {
|
||||||
OPT_FLAG,
|
OPT_FLAG,
|
||||||
|
OPT_NOOP,
|
||||||
OPT_STRING,
|
OPT_STRING,
|
||||||
OPT_POS_INT,
|
OPT_POS_INT,
|
||||||
OPT_NONNEG_INT,
|
OPT_NONNEG_INT,
|
||||||
OPT_ULL,
|
OPT_ULL,
|
||||||
|
OPT_UNSUPPORTED,
|
||||||
} OptKind;
|
} OptKind;
|
||||||
|
|
||||||
typedef struct {
|
typedef struct {
|
||||||
const char* name;
|
const char* name;
|
||||||
const char* alias;
|
const char* alias;
|
||||||
OptKind kind;
|
OptKind kind;
|
||||||
size_t offset; /* offsetof of the target field in Config */
|
size_t offset; /* offsetof of the target field in Config, or 0 for OPT_NOOP */
|
||||||
} OptionEntry;
|
} OptionEntry;
|
||||||
|
|
||||||
|
/* Options parsed directly into Config, plus compatibility options with no effect. */
|
||||||
|
typedef struct {
|
||||||
|
const char* name;
|
||||||
|
const char* alias;
|
||||||
|
size_t offset; /* offsetof of the boolean target field in Config */
|
||||||
|
} NegatableOption;
|
||||||
|
|
||||||
/* Options that map directly onto a Config field with no side effects. */
|
/* Options that map directly onto a Config field with no side effects. */
|
||||||
static const OptionEntry OPTION_TABLE[] = {
|
static const OptionEntry OPTION_TABLE[] = {
|
||||||
{"--dry-run", "-n", OPT_FLAG, offsetof(Config, dry_run)},
|
{"--dry-run", "-n", OPT_FLAG, offsetof(Config, dry_run)},
|
||||||
|
{"--remove-source-files", NULL, OPT_FLAG, offsetof(Config, remove_source_files)},
|
||||||
{"--delete", NULL, OPT_FLAG, offsetof(Config, use_delete)},
|
{"--delete", NULL, OPT_FLAG, offsetof(Config, use_delete)},
|
||||||
{"--incremental", NULL, OPT_FLAG, offsetof(Config, use_incremental)},
|
{"--incremental", NULL, OPT_FLAG, offsetof(Config, use_incremental)},
|
||||||
|
{"--size-only", NULL, OPT_FLAG, offsetof(Config, size_only)},
|
||||||
|
{"--ignore-times", "-I", OPT_FLAG, offsetof(Config, ignore_times)},
|
||||||
|
{"--modify-window", "-@", OPT_NONNEG_INT, offsetof(Config, modify_window)},
|
||||||
{"--delta", NULL, OPT_FLAG, offsetof(Config, use_delta)},
|
{"--delta", NULL, OPT_FLAG, offsetof(Config, use_delta)},
|
||||||
|
{"--whole-file", "-W", OPT_FLAG, offsetof(Config, whole_file)},
|
||||||
{"--save-to-disk", NULL, OPT_FLAG, offsetof(Config, save_to_disk)},
|
{"--save-to-disk", NULL, OPT_FLAG, offsetof(Config, save_to_disk)},
|
||||||
{"--progress", NULL, OPT_FLAG, offsetof(Config, show_progress)},
|
{"--progress", NULL, OPT_FLAG, offsetof(Config, show_progress)},
|
||||||
{"--tls", NULL, OPT_FLAG, offsetof(Config, use_tls)},
|
{"--tls", NULL, OPT_FLAG, offsetof(Config, use_tls)},
|
||||||
{"--backup", NULL, OPT_FLAG, offsetof(Config, backup)},
|
{"--backup", NULL, OPT_FLAG, offsetof(Config, backup)},
|
||||||
{"--stats", NULL, OPT_FLAG, offsetof(Config, stats)},
|
{"--stats", NULL, OPT_FLAG, offsetof(Config, stats)},
|
||||||
|
{"--human-readable", "-h", OPT_FLAG, offsetof(Config, human_readable)},
|
||||||
{"--partial", NULL, OPT_FLAG, offsetof(Config, partial)},
|
{"--partial", NULL, OPT_FLAG, offsetof(Config, partial)},
|
||||||
|
{"--secluded-args", NULL, OPT_NOOP, 0},
|
||||||
|
{"--update", "-u", OPT_FLAG, offsetof(Config, update)},
|
||||||
|
{"--old-args", NULL, OPT_FLAG, offsetof(Config, old_args)},
|
||||||
{"--links", "-l", OPT_FLAG, offsetof(Config, follow_symlinks)},
|
{"--links", "-l", OPT_FLAG, offsetof(Config, follow_symlinks)},
|
||||||
{"--copy-links", NULL, OPT_FLAG, offsetof(Config, copy_links)},
|
{"--copy-links", NULL, OPT_FLAG, offsetof(Config, copy_links)},
|
||||||
{"--safe-links", NULL, OPT_FLAG, offsetof(Config, safe_links)},
|
{"--safe-links", NULL, OPT_FLAG, offsetof(Config, safe_links)},
|
||||||
{"--copy-unsafe-links", NULL, OPT_FLAG, offsetof(Config, copy_unsafe_links)},
|
{"--copy-unsafe-links", NULL, OPT_FLAG, offsetof(Config, copy_unsafe_links)},
|
||||||
{"--sparse", "-S", OPT_FLAG, offsetof(Config, preserve_sparse)},
|
{"--sparse", "-S", OPT_FLAG, offsetof(Config, preserve_sparse)},
|
||||||
{"--inplace", NULL, OPT_FLAG, offsetof(Config, inplace)},
|
{"--inplace", NULL, OPT_FLAG, offsetof(Config, inplace)},
|
||||||
|
{"--fsync", NULL, OPT_FLAG, offsetof(Config, use_fsync)},
|
||||||
{"--checksum", NULL, OPT_FLAG, offsetof(Config, checksum)},
|
{"--checksum", NULL, OPT_FLAG, offsetof(Config, checksum)},
|
||||||
|
{"--8-bit-output", "-8", OPT_FLAG, offsetof(Config, eight_bit_output)},
|
||||||
|
{"--itemize-changes", "-i", OPT_FLAG, offsetof(Config, itemize_changes)},
|
||||||
|
{"--list-only", NULL, OPT_FLAG, offsetof(Config, list_only)},
|
||||||
|
{"--out-format", NULL, OPT_STRING, offsetof(Config, out_format)},
|
||||||
|
{"--log-file-format", NULL, OPT_STRING, offsetof(Config, log_file_format)},
|
||||||
|
{"--existing", NULL, OPT_FLAG, offsetof(Config, existing)},
|
||||||
|
{"--ignore-existing", NULL, OPT_FLAG, offsetof(Config, ignore_existing)},
|
||||||
{"--chmod", NULL, OPT_STRING, offsetof(Config, chmod_spec)},
|
{"--chmod", NULL, OPT_STRING, offsetof(Config, chmod_spec)},
|
||||||
|
{"--dirs", "--old-dirs", OPT_UNSUPPORTED, 0},
|
||||||
|
{"--old-d", NULL, OPT_UNSUPPORTED, 0},
|
||||||
|
{"--delete-during", "--del", OPT_UNSUPPORTED, 0},
|
||||||
|
|
||||||
{"--source-dir", NULL, OPT_STRING, offsetof(Config, send_directory)},
|
{"--source-dir", NULL, OPT_STRING, offsetof(Config, send_directory)},
|
||||||
{"--dest-dir", NULL, OPT_STRING, offsetof(Config, receive_root_directory)},
|
{"--dest-dir", NULL, OPT_STRING, offsetof(Config, receive_root_directory)},
|
||||||
@@ -166,6 +411,8 @@ static const OptionEntry OPTION_TABLE[] = {
|
|||||||
{"--fastsync-server-path", NULL, OPT_STRING, offsetof(Config, fastsync_server_path)},
|
{"--fastsync-server-path", NULL, OPT_STRING, offsetof(Config, fastsync_server_path)},
|
||||||
{"--partial-dir", NULL, OPT_STRING, offsetof(Config, partial_dir)},
|
{"--partial-dir", NULL, OPT_STRING, offsetof(Config, partial_dir)},
|
||||||
{"--suffix", NULL, OPT_STRING, offsetof(Config, suffix)},
|
{"--suffix", NULL, OPT_STRING, offsetof(Config, suffix)},
|
||||||
|
{"--compress-choice", "--zc", OPT_STRING, offsetof(Config, compress_choice)},
|
||||||
|
{"--compress-level", "--zl", OPT_POS_INT, offsetof(Config, compression_level)},
|
||||||
|
|
||||||
{"--timeout", NULL, OPT_POS_INT, offsetof(Config, timeout)},
|
{"--timeout", NULL, OPT_POS_INT, offsetof(Config, timeout)},
|
||||||
{"--contimeout", NULL, OPT_POS_INT, offsetof(Config, contimeout)},
|
{"--contimeout", NULL, OPT_POS_INT, offsetof(Config, contimeout)},
|
||||||
@@ -175,6 +422,35 @@ static const OptionEntry OPTION_TABLE[] = {
|
|||||||
{"--min-size", NULL, OPT_ULL, offsetof(Config, min_size)},
|
{"--min-size", NULL, OPT_ULL, offsetof(Config, min_size)},
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/* Only boolean options with no required argument are safe to negate. */
|
||||||
|
static const NegatableOption NEGATABLE_OPTIONS[] = {
|
||||||
|
{"dry-run", "n", offsetof(Config, dry_run)},
|
||||||
|
{"delete", NULL, offsetof(Config, use_delete)},
|
||||||
|
{"incremental", NULL, offsetof(Config, use_incremental)},
|
||||||
|
{"delta", NULL, offsetof(Config, use_delta)},
|
||||||
|
{"save-to-disk", NULL, offsetof(Config, save_to_disk)},
|
||||||
|
{"progress", NULL, offsetof(Config, show_progress)},
|
||||||
|
{"tls", NULL, offsetof(Config, use_tls)},
|
||||||
|
{"backup", NULL, offsetof(Config, backup)},
|
||||||
|
{"stats", NULL, offsetof(Config, stats)},
|
||||||
|
{"partial", NULL, offsetof(Config, partial)},
|
||||||
|
{"links", "l", offsetof(Config, follow_symlinks)},
|
||||||
|
{"copy-links", NULL, offsetof(Config, copy_links)},
|
||||||
|
{"safe-links", NULL, offsetof(Config, safe_links)},
|
||||||
|
{"copy-unsafe-links", NULL, offsetof(Config, copy_unsafe_links)},
|
||||||
|
{"sparse", "S", offsetof(Config, preserve_sparse)},
|
||||||
|
{"inplace", NULL, offsetof(Config, inplace)},
|
||||||
|
{"checksum", NULL, offsetof(Config, checksum)},
|
||||||
|
|
||||||
|
/* These options are also implied by --archive or handled outside the table. */
|
||||||
|
{"compress", "c", offsetof(Config, use_compression)},
|
||||||
|
{"compress", "z", offsetof(Config, use_compression)},
|
||||||
|
{"multithreading", "m", offsetof(Config, use_multithreading)},
|
||||||
|
{"preserve", "M", offsetof(Config, use_metadata)},
|
||||||
|
{"sendfile", "f", offsetof(Config, use_sendfile)},
|
||||||
|
{"chunk-serialization", "s", offsetof(Config, use_chunk_serialization)},
|
||||||
|
};
|
||||||
|
|
||||||
static bool opt_is(const char* arg, const char* name, const char* alias) {
|
static bool opt_is(const char* arg, const char* name, const char* alias) {
|
||||||
return strcmp(arg, name) == 0 || (alias && strcmp(arg, alias) == 0);
|
return strcmp(arg, name) == 0 || (alias && strcmp(arg, alias) == 0);
|
||||||
}
|
}
|
||||||
@@ -186,11 +462,65 @@ static const OptionEntry* find_table_option(const char* arg) {
|
|||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
static int apply_table_option(Config* config, const OptionEntry* entry, const char* value) {
|
/* Match a "--opt=value" argument against table options that take a value. Flags,
|
||||||
|
* no-ops, and unsupported options do not accept an inline "=" value. */
|
||||||
|
static const OptionEntry* find_table_option_with_equals(const char* arg, const char** value) {
|
||||||
|
const char* equals = strchr(arg, '=');
|
||||||
|
if (!equals || equals == arg)
|
||||||
|
return NULL;
|
||||||
|
size_t name_len = (size_t)(equals - arg);
|
||||||
|
for (size_t i = 0; i < sizeof(OPTION_TABLE) / sizeof(OPTION_TABLE[0]); i++) {
|
||||||
|
const OptionEntry* entry = &OPTION_TABLE[i];
|
||||||
|
if ((strlen(entry->name) == name_len && strncmp(arg, entry->name, name_len) == 0) ||
|
||||||
|
(entry->alias && strlen(entry->alias) == name_len &&
|
||||||
|
strncmp(arg, entry->alias, name_len) == 0)) {
|
||||||
|
if (entry->kind == OPT_STRING || entry->kind == OPT_POS_INT ||
|
||||||
|
entry->kind == OPT_NONNEG_INT || entry->kind == OPT_ULL) {
|
||||||
|
*value = equals + 1;
|
||||||
|
return entry;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
static const NegatableOption* find_negatable_option(const char* name) {
|
||||||
|
for (size_t i = 0; i < sizeof(NEGATABLE_OPTIONS) / sizeof(NEGATABLE_OPTIONS[0]); i++)
|
||||||
|
if (strcmp(name, NEGATABLE_OPTIONS[i].name) == 0 ||
|
||||||
|
(NEGATABLE_OPTIONS[i].alias && strcmp(name, NEGATABLE_OPTIONS[i].alias) == 0))
|
||||||
|
return &NEGATABLE_OPTIONS[i];
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
static int apply_negation(Config* config, const char* arg) {
|
||||||
|
const char* name = arg + strlen("--no-");
|
||||||
|
if (*name == '\0') {
|
||||||
|
fprintf(stderr, "Cannot negate an empty option name: %s\n", arg);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
const NegatableOption* entry = find_negatable_option(name);
|
||||||
|
if (!entry) {
|
||||||
|
fprintf(stderr, "Cannot negate unsupported or unsafe option: %s\n", arg);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
*(bool*)((char*)config + entry->offset) = false;
|
||||||
|
if (entry->offset == offsetof(Config, use_metadata))
|
||||||
|
config->metadata_explicitly_disabled = true;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
static int apply_table_option(Config* config, const OptionEntry* entry, const char* option_name,
|
||||||
|
const char* value) {
|
||||||
|
if (entry->kind == OPT_NOOP)
|
||||||
|
return 0;
|
||||||
void* field = (char*)config + entry->offset;
|
void* field = (char*)config + entry->offset;
|
||||||
switch (entry->kind) {
|
switch (entry->kind) {
|
||||||
case OPT_FLAG:
|
case OPT_FLAG:
|
||||||
*(bool*)field = true;
|
*(bool*)field = true;
|
||||||
|
if (entry->offset == offsetof(Config, update))
|
||||||
|
config->use_metadata = true;
|
||||||
|
return 0;
|
||||||
|
case OPT_NOOP:
|
||||||
return 0;
|
return 0;
|
||||||
case OPT_STRING:
|
case OPT_STRING:
|
||||||
return set_string_option((char**)field, value, entry->name);
|
return set_string_option((char**)field, value, entry->name);
|
||||||
@@ -200,11 +530,23 @@ static int apply_table_option(Config* config, const OptionEntry* entry, const ch
|
|||||||
return set_nonneg_int_option((int*)field, value, entry->name);
|
return set_nonneg_int_option((int*)field, value, entry->name);
|
||||||
case OPT_ULL: {
|
case OPT_ULL: {
|
||||||
unsigned long long v;
|
unsigned long long v;
|
||||||
if (parse_ull_arg(value, &v, entry->name) != 0)
|
/* Size-limit options accept rsync-style suffixes (e.g. --max-size=2G); a
|
||||||
|
* plain byte count, including 0 ("no limit"), stays valid. */
|
||||||
|
if (parse_size_arg_allow_zero(value, &v, true) != 0) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "%s must be a non-negative size (B, K, M, G, T, P, or E)",
|
||||||
|
entry->name);
|
||||||
return -1;
|
return -1;
|
||||||
|
}
|
||||||
*(unsigned long long*)field = v;
|
*(unsigned long long*)field = v;
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
case OPT_UNSUPPORTED: {
|
||||||
|
const char* reason = "directory-only transfer is not implemented";
|
||||||
|
if (strcmp(entry->name, "--delete-during") == 0)
|
||||||
|
reason = "delete-during is not implemented";
|
||||||
|
log_message(LOG_LEVEL_ERROR, "%s: %s; refusing to ignore option", option_name, reason);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
@@ -212,30 +554,114 @@ static int apply_table_option(Config* config, const OptionEntry* entry, const ch
|
|||||||
/* Parse CLI arguments into config. Returns 0 on success, -1 on error, 1 for help/clean-exit. */
|
/* Parse CLI arguments into config. Returns 0 on success, -1 on error, 1 for help/clean-exit. */
|
||||||
int parse_args(Config* config, int argc, char* argv[], int* positional_args,
|
int parse_args(Config* config, int argc, char* argv[], int* positional_args,
|
||||||
int* positional_count) {
|
int* positional_count) {
|
||||||
|
bool verbose = false;
|
||||||
|
protocol_set_8_bit_output(config->eight_bit_output);
|
||||||
|
|
||||||
|
/* Apply output controls before processing other options so their order is irrelevant. */
|
||||||
for (int i = 1; i < argc; i++) {
|
for (int i = 1; i < argc; i++) {
|
||||||
const OptionEntry* entry = find_table_option(argv[i]);
|
if (strcmp(argv[i], "-v") == 0 || strcmp(argv[i], "--verbose") == 0) {
|
||||||
if (entry) {
|
set_log_level(LOG_LEVEL_DEBUG);
|
||||||
if (entry->kind != OPT_FLAG) {
|
} else if (strncmp(argv[i], "--info=", 7) == 0) {
|
||||||
|
if (parse_info_flags(argv[i] + 7, config) != 0)
|
||||||
|
return -1;
|
||||||
|
} else if (strcmp(argv[i], "--info") == 0) {
|
||||||
|
if (i + 1 >= argc || parse_info_flags(argv[++i], config) != 0)
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (int i = 1; i < argc; i++) {
|
||||||
|
if (strcmp(argv[i], "-P") == 0) {
|
||||||
|
config->partial = true;
|
||||||
|
config->show_progress = true;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (strncmp(argv[i], "--no-", strlen("--no-")) == 0) {
|
||||||
|
if (apply_negation(config, argv[i]) != 0)
|
||||||
|
return -1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const char* modify_window_prefix = "--modify-window=";
|
||||||
|
if (strncmp(argv[i], modify_window_prefix, strlen(modify_window_prefix)) == 0) {
|
||||||
|
if (set_nonneg_int_option(&config->modify_window, argv[i] + strlen(modify_window_prefix),
|
||||||
|
"--modify-window") != 0)
|
||||||
|
return -1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (strncmp(argv[i], "-@", 2) == 0 && argv[i][2] != '\0') {
|
||||||
|
if (set_nonneg_int_option(&config->modify_window, argv[i] + 2, "-@") != 0)
|
||||||
|
return -1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const char* threads_prefix = "--compress-threads=";
|
||||||
|
if (strncmp(argv[i], threads_prefix, strlen(threads_prefix)) == 0) {
|
||||||
|
if (set_compression_threads_option(&config->compression_threads,
|
||||||
|
argv[i] + strlen(threads_prefix)) != 0)
|
||||||
|
return -1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (strncmp(argv[i], "--max-alloc=", 12) == 0 || strcmp(argv[i], "--max-alloc") == 0) {
|
||||||
|
const char* value = strcmp(argv[i], "--max-alloc") == 0 ? "" : argv[i] + 12;
|
||||||
|
if (*value == '\0') {
|
||||||
if (i + 1 >= argc) {
|
if (i + 1 >= argc) {
|
||||||
log_message(LOG_LEVEL_ERROR, "missing argument for %s", entry->name);
|
log_message(LOG_LEVEL_ERROR, "missing argument for --max-alloc");
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
if (apply_table_option(config, entry, argv[++i]) != 0)
|
value = argv[++i];
|
||||||
return -1;
|
}
|
||||||
if (entry->offset == offsetof(Config, chmod_spec)) {
|
if (parse_size_arg(value, &config->max_alloc) != 0) {
|
||||||
mode_t ignored;
|
log_message(LOG_LEVEL_ERROR,
|
||||||
if (!chmod_apply(0, config->chmod_spec, &ignored)) {
|
"--max-alloc must be a positive size (B, K, M, G, T, P, or E)");
|
||||||
log_message(LOG_LEVEL_ERROR, "--chmod has invalid permission changes");
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
config->use_metadata = true;
|
|
||||||
}
|
|
||||||
} else if (apply_table_option(config, entry, NULL) != 0) {
|
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const OptionEntry* entry = find_table_option(argv[i]);
|
||||||
|
const char* inline_value = NULL;
|
||||||
|
if (!entry)
|
||||||
|
entry = find_table_option_with_equals(argv[i], &inline_value);
|
||||||
|
if (entry) {
|
||||||
|
const char* option_name = argv[i];
|
||||||
|
const char* value = NULL;
|
||||||
|
if (entry->kind != OPT_FLAG) {
|
||||||
|
if (entry->kind != OPT_UNSUPPORTED) {
|
||||||
|
value = inline_value;
|
||||||
|
if (!value && i + 1 < argc)
|
||||||
|
value = argv[++i];
|
||||||
|
if (!value) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "missing argument for %s", entry->name);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (strcmp(entry->name, "--compress-choice") == 0) {
|
||||||
|
if (set_compression_choice(config, value) != 0)
|
||||||
|
return -1;
|
||||||
|
} else {
|
||||||
|
if (apply_table_option(config, entry, option_name, value) != 0)
|
||||||
|
return -1;
|
||||||
|
if (strcmp(entry->name, "--compress-level") == 0 &&
|
||||||
|
(config->compression_level < 1 || config->compression_level > 22)) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "--compress-level must be between 1 and 22");
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
if (entry->offset == offsetof(Config, chmod_spec)) {
|
||||||
|
mode_t ignored;
|
||||||
|
if (!chmod_apply(0, config->chmod_spec, &ignored)) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "--chmod has invalid permission changes");
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
config->use_metadata = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if (apply_table_option(config, entry, option_name, NULL) != 0) {
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
if (entry->offset == offsetof(Config, eight_bit_output))
|
||||||
|
protocol_set_8_bit_output(true);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
if (strncmp(argv[i], "--chmod=", 8) == 0) {
|
if (strncmp(argv[i], "--chmod=", 8) == 0) {
|
||||||
if (set_string_option(&config->chmod_spec, argv[i] + 8, "--chmod") != 0)
|
if (set_string_option(&config->chmod_spec, argv[i] + 8, "--chmod") != 0)
|
||||||
return -1;
|
return -1;
|
||||||
@@ -255,26 +681,43 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args,
|
|||||||
printf("fastsync version %s\n", PROTOCOL_VERSION);
|
printf("fastsync version %s\n", PROTOCOL_VERSION);
|
||||||
return 1;
|
return 1;
|
||||||
} else if (opt_is(argv[i], "-a", "--archive")) {
|
} else if (opt_is(argv[i], "-a", "--archive")) {
|
||||||
config->use_compression = true;
|
config->use_compression =
|
||||||
|
!config->compress_choice || strcmp(config->compress_choice, "zstd") == 0;
|
||||||
config->use_multithreading = true;
|
config->use_multithreading = true;
|
||||||
config->use_metadata = true;
|
config->use_metadata = true;
|
||||||
log_message(LOG_LEVEL_INFO, "Enabled archive mode (-c -m -M)");
|
log_info_message(LOG_INFO_MISC, "Enabled archive mode (-c -m -M)");
|
||||||
} else if (opt_is(argv[i], "-p", NULL) && i + 1 < argc) {
|
} else if (opt_is(argv[i], "-p", NULL)) {
|
||||||
|
if (i + 1 >= argc) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "missing argument for %s", argv[i]);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
if (set_positive_int_option(&config->ssh_port, argv[++i], "-p") != 0)
|
if (set_positive_int_option(&config->ssh_port, argv[++i], "-p") != 0)
|
||||||
return -1;
|
return -1;
|
||||||
if (config->ssh_port > 65535) {
|
if (config->ssh_port > 65535) {
|
||||||
log_message(LOG_LEVEL_ERROR, "SSH port must be 1-65535");
|
log_message(LOG_LEVEL_ERROR, "SSH port must be 1-65535");
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
} else if (opt_is(argv[i], "--exclude", NULL) && i + 1 < argc) {
|
} else if (opt_is(argv[i], "--exclude", NULL)) {
|
||||||
|
if (i + 1 >= argc) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "missing argument for %s", argv[i]);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
if (config_add_pattern(&config->exclude_patterns, &config->exclude_count, argv[++i],
|
if (config_add_pattern(&config->exclude_patterns, &config->exclude_count, argv[++i],
|
||||||
"--exclude") != 0)
|
"--exclude") != 0)
|
||||||
return -1;
|
return -1;
|
||||||
} else if (opt_is(argv[i], "--include", NULL) && i + 1 < argc) {
|
} else if (opt_is(argv[i], "--include", NULL)) {
|
||||||
|
if (i + 1 >= argc) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "missing argument for %s", argv[i]);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
if (config_add_pattern(&config->include_patterns, &config->include_count, argv[++i],
|
if (config_add_pattern(&config->include_patterns, &config->include_count, argv[++i],
|
||||||
"--include") != 0)
|
"--include") != 0)
|
||||||
return -1;
|
return -1;
|
||||||
} else if (opt_is(argv[i], "--delta-block", NULL) && i + 1 < argc) {
|
} else if (opt_is(argv[i], "--delta-block", NULL)) {
|
||||||
|
if (i + 1 >= argc) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "missing argument for %s", argv[i]);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
unsigned long long val;
|
unsigned long long val;
|
||||||
if (parse_ull_arg(argv[++i], &val, "--delta-block") != 0)
|
if (parse_ull_arg(argv[++i], &val, "--delta-block") != 0)
|
||||||
return -1;
|
return -1;
|
||||||
@@ -282,7 +725,11 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args,
|
|||||||
config->delta_block_size = (uint32_t)val;
|
config->delta_block_size = (uint32_t)val;
|
||||||
else
|
else
|
||||||
log_message(LOG_LEVEL_WARNING, "--delta-block value %llu out of range, using default", val);
|
log_message(LOG_LEVEL_WARNING, "--delta-block value %llu out of range, using default", val);
|
||||||
} else if (opt_is(argv[i], "--delta-max", NULL) && i + 1 < argc) {
|
} else if (opt_is(argv[i], "--delta-max", NULL)) {
|
||||||
|
if (i + 1 >= argc) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "missing argument for %s", argv[i]);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
unsigned long long val;
|
unsigned long long val;
|
||||||
if (parse_ull_arg(argv[++i], &val, "--delta-max") != 0)
|
if (parse_ull_arg(argv[++i], &val, "--delta-max") != 0)
|
||||||
return -1;
|
return -1;
|
||||||
@@ -291,8 +738,9 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args,
|
|||||||
else
|
else
|
||||||
log_message(LOG_LEVEL_WARNING, "--delta-max value %llu too small, using default", val);
|
log_message(LOG_LEVEL_WARNING, "--delta-max value %llu too small, using default", val);
|
||||||
} else if (opt_is(argv[i], "-c", "-z")) {
|
} else if (opt_is(argv[i], "-c", "-z")) {
|
||||||
config->use_compression = true;
|
config->use_compression =
|
||||||
log_message(LOG_LEVEL_INFO, "Enabled Compression");
|
!config->compress_choice || strcmp(config->compress_choice, "zstd") == 0;
|
||||||
|
log_info_message(LOG_INFO_MISC, "Enabled Compression");
|
||||||
if (i + 1 < argc) {
|
if (i + 1 < argc) {
|
||||||
char* end_ptr;
|
char* end_ptr;
|
||||||
long level = strtol(argv[i + 1], &end_ptr, 10);
|
long level = strtol(argv[i + 1], &end_ptr, 10);
|
||||||
@@ -302,32 +750,47 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args,
|
|||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
config->compression_level = (int)level;
|
config->compression_level = (int)level;
|
||||||
log_message(LOG_LEVEL_INFO, "Set Compression level to %ld", level);
|
log_info_message(LOG_INFO_MISC, "Set Compression level to %ld", level);
|
||||||
i++;
|
i++;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else if (opt_is(argv[i], "-M", "--preserve")) {
|
} else if (opt_is(argv[i], "-M", "--preserve")) {
|
||||||
config->use_metadata = true;
|
config->use_metadata = true;
|
||||||
log_message(LOG_LEVEL_INFO, "Enabled metadata preservation");
|
log_info_message(LOG_INFO_MISC, "Enabled metadata preservation");
|
||||||
|
} else if (opt_is(argv[i], "-E", "--executability")) {
|
||||||
|
config->use_metadata = true;
|
||||||
|
config->use_executability = true;
|
||||||
|
log_info_message(LOG_INFO_MISC, "Enabled executable permission preservation");
|
||||||
} else if (opt_is(argv[i], "-f", "--sendfile")) {
|
} else if (opt_is(argv[i], "-f", "--sendfile")) {
|
||||||
config->use_sendfile = true;
|
config->use_sendfile = true;
|
||||||
log_message(LOG_LEVEL_INFO, "Enabled sendfile");
|
log_info_message(LOG_INFO_MISC, "Enabled sendfile");
|
||||||
} else if (opt_is(argv[i], "-m", NULL)) {
|
} else if (opt_is(argv[i], "-m", NULL)) {
|
||||||
config->use_multithreading = true;
|
config->use_multithreading = true;
|
||||||
log_message(LOG_LEVEL_INFO, "Enabled Multithreading");
|
log_info_message(LOG_INFO_MISC, "Enabled Multithreading");
|
||||||
} else if (opt_is(argv[i], "-s", NULL)) {
|
} else if (opt_is(argv[i], "-s", NULL)) {
|
||||||
config->use_chunk_serialization = true;
|
config->use_chunk_serialization = true;
|
||||||
log_message(LOG_LEVEL_INFO, "Enabled Chunk Serialization");
|
log_info_message(LOG_INFO_MISC, "Enabled Chunk Serialization");
|
||||||
} else if (opt_is(argv[i], "--server-port", NULL) && i + 1 < argc) {
|
} else if (opt_is(argv[i], "--server-port", NULL)) {
|
||||||
|
if (i + 1 >= argc) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "missing argument for %s", argv[i]);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
if (!parse_positive_int(argv[++i], &config->server_port)) {
|
if (!parse_positive_int(argv[++i], &config->server_port)) {
|
||||||
log_message(LOG_LEVEL_ERROR, "invalid --server-port value: %s", argv[i]);
|
char* escaped = output_escape(argv[i], false);
|
||||||
|
log_message(LOG_LEVEL_ERROR, "invalid --server-port value: %s",
|
||||||
|
escaped ? escaped : "<allocation failed>");
|
||||||
|
free(escaped);
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
if (config->server_port > 65535) {
|
if (config->server_port > 65535) {
|
||||||
log_message(LOG_LEVEL_ERROR, "server port must be 1-65535");
|
log_message(LOG_LEVEL_ERROR, "server port must be 1-65535");
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
} else if (opt_is(argv[i], "--bwlimit", NULL) && i + 1 < argc) {
|
} else if (opt_is(argv[i], "--bwlimit", NULL)) {
|
||||||
|
if (i + 1 >= argc) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "missing argument for %s", argv[i]);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
unsigned long long kbps;
|
unsigned long long kbps;
|
||||||
if (parse_ull_arg(argv[++i], &kbps, "--bwlimit") != 0)
|
if (parse_ull_arg(argv[++i], &kbps, "--bwlimit") != 0)
|
||||||
return -1;
|
return -1;
|
||||||
@@ -340,8 +803,12 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args,
|
|||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
io_set_bwlimit(kbps * 1024);
|
io_set_bwlimit(kbps * 1024);
|
||||||
log_message(LOG_LEVEL_INFO, "Set bandwidth limit to %llu KB/s", kbps);
|
log_info_message(LOG_INFO_MISC, "Set bandwidth limit to %llu KB/s", kbps);
|
||||||
} else if (opt_is(argv[i], "--chunk-size", NULL) && i + 1 < argc) {
|
} else if (opt_is(argv[i], "--chunk-size", NULL)) {
|
||||||
|
if (i + 1 >= argc) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "missing argument for %s", argv[i]);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
unsigned long long val;
|
unsigned long long val;
|
||||||
if (parse_ull_arg(argv[++i], &val, "--chunk-size") != 0)
|
if (parse_ull_arg(argv[++i], &val, "--chunk-size") != 0)
|
||||||
return -1;
|
return -1;
|
||||||
@@ -350,7 +817,11 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args,
|
|||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
config->chunk_size = val;
|
config->chunk_size = val;
|
||||||
} else if (opt_is(argv[i], "--log-file", NULL) && i + 1 < argc) {
|
} else if (opt_is(argv[i], "--log-file", NULL)) {
|
||||||
|
if (i + 1 >= argc) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "missing argument for %s", argv[i]);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
if (config->log_file) {
|
if (config->log_file) {
|
||||||
fclose(config->log_file);
|
fclose(config->log_file);
|
||||||
config->log_file = NULL;
|
config->log_file = NULL;
|
||||||
@@ -358,52 +829,126 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args,
|
|||||||
}
|
}
|
||||||
FILE* lf = fopen(argv[++i], "a");
|
FILE* lf = fopen(argv[++i], "a");
|
||||||
if (!lf) {
|
if (!lf) {
|
||||||
log_message(LOG_LEVEL_ERROR, "could not open log file '%s': %s", argv[i], strerror(errno));
|
char* escaped = output_escape(argv[i], false);
|
||||||
|
log_message(LOG_LEVEL_ERROR, "could not open log file '%s': %s",
|
||||||
|
escaped ? escaped : "<allocation failed>", strerror(errno));
|
||||||
|
free(escaped);
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
config->log_file = lf;
|
config->log_file = lf;
|
||||||
log_set_file(lf);
|
log_set_file(lf);
|
||||||
} else if (opt_is(argv[i], "--exclude-from", NULL) && i + 1 < argc) {
|
} else if (strncmp(argv[i], "--stderr=", 9) == 0) {
|
||||||
|
if (set_stderr_mode(argv[i] + 9) != 0)
|
||||||
|
return -1;
|
||||||
|
} else if (opt_is(argv[i], "--stderr", NULL)) {
|
||||||
|
if (i + 1 >= argc || set_stderr_mode(argv[++i]) != 0)
|
||||||
|
return -1;
|
||||||
|
} else if (opt_is(argv[i], "--exclude-from", NULL)) {
|
||||||
|
if (i + 1 >= argc) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "missing argument for %s", argv[i]);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
if (read_patterns_from_file(argv[++i], &config->exclude_patterns, &config->exclude_count) !=
|
if (read_patterns_from_file(argv[++i], &config->exclude_patterns, &config->exclude_count) !=
|
||||||
0)
|
0)
|
||||||
return -1;
|
return -1;
|
||||||
} else if (opt_is(argv[i], "--include-from", NULL) && i + 1 < argc) {
|
} else if (opt_is(argv[i], "--include-from", NULL)) {
|
||||||
|
if (i + 1 >= argc) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "missing argument for %s", argv[i]);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
if (read_patterns_from_file(argv[++i], &config->include_patterns, &config->include_count) !=
|
if (read_patterns_from_file(argv[++i], &config->include_patterns, &config->include_count) !=
|
||||||
0)
|
0)
|
||||||
return -1;
|
return -1;
|
||||||
} else if (opt_is(argv[i], "-v", "--verbose")) {
|
} else if (opt_is(argv[i], "-v", "--verbose")) {
|
||||||
|
verbose = true;
|
||||||
set_log_level(LOG_LEVEL_DEBUG);
|
set_log_level(LOG_LEVEL_DEBUG);
|
||||||
} else if (opt_is(argv[i], "-T", NULL) && i + 1 < argc) {
|
} else if (opt_is(argv[i], "-q", "--quiet")) {
|
||||||
if (set_positive_int_option(&config->timeout, argv[++i], "-T") != 0)
|
config->quiet = true;
|
||||||
|
} else if (strncmp(argv[i], "--debug=", 8) == 0) {
|
||||||
|
int debug_ret = parse_debug_flags(argv[i] + 8, config);
|
||||||
|
if (debug_ret != 0)
|
||||||
|
return debug_ret;
|
||||||
|
} else if (opt_is(argv[i], "--debug", NULL)) {
|
||||||
|
if (i + 1 >= argc)
|
||||||
|
return parse_debug_flags(NULL, config);
|
||||||
|
int debug_ret = parse_debug_flags(argv[++i], config);
|
||||||
|
if (debug_ret != 0)
|
||||||
|
return debug_ret;
|
||||||
|
} else if (strncmp(argv[i], "--info=", 7) == 0) {
|
||||||
|
if (parse_info_flags(argv[i] + 7, config) != 0)
|
||||||
return -1;
|
return -1;
|
||||||
} else if (opt_is(argv[i], "--compress-level", NULL) && i + 1 < argc) {
|
} else if (opt_is(argv[i], "--info", NULL)) {
|
||||||
if (set_positive_int_option(&config->compression_level, argv[++i], "--compress-level") != 0)
|
if (i + 1 >= argc || parse_info_flags(argv[++i], config) != 0)
|
||||||
return -1;
|
return -1;
|
||||||
if (config->compression_level < 1 || config->compression_level > 22) {
|
} else if (opt_is(argv[i], "-T", NULL)) {
|
||||||
log_message(LOG_LEVEL_ERROR, "--compress-level must be between 1 and 22");
|
if (i + 1 >= argc) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "missing argument for %s", argv[i]);
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
if (set_positive_int_option(&config->timeout, argv[++i], "-T") != 0)
|
||||||
|
return -1;
|
||||||
|
} else if (strncmp(argv[i], "--skip-compress=", 16) == 0) {
|
||||||
|
if (parse_skip_compress(config, argv[i] + 16) != 0)
|
||||||
|
return -1;
|
||||||
|
} else if (opt_is(argv[i], "--skip-compress", NULL)) {
|
||||||
|
if (i + 1 >= argc) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "missing argument for %s", argv[i]);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
if (parse_skip_compress(config, argv[++i]) != 0)
|
||||||
|
return -1;
|
||||||
|
} else if (opt_is(argv[i], "--compress-threads", NULL)) {
|
||||||
|
if (i + 1 >= argc) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "missing argument for %s", argv[i]);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
if (set_compression_threads_option(&config->compression_threads, argv[++i]) != 0)
|
||||||
|
return -1;
|
||||||
|
} else if (opt_is(argv[i], "--checksum-choice", "--cc")) {
|
||||||
|
if (i + 1 >= argc) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "missing argument for %s", argv[i]);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
log_message(LOG_LEVEL_ERROR, "%s is not supported yet (xxHash64 is used)", argv[i]);
|
||||||
|
return -1;
|
||||||
} else if (argv[i][0] == '-') {
|
} else if (argv[i][0] == '-') {
|
||||||
fprintf(stderr, "Unknown option: %s\n", argv[i]);
|
char* escaped = output_escape(argv[i], false);
|
||||||
|
fprintf(stderr, "Unknown option: %s\n", escaped ? escaped : "<allocation failed>");
|
||||||
|
free(escaped);
|
||||||
print_usage();
|
print_usage();
|
||||||
return -1;
|
return -1;
|
||||||
} else {
|
} else {
|
||||||
if (*positional_count < 2)
|
if (*positional_count < 2)
|
||||||
positional_args[(*positional_count)++] = i;
|
positional_args[(*positional_count)++] = i;
|
||||||
else {
|
else {
|
||||||
fprintf(stderr, "Unexpected argument: %s\n", argv[i]);
|
char* escaped = output_escape(argv[i], false);
|
||||||
|
fprintf(stderr, "Unexpected argument: %s\n", escaped ? escaped : "<allocation failed>");
|
||||||
|
free(escaped);
|
||||||
print_usage();
|
print_usage();
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
set_log_level(config->quiet ? LOG_LEVEL_ERROR : (verbose ? LOG_LEVEL_DEBUG : LOG_LEVEL_WARNING));
|
||||||
|
if (config->compress_choice)
|
||||||
|
config->use_compression = strcmp(config->compress_choice, "zstd") == 0;
|
||||||
|
|
||||||
|
/* Incremental and delta transfers need metadata unless the user disabled it. */
|
||||||
|
if ((config->use_incremental || config->use_delta) && !config->use_metadata &&
|
||||||
|
!config->metadata_explicitly_disabled) {
|
||||||
|
log_message(LOG_LEVEL_INFO, "Enabling metadata preservation for incremental/delta transfer");
|
||||||
|
config->use_metadata = true;
|
||||||
|
}
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
static int read_patterns_from_file(const char* filepath, char*** patterns, int* count) {
|
static int read_patterns_from_file(const char* filepath, char*** patterns, int* count) {
|
||||||
FILE* fp = fopen(filepath, "r");
|
FILE* fp = fopen(filepath, "r");
|
||||||
if (!fp) {
|
if (!fp) {
|
||||||
log_message(LOG_LEVEL_ERROR, "could not open pattern file '%s': %s", filepath, strerror(errno));
|
char* escaped = output_escape(filepath, false);
|
||||||
|
log_message(LOG_LEVEL_ERROR, "could not open pattern file '%s': %s",
|
||||||
|
escaped ? escaped : "<allocation failed>", strerror(errno));
|
||||||
|
free(escaped);
|
||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
char* line = NULL;
|
char* line = NULL;
|
||||||
@@ -433,6 +978,10 @@ static int read_patterns_from_file(const char* filepath, char*** patterns, int*
|
|||||||
|
|
||||||
#ifndef FASTSYNC_TEST_BUILD
|
#ifndef FASTSYNC_TEST_BUILD
|
||||||
int main(int argc, char* argv[]) {
|
int main(int argc, char* argv[]) {
|
||||||
|
/* The server may close a connection mid-stream (e.g. when it rejects an
|
||||||
|
oversized delta). Ignore SIGPIPE so that a broken TCP connection
|
||||||
|
surfaces as a clean write error instead of killing the client. */
|
||||||
|
signal(SIGPIPE, SIG_IGN);
|
||||||
const char* env_source = NULL;
|
const char* env_source = NULL;
|
||||||
const char* env_dest = NULL;
|
const char* env_dest = NULL;
|
||||||
bool save_to_disk = false;
|
bool save_to_disk = false;
|
||||||
@@ -503,16 +1052,6 @@ int main(int argc, char* argv[]) {
|
|||||||
goto cleanup;
|
goto cleanup;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Enable implicit flags */
|
|
||||||
if (config->use_incremental && !config->use_metadata) {
|
|
||||||
log_message(LOG_LEVEL_INFO, "Enabling metadata preservation for --incremental");
|
|
||||||
config->use_metadata = true;
|
|
||||||
}
|
|
||||||
if (config->use_delta && !config->use_metadata) {
|
|
||||||
log_message(LOG_LEVEL_INFO, "Enabling metadata preservation for --delta");
|
|
||||||
config->use_metadata = true;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Initialize TLS if needed */
|
/* Initialize TLS if needed */
|
||||||
if (config->use_tls)
|
if (config->use_tls)
|
||||||
tls_global_init();
|
tls_global_init();
|
||||||
|
|||||||
+413
-56
@@ -1,5 +1,6 @@
|
|||||||
#include "client_send.h"
|
#include "client_send.h"
|
||||||
#include "array_list.h"
|
#include "array_list.h"
|
||||||
|
#include "change_list.h"
|
||||||
#include "chunk.h"
|
#include "chunk.h"
|
||||||
#include "compression.h"
|
#include "compression.h"
|
||||||
#include "config.h"
|
#include "config.h"
|
||||||
@@ -16,11 +17,14 @@
|
|||||||
#include "transport_ssh.h"
|
#include "transport_ssh.h"
|
||||||
#include "transport_tls.h"
|
#include "transport_tls.h"
|
||||||
#include "utils.h"
|
#include "utils.h"
|
||||||
|
#include <fcntl.h>
|
||||||
|
#include <limits.h>
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
#include <threads.h>
|
#include <threads.h>
|
||||||
#include <time.h>
|
#include <time.h>
|
||||||
|
#include <sys/stat.h>
|
||||||
#include <unistd.h>
|
#include <unistd.h>
|
||||||
|
|
||||||
#define STREAM_THRESHOLD (64ULL * 1024 * 1024)
|
#define STREAM_THRESHOLD (64ULL * 1024 * 1024)
|
||||||
@@ -28,6 +32,14 @@
|
|||||||
/* Forward declaration for progress-reporting thread used in multithreaded send. */
|
/* Forward declaration for progress-reporting thread used in multithreaded send. */
|
||||||
static int progress_thread_fn(void* arg);
|
static int progress_thread_fn(void* arg);
|
||||||
|
|
||||||
|
static const char* display_bytes(unsigned long long bytes, bool human_readable, char* buffer,
|
||||||
|
size_t buffer_size) {
|
||||||
|
if (human_readable && format_human_bytes(bytes, buffer, buffer_size))
|
||||||
|
return buffer;
|
||||||
|
snprintf(buffer, buffer_size, "%.1f MB", bytes / 1048576.0);
|
||||||
|
return buffer;
|
||||||
|
}
|
||||||
|
|
||||||
static ScannerOptions scanner_options_from_config(const Config* config, int num_threads) {
|
static ScannerOptions scanner_options_from_config(const Config* config, int num_threads) {
|
||||||
ScannerOptions options = {
|
ScannerOptions options = {
|
||||||
config->use_metadata, config->chunk_size, config->exclude_patterns,
|
config->use_metadata, config->chunk_size, config->exclude_patterns,
|
||||||
@@ -46,7 +58,7 @@ static Client* connect_transfer_client(const Config* config) {
|
|||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
return client_connect_ssh(config->ssh_destination, config->ssh_port,
|
return client_connect_ssh(config->ssh_destination, config->ssh_port,
|
||||||
config->fastsync_server_path);
|
config->fastsync_server_path, config->old_args);
|
||||||
}
|
}
|
||||||
|
|
||||||
Client* client = client_create();
|
Client* client = client_create();
|
||||||
@@ -98,10 +110,92 @@ static bool add_chunk_to_manifest(ArrayList* manifest, const Chunk* chunk) {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
static bool finalize_transfer(Client* client) {
|
/* (finalize_transfer is defined after the SourceFile helpers below.) */
|
||||||
Status status;
|
|
||||||
return send_status(client->file_descriptor, STATUS_FINISHED) &&
|
typedef struct SourceFile {
|
||||||
receive_status(client->file_descriptor, &status) && status == STATUS_OK;
|
char* path;
|
||||||
|
dev_t device;
|
||||||
|
ino_t inode;
|
||||||
|
bool skipped; /* receiver reported the file was not written */
|
||||||
|
} SourceFile;
|
||||||
|
|
||||||
|
static void source_file_destroy(void* item) {
|
||||||
|
SourceFile* source = item;
|
||||||
|
if (source) {
|
||||||
|
free(source->path);
|
||||||
|
free(source);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Remove only the same regular source file that was sent. */
|
||||||
|
static void remove_transferred_sources(const Config* config, ArrayList* paths) {
|
||||||
|
if (!config->remove_source_files || !paths)
|
||||||
|
return;
|
||||||
|
for (int i = 0; i < paths->size; i++) {
|
||||||
|
SourceFile* source = paths->items[i];
|
||||||
|
if (source->skipped)
|
||||||
|
continue;
|
||||||
|
const char* slash = strrchr(source->path, '/');
|
||||||
|
const char* leaf = slash ? slash + 1 : source->path;
|
||||||
|
char parent[PATH_MAX];
|
||||||
|
if (slash) {
|
||||||
|
size_t parent_length = (size_t)(slash - source->path);
|
||||||
|
if (parent_length == 0)
|
||||||
|
parent_length = 1;
|
||||||
|
if (parent_length >= sizeof(parent))
|
||||||
|
continue;
|
||||||
|
memcpy(parent, source->path, parent_length);
|
||||||
|
parent[parent_length] = '\0';
|
||||||
|
} else {
|
||||||
|
(void)snprintf(parent, sizeof(parent), ".");
|
||||||
|
}
|
||||||
|
|
||||||
|
int dirfd = open(parent, O_RDONLY | O_DIRECTORY | O_CLOEXEC);
|
||||||
|
if (dirfd < 0)
|
||||||
|
continue;
|
||||||
|
struct stat st;
|
||||||
|
if (fstatat(dirfd, leaf, &st, AT_SYMLINK_NOFOLLOW) != 0 || !S_ISREG(st.st_mode) ||
|
||||||
|
st.st_dev != source->device || st.st_ino != source->inode) {
|
||||||
|
close(dirfd);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (unlinkat(dirfd, leaf, 0) != 0)
|
||||||
|
log_message(LOG_LEVEL_WARNING, "Could not remove source file %s", source->path);
|
||||||
|
close(dirfd);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static SourceFile* source_file_create(const File* file) {
|
||||||
|
if (!file || !file->path)
|
||||||
|
return NULL;
|
||||||
|
struct stat st;
|
||||||
|
if (lstat(file->path, &st) != 0 || !S_ISREG(st.st_mode))
|
||||||
|
return NULL;
|
||||||
|
SourceFile* source = malloc(sizeof(*source));
|
||||||
|
if (!source)
|
||||||
|
return NULL;
|
||||||
|
source->path = str_dup(file->path);
|
||||||
|
source->device = st.st_dev;
|
||||||
|
source->inode = st.st_ino;
|
||||||
|
source->skipped = false;
|
||||||
|
if (!source->path) {
|
||||||
|
source_file_destroy(source);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
return source;
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool remember_source_file(ArrayList* paths, const File* file) {
|
||||||
|
if (!paths || !file || !file->path)
|
||||||
|
return true;
|
||||||
|
SourceFile* source = source_file_create(file);
|
||||||
|
if (!source)
|
||||||
|
return true;
|
||||||
|
if (!array_list_add(paths, source)) {
|
||||||
|
source_file_destroy(source);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
static void mark_sender_done(PipelineContextSender* context) {
|
static void mark_sender_done(PipelineContextSender* context) {
|
||||||
@@ -110,6 +204,33 @@ static void mark_sender_done(PipelineContextSender* context) {
|
|||||||
mtx_unlock(&context->mutex_progress);
|
mtx_unlock(&context->mutex_progress);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Send the final STATUS_FINISHED frame and await the receiver's verdict.
|
||||||
|
When --remove-source-files is active the receiver acknowledges each data
|
||||||
|
file it processed, in send order: STATUS_NEXT means the file was written,
|
||||||
|
STATUS_OK means the file was skipped/unchanged. Skipped sources are marked
|
||||||
|
so the later removal pass keeps them. */
|
||||||
|
static bool finalize_transfer(Client* client, const Config* config, ArrayList* remove_sources) {
|
||||||
|
if (!send_status(client->file_descriptor, STATUS_FINISHED))
|
||||||
|
return false;
|
||||||
|
if (config->remove_source_files && remove_sources) {
|
||||||
|
for (int i = 0; i < remove_sources->size; i++) {
|
||||||
|
Status per_file;
|
||||||
|
if (!receive_status(client->file_descriptor, &per_file))
|
||||||
|
return false;
|
||||||
|
if (per_file == STATUS_ERROR)
|
||||||
|
return false;
|
||||||
|
if (per_file == STATUS_OK) {
|
||||||
|
((SourceFile*)remove_sources->items[i])->skipped = true;
|
||||||
|
} else if (per_file != STATUS_NEXT) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Unexpected per-file status from receiver");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Status status;
|
||||||
|
return receive_status(client->file_descriptor, &status) && status == STATUS_OK;
|
||||||
|
}
|
||||||
|
|
||||||
static void pipeline_cancel(PipelineContextSender* context) {
|
static void pipeline_cancel(PipelineContextSender* context) {
|
||||||
mtx_lock(&context->mutex_scanner);
|
mtx_lock(&context->mutex_scanner);
|
||||||
mtx_lock(&context->mutex_loader);
|
mtx_lock(&context->mutex_loader);
|
||||||
@@ -134,17 +255,146 @@ static int send_dry_run_manifest(const Config* config) {
|
|||||||
Chunk* chunk;
|
Chunk* chunk;
|
||||||
int file_count = 0;
|
int file_count = 0;
|
||||||
unsigned long long total_bytes = 0;
|
unsigned long long total_bytes = 0;
|
||||||
printf("Dry run: files to be transferred\n");
|
char size_buffer[32];
|
||||||
|
if (!config->quiet)
|
||||||
|
printf("Dry run: files to be transferred\n");
|
||||||
while ((chunk = directory_scanner_next(scanner)) != NULL) {
|
while ((chunk = directory_scanner_next(scanner)) != NULL) {
|
||||||
for (int i = 0; i < chunk->element_count; i++) {
|
for (int i = 0; i < chunk->element_count; i++) {
|
||||||
printf(" %s (%zu bytes)\n", chunk->items[i]->path, chunk->items[i]->data->size);
|
if (!config->quiet) {
|
||||||
|
char* escaped_path = output_escape(chunk->items[i]->path, config->eight_bit_output);
|
||||||
|
if (!escaped_path) {
|
||||||
|
chunk_destroy(chunk);
|
||||||
|
directory_scanner_destroy(scanner);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
if (config->human_readable)
|
||||||
|
printf(
|
||||||
|
" %s (%s)\n", escaped_path,
|
||||||
|
display_bytes(chunk->items[i]->data->size, true, size_buffer, sizeof(size_buffer)));
|
||||||
|
else
|
||||||
|
printf(" %s (%zu bytes)\n", escaped_path, chunk->items[i]->data->size);
|
||||||
|
free(escaped_path);
|
||||||
|
}
|
||||||
total_bytes += chunk->items[i]->data->size;
|
total_bytes += chunk->items[i]->data->size;
|
||||||
file_count++;
|
file_count++;
|
||||||
}
|
}
|
||||||
chunk_destroy(chunk);
|
chunk_destroy(chunk);
|
||||||
}
|
}
|
||||||
directory_scanner_destroy(scanner);
|
directory_scanner_destroy(scanner);
|
||||||
printf("Total: %d files, %.1f MB\n", file_count, total_bytes / 1048576.0);
|
if (!config->quiet) {
|
||||||
|
if (config->human_readable)
|
||||||
|
printf("Total: %d files, %s\n", file_count,
|
||||||
|
display_bytes(total_bytes, true, size_buffer, sizeof(size_buffer)));
|
||||||
|
else
|
||||||
|
printf("Total: %d files, %.1f MB\n", file_count, total_bytes / 1048576.0);
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
char* path;
|
||||||
|
mode_t mode;
|
||||||
|
unsigned long long size;
|
||||||
|
time_t mtime;
|
||||||
|
} ListEntry;
|
||||||
|
|
||||||
|
static void list_entries_destroy(ListEntry* entries, size_t count) {
|
||||||
|
if (entries == NULL)
|
||||||
|
return;
|
||||||
|
for (size_t i = 0; i < count; i++)
|
||||||
|
free(entries[i].path);
|
||||||
|
free(entries);
|
||||||
|
}
|
||||||
|
|
||||||
|
static int compare_list_entries(const void* left, const void* right) {
|
||||||
|
const ListEntry* a = (const ListEntry*)left;
|
||||||
|
const ListEntry* b = (const ListEntry*)right;
|
||||||
|
return strcmp(a->path, b->path);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* --list-only: print an ls-style listing of the files that WOULD be
|
||||||
|
* transferred and exit without contacting the server or writing anything.
|
||||||
|
* Directory lines are not printed because the scanner only yields regular
|
||||||
|
* transfer candidates. Returns 0 on success, 1 on error. */
|
||||||
|
static int send_list_only(const Config* config) {
|
||||||
|
ScannerOptions options = scanner_options_from_config(config, 0);
|
||||||
|
options.use_metadata = true; /* capture mode + mtime for the listing */
|
||||||
|
DirectoryScanner* scanner =
|
||||||
|
directory_scanner_create_with_options(config->send_directory, &options);
|
||||||
|
if (!scanner)
|
||||||
|
return 1;
|
||||||
|
ListEntry* entries = NULL;
|
||||||
|
size_t count = 0;
|
||||||
|
size_t capacity = 0;
|
||||||
|
Chunk* chunk;
|
||||||
|
bool oom = false;
|
||||||
|
while ((chunk = directory_scanner_next(scanner)) != NULL) {
|
||||||
|
for (int i = 0; i < chunk->element_count; i++) {
|
||||||
|
File* f = chunk->items[i];
|
||||||
|
if (f == NULL)
|
||||||
|
continue;
|
||||||
|
if (count == capacity) {
|
||||||
|
size_t new_capacity = capacity > 0 ? capacity * 2 : 64;
|
||||||
|
if (new_capacity <= capacity) {
|
||||||
|
oom = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
ListEntry* grown = realloc(entries, new_capacity * sizeof(ListEntry));
|
||||||
|
if (!grown) {
|
||||||
|
oom = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
entries = grown;
|
||||||
|
capacity = new_capacity;
|
||||||
|
}
|
||||||
|
char* path = str_dup(f->path);
|
||||||
|
if (!path) {
|
||||||
|
oom = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
mode_t mode = 0;
|
||||||
|
time_t mtime = 0;
|
||||||
|
if (f->metadata != NULL) {
|
||||||
|
mode = f->metadata->mode;
|
||||||
|
mtime = f->metadata->mtime_sec;
|
||||||
|
} else {
|
||||||
|
struct stat st;
|
||||||
|
if (stat(f->path, &st) == 0) {
|
||||||
|
mode = st.st_mode;
|
||||||
|
mtime = st.st_mtime;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
entries[count].path = path;
|
||||||
|
entries[count].mode = mode;
|
||||||
|
entries[count].mtime = mtime;
|
||||||
|
entries[count].size = f->data != NULL ? f->data->size : 0;
|
||||||
|
count++;
|
||||||
|
}
|
||||||
|
chunk_destroy(chunk);
|
||||||
|
if (oom)
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
bool failed = oom || directory_scanner_failed(scanner);
|
||||||
|
directory_scanner_destroy(scanner);
|
||||||
|
if (failed) {
|
||||||
|
list_entries_destroy(entries, count);
|
||||||
|
if (oom)
|
||||||
|
log_message(LOG_LEVEL_ERROR, "memory allocation failed while listing");
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
if (count > 1)
|
||||||
|
qsort(entries, count, sizeof(ListEntry), compare_list_entries);
|
||||||
|
for (size_t i = 0; i < count; i++) {
|
||||||
|
char* line = change_render_list_line(entries[i].mode, entries[i].size, entries[i].mtime,
|
||||||
|
entries[i].path);
|
||||||
|
if (line != NULL) {
|
||||||
|
char* escaped = output_escape(line, config->eight_bit_output);
|
||||||
|
printf("%s\n", escaped != NULL ? escaped : line);
|
||||||
|
free(escaped);
|
||||||
|
free(line);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
list_entries_destroy(entries, count);
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -172,10 +422,13 @@ static int incremental_check(Client* client, File* file, const Config* config,
|
|||||||
return -1;
|
return -1;
|
||||||
unsigned long long fsize = file->data->size;
|
unsigned long long fsize = file->data->size;
|
||||||
long long mtime = file->metadata ? file->metadata->mtime_sec : 0;
|
long long mtime = file->metadata ? file->metadata->mtime_sec : 0;
|
||||||
|
long long mtime_nsec = file->metadata ? file->metadata->mtime_nsec : 0;
|
||||||
if (!send_n_data(client->file_descriptor, &fsize, sizeof(fsize)))
|
if (!send_n_data(client->file_descriptor, &fsize, sizeof(fsize)))
|
||||||
return -1;
|
return -1;
|
||||||
if (!send_n_data(client->file_descriptor, &mtime, sizeof(mtime)))
|
if (!send_n_data(client->file_descriptor, &mtime, sizeof(mtime)))
|
||||||
return -1;
|
return -1;
|
||||||
|
if (!send_n_data(client->file_descriptor, &mtime_nsec, sizeof(mtime_nsec)))
|
||||||
|
return -1;
|
||||||
if (config->checksum) {
|
if (config->checksum) {
|
||||||
uint64_t checksum;
|
uint64_t checksum;
|
||||||
if (!file_checksum(file, &checksum) ||
|
if (!file_checksum(file, &checksum) ||
|
||||||
@@ -234,8 +487,11 @@ static int send_delta(Client* client, File* file, DeltaSignature* sig, Config* c
|
|||||||
return send_status(client->file_descriptor, STATUS_NEXT) ? 1 : -1;
|
return send_status(client->file_descriptor, STATUS_NEXT) ? 1 : -1;
|
||||||
|
|
||||||
Data* to_send = delta_data;
|
Data* to_send = delta_data;
|
||||||
if (config->use_compression) {
|
int skip_count = config->skip_compress_set ? config->skip_compress_count : -1;
|
||||||
to_send = data_compress(delta_data, config->compression_level);
|
if (config->use_compression && !compression_should_skip_with_suffixes(
|
||||||
|
file->path, config->skip_compress_suffixes, skip_count)) {
|
||||||
|
to_send = data_compress_with_threads(delta_data, config->compression_level,
|
||||||
|
config->compression_threads);
|
||||||
data_destroy(delta_data);
|
data_destroy(delta_data);
|
||||||
if (!to_send)
|
if (!to_send)
|
||||||
return send_status(client->file_descriptor, STATUS_NEXT) ? 1 : -1;
|
return send_status(client->file_descriptor, STATUS_NEXT) ? 1 : -1;
|
||||||
@@ -251,20 +507,25 @@ static int send_delta(Client* client, File* file, DeltaSignature* sig, Config* c
|
|||||||
return ok ? 0 : -1;
|
return ok ? 0 : -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
typedef bool (*file_send_fn)(File*, int, bool, int, bool);
|
|
||||||
|
|
||||||
// Send a single file directly (non-incremental path).
|
// Send a single file directly (non-incremental path).
|
||||||
static bool send_file_direct(File* file, int fd, bool use_metadata, int compression_level) {
|
static bool send_file_direct(File* file, int fd, bool use_metadata, int compression_level,
|
||||||
|
const Config* config) {
|
||||||
if (!send_status(fd, STATUS_NEXT))
|
if (!send_status(fd, STATUS_NEXT))
|
||||||
return false;
|
return false;
|
||||||
return file_send_single_calls(file, fd, use_metadata, compression_level, true);
|
int skip_count = config->skip_compress_set ? config->skip_compress_count : -1;
|
||||||
|
return file_send_single_calls_with_skip(file, fd, use_metadata, compression_level, true,
|
||||||
|
config->skip_compress_suffixes, skip_count,
|
||||||
|
config->compression_threads);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Send a single file directly via sendfile (non-incremental path).
|
// Send a single file directly via sendfile (non-incremental path).
|
||||||
static bool send_file_direct_sendfile(File* file, int fd, bool use_metadata) {
|
static bool send_file_direct_sendfile(File* file, int fd, bool use_metadata, const Config* config) {
|
||||||
if (!send_status(fd, STATUS_NEXT))
|
if (!send_status(fd, STATUS_NEXT))
|
||||||
return false;
|
return false;
|
||||||
return file_send_sendfile(file, fd, use_metadata, 0, true);
|
int skip_count = config->skip_compress_set ? config->skip_compress_count : -1;
|
||||||
|
return file_send_sendfile_with_skip(file, fd, use_metadata, 0, true,
|
||||||
|
config->skip_compress_suffixes, skip_count,
|
||||||
|
config->compression_threads);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Process one file in a chunk: either via incremental check or direct send.
|
// Process one file in a chunk: either via incremental check or direct send.
|
||||||
@@ -272,13 +533,16 @@ static bool send_file_direct_sendfile(File* file, int fd, bool use_metadata) {
|
|||||||
static int send_single_file(Client* client, File* file, Config* config, bool use_incremental,
|
static int send_single_file(Client* client, File* file, Config* config, bool use_incremental,
|
||||||
bool use_sendfile) {
|
bool use_sendfile) {
|
||||||
int compression_level = config->use_compression ? config->compression_level : 0;
|
int compression_level = config->use_compression ? config->compression_level : 0;
|
||||||
|
log_info_message(LOG_INFO_COPY, "Transferring %s", file->path);
|
||||||
|
|
||||||
if (!use_incremental) {
|
if (!use_incremental) {
|
||||||
if (use_sendfile) {
|
if (use_sendfile) {
|
||||||
return send_file_direct_sendfile(file, client->file_descriptor, config->use_metadata) ? 0
|
return send_file_direct_sendfile(file, client->file_descriptor, config->use_metadata, config)
|
||||||
: -1;
|
? 0
|
||||||
|
: -1;
|
||||||
}
|
}
|
||||||
return send_file_direct(file, client->file_descriptor, config->use_metadata, compression_level)
|
return send_file_direct(file, client->file_descriptor, config->use_metadata, compression_level,
|
||||||
|
config)
|
||||||
? 0
|
? 0
|
||||||
: -1;
|
: -1;
|
||||||
}
|
}
|
||||||
@@ -288,6 +552,7 @@ static int send_single_file(Client* client, File* file, Config* config, bool use
|
|||||||
DeltaSignature* sig = NULL;
|
DeltaSignature* sig = NULL;
|
||||||
int rc = incremental_check(client, file, config, &sig);
|
int rc = incremental_check(client, file, config, &sig);
|
||||||
if (rc == 1) {
|
if (rc == 1) {
|
||||||
|
log_info_message(LOG_INFO_SKIP, "Skipping unchanged %s", file->path);
|
||||||
delta_signature_destroy(sig);
|
delta_signature_destroy(sig);
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
@@ -304,13 +569,15 @@ static int send_single_file(Client* client, File* file, Config* config, bool use
|
|||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
// Fall through: send full file via sendfile (pass 0 for compression_level)
|
// Fall through: send full file via sendfile (pass 0 for compression_level)
|
||||||
if (!file_send_sendfile(file, client->file_descriptor, config->use_metadata, 0, false))
|
int skip_count = config->skip_compress_set ? config->skip_compress_count : -1;
|
||||||
|
if (!file_send_sendfile_with_skip(file, client->file_descriptor, config->use_metadata, 0, false,
|
||||||
|
config->skip_compress_suffixes, skip_count,
|
||||||
|
config->compression_threads))
|
||||||
return -1;
|
return -1;
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Incremental path with single_calls (supports compression and delta)
|
// Incremental path with single_calls (supports compression and delta)
|
||||||
file_send_fn send_fn = (file_send_fn)file_send_single_calls;
|
|
||||||
DeltaSignature* sig = NULL;
|
DeltaSignature* sig = NULL;
|
||||||
int rc = incremental_check(client, file, config, &sig);
|
int rc = incremental_check(client, file, config, &sig);
|
||||||
if (rc < 0) {
|
if (rc < 0) {
|
||||||
@@ -318,10 +585,11 @@ static int send_single_file(Client* client, File* file, Config* config, bool use
|
|||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
if (rc == 1) {
|
if (rc == 1) {
|
||||||
|
log_info_message(LOG_INFO_SKIP, "Skipping unchanged %s", file->path);
|
||||||
delta_signature_destroy(sig);
|
delta_signature_destroy(sig);
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
if (rc == 2 && config->use_delta) {
|
if (rc == 2 && config->use_delta && !config->whole_file) {
|
||||||
int drc = send_delta(client, file, sig, config);
|
int drc = send_delta(client, file, sig, config);
|
||||||
delta_signature_destroy(sig);
|
delta_signature_destroy(sig);
|
||||||
if (drc == 0)
|
if (drc == 0)
|
||||||
@@ -338,18 +606,29 @@ static int send_single_file(Client* client, File* file, Config* config, bool use
|
|||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (!send_fn(file, client->file_descriptor, config->use_metadata, compression_level, false))
|
int skip_count = config->skip_compress_set ? config->skip_compress_count : -1;
|
||||||
|
if (!file_send_single_calls_with_skip(file, client->file_descriptor, config->use_metadata,
|
||||||
|
compression_level, false, config->skip_compress_suffixes,
|
||||||
|
skip_count, config->compression_threads))
|
||||||
return -1;
|
return -1;
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
int send_chunk(Client* client, Chunk* chunk, Config* config) {
|
static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config,
|
||||||
|
ArrayList* remove_sources) {
|
||||||
if (config->use_chunk_serialization) {
|
if (config->use_chunk_serialization) {
|
||||||
|
if (remove_sources) {
|
||||||
|
for (int i = 0; i < chunk->element_count; i++) {
|
||||||
|
if (!remember_source_file(remove_sources, chunk->items[i]))
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
}
|
||||||
if (!send_status(client->file_descriptor, STATUS_CHUNK))
|
if (!send_status(client->file_descriptor, STATUS_CHUNK))
|
||||||
return -1;
|
return -1;
|
||||||
Data* data;
|
Data* data;
|
||||||
if (config->use_compression) {
|
if (config->use_compression) {
|
||||||
data = chunk_compress(chunk, config->compression_level, config->use_metadata);
|
data = chunk_compress_with_threads(chunk, config->compression_level, config->use_metadata,
|
||||||
|
config->compression_threads);
|
||||||
} else {
|
} else {
|
||||||
data = chunk_serialize(chunk, config->use_metadata);
|
data = chunk_serialize(chunk, config->use_metadata);
|
||||||
}
|
}
|
||||||
@@ -360,6 +639,10 @@ int send_chunk(Client* client, Chunk* chunk, Config* config) {
|
|||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
data_destroy(data);
|
data_destroy(data);
|
||||||
|
for (int i = 0; i < chunk->element_count; i++) {
|
||||||
|
if (chunk->items[i] != NULL)
|
||||||
|
change_emit_file_sent(config, chunk->items[i]);
|
||||||
|
}
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -370,15 +653,29 @@ int send_chunk(Client* client, Chunk* chunk, Config* config) {
|
|||||||
bool stream = f->data->data == NULL && f->data->size > 0;
|
bool stream = f->data->data == NULL && f->data->size > 0;
|
||||||
bool use_sendfile =
|
bool use_sendfile =
|
||||||
(config->use_sendfile && !config->use_compression) || (stream && !config->use_compression);
|
(config->use_sendfile && !config->use_compression) || (stream && !config->use_compression);
|
||||||
|
SourceFile* source = remove_sources ? source_file_create(f) : NULL;
|
||||||
int rc = send_single_file(client, f, config, config->use_incremental, use_sendfile);
|
int rc = send_single_file(client, f, config, config->use_incremental, use_sendfile);
|
||||||
if (rc == 1)
|
if (rc == 1) {
|
||||||
|
source_file_destroy(source);
|
||||||
continue;
|
continue;
|
||||||
if (rc < 0)
|
}
|
||||||
|
if (rc < 0) {
|
||||||
|
source_file_destroy(source);
|
||||||
return -1;
|
return -1;
|
||||||
|
}
|
||||||
|
change_emit_file_sent(config, f);
|
||||||
|
if (source && !array_list_add(remove_sources, source)) {
|
||||||
|
source_file_destroy(source);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
int send_chunk(Client* client, Chunk* chunk, Config* config) {
|
||||||
|
return send_chunk_with_removal(client, chunk, config, NULL);
|
||||||
|
}
|
||||||
|
|
||||||
static int send_chunks_multithreaded(void* pipeline_context) {
|
static int send_chunks_multithreaded(void* pipeline_context) {
|
||||||
PipelineContextSender* context = (PipelineContextSender*)pipeline_context;
|
PipelineContextSender* context = (PipelineContextSender*)pipeline_context;
|
||||||
Client* client = connect_transfer_client(context->config);
|
Client* client = connect_transfer_client(context->config);
|
||||||
@@ -418,7 +715,17 @@ static int send_chunks_multithreaded(void* pipeline_context) {
|
|||||||
if (send_delete_manifest(client->file_descriptor, context->manifest) != 0)
|
if (send_delete_manifest(client->file_descriptor, context->manifest) != 0)
|
||||||
goto send_fail;
|
goto send_fail;
|
||||||
}
|
}
|
||||||
bool ok = finalize_transfer(client);
|
bool ok = finalize_transfer(client, context->config, context->remove_source_files);
|
||||||
|
if (ok)
|
||||||
|
remove_transferred_sources(context->config, context->remove_source_files);
|
||||||
|
mtx_lock(&context->mutex_progress);
|
||||||
|
int total_files = context->total_files;
|
||||||
|
unsigned long long total_bytes = context->total_bytes;
|
||||||
|
mtx_unlock(&context->mutex_progress);
|
||||||
|
if (context->config->stats)
|
||||||
|
fprintf(stderr, "Stats: %d files, %.1f MB\n", total_files, total_bytes / 1048576.0);
|
||||||
|
log_info_message(LOG_INFO_STATS, "Transfer summary: %d files, %.1f MB", total_files,
|
||||||
|
total_bytes / 1048576.0);
|
||||||
disconnect_transfer_client(client);
|
disconnect_transfer_client(client);
|
||||||
mark_sender_done(context);
|
mark_sender_done(context);
|
||||||
protocol_session_unbind();
|
protocol_session_unbind();
|
||||||
@@ -431,7 +738,8 @@ static int send_chunks_multithreaded(void* pipeline_context) {
|
|||||||
protocol_session_unbind();
|
protocol_session_unbind();
|
||||||
return thrd_error;
|
return thrd_error;
|
||||||
}
|
}
|
||||||
if (send_chunk(client, current_chunk, context->config) != 0) {
|
if (send_chunk_with_removal(client, current_chunk, context->config,
|
||||||
|
context->remove_source_files) != 0) {
|
||||||
log_message(LOG_LEVEL_ERROR, "unexpected error while sending chunk");
|
log_message(LOG_LEVEL_ERROR, "unexpected error while sending chunk");
|
||||||
chunk_destroy(current_chunk);
|
chunk_destroy(current_chunk);
|
||||||
pipeline_cancel(context);
|
pipeline_cancel(context);
|
||||||
@@ -440,30 +748,35 @@ static int send_chunks_multithreaded(void* pipeline_context) {
|
|||||||
protocol_session_unbind();
|
protocol_session_unbind();
|
||||||
return thrd_error;
|
return thrd_error;
|
||||||
}
|
}
|
||||||
if (context->config->show_progress) {
|
unsigned long long chunk_bytes = 0;
|
||||||
unsigned long long chunk_bytes = 0;
|
int chunk_files = 0;
|
||||||
for (int i = 0; i < current_chunk->element_count; i++) {
|
for (int i = 0; i < current_chunk->element_count; i++) {
|
||||||
if (current_chunk->items[i] && current_chunk->items[i]->data)
|
if (current_chunk->items[i] && current_chunk->items[i]->data) {
|
||||||
chunk_bytes += current_chunk->items[i]->data->size;
|
chunk_files++;
|
||||||
|
chunk_bytes += current_chunk->items[i]->data->size;
|
||||||
}
|
}
|
||||||
mtx_lock(&context->mutex_progress);
|
|
||||||
context->progress_bytes += chunk_bytes;
|
|
||||||
mtx_unlock(&context->mutex_progress);
|
|
||||||
}
|
}
|
||||||
|
mtx_lock(&context->mutex_progress);
|
||||||
|
context->total_files += chunk_files;
|
||||||
|
context->total_bytes += chunk_bytes;
|
||||||
|
context->progress_bytes = context->total_bytes;
|
||||||
|
mtx_unlock(&context->mutex_progress);
|
||||||
chunk_destroy(current_chunk);
|
chunk_destroy(current_chunk);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static int scan_directory_multithreaded(void* pipeline_context) {
|
static int scan_directory_multithreaded(void* pipeline_context) {
|
||||||
PipelineContextSender* context = (PipelineContextSender*)pipeline_context;
|
PipelineContextSender* context = (PipelineContextSender*)pipeline_context;
|
||||||
|
protocol_session_bind(&context->allocation_session);
|
||||||
ScannerOptions options = scanner_options_from_config(context->config, 4);
|
ScannerOptions options = scanner_options_from_config(context->config, 4);
|
||||||
ParallelScanner* scanner =
|
ParallelScanner* scanner = parallel_scanner_create_with_options(
|
||||||
parallel_scanner_create_with_options(context->config->send_directory, &options);
|
context->config->send_directory, &options, &context->allocation_session);
|
||||||
|
|
||||||
Chunk* current_chunk;
|
Chunk* current_chunk;
|
||||||
if (scanner == NULL) {
|
if (scanner == NULL) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Failed to create parallel scanner");
|
log_message(LOG_LEVEL_ERROR, "Failed to create parallel scanner");
|
||||||
pipeline_cancel(context);
|
pipeline_cancel(context);
|
||||||
|
protocol_session_unbind();
|
||||||
return thrd_error;
|
return thrd_error;
|
||||||
}
|
}
|
||||||
while ((current_chunk = parallel_scanner_next(scanner)) != NULL) {
|
while ((current_chunk = parallel_scanner_next(scanner)) != NULL) {
|
||||||
@@ -475,6 +788,7 @@ static int scan_directory_multithreaded(void* pipeline_context) {
|
|||||||
pipeline_cancel(context);
|
pipeline_cancel(context);
|
||||||
chunk_destroy(current_chunk);
|
chunk_destroy(current_chunk);
|
||||||
parallel_scanner_destroy(scanner);
|
parallel_scanner_destroy(scanner);
|
||||||
|
protocol_session_unbind();
|
||||||
return thrd_error;
|
return thrd_error;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -485,6 +799,7 @@ static int scan_directory_multithreaded(void* pipeline_context) {
|
|||||||
chunk_destroy(current_chunk);
|
chunk_destroy(current_chunk);
|
||||||
pipeline_cancel(context);
|
pipeline_cancel(context);
|
||||||
parallel_scanner_destroy(scanner);
|
parallel_scanner_destroy(scanner);
|
||||||
|
protocol_session_unbind();
|
||||||
return thrd_error;
|
return thrd_error;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -496,6 +811,7 @@ static int scan_directory_multithreaded(void* pipeline_context) {
|
|||||||
cnd_broadcast(&context->condition_not_full_scanner);
|
cnd_broadcast(&context->condition_not_full_scanner);
|
||||||
mtx_unlock(&context->mutex_scanner);
|
mtx_unlock(&context->mutex_scanner);
|
||||||
pipeline_cancel(context);
|
pipeline_cancel(context);
|
||||||
|
protocol_session_unbind();
|
||||||
return thrd_error;
|
return thrd_error;
|
||||||
}
|
}
|
||||||
mtx_lock(&context->mutex_scanner);
|
mtx_lock(&context->mutex_scanner);
|
||||||
@@ -504,11 +820,13 @@ static int scan_directory_multithreaded(void* pipeline_context) {
|
|||||||
mtx_unlock(&context->mutex_scanner);
|
mtx_unlock(&context->mutex_scanner);
|
||||||
|
|
||||||
parallel_scanner_destroy(scanner);
|
parallel_scanner_destroy(scanner);
|
||||||
|
protocol_session_unbind();
|
||||||
return thrd_success;
|
return thrd_success;
|
||||||
}
|
}
|
||||||
|
|
||||||
static int load_files_multithreaded(void* pipeline_context) {
|
static int load_files_multithreaded(void* pipeline_context) {
|
||||||
PipelineContextSender* context = (PipelineContextSender*)pipeline_context;
|
PipelineContextSender* context = (PipelineContextSender*)pipeline_context;
|
||||||
|
protocol_session_bind(&context->allocation_session);
|
||||||
while (true) {
|
while (true) {
|
||||||
Chunk* chunk = queue_dequeue_multithreaded(
|
Chunk* chunk = queue_dequeue_multithreaded(
|
||||||
context->queue_scanner, &context->mutex_scanner, &context->condition_not_empty_scanner,
|
context->queue_scanner, &context->mutex_scanner, &context->condition_not_empty_scanner,
|
||||||
@@ -518,6 +836,7 @@ static int load_files_multithreaded(void* pipeline_context) {
|
|||||||
context->loader_done = true;
|
context->loader_done = true;
|
||||||
cnd_signal(&context->condition_not_empty_loader);
|
cnd_signal(&context->condition_not_empty_loader);
|
||||||
mtx_unlock(&context->mutex_loader);
|
mtx_unlock(&context->mutex_loader);
|
||||||
|
protocol_session_unbind();
|
||||||
return thrd_success;
|
return thrd_success;
|
||||||
}
|
}
|
||||||
if (!context->config->use_sendfile) {
|
if (!context->config->use_sendfile) {
|
||||||
@@ -529,6 +848,7 @@ static int load_files_multithreaded(void* pipeline_context) {
|
|||||||
log_message(LOG_LEVEL_ERROR, "Failed to load file data");
|
log_message(LOG_LEVEL_ERROR, "Failed to load file data");
|
||||||
chunk_destroy(chunk);
|
chunk_destroy(chunk);
|
||||||
pipeline_cancel(context);
|
pipeline_cancel(context);
|
||||||
|
protocol_session_unbind();
|
||||||
return thrd_error;
|
return thrd_error;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -539,6 +859,7 @@ static int load_files_multithreaded(void* pipeline_context) {
|
|||||||
&context->cancelled)) {
|
&context->cancelled)) {
|
||||||
chunk_destroy(chunk);
|
chunk_destroy(chunk);
|
||||||
pipeline_cancel(context);
|
pipeline_cancel(context);
|
||||||
|
protocol_session_unbind();
|
||||||
return thrd_error;
|
return thrd_error;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -548,10 +869,20 @@ static int load_files_multithreaded(void* pipeline_context) {
|
|||||||
line (e.g. "Done.\n") or is "" for in-place refresh. Shared by the
|
line (e.g. "Done.\n") or is "" for in-place refresh. Shared by the
|
||||||
single-threaded loop and the multithreaded progress thread. */
|
single-threaded loop and the multithreaded progress thread. */
|
||||||
static void print_transfer_progress(unsigned long long total_bytes, time_t start,
|
static void print_transfer_progress(unsigned long long total_bytes, time_t start,
|
||||||
const char* suffix) {
|
const char* suffix, bool human_readable) {
|
||||||
double elapsed = difftime(time(NULL), start);
|
double elapsed = difftime(time(NULL), start);
|
||||||
double rate = elapsed > 0.0 ? total_bytes / (1048576.0 * elapsed) : 0.0;
|
double rate = elapsed > 0.0 ? total_bytes / (1048576.0 * elapsed) : 0.0;
|
||||||
fprintf(stderr, "\rSent %.1f MB (%.1f MB/s) %s", total_bytes / 1048576.0, rate, suffix);
|
if (human_readable) {
|
||||||
|
char total_buffer[32];
|
||||||
|
char rate_buffer[32];
|
||||||
|
fprintf(stderr, "\rSent %s (%s/s) %s",
|
||||||
|
display_bytes(total_bytes, true, total_buffer, sizeof(total_buffer)),
|
||||||
|
display_bytes((unsigned long long)(rate * 1048576.0), true, rate_buffer,
|
||||||
|
sizeof(rate_buffer)),
|
||||||
|
suffix);
|
||||||
|
} else {
|
||||||
|
fprintf(stderr, "\rSent %.1f MB (%.1f MB/s) %s", total_bytes / 1048576.0, rate, suffix);
|
||||||
|
}
|
||||||
fflush(stderr);
|
fflush(stderr);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -569,14 +900,14 @@ static int progress_thread_fn(void* arg) {
|
|||||||
mtx_unlock(&context->mutex_progress);
|
mtx_unlock(&context->mutex_progress);
|
||||||
|
|
||||||
if (done) {
|
if (done) {
|
||||||
print_transfer_progress(total, start, "Done.\n");
|
print_transfer_progress(total, start, "Done.\n", context->config->human_readable);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
time_t now = time(NULL);
|
time_t now = time(NULL);
|
||||||
if (now - last_progress >= 1) {
|
if (now - last_progress >= 1) {
|
||||||
last_progress = now;
|
last_progress = now;
|
||||||
print_transfer_progress(total, start, "");
|
print_transfer_progress(total, start, "", context->config->human_readable);
|
||||||
}
|
}
|
||||||
|
|
||||||
struct timespec ts = {0, 100 * 1000000L}; /* 100 ms */
|
struct timespec ts = {0, 100 * 1000000L}; /* 100 ms */
|
||||||
@@ -586,6 +917,8 @@ static int progress_thread_fn(void* arg) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
int send_files(Config* config) {
|
int send_files(Config* config) {
|
||||||
|
if (config->list_only)
|
||||||
|
return send_list_only(config);
|
||||||
if (config->dry_run)
|
if (config->dry_run)
|
||||||
return send_dry_run_manifest(config);
|
return send_dry_run_manifest(config);
|
||||||
|
|
||||||
@@ -603,12 +936,16 @@ int send_files(Config* config) {
|
|||||||
int ret = 1;
|
int ret = 1;
|
||||||
DirectoryScanner* scanner = NULL;
|
DirectoryScanner* scanner = NULL;
|
||||||
ArrayList* manifest = NULL;
|
ArrayList* manifest = NULL;
|
||||||
|
ArrayList* remove_sources = NULL;
|
||||||
if (!config_send(client->file_descriptor, config))
|
if (!config_send(client->file_descriptor, config))
|
||||||
goto send_fail;
|
goto send_fail;
|
||||||
ScannerOptions scanner_options = scanner_options_from_config(config, 0);
|
ScannerOptions scanner_options = scanner_options_from_config(config, 0);
|
||||||
scanner = directory_scanner_create_with_options(config->send_directory, &scanner_options);
|
scanner = directory_scanner_create_with_options(config->send_directory, &scanner_options);
|
||||||
manifest = create_transfer_manifest(config);
|
manifest = create_transfer_manifest(config);
|
||||||
if (!scanner || (config->use_delete && !manifest))
|
if (config->remove_source_files)
|
||||||
|
remove_sources = array_list_create(source_file_destroy);
|
||||||
|
if (!scanner || (config->use_delete && !manifest) ||
|
||||||
|
(config->remove_source_files && !remove_sources))
|
||||||
goto send_fail;
|
goto send_fail;
|
||||||
Chunk* current_chunk;
|
Chunk* current_chunk;
|
||||||
unsigned long long total_bytes = 0;
|
unsigned long long total_bytes = 0;
|
||||||
@@ -642,7 +979,7 @@ int send_files(Config* config) {
|
|||||||
goto send_fail;
|
goto send_fail;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (send_chunk(client, current_chunk, config) != 0) {
|
if (send_chunk_with_removal(client, current_chunk, config, remove_sources) != 0) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Failed to send chunk");
|
log_message(LOG_LEVEL_ERROR, "Failed to send chunk");
|
||||||
chunk_destroy(current_chunk);
|
chunk_destroy(current_chunk);
|
||||||
if (manifest)
|
if (manifest)
|
||||||
@@ -650,12 +987,12 @@ int send_files(Config* config) {
|
|||||||
manifest = NULL;
|
manifest = NULL;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
if (config->show_progress) {
|
total_bytes += chunk_bytes;
|
||||||
total_bytes += chunk_bytes;
|
if (config->show_progress && !config->quiet) {
|
||||||
time_t now = time(NULL);
|
time_t now = time(NULL);
|
||||||
if (now - last_progress >= 1) {
|
if (now - last_progress >= 1) {
|
||||||
last_progress = now;
|
last_progress = now;
|
||||||
print_transfer_progress(total_bytes, start, "");
|
print_transfer_progress(total_bytes, start, "", config->human_readable);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
chunk_destroy(current_chunk);
|
chunk_destroy(current_chunk);
|
||||||
@@ -671,15 +1008,28 @@ int send_files(Config* config) {
|
|||||||
array_list_delete(manifest);
|
array_list_delete(manifest);
|
||||||
manifest = NULL;
|
manifest = NULL;
|
||||||
}
|
}
|
||||||
bool ok = finalize_transfer(client);
|
bool ok = finalize_transfer(client, config, remove_sources);
|
||||||
if (config->show_progress)
|
if (ok)
|
||||||
print_transfer_progress(total_bytes, start, "Done.\n");
|
remove_transferred_sources(config, remove_sources);
|
||||||
if (config->stats) {
|
if (config->show_progress && !config->quiet)
|
||||||
|
print_transfer_progress(total_bytes, start, "Done.\n", config->human_readable);
|
||||||
|
if (config->stats && !config->quiet) {
|
||||||
double elapsed_total = difftime(time(NULL), start);
|
double elapsed_total = difftime(time(NULL), start);
|
||||||
double rate = elapsed_total > 0 ? total_bytes / (1048576.0 * elapsed_total) : 0;
|
double rate = elapsed_total > 0 ? total_bytes / (1048576.0 * elapsed_total) : 0;
|
||||||
fprintf(stderr, "Stats: %d files, %.1f MB, %.1f MB/s\n", total_files, total_bytes / 1048576.0,
|
if (config->human_readable) {
|
||||||
rate);
|
char total_buffer[32];
|
||||||
|
char rate_buffer[32];
|
||||||
|
fprintf(stderr, "Stats: %d files, %s, %s/s\n", total_files,
|
||||||
|
display_bytes(total_bytes, true, total_buffer, sizeof(total_buffer)),
|
||||||
|
display_bytes((unsigned long long)(rate * 1048576.0), true, rate_buffer,
|
||||||
|
sizeof(rate_buffer)));
|
||||||
|
} else {
|
||||||
|
fprintf(stderr, "Stats: %d files, %.1f MB, %.1f MB/s\n", total_files, total_bytes / 1048576.0,
|
||||||
|
rate);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
log_info_message(LOG_INFO_STATS, "Transfer summary: %d files, %.1f MB", total_files,
|
||||||
|
total_bytes / 1048576.0);
|
||||||
ret = ok ? 0 : 1;
|
ret = ok ? 0 : 1;
|
||||||
|
|
||||||
send_fail:
|
send_fail:
|
||||||
@@ -687,6 +1037,8 @@ send_fail:
|
|||||||
here even on success without --delete, fixing a pre-existing leak. */
|
here even on success without --delete, fixing a pre-existing leak. */
|
||||||
if (manifest)
|
if (manifest)
|
||||||
array_list_delete(manifest);
|
array_list_delete(manifest);
|
||||||
|
if (remove_sources)
|
||||||
|
array_list_delete(remove_sources);
|
||||||
if (scanner)
|
if (scanner)
|
||||||
directory_scanner_destroy(scanner);
|
directory_scanner_destroy(scanner);
|
||||||
disconnect_transfer_client(client);
|
disconnect_transfer_client(client);
|
||||||
@@ -698,6 +1050,8 @@ int send_files_multithreaded(Config** config_ptr) {
|
|||||||
if (!config_ptr || !*config_ptr)
|
if (!config_ptr || !*config_ptr)
|
||||||
return 1;
|
return 1;
|
||||||
Config* config = *config_ptr;
|
Config* config = *config_ptr;
|
||||||
|
if (config->list_only)
|
||||||
|
return send_list_only(config);
|
||||||
if (config->dry_run)
|
if (config->dry_run)
|
||||||
return send_dry_run_manifest(config);
|
return send_dry_run_manifest(config);
|
||||||
|
|
||||||
@@ -731,7 +1085,10 @@ int send_files_multithreaded(Config** config_ptr) {
|
|||||||
*config_ptr = NULL; /* context now owns config through all remaining paths */
|
*config_ptr = NULL; /* context now owns config through all remaining paths */
|
||||||
if (config->use_delete)
|
if (config->use_delete)
|
||||||
context->manifest = array_list_create(free);
|
context->manifest = array_list_create(free);
|
||||||
if (config->use_delete && !context->manifest) {
|
if (config->remove_source_files)
|
||||||
|
context->remove_source_files = array_list_create(source_file_destroy);
|
||||||
|
if ((config->use_delete && !context->manifest) ||
|
||||||
|
(config->remove_source_files && !context->remove_source_files)) {
|
||||||
pipeline_context_sender_destroy(context);
|
pipeline_context_sender_destroy(context);
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
@@ -765,7 +1122,7 @@ int send_files_multithreaded(Config** config_ptr) {
|
|||||||
|
|
||||||
thrd_t progress;
|
thrd_t progress;
|
||||||
bool progress_created = false;
|
bool progress_created = false;
|
||||||
if (config->show_progress) {
|
if (config->show_progress && !config->quiet) {
|
||||||
progress_created = (thrd_create(&progress, progress_thread_fn, context) == thrd_success);
|
progress_created = (thrd_create(&progress, progress_thread_fn, context) == thrd_success);
|
||||||
if (!progress_created) {
|
if (!progress_created) {
|
||||||
log_perror("Error creating progress thread");
|
log_perror("Error creating progress thread");
|
||||||
|
|||||||
@@ -15,6 +15,10 @@ bool validate_config(const Config* config) {
|
|||||||
"(chunk serialization)");
|
"(chunk serialization)");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
if (config->compression_threads > 0 && !config->use_compression) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "--compress-threads requires compression (-c or -z)");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
if (config->transport == TRANSPORT_SSH && config->use_sendfile) {
|
if (config->transport == TRANSPORT_SSH && config->use_sendfile) {
|
||||||
log_message(LOG_LEVEL_ERROR, "-f/--sendfile is not supported with SSH transport");
|
log_message(LOG_LEVEL_ERROR, "-f/--sendfile is not supported with SSH transport");
|
||||||
return false;
|
return false;
|
||||||
@@ -23,18 +27,27 @@ bool validate_config(const Config* config) {
|
|||||||
log_message(LOG_LEVEL_ERROR, "--incremental is not supported with -s (chunk serialization)");
|
log_message(LOG_LEVEL_ERROR, "--incremental is not supported with -s (chunk serialization)");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
if (config->use_delta && !config->use_incremental) {
|
if (config->skip_compress_set && config->use_chunk_serialization) {
|
||||||
|
log_message(LOG_LEVEL_ERROR,
|
||||||
|
"--skip-compress cannot be combined with -s (chunk serialization)");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (config->use_delta && !config->whole_file && !config->use_incremental) {
|
||||||
log_message(LOG_LEVEL_ERROR, "--delta requires --incremental");
|
log_message(LOG_LEVEL_ERROR, "--delta requires --incremental");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
if (config->use_delta && config->use_chunk_serialization) {
|
if (config->use_delta && !config->whole_file && config->use_chunk_serialization) {
|
||||||
log_message(LOG_LEVEL_ERROR, "--delta cannot be combined with -s (chunk serialization)");
|
log_message(LOG_LEVEL_ERROR, "--delta cannot be combined with -s (chunk serialization)");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
if (config->use_delta && config->use_sendfile) {
|
if (config->use_delta && !config->whole_file && config->use_sendfile) {
|
||||||
log_message(LOG_LEVEL_ERROR, "--delta cannot be combined with -f (sendfile)");
|
log_message(LOG_LEVEL_ERROR, "--delta cannot be combined with -f (sendfile)");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
if (config->log_file_format && !config->log_file) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "--log-file-format requires --log-file");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
if (config->append || config->append_verify) {
|
if (config->append || config->append_verify) {
|
||||||
fprintf(
|
fprintf(
|
||||||
stderr,
|
stderr,
|
||||||
|
|||||||
+10
-1
@@ -351,10 +351,14 @@ typedef struct {
|
|||||||
char** dirs;
|
char** dirs;
|
||||||
int dir_count;
|
int dir_count;
|
||||||
ScannerOptions options;
|
ScannerOptions options;
|
||||||
|
ProtocolSession* allocation_session;
|
||||||
} ParallelWorkerArg;
|
} ParallelWorkerArg;
|
||||||
|
|
||||||
static int parallel_worker_thread(void* arg) {
|
static int parallel_worker_thread(void* arg) {
|
||||||
ParallelWorkerArg* wa = (ParallelWorkerArg*)arg;
|
ParallelWorkerArg* wa = (ParallelWorkerArg*)arg;
|
||||||
|
ProtocolSession* allocation_session = wa->allocation_session;
|
||||||
|
if (allocation_session)
|
||||||
|
protocol_session_bind(allocation_session);
|
||||||
for (int i = 0; i < wa->dir_count; i++) {
|
for (int i = 0; i < wa->dir_count; i++) {
|
||||||
DirectoryScanner* ds = directory_scanner_create_with_options(wa->dirs[i], &wa->options);
|
DirectoryScanner* ds = directory_scanner_create_with_options(wa->dirs[i], &wa->options);
|
||||||
if (!ds) {
|
if (!ds) {
|
||||||
@@ -398,6 +402,8 @@ static int parallel_worker_thread(void* arg) {
|
|||||||
cnd_signal(&ps->result_not_empty);
|
cnd_signal(&ps->result_not_empty);
|
||||||
}
|
}
|
||||||
mtx_unlock(&ps->result_mutex);
|
mtx_unlock(&ps->result_mutex);
|
||||||
|
if (allocation_session)
|
||||||
|
protocol_session_unbind();
|
||||||
return thrd_success;
|
return thrd_success;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -633,6 +639,7 @@ static void spawn_parallel_workers(ParallelScanner* ps, ArrayList* subdirs,
|
|||||||
wa->dir_count = count;
|
wa->dir_count = count;
|
||||||
wa->options = *options;
|
wa->options = *options;
|
||||||
wa->options.chunk_size = cs;
|
wa->options.chunk_size = cs;
|
||||||
|
wa->allocation_session = ps->allocation_session;
|
||||||
start += count;
|
start += count;
|
||||||
if (thrd_create(&ps->threads[t], parallel_worker_thread, wa) != thrd_success) {
|
if (thrd_create(&ps->threads[t], parallel_worker_thread, wa) != thrd_success) {
|
||||||
for (int j = 0; j < count; j++)
|
for (int j = 0; j < count; j++)
|
||||||
@@ -648,7 +655,8 @@ static void spawn_parallel_workers(ParallelScanner* ps, ArrayList* subdirs,
|
|||||||
}
|
}
|
||||||
|
|
||||||
ParallelScanner* parallel_scanner_create_with_options(const char* root_directory,
|
ParallelScanner* parallel_scanner_create_with_options(const char* root_directory,
|
||||||
const ScannerOptions* options) {
|
const ScannerOptions* options,
|
||||||
|
ProtocolSession* allocation_session) {
|
||||||
if (!root_directory || !options)
|
if (!root_directory || !options)
|
||||||
return NULL;
|
return NULL;
|
||||||
ParallelScanner* ps = calloc(1, sizeof(ParallelScanner));
|
ParallelScanner* ps = calloc(1, sizeof(ParallelScanner));
|
||||||
@@ -658,6 +666,7 @@ ParallelScanner* parallel_scanner_create_with_options(const char* root_directory
|
|||||||
free(ps);
|
free(ps);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
ps->allocation_session = allocation_session;
|
||||||
|
|
||||||
ArrayList* root_files = array_list_create(file_destroy);
|
ArrayList* root_files = array_list_create(file_destroy);
|
||||||
ArrayList* subdirs = array_list_create(free);
|
ArrayList* subdirs = array_list_create(free);
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
#define SCANNER_H
|
#define SCANNER_H
|
||||||
|
|
||||||
#include "chunk.h"
|
#include "chunk.h"
|
||||||
|
#include "protocol.h"
|
||||||
#include "queue.h"
|
#include "queue.h"
|
||||||
#include <dirent.h>
|
#include <dirent.h>
|
||||||
#include <stdbool.h>
|
#include <stdbool.h>
|
||||||
@@ -62,6 +63,7 @@ typedef struct {
|
|||||||
atomic_bool cancelled;
|
atomic_bool cancelled;
|
||||||
int completed;
|
int completed;
|
||||||
Chunk* initial_chunk;
|
Chunk* initial_chunk;
|
||||||
|
ProtocolSession* allocation_session;
|
||||||
} ParallelScanner;
|
} ParallelScanner;
|
||||||
|
|
||||||
DirectoryScanner* directory_scanner_create(const char* root_directory, bool use_metadata,
|
DirectoryScanner* directory_scanner_create(const char* root_directory, bool use_metadata,
|
||||||
@@ -78,7 +80,8 @@ bool directory_scanner_failed(const DirectoryScanner* scanner);
|
|||||||
void directory_scanner_destroy(DirectoryScanner* scanner);
|
void directory_scanner_destroy(DirectoryScanner* scanner);
|
||||||
|
|
||||||
ParallelScanner* parallel_scanner_create_with_options(const char* root_directory,
|
ParallelScanner* parallel_scanner_create_with_options(const char* root_directory,
|
||||||
const ScannerOptions* options);
|
const ScannerOptions* options,
|
||||||
|
ProtocolSession* allocation_session);
|
||||||
Chunk* parallel_scanner_next(ParallelScanner* scanner);
|
Chunk* parallel_scanner_next(ParallelScanner* scanner);
|
||||||
bool parallel_scanner_failed(const ParallelScanner* scanner);
|
bool parallel_scanner_failed(const ParallelScanner* scanner);
|
||||||
void parallel_scanner_destroy(ParallelScanner* scanner);
|
void parallel_scanner_destroy(ParallelScanner* scanner);
|
||||||
|
|||||||
@@ -18,26 +18,50 @@ void print_usage(void) {
|
|||||||
printf(" -z [level] Alias for -c\n");
|
printf(" -z [level] Alias for -c\n");
|
||||||
printf(" -a, --archive Archive mode (-c -m -M)\n");
|
printf(" -a, --archive Archive mode (-c -m -M)\n");
|
||||||
printf(" -n, --dry-run Show what would be transferred\n");
|
printf(" -n, --dry-run Show what would be transferred\n");
|
||||||
|
printf(" --remove-source-files Remove regular source files after successful transfer\n");
|
||||||
printf(" -p <port> SSH port (default: 22)\n");
|
printf(" -p <port> SSH port (default: 22)\n");
|
||||||
printf(" --progress Show transfer progress\n");
|
printf(" --progress Show transfer progress\n");
|
||||||
|
printf(" -P Partial mode with progress (retention incomplete)\n");
|
||||||
|
printf(" -8, --8-bit-output Leave high-bit characters unescaped in output\n");
|
||||||
printf(" --delete Delete files on receiver not in source\n");
|
printf(" --delete Delete files on receiver not in source\n");
|
||||||
|
printf(" --ignore-existing Skip files that already exist on receiver\n");
|
||||||
|
printf(
|
||||||
|
" --dirs, --old-dirs, --old-d Transfer directories without recursing (not implemented)\n");
|
||||||
|
printf(" --del Alias for --delete-during (not implemented)\n");
|
||||||
printf(" --exclude <pattern> Exclude files matching pattern\n");
|
printf(" --exclude <pattern> Exclude files matching pattern\n");
|
||||||
printf(" --include <pattern> Only include files matching pattern\n");
|
printf(" --include <pattern> Only include files matching pattern\n");
|
||||||
printf(" --exclude-from <file> Read exclude patterns from file\n");
|
printf(" --exclude-from <file> Read exclude patterns from file\n");
|
||||||
printf(" --include-from <file> Read include patterns from file\n");
|
printf(" --include-from <file> Read include patterns from file\n");
|
||||||
printf(" --max-size <n> Skip files larger than n bytes\n");
|
printf(" --max-size <n> Skip files larger than n bytes\n");
|
||||||
printf(" --min-size <n> Skip files smaller than n bytes\n");
|
printf(" --min-size <n> Skip files smaller than n bytes\n");
|
||||||
|
printf(" --max-alloc <SIZE> Maximum single allocation (default: 1G)\n");
|
||||||
printf(" --incremental Skip files unchanged since last transfer\n");
|
printf(" --incremental Skip files unchanged since last transfer\n");
|
||||||
|
printf(" --size-only Skip incremental files matching in size, ignoring mtime\n");
|
||||||
|
printf(" -I, --ignore-times Transfer files even when size and mtime match\n");
|
||||||
|
printf(" -@, --modify-window <sec> Modification time tolerance\n");
|
||||||
|
printf(" -u, --update Skip files newer than the source on receiver\n");
|
||||||
|
printf(" --existing Skip files not already present at destination\n");
|
||||||
|
printf(" --checksum-choice, --cc <alg> Checksum algorithm (not supported yet; xxHash64 is "
|
||||||
|
"used)\n");
|
||||||
printf(" --delta Delta transfer for changed files (requires --incremental)\n");
|
printf(" --delta Delta transfer for changed files (requires --incremental)\n");
|
||||||
|
printf(" -W, --whole-file Transfer changed files without delta processing\n");
|
||||||
printf(" --delta-block <n> Delta block size in bytes (default: %d)\n",
|
printf(" --delta-block <n> Delta block size in bytes (default: %d)\n",
|
||||||
DELTA_BLOCK_SIZE_DEFAULT);
|
DELTA_BLOCK_SIZE_DEFAULT);
|
||||||
printf(" --delta-max <n> Max file size for delta transfer (default: %llu)\n",
|
printf(" --delta-max <n> Max file size for delta transfer (default: %llu)\n",
|
||||||
DELTA_MAX_FILE_SIZE);
|
DELTA_MAX_FILE_SIZE);
|
||||||
printf(" -m Enable multithreading\n");
|
printf(" -m Enable multithreading\n");
|
||||||
printf(" -s Enable chunk serialization\n");
|
printf(" -s Enable chunk serialization\n");
|
||||||
|
printf(" --secluded-args Accept rsync compatibility option (no effect)\n");
|
||||||
printf(" -f Enable sendfile (TCP only, not with -c or -s)\n");
|
printf(" -f Enable sendfile (TCP only, not with -c or -s)\n");
|
||||||
|
printf(" --compress-choice <alg> Compression algorithm (default: zstd)\n");
|
||||||
|
printf(" --zc <alg> Alias for --compress-choice\n");
|
||||||
printf(" -v, --verbose Enable debug logging\n");
|
printf(" -v, --verbose Enable debug logging\n");
|
||||||
|
printf(" -q, --quiet Suppress non-error output\n");
|
||||||
|
printf(" --debug=FLAGS Fine-grained debug logging (use --debug=help for flags)\n");
|
||||||
|
printf(" --info=FLAGS Fine-grained info: copy,misc,skip,stats,all,none\n");
|
||||||
|
printf(" none suppresses info even with --verbose\n");
|
||||||
printf(" -M, --preserve Preserve file metadata\n");
|
printf(" -M, --preserve Preserve file metadata\n");
|
||||||
|
printf(" -E, --executability Preserve executable permission bits\n");
|
||||||
printf(" --chmod <changes> Modify transferred permissions (rsync syntax)\n");
|
printf(" --chmod <changes> Modify transferred permissions (rsync syntax)\n");
|
||||||
printf(" --chunk-size <n> Chunk size in bytes (default: %d)\n", DEFAULT_CHUNK_SIZE);
|
printf(" --chunk-size <n> Chunk size in bytes (default: %d)\n", DEFAULT_CHUNK_SIZE);
|
||||||
printf(" --source-dir <path> Source directory\n");
|
printf(" --source-dir <path> Source directory\n");
|
||||||
@@ -57,19 +81,38 @@ void print_usage(void) {
|
|||||||
printf(" --backup-dir <dir> Directory for backups (requires --backup)\n");
|
printf(" --backup-dir <dir> Directory for backups (requires --backup)\n");
|
||||||
printf(" --suffix <str> Backup suffix (default: ~)\n");
|
printf(" --suffix <str> Backup suffix (default: ~)\n");
|
||||||
printf(" --stats Print transfer statistics at end\n");
|
printf(" --stats Print transfer statistics at end\n");
|
||||||
|
printf(" -i, --itemize-changes Print an rsync-style per-file change line\n");
|
||||||
|
printf(" --out-format=FORMAT Output format for changed files (%%f %%n %%l %%b %%M %%%%)\n");
|
||||||
|
printf(" --list-only List source files instead of transferring\n");
|
||||||
|
printf(" --log-file-format=FORMAT Per-file log line format (needs --log-file)\n");
|
||||||
|
printf(" -h, --human-readable Print byte sizes in human-readable form\n");
|
||||||
printf(" --max-depth <n> Maximum directory depth (0=unlimited)\n");
|
printf(" --max-depth <n> Maximum directory depth (0=unlimited)\n");
|
||||||
printf(" --log-file <path> Write log messages to file\n");
|
printf(" --log-file <path> Write log messages to file\n");
|
||||||
|
printf(" --stderr=MODE Route logging to stderr: errors or all\n");
|
||||||
printf(" --partial Keep partial files on interrupted transfer\n");
|
printf(" --partial Keep partial files on interrupted transfer\n");
|
||||||
printf(" --partial-dir <dir> Directory for partial files\n");
|
printf(" --partial-dir <dir> Directory for partial files\n");
|
||||||
printf(" --fastsync-server-path <path>\n");
|
printf(" --fastsync-server-path <path>\n");
|
||||||
printf(" Path to fastsync-server on remote (default: fastsync-server)\n");
|
printf(" Path to fastsync-server on remote (default: fastsync-server)\n");
|
||||||
|
printf(
|
||||||
|
" --old-args Disable safe SSH command argument quoting (legacy compatibility)\n");
|
||||||
printf(" -l, --links Copy symlinks as symlinks\n");
|
printf(" -l, --links Copy symlinks as symlinks\n");
|
||||||
printf(" --copy-links Transform symlinks into referent files\n");
|
printf(" --copy-links Transform symlinks into referent files\n");
|
||||||
printf(" --safe-links Skip symlinks that point outside transfer tree\n");
|
printf(" --safe-links Skip symlinks that point outside transfer tree\n");
|
||||||
printf(" --copy-unsafe-links Only transform unsafe symlinks into referent files\n");
|
printf(" --copy-unsafe-links Only transform unsafe symlinks into referent files\n");
|
||||||
printf(" -S, --sparse Handle sparse files efficiently\n");
|
printf(" -S, --sparse Handle sparse files efficiently\n");
|
||||||
printf(" --inplace Update files in-place (no temp+rename)\n");
|
printf(" --inplace Update files in-place (no temp+rename)\n");
|
||||||
|
printf(" --fsync Fsync every written file before publication\n");
|
||||||
printf(" --compress-level <n> Compression level (default: 5)\n");
|
printf(" --compress-level <n> Compression level (default: 5)\n");
|
||||||
|
printf(" --zl <n> Alias for --compress-level\n");
|
||||||
|
printf(" --skip-compress=LIST Skip compression for comma-separated suffixes\n");
|
||||||
|
printf(" --compress-threads <n> Compression worker threads (requires zstd threaded support)\n");
|
||||||
|
printf(" --no-OPTION Disable a supported boolean option\n");
|
||||||
printf(" --help Show this help\n");
|
printf(" --help Show this help\n");
|
||||||
printf(" -V, --version Show version\n");
|
printf(" -V, --version Show version\n");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void print_debug_usage(void) {
|
||||||
|
printf("Supported debug flags: IO,PROTO,PACK,UTIL,ALL,NONE\n");
|
||||||
|
printf("Flags may be comma-separated, for example: --debug=io,proto\n");
|
||||||
|
printf("Other rsync debug flags are unsupported and rejected.\n");
|
||||||
|
}
|
||||||
@@ -2,5 +2,6 @@
|
|||||||
#define USAGE_H
|
#define USAGE_H
|
||||||
|
|
||||||
void print_usage(void);
|
void print_usage(void);
|
||||||
|
void print_debug_usage(void);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
+100
-13
@@ -1,12 +1,56 @@
|
|||||||
#include "receiver.h"
|
#include "receiver.h"
|
||||||
|
|
||||||
#include "chunk.h"
|
#include "chunk.h"
|
||||||
|
#include "file_receive.h"
|
||||||
#include "log.h"
|
#include "log.h"
|
||||||
|
#include "metadata.h"
|
||||||
#include "protocol.h"
|
#include "protocol.h"
|
||||||
#include "utils.h"
|
#include "utils.h"
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <sys/stat.h>
|
#include <sys/stat.h>
|
||||||
|
|
||||||
|
bool receiver_outcomes_append(ReceiverOutcomes* outcomes, unsigned char code) {
|
||||||
|
if (!outcomes)
|
||||||
|
return false;
|
||||||
|
if (outcomes->count == outcomes->capacity) {
|
||||||
|
size_t new_capacity = outcomes->capacity == 0 ? 64 : outcomes->capacity * 2;
|
||||||
|
if (new_capacity < outcomes->capacity)
|
||||||
|
return false;
|
||||||
|
unsigned char* grown = realloc(outcomes->entries, new_capacity);
|
||||||
|
if (!grown)
|
||||||
|
return false;
|
||||||
|
outcomes->entries = grown;
|
||||||
|
outcomes->capacity = new_capacity;
|
||||||
|
}
|
||||||
|
outcomes->entries[outcomes->count++] = code;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
void receiver_outcomes_destroy(ReceiverOutcomes* outcomes) {
|
||||||
|
if (!outcomes)
|
||||||
|
return;
|
||||||
|
free(outcomes->entries);
|
||||||
|
outcomes->entries = NULL;
|
||||||
|
outcomes->count = 0;
|
||||||
|
outcomes->capacity = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* End-of-transfer success frame. When --remove-source-files was negotiated
|
||||||
|
each processed data file is acknowledged first (STATUS_NEXT = written,
|
||||||
|
STATUS_OK = skipped) so the sender never removes a source the receiver did
|
||||||
|
not actually store. The frame always ends with a plain STATUS_OK. */
|
||||||
|
bool receiver_send_final_success(int fd, const Config* config, const ReceiverOutcomes* outcomes) {
|
||||||
|
if (!config->remove_source_files)
|
||||||
|
return send_status(fd, STATUS_OK);
|
||||||
|
size_t count = outcomes ? outcomes->count : 0;
|
||||||
|
for (size_t i = 0; i < count; i++) {
|
||||||
|
Status per_file = outcomes->entries[i] == FILE_SAVE_WRITTEN ? STATUS_NEXT : STATUS_OK;
|
||||||
|
if (!send_status(fd, per_file))
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return send_status(fd, STATUS_OK);
|
||||||
|
}
|
||||||
|
|
||||||
static bool receiver_process_chunk(Chunk* chunk, const ReceiverSink* sink) {
|
static bool receiver_process_chunk(Chunk* chunk, const ReceiverSink* sink) {
|
||||||
if (!chunk || !sink || !sink->store_file)
|
if (!chunk || !sink || !sink->store_file)
|
||||||
return false;
|
return false;
|
||||||
@@ -37,9 +81,13 @@ static bool receiver_process_batch(Config* config, int file_descriptor) {
|
|||||||
return false;
|
return false;
|
||||||
unsigned long long check_size;
|
unsigned long long check_size;
|
||||||
long long check_mtime;
|
long long check_mtime;
|
||||||
|
long long check_mtime_nsec;
|
||||||
if (!receive_n_data(file_descriptor, &check_size, sizeof(check_size)) ||
|
if (!receive_n_data(file_descriptor, &check_size, sizeof(check_size)) ||
|
||||||
!receive_n_data(file_descriptor, &check_mtime, sizeof(check_mtime))) {
|
!receive_n_data(file_descriptor, &check_mtime, sizeof(check_mtime)) ||
|
||||||
|
!receive_n_data(file_descriptor, &check_mtime_nsec, sizeof(check_mtime_nsec)) ||
|
||||||
|
check_mtime_nsec < 0 || check_mtime_nsec >= 1000000000LL) {
|
||||||
free(check_path);
|
free(check_path);
|
||||||
|
send_status(file_descriptor, STATUS_ERROR);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
if (!utils_valid_batch_path(check_path)) {
|
if (!utils_valid_batch_path(check_path)) {
|
||||||
@@ -47,7 +95,7 @@ static bool receiver_process_batch(Config* config, int file_descriptor) {
|
|||||||
send_status(file_descriptor, STATUS_ERROR);
|
send_status(file_descriptor, STATUS_ERROR);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
if (check_size > MAX_RECEIVE_FILE_SIZE) {
|
if (check_size > MAX_RECEIVE_WHOLE_FILE_SIZE) {
|
||||||
free(check_path);
|
free(check_path);
|
||||||
send_status(file_descriptor, STATUS_ERROR);
|
send_status(file_descriptor, STATUS_ERROR);
|
||||||
return false;
|
return false;
|
||||||
@@ -60,8 +108,15 @@ static bool receiver_process_batch(Config* config, int file_descriptor) {
|
|||||||
}
|
}
|
||||||
struct stat st;
|
struct stat st;
|
||||||
bool has_old = file_stat_secure(full_path, &st);
|
bool has_old = file_stat_secure(full_path, &st);
|
||||||
bool match = has_old && (unsigned long long)st.st_size == check_size &&
|
long long old_mtime_nsec = 0;
|
||||||
(long long)st.st_mtime == check_mtime;
|
if (has_old) {
|
||||||
|
#ifdef __linux__
|
||||||
|
old_mtime_nsec = st.st_mtim.tv_nsec;
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
bool match = !config->ignore_times && has_old && (unsigned long long)st.st_size == check_size &&
|
||||||
|
metadata_mtime_matches(st.st_mtime, old_mtime_nsec, (time_t)check_mtime,
|
||||||
|
(long)check_mtime_nsec, config->modify_window);
|
||||||
bool sent = send_status(file_descriptor, match ? STATUS_OK : STATUS_NEXT);
|
bool sent = send_status(file_descriptor, match ? STATUS_OK : STATUS_NEXT);
|
||||||
free(full_path);
|
free(full_path);
|
||||||
free(check_path);
|
free(check_path);
|
||||||
@@ -118,8 +173,14 @@ int receiver_process(Config* config, int file_descriptor, const ReceiverSink* si
|
|||||||
log_message(LOG_LEVEL_ERROR, "Did not receive FINISHED Status");
|
log_message(LOG_LEVEL_ERROR, "Did not receive FINISHED Status");
|
||||||
goto receive_error;
|
goto receive_error;
|
||||||
}
|
}
|
||||||
if (sink->send_success && !send_status(file_descriptor, STATUS_OK))
|
if (sink->send_success) {
|
||||||
return -1;
|
if (sink->send_success_frame) {
|
||||||
|
if (!sink->send_success_frame(file_descriptor, sink->context))
|
||||||
|
return -1;
|
||||||
|
} else if (!send_status(file_descriptor, STATUS_OK)) {
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
}
|
||||||
return 0;
|
return 0;
|
||||||
|
|
||||||
receive_error:
|
receive_error:
|
||||||
@@ -128,15 +189,41 @@ receive_error:
|
|||||||
return -1;
|
return -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
static bool receiver_save_file(File* file, void* context) {
|
/* ---- Single-threaded sink (used by receiver_receive_files) ---- */
|
||||||
Config* config = context;
|
|
||||||
bool success =
|
typedef struct {
|
||||||
!config->save_to_disk || file_save_to_disk(config->receive_root_directory, file, config);
|
Config* config;
|
||||||
|
ReceiverOutcomes outcomes;
|
||||||
|
} ReceiverSaveContext;
|
||||||
|
|
||||||
|
static bool receiver_save_file(File* file, void* context_pointer) {
|
||||||
|
ReceiverSaveContext* context = context_pointer;
|
||||||
|
FileSaveResult result = FILE_SAVE_ERROR;
|
||||||
|
if (!context->config->save_to_disk) {
|
||||||
|
/* Nothing is stored; report the file as not-written so a
|
||||||
|
--remove-source-files sender keeps its source. */
|
||||||
|
result = FILE_SAVE_SKIPPED;
|
||||||
|
} else {
|
||||||
|
result = file_save_to_disk_full(context->config->receive_root_directory, file, context->config);
|
||||||
|
}
|
||||||
|
if (result != FILE_SAVE_ERROR && context->config->remove_source_files &&
|
||||||
|
!receiver_outcomes_append(&context->outcomes, (unsigned char)result)) {
|
||||||
|
file_destroy(file);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
file_destroy(file);
|
file_destroy(file);
|
||||||
return success;
|
return result != FILE_SAVE_ERROR;
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool receiver_send_success_frame(int fd, void* context_pointer) {
|
||||||
|
ReceiverSaveContext* context = context_pointer;
|
||||||
|
return receiver_send_final_success(fd, context->config, &context->outcomes);
|
||||||
}
|
}
|
||||||
|
|
||||||
int receiver_receive_files(Config* config, int file_descriptor) {
|
int receiver_receive_files(Config* config, int file_descriptor) {
|
||||||
ReceiverSink sink = {receiver_save_file, config, true, true};
|
ReceiverSaveContext context = {.config = config, .outcomes = {0}};
|
||||||
return receiver_process(config, file_descriptor, &sink);
|
ReceiverSink sink = {receiver_save_file, &context, true, true, receiver_send_success_frame};
|
||||||
|
int ret = receiver_process(config, file_descriptor, &sink);
|
||||||
|
receiver_outcomes_destroy(&context.outcomes);
|
||||||
|
return ret;
|
||||||
}
|
}
|
||||||
@@ -3,16 +3,37 @@
|
|||||||
|
|
||||||
#include "config.h"
|
#include "config.h"
|
||||||
#include "file.h"
|
#include "file.h"
|
||||||
|
#include "file_receive.h"
|
||||||
|
|
||||||
typedef bool (*ReceiverFileSink)(File* file, void* context);
|
typedef bool (*ReceiverFileSink)(File* file, void* context);
|
||||||
|
|
||||||
|
/* Ordered per-file save outcomes for one connection. One entry is appended
|
||||||
|
for every data-bearing file the receiver processes (in the order the files
|
||||||
|
were sent) so the sender of a --remove-source-files transfer can be told
|
||||||
|
which sources were actually written versus skipped on the receiver. */
|
||||||
|
typedef struct {
|
||||||
|
unsigned char* entries; /* FILE_SAVE_WRITTEN or FILE_SAVE_SKIPPED */
|
||||||
|
size_t count;
|
||||||
|
size_t capacity;
|
||||||
|
} ReceiverOutcomes;
|
||||||
|
|
||||||
|
typedef bool (*ReceiverSuccessFrame)(int fd, void* context);
|
||||||
|
|
||||||
typedef struct {
|
typedef struct {
|
||||||
ReceiverFileSink store_file;
|
ReceiverFileSink store_file;
|
||||||
void* context;
|
void* context;
|
||||||
bool send_error;
|
bool send_error;
|
||||||
bool send_success;
|
bool send_success;
|
||||||
|
/* Emits the end-of-transfer success frame. When the sender requested
|
||||||
|
--remove-source-files this includes one per-file status per processed
|
||||||
|
data file followed by the final STATUS_OK; otherwise just STATUS_OK. */
|
||||||
|
ReceiverSuccessFrame send_success_frame;
|
||||||
} ReceiverSink;
|
} ReceiverSink;
|
||||||
|
|
||||||
|
bool receiver_outcomes_append(ReceiverOutcomes* outcomes, unsigned char code);
|
||||||
|
void receiver_outcomes_destroy(ReceiverOutcomes* outcomes);
|
||||||
|
bool receiver_send_final_success(int fd, const Config* config, const ReceiverOutcomes* outcomes);
|
||||||
|
|
||||||
int receiver_process(Config* config, int file_descriptor, const ReceiverSink* sink);
|
int receiver_process(Config* config, int file_descriptor, const ReceiverSink* sink);
|
||||||
int receiver_receive_files(Config* config, int file_descriptor);
|
int receiver_receive_files(Config* config, int file_descriptor);
|
||||||
|
|
||||||
|
|||||||
+34
-129
@@ -1,13 +1,12 @@
|
|||||||
#include "config.h"
|
#include "config.h"
|
||||||
#include "chunk.h"
|
|
||||||
#include "file.h"
|
#include "file.h"
|
||||||
#include "log.h"
|
#include "log.h"
|
||||||
#include "multiprocessing.h"
|
#include "multiprocessing.h"
|
||||||
|
#include "protocol.h"
|
||||||
#include "queue.h"
|
#include "queue.h"
|
||||||
#include "receiver.h"
|
#include "receiver.h"
|
||||||
#include "transport_tcp.h"
|
#include "transport_tcp.h"
|
||||||
#include "transport_tls.h"
|
#include "transport_tls.h"
|
||||||
#include "unistd.h"
|
|
||||||
#include "utils.h"
|
#include "utils.h"
|
||||||
#include <fcntl.h>
|
#include <fcntl.h>
|
||||||
#include <limits.h>
|
#include <limits.h>
|
||||||
@@ -15,7 +14,6 @@
|
|||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
#include <sys/stat.h>
|
|
||||||
#include <unistd.h>
|
#include <unistd.h>
|
||||||
#include <openssl/x509.h>
|
#include <openssl/x509.h>
|
||||||
|
|
||||||
@@ -25,6 +23,13 @@ static bool allow_delete;
|
|||||||
static bool allow_unauthenticated;
|
static bool allow_unauthenticated;
|
||||||
static const char* required_client_cn;
|
static const char* required_client_cn;
|
||||||
|
|
||||||
|
/* Aggregate payload bytes the multithreaded receiver may buffer ahead of the
|
||||||
|
slow disk writer. Receiving one more chunk adds up to ~2 * MAX_CHUNK_SIZE
|
||||||
|
of transient wire/decompression buffers on top of the queued payloads, so
|
||||||
|
this ceiling keeps total per-connection receive memory (decompressed and
|
||||||
|
per-file copied chunk buffers included) within MAX_CONNECTION_MEMORY. */
|
||||||
|
#define RECEIVER_QUEUE_MAX_BYTES (MAX_CONNECTION_MEMORY - 2 * MAX_CHUNK_SIZE)
|
||||||
|
|
||||||
static bool tls_client_identity_allowed(SSL* ssl) {
|
static bool tls_client_identity_allowed(SSL* ssl) {
|
||||||
if (!ssl || !required_client_cn)
|
if (!ssl || !required_client_cn)
|
||||||
return false;
|
return false;
|
||||||
@@ -100,125 +105,6 @@ static bool __attribute__((unused)) configure_authorization(const char* root) {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
int receive_files(Config* config, int fd) {
|
|
||||||
Status status;
|
|
||||||
if (!receive_status(fd, &status))
|
|
||||||
return -1;
|
|
||||||
|
|
||||||
while (status == STATUS_NEXT || status == STATUS_CHUNK || status == STATUS_CHECK ||
|
|
||||||
status == STATUS_KEEPALIVE || status == STATUS_ABORT || status == STATUS_CHECK_BATCH) {
|
|
||||||
if (status == STATUS_KEEPALIVE) {
|
|
||||||
send_status(fd, STATUS_KEEPALIVE);
|
|
||||||
goto next;
|
|
||||||
}
|
|
||||||
if (status == STATUS_ABORT) {
|
|
||||||
log_message(LOG_LEVEL_INFO, "Received abort from client, cleaning up");
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
if (status == STATUS_CHECK) {
|
|
||||||
bool skipped;
|
|
||||||
File* file = receive_incremental_check(fd, config, &skipped);
|
|
||||||
if (skipped)
|
|
||||||
goto next;
|
|
||||||
if (file == NULL && !skipped)
|
|
||||||
return -1;
|
|
||||||
if (config->save_to_disk &&
|
|
||||||
!file_save_to_disk(config->receive_root_directory, file, config)) {
|
|
||||||
file_destroy(file);
|
|
||||||
send_status(fd, STATUS_ERROR);
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
file_destroy(file);
|
|
||||||
} else if (status == STATUS_CHUNK) {
|
|
||||||
Chunk* chunk = receive_chunk_data(fd, config);
|
|
||||||
if (chunk == NULL) {
|
|
||||||
send_status(fd, STATUS_ERROR);
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
for (int i = 0; i < chunk->element_count; i++) {
|
|
||||||
if (config->save_to_disk &&
|
|
||||||
!file_save_to_disk(config->receive_root_directory, chunk->items[i], config)) {
|
|
||||||
chunk_destroy(chunk);
|
|
||||||
send_status(fd, STATUS_ERROR);
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
chunk_destroy(chunk);
|
|
||||||
} else if (status == STATUS_CHECK_BATCH) {
|
|
||||||
int count;
|
|
||||||
/* Batch framing has no checksum field yet; never silently downgrade a
|
|
||||||
checksum-enabled transfer into mtime-only matching. */
|
|
||||||
if (config->checksum || !receive_int(fd, &count) || count < 0 || count > MAX_MANIFEST_ENTRIES)
|
|
||||||
return -1;
|
|
||||||
for (int i = 0; i < count; i++) {
|
|
||||||
char* check_path = receive_str(fd);
|
|
||||||
if (!check_path)
|
|
||||||
return -1;
|
|
||||||
unsigned long long check_size;
|
|
||||||
long long check_mtime;
|
|
||||||
if (!receive_n_data(fd, &check_size, sizeof(check_size)) ||
|
|
||||||
!receive_n_data(fd, &check_mtime, sizeof(check_mtime))) {
|
|
||||||
free(check_path);
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
if (!utils_valid_batch_path(check_path)) {
|
|
||||||
free(check_path);
|
|
||||||
send_status(fd, STATUS_ERROR);
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
struct stat st;
|
|
||||||
char* full_path = path_cat(config->receive_root_directory, check_path);
|
|
||||||
if (!full_path) {
|
|
||||||
free(check_path);
|
|
||||||
send_status(fd, STATUS_ERROR);
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
bool has_old = full_path && file_stat_secure(full_path, &st);
|
|
||||||
bool match = has_old && (unsigned long long)st.st_size == check_size &&
|
|
||||||
(long long)st.st_mtime == check_mtime;
|
|
||||||
bool sent = send_status(fd, match ? STATUS_OK : STATUS_NEXT);
|
|
||||||
free(full_path);
|
|
||||||
free(check_path);
|
|
||||||
if (!sent)
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
goto next;
|
|
||||||
} else {
|
|
||||||
File* file = file_receive(config, fd);
|
|
||||||
if (file == NULL) {
|
|
||||||
log_message(LOG_LEVEL_ERROR, "Failed to receive file");
|
|
||||||
send_status(fd, STATUS_ERROR);
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
if (config->save_to_disk &&
|
|
||||||
!file_save_to_disk(config->receive_root_directory, file, config)) {
|
|
||||||
file_destroy(file);
|
|
||||||
send_status(fd, STATUS_ERROR);
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
file_destroy(file);
|
|
||||||
}
|
|
||||||
next:
|
|
||||||
if (!receive_status(fd, &status)) {
|
|
||||||
send_status(fd, STATUS_ERROR);
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (status == STATUS_MANIFEST) {
|
|
||||||
if (receive_manifest(fd, config, &status) != 0) {
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (status != STATUS_FINISHED) {
|
|
||||||
log_message(LOG_LEVEL_ERROR, "Did not receive FINISHED Status");
|
|
||||||
send_status(fd, STATUS_ERROR);
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
send_status(fd, STATUS_OK);
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
void handler(int file_descriptor) {
|
void handler(int file_descriptor) {
|
||||||
SSL* ssl = io_get_ssl();
|
SSL* ssl = io_get_ssl();
|
||||||
ProtocolSession session;
|
ProtocolSession session;
|
||||||
@@ -232,6 +118,7 @@ void handler(int file_descriptor) {
|
|||||||
protocol_session_unbind();
|
protocol_session_unbind();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
protocol_set_8_bit_output(config->eight_bit_output);
|
||||||
if (!authorized_root) {
|
if (!authorized_root) {
|
||||||
log_message(LOG_LEVEL_ERROR, "No server-side destination root configured");
|
log_message(LOG_LEVEL_ERROR, "No server-side destination root configured");
|
||||||
config_delete(config);
|
config_delete(config);
|
||||||
@@ -294,7 +181,10 @@ void handler(int file_descriptor) {
|
|||||||
protocol_session_unbind();
|
protocol_session_unbind();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
context->session.total_allocated_bytes = session.total_allocated_bytes;
|
protocol_session_set_max_alloc(&context->session, config->max_alloc);
|
||||||
|
atomic_store(&context->session.total_allocated_bytes,
|
||||||
|
atomic_load(&session.total_allocated_bytes));
|
||||||
|
pipeline_context_receiver_set_queue_byte_limit(context, RECEIVER_QUEUE_MAX_BYTES);
|
||||||
thrd_t receiver, writer;
|
thrd_t receiver, writer;
|
||||||
bool receiver_created = thrd_create(&receiver, receive_thread, context) == thrd_success;
|
bool receiver_created = thrd_create(&receiver, receive_thread, context) == thrd_success;
|
||||||
bool writer_created = false;
|
bool writer_created = false;
|
||||||
@@ -323,9 +213,15 @@ void handler(int file_descriptor) {
|
|||||||
int writer_result;
|
int writer_result;
|
||||||
thrd_join(receiver, &receiver_result);
|
thrd_join(receiver, &receiver_result);
|
||||||
thrd_join(writer, &writer_result);
|
thrd_join(writer, &writer_result);
|
||||||
send_status(file_descriptor, receiver_result == thrd_success && writer_result == thrd_success
|
bool transfer_ok = receiver_result == thrd_success && writer_result == thrd_success;
|
||||||
? STATUS_OK
|
if (transfer_ok) {
|
||||||
: STATUS_ERROR);
|
if (!receiver_send_final_success(file_descriptor, config, &context->outcomes))
|
||||||
|
transfer_ok = false;
|
||||||
|
} else {
|
||||||
|
send_status(file_descriptor, STATUS_ERROR);
|
||||||
|
}
|
||||||
|
if (!transfer_ok)
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Transfer failed");
|
||||||
pipeline_context_receiver_destroy(context);
|
pipeline_context_receiver_destroy(context);
|
||||||
} else {
|
} else {
|
||||||
if (receiver_receive_files(config, file_descriptor) != 0)
|
if (receiver_receive_files(config, file_descriptor) != 0)
|
||||||
@@ -380,6 +276,7 @@ int main(int argc, char* argv[]) {
|
|||||||
stdio_mode = true;
|
stdio_mode = true;
|
||||||
} else if (strcmp(argv[i], "-v") == 0 || strcmp(argv[i], "--verbose") == 0) {
|
} else if (strcmp(argv[i], "-v") == 0 || strcmp(argv[i], "--verbose") == 0) {
|
||||||
set_log_level(LOG_LEVEL_DEBUG);
|
set_log_level(LOG_LEVEL_DEBUG);
|
||||||
|
set_log_debug_flags(LOG_DEBUG_ALL);
|
||||||
} else if (strcmp(argv[i], "--tls") == 0) {
|
} else if (strcmp(argv[i], "--tls") == 0) {
|
||||||
use_tls = true;
|
use_tls = true;
|
||||||
} else if (strcmp(argv[i], "--cert") == 0 && i + 1 < argc) {
|
} else if (strcmp(argv[i], "--cert") == 0 && i + 1 < argc) {
|
||||||
@@ -400,12 +297,17 @@ int main(int argc, char* argv[]) {
|
|||||||
char* end;
|
char* end;
|
||||||
long p = strtol(argv[++i], &end, 10);
|
long p = strtol(argv[++i], &end, 10);
|
||||||
if (*end || p <= 0 || p > 65535) {
|
if (*end || p <= 0 || p > 65535) {
|
||||||
fprintf(stderr, "Error: invalid port '%s' (must be 1-65535)\n", argv[i]);
|
char* escaped = output_escape(argv[i], false);
|
||||||
|
fprintf(stderr, "Error: invalid port '%s' (must be 1-65535)\n",
|
||||||
|
escaped ? escaped : "<allocation failed>");
|
||||||
|
free(escaped);
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
port = (int)p;
|
port = (int)p;
|
||||||
} else if (argv[i][0] == '-') {
|
} else if (argv[i][0] == '-') {
|
||||||
fprintf(stderr, "Unknown option: %s\n", argv[i]);
|
char* escaped = output_escape(argv[i], false);
|
||||||
|
fprintf(stderr, "Unknown option: %s\n", escaped ? escaped : "<allocation failed>");
|
||||||
|
free(escaped);
|
||||||
print_server_usage();
|
print_server_usage();
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
@@ -415,7 +317,10 @@ int main(int argc, char* argv[]) {
|
|||||||
signal(SIGINT, cleanup);
|
signal(SIGINT, cleanup);
|
||||||
signal(SIGTERM, cleanup);
|
signal(SIGTERM, cleanup);
|
||||||
if (!configure_authorization(destination_root)) {
|
if (!configure_authorization(destination_root)) {
|
||||||
fprintf(stderr, "Error: invalid destination root '%s'\n", destination_root);
|
char* escaped = output_escape(destination_root, false);
|
||||||
|
fprintf(stderr, "Error: invalid destination root '%s'\n",
|
||||||
|
escaped ? escaped : "<allocation failed>");
|
||||||
|
free(escaped);
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
if (stdio_mode) {
|
if (stdio_mode) {
|
||||||
|
|||||||
@@ -1,17 +1,18 @@
|
|||||||
#include "log.h"
|
#include "log.h"
|
||||||
#include "array_list.h"
|
#include "array_list.h"
|
||||||
|
#include "protocol.h"
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
|
|
||||||
ArrayList* array_list_create(void (*item_destroyer)(void* item)) {
|
ArrayList* array_list_create(void (*item_destroyer)(void* item)) {
|
||||||
ArrayList* list = (ArrayList*)malloc(sizeof(ArrayList));
|
ArrayList* list = (ArrayList*)protocol_alloc(sizeof(ArrayList));
|
||||||
if (list == NULL) {
|
if (list == NULL) {
|
||||||
log_perror("ERROR: Could not allocate memory for array list struct");
|
log_perror("ERROR: Could not allocate memory for array list struct");
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
list->items = malloc(INITIAL_ARRAY_SIZE * sizeof(void*));
|
list->items = protocol_alloc(INITIAL_ARRAY_SIZE * sizeof(void*));
|
||||||
if (list->items == NULL) {
|
if (list->items == NULL) {
|
||||||
free(list);
|
free(list);
|
||||||
return NULL;
|
return NULL;
|
||||||
@@ -41,7 +42,7 @@ static bool array_list_extend(ArrayList* array_list) {
|
|||||||
int new_capacity = array_list->capacity * 2;
|
int new_capacity = array_list->capacity * 2;
|
||||||
if (new_capacity == 0)
|
if (new_capacity == 0)
|
||||||
new_capacity = INITIAL_ARRAY_SIZE;
|
new_capacity = INITIAL_ARRAY_SIZE;
|
||||||
void* new_items = realloc(array_list->items, new_capacity * sizeof(void*));
|
void* new_items = protocol_realloc(array_list->items, new_capacity * sizeof(void*));
|
||||||
if (new_items == NULL) {
|
if (new_items == NULL) {
|
||||||
log_perror("ERROR: Could not reallocate memory for array list items");
|
log_perror("ERROR: Could not reallocate memory for array list items");
|
||||||
return false;
|
return false;
|
||||||
@@ -67,7 +68,7 @@ void** array_list_to_array(const ArrayList* array_list) {
|
|||||||
if (array_list == NULL) {
|
if (array_list == NULL) {
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
void** array = malloc(array_list->size * sizeof(void*));
|
void** array = protocol_alloc(array_list->size * sizeof(void*));
|
||||||
if (array == NULL) {
|
if (array == NULL) {
|
||||||
log_perror("Could not malloc space for array from array list!");
|
log_perror("Could not malloc space for array from array list!");
|
||||||
return NULL;
|
return NULL;
|
||||||
|
|||||||
+11
-6
@@ -22,7 +22,7 @@
|
|||||||
Chunk* chunk_create(File** items, int element_count) {
|
Chunk* chunk_create(File** items, int element_count) {
|
||||||
if (element_count < 0 || (element_count > 0 && items == NULL))
|
if (element_count < 0 || (element_count > 0 && items == NULL))
|
||||||
return NULL;
|
return NULL;
|
||||||
Chunk* chunk = (Chunk*)malloc(sizeof(Chunk));
|
Chunk* chunk = (Chunk*)protocol_alloc(sizeof(Chunk));
|
||||||
if (chunk == NULL) {
|
if (chunk == NULL) {
|
||||||
log_perror("ERROR: Could not allocate memory for chunk structure");
|
log_perror("ERROR: Could not allocate memory for chunk structure");
|
||||||
return NULL;
|
return NULL;
|
||||||
@@ -35,7 +35,7 @@ Chunk* chunk_create(File** items, int element_count) {
|
|||||||
free(chunk);
|
free(chunk);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
chunk->items = (File**)malloc((size_t)element_count * sizeof(File*));
|
chunk->items = (File**)protocol_alloc((size_t)element_count * sizeof(File*));
|
||||||
if (chunk->items == NULL) {
|
if (chunk->items == NULL) {
|
||||||
free(chunk);
|
free(chunk);
|
||||||
return NULL;
|
return NULL;
|
||||||
@@ -158,7 +158,7 @@ Chunk* chunk_deserialize(Data* data, bool use_metadata) {
|
|||||||
array_list_delete(files);
|
array_list_delete(files);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
char* path = malloc(path_len + 1);
|
char* path = protocol_alloc(path_len + 1);
|
||||||
if (path == NULL) {
|
if (path == NULL) {
|
||||||
log_perror("Could not allocate memory for file path");
|
log_perror("Could not allocate memory for file path");
|
||||||
array_list_delete(files);
|
array_list_delete(files);
|
||||||
@@ -245,7 +245,7 @@ Chunk* chunk_deserialize(Data* data, bool use_metadata) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
size_t allocation_size = file_data_size > 0 ? file_data_size : 1;
|
size_t allocation_size = file_data_size > 0 ? file_data_size : 1;
|
||||||
void* file_data = malloc(allocation_size);
|
void* file_data = protocol_alloc(allocation_size);
|
||||||
if (file_data == NULL) {
|
if (file_data == NULL) {
|
||||||
log_perror("Could not allocate memory for file data");
|
log_perror("Could not allocate memory for file data");
|
||||||
file_destroy(file);
|
file_destroy(file);
|
||||||
@@ -290,15 +290,20 @@ Chunk* chunk_deserialize(Data* data, bool use_metadata) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
Data* chunk_compress(Chunk* chunk, int compression_level, bool use_metadata) {
|
Data* chunk_compress(Chunk* chunk, int compression_level, bool use_metadata) {
|
||||||
|
return chunk_compress_with_threads(chunk, compression_level, use_metadata, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
Data* chunk_compress_with_threads(Chunk* chunk, int compression_level, bool use_metadata,
|
||||||
|
int compression_threads) {
|
||||||
log_message(LOG_LEVEL_DEBUG, "Starting to compress chunk");
|
log_message(LOG_LEVEL_DEBUG, "Starting to compress chunk");
|
||||||
Data* serialized = chunk_serialize(chunk, use_metadata);
|
Data* serialized = chunk_serialize(chunk, use_metadata);
|
||||||
if (serialized == NULL)
|
if (serialized == NULL)
|
||||||
return NULL;
|
return NULL;
|
||||||
Data* compressed = data_compress(serialized, compression_level);
|
Data* compressed = data_compress_with_threads(serialized, compression_level, compression_threads);
|
||||||
data_destroy(serialized);
|
data_destroy(serialized);
|
||||||
if (compressed == NULL)
|
if (compressed == NULL)
|
||||||
return NULL;
|
return NULL;
|
||||||
log_message(LOG_LEVEL_DEBUG, "Chunk successfully compressed");
|
log_debug_message(LOG_DEBUG_PACK, "Chunk successfully compressed");
|
||||||
return compressed;
|
return compressed;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -19,6 +19,8 @@ void chunk_destroy(void* chunk);
|
|||||||
Data* chunk_serialize(Chunk* chunk, bool use_metadata);
|
Data* chunk_serialize(Chunk* chunk, bool use_metadata);
|
||||||
Chunk* chunk_deserialize(Data* data, bool use_metadata);
|
Chunk* chunk_deserialize(Data* data, bool use_metadata);
|
||||||
Data* chunk_compress(Chunk* chunk, int compression_level, bool use_metadata);
|
Data* chunk_compress(Chunk* chunk, int compression_level, bool use_metadata);
|
||||||
|
Data* chunk_compress_with_threads(Chunk* chunk, int compression_level, bool use_metadata,
|
||||||
|
int compression_threads);
|
||||||
Chunk* receive_chunk_data(int fd, const Config* config);
|
Chunk* receive_chunk_data(int fd, const Config* config);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
@@ -1,33 +1,53 @@
|
|||||||
#include "compression.h"
|
#include "compression.h"
|
||||||
#include "data.h"
|
#include "data.h"
|
||||||
#include "log.h"
|
#include "log.h"
|
||||||
|
#include "protocol.h"
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <limits.h>
|
#include <limits.h>
|
||||||
#include <stdint.h>
|
#include <stdint.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
#include <strings.h>
|
#include <strings.h>
|
||||||
|
#include <unistd.h>
|
||||||
#include <zstd.h>
|
#include <zstd.h>
|
||||||
|
|
||||||
#define INITIAL_DECOMPRESS_BUF_SIZE (1024 * 1024)
|
#define INITIAL_DECOMPRESS_BUF_SIZE (1024 * 1024)
|
||||||
#define MAX_DECOMPRESSED_SIZE (100ULL * 1024 * 1024) /* 100 MB hard ceiling */
|
#define MAX_DECOMPRESSED_SIZE (100ULL * 1024 * 1024) /* 100 MB hard ceiling */
|
||||||
|
|
||||||
static const char* SKIP_COMPRESSION_EXTENSIONS[] = {".jpg", ".jpeg", ".png", ".gif", ".mp4", ".mkv",
|
static char* SKIP_COMPRESSION_EXTENSIONS[] = {".jpg", ".jpeg", ".png", ".gif", ".mp4", ".mkv",
|
||||||
".zip", ".gz", ".xz", ".zst", NULL};
|
".zip", ".gz", ".xz", ".zst", NULL};
|
||||||
|
|
||||||
bool compression_should_skip(const char* path) {
|
bool compression_should_skip(const char* path) {
|
||||||
|
return compression_should_skip_with_suffixes(path, NULL, -1);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool compression_should_skip_with_suffixes(const char* path, char* const* suffixes, int count) {
|
||||||
if (!path)
|
if (!path)
|
||||||
return false;
|
return false;
|
||||||
const char* dot = strrchr(path, '.');
|
const char* dot = strrchr(path, '.');
|
||||||
if (!dot)
|
if (!dot)
|
||||||
return false;
|
return false;
|
||||||
for (int i = 0; SKIP_COMPRESSION_EXTENSIONS[i]; i++) {
|
if (count < 0) {
|
||||||
if (strcasecmp(dot, SKIP_COMPRESSION_EXTENSIONS[i]) == 0)
|
suffixes = SKIP_COMPRESSION_EXTENSIONS;
|
||||||
|
count = 0;
|
||||||
|
while (SKIP_COMPRESSION_EXTENSIONS[count])
|
||||||
|
count++;
|
||||||
|
}
|
||||||
|
for (int i = 0; i < count; i++) {
|
||||||
|
if (strcasecmp(dot, suffixes[i]) == 0)
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
Data* data_compress(Data* data_to_compress, int compression_level) {
|
Data* data_compress(Data* data_to_compress, int compression_level) {
|
||||||
|
return data_compress_with_threads(data_to_compress, compression_level, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
Data* data_compress_with_threads(Data* data_to_compress, int compression_level,
|
||||||
|
int compression_threads) {
|
||||||
|
if (!data_to_compress || (!data_to_compress->data && data_to_compress->size != 0) ||
|
||||||
|
compression_threads < 0 || compression_threads > COMPRESSION_MAX_THREADS)
|
||||||
|
return NULL;
|
||||||
log_message(LOG_LEVEL_DEBUG, "Starting to compress data");
|
log_message(LOG_LEVEL_DEBUG, "Starting to compress data");
|
||||||
size_t dst_size = ZSTD_compressBound(data_to_compress->size);
|
size_t dst_size = ZSTD_compressBound(data_to_compress->size);
|
||||||
Data* compressed_data = data_create_empty(dst_size);
|
Data* compressed_data = data_create_empty(dst_size);
|
||||||
@@ -49,6 +69,30 @@ Data* data_compress(Data* data_to_compress, int compression_level) {
|
|||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (compression_threads > 0) {
|
||||||
|
long online_cpus = sysconf(_SC_NPROCESSORS_ONLN);
|
||||||
|
int available_threads = online_cpus > 0 && online_cpus < compression_threads
|
||||||
|
? (int)online_cpus
|
||||||
|
: compression_threads;
|
||||||
|
zret = ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, available_threads);
|
||||||
|
if (ZSTD_isError(zret)) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Failed to set compression threads: %s",
|
||||||
|
ZSTD_getErrorName(zret));
|
||||||
|
ZSTD_freeCCtx(cctx);
|
||||||
|
data_destroy(compressed_data);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
/* Streaming compression needs the source size before threaded mode can end a frame. */
|
||||||
|
zret = ZSTD_CCtx_setPledgedSrcSize(cctx, data_to_compress->size);
|
||||||
|
if (ZSTD_isError(zret)) {
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Failed to set compression source size: %s",
|
||||||
|
ZSTD_getErrorName(zret));
|
||||||
|
ZSTD_freeCCtx(cctx);
|
||||||
|
data_destroy(compressed_data);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
ZSTD_inBuffer input = {data_to_compress->data, data_to_compress->size, 0};
|
ZSTD_inBuffer input = {data_to_compress->data, data_to_compress->size, 0};
|
||||||
ZSTD_outBuffer output = {compressed_data->data, dst_size, 0};
|
ZSTD_outBuffer output = {compressed_data->data, dst_size, 0};
|
||||||
|
|
||||||
@@ -66,8 +110,8 @@ Data* data_compress(Data* data_to_compress, int compression_level) {
|
|||||||
compressed_data->size = output.pos;
|
compressed_data->size = output.pos;
|
||||||
ZSTD_freeCCtx(cctx);
|
ZSTD_freeCCtx(cctx);
|
||||||
|
|
||||||
log_message(LOG_LEVEL_DEBUG, "Data succesfully compressed from %zu to %zu",
|
log_debug_message(LOG_DEBUG_UTIL, "Data succesfully compressed from %zu to %zu",
|
||||||
data_to_compress->size, compressed_data->size);
|
data_to_compress->size, compressed_data->size);
|
||||||
return compressed_data;
|
return compressed_data;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -75,7 +119,7 @@ Data* data_decompress_limited(Data* compressed_data, size_t maximum_size) {
|
|||||||
if (!compressed_data || (!compressed_data->data && compressed_data->size != 0) ||
|
if (!compressed_data || (!compressed_data->data && compressed_data->size != 0) ||
|
||||||
maximum_size == 0)
|
maximum_size == 0)
|
||||||
return NULL;
|
return NULL;
|
||||||
log_message(LOG_LEVEL_DEBUG, "Start to decompress data");
|
log_debug_message(LOG_DEBUG_UTIL, "Start to decompress data");
|
||||||
unsigned long long dst_size =
|
unsigned long long dst_size =
|
||||||
ZSTD_getFrameContentSize(compressed_data->data, compressed_data->size);
|
ZSTD_getFrameContentSize(compressed_data->data, compressed_data->size);
|
||||||
if (ZSTD_isError(dst_size)) {
|
if (ZSTD_isError(dst_size)) {
|
||||||
@@ -139,7 +183,7 @@ Data* data_decompress_limited(Data* compressed_data, size_t maximum_size) {
|
|||||||
buf_size *= 2;
|
buf_size *= 2;
|
||||||
if (buf_size > hard_limit)
|
if (buf_size > hard_limit)
|
||||||
buf_size = (size_t)hard_limit;
|
buf_size = (size_t)hard_limit;
|
||||||
void* new_data = realloc(uncompressed_data->data, buf_size);
|
void* new_data = protocol_realloc(uncompressed_data->data, buf_size);
|
||||||
if (!new_data) {
|
if (!new_data) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Failed to grow decompression buffer");
|
log_message(LOG_LEVEL_ERROR, "Failed to grow decompression buffer");
|
||||||
ZSTD_freeDCtx(dctx);
|
ZSTD_freeDCtx(dctx);
|
||||||
@@ -155,7 +199,7 @@ Data* data_decompress_limited(Data* compressed_data, size_t maximum_size) {
|
|||||||
uncompressed_data->size = output.pos;
|
uncompressed_data->size = output.pos;
|
||||||
ZSTD_freeDCtx(dctx);
|
ZSTD_freeDCtx(dctx);
|
||||||
|
|
||||||
log_message(LOG_LEVEL_DEBUG, "Decompressed data successfully");
|
log_debug_message(LOG_DEBUG_UTIL, "Decompressed data successfully");
|
||||||
return uncompressed_data;
|
return uncompressed_data;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -4,9 +4,14 @@
|
|||||||
#include "data.h"
|
#include "data.h"
|
||||||
#include <stdbool.h>
|
#include <stdbool.h>
|
||||||
|
|
||||||
|
#define COMPRESSION_MAX_THREADS 64
|
||||||
|
|
||||||
Data* data_compress(Data* data_to_compress, int compression_level);
|
Data* data_compress(Data* data_to_compress, int compression_level);
|
||||||
|
Data* data_compress_with_threads(Data* data_to_compress, int compression_level,
|
||||||
|
int compression_threads);
|
||||||
Data* data_decompress(Data* compressed_data);
|
Data* data_decompress(Data* compressed_data);
|
||||||
Data* data_decompress_limited(Data* compressed_data, size_t maximum_size);
|
Data* data_decompress_limited(Data* compressed_data, size_t maximum_size);
|
||||||
bool compression_should_skip(const char* path);
|
bool compression_should_skip(const char* path);
|
||||||
|
bool compression_should_skip_with_suffixes(const char* path, char* const* suffixes, int count);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
+155
-36
@@ -18,10 +18,14 @@ static void config_set_defaults(Config* config) {
|
|||||||
config->use_chunk_serialization = false;
|
config->use_chunk_serialization = false;
|
||||||
config->use_compression = false;
|
config->use_compression = false;
|
||||||
config->use_metadata = false;
|
config->use_metadata = false;
|
||||||
|
config->use_executability = false;
|
||||||
|
config->metadata_explicitly_disabled = false;
|
||||||
config->show_progress = false;
|
config->show_progress = false;
|
||||||
config->dry_run = false;
|
config->dry_run = false;
|
||||||
|
config->remove_source_files = false;
|
||||||
config->use_delete = false;
|
config->use_delete = false;
|
||||||
config->compression_level = 5;
|
config->compression_level = 5;
|
||||||
|
config->compression_threads = 0;
|
||||||
config->use_sendfile = false;
|
config->use_sendfile = false;
|
||||||
config->chunk_size = DEFAULT_CHUNK_SIZE;
|
config->chunk_size = DEFAULT_CHUNK_SIZE;
|
||||||
config->ssh_port = 22;
|
config->ssh_port = 22;
|
||||||
@@ -34,8 +38,13 @@ static void config_set_defaults(Config* config) {
|
|||||||
config->include_count = 0;
|
config->include_count = 0;
|
||||||
config->max_size = 0;
|
config->max_size = 0;
|
||||||
config->min_size = 0;
|
config->min_size = 0;
|
||||||
|
config->max_alloc = DEFAULT_MAX_ALLOC;
|
||||||
config->use_incremental = false;
|
config->use_incremental = false;
|
||||||
|
config->ignore_times = false;
|
||||||
|
config->size_only = false;
|
||||||
config->use_delta = false;
|
config->use_delta = false;
|
||||||
|
config->whole_file = false;
|
||||||
|
config->modify_window = 0;
|
||||||
config->delta_block_size = DELTA_BLOCK_SIZE_DEFAULT;
|
config->delta_block_size = DELTA_BLOCK_SIZE_DEFAULT;
|
||||||
config->delta_max_file_size = DELTA_MAX_FILE_SIZE;
|
config->delta_max_file_size = DELTA_MAX_FILE_SIZE;
|
||||||
config->use_tls = false;
|
config->use_tls = false;
|
||||||
@@ -65,12 +74,17 @@ static void config_set_defaults(Config* config) {
|
|||||||
config->preserve_sparse = false;
|
config->preserve_sparse = false;
|
||||||
config->itemize_changes = false;
|
config->itemize_changes = false;
|
||||||
config->out_format = NULL;
|
config->out_format = NULL;
|
||||||
|
config->log_file_format = NULL;
|
||||||
config->info_level = 0;
|
config->info_level = 0;
|
||||||
config->debug_level = 0;
|
config->debug_level = 0;
|
||||||
config->list_only = false;
|
config->list_only = false;
|
||||||
config->human_readable = false;
|
config->human_readable = false;
|
||||||
|
config->eight_bit_output = false;
|
||||||
|
config->existing = false;
|
||||||
|
config->ignore_existing = false;
|
||||||
config->update = false;
|
config->update = false;
|
||||||
config->inplace = false;
|
config->inplace = false;
|
||||||
|
config->use_fsync = false;
|
||||||
config->append = false;
|
config->append = false;
|
||||||
config->append_verify = false;
|
config->append_verify = false;
|
||||||
config->delete_excluded = false;
|
config->delete_excluded = false;
|
||||||
@@ -83,6 +97,7 @@ static void config_set_defaults(Config* config) {
|
|||||||
config->relative = false;
|
config->relative = false;
|
||||||
config->rsh_command = NULL;
|
config->rsh_command = NULL;
|
||||||
config->rsync_path = NULL;
|
config->rsync_path = NULL;
|
||||||
|
config->old_args = false;
|
||||||
config->temp_dir = NULL;
|
config->temp_dir = NULL;
|
||||||
config->compare_dest = NULL;
|
config->compare_dest = NULL;
|
||||||
config->copy_dest = NULL;
|
config->copy_dest = NULL;
|
||||||
@@ -100,6 +115,9 @@ static void config_set_defaults(Config* config) {
|
|||||||
config->checksum = false;
|
config->checksum = false;
|
||||||
config->compress_choice = NULL;
|
config->compress_choice = NULL;
|
||||||
config->chmod_spec = NULL;
|
config->chmod_spec = NULL;
|
||||||
|
config->skip_compress_suffixes = NULL;
|
||||||
|
config->skip_compress_count = 0;
|
||||||
|
config->skip_compress_set = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
static bool valid_wire_bool(int value) {
|
static bool valid_wire_bool(int value) {
|
||||||
@@ -118,25 +136,32 @@ static bool validate_received_config(const Config* config) {
|
|||||||
return valid_wire_bool(config->save_to_disk) && valid_wire_bool(config->use_multithreading) &&
|
return valid_wire_bool(config->save_to_disk) && valid_wire_bool(config->use_multithreading) &&
|
||||||
valid_wire_bool(config->use_chunk_serialization) &&
|
valid_wire_bool(config->use_chunk_serialization) &&
|
||||||
valid_wire_bool(config->use_compression) && valid_wire_bool(config->use_metadata) &&
|
valid_wire_bool(config->use_compression) && valid_wire_bool(config->use_metadata) &&
|
||||||
valid_wire_bool(config->use_sendfile) && valid_wire_bool(config->use_delete) &&
|
valid_wire_bool(config->use_executability) && valid_wire_bool(config->use_sendfile) &&
|
||||||
valid_wire_bool(config->use_incremental) && valid_wire_bool(config->use_delta) &&
|
valid_wire_bool(config->use_delete) && valid_wire_bool(config->use_incremental) &&
|
||||||
valid_wire_bool(config->backup) && valid_wire_bool(config->follow_symlinks) &&
|
valid_wire_bool(config->size_only) && valid_wire_bool(config->ignore_times) &&
|
||||||
|
valid_wire_bool(config->use_delta) && valid_wire_bool(config->backup) &&
|
||||||
|
valid_wire_bool(config->remove_source_files) && valid_wire_bool(config->follow_symlinks) &&
|
||||||
valid_wire_bool(config->copy_links) && valid_wire_bool(config->safe_links) &&
|
valid_wire_bool(config->copy_links) && valid_wire_bool(config->safe_links) &&
|
||||||
valid_wire_bool(config->copy_unsafe_links) &&
|
valid_wire_bool(config->copy_unsafe_links) &&
|
||||||
valid_wire_bool(config->preserve_hard_links) && valid_wire_bool(config->preserve_acls) &&
|
valid_wire_bool(config->preserve_hard_links) && valid_wire_bool(config->preserve_acls) &&
|
||||||
valid_wire_bool(config->preserve_xattrs) && valid_wire_bool(config->preserve_devices) &&
|
valid_wire_bool(config->preserve_xattrs) && valid_wire_bool(config->preserve_devices) &&
|
||||||
valid_wire_bool(config->preserve_sparse) && valid_wire_bool(config->update) &&
|
valid_wire_bool(config->preserve_sparse) && valid_wire_bool(config->ignore_existing) &&
|
||||||
|
valid_wire_bool(config->existing) && valid_wire_bool(config->update) &&
|
||||||
valid_wire_bool(config->inplace) && valid_wire_bool(config->append) &&
|
valid_wire_bool(config->inplace) && valid_wire_bool(config->append) &&
|
||||||
valid_wire_bool(config->append_verify) && valid_wire_bool(config->delete_excluded) &&
|
valid_wire_bool(config->use_fsync) && valid_wire_bool(config->append_verify) &&
|
||||||
valid_wire_bool(config->delete_after) && valid_wire_bool(config->relative) &&
|
valid_wire_bool(config->delete_excluded) && valid_wire_bool(config->delete_after) &&
|
||||||
valid_wire_bool(config->prune_empty_dirs) && valid_wire_bool(config->partial) &&
|
valid_wire_bool(config->relative) && valid_wire_bool(config->prune_empty_dirs) &&
|
||||||
valid_wire_bool(config->delete_before) && valid_wire_bool(config->checksum) &&
|
valid_wire_bool(config->partial) && valid_wire_bool(config->delete_before) &&
|
||||||
|
valid_wire_bool(config->checksum) && valid_wire_bool(config->eight_bit_output) &&
|
||||||
|
!(config->skip_compress_set && config->use_chunk_serialization) &&
|
||||||
(!config->use_compression ||
|
(!config->use_compression ||
|
||||||
(config->compression_level >= 1 && config->compression_level <= 22)) &&
|
(config->compression_level >= 1 && config->compression_level <= 22)) &&
|
||||||
config->chunk_size > 0 && config->chunk_size <= MAX_CHUNK_SIZE &&
|
config->chunk_size > 0 && config->chunk_size <= MAX_CHUNK_SIZE &&
|
||||||
config->delta_block_size >= DELTA_BLOCK_SIZE_MIN &&
|
config->delta_block_size >= DELTA_BLOCK_SIZE_MIN &&
|
||||||
config->delta_block_size <= DELTA_BLOCK_SIZE_MAX &&
|
config->delta_block_size <= DELTA_BLOCK_SIZE_MAX &&
|
||||||
config->delta_max_file_size <= DELTA_MAX_FILE_SIZE && config->max_delete >= 0 &&
|
config->delta_max_file_size <= DELTA_MAX_FILE_SIZE && config->modify_window >= 0 &&
|
||||||
|
config->max_delete >= 0 && config->skip_compress_count >= 0 &&
|
||||||
|
config->skip_compress_count <= 10000 && config->max_alloc > 0 &&
|
||||||
(!config->chmod_spec || !*config->chmod_spec ||
|
(!config->chmod_spec || !*config->chmod_spec ||
|
||||||
chmod_apply(0, config->chmod_spec, &(mode_t){0}));
|
chmod_apply(0, config->chmod_spec, &(mode_t){0}));
|
||||||
}
|
}
|
||||||
@@ -199,6 +224,7 @@ void config_delete(Config* config) {
|
|||||||
free(config->backup_dir);
|
free(config->backup_dir);
|
||||||
free(config->server_host);
|
free(config->server_host);
|
||||||
free(config->out_format);
|
free(config->out_format);
|
||||||
|
free(config->log_file_format);
|
||||||
free(config->files_from);
|
free(config->files_from);
|
||||||
free(config->rsh_command);
|
free(config->rsh_command);
|
||||||
free(config->rsync_path);
|
free(config->rsync_path);
|
||||||
@@ -213,6 +239,11 @@ void config_delete(Config* config) {
|
|||||||
free(config->daemon_config);
|
free(config->daemon_config);
|
||||||
free(config->compress_choice);
|
free(config->compress_choice);
|
||||||
free(config->chmod_spec);
|
free(config->chmod_spec);
|
||||||
|
if (config->skip_compress_suffixes) {
|
||||||
|
for (int i = 0; i < config->skip_compress_count; i++)
|
||||||
|
free(config->skip_compress_suffixes[i]);
|
||||||
|
free(config->skip_compress_suffixes);
|
||||||
|
}
|
||||||
if (config->filters) {
|
if (config->filters) {
|
||||||
array_list_delete(config->filters);
|
array_list_delete(config->filters);
|
||||||
}
|
}
|
||||||
@@ -223,54 +254,81 @@ void config_delete(Config* config) {
|
|||||||
* helper call order in config_send and config_receive unchanged when adding
|
* helper call order in config_send and config_receive unchanged when adding
|
||||||
* fields. */
|
* fields. */
|
||||||
static bool send_core_fields(int fd, const Config* c) {
|
static bool send_core_fields(int fd, const Config* c) {
|
||||||
return send_str(fd, c->version) && send_str(fd, c->send_directory) &&
|
if (!send_str(fd, c->version) || !send_int(fd, c->eight_bit_output))
|
||||||
send_str(fd, c->receive_root_directory) && send_int(fd, c->save_to_disk) &&
|
return false;
|
||||||
send_int(fd, c->use_multithreading) && send_int(fd, c->use_chunk_serialization) &&
|
protocol_set_8_bit_output(c->eight_bit_output);
|
||||||
send_int(fd, c->use_compression) && send_int(fd, c->use_metadata) &&
|
if (!send_n_data(fd, &c->max_alloc, sizeof(c->max_alloc)))
|
||||||
|
return false;
|
||||||
|
return send_str(fd, c->send_directory) && send_str(fd, c->receive_root_directory) &&
|
||||||
|
send_int(fd, c->save_to_disk) && send_int(fd, c->use_multithreading) &&
|
||||||
|
send_int(fd, c->use_chunk_serialization) && send_int(fd, c->use_compression) &&
|
||||||
|
send_int(fd, c->use_metadata) && send_int(fd, c->use_executability) &&
|
||||||
send_int(fd, c->compression_level) &&
|
send_int(fd, c->compression_level) &&
|
||||||
send_n_data(fd, &c->chunk_size, sizeof(c->chunk_size)) && send_int(fd, c->use_sendfile);
|
send_n_data(fd, &c->chunk_size, sizeof(c->chunk_size)) && send_int(fd, c->use_sendfile);
|
||||||
}
|
}
|
||||||
|
|
||||||
static bool send_delta_fields(int fd, const Config* c) {
|
static bool send_delta_fields(int fd, const Config* c) {
|
||||||
return send_int(fd, c->use_delete) && send_int(fd, c->use_incremental) &&
|
return send_int(fd, c->use_delete) && send_int(fd, c->use_incremental) &&
|
||||||
send_int(fd, c->use_delta) &&
|
send_int(fd, c->size_only) && send_int(fd, c->ignore_times) &&
|
||||||
|
send_int(fd, c->use_delta && !c->whole_file) &&
|
||||||
send_n_data(fd, &c->delta_block_size, sizeof(c->delta_block_size)) &&
|
send_n_data(fd, &c->delta_block_size, sizeof(c->delta_block_size)) &&
|
||||||
send_n_data(fd, &c->delta_max_file_size, sizeof(unsigned long long));
|
send_n_data(fd, &c->delta_max_file_size, sizeof(unsigned long long));
|
||||||
}
|
}
|
||||||
|
|
||||||
static bool send_file_options(int fd, const Config* c) {
|
static bool send_file_options(int fd, const Config* c) {
|
||||||
return send_int(fd, c->backup) && send_str(fd, c->backup_dir ? c->backup_dir : "") &&
|
return send_int(fd, c->backup) && send_str(fd, c->backup_dir ? c->backup_dir : "") &&
|
||||||
send_int(fd, c->follow_symlinks) && send_int(fd, c->copy_links) &&
|
send_int(fd, c->remove_source_files) && send_int(fd, c->follow_symlinks) &&
|
||||||
send_int(fd, c->safe_links) && send_int(fd, c->copy_unsafe_links) &&
|
send_int(fd, c->copy_links) && send_int(fd, c->safe_links) &&
|
||||||
send_int(fd, c->preserve_hard_links) && send_int(fd, c->preserve_acls) &&
|
send_int(fd, c->copy_unsafe_links) && send_int(fd, c->preserve_hard_links) &&
|
||||||
send_int(fd, c->preserve_xattrs) && send_int(fd, c->preserve_devices) &&
|
send_int(fd, c->preserve_acls) && send_int(fd, c->preserve_xattrs) &&
|
||||||
send_int(fd, c->preserve_sparse);
|
send_int(fd, c->preserve_devices) && send_int(fd, c->preserve_sparse);
|
||||||
}
|
}
|
||||||
|
|
||||||
static bool send_selection_options(int fd, const Config* c) {
|
static bool send_selection_options(int fd, const Config* c) {
|
||||||
return send_int(fd, c->update) && send_int(fd, c->inplace) && send_int(fd, c->append) &&
|
return send_int(fd, c->ignore_existing) && send_int(fd, c->existing) && send_int(fd, c->update) &&
|
||||||
|
send_int(fd, c->inplace) && send_int(fd, c->append) && send_int(fd, c->use_fsync) &&
|
||||||
send_int(fd, c->append_verify) && send_int(fd, c->delete_excluded) &&
|
send_int(fd, c->append_verify) && send_int(fd, c->delete_excluded) &&
|
||||||
send_int(fd, c->delete_after) && send_n_data(fd, &c->max_delete, sizeof(c->max_delete)) &&
|
send_int(fd, c->delete_after) && send_n_data(fd, &c->max_delete, sizeof(c->max_delete)) &&
|
||||||
send_int(fd, c->relative) && send_int(fd, c->prune_empty_dirs);
|
send_int(fd, c->relative) && send_int(fd, c->prune_empty_dirs);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static bool send_skip_compress_options(int fd, const Config* c) {
|
||||||
|
if (!send_int(fd, c->skip_compress_set) || !send_int(fd, c->skip_compress_count))
|
||||||
|
return false;
|
||||||
|
for (int i = 0; i < c->skip_compress_count; i++) {
|
||||||
|
if (!send_str(fd, c->skip_compress_suffixes[i]))
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
static bool send_resume_options(int fd, const Config* c) {
|
static bool send_resume_options(int fd, const Config* c) {
|
||||||
return send_str(fd, c->temp_dir ? c->temp_dir : "") && send_int(fd, c->partial) &&
|
return send_str(fd, c->temp_dir ? c->temp_dir : "") && send_int(fd, c->partial) &&
|
||||||
send_str(fd, c->partial_dir ? c->partial_dir : "") &&
|
send_str(fd, c->partial_dir ? c->partial_dir : "") &&
|
||||||
send_str(fd, c->suffix ? c->suffix : "") && send_int(fd, c->delete_before) &&
|
send_str(fd, c->suffix ? c->suffix : "") && send_int(fd, c->delete_before) &&
|
||||||
send_int(fd, c->checksum) && send_str(fd, c->compress_choice ? c->compress_choice : "") &&
|
send_int(fd, c->checksum) && send_int(fd, c->modify_window) &&
|
||||||
send_str(fd, c->chmod_spec ? c->chmod_spec : "");
|
send_str(fd, c->compress_choice ? c->compress_choice : "") &&
|
||||||
|
send_str(fd, c->chmod_spec ? c->chmod_spec : "") && send_skip_compress_options(fd, c);
|
||||||
}
|
}
|
||||||
|
|
||||||
static bool receive_core_fields(int fd, Config* c) {
|
static bool receive_core_fields(int fd, Config* c) {
|
||||||
int value;
|
int value;
|
||||||
|
if (!receive_wire_bool(fd, &c->eight_bit_output))
|
||||||
|
return false;
|
||||||
|
protocol_set_8_bit_output(c->eight_bit_output);
|
||||||
|
if (!receive_n_data(fd, &c->max_alloc, sizeof(c->max_alloc)) || c->max_alloc == 0)
|
||||||
|
return false;
|
||||||
|
if (c->max_alloc > MAX_SERVER_ALLOC)
|
||||||
|
c->max_alloc = MAX_SERVER_ALLOC;
|
||||||
|
protocol_session_set_max_alloc(NULL, c->max_alloc);
|
||||||
c->send_directory = receive_str(fd);
|
c->send_directory = receive_str(fd);
|
||||||
c->receive_root_directory = receive_str(fd);
|
c->receive_root_directory = receive_str(fd);
|
||||||
if (!c->send_directory || !c->receive_root_directory)
|
if (!c->send_directory || !c->receive_root_directory)
|
||||||
return false;
|
return false;
|
||||||
if (!receive_wire_bool(fd, &c->save_to_disk) || !receive_wire_bool(fd, &c->use_multithreading) ||
|
if (!receive_wire_bool(fd, &c->save_to_disk) || !receive_wire_bool(fd, &c->use_multithreading) ||
|
||||||
!receive_wire_bool(fd, &c->use_chunk_serialization) ||
|
!receive_wire_bool(fd, &c->use_chunk_serialization) ||
|
||||||
!receive_wire_bool(fd, &c->use_compression) || !receive_wire_bool(fd, &c->use_metadata))
|
!receive_wire_bool(fd, &c->use_compression) || !receive_wire_bool(fd, &c->use_metadata) ||
|
||||||
|
!receive_wire_bool(fd, &c->use_executability))
|
||||||
return false;
|
return false;
|
||||||
if (!receive_int(fd, &value))
|
if (!receive_int(fd, &value))
|
||||||
return false;
|
return false;
|
||||||
@@ -287,6 +345,10 @@ static bool receive_delta_fields(int fd, Config* c) {
|
|||||||
return false;
|
return false;
|
||||||
if (!receive_wire_bool(fd, &c->use_incremental))
|
if (!receive_wire_bool(fd, &c->use_incremental))
|
||||||
return false;
|
return false;
|
||||||
|
if (!receive_wire_bool(fd, &c->size_only))
|
||||||
|
return false;
|
||||||
|
if (!receive_wire_bool(fd, &c->ignore_times))
|
||||||
|
return false;
|
||||||
if (!receive_wire_bool(fd, &c->use_delta))
|
if (!receive_wire_bool(fd, &c->use_delta))
|
||||||
return false;
|
return false;
|
||||||
return receive_n_data(fd, &c->delta_block_size, sizeof(c->delta_block_size)) &&
|
return receive_n_data(fd, &c->delta_block_size, sizeof(c->delta_block_size)) &&
|
||||||
@@ -296,8 +358,17 @@ static bool receive_delta_fields(int fd, Config* c) {
|
|||||||
static bool receive_file_options(int fd, Config* c) {
|
static bool receive_file_options(int fd, Config* c) {
|
||||||
if (!receive_wire_bool(fd, &c->backup))
|
if (!receive_wire_bool(fd, &c->backup))
|
||||||
return false;
|
return false;
|
||||||
c->backup_dir = receive_str(fd);
|
char* backup_dir = receive_str(fd);
|
||||||
if (!c->backup_dir)
|
if (!backup_dir)
|
||||||
|
return false;
|
||||||
|
if (*backup_dir != '\0') {
|
||||||
|
c->backup_dir = backup_dir;
|
||||||
|
} else {
|
||||||
|
/* The sender serializes an unset (NULL) string as "", so canonicalize the
|
||||||
|
empty wire value back to NULL to preserve NULL-vs-empty semantics. */
|
||||||
|
free(backup_dir);
|
||||||
|
}
|
||||||
|
if (!receive_wire_bool(fd, &c->remove_source_files))
|
||||||
return false;
|
return false;
|
||||||
bool* flags[] = {&c->follow_symlinks, &c->copy_links, &c->safe_links,
|
bool* flags[] = {&c->follow_symlinks, &c->copy_links, &c->safe_links,
|
||||||
&c->copy_unsafe_links, &c->preserve_hard_links, &c->preserve_acls,
|
&c->copy_unsafe_links, &c->preserve_hard_links, &c->preserve_acls,
|
||||||
@@ -310,8 +381,9 @@ static bool receive_file_options(int fd, Config* c) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
static bool receive_selection_options(int fd, Config* c) {
|
static bool receive_selection_options(int fd, Config* c) {
|
||||||
bool* flags[] = {&c->update, &c->inplace, &c->append,
|
bool* flags[] = {&c->ignore_existing, &c->existing, &c->update,
|
||||||
&c->append_verify, &c->delete_excluded, &c->delete_after};
|
&c->inplace, &c->append, &c->use_fsync,
|
||||||
|
&c->append_verify, &c->delete_excluded, &c->delete_after};
|
||||||
for (size_t i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
|
for (size_t i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) {
|
||||||
if (!receive_wire_bool(fd, flags[i]))
|
if (!receive_wire_bool(fd, flags[i]))
|
||||||
return false;
|
return false;
|
||||||
@@ -326,23 +398,65 @@ static bool receive_selection_options(int fd, Config* c) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
static bool receive_resume_options(int fd, Config* c) {
|
static bool receive_resume_options(int fd, Config* c) {
|
||||||
c->temp_dir = receive_str(fd);
|
char* temp_dir = receive_str(fd);
|
||||||
if (!c->temp_dir || !receive_wire_bool(fd, &c->partial))
|
if (!temp_dir)
|
||||||
return false;
|
return false;
|
||||||
c->partial_dir = receive_str(fd);
|
if (*temp_dir != '\0') {
|
||||||
c->suffix = c->partial_dir ? receive_str(fd) : NULL;
|
c->temp_dir = temp_dir;
|
||||||
if (!c->partial_dir || !c->suffix || !receive_wire_bool(fd, &c->delete_before))
|
} else {
|
||||||
|
free(temp_dir);
|
||||||
|
}
|
||||||
|
if (!receive_wire_bool(fd, &c->partial))
|
||||||
|
return false;
|
||||||
|
/* These options have NULL client defaults, so the sender transmits an empty
|
||||||
|
string for "unset". Canonicalize the empty wire value back to NULL so
|
||||||
|
receivers observe exactly what the client configured (plain --backup, for
|
||||||
|
example, must not look like --backup-dir ""). */
|
||||||
|
char* partial_dir = receive_str(fd);
|
||||||
|
if (!partial_dir)
|
||||||
|
return false;
|
||||||
|
if (*partial_dir != '\0') {
|
||||||
|
c->partial_dir = partial_dir;
|
||||||
|
} else {
|
||||||
|
free(partial_dir);
|
||||||
|
}
|
||||||
|
char* suffix = receive_str(fd);
|
||||||
|
if (!suffix)
|
||||||
|
return false;
|
||||||
|
if (*suffix != '\0') {
|
||||||
|
c->suffix = suffix;
|
||||||
|
} else {
|
||||||
|
free(suffix);
|
||||||
|
}
|
||||||
|
if (!receive_wire_bool(fd, &c->delete_before))
|
||||||
return false;
|
return false;
|
||||||
if (!receive_wire_bool(fd, &c->checksum))
|
if (!receive_wire_bool(fd, &c->checksum))
|
||||||
return false;
|
return false;
|
||||||
|
if (!receive_n_data(fd, &c->modify_window, sizeof(c->modify_window)))
|
||||||
|
return false;
|
||||||
c->compress_choice = receive_str(fd);
|
c->compress_choice = receive_str(fd);
|
||||||
if (!c->compress_choice)
|
if (!c->compress_choice)
|
||||||
return false;
|
return false;
|
||||||
c->chmod_spec = receive_str(fd);
|
c->chmod_spec = receive_str(fd);
|
||||||
return c->chmod_spec != NULL;
|
if (!c->chmod_spec || !receive_wire_bool(fd, &c->skip_compress_set) ||
|
||||||
|
!receive_int(fd, &c->skip_compress_count) || c->skip_compress_count < 0 ||
|
||||||
|
c->skip_compress_count > 10000)
|
||||||
|
return false;
|
||||||
|
if (c->skip_compress_count > 0) {
|
||||||
|
c->skip_compress_suffixes = calloc((size_t)c->skip_compress_count, sizeof(char*));
|
||||||
|
if (!c->skip_compress_suffixes)
|
||||||
|
return false;
|
||||||
|
for (int i = 0; i < c->skip_compress_count; i++) {
|
||||||
|
c->skip_compress_suffixes[i] = receive_str(fd);
|
||||||
|
if (!c->skip_compress_suffixes[i])
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool config_send(int file_descriptor, const Config* config) {
|
bool config_send(int file_descriptor, const Config* config) {
|
||||||
|
protocol_session_set_max_alloc(NULL, config->max_alloc);
|
||||||
if (!send_core_fields(file_descriptor, config) || !send_delta_fields(file_descriptor, config) ||
|
if (!send_core_fields(file_descriptor, config) || !send_delta_fields(file_descriptor, config) ||
|
||||||
!send_file_options(file_descriptor, config) ||
|
!send_file_options(file_descriptor, config) ||
|
||||||
!send_selection_options(file_descriptor, config) ||
|
!send_selection_options(file_descriptor, config) ||
|
||||||
@@ -367,8 +481,10 @@ Config* config_receive(int file_descriptor) {
|
|||||||
if (!config->version)
|
if (!config->version)
|
||||||
goto error;
|
goto error;
|
||||||
if (strcmp(config->version, PROTOCOL_VERSION) != 0) {
|
if (strcmp(config->version, PROTOCOL_VERSION) != 0) {
|
||||||
fprintf(stderr, "Protocol version mismatch: client=%s, server=%s\n", config->version,
|
char* escaped_version = output_escape(config->version, false);
|
||||||
PROTOCOL_VERSION);
|
fprintf(stderr, "Protocol version mismatch: client=%s, server=%s\n",
|
||||||
|
escaped_version ? escaped_version : "<allocation failed>", PROTOCOL_VERSION);
|
||||||
|
free(escaped_version);
|
||||||
send_status(file_descriptor, STATUS_ERROR);
|
send_status(file_descriptor, STATUS_ERROR);
|
||||||
goto error;
|
goto error;
|
||||||
}
|
}
|
||||||
@@ -380,7 +496,10 @@ Config* config_receive(int file_descriptor) {
|
|||||||
goto error;
|
goto error;
|
||||||
if (config->compress_choice[0] != '\0' && strcmp(config->compress_choice, "zstd") != 0 &&
|
if (config->compress_choice[0] != '\0' && strcmp(config->compress_choice, "zstd") != 0 &&
|
||||||
strcmp(config->compress_choice, "none") != 0) {
|
strcmp(config->compress_choice, "none") != 0) {
|
||||||
fprintf(stderr, "Unsupported compression choice: %s\n", config->compress_choice);
|
char* escaped_choice = output_escape(config->compress_choice, config->eight_bit_output);
|
||||||
|
fprintf(stderr, "Unsupported compression choice: %s\n",
|
||||||
|
escaped_choice ? escaped_choice : "<allocation failed>");
|
||||||
|
free(escaped_choice);
|
||||||
send_status(file_descriptor, STATUS_ERROR);
|
send_status(file_descriptor, STATUS_ERROR);
|
||||||
goto error;
|
goto error;
|
||||||
}
|
}
|
||||||
|
|||||||
+20
-1
@@ -18,10 +18,14 @@ typedef struct Config {
|
|||||||
bool use_compression;
|
bool use_compression;
|
||||||
bool use_sendfile;
|
bool use_sendfile;
|
||||||
bool use_metadata;
|
bool use_metadata;
|
||||||
|
bool use_executability;
|
||||||
|
bool metadata_explicitly_disabled;
|
||||||
bool show_progress;
|
bool show_progress;
|
||||||
bool dry_run;
|
bool dry_run;
|
||||||
|
bool remove_source_files;
|
||||||
bool use_delete;
|
bool use_delete;
|
||||||
int compression_level;
|
int compression_level;
|
||||||
|
int compression_threads;
|
||||||
unsigned long long chunk_size;
|
unsigned long long chunk_size;
|
||||||
int ssh_port;
|
int ssh_port;
|
||||||
TransportType transport;
|
TransportType transport;
|
||||||
@@ -33,8 +37,13 @@ typedef struct Config {
|
|||||||
int include_count;
|
int include_count;
|
||||||
unsigned long long max_size;
|
unsigned long long max_size;
|
||||||
unsigned long long min_size;
|
unsigned long long min_size;
|
||||||
|
unsigned long long max_alloc;
|
||||||
bool use_incremental;
|
bool use_incremental;
|
||||||
|
bool ignore_times;
|
||||||
|
bool size_only;
|
||||||
bool use_delta;
|
bool use_delta;
|
||||||
|
bool whole_file;
|
||||||
|
int modify_window;
|
||||||
uint32_t delta_block_size;
|
uint32_t delta_block_size;
|
||||||
unsigned long long delta_max_file_size;
|
unsigned long long delta_max_file_size;
|
||||||
bool use_tls;
|
bool use_tls;
|
||||||
@@ -70,14 +79,19 @@ typedef struct Config {
|
|||||||
// Issue #122: Output/logging options
|
// Issue #122: Output/logging options
|
||||||
bool itemize_changes;
|
bool itemize_changes;
|
||||||
char* out_format;
|
char* out_format;
|
||||||
|
char* log_file_format;
|
||||||
int info_level;
|
int info_level;
|
||||||
int debug_level;
|
int debug_level;
|
||||||
bool list_only;
|
bool list_only;
|
||||||
bool human_readable;
|
bool human_readable;
|
||||||
|
bool eight_bit_output;
|
||||||
|
|
||||||
// Issue #127: Transfer modes
|
// Issue #127: Transfer modes
|
||||||
|
bool existing;
|
||||||
|
bool ignore_existing;
|
||||||
bool update;
|
bool update;
|
||||||
bool inplace;
|
bool inplace;
|
||||||
|
bool use_fsync;
|
||||||
bool append;
|
bool append;
|
||||||
bool append_verify;
|
bool append_verify;
|
||||||
|
|
||||||
@@ -96,6 +110,7 @@ typedef struct Config {
|
|||||||
// Issue #130: Remote shell/connection options
|
// Issue #130: Remote shell/connection options
|
||||||
char* rsh_command;
|
char* rsh_command;
|
||||||
char* rsync_path;
|
char* rsync_path;
|
||||||
|
bool old_args;
|
||||||
char* temp_dir;
|
char* temp_dir;
|
||||||
char* compare_dest;
|
char* compare_dest;
|
||||||
char* copy_dest;
|
char* copy_dest;
|
||||||
@@ -127,9 +142,13 @@ typedef struct Config {
|
|||||||
// PR #184: Compression algorithm negotiation
|
// PR #184: Compression algorithm negotiation
|
||||||
char* compress_choice;
|
char* compress_choice;
|
||||||
char* chmod_spec;
|
char* chmod_spec;
|
||||||
|
|
||||||
|
char** skip_compress_suffixes;
|
||||||
|
int skip_compress_count;
|
||||||
|
bool skip_compress_set;
|
||||||
} Config;
|
} Config;
|
||||||
|
|
||||||
#define PROTOCOL_VERSION "2.3.0"
|
#define PROTOCOL_VERSION "2.5.0"
|
||||||
#define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024)
|
#define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024)
|
||||||
|
|
||||||
Config* config_create(void);
|
Config* config_create(void);
|
||||||
|
|||||||
+4
-3
@@ -1,11 +1,12 @@
|
|||||||
#include "data.h"
|
#include "data.h"
|
||||||
#include "log.h"
|
#include "log.h"
|
||||||
|
#include "protocol.h"
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
|
|
||||||
Data* data_create_empty(size_t data_size) {
|
Data* data_create_empty(size_t data_size) {
|
||||||
/* malloc(0) is UB; allocate at least 1 byte but preserve requested size */
|
/* malloc(0) is UB; allocate at least 1 byte but preserve requested size */
|
||||||
size_t alloc_size = data_size > 0 ? data_size : 1;
|
size_t alloc_size = data_size > 0 ? data_size : 1;
|
||||||
void* data = malloc(alloc_size);
|
void* data = protocol_alloc(alloc_size);
|
||||||
if (data == NULL) {
|
if (data == NULL) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Could not allocate memory for empty data");
|
log_message(LOG_LEVEL_ERROR, "Could not allocate memory for empty data");
|
||||||
return NULL;
|
return NULL;
|
||||||
@@ -14,7 +15,7 @@ Data* data_create_empty(size_t data_size) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
Data* data_create_reserve(size_t size) {
|
Data* data_create_reserve(size_t size) {
|
||||||
Data* d = malloc(sizeof(Data));
|
Data* d = protocol_alloc(sizeof(Data));
|
||||||
if (d == NULL) {
|
if (d == NULL) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Could not allocate memory for data");
|
log_message(LOG_LEVEL_ERROR, "Could not allocate memory for data");
|
||||||
return NULL;
|
return NULL;
|
||||||
@@ -26,7 +27,7 @@ Data* data_create_reserve(size_t size) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
Data* data_create(void* data, size_t data_size) {
|
Data* data_create(void* data, size_t data_size) {
|
||||||
Data* new_data = malloc(sizeof(Data));
|
Data* new_data = protocol_alloc(sizeof(Data));
|
||||||
if (new_data == NULL) {
|
if (new_data == NULL) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Could not allocate memory for data");
|
log_message(LOG_LEVEL_ERROR, "Could not allocate memory for data");
|
||||||
free(data);
|
free(data);
|
||||||
|
|||||||
+176
-42
@@ -1,5 +1,6 @@
|
|||||||
#include "delta.h"
|
#include "delta.h"
|
||||||
#include "log.h"
|
#include "log.h"
|
||||||
|
#include "protocol.h"
|
||||||
#include <stdint.h>
|
#include <stdint.h>
|
||||||
#include <limits.h>
|
#include <limits.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
@@ -43,7 +44,7 @@ DeltaSignature* delta_signature_create(const void* old_file_data, uint64_t old_f
|
|||||||
|
|
||||||
uint32_t block_count = (uint32_t)((old_file_size + block_size - 1) / block_size);
|
uint32_t block_count = (uint32_t)((old_file_size + block_size - 1) / block_size);
|
||||||
|
|
||||||
DeltaSignature* sig = malloc(sizeof(DeltaSignature));
|
DeltaSignature* sig = protocol_alloc(sizeof(DeltaSignature));
|
||||||
if (!sig)
|
if (!sig)
|
||||||
return NULL;
|
return NULL;
|
||||||
|
|
||||||
@@ -54,7 +55,7 @@ DeltaSignature* delta_signature_create(const void* old_file_data, uint64_t old_f
|
|||||||
free(sig);
|
free(sig);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
sig->blocks = malloc((size_t)block_count * sizeof(DeltaBlockSig));
|
sig->blocks = protocol_alloc((size_t)block_count * sizeof(DeltaBlockSig));
|
||||||
if (!sig->blocks) {
|
if (!sig->blocks) {
|
||||||
free(sig);
|
free(sig);
|
||||||
return NULL;
|
return NULL;
|
||||||
@@ -82,7 +83,7 @@ Data* delta_signature_serialize(const DeltaSignature* sig) {
|
|||||||
total > SIZE_MAX)
|
total > SIZE_MAX)
|
||||||
return NULL;
|
return NULL;
|
||||||
|
|
||||||
uint8_t* buf = malloc((size_t)total);
|
uint8_t* buf = protocol_alloc((size_t)total);
|
||||||
if (!buf)
|
if (!buf)
|
||||||
return NULL;
|
return NULL;
|
||||||
|
|
||||||
@@ -111,7 +112,7 @@ DeltaSignature* delta_signature_deserialize(const Data* data) {
|
|||||||
const uint8_t* buf = (const uint8_t*)data->data;
|
const uint8_t* buf = (const uint8_t*)data->data;
|
||||||
size_t pos = 0;
|
size_t pos = 0;
|
||||||
|
|
||||||
DeltaSignature* sig = malloc(sizeof(DeltaSignature));
|
DeltaSignature* sig = protocol_alloc(sizeof(DeltaSignature));
|
||||||
if (!sig)
|
if (!sig)
|
||||||
return NULL;
|
return NULL;
|
||||||
|
|
||||||
@@ -149,7 +150,7 @@ DeltaSignature* delta_signature_deserialize(const Data* data) {
|
|||||||
free(sig);
|
free(sig);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
sig->blocks = malloc((size_t)blocks_size);
|
sig->blocks = protocol_alloc((size_t)blocks_size);
|
||||||
if (!sig->blocks) {
|
if (!sig->blocks) {
|
||||||
free(sig);
|
free(sig);
|
||||||
return NULL;
|
return NULL;
|
||||||
@@ -178,7 +179,7 @@ static bool ensure_capacity(DeltaInstruction** instrs, uint32_t* capacity, uint3
|
|||||||
if (*capacity > MAX_DELTA_INSTRUCTIONS / 2)
|
if (*capacity > MAX_DELTA_INSTRUCTIONS / 2)
|
||||||
return false;
|
return false;
|
||||||
uint32_t new_cap = *capacity * 2;
|
uint32_t new_cap = *capacity * 2;
|
||||||
DeltaInstruction* tmp = realloc(*instrs, (size_t)new_cap * sizeof(DeltaInstruction));
|
DeltaInstruction* tmp = protocol_realloc(*instrs, (size_t)new_cap * sizeof(DeltaInstruction));
|
||||||
if (!tmp)
|
if (!tmp)
|
||||||
return false;
|
return false;
|
||||||
*instrs = tmp;
|
*instrs = tmp;
|
||||||
@@ -195,7 +196,7 @@ static bool flush_literal(DeltaInstruction** instrs, uint32_t* capacity, uint32_
|
|||||||
uint32_t lit_len = (uint32_t)(end - start);
|
uint32_t lit_len = (uint32_t)(end - start);
|
||||||
if (!ensure_capacity(instrs, capacity, *count))
|
if (!ensure_capacity(instrs, capacity, *count))
|
||||||
return false;
|
return false;
|
||||||
uint8_t* lit_data = malloc(lit_len);
|
uint8_t* lit_data = protocol_alloc(lit_len);
|
||||||
if (!lit_data)
|
if (!lit_data)
|
||||||
return false;
|
return false;
|
||||||
memcpy(lit_data, data + start, lit_len);
|
memcpy(lit_data, data + start, lit_len);
|
||||||
@@ -215,6 +216,119 @@ static void free_instructions(DeltaInstruction* instrs, uint32_t count) {
|
|||||||
free(instrs);
|
free(instrs);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Sentinel meaning "no signature block" in the lookup index chains. Block
|
||||||
|
* counts are bounded well below UINT32_MAX, so it doubles as a null link. */
|
||||||
|
#define DELTA_NO_BLOCK UINT32_MAX
|
||||||
|
|
||||||
|
/* Avalanche mix for the rolling checksum so blocks do not cluster in the
|
||||||
|
* bucket table when the weak checksum has little entropy (e.g. all-zero or
|
||||||
|
* patterned files). */
|
||||||
|
static uint32_t delta_adler_mix(uint32_t h) {
|
||||||
|
h ^= h >> 16;
|
||||||
|
h *= 0x7feb352dU;
|
||||||
|
h ^= h >> 15;
|
||||||
|
h *= 0x846ca68bU;
|
||||||
|
h ^= h >> 16;
|
||||||
|
return h;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Smallest power of two >= v. v must be non-zero. */
|
||||||
|
static uint32_t delta_next_pow2(uint32_t v) {
|
||||||
|
v--;
|
||||||
|
v |= v >> 1;
|
||||||
|
v |= v >> 2;
|
||||||
|
v |= v >> 4;
|
||||||
|
v |= v >> 8;
|
||||||
|
v |= v >> 16;
|
||||||
|
return v + 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Build a hash index over sig->blocks keyed by the (mixed) rolling checksum.
|
||||||
|
* All blocks sharing an Adler-32 value land in the same bucket; collisions
|
||||||
|
* are chained through a single contiguous allocation:
|
||||||
|
*
|
||||||
|
* [0, bucket_count) heads (first block per bucket)
|
||||||
|
* [bucket_count, 2*bucket_count) tails (last block per bucket)
|
||||||
|
* [2*bucket_count, ...) per-block chain links
|
||||||
|
*
|
||||||
|
* Blocks are inserted in ascending index order so every bucket chain is
|
||||||
|
* ordered exactly like the historical linear scan. Returns the base pointer
|
||||||
|
* (also the heads array) or NULL when no index could be allocated; callers
|
||||||
|
* then fall back to the linear scan. */
|
||||||
|
static uint32_t* delta_build_index(const DeltaSignature* sig, uint32_t bucket_count) {
|
||||||
|
if (sig->block_count == 0 || bucket_count == 0)
|
||||||
|
return NULL;
|
||||||
|
|
||||||
|
size_t entries = (size_t)2 * bucket_count + sig->block_count;
|
||||||
|
if (entries > SIZE_MAX / sizeof(uint32_t))
|
||||||
|
return NULL;
|
||||||
|
|
||||||
|
uint32_t* index = protocol_alloc(entries * sizeof(uint32_t));
|
||||||
|
if (!index)
|
||||||
|
return NULL;
|
||||||
|
|
||||||
|
uint32_t* heads = index;
|
||||||
|
uint32_t* tails = index + bucket_count;
|
||||||
|
uint32_t* next = index + 2 * bucket_count;
|
||||||
|
uint32_t mask = bucket_count - 1;
|
||||||
|
|
||||||
|
memset(heads, 0xFF, (size_t)bucket_count * sizeof(uint32_t));
|
||||||
|
memset(tails, 0xFF, (size_t)bucket_count * sizeof(uint32_t));
|
||||||
|
|
||||||
|
for (uint32_t j = 0; j < sig->block_count; j++) {
|
||||||
|
uint32_t b = delta_adler_mix(sig->blocks[j].adler32) & mask;
|
||||||
|
if (heads[b] == DELTA_NO_BLOCK)
|
||||||
|
heads[b] = j;
|
||||||
|
else
|
||||||
|
next[tails[b]] = j;
|
||||||
|
tails[b] = j;
|
||||||
|
next[j] = DELTA_NO_BLOCK;
|
||||||
|
}
|
||||||
|
return index;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Locate the signature block matching the byte window at new_data[i].
|
||||||
|
*
|
||||||
|
* Mirrors the original per-window behaviour exactly: only a full block_size
|
||||||
|
* window can match, candidates are accepted only when the weak (Adler-32) and
|
||||||
|
* strong (xxHash32) checksums both agree, and the lowest block index wins so
|
||||||
|
* the emitted op stream is byte-identical to the linear scan. When heads is
|
||||||
|
* non-NULL the candidate set is reached through the bucket index (expected
|
||||||
|
* O(1) per window); otherwise an exact linear scan is used. */
|
||||||
|
static uint32_t delta_find_match(const uint8_t* window, uint32_t window_len, uint32_t adler,
|
||||||
|
bool full_window, const DeltaSignature* sig, const uint32_t* heads,
|
||||||
|
const uint32_t* next, uint32_t mask) {
|
||||||
|
if (!full_window || sig->block_count == 0)
|
||||||
|
return DELTA_NO_BLOCK;
|
||||||
|
|
||||||
|
if (heads) {
|
||||||
|
uint32_t b = delta_adler_mix(adler) & mask;
|
||||||
|
uint32_t window_xxh = 0;
|
||||||
|
bool have_xxh = false;
|
||||||
|
for (uint32_t j = heads[b]; j != DELTA_NO_BLOCK; j = next[j]) {
|
||||||
|
if (sig->blocks[j].adler32 != adler)
|
||||||
|
continue;
|
||||||
|
if (!have_xxh) {
|
||||||
|
window_xxh = delta_xxhash32(window, window_len);
|
||||||
|
have_xxh = true;
|
||||||
|
}
|
||||||
|
if (window_xxh == sig->blocks[j].xxhash)
|
||||||
|
return j;
|
||||||
|
}
|
||||||
|
return DELTA_NO_BLOCK;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Fallback used when the index could not be allocated. */
|
||||||
|
for (uint32_t j = 0; j < sig->block_count; j++) {
|
||||||
|
if (sig->blocks[j].adler32 == adler) {
|
||||||
|
uint32_t window_xxh = delta_xxhash32(window, window_len);
|
||||||
|
if (window_xxh == sig->blocks[j].xxhash)
|
||||||
|
return j;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return DELTA_NO_BLOCK;
|
||||||
|
}
|
||||||
|
|
||||||
Delta* delta_compute(const void* new_file_data, uint64_t new_file_size, const DeltaSignature* sig,
|
Delta* delta_compute(const void* new_file_data, uint64_t new_file_size, const DeltaSignature* sig,
|
||||||
uint32_t block_size) {
|
uint32_t block_size) {
|
||||||
if (!new_file_data || !sig || !sig->blocks || new_file_size == 0 || block_size == 0 ||
|
if (!new_file_data || !sig || !sig->blocks || new_file_size == 0 || block_size == 0 ||
|
||||||
@@ -225,10 +339,29 @@ Delta* delta_compute(const void* new_file_data, uint64_t new_file_size, const De
|
|||||||
|
|
||||||
uint32_t capacity = 64;
|
uint32_t capacity = 64;
|
||||||
uint32_t count = 0;
|
uint32_t count = 0;
|
||||||
DeltaInstruction* instrs = malloc((size_t)capacity * sizeof(DeltaInstruction));
|
DeltaInstruction* instrs = protocol_alloc((size_t)capacity * sizeof(DeltaInstruction));
|
||||||
if (!instrs)
|
if (!instrs)
|
||||||
return NULL;
|
return NULL;
|
||||||
|
|
||||||
|
/* Build a one-time bucket index over the signature blocks keyed by the weak
|
||||||
|
* checksum. This turns the per-byte-window candidate lookup from an
|
||||||
|
* O(block_count) linear scan into an expected O(1) probe, which dominates
|
||||||
|
* the cost for large mostly-matching files (the diff steps one byte at a
|
||||||
|
* time through changed regions). On allocation failure the probe falls back
|
||||||
|
* to the original linear scan, so behaviour is unchanged under memory
|
||||||
|
* pressure. */
|
||||||
|
uint32_t* index = NULL;
|
||||||
|
const uint32_t* chain_next = NULL;
|
||||||
|
uint32_t mask = 0;
|
||||||
|
if (sig->block_count > 0) {
|
||||||
|
uint32_t bucket_count = delta_next_pow2(sig->block_count);
|
||||||
|
index = delta_build_index(sig, bucket_count);
|
||||||
|
if (index) {
|
||||||
|
chain_next = index + 2 * bucket_count;
|
||||||
|
mask = bucket_count - 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
uint64_t literal_start = 0;
|
uint64_t literal_start = 0;
|
||||||
bool has_literal = false;
|
bool has_literal = false;
|
||||||
|
|
||||||
@@ -263,34 +396,32 @@ Delta* delta_compute(const void* new_file_data, uint64_t new_file_size, const De
|
|||||||
}
|
}
|
||||||
|
|
||||||
bool matched = false;
|
bool matched = false;
|
||||||
for (uint32_t j = 0; j < sig->block_count; j++) {
|
uint32_t match_block = delta_find_match(new_data + i, window_len, adler, full_window, sig,
|
||||||
if (adler == sig->blocks[j].adler32 && full_window) {
|
index, chain_next, mask);
|
||||||
uint32_t xxh = delta_xxhash32(new_data + i, window_len);
|
if (match_block != DELTA_NO_BLOCK) {
|
||||||
if (xxh == sig->blocks[j].xxhash) {
|
if (has_literal) {
|
||||||
if (has_literal) {
|
if (!flush_literal(&instrs, &capacity, &count, new_data, literal_start, i)) {
|
||||||
if (!flush_literal(&instrs, &capacity, &count, new_data, literal_start, i)) {
|
free_instructions(instrs, count);
|
||||||
free_instructions(instrs, count);
|
free(index);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
|
||||||
has_literal = false;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!ensure_capacity(&instrs, &capacity, count)) {
|
|
||||||
free_instructions(instrs, count);
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
instrs[count].type = DELTA_INSTR_BLOCK_MATCH;
|
|
||||||
instrs[count].match.block_index = j;
|
|
||||||
instrs[count].match.block_offset = 0;
|
|
||||||
instrs[count].match.length = window_len;
|
|
||||||
count++;
|
|
||||||
|
|
||||||
i += window_len;
|
|
||||||
rolling_valid = false;
|
|
||||||
matched = true;
|
|
||||||
break;
|
|
||||||
}
|
}
|
||||||
|
has_literal = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (!ensure_capacity(&instrs, &capacity, count)) {
|
||||||
|
free_instructions(instrs, count);
|
||||||
|
free(index);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
instrs[count].type = DELTA_INSTR_BLOCK_MATCH;
|
||||||
|
instrs[count].match.block_index = match_block;
|
||||||
|
instrs[count].match.block_offset = 0;
|
||||||
|
instrs[count].match.length = window_len;
|
||||||
|
count++;
|
||||||
|
|
||||||
|
i += window_len;
|
||||||
|
rolling_valid = false;
|
||||||
|
matched = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!matched) {
|
if (!matched) {
|
||||||
@@ -302,6 +433,8 @@ Delta* delta_compute(const void* new_file_data, uint64_t new_file_size, const De
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
free(index);
|
||||||
|
|
||||||
if (has_literal) {
|
if (has_literal) {
|
||||||
if (!flush_literal(&instrs, &capacity, &count, new_data, literal_start, new_file_size)) {
|
if (!flush_literal(&instrs, &capacity, &count, new_data, literal_start, new_file_size)) {
|
||||||
free_instructions(instrs, count);
|
free_instructions(instrs, count);
|
||||||
@@ -309,7 +442,7 @@ Delta* delta_compute(const void* new_file_data, uint64_t new_file_size, const De
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Delta* delta = malloc(sizeof(Delta));
|
Delta* delta = protocol_alloc(sizeof(Delta));
|
||||||
if (!delta) {
|
if (!delta) {
|
||||||
free_instructions(instrs, count);
|
free_instructions(instrs, count);
|
||||||
return NULL;
|
return NULL;
|
||||||
@@ -355,7 +488,7 @@ Data* delta_serialize(const Delta* delta) {
|
|||||||
if (delta->delta_size > UINT64_MAX - header_size || header_size + delta->delta_size > SIZE_MAX)
|
if (delta->delta_size > UINT64_MAX - header_size || header_size + delta->delta_size > SIZE_MAX)
|
||||||
return NULL;
|
return NULL;
|
||||||
uint64_t total = header_size + delta->delta_size;
|
uint64_t total = header_size + delta->delta_size;
|
||||||
uint8_t* buf = malloc((size_t)total);
|
uint8_t* buf = protocol_alloc((size_t)total);
|
||||||
if (!buf)
|
if (!buf)
|
||||||
return NULL;
|
return NULL;
|
||||||
|
|
||||||
@@ -395,7 +528,7 @@ Delta* delta_deserialize(const Data* data) {
|
|||||||
const uint8_t* buf = (const uint8_t*)data->data;
|
const uint8_t* buf = (const uint8_t*)data->data;
|
||||||
size_t pos = 0;
|
size_t pos = 0;
|
||||||
|
|
||||||
Delta* delta = malloc(sizeof(Delta));
|
Delta* delta = protocol_alloc(sizeof(Delta));
|
||||||
if (!delta)
|
if (!delta)
|
||||||
return NULL;
|
return NULL;
|
||||||
|
|
||||||
@@ -412,9 +545,10 @@ Delta* delta_deserialize(const Data* data) {
|
|||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
delta->instructions = delta->instruction_count == 0
|
delta->instructions =
|
||||||
? NULL
|
delta->instruction_count == 0
|
||||||
: malloc((size_t)delta->instruction_count * sizeof(DeltaInstruction));
|
? NULL
|
||||||
|
: protocol_alloc((size_t)delta->instruction_count * sizeof(DeltaInstruction));
|
||||||
if (delta->instruction_count > 0 && !delta->instructions) {
|
if (delta->instruction_count > 0 && !delta->instructions) {
|
||||||
free(delta);
|
free(delta);
|
||||||
return NULL;
|
return NULL;
|
||||||
@@ -465,7 +599,7 @@ Delta* delta_deserialize(const Data* data) {
|
|||||||
free(delta);
|
free(delta);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
delta->instructions[i].literal.data = malloc(lit_len ? lit_len : 1);
|
delta->instructions[i].literal.data = protocol_alloc(lit_len ? lit_len : 1);
|
||||||
if (!delta->instructions[i].literal.data) {
|
if (!delta->instructions[i].literal.data) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Failed to allocate %u bytes for literal data", lit_len);
|
log_message(LOG_LEVEL_ERROR, "Failed to allocate %u bytes for literal data", lit_len);
|
||||||
free_instructions(delta->instructions, i);
|
free_instructions(delta->instructions, i);
|
||||||
@@ -492,7 +626,7 @@ void* delta_apply(const void* old_data, uint64_t old_size, const Delta* delta,
|
|||||||
delta->new_file_size > DELTA_MAX_FILE_SIZE || delta->new_file_size > SIZE_MAX)
|
delta->new_file_size > DELTA_MAX_FILE_SIZE || delta->new_file_size > SIZE_MAX)
|
||||||
return NULL;
|
return NULL;
|
||||||
|
|
||||||
void* output = malloc(delta->new_file_size ? (size_t)delta->new_file_size : 1);
|
void* output = protocol_alloc(delta->new_file_size ? (size_t)delta->new_file_size : 1);
|
||||||
if (!output)
|
if (!output)
|
||||||
return NULL;
|
return NULL;
|
||||||
|
|
||||||
|
|||||||
+127
-20
@@ -14,6 +14,7 @@
|
|||||||
#include "log.h"
|
#include "log.h"
|
||||||
#include "metadata.h"
|
#include "metadata.h"
|
||||||
#include "utils.h"
|
#include "utils.h"
|
||||||
|
#include "protocol.h"
|
||||||
|
|
||||||
static bool write_all(int fd, const void* data, unsigned long long size) {
|
static bool write_all(int fd, const void* data, unsigned long long size) {
|
||||||
const unsigned char* p = data;
|
const unsigned char* p = data;
|
||||||
@@ -45,14 +46,14 @@ bool file_checksum(File* file, uint64_t* checksum) {
|
|||||||
File* file_create(const char* path) {
|
File* file_create(const char* path) {
|
||||||
if (!path)
|
if (!path)
|
||||||
return NULL;
|
return NULL;
|
||||||
File* file = (File*)malloc(sizeof(File));
|
File* file = (File*)protocol_alloc(sizeof(File));
|
||||||
if (file == NULL) {
|
if (file == NULL) {
|
||||||
log_perror("ERROR: Could not allocate memory for file struct");
|
log_perror("ERROR: Could not allocate memory for file struct");
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t path_len = strlen(path);
|
size_t path_len = strlen(path);
|
||||||
file->path = (char*)malloc(path_len + 1);
|
file->path = (char*)protocol_alloc(path_len + 1);
|
||||||
if (file->path == NULL) {
|
if (file->path == NULL) {
|
||||||
free(file);
|
free(file);
|
||||||
return NULL;
|
return NULL;
|
||||||
@@ -85,7 +86,7 @@ void file_destroy(void* item) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
FileMetadata* file_metadata_create(const struct stat* stats) {
|
FileMetadata* file_metadata_create(const struct stat* stats) {
|
||||||
FileMetadata* m = malloc(sizeof(FileMetadata));
|
FileMetadata* m = protocol_alloc(sizeof(FileMetadata));
|
||||||
if (m == NULL) {
|
if (m == NULL) {
|
||||||
log_perror("ERROR: Could not allocate memory for file metadata");
|
log_perror("ERROR: Could not allocate memory for file metadata");
|
||||||
return NULL;
|
return NULL;
|
||||||
@@ -112,7 +113,7 @@ bool file_load_data(File* file) {
|
|||||||
if (file->data->data == NULL) {
|
if (file->data->data == NULL) {
|
||||||
if (file->data->size == 0)
|
if (file->data->size == 0)
|
||||||
return true;
|
return true;
|
||||||
file->data->data = malloc(file->data->size);
|
file->data->data = protocol_alloc(file->data->size);
|
||||||
if (file->data->data == NULL) {
|
if (file->data->data == NULL) {
|
||||||
log_perror("Could not allocate memory for file data");
|
log_perror("Could not allocate memory for file data");
|
||||||
return false;
|
return false;
|
||||||
@@ -172,8 +173,17 @@ bool file_set_authorized_root(int fd, const char* canonical_path) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
bool file_path_exists_secure(const char* path) {
|
bool file_path_exists_secure(const char* path) {
|
||||||
|
if (!path)
|
||||||
|
return false;
|
||||||
|
char* leaf = NULL;
|
||||||
|
int parent_fd = file_open_secure_parent(path, &leaf, false);
|
||||||
|
if (parent_fd < 0)
|
||||||
|
return false;
|
||||||
struct stat st;
|
struct stat st;
|
||||||
return file_stat_secure(path, &st);
|
bool exists = fstatat(parent_fd, leaf, &st, AT_SYMLINK_NOFOLLOW) == 0;
|
||||||
|
close(parent_fd);
|
||||||
|
free(leaf);
|
||||||
|
return exists;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool file_stat_secure(const char* path, struct stat* st) {
|
bool file_stat_secure(const char* path, struct stat* st) {
|
||||||
@@ -183,15 +193,29 @@ bool file_stat_secure(const char* path, struct stat* st) {
|
|||||||
int parent_fd = file_open_secure_parent(path, &leaf, false);
|
int parent_fd = file_open_secure_parent(path, &leaf, false);
|
||||||
if (parent_fd < 0)
|
if (parent_fd < 0)
|
||||||
return false;
|
return false;
|
||||||
int fd = openat(parent_fd, leaf, O_RDONLY | O_NONBLOCK | O_CLOEXEC | O_NOFOLLOW);
|
bool exists = fstatat(parent_fd, leaf, st, AT_SYMLINK_NOFOLLOW) == 0 && S_ISREG(st->st_mode);
|
||||||
bool exists = fd >= 0 && fstat(fd, st) == 0 && S_ISREG(st->st_mode);
|
|
||||||
if (fd >= 0)
|
|
||||||
close(fd);
|
|
||||||
close(parent_fd);
|
close(parent_fd);
|
||||||
free(leaf);
|
free(leaf);
|
||||||
return exists;
|
return exists;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static bool stat_is_newer(const struct stat* st, const FileMetadata* metadata) {
|
||||||
|
if (!st || !metadata)
|
||||||
|
return false;
|
||||||
|
#ifdef __linux__
|
||||||
|
long mtime_nsec = st->st_mtim.tv_nsec;
|
||||||
|
#else
|
||||||
|
long mtime_nsec = 0;
|
||||||
|
#endif
|
||||||
|
return st->st_mtime > metadata->mtime_sec ||
|
||||||
|
(st->st_mtime == metadata->mtime_sec && mtime_nsec > metadata->mtime_nsec);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool file_destination_is_newer_secure(const char* path, const FileMetadata* metadata) {
|
||||||
|
struct stat st;
|
||||||
|
return file_stat_secure(path, &st) && stat_is_newer(&st, metadata);
|
||||||
|
}
|
||||||
|
|
||||||
int file_open_secure_parent(const char* path, char** leaf_out, bool create_dirs) {
|
int file_open_secure_parent(const char* path, char** leaf_out, bool create_dirs) {
|
||||||
char* copy = str_dup(path);
|
char* copy = str_dup(path);
|
||||||
if (!copy)
|
if (!copy)
|
||||||
@@ -302,8 +326,10 @@ bool file_rename_secure(const char* old_path, const char* new_path) {
|
|||||||
return ok;
|
return ok;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool file_to_disk_secure(const char* path, const void* data, unsigned long long data_size,
|
static bool file_to_disk_secure_impl(const char* path, const void* data,
|
||||||
bool inplace, bool sparse, const FileMetadata* metadata) {
|
unsigned long long data_size, bool inplace, bool sparse,
|
||||||
|
const FileMetadata* metadata, bool preserve_executability,
|
||||||
|
bool update, bool no_replace, bool use_fsync) {
|
||||||
char* leaf = NULL;
|
char* leaf = NULL;
|
||||||
int dirfd = file_open_secure_parent(path, &leaf, true);
|
int dirfd = file_open_secure_parent(path, &leaf, true);
|
||||||
if (dirfd < 0)
|
if (dirfd < 0)
|
||||||
@@ -311,15 +337,53 @@ bool file_to_disk_secure(const char* path, const void* data, unsigned long long
|
|||||||
int fd = -1;
|
int fd = -1;
|
||||||
bool ok = false;
|
bool ok = false;
|
||||||
if (inplace) {
|
if (inplace) {
|
||||||
fd = openat(dirfd, leaf, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC | O_NOFOLLOW, 0644);
|
fd = openat(dirfd, leaf, O_WRONLY | O_CREAT | O_CLOEXEC | O_NOFOLLOW, 0644);
|
||||||
if (fd >= 0) {
|
if (fd >= 0) {
|
||||||
if (!sparse || data_size == 0 || ftruncate(fd, (off_t)data_size) == 0)
|
struct stat destination_stat;
|
||||||
ok = write_all(fd, data, data_size);
|
bool newer = false;
|
||||||
if (ok && metadata)
|
if (update && metadata && fstat(fd, &destination_stat) == 0 &&
|
||||||
ok = file_restore_metadata_fd(fd, metadata);
|
S_ISREG(destination_stat.st_mode)) {
|
||||||
|
newer = stat_is_newer(&destination_stat, metadata);
|
||||||
|
}
|
||||||
|
if (newer) {
|
||||||
|
ok = true;
|
||||||
|
} else {
|
||||||
|
/* In-place overwrites: pre-size sparse targets and always trim the
|
||||||
|
file to the new payload length afterwards so shorter payloads can
|
||||||
|
never leave stale trailing bytes from a previous version. */
|
||||||
|
if (sparse && data_size > 0)
|
||||||
|
ok = ftruncate(fd, (off_t)data_size) == 0;
|
||||||
|
if (ok || !sparse || data_size == 0)
|
||||||
|
ok = write_all(fd, data, data_size);
|
||||||
|
if (ok)
|
||||||
|
ok = ftruncate(fd, (off_t)data_size) == 0;
|
||||||
|
/* Normalize the mode: apply the metadata-derived safe mode when the
|
||||||
|
sender supplied metadata (setuid/setgid/sticky are never honored);
|
||||||
|
otherwise fall back to a safe default so dangerous bits on an
|
||||||
|
existing destination cannot survive an overwrite. */
|
||||||
|
if (ok) {
|
||||||
|
if (metadata)
|
||||||
|
ok = file_restore_metadata_fd(fd, metadata, preserve_executability);
|
||||||
|
else if (fchmod(fd, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH) != 0)
|
||||||
|
ok = false;
|
||||||
|
}
|
||||||
|
if (ok && use_fsync)
|
||||||
|
ok = fsync(fd) == 0;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
char tmp[NAME_MAX];
|
char tmp[NAME_MAX];
|
||||||
|
if (update && metadata) {
|
||||||
|
/* This check protects the normal atomic path as far as possible. A
|
||||||
|
concurrent replacement can still occur before the final rename. */
|
||||||
|
struct stat destination_stat;
|
||||||
|
if (fstatat(dirfd, leaf, &destination_stat, AT_SYMLINK_NOFOLLOW) == 0 &&
|
||||||
|
S_ISREG(destination_stat.st_mode) && stat_is_newer(&destination_stat, metadata)) {
|
||||||
|
close(dirfd);
|
||||||
|
free(leaf);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
for (unsigned int i = 0; i < 100 && !ok; ++i) {
|
for (unsigned int i = 0; i < 100 && !ok; ++i) {
|
||||||
snprintf(tmp, sizeof(tmp), ".%s.tmp.%ld.%u", leaf, (long)getpid(), i);
|
snprintf(tmp, sizeof(tmp), ".%s.tmp.%ld.%u", leaf, (long)getpid(), i);
|
||||||
fd = openat(dirfd, tmp, O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC | O_NOFOLLOW, 0600);
|
fd = openat(dirfd, tmp, O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC | O_NOFOLLOW, 0600);
|
||||||
@@ -330,12 +394,26 @@ bool file_to_disk_secure(const char* path, const void* data, unsigned long long
|
|||||||
if (ok || (!sparse || data_size == 0))
|
if (ok || (!sparse || data_size == 0))
|
||||||
ok = write_all(fd, data, data_size);
|
ok = write_all(fd, data, data_size);
|
||||||
if (ok && metadata)
|
if (ok && metadata)
|
||||||
ok = file_restore_metadata_fd(fd, metadata);
|
ok = file_restore_metadata_fd(fd, metadata, preserve_executability);
|
||||||
|
if (ok && use_fsync)
|
||||||
|
ok = fsync(fd) == 0;
|
||||||
if (close(fd) != 0)
|
if (close(fd) != 0)
|
||||||
ok = false;
|
ok = false;
|
||||||
fd = -1;
|
fd = -1;
|
||||||
if (ok && renameat(dirfd, tmp, dirfd, leaf) != 0)
|
if (ok) {
|
||||||
ok = false;
|
if (no_replace) {
|
||||||
|
/* The probe and commit cannot be one operation. A concurrent
|
||||||
|
creator may win; EEXIST is then the requested skip. */
|
||||||
|
if (linkat(dirfd, tmp, dirfd, leaf, 0) == 0 || errno == EEXIST) {
|
||||||
|
if (unlinkat(dirfd, tmp, 0) != 0 && errno != ENOENT)
|
||||||
|
ok = false;
|
||||||
|
} else {
|
||||||
|
ok = false;
|
||||||
|
}
|
||||||
|
} else if (renameat(dirfd, tmp, dirfd, leaf) != 0) {
|
||||||
|
ok = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
if (!ok)
|
if (!ok)
|
||||||
unlinkat(dirfd, tmp, 0);
|
unlinkat(dirfd, tmp, 0);
|
||||||
}
|
}
|
||||||
@@ -347,9 +425,38 @@ bool file_to_disk_secure(const char* path, const void* data, unsigned long long
|
|||||||
return ok;
|
return ok;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool file_to_disk_secure(const char* path, const void* data, unsigned long long data_size,
|
||||||
|
bool inplace, bool sparse, const FileMetadata* metadata,
|
||||||
|
bool preserve_executability) {
|
||||||
|
return file_to_disk_secure_impl(path, data, data_size, inplace, sparse, metadata,
|
||||||
|
preserve_executability, false, false, false);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool file_to_disk_secure_update(const char* path, const void* data, unsigned long long data_size,
|
||||||
|
bool inplace, bool sparse, const FileMetadata* metadata,
|
||||||
|
bool preserve_executability) {
|
||||||
|
return file_to_disk_secure_impl(path, data, data_size, inplace, sparse, metadata,
|
||||||
|
preserve_executability, true, false, false);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool file_to_disk_secure_with_fsync(const char* path, const void* data,
|
||||||
|
unsigned long long data_size, bool inplace, bool sparse,
|
||||||
|
const FileMetadata* metadata, bool preserve_executability,
|
||||||
|
bool use_fsync) {
|
||||||
|
return file_to_disk_secure_impl(path, data, data_size, inplace, sparse, metadata,
|
||||||
|
preserve_executability, false, false, use_fsync);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool file_to_disk_secure_no_replace(const char* path, const void* data,
|
||||||
|
unsigned long long data_size, bool sparse,
|
||||||
|
const FileMetadata* metadata, bool preserve_executability) {
|
||||||
|
return file_to_disk_secure_impl(path, data, data_size, false, sparse, metadata,
|
||||||
|
preserve_executability, false, true, false);
|
||||||
|
}
|
||||||
|
|
||||||
bool file_write_to_disk(const char* path, const void* data, unsigned long long data_size,
|
bool file_write_to_disk(const char* path, const void* data, unsigned long long data_size,
|
||||||
bool inplace, bool sparse) {
|
bool inplace, bool sparse) {
|
||||||
if (!path || (!data && data_size != 0) || has_path_traversal(path))
|
if (!path || (!data && data_size != 0) || has_path_traversal(path))
|
||||||
return false;
|
return false;
|
||||||
return file_to_disk_secure(path, data, data_size, inplace, sparse, NULL);
|
return file_to_disk_secure(path, data, data_size, inplace, sparse, NULL, false);
|
||||||
}
|
}
|
||||||
+16
-1
@@ -27,10 +27,25 @@ bool file_set_authorized_root(int fd, const char* canonical_path);
|
|||||||
/* Secure path/filesystem primitives (symlink-safe, O_NOFOLLOW, root-confined). */
|
/* Secure path/filesystem primitives (symlink-safe, O_NOFOLLOW, root-confined). */
|
||||||
bool file_path_exists_secure(const char* path);
|
bool file_path_exists_secure(const char* path);
|
||||||
bool file_stat_secure(const char* path, struct stat* st);
|
bool file_stat_secure(const char* path, struct stat* st);
|
||||||
|
bool file_destination_is_newer_secure(const char* path, const FileMetadata* metadata);
|
||||||
int file_open_secure_parent(const char* path, char** leaf_out, bool create_dirs);
|
int file_open_secure_parent(const char* path, char** leaf_out, bool create_dirs);
|
||||||
bool file_ensure_directory_secure(const char* path);
|
bool file_ensure_directory_secure(const char* path);
|
||||||
bool file_rename_secure(const char* old_path, const char* new_path);
|
bool file_rename_secure(const char* old_path, const char* new_path);
|
||||||
bool file_to_disk_secure(const char* path, const void* data, unsigned long long data_size,
|
bool file_to_disk_secure(const char* path, const void* data, unsigned long long data_size,
|
||||||
bool inplace, bool sparse, const FileMetadata* metadata);
|
bool inplace, bool sparse, const FileMetadata* metadata,
|
||||||
|
bool preserve_executability);
|
||||||
|
bool file_to_disk_secure_with_fsync(const char* path, const void* data,
|
||||||
|
unsigned long long data_size, bool inplace, bool sparse,
|
||||||
|
const FileMetadata* metadata, bool preserve_executability,
|
||||||
|
bool use_fsync);
|
||||||
|
/* With update enabled, an existing newer destination is left untouched. The
|
||||||
|
check is descriptor-based for inplace writes; atomic replacement still has
|
||||||
|
an unavoidable final rename race without filesystem locking. */
|
||||||
|
bool file_to_disk_secure_update(const char* path, const void* data, unsigned long long data_size,
|
||||||
|
bool inplace, bool sparse, const FileMetadata* metadata,
|
||||||
|
bool preserve_executability);
|
||||||
|
bool file_to_disk_secure_no_replace(const char* path, const void* data,
|
||||||
|
unsigned long long data_size, bool sparse,
|
||||||
|
const FileMetadata* metadata, bool preserve_executability);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
+185
-70
@@ -20,16 +20,26 @@
|
|||||||
#include "utils.h"
|
#include "utils.h"
|
||||||
|
|
||||||
#define MAX_SERVER_DELETE_COUNT 100000U
|
#define MAX_SERVER_DELETE_COUNT 100000U
|
||||||
#define MAX_FILE_DATA_SIZE MAX_RECEIVE_FILE_SIZE
|
#define MAX_FILE_DATA_SIZE MAX_RECEIVE_WHOLE_FILE_SIZE
|
||||||
|
|
||||||
bool file_save_to_disk(const char* root_directory, const File* file, const Config* config) {
|
bool file_save_to_disk(const char* root_directory, const File* file, const Config* config) {
|
||||||
bool backup_enabled = config && config->backup;
|
return file_save_to_disk_full(root_directory, file, config) != FILE_SAVE_ERROR;
|
||||||
|
}
|
||||||
|
|
||||||
|
FileSaveResult file_save_to_disk_full(const char* root_directory, const File* file,
|
||||||
|
const Config* config) {
|
||||||
|
/* Backups are incompatible with ignore-existing: moving the entry first
|
||||||
|
would make a concurrent no-replace commit overwrite its old name. */
|
||||||
|
bool backup_enabled = config && config->backup && !config->ignore_existing;
|
||||||
bool inplace = config && config->inplace;
|
bool inplace = config && config->inplace;
|
||||||
bool sparse = config && config->preserve_sparse;
|
bool sparse = config && config->preserve_sparse;
|
||||||
|
bool preserve_executability = config && config->use_executability;
|
||||||
const char* backup_suffix = (config && config->suffix) ? config->suffix : "~";
|
const char* backup_suffix = (config && config->suffix) ? config->suffix : "~";
|
||||||
const char* backup_dir = (config && config->backup_dir) ? config->backup_dir : NULL;
|
const char* backup_dir = (config && config->backup_dir) ? config->backup_dir : NULL;
|
||||||
const char* partial_dir = (config && config->partial_dir) ? config->partial_dir : NULL;
|
const char* partial_dir = (config && config->partial_dir) ? config->partial_dir : NULL;
|
||||||
|
bool use_partial_root = partial_dir && config && config->partial;
|
||||||
char *confined_backup = NULL, *confined_partial = NULL, *disk_path = NULL;
|
char *confined_backup = NULL, *confined_partial = NULL, *disk_path = NULL;
|
||||||
|
char* destination_path = NULL;
|
||||||
char *backup_path = NULL, *parent_copy = NULL;
|
char *backup_path = NULL, *parent_copy = NULL;
|
||||||
|
|
||||||
if (!file || !file->path || !file->data || (file->data->size != 0 && !file->data->data) ||
|
if (!file || !file->path || !file->data || (file->data->size != 0 && !file->data->data) ||
|
||||||
@@ -38,55 +48,84 @@ bool file_save_to_disk(const char* root_directory, const File* file, const Confi
|
|||||||
(!backup_suffix || backup_suffix[0] == '\0' || strchr(backup_suffix, '/') != NULL ||
|
(!backup_suffix || backup_suffix[0] == '\0' || strchr(backup_suffix, '/') != NULL ||
|
||||||
strcmp(backup_suffix, ".") == 0 || strcmp(backup_suffix, "..") == 0))) {
|
strcmp(backup_suffix, ".") == 0 || strcmp(backup_suffix, "..") == 0))) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Invalid file or path received");
|
log_message(LOG_LEVEL_ERROR, "Invalid file or path received");
|
||||||
return false;
|
return FILE_SAVE_ERROR;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* These options arrive from the client. They are names below the server
|
/* These options arrive from the client. They are names below the server
|
||||||
root, never independent filesystem roots. */
|
root, never independent filesystem roots. */
|
||||||
if ((backup_dir && (backup_dir[0] == '/' || has_path_traversal(backup_dir))) ||
|
if ((backup_dir && (backup_dir[0] == '/' || has_path_traversal(backup_dir))) ||
|
||||||
(partial_dir && (partial_dir[0] == '/' || has_path_traversal(partial_dir))))
|
(partial_dir && (partial_dir[0] == '/' || has_path_traversal(partial_dir))))
|
||||||
return false;
|
return FILE_SAVE_ERROR;
|
||||||
if (backup_dir && !(confined_backup = path_cat(root_directory, backup_dir)))
|
if (backup_dir && !(confined_backup = path_cat(root_directory, backup_dir)))
|
||||||
return false;
|
return FILE_SAVE_ERROR;
|
||||||
if (partial_dir && !(confined_partial = path_cat(root_directory, partial_dir))) {
|
if (partial_dir && !(confined_partial = path_cat(root_directory, partial_dir))) {
|
||||||
free(confined_backup);
|
free(confined_backup);
|
||||||
return false;
|
return FILE_SAVE_ERROR;
|
||||||
}
|
}
|
||||||
|
|
||||||
const char* actual_root =
|
const char* actual_root = use_partial_root ? confined_partial : root_directory;
|
||||||
(partial_dir && config && config->partial) ? confined_partial : root_directory;
|
destination_path = path_cat(root_directory, file->path);
|
||||||
disk_path = path_cat(actual_root, file->path);
|
disk_path = path_cat(actual_root, file->path);
|
||||||
if (disk_path == NULL) {
|
if (destination_path == NULL || disk_path == NULL) {
|
||||||
free(confined_backup);
|
free(confined_backup);
|
||||||
free(confined_partial);
|
free(confined_partial);
|
||||||
return false;
|
free(destination_path);
|
||||||
|
free(disk_path);
|
||||||
|
return FILE_SAVE_ERROR;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* --update is receiver-side policy: never replace a newer destination. */
|
/* --existing checks the final destination, not a temporary partial path. */
|
||||||
if (config && config->update) {
|
if (config && config->existing && !file_path_exists_secure(destination_path)) {
|
||||||
struct stat destination_stat;
|
free(confined_backup);
|
||||||
if (file_stat_secure(disk_path, &destination_stat) && file->metadata &&
|
free(confined_partial);
|
||||||
destination_stat.st_mtime > file->metadata->mtime_sec) {
|
free(destination_path);
|
||||||
|
free(disk_path);
|
||||||
|
return FILE_SAVE_SKIPPED;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* --ignore-existing checks the final destination before partial files or
|
||||||
|
overwrite policies can modify it. */
|
||||||
|
if (config && config->ignore_existing) {
|
||||||
|
bool exists = file_path_exists_secure(destination_path);
|
||||||
|
if (exists) {
|
||||||
free(confined_backup);
|
free(confined_backup);
|
||||||
free(confined_partial);
|
free(confined_partial);
|
||||||
|
free(destination_path);
|
||||||
free(disk_path);
|
free(disk_path);
|
||||||
return true;
|
return FILE_SAVE_SKIPPED;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* --update is receiver-side policy: never replace a newer destination.
|
||||||
|
In partial-dir mode the entry that would be replaced is the real
|
||||||
|
destination, not the temporary partial file. The secure stat does not
|
||||||
|
require read permission on the destination. */
|
||||||
|
const char* update_target = use_partial_root ? destination_path : disk_path;
|
||||||
|
if (config && config->update && file_destination_is_newer_secure(update_target, file->metadata)) {
|
||||||
|
free(confined_backup);
|
||||||
|
free(confined_partial);
|
||||||
|
free(destination_path);
|
||||||
|
free(disk_path);
|
||||||
|
return FILE_SAVE_SKIPPED;
|
||||||
|
}
|
||||||
|
|
||||||
if (backup_enabled) {
|
if (backup_enabled) {
|
||||||
|
/* Back up the entry that the incoming write will replace. When writing
|
||||||
|
through a partial dir the pre-existing destination file is the one to
|
||||||
|
preserve; any stale partial file is overwritten without a backup. */
|
||||||
|
const char* replace_target = use_partial_root ? destination_path : disk_path;
|
||||||
struct stat backup_stat;
|
struct stat backup_stat;
|
||||||
if (file_stat_secure(disk_path, &backup_stat)) {
|
if (file_stat_secure(replace_target, &backup_stat)) {
|
||||||
if (backup_dir) {
|
if (backup_dir) {
|
||||||
backup_path = path_cat(confined_backup, file->path);
|
backup_path = path_cat(confined_backup, file->path);
|
||||||
} else {
|
} else {
|
||||||
size_t path_len = strlen(disk_path);
|
size_t path_len = strlen(replace_target);
|
||||||
size_t suffix_len = strlen(backup_suffix);
|
size_t suffix_len = strlen(backup_suffix);
|
||||||
if (path_len > SIZE_MAX - suffix_len - 1)
|
if (path_len > SIZE_MAX - suffix_len - 1)
|
||||||
goto fail;
|
goto fail;
|
||||||
backup_path = malloc(path_len + suffix_len + 1);
|
backup_path = malloc(path_len + suffix_len + 1);
|
||||||
if (backup_path) {
|
if (backup_path) {
|
||||||
memcpy(backup_path, disk_path, path_len);
|
memcpy(backup_path, replace_target, path_len);
|
||||||
memcpy(backup_path + path_len, backup_suffix, suffix_len + 1);
|
memcpy(backup_path + path_len, backup_suffix, suffix_len + 1);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -97,7 +136,7 @@ bool file_save_to_disk(const char* root_directory, const File* file, const Confi
|
|||||||
goto fail;
|
goto fail;
|
||||||
free(parent_copy);
|
free(parent_copy);
|
||||||
parent_copy = NULL;
|
parent_copy = NULL;
|
||||||
if (!file_rename_secure(disk_path, backup_path))
|
if (!file_rename_secure(replace_target, backup_path))
|
||||||
goto fail;
|
goto fail;
|
||||||
free(backup_path);
|
free(backup_path);
|
||||||
backup_path = NULL;
|
backup_path = NULL;
|
||||||
@@ -112,28 +151,51 @@ bool file_save_to_disk(const char* root_directory, const File* file, const Confi
|
|||||||
goto fail;
|
goto fail;
|
||||||
metadata = &adjusted_metadata;
|
metadata = &adjusted_metadata;
|
||||||
}
|
}
|
||||||
bool ok =
|
bool ok = config && config->ignore_existing
|
||||||
file_to_disk_secure(disk_path, file->data->data, file->data->size, inplace, sparse, metadata);
|
? file_to_disk_secure_no_replace(disk_path, file->data->data, file->data->size,
|
||||||
|
sparse, metadata, preserve_executability)
|
||||||
|
: config && config->update
|
||||||
|
? file_to_disk_secure_update(disk_path, file->data->data, file->data->size, inplace,
|
||||||
|
sparse, metadata, preserve_executability)
|
||||||
|
: file_to_disk_secure_with_fsync(disk_path, file->data->data, file->data->size,
|
||||||
|
inplace, sparse, metadata, preserve_executability,
|
||||||
|
config && config->use_fsync);
|
||||||
|
if (!ok)
|
||||||
|
goto fail;
|
||||||
|
|
||||||
|
/* --partial --partial-dir writes the complete file under the partial dir so
|
||||||
|
interrupted transfers leave a resumable copy there. Once the file is
|
||||||
|
fully written it must be atomically installed at the real destination;
|
||||||
|
otherwise completed transfers would linger under the partial dir. */
|
||||||
|
if (use_partial_root) {
|
||||||
|
if (!file_rename_secure(disk_path, destination_path))
|
||||||
|
goto fail;
|
||||||
|
}
|
||||||
|
|
||||||
free(parent_copy);
|
free(parent_copy);
|
||||||
free(backup_path);
|
free(backup_path);
|
||||||
free(confined_backup);
|
free(confined_backup);
|
||||||
free(confined_partial);
|
free(confined_partial);
|
||||||
|
free(destination_path);
|
||||||
free(disk_path);
|
free(disk_path);
|
||||||
return ok;
|
return FILE_SAVE_WRITTEN;
|
||||||
|
|
||||||
fail:
|
fail:
|
||||||
free(parent_copy);
|
free(parent_copy);
|
||||||
free(backup_path);
|
free(backup_path);
|
||||||
free(confined_backup);
|
free(confined_backup);
|
||||||
free(confined_partial);
|
free(confined_partial);
|
||||||
|
free(destination_path);
|
||||||
free(disk_path);
|
free(disk_path);
|
||||||
return false;
|
return FILE_SAVE_ERROR;
|
||||||
}
|
}
|
||||||
|
|
||||||
static File* receive_delta_file(int fd, const Config* config, const char* check_path,
|
static File* receive_delta_file(int fd, const Config* config, const char* check_path,
|
||||||
void* old_data, unsigned long long old_size, bool* failed) {
|
void* old_data, unsigned long long old_size, bool* failed) {
|
||||||
if (!old_data)
|
if (!old_data) {
|
||||||
|
*failed = true;
|
||||||
return NULL;
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
DeltaSignature* sig = delta_signature_create(old_data, old_size, config->delta_block_size);
|
DeltaSignature* sig = delta_signature_create(old_data, old_size, config->delta_block_size);
|
||||||
if (!sig) {
|
if (!sig) {
|
||||||
@@ -169,7 +231,7 @@ static File* receive_delta_file(int fd, const Config* config, const char* check_
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (resp == STATUS_DELTA_DATA) {
|
if (resp == STATUS_DELTA_DATA) {
|
||||||
Data* delta_data = receive_data_limited(fd, MAX_RECEIVE_FILE_SIZE);
|
Data* delta_data = receive_data_limited(fd, MAX_RECEIVE_WHOLE_FILE_SIZE);
|
||||||
if (!delta_data) {
|
if (!delta_data) {
|
||||||
delta_signature_destroy(sig);
|
delta_signature_destroy(sig);
|
||||||
free(old_data);
|
free(old_data);
|
||||||
@@ -178,8 +240,11 @@ static File* receive_delta_file(int fd, const Config* config, const char* check_
|
|||||||
}
|
}
|
||||||
|
|
||||||
Data* raw_delta = delta_data;
|
Data* raw_delta = delta_data;
|
||||||
if (config->use_compression) {
|
if (config->use_compression &&
|
||||||
raw_delta = data_decompress_limited(delta_data, MAX_RECEIVE_FILE_SIZE);
|
!compression_should_skip_with_suffixes(
|
||||||
|
check_path, config->skip_compress_suffixes,
|
||||||
|
config->skip_compress_set ? config->skip_compress_count : -1)) {
|
||||||
|
raw_delta = data_decompress_limited(delta_data, MAX_RECEIVE_WHOLE_FILE_SIZE);
|
||||||
data_destroy(delta_data);
|
data_destroy(delta_data);
|
||||||
if (!raw_delta) {
|
if (!raw_delta) {
|
||||||
free(old_data);
|
free(old_data);
|
||||||
@@ -199,11 +264,12 @@ static File* receive_delta_file(int fd, const Config* config, const char* check_
|
|||||||
}
|
}
|
||||||
|
|
||||||
uint64_t new_size = delta->new_file_size;
|
uint64_t new_size = delta->new_file_size;
|
||||||
if (new_size > MAX_RECEIVE_FILE_SIZE || new_size > SIZE_MAX) {
|
if (new_size > MAX_RECEIVE_WHOLE_FILE_SIZE || new_size > SIZE_MAX) {
|
||||||
delta_destroy(delta);
|
delta_destroy(delta);
|
||||||
free(old_data);
|
free(old_data);
|
||||||
delta_signature_destroy(sig);
|
delta_signature_destroy(sig);
|
||||||
send_status(fd, STATUS_ERROR);
|
send_status(fd, STATUS_ERROR);
|
||||||
|
*failed = true;
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
void* new_data = delta_apply(old_data, old_size, delta, config->delta_block_size);
|
void* new_data = delta_apply(old_data, old_size, delta, config->delta_block_size);
|
||||||
@@ -244,6 +310,7 @@ static File* receive_delta_file(int fd, const Config* config, const char* check_
|
|||||||
free(old_data);
|
free(old_data);
|
||||||
delta_signature_destroy(sig);
|
delta_signature_destroy(sig);
|
||||||
send_status(fd, STATUS_ERROR);
|
send_status(fd, STATUS_ERROR);
|
||||||
|
*failed = true;
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
data_destroy(file->data);
|
data_destroy(file->data);
|
||||||
@@ -274,15 +341,18 @@ static File* receive_delta_file(int fd, const Config* config, const char* check_
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Data* file_data = receive_data_limited(fd, MAX_RECEIVE_FILE_SIZE);
|
Data* file_data = receive_data_limited(fd, MAX_RECEIVE_WHOLE_FILE_SIZE);
|
||||||
if (file_data == NULL) {
|
if (file_data == NULL) {
|
||||||
file_destroy(file);
|
file_destroy(file);
|
||||||
*failed = true;
|
*failed = true;
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (config->use_compression) {
|
if (config->use_compression &&
|
||||||
Data* uncompressed = data_decompress_limited(file_data, MAX_RECEIVE_FILE_SIZE);
|
!compression_should_skip_with_suffixes(
|
||||||
|
file->path, config->skip_compress_suffixes,
|
||||||
|
config->skip_compress_set ? config->skip_compress_count : -1)) {
|
||||||
|
Data* uncompressed = data_decompress_limited(file_data, MAX_RECEIVE_WHOLE_FILE_SIZE);
|
||||||
data_destroy(file_data);
|
data_destroy(file_data);
|
||||||
if (uncompressed == NULL) {
|
if (uncompressed == NULL) {
|
||||||
file_destroy(file);
|
file_destroy(file);
|
||||||
@@ -293,6 +363,7 @@ static File* receive_delta_file(int fd, const Config* config, const char* check_
|
|||||||
data_destroy(uncompressed);
|
data_destroy(uncompressed);
|
||||||
file_destroy(file);
|
file_destroy(file);
|
||||||
send_status(fd, STATUS_ERROR);
|
send_status(fd, STATUS_ERROR);
|
||||||
|
*failed = true;
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
file_data = uncompressed;
|
file_data = uncompressed;
|
||||||
@@ -323,25 +394,35 @@ File* receive_incremental_check(int fd, const Config* config, bool* skipped) {
|
|||||||
|
|
||||||
unsigned long long check_size;
|
unsigned long long check_size;
|
||||||
long long check_mtime;
|
long long check_mtime;
|
||||||
|
long long check_mtime_nsec;
|
||||||
uint64_t check_checksum = 0;
|
uint64_t check_checksum = 0;
|
||||||
if (!receive_n_data(fd, &check_size, sizeof(check_size)) ||
|
if (!receive_n_data(fd, &check_size, sizeof(check_size)) ||
|
||||||
!receive_n_data(fd, &check_mtime, sizeof(check_mtime))) {
|
!receive_n_data(fd, &check_mtime, sizeof(check_mtime))) {
|
||||||
free(check_path);
|
free(check_path);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
if (!receive_n_data(fd, &check_mtime_nsec, sizeof(check_mtime_nsec)) || check_mtime_nsec < 0 ||
|
||||||
|
check_mtime_nsec >= 1000000000LL) {
|
||||||
|
free(check_path);
|
||||||
|
send_status(fd, STATUS_ERROR);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
if (config->checksum && !receive_n_data(fd, &check_checksum, sizeof(check_checksum))) {
|
if (config->checksum && !receive_n_data(fd, &check_checksum, sizeof(check_checksum))) {
|
||||||
free(check_path);
|
free(check_path);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (check_size > MAX_RECEIVE_FILE_SIZE) {
|
if (check_size > MAX_RECEIVE_WHOLE_FILE_SIZE) {
|
||||||
free(check_path);
|
free(check_path);
|
||||||
send_status(fd, STATUS_ERROR);
|
send_status(fd, STATUS_ERROR);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (has_path_traversal(check_path)) {
|
if (has_path_traversal(check_path)) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Path traversal detected: %s", check_path);
|
char* escaped_path = output_escape(check_path, log_get_8_bit_output());
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Path traversal detected: %s",
|
||||||
|
escaped_path ? escaped_path : "<allocation failed>");
|
||||||
|
free(escaped_path);
|
||||||
free(check_path);
|
free(check_path);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
@@ -352,6 +433,9 @@ File* receive_incremental_check(int fd, const Config* config, bool* skipped) {
|
|||||||
send_status(fd, STATUS_ERROR);
|
send_status(fd, STATUS_ERROR);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Open the existing destination entry (if any) once and keep the descriptor
|
||||||
|
until the quick-check below decides whether the old contents are needed. */
|
||||||
struct stat st;
|
struct stat st;
|
||||||
bool has_old_file = false;
|
bool has_old_file = false;
|
||||||
int old_fd = -1;
|
int old_fd = -1;
|
||||||
@@ -363,10 +447,35 @@ File* receive_incremental_check(int fd, const Config* config, bool* skipped) {
|
|||||||
close(parent_fd);
|
close(parent_fd);
|
||||||
has_old_file = old_fd >= 0 && fstat(old_fd, &st) == 0 && S_ISREG(st.st_mode);
|
has_old_file = old_fd >= 0 && fstat(old_fd, &st) == 0 && S_ISREG(st.st_mode);
|
||||||
}
|
}
|
||||||
|
if (!has_old_file && old_fd >= 0) {
|
||||||
|
close(old_fd);
|
||||||
|
old_fd = -1;
|
||||||
|
}
|
||||||
unsigned long long old_size = has_old_file ? (unsigned long long)st.st_size : 0;
|
unsigned long long old_size = has_old_file ? (unsigned long long)st.st_size : 0;
|
||||||
|
|
||||||
|
/* Decide from metadata alone whether the receiver already holds the file
|
||||||
|
the sender is offering. The old contents are only read into memory when
|
||||||
|
a checksum comparison or a delta transfer actually requires them. */
|
||||||
|
bool size_equal = has_old_file && old_size == check_size;
|
||||||
|
bool match_by_metadata = false;
|
||||||
|
if (size_equal && !config->ignore_times && !config->size_only) {
|
||||||
|
long long old_mtime_nsec = 0;
|
||||||
|
#ifdef __linux__
|
||||||
|
old_mtime_nsec = st.st_mtim.tv_nsec;
|
||||||
|
#endif
|
||||||
|
match_by_metadata = metadata_mtime_matches(st.st_mtime, old_mtime_nsec, (time_t)check_mtime,
|
||||||
|
(long)check_mtime_nsec, config->modify_window);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool try_delta = config->use_delta && !config->whole_file && has_old_file &&
|
||||||
|
delta_should_attempt(old_size, check_size, config->delta_max_file_size);
|
||||||
|
bool checksum_needs_read = size_equal && !config->ignore_times && config->checksum;
|
||||||
|
bool need_old_data = checksum_needs_read || try_delta;
|
||||||
|
|
||||||
void* old_data = NULL;
|
void* old_data = NULL;
|
||||||
if (has_old_file && old_size > 0 && old_size <= MAX_RECEIVE_FILE_SIZE && old_size <= SIZE_MAX) {
|
if (need_old_data && has_old_file && old_size > 0 && old_size <= MAX_RECEIVE_WHOLE_FILE_SIZE &&
|
||||||
old_data = malloc((size_t)old_size);
|
old_size <= SIZE_MAX) {
|
||||||
|
old_data = protocol_alloc((size_t)old_size);
|
||||||
if (old_data) {
|
if (old_data) {
|
||||||
size_t got = 0;
|
size_t got = 0;
|
||||||
while (got < (size_t)old_size) {
|
while (got < (size_t)old_size) {
|
||||||
@@ -380,65 +489,63 @@ File* receive_incremental_check(int fd, const Config* config, bool* skipped) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (old_fd >= 0) {
|
|
||||||
close(old_fd);
|
|
||||||
}
|
|
||||||
|
|
||||||
bool match = has_old_file && (unsigned long long)st.st_size == check_size;
|
/* Quick-skip decision. If no content comparison is required this is final
|
||||||
if (match && config->checksum) {
|
and the old file was never read; if the read failed the file is not
|
||||||
uint64_t old_checksum = old_size == 0 ? delta_xxhash64("", 0) : 0;
|
skipped and the transfer proceeds with the full new contents. */
|
||||||
if (old_data)
|
bool match = false;
|
||||||
old_checksum = delta_xxhash64(old_data, (size_t)old_size);
|
if (checksum_needs_read) {
|
||||||
match = (old_size == 0 || old_data) && old_checksum == check_checksum;
|
if (old_size == 0)
|
||||||
free(old_data);
|
match = delta_xxhash64("", 0) == check_checksum;
|
||||||
old_data = NULL;
|
else
|
||||||
} else if (match) {
|
match = old_data != NULL && delta_xxhash64(old_data, (size_t)old_size) == check_checksum;
|
||||||
match = (long long)st.st_mtime == check_mtime;
|
} else if (size_equal && !config->ignore_times) {
|
||||||
|
match = config->size_only || match_by_metadata;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (match) {
|
if (match) {
|
||||||
free(old_data);
|
free(old_data);
|
||||||
if (!send_status(fd, STATUS_OK)) {
|
if (!send_status(fd, STATUS_OK)) {
|
||||||
|
close(old_fd);
|
||||||
free(full_path);
|
free(full_path);
|
||||||
free(check_path);
|
free(check_path);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
close(old_fd);
|
||||||
free(full_path);
|
free(full_path);
|
||||||
free(check_path);
|
free(check_path);
|
||||||
*skipped = true;
|
*skipped = true;
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool try_delta = config->use_delta && has_old_file && old_data != NULL &&
|
if (try_delta && old_data != NULL) {
|
||||||
delta_should_attempt(old_size, check_size, config->delta_max_file_size);
|
|
||||||
|
|
||||||
if (try_delta) {
|
|
||||||
bool delta_failed = false;
|
bool delta_failed = false;
|
||||||
File* delta_file =
|
File* delta_file =
|
||||||
receive_delta_file(fd, config, check_path, old_data, old_size, &delta_failed);
|
receive_delta_file(fd, config, check_path, old_data, old_size, &delta_failed);
|
||||||
old_data = NULL; /* receive_delta_file consumes the snapshot on every path */
|
old_data = NULL; /* receive_delta_file consumes the snapshot on every path */
|
||||||
if (delta_file) {
|
if (delta_file) {
|
||||||
|
close(old_fd);
|
||||||
free(full_path);
|
free(full_path);
|
||||||
free(check_path);
|
free(check_path);
|
||||||
return delta_file;
|
return delta_file;
|
||||||
}
|
}
|
||||||
if (delta_failed) {
|
if (delta_failed) {
|
||||||
|
close(old_fd);
|
||||||
free(full_path);
|
free(full_path);
|
||||||
free(check_path);
|
free(check_path);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
free(old_data);
|
|
||||||
old_data = NULL;
|
|
||||||
try_delta = false;
|
|
||||||
}
|
}
|
||||||
|
free(old_data);
|
||||||
|
old_data = NULL;
|
||||||
|
|
||||||
if (!try_delta) {
|
if (!send_status(fd, STATUS_NEXT)) {
|
||||||
if (!send_status(fd, STATUS_NEXT)) {
|
close(old_fd);
|
||||||
free(full_path);
|
free(full_path);
|
||||||
free(check_path);
|
free(check_path);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
close(old_fd);
|
||||||
|
|
||||||
File* file = file_create(check_path);
|
File* file = file_create(check_path);
|
||||||
free(check_path);
|
free(check_path);
|
||||||
@@ -456,14 +563,17 @@ File* receive_incremental_check(int fd, const Config* config, bool* skipped) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Data* file_data = receive_data_limited(fd, MAX_RECEIVE_FILE_SIZE);
|
Data* file_data = receive_data_limited(fd, MAX_RECEIVE_WHOLE_FILE_SIZE);
|
||||||
if (file_data == NULL) {
|
if (file_data == NULL) {
|
||||||
file_destroy(file);
|
file_destroy(file);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (config->use_compression) {
|
if (config->use_compression &&
|
||||||
Data* uncompressed = data_decompress_limited(file_data, MAX_RECEIVE_FILE_SIZE);
|
!compression_should_skip_with_suffixes(file->path, config->skip_compress_suffixes,
|
||||||
|
config->skip_compress_set ? config->skip_compress_count
|
||||||
|
: -1)) {
|
||||||
|
Data* uncompressed = data_decompress_limited(file_data, MAX_RECEIVE_WHOLE_FILE_SIZE);
|
||||||
data_destroy(file_data);
|
data_destroy(file_data);
|
||||||
if (uncompressed == NULL) {
|
if (uncompressed == NULL) {
|
||||||
file_destroy(file);
|
file_destroy(file);
|
||||||
@@ -472,7 +582,6 @@ File* receive_incremental_check(int fd, const Config* config, bool* skipped) {
|
|||||||
if (uncompressed->size > MAX_FILE_DATA_SIZE) {
|
if (uncompressed->size > MAX_FILE_DATA_SIZE) {
|
||||||
data_destroy(uncompressed);
|
data_destroy(uncompressed);
|
||||||
file_destroy(file);
|
file_destroy(file);
|
||||||
send_status(fd, STATUS_ERROR);
|
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
file_data = uncompressed;
|
file_data = uncompressed;
|
||||||
@@ -488,7 +597,10 @@ File* file_receive(const Config* config, int file_descriptor) {
|
|||||||
if (path == NULL)
|
if (path == NULL)
|
||||||
return NULL;
|
return NULL;
|
||||||
if (path[0] == '\0' || has_path_traversal(path)) {
|
if (path[0] == '\0' || has_path_traversal(path)) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Invalid received file path: %s", path);
|
char* escaped_path = output_escape(path, log_get_8_bit_output());
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Invalid received file path: %s",
|
||||||
|
escaped_path ? escaped_path : "<allocation failed>");
|
||||||
|
free(escaped_path);
|
||||||
free(path);
|
free(path);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
@@ -504,13 +616,16 @@ File* file_receive(const Config* config, int file_descriptor) {
|
|||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Data* file_data = receive_data_limited(file_descriptor, MAX_RECEIVE_FILE_SIZE);
|
Data* file_data = receive_data_limited(file_descriptor, MAX_RECEIVE_WHOLE_FILE_SIZE);
|
||||||
if (file_data == NULL) {
|
if (file_data == NULL) {
|
||||||
file_destroy(file);
|
file_destroy(file);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
if (config->use_compression && !compression_should_skip(file->path)) {
|
if (config->use_compression &&
|
||||||
Data* file_data_uncompressed = data_decompress_limited(file_data, MAX_RECEIVE_FILE_SIZE);
|
!compression_should_skip_with_suffixes(file->path, config->skip_compress_suffixes,
|
||||||
|
config->skip_compress_set ? config->skip_compress_count
|
||||||
|
: -1)) {
|
||||||
|
Data* file_data_uncompressed = data_decompress_limited(file_data, MAX_RECEIVE_WHOLE_FILE_SIZE);
|
||||||
data_destroy(file_data);
|
data_destroy(file_data);
|
||||||
if (file_data_uncompressed == NULL) {
|
if (file_data_uncompressed == NULL) {
|
||||||
file_destroy(file);
|
file_destroy(file);
|
||||||
|
|||||||
@@ -10,6 +10,14 @@
|
|||||||
File* file_receive(const Config* config, int file_descriptor);
|
File* file_receive(const Config* config, int file_descriptor);
|
||||||
File* receive_incremental_check(int fd, const Config* config, bool* skipped);
|
File* receive_incremental_check(int fd, const Config* config, bool* skipped);
|
||||||
int receive_manifest(int fd, const Config* config, int* next_status);
|
int receive_manifest(int fd, const Config* config, int* next_status);
|
||||||
|
|
||||||
|
/* Outcome of a single file_save_to_disk operation. The receiver needs to
|
||||||
|
distinguish "written" from "skipped" so --remove-source-files can be told
|
||||||
|
which sources were actually stored. */
|
||||||
|
typedef enum { FILE_SAVE_ERROR = 0, FILE_SAVE_WRITTEN = 1, FILE_SAVE_SKIPPED = 2 } FileSaveResult;
|
||||||
|
|
||||||
|
FileSaveResult file_save_to_disk_full(const char* root_directory, const File* file,
|
||||||
|
const Config* config);
|
||||||
bool file_save_to_disk(const char* root_directory, const File* file, const Config* config);
|
bool file_save_to_disk(const char* root_directory, const File* file, const Config* config);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
+22
-4
@@ -19,12 +19,22 @@
|
|||||||
|
|
||||||
bool file_send_single_calls(File* file, int file_descriptor, bool use_metadata,
|
bool file_send_single_calls(File* file, int file_descriptor, bool use_metadata,
|
||||||
int compression_level, bool send_path) {
|
int compression_level, bool send_path) {
|
||||||
|
return file_send_single_calls_with_skip(file, file_descriptor, use_metadata, compression_level,
|
||||||
|
send_path, NULL, -1, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool file_send_single_calls_with_skip(File* file, int file_descriptor, bool use_metadata,
|
||||||
|
int compression_level, bool send_path,
|
||||||
|
char* const* skip_suffixes, int skip_count,
|
||||||
|
int compression_threads) {
|
||||||
if (!file || !file->path || !file->data || (file->data->size != 0 && !file->data->data))
|
if (!file || !file->path || !file->data || (file->data->size != 0 && !file->data->data))
|
||||||
return false;
|
return false;
|
||||||
const Data* data_to_send = file->data;
|
const Data* data_to_send = file->data;
|
||||||
Data* compressed_data = NULL;
|
Data* compressed_data = NULL;
|
||||||
if (compression_level > 0 && !compression_should_skip(file->path)) {
|
if (compression_level > 0 &&
|
||||||
compressed_data = data_compress(file->data, compression_level);
|
!compression_should_skip_with_suffixes(file->path, skip_suffixes, skip_count)) {
|
||||||
|
compressed_data =
|
||||||
|
data_compress_with_threads(file->data, compression_level, compression_threads);
|
||||||
if (compressed_data == NULL) {
|
if (compressed_data == NULL) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Failed to compress file data");
|
log_message(LOG_LEVEL_ERROR, "Failed to compress file data");
|
||||||
return false;
|
return false;
|
||||||
@@ -49,11 +59,19 @@ bool file_send_single_calls(File* file, int file_descriptor, bool use_metadata,
|
|||||||
|
|
||||||
bool file_send_sendfile(File* file, int file_descriptor, bool use_metadata, int compression_level,
|
bool file_send_sendfile(File* file, int file_descriptor, bool use_metadata, int compression_level,
|
||||||
bool send_path) {
|
bool send_path) {
|
||||||
|
return file_send_sendfile_with_skip(file, file_descriptor, use_metadata, compression_level,
|
||||||
|
send_path, NULL, -1, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool file_send_sendfile_with_skip(File* file, int file_descriptor, bool use_metadata,
|
||||||
|
int compression_level, bool send_path, char* const* skip_suffixes,
|
||||||
|
int skip_count, int compression_threads) {
|
||||||
if (!file || !file->path || !file->data)
|
if (!file || !file->path || !file->data)
|
||||||
return false;
|
return false;
|
||||||
if (compression_level > 0)
|
if (compression_level > 0)
|
||||||
return file_send_single_calls(file, file_descriptor, use_metadata, compression_level,
|
return file_send_single_calls_with_skip(file, file_descriptor, use_metadata, compression_level,
|
||||||
send_path);
|
send_path, skip_suffixes, skip_count,
|
||||||
|
compression_threads);
|
||||||
|
|
||||||
if (send_path && !send_str(file_descriptor, file->path))
|
if (send_path && !send_str(file_descriptor, file->path))
|
||||||
return false;
|
return false;
|
||||||
|
|||||||
@@ -8,7 +8,14 @@
|
|||||||
|
|
||||||
bool file_send_single_calls(File* file, int file_descriptor, bool use_metadata,
|
bool file_send_single_calls(File* file, int file_descriptor, bool use_metadata,
|
||||||
int compression_level, bool send_path);
|
int compression_level, bool send_path);
|
||||||
|
bool file_send_single_calls_with_skip(File* file, int file_descriptor, bool use_metadata,
|
||||||
|
int compression_level, bool send_path,
|
||||||
|
char* const* skip_suffixes, int skip_count,
|
||||||
|
int compression_threads);
|
||||||
bool file_send_sendfile(File* file, int file_descriptor, bool use_metadata, int compression_level,
|
bool file_send_sendfile(File* file, int file_descriptor, bool use_metadata, int compression_level,
|
||||||
bool send_path);
|
bool send_path);
|
||||||
|
bool file_send_sendfile_with_skip(File* file, int file_descriptor, bool use_metadata,
|
||||||
|
int compression_level, bool send_path, char* const* skip_suffixes,
|
||||||
|
int skip_count, int compression_threads);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
@@ -140,7 +140,8 @@ static bool write_all(int fd, const void* data, unsigned long long size) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
bool file_store_write_secure(const char* path, const void* data, unsigned long long data_size,
|
bool file_store_write_secure(const char* path, const void* data, unsigned long long data_size,
|
||||||
bool inplace, bool sparse, const FileMetadata* metadata) {
|
bool inplace, bool sparse, const FileMetadata* metadata,
|
||||||
|
bool preserve_executability) {
|
||||||
char* leaf = NULL;
|
char* leaf = NULL;
|
||||||
int dirfd = file_store_open_secure_parent(path, &leaf);
|
int dirfd = file_store_open_secure_parent(path, &leaf);
|
||||||
if (dirfd < 0)
|
if (dirfd < 0)
|
||||||
@@ -153,7 +154,7 @@ bool file_store_write_secure(const char* path, const void* data, unsigned long l
|
|||||||
if (!sparse || data_size == 0 || ftruncate(fd, (off_t)data_size) == 0)
|
if (!sparse || data_size == 0 || ftruncate(fd, (off_t)data_size) == 0)
|
||||||
ok = write_all(fd, data, data_size);
|
ok = write_all(fd, data, data_size);
|
||||||
if (ok && metadata)
|
if (ok && metadata)
|
||||||
ok = file_restore_metadata_fd(fd, metadata);
|
ok = file_restore_metadata_fd(fd, metadata, preserve_executability);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
int tmp_size = snprintf(NULL, 0, ".%s.tmp.%ld.%u", leaf, (long)getpid(), 99U);
|
int tmp_size = snprintf(NULL, 0, ".%s.tmp.%ld.%u", leaf, (long)getpid(), 99U);
|
||||||
@@ -178,7 +179,7 @@ bool file_store_write_secure(const char* path, const void* data, unsigned long l
|
|||||||
if (ok || (!sparse || data_size == 0))
|
if (ok || (!sparse || data_size == 0))
|
||||||
ok = write_all(fd, data, data_size);
|
ok = write_all(fd, data, data_size);
|
||||||
if (ok && metadata)
|
if (ok && metadata)
|
||||||
ok = file_restore_metadata_fd(fd, metadata);
|
ok = file_restore_metadata_fd(fd, metadata, preserve_executability);
|
||||||
if (close(fd) != 0)
|
if (close(fd) != 0)
|
||||||
ok = false;
|
ok = false;
|
||||||
fd = -1;
|
fd = -1;
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ bool file_store_set_authorized_root(int fd, const char* canonical_path);
|
|||||||
int file_store_open_secure_parent(const char* path, char** leaf_out);
|
int file_store_open_secure_parent(const char* path, char** leaf_out);
|
||||||
bool file_store_rename_secure(const char* old_path, const char* new_path);
|
bool file_store_rename_secure(const char* old_path, const char* new_path);
|
||||||
bool file_store_write_secure(const char* path, const void* data, unsigned long long data_size,
|
bool file_store_write_secure(const char* path, const void* data, unsigned long long data_size,
|
||||||
bool inplace, bool sparse, const FileMetadata* metadata);
|
bool inplace, bool sparse, const FileMetadata* metadata,
|
||||||
|
bool preserve_executability);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
+83
-1
@@ -1,5 +1,6 @@
|
|||||||
#include "log.h"
|
#include "log.h"
|
||||||
#include <errno.h>
|
#include <errno.h>
|
||||||
|
#include <stdbool.h>
|
||||||
#include <stdarg.h>
|
#include <stdarg.h>
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
@@ -7,16 +8,54 @@
|
|||||||
|
|
||||||
static const char* log_level_strings[] = {"DEBUG", "INFO", "WARN", "ERROR"};
|
static const char* log_level_strings[] = {"DEBUG", "INFO", "WARN", "ERROR"};
|
||||||
static LogLevel current_log_level = LOG_LEVEL_WARNING;
|
static LogLevel current_log_level = LOG_LEVEL_WARNING;
|
||||||
|
static uint32_t current_debug_flags = 0;
|
||||||
|
static uint32_t info_flags = 0;
|
||||||
|
static bool info_flags_explicit = false;
|
||||||
static FILE* log_fp = NULL;
|
static FILE* log_fp = NULL;
|
||||||
|
static _Thread_local bool eight_bit_output;
|
||||||
|
static LogStderrMode stderr_mode = LOG_STDERR_ERRORS;
|
||||||
|
|
||||||
void set_log_level(LogLevel level) {
|
void set_log_level(LogLevel level) {
|
||||||
current_log_level = level;
|
current_log_level = level;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void set_log_debug_flags(uint32_t flags) {
|
||||||
|
current_debug_flags = flags;
|
||||||
|
}
|
||||||
|
|
||||||
|
uint32_t get_log_debug_flags(void) {
|
||||||
|
return current_debug_flags;
|
||||||
|
}
|
||||||
|
|
||||||
|
void set_log_info_flags(uint32_t flags) {
|
||||||
|
info_flags = flags;
|
||||||
|
info_flags_explicit = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
uint32_t get_log_info_flags(void) {
|
||||||
|
return info_flags;
|
||||||
|
}
|
||||||
|
|
||||||
void log_set_file(FILE* fp) {
|
void log_set_file(FILE* fp) {
|
||||||
log_fp = fp;
|
log_fp = fp;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void log_set_8_bit_output(bool enabled) {
|
||||||
|
eight_bit_output = enabled;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool log_get_8_bit_output(void) {
|
||||||
|
return eight_bit_output;
|
||||||
|
}
|
||||||
|
|
||||||
|
void log_set_stderr_mode(LogStderrMode mode) {
|
||||||
|
stderr_mode = mode;
|
||||||
|
}
|
||||||
|
|
||||||
|
LogStderrMode log_get_stderr_mode(void) {
|
||||||
|
return stderr_mode;
|
||||||
|
}
|
||||||
|
|
||||||
static inline void write_message(FILE* dest_io, LogLevel log_level, struct tm t, const char* format,
|
static inline void write_message(FILE* dest_io, LogLevel log_level, struct tm t, const char* format,
|
||||||
va_list args) {
|
va_list args) {
|
||||||
fprintf(dest_io, "%04d-%02d-%02d %02d:%02d:%02d [%s]: ", t.tm_year + 1900, t.tm_mon + 1,
|
fprintf(dest_io, "%04d-%02d-%02d %02d:%02d:%02d [%s]: ", t.tm_year + 1900, t.tm_mon + 1,
|
||||||
@@ -37,7 +76,7 @@ void log_message(LogLevel log_level, const char* format, ...) {
|
|||||||
return;
|
return;
|
||||||
|
|
||||||
FILE* dest_io = stdout;
|
FILE* dest_io = stdout;
|
||||||
if (log_level == LOG_LEVEL_ERROR) {
|
if (stderr_mode == LOG_STDERR_ALL || log_level == LOG_LEVEL_ERROR) {
|
||||||
dest_io = stderr;
|
dest_io = stderr;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -53,6 +92,49 @@ void log_message(LogLevel log_level, const char* format, ...) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void log_debug_message(LogDebugFlag flag, const char* format, ...) {
|
||||||
|
if (current_log_level > LOG_LEVEL_DEBUG || !(current_debug_flags & flag))
|
||||||
|
return;
|
||||||
|
|
||||||
|
time_t now = time(NULL);
|
||||||
|
struct tm t;
|
||||||
|
if (!localtime_r(&now, &t))
|
||||||
|
return;
|
||||||
|
|
||||||
|
va_list args;
|
||||||
|
va_start(args, format);
|
||||||
|
write_message(stdout, LOG_LEVEL_DEBUG, t, format, args);
|
||||||
|
va_end(args);
|
||||||
|
|
||||||
|
if (log_fp) {
|
||||||
|
va_start(args, format);
|
||||||
|
write_message(log_fp, LOG_LEVEL_DEBUG, t, format, args);
|
||||||
|
va_end(args);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void log_info_message(LogInfoFlag flag, const char* format, ...) {
|
||||||
|
if ((info_flags_explicit && (info_flags & flag) == 0) ||
|
||||||
|
(!info_flags_explicit && current_log_level > LOG_LEVEL_DEBUG))
|
||||||
|
return;
|
||||||
|
|
||||||
|
time_t now = time(NULL);
|
||||||
|
struct tm t;
|
||||||
|
if (!localtime_r(&now, &t))
|
||||||
|
return;
|
||||||
|
|
||||||
|
va_list args;
|
||||||
|
va_start(args, format);
|
||||||
|
write_message(stdout, LOG_LEVEL_INFO, t, format, args);
|
||||||
|
va_end(args);
|
||||||
|
|
||||||
|
if (log_fp) {
|
||||||
|
va_start(args, format);
|
||||||
|
write_message(log_fp, LOG_LEVEL_INFO, t, format, args);
|
||||||
|
va_end(args);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
void log_perror(const char* context) {
|
void log_perror(const char* context) {
|
||||||
log_message(LOG_LEVEL_ERROR, "%s: %s", context, strerror(errno));
|
log_message(LOG_LEVEL_ERROR, "%s: %s", context, strerror(errno));
|
||||||
}
|
}
|
||||||
@@ -2,12 +2,41 @@
|
|||||||
#define LOG_H
|
#define LOG_H
|
||||||
|
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
|
#include <stdbool.h>
|
||||||
|
#include <stdint.h>
|
||||||
|
|
||||||
typedef enum { LOG_LEVEL_DEBUG, LOG_LEVEL_INFO, LOG_LEVEL_WARNING, LOG_LEVEL_ERROR } LogLevel;
|
typedef enum { LOG_LEVEL_DEBUG, LOG_LEVEL_INFO, LOG_LEVEL_WARNING, LOG_LEVEL_ERROR } LogLevel;
|
||||||
|
typedef enum { LOG_STDERR_ERRORS, LOG_STDERR_ALL } LogStderrMode;
|
||||||
|
|
||||||
|
typedef enum {
|
||||||
|
LOG_DEBUG_IO = 1u << 0,
|
||||||
|
LOG_DEBUG_PROTO = 1u << 1,
|
||||||
|
LOG_DEBUG_PACK = 1u << 2,
|
||||||
|
LOG_DEBUG_UTIL = 1u << 3,
|
||||||
|
LOG_DEBUG_ALL = (1u << 4) - 1,
|
||||||
|
} LogDebugFlag;
|
||||||
|
|
||||||
|
typedef enum {
|
||||||
|
LOG_INFO_COPY = 1u << 0,
|
||||||
|
LOG_INFO_MISC = 1u << 1,
|
||||||
|
LOG_INFO_SKIP = 1u << 2,
|
||||||
|
LOG_INFO_STATS = 1u << 3,
|
||||||
|
LOG_INFO_ALL = LOG_INFO_COPY | LOG_INFO_MISC | LOG_INFO_SKIP | LOG_INFO_STATS,
|
||||||
|
} LogInfoFlag;
|
||||||
|
|
||||||
void log_message(LogLevel log_level, const char* message, ...);
|
void log_message(LogLevel log_level, const char* message, ...);
|
||||||
void log_perror(const char* context);
|
void log_perror(const char* context);
|
||||||
void set_log_level(LogLevel level);
|
void set_log_level(LogLevel level);
|
||||||
|
void set_log_debug_flags(uint32_t flags);
|
||||||
|
uint32_t get_log_debug_flags(void);
|
||||||
|
void log_debug_message(LogDebugFlag flag, const char* message, ...);
|
||||||
|
void set_log_info_flags(uint32_t flags);
|
||||||
|
uint32_t get_log_info_flags(void);
|
||||||
|
void log_info_message(LogInfoFlag flag, const char* message, ...);
|
||||||
void log_set_file(FILE* fp);
|
void log_set_file(FILE* fp);
|
||||||
|
void log_set_8_bit_output(bool enabled);
|
||||||
|
bool log_get_8_bit_output(void);
|
||||||
|
void log_set_stderr_mode(LogStderrMode mode);
|
||||||
|
LogStderrMode log_get_stderr_mode(void);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
+56
-10
@@ -2,6 +2,7 @@
|
|||||||
#include "file.h"
|
#include "file.h"
|
||||||
#include "log.h"
|
#include "log.h"
|
||||||
#include "protocol.h"
|
#include "protocol.h"
|
||||||
|
#include "utils.h"
|
||||||
#include <errno.h>
|
#include <errno.h>
|
||||||
#include <fcntl.h>
|
#include <fcntl.h>
|
||||||
#include <stdint.h>
|
#include <stdint.h>
|
||||||
@@ -24,6 +25,29 @@ typedef char static_assert_mode_t_fits[(sizeof(mode_t) <= sizeof(int32_t)) ? 1 :
|
|||||||
typedef char static_assert_uid_t_fits[(sizeof(uid_t) <= sizeof(int32_t)) ? 1 : -1];
|
typedef char static_assert_uid_t_fits[(sizeof(uid_t) <= sizeof(int32_t)) ? 1 : -1];
|
||||||
typedef char static_assert_gid_t_fits[(sizeof(gid_t) <= sizeof(int32_t)) ? 1 : -1];
|
typedef char static_assert_gid_t_fits[(sizeof(gid_t) <= sizeof(int32_t)) ? 1 : -1];
|
||||||
|
|
||||||
|
bool metadata_mtime_matches(time_t left_sec, long left_nsec, time_t right_sec, long right_nsec,
|
||||||
|
int modify_window) {
|
||||||
|
int64_t left = (int64_t)left_sec;
|
||||||
|
int64_t right = (int64_t)right_sec;
|
||||||
|
int64_t seconds;
|
||||||
|
int64_t nanoseconds;
|
||||||
|
|
||||||
|
if (left > right || (left == right && left_nsec >= right_nsec)) {
|
||||||
|
seconds = left - right;
|
||||||
|
nanoseconds = (int64_t)left_nsec - (int64_t)right_nsec;
|
||||||
|
} else {
|
||||||
|
seconds = right - left;
|
||||||
|
nanoseconds = (int64_t)right_nsec - (int64_t)left_nsec;
|
||||||
|
}
|
||||||
|
if (nanoseconds < 0) {
|
||||||
|
seconds--;
|
||||||
|
nanoseconds += 1000000000LL;
|
||||||
|
}
|
||||||
|
if (modify_window == 0)
|
||||||
|
return left == right;
|
||||||
|
return seconds < modify_window || (seconds == modify_window && nanoseconds == 0);
|
||||||
|
}
|
||||||
|
|
||||||
void metadata_to_buf(char** buf, const FileMetadata* m) {
|
void metadata_to_buf(char** buf, const FileMetadata* m) {
|
||||||
int32_t present = (m != NULL) ? 1 : 0;
|
int32_t present = (m != NULL) ? 1 : 0;
|
||||||
memcpy(*buf, &present, sizeof(present));
|
memcpy(*buf, &present, sizeof(present));
|
||||||
@@ -55,7 +79,7 @@ FileMetadata* metadata_from_buf(char** buf) {
|
|||||||
return NULL;
|
return NULL;
|
||||||
if (!present)
|
if (!present)
|
||||||
return NULL;
|
return NULL;
|
||||||
FileMetadata* m = malloc(sizeof(FileMetadata));
|
FileMetadata* m = protocol_alloc(sizeof(FileMetadata));
|
||||||
if (m == NULL)
|
if (m == NULL)
|
||||||
return NULL;
|
return NULL;
|
||||||
int32_t mode;
|
int32_t mode;
|
||||||
@@ -122,7 +146,7 @@ FileMetadata* metadata_receive(int file_descriptor, int* ok) {
|
|||||||
*ok = 0;
|
*ok = 0;
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
FileMetadata* m = malloc(sizeof(FileMetadata));
|
FileMetadata* m = protocol_alloc(sizeof(FileMetadata));
|
||||||
if (m == NULL) {
|
if (m == NULL) {
|
||||||
if (ok)
|
if (ok)
|
||||||
*ok = 0;
|
*ok = 0;
|
||||||
@@ -179,12 +203,27 @@ FileMetadata* metadata_receive(int file_descriptor, int* ok) {
|
|||||||
return m;
|
return m;
|
||||||
}
|
}
|
||||||
|
|
||||||
void file_restore_metadata(const char* path, const FileMetadata* metadata) {
|
static mode_t metadata_mode(const FileMetadata* metadata, mode_t current_mode,
|
||||||
|
bool preserve_executability) {
|
||||||
|
const mode_t execute_bits = S_IXUSR | S_IXGRP | S_IXOTH;
|
||||||
|
if (preserve_executability)
|
||||||
|
return (current_mode & 0777 & ~execute_bits) | (metadata->mode & execute_bits);
|
||||||
|
return metadata->mode & 0777 & ~(S_IWGRP | S_IWOTH);
|
||||||
|
}
|
||||||
|
|
||||||
|
void file_restore_metadata(const char* path, const FileMetadata* metadata,
|
||||||
|
bool preserve_executability) {
|
||||||
if (metadata == NULL)
|
if (metadata == NULL)
|
||||||
return;
|
return;
|
||||||
mode_t safe_mode = metadata->mode & 0777 & ~(S_IWGRP | S_IWOTH);
|
struct stat current;
|
||||||
if (chmod(path, safe_mode) != 0)
|
mode_t current_mode = stat(path, ¤t) == 0 ? current.st_mode : 0;
|
||||||
log_message(LOG_LEVEL_WARNING, "Failed to chmod %s: %s", path, strerror(errno));
|
mode_t safe_mode = metadata_mode(metadata, current_mode, preserve_executability);
|
||||||
|
if (chmod(path, safe_mode) != 0) {
|
||||||
|
char* escaped_path = output_escape(path, log_get_8_bit_output());
|
||||||
|
log_message(LOG_LEVEL_WARNING, "Failed to chmod %s: %s",
|
||||||
|
escaped_path ? escaped_path : "<allocation failed>", strerror(errno));
|
||||||
|
free(escaped_path);
|
||||||
|
}
|
||||||
/* Never apply client-supplied ownership. The descriptor API below is the
|
/* Never apply client-supplied ownership. The descriptor API below is the
|
||||||
receiver write path; retain this legacy API only for compatibility. */
|
receiver write path; retain this legacy API only for compatibility. */
|
||||||
struct timespec times[2];
|
struct timespec times[2];
|
||||||
@@ -192,15 +231,22 @@ void file_restore_metadata(const char* path, const FileMetadata* metadata) {
|
|||||||
times[0].tv_nsec = UTIME_OMIT;
|
times[0].tv_nsec = UTIME_OMIT;
|
||||||
times[1].tv_sec = metadata->mtime_sec;
|
times[1].tv_sec = metadata->mtime_sec;
|
||||||
times[1].tv_nsec = metadata->mtime_nsec;
|
times[1].tv_nsec = metadata->mtime_nsec;
|
||||||
if (utimensat(AT_FDCWD, path, times, 0) != 0)
|
if (utimensat(AT_FDCWD, path, times, 0) != 0) {
|
||||||
log_message(LOG_LEVEL_WARNING, "Failed to set timestamps on %s: %s", path, strerror(errno));
|
char* escaped_path = output_escape(path, log_get_8_bit_output());
|
||||||
|
log_message(LOG_LEVEL_WARNING, "Failed to set timestamps on %s: %s",
|
||||||
|
escaped_path ? escaped_path : "<allocation failed>", strerror(errno));
|
||||||
|
free(escaped_path);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
bool file_restore_metadata_fd(int fd, const FileMetadata* metadata) {
|
bool file_restore_metadata_fd(int fd, const FileMetadata* metadata, bool preserve_executability) {
|
||||||
if (fd < 0 || metadata == NULL)
|
if (fd < 0 || metadata == NULL)
|
||||||
return metadata == NULL;
|
return metadata == NULL;
|
||||||
bool ok = true;
|
bool ok = true;
|
||||||
mode_t safe_mode = metadata->mode & 0777 & ~(S_IWGRP | S_IWOTH);
|
struct stat current;
|
||||||
|
if (fstat(fd, ¤t) != 0)
|
||||||
|
return false;
|
||||||
|
mode_t safe_mode = metadata_mode(metadata, current.st_mode, preserve_executability);
|
||||||
if (fchmod(fd, safe_mode) != 0)
|
if (fchmod(fd, safe_mode) != 0)
|
||||||
ok = false;
|
ok = false;
|
||||||
/* Client uid/gid values are deliberately not authoritative. */
|
/* Client uid/gid values are deliberately not authoritative. */
|
||||||
|
|||||||
@@ -5,6 +5,7 @@
|
|||||||
#include <stdbool.h>
|
#include <stdbool.h>
|
||||||
#include <stdint.h>
|
#include <stdint.h>
|
||||||
#include <sys/stat.h>
|
#include <sys/stat.h>
|
||||||
|
#include <time.h>
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Wire format (introduced in protocol version 2.0.0):
|
* Wire format (introduced in protocol version 2.0.0):
|
||||||
@@ -29,7 +30,12 @@ void metadata_to_buf(char** buf, const FileMetadata* m);
|
|||||||
FileMetadata* metadata_from_buf(char** buf);
|
FileMetadata* metadata_from_buf(char** buf);
|
||||||
bool metadata_send(int file_descriptor, const FileMetadata* m);
|
bool metadata_send(int file_descriptor, const FileMetadata* m);
|
||||||
FileMetadata* metadata_receive(int file_descriptor, int* ok);
|
FileMetadata* metadata_receive(int file_descriptor, int* ok);
|
||||||
void file_restore_metadata(const char* path, const FileMetadata* metadata);
|
void file_restore_metadata(const char* path, const FileMetadata* metadata,
|
||||||
bool file_restore_metadata_fd(int fd, const FileMetadata* metadata);
|
bool preserve_executability);
|
||||||
|
bool file_restore_metadata_fd(int fd, const FileMetadata* metadata, bool preserve_executability);
|
||||||
|
|
||||||
|
/* Compare timestamps using rsync's whole-second modification window. */
|
||||||
|
bool metadata_mtime_matches(time_t left_sec, long left_nsec, time_t right_sec, long right_nsec,
|
||||||
|
int modify_window);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
@@ -26,9 +26,14 @@ PipelineContextSender* pipeline_context_sender_create(Config* config, Queue* que
|
|||||||
context->scanner_done = false;
|
context->scanner_done = false;
|
||||||
context->loader_done = false;
|
context->loader_done = false;
|
||||||
context->manifest = NULL;
|
context->manifest = NULL;
|
||||||
|
context->remove_source_files = NULL;
|
||||||
|
context->total_files = 0;
|
||||||
context->progress_bytes = 0;
|
context->progress_bytes = 0;
|
||||||
|
context->total_bytes = 0;
|
||||||
context->sender_done = false;
|
context->sender_done = false;
|
||||||
atomic_init(&context->cancelled, false);
|
atomic_init(&context->cancelled, false);
|
||||||
|
protocol_session_init(&context->allocation_session, -1, -1);
|
||||||
|
protocol_session_set_max_alloc(&context->allocation_session, config->max_alloc);
|
||||||
int init = 0;
|
int init = 0;
|
||||||
if (mtx_init(&context->mutex_scanner, mtx_plain) != thrd_success)
|
if (mtx_init(&context->mutex_scanner, mtx_plain) != thrd_success)
|
||||||
goto fail;
|
goto fail;
|
||||||
@@ -76,6 +81,8 @@ void pipeline_context_sender_destroy(PipelineContextSender* context) {
|
|||||||
if (context->manifest) {
|
if (context->manifest) {
|
||||||
array_list_delete(context->manifest);
|
array_list_delete(context->manifest);
|
||||||
}
|
}
|
||||||
|
if (context->remove_source_files)
|
||||||
|
array_list_delete(context->remove_source_files);
|
||||||
config_delete(context->config);
|
config_delete(context->config);
|
||||||
queue_destroy(context->queue_scanner);
|
queue_destroy(context->queue_scanner);
|
||||||
queue_destroy(context->queue_loader);
|
queue_destroy(context->queue_loader);
|
||||||
@@ -98,9 +105,14 @@ PipelineContextReceiver* pipeline_context_receiver_create(Config* config, Queue*
|
|||||||
context->queue = queue;
|
context->queue = queue;
|
||||||
context->file_descriptor = file_descriptor;
|
context->file_descriptor = file_descriptor;
|
||||||
context->ssl = ssl;
|
context->ssl = ssl;
|
||||||
|
context->outcomes.entries = NULL;
|
||||||
|
context->outcomes.count = 0;
|
||||||
|
context->outcomes.capacity = 0;
|
||||||
protocol_session_init(&context->session, file_descriptor, file_descriptor);
|
protocol_session_init(&context->session, file_descriptor, file_descriptor);
|
||||||
protocol_session_set_ssl(&context->session, ssl);
|
protocol_session_set_ssl(&context->session, ssl);
|
||||||
context->receiver_done = false;
|
context->receiver_done = false;
|
||||||
|
context->queued_bytes = 0;
|
||||||
|
context->max_queue_bytes = 0;
|
||||||
atomic_init(&context->cancelled, false);
|
atomic_init(&context->cancelled, false);
|
||||||
int init = 0;
|
int init = 0;
|
||||||
if (mtx_init(&context->mutex, mtx_plain) != thrd_success)
|
if (mtx_init(&context->mutex, mtx_plain) != thrd_success)
|
||||||
@@ -130,20 +142,80 @@ fail:
|
|||||||
void pipeline_context_receiver_destroy(PipelineContextReceiver* context) {
|
void pipeline_context_receiver_destroy(PipelineContextReceiver* context) {
|
||||||
config_delete(context->config);
|
config_delete(context->config);
|
||||||
queue_destroy(context->queue);
|
queue_destroy(context->queue);
|
||||||
|
receiver_outcomes_destroy(&context->outcomes);
|
||||||
mtx_destroy(&context->mutex);
|
mtx_destroy(&context->mutex);
|
||||||
cnd_destroy(&context->condition_not_full);
|
cnd_destroy(&context->condition_not_full);
|
||||||
cnd_destroy(&context->condition_not_empty);
|
cnd_destroy(&context->condition_not_empty);
|
||||||
free(context);
|
free(context);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void pipeline_context_receiver_set_queue_byte_limit(PipelineContextReceiver* context,
|
||||||
|
size_t max_bytes) {
|
||||||
|
if (context == NULL)
|
||||||
|
return;
|
||||||
|
mtx_lock(&context->mutex);
|
||||||
|
context->max_queue_bytes = max_bytes;
|
||||||
|
context->queued_bytes = 0;
|
||||||
|
cnd_broadcast(&context->condition_not_full);
|
||||||
|
mtx_unlock(&context->mutex);
|
||||||
|
}
|
||||||
|
|
||||||
|
void pipeline_context_receiver_note_bytes_released(PipelineContextReceiver* context,
|
||||||
|
size_t released_bytes) {
|
||||||
|
if (context == NULL || context->max_queue_bytes == 0 || released_bytes == 0)
|
||||||
|
return;
|
||||||
|
mtx_lock(&context->mutex);
|
||||||
|
if (released_bytes >= context->queued_bytes)
|
||||||
|
context->queued_bytes = 0;
|
||||||
|
else
|
||||||
|
context->queued_bytes -= released_bytes;
|
||||||
|
cnd_signal(&context->condition_not_full);
|
||||||
|
mtx_unlock(&context->mutex);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool pipeline_context_receiver_enqueue_file(PipelineContextReceiver* context, File* file) {
|
||||||
|
if (context == NULL || file == NULL)
|
||||||
|
return false;
|
||||||
|
size_t file_bytes = file->data ? file->data->size : 0;
|
||||||
|
mtx_lock(&context->mutex);
|
||||||
|
while (!atomic_load(&context->cancelled)) {
|
||||||
|
bool blocked_by_count = queue_is_full(context->queue);
|
||||||
|
bool blocked_by_budget = false;
|
||||||
|
if (context->max_queue_bytes > 0) {
|
||||||
|
size_t budget = context->max_queue_bytes;
|
||||||
|
size_t used = context->queued_bytes;
|
||||||
|
if (used >= budget) {
|
||||||
|
blocked_by_budget = true;
|
||||||
|
} else if (file_bytes > budget - used) {
|
||||||
|
/* A single payload larger than the whole budget (not possible with
|
||||||
|
the per-file receive cap) is only admitted to an empty pipeline so
|
||||||
|
the wait can never deadlock. */
|
||||||
|
blocked_by_budget = used != 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!blocked_by_count && !blocked_by_budget)
|
||||||
|
break;
|
||||||
|
cnd_wait(&context->condition_not_full, &context->mutex);
|
||||||
|
}
|
||||||
|
if (atomic_load(&context->cancelled)) {
|
||||||
|
mtx_unlock(&context->mutex);
|
||||||
|
file_destroy(file);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (!queue_enqueue(context->queue, file)) {
|
||||||
|
mtx_unlock(&context->mutex);
|
||||||
|
file_destroy(file);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
context->queued_bytes += file_bytes;
|
||||||
|
cnd_signal(&context->condition_not_empty);
|
||||||
|
mtx_unlock(&context->mutex);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
static bool receiver_enqueue_file(File* file, void* context_pointer) {
|
static bool receiver_enqueue_file(File* file, void* context_pointer) {
|
||||||
PipelineContextReceiver* context = context_pointer;
|
PipelineContextReceiver* context = (PipelineContextReceiver*)context_pointer;
|
||||||
if (queue_enqueue_multithreaded_cancel(context->queue, file, &context->mutex,
|
return pipeline_context_receiver_enqueue_file(context, file);
|
||||||
&context->condition_not_empty,
|
|
||||||
&context->condition_not_full, &context->cancelled))
|
|
||||||
return true;
|
|
||||||
file_destroy(file);
|
|
||||||
return false;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
static void receiver_thread_fail(PipelineContextReceiver* context) {
|
static void receiver_thread_fail(PipelineContextReceiver* context) {
|
||||||
@@ -163,7 +235,7 @@ int receive_thread(void* pipeline_context) {
|
|||||||
const Config* config = context->config;
|
const Config* config = context->config;
|
||||||
mtx_unlock(&context->mutex);
|
mtx_unlock(&context->mutex);
|
||||||
|
|
||||||
ReceiverSink sink = {receiver_enqueue_file, context, false, false};
|
ReceiverSink sink = {receiver_enqueue_file, context, false, false, NULL};
|
||||||
if (receiver_process((Config*)config, file_descriptor, &sink) != 0) {
|
if (receiver_process((Config*)config, file_descriptor, &sink) != 0) {
|
||||||
receiver_thread_fail(context);
|
receiver_thread_fail(context);
|
||||||
protocol_session_unbind();
|
protocol_session_unbind();
|
||||||
@@ -179,6 +251,7 @@ int receive_thread(void* pipeline_context) {
|
|||||||
|
|
||||||
int write_thread(void* pipeline_context) {
|
int write_thread(void* pipeline_context) {
|
||||||
PipelineContextReceiver* context = (PipelineContextReceiver*)pipeline_context;
|
PipelineContextReceiver* context = (PipelineContextReceiver*)pipeline_context;
|
||||||
|
protocol_session_bind(&context->session);
|
||||||
mtx_lock(&context->mutex);
|
mtx_lock(&context->mutex);
|
||||||
bool save_to_disk = context->config->save_to_disk;
|
bool save_to_disk = context->config->save_to_disk;
|
||||||
char* root_directory = str_dup(context->config->receive_root_directory);
|
char* root_directory = str_dup(context->config->receive_root_directory);
|
||||||
@@ -190,6 +263,7 @@ int write_thread(void* pipeline_context) {
|
|||||||
cnd_broadcast(&context->condition_not_full);
|
cnd_broadcast(&context->condition_not_full);
|
||||||
cnd_broadcast(&context->condition_not_empty);
|
cnd_broadcast(&context->condition_not_empty);
|
||||||
mtx_unlock(&context->mutex);
|
mtx_unlock(&context->mutex);
|
||||||
|
protocol_session_unbind();
|
||||||
return thrd_error;
|
return thrd_error;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -199,10 +273,33 @@ int write_thread(void* pipeline_context) {
|
|||||||
&context->condition_not_full, &context->receiver_done);
|
&context->condition_not_full, &context->receiver_done);
|
||||||
if (file == NULL) {
|
if (file == NULL) {
|
||||||
free(root_directory);
|
free(root_directory);
|
||||||
|
protocol_session_unbind();
|
||||||
return thrd_success;
|
return thrd_success;
|
||||||
}
|
}
|
||||||
if (save_to_disk && !file_save_to_disk(root_directory, file, context->config)) {
|
size_t file_bytes = file->data ? file->data->size : 0;
|
||||||
|
FileSaveResult result = FILE_SAVE_SKIPPED;
|
||||||
|
if (save_to_disk) {
|
||||||
|
result = file_save_to_disk_full(root_directory, file, context->config);
|
||||||
|
if (result == FILE_SAVE_ERROR) {
|
||||||
|
file_destroy(file);
|
||||||
|
pipeline_context_receiver_note_bytes_released(context, file_bytes);
|
||||||
|
mtx_lock(&context->mutex);
|
||||||
|
atomic_store(&context->cancelled, true);
|
||||||
|
context->receiver_done = true;
|
||||||
|
cnd_broadcast(&context->condition_not_full);
|
||||||
|
cnd_broadcast(&context->condition_not_empty);
|
||||||
|
mtx_unlock(&context->mutex);
|
||||||
|
free(root_directory);
|
||||||
|
protocol_session_unbind();
|
||||||
|
return thrd_error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
/* Record the per-file outcome so a --remove-source-files sender learns
|
||||||
|
which sources were actually written versus skipped on the receiver. */
|
||||||
|
if (context->config->remove_source_files &&
|
||||||
|
!receiver_outcomes_append(&context->outcomes, (unsigned char)result)) {
|
||||||
file_destroy(file);
|
file_destroy(file);
|
||||||
|
pipeline_context_receiver_note_bytes_released(context, file_bytes);
|
||||||
mtx_lock(&context->mutex);
|
mtx_lock(&context->mutex);
|
||||||
atomic_store(&context->cancelled, true);
|
atomic_store(&context->cancelled, true);
|
||||||
context->receiver_done = true;
|
context->receiver_done = true;
|
||||||
@@ -210,8 +307,10 @@ int write_thread(void* pipeline_context) {
|
|||||||
cnd_broadcast(&context->condition_not_empty);
|
cnd_broadcast(&context->condition_not_empty);
|
||||||
mtx_unlock(&context->mutex);
|
mtx_unlock(&context->mutex);
|
||||||
free(root_directory);
|
free(root_directory);
|
||||||
|
protocol_session_unbind();
|
||||||
return thrd_error;
|
return thrd_error;
|
||||||
}
|
}
|
||||||
file_destroy(file);
|
file_destroy(file);
|
||||||
|
pipeline_context_receiver_note_bytes_released(context, file_bytes);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -9,6 +9,7 @@
|
|||||||
#include "file.h"
|
#include "file.h"
|
||||||
#include "protocol.h"
|
#include "protocol.h"
|
||||||
#include "queue.h"
|
#include "queue.h"
|
||||||
|
#include "receiver.h"
|
||||||
#include <openssl/ssl.h>
|
#include <openssl/ssl.h>
|
||||||
|
|
||||||
typedef struct {
|
typedef struct {
|
||||||
@@ -24,10 +25,14 @@ typedef struct {
|
|||||||
cnd_t condition_not_empty_loader;
|
cnd_t condition_not_empty_loader;
|
||||||
bool loader_done;
|
bool loader_done;
|
||||||
ArrayList* manifest;
|
ArrayList* manifest;
|
||||||
|
ArrayList* remove_source_files;
|
||||||
mtx_t mutex_progress;
|
mtx_t mutex_progress;
|
||||||
|
int total_files;
|
||||||
unsigned long long progress_bytes;
|
unsigned long long progress_bytes;
|
||||||
|
unsigned long long total_bytes;
|
||||||
bool sender_done;
|
bool sender_done;
|
||||||
atomic_bool cancelled;
|
atomic_bool cancelled;
|
||||||
|
ProtocolSession allocation_session;
|
||||||
} PipelineContextSender;
|
} PipelineContextSender;
|
||||||
|
|
||||||
typedef struct PipelineContextReceiver {
|
typedef struct PipelineContextReceiver {
|
||||||
@@ -36,11 +41,20 @@ typedef struct PipelineContextReceiver {
|
|||||||
int file_descriptor;
|
int file_descriptor;
|
||||||
SSL* ssl;
|
SSL* ssl;
|
||||||
ProtocolSession session;
|
ProtocolSession session;
|
||||||
|
ReceiverOutcomes outcomes;
|
||||||
mtx_t mutex;
|
mtx_t mutex;
|
||||||
cnd_t condition_not_full;
|
cnd_t condition_not_full;
|
||||||
cnd_t condition_not_empty;
|
cnd_t condition_not_empty;
|
||||||
bool receiver_done;
|
bool receiver_done;
|
||||||
atomic_bool cancelled;
|
atomic_bool cancelled;
|
||||||
|
/* Aggregate payload bytes that have been received but not yet released by
|
||||||
|
the disk writer (queued or in the writer's hand). Guarded by `mutex`.
|
||||||
|
When `max_queue_bytes` is non-zero the receiver blocks before enqueuing
|
||||||
|
once this total would exceed it, so decompressed/copied file payloads
|
||||||
|
buffered ahead of a slow disk writer respect the per-connection memory
|
||||||
|
budget instead of growing without bound. */
|
||||||
|
size_t queued_bytes;
|
||||||
|
size_t max_queue_bytes;
|
||||||
} PipelineContextReceiver;
|
} PipelineContextReceiver;
|
||||||
|
|
||||||
PipelineContextSender* pipeline_context_sender_create(Config* config, Queue* queue_scanner,
|
PipelineContextSender* pipeline_context_sender_create(Config* config, Queue* queue_scanner,
|
||||||
@@ -49,6 +63,18 @@ void pipeline_context_sender_destroy(PipelineContextSender* context);
|
|||||||
PipelineContextReceiver* pipeline_context_receiver_create(Config* config, Queue* queue_receiver,
|
PipelineContextReceiver* pipeline_context_receiver_create(Config* config, Queue* queue_receiver,
|
||||||
int file_descriptor, SSL* ssl);
|
int file_descriptor, SSL* ssl);
|
||||||
void pipeline_context_receiver_destroy(PipelineContextReceiver* context);
|
void pipeline_context_receiver_destroy(PipelineContextReceiver* context);
|
||||||
|
/* Bound the bytes buffered ahead of the disk writer (see max_queue_bytes). */
|
||||||
|
void pipeline_context_receiver_set_queue_byte_limit(PipelineContextReceiver* context,
|
||||||
|
size_t max_bytes);
|
||||||
|
/* Blocking enqueue used by the receive pipeline sink. Blocks while the queue
|
||||||
|
is full by element count or when adding `file` would push queued_bytes over
|
||||||
|
the configured byte limit; waits until the disk writer releases bytes.
|
||||||
|
Takes ownership of `file` on success and destroys it on failure/cancel. */
|
||||||
|
bool pipeline_context_receiver_enqueue_file(PipelineContextReceiver* context, File* file);
|
||||||
|
/* Account for `released_bytes` of payload memory that has been freed by the
|
||||||
|
disk writer, unblocking a receiver that is waiting on the byte limit. */
|
||||||
|
void pipeline_context_receiver_note_bytes_released(PipelineContextReceiver* context,
|
||||||
|
size_t released_bytes);
|
||||||
int receive_thread(void* pipeline_context);
|
int receive_thread(void* pipeline_context);
|
||||||
int write_thread(void* pipeline_context);
|
int write_thread(void* pipeline_context);
|
||||||
#endif
|
#endif
|
||||||
+105
-33
@@ -1,5 +1,6 @@
|
|||||||
#include "protocol.h"
|
#include "protocol.h"
|
||||||
#include "log.h"
|
#include "log.h"
|
||||||
|
#include "utils.h"
|
||||||
#include <errno.h>
|
#include <errno.h>
|
||||||
#include <limits.h>
|
#include <limits.h>
|
||||||
#include <openssl/ssl.h>
|
#include <openssl/ssl.h>
|
||||||
@@ -13,13 +14,13 @@
|
|||||||
|
|
||||||
#define RECEIVE_TIMEOUT_SEC 60 /* 60 second per-message timeout */
|
#define RECEIVE_TIMEOUT_SEC 60 /* 60 second per-message timeout */
|
||||||
#define SEND_TIMEOUT_SEC 60
|
#define SEND_TIMEOUT_SEC 60
|
||||||
#define MAX_CONNECTION_MEMORY (256ULL * 1024 * 1024) /* bounded cumulative receive budget */
|
|
||||||
|
|
||||||
static __thread int io_read_fd = -1;
|
static __thread int io_read_fd = -1;
|
||||||
static __thread int io_write_fd = -1;
|
static __thread int io_write_fd = -1;
|
||||||
static __thread SSL* io_ssl;
|
static __thread SSL* io_ssl;
|
||||||
static __thread ProtocolSession* bound_session;
|
static __thread ProtocolSession* bound_session;
|
||||||
static __thread ProtocolSession legacy_io_session = {.read_fd = -1, .write_fd = -1};
|
static __thread ProtocolSession legacy_io_session = {
|
||||||
|
.read_fd = -1, .write_fd = -1, .max_alloc = DEFAULT_MAX_ALLOC};
|
||||||
|
|
||||||
static unsigned long long io_bwlimit = 0;
|
static unsigned long long io_bwlimit = 0;
|
||||||
static mtx_t bw_mutex;
|
static mtx_t bw_mutex;
|
||||||
@@ -27,12 +28,30 @@ static once_flag bw_mutex_once = ONCE_FLAG_INIT;
|
|||||||
|
|
||||||
static unsigned long long global_bwlimit(void);
|
static unsigned long long global_bwlimit(void);
|
||||||
|
|
||||||
|
static bool protocol_reserve_memory(ProtocolSession* session, size_t charge) {
|
||||||
|
unsigned long long allocated = atomic_load(&session->total_allocated_bytes);
|
||||||
|
while (true) {
|
||||||
|
if (allocated > MAX_CONNECTION_MEMORY ||
|
||||||
|
(unsigned long long)charge > MAX_CONNECTION_MEMORY - allocated)
|
||||||
|
return false;
|
||||||
|
if (atomic_compare_exchange_weak(&session->total_allocated_bytes, &allocated,
|
||||||
|
allocated + (unsigned long long)charge))
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static void protocol_release_memory_for_session(ProtocolSession* session, size_t charge) {
|
||||||
|
unsigned long long allocated = atomic_load(&session->total_allocated_bytes);
|
||||||
|
while (true) {
|
||||||
|
unsigned long long remaining = (unsigned long long)charge >= allocated ? 0 : allocated - charge;
|
||||||
|
if (atomic_compare_exchange_weak(&session->total_allocated_bytes, &allocated, remaining))
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
void protocol_release_memory(size_t charge) {
|
void protocol_release_memory(size_t charge) {
|
||||||
ProtocolSession* session = bound_session ? bound_session : &legacy_io_session;
|
ProtocolSession* session = bound_session ? bound_session : &legacy_io_session;
|
||||||
if ((unsigned long long)charge >= session->total_allocated_bytes)
|
protocol_release_memory_for_session(session, charge);
|
||||||
session->total_allocated_bytes = 0;
|
|
||||||
else
|
|
||||||
session->total_allocated_bytes -= charge;
|
|
||||||
}
|
}
|
||||||
void io_set_fds(int read_fd, int write_fd) {
|
void io_set_fds(int read_fd, int write_fd) {
|
||||||
bound_session = NULL;
|
bound_session = NULL;
|
||||||
@@ -44,7 +63,9 @@ void io_set_fds(int read_fd, int write_fd) {
|
|||||||
legacy_io_session.read_fd = read_fd;
|
legacy_io_session.read_fd = read_fd;
|
||||||
legacy_io_session.write_fd = write_fd;
|
legacy_io_session.write_fd = write_fd;
|
||||||
legacy_io_session.ssl = NULL;
|
legacy_io_session.ssl = NULL;
|
||||||
legacy_io_session.total_allocated_bytes = 0;
|
legacy_io_session.eight_bit_output = false;
|
||||||
|
atomic_store(&legacy_io_session.total_allocated_bytes, 0);
|
||||||
|
legacy_io_session.max_alloc = DEFAULT_MAX_ALLOC;
|
||||||
protocol_session_set_bwlimit(&legacy_io_session, global_bwlimit());
|
protocol_session_set_bwlimit(&legacy_io_session, global_bwlimit());
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -54,11 +75,46 @@ void protocol_session_init(ProtocolSession* session, int read_fd, int write_fd)
|
|||||||
memset(session, 0, sizeof(*session));
|
memset(session, 0, sizeof(*session));
|
||||||
session->read_fd = read_fd;
|
session->read_fd = read_fd;
|
||||||
session->write_fd = write_fd;
|
session->write_fd = write_fd;
|
||||||
|
session->max_alloc = DEFAULT_MAX_ALLOC;
|
||||||
|
atomic_init(&session->total_allocated_bytes, 0);
|
||||||
protocol_session_set_bwlimit(session, global_bwlimit());
|
protocol_session_set_bwlimit(session, global_bwlimit());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void protocol_session_set_max_alloc(ProtocolSession* session, unsigned long long max_alloc) {
|
||||||
|
if (!session)
|
||||||
|
session = bound_session ? bound_session : &legacy_io_session;
|
||||||
|
session->max_alloc = max_alloc;
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool allocation_allowed(const ProtocolSession* session, size_t size) {
|
||||||
|
return (unsigned long long)size <= session->max_alloc;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void* protocol_alloc_for_session(const ProtocolSession* session, size_t size) {
|
||||||
|
if (!allocation_allowed(session, size))
|
||||||
|
return NULL;
|
||||||
|
return malloc(size);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void* protocol_realloc_for_session(const ProtocolSession* session, void* ptr, size_t size) {
|
||||||
|
if (!allocation_allowed(session, size))
|
||||||
|
return NULL;
|
||||||
|
return realloc(ptr, size);
|
||||||
|
}
|
||||||
|
|
||||||
|
void* protocol_alloc(size_t size) {
|
||||||
|
const ProtocolSession* session = bound_session ? bound_session : &legacy_io_session;
|
||||||
|
return protocol_alloc_for_session(session, size);
|
||||||
|
}
|
||||||
|
|
||||||
|
void* protocol_realloc(void* ptr, size_t size) {
|
||||||
|
const ProtocolSession* session = bound_session ? bound_session : &legacy_io_session;
|
||||||
|
return protocol_realloc_for_session(session, ptr, size);
|
||||||
|
}
|
||||||
|
|
||||||
void protocol_session_bind(ProtocolSession* session) {
|
void protocol_session_bind(ProtocolSession* session) {
|
||||||
bound_session = session;
|
bound_session = session;
|
||||||
|
log_set_8_bit_output(session && session->eight_bit_output);
|
||||||
}
|
}
|
||||||
|
|
||||||
void protocol_session_unbind(void) {
|
void protocol_session_unbind(void) {
|
||||||
@@ -103,6 +159,19 @@ void protocol_session_set_bwlimit(ProtocolSession* session, unsigned long long b
|
|||||||
session->bw_last_refill_nsec = now.tv_nsec;
|
session->bw_last_refill_nsec = now.tv_nsec;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void protocol_session_set_8_bit_output(ProtocolSession* session, bool enabled) {
|
||||||
|
if (!session)
|
||||||
|
return;
|
||||||
|
session->eight_bit_output = enabled;
|
||||||
|
if (session == bound_session)
|
||||||
|
log_set_8_bit_output(enabled);
|
||||||
|
}
|
||||||
|
|
||||||
|
void protocol_set_8_bit_output(bool enabled) {
|
||||||
|
ProtocolSession* session = bound_session ? bound_session : &legacy_io_session;
|
||||||
|
protocol_session_set_8_bit_output(session, enabled);
|
||||||
|
}
|
||||||
|
|
||||||
static void bw_throttle_session(ProtocolSession* session, size_t bytes_written) {
|
static void bw_throttle_session(ProtocolSession* session, size_t bytes_written) {
|
||||||
if (session->bwlimit == 0)
|
if (session->bwlimit == 0)
|
||||||
return;
|
return;
|
||||||
@@ -153,7 +222,8 @@ static ProtocolSession* legacy_session(int read_fd, int write_fd) {
|
|||||||
legacy_io_session.write_fd != target_write_fd) {
|
legacy_io_session.write_fd != target_write_fd) {
|
||||||
legacy_io_session.read_fd = target_read_fd;
|
legacy_io_session.read_fd = target_read_fd;
|
||||||
legacy_io_session.write_fd = target_write_fd;
|
legacy_io_session.write_fd = target_write_fd;
|
||||||
legacy_io_session.total_allocated_bytes = 0;
|
atomic_store(&legacy_io_session.total_allocated_bytes, 0);
|
||||||
|
legacy_io_session.max_alloc = DEFAULT_MAX_ALLOC;
|
||||||
protocol_session_set_bwlimit(&legacy_io_session, global_bwlimit());
|
protocol_session_set_bwlimit(&legacy_io_session, global_bwlimit());
|
||||||
} else if (legacy_io_session.bwlimit != global_bwlimit()) {
|
} else if (legacy_io_session.bwlimit != global_bwlimit()) {
|
||||||
protocol_session_set_bwlimit(&legacy_io_session, global_bwlimit());
|
protocol_session_set_bwlimit(&legacy_io_session, global_bwlimit());
|
||||||
@@ -184,7 +254,7 @@ static int deadline_remaining_ms(const struct timespec* deadline) {
|
|||||||
bool protocol_send_n_data(ProtocolSession* session, const void* data, size_t data_size) {
|
bool protocol_send_n_data(ProtocolSession* session, const void* data, size_t data_size) {
|
||||||
if (!data && data_size != 0)
|
if (!data && data_size != 0)
|
||||||
return false;
|
return false;
|
||||||
log_message(LOG_LEVEL_DEBUG, " Sending n Data: %zu", data_size);
|
log_debug_message(LOG_DEBUG_IO, " Sending n Data: %zu", data_size);
|
||||||
if (!session)
|
if (!session)
|
||||||
return false;
|
return false;
|
||||||
int fd = session->write_fd;
|
int fd = session->write_fd;
|
||||||
@@ -228,12 +298,12 @@ bool protocol_send_n_data(ProtocolSession* session, const void* data, size_t dat
|
|||||||
if (session->ssl)
|
if (session->ssl)
|
||||||
wait_events = POLLOUT;
|
wait_events = POLLOUT;
|
||||||
}
|
}
|
||||||
log_message(LOG_LEVEL_DEBUG, " Send n Data: %zu", total_bytes_send);
|
log_debug_message(LOG_DEBUG_IO, " Send n Data: %zu", total_bytes_send);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool protocol_receive_n_data(ProtocolSession* session, void* data, size_t data_size) {
|
bool protocol_receive_n_data(ProtocolSession* session, void* data, size_t data_size) {
|
||||||
log_message(LOG_LEVEL_DEBUG, " Receiving n Data: %zu", data_size);
|
log_debug_message(LOG_DEBUG_IO, " Receiving n Data: %zu", data_size);
|
||||||
if (!session)
|
if (!session)
|
||||||
return false;
|
return false;
|
||||||
int fd = session->read_fd;
|
int fd = session->read_fd;
|
||||||
@@ -287,7 +357,7 @@ bool protocol_receive_n_data(ProtocolSession* session, void* data, size_t data_s
|
|||||||
if (session->ssl)
|
if (session->ssl)
|
||||||
wait_events = POLLIN;
|
wait_events = POLLIN;
|
||||||
}
|
}
|
||||||
log_message(LOG_LEVEL_DEBUG, " Received n Data: %zu", total_bytes_received);
|
log_debug_message(LOG_DEBUG_IO, " Received n Data: %zu", total_bytes_received);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -328,7 +398,7 @@ bool protocol_send_str(ProtocolSession* session, const char* data) {
|
|||||||
return false;
|
return false;
|
||||||
if (!protocol_send_n_data(session, data, size))
|
if (!protocol_send_n_data(session, data, size))
|
||||||
return false;
|
return false;
|
||||||
log_message(LOG_LEVEL_DEBUG, "Send String: %s", data);
|
log_debug_message(LOG_DEBUG_PROTO, "Send String: %s", data);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -336,13 +406,12 @@ char* protocol_receive_str(ProtocolSession* session) {
|
|||||||
size_t size;
|
size_t size;
|
||||||
if (!protocol_receive_n_data(session, &size, sizeof(size_t)))
|
if (!protocol_receive_n_data(session, &size, sizeof(size_t)))
|
||||||
return NULL;
|
return NULL;
|
||||||
if (size > MAX_STRING_SIZE || size > SIZE_MAX - 1 ||
|
if (size > MAX_STRING_SIZE || size > SIZE_MAX - 1) {
|
||||||
size + 1 > MAX_CONNECTION_MEMORY - session->total_allocated_bytes) {
|
|
||||||
log_message(LOG_LEVEL_ERROR, "String size %zu exceeds maximum %llu", size,
|
log_message(LOG_LEVEL_ERROR, "String size %zu exceeds maximum %llu", size,
|
||||||
(unsigned long long)MAX_STRING_SIZE);
|
(unsigned long long)MAX_STRING_SIZE);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
char* data = (char*)malloc(size + 1);
|
char* data = (char*)protocol_alloc_for_session(session, size + 1);
|
||||||
if (data == NULL)
|
if (data == NULL)
|
||||||
return NULL;
|
return NULL;
|
||||||
if (!protocol_receive_n_data(session, data, size)) {
|
if (!protocol_receive_n_data(session, data, size)) {
|
||||||
@@ -355,8 +424,7 @@ char* protocol_receive_str(ProtocolSession* session) {
|
|||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
data[size] = '\0';
|
data[size] = '\0';
|
||||||
session->total_allocated_bytes += size + 1;
|
log_debug_message(LOG_DEBUG_PROTO, "Received String: %s", data);
|
||||||
log_message(LOG_LEVEL_DEBUG, "Received String: %s", data);
|
|
||||||
return data;
|
return data;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -370,7 +438,7 @@ bool protocol_send_data(ProtocolSession* session, const Data* data) {
|
|||||||
return false;
|
return false;
|
||||||
if (!protocol_send_n_data(session, data->data, data_size))
|
if (!protocol_send_n_data(session, data->data, data_size))
|
||||||
return false;
|
return false;
|
||||||
log_message(LOG_LEVEL_DEBUG, "Send %lld data", data_size);
|
log_debug_message(LOG_DEBUG_PROTO, "Send %lld data", data_size);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -385,25 +453,29 @@ Data* protocol_receive_data_limited(ProtocolSession* session, unsigned long long
|
|||||||
(unsigned long long)MAX_DATA_PAYLOAD_SIZE);
|
(unsigned long long)MAX_DATA_PAYLOAD_SIZE);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
if (size > SIZE_MAX)
|
||||||
|
return NULL;
|
||||||
size_t allocation_size = size == 0 ? 1 : (size_t)size;
|
size_t allocation_size = size == 0 ? 1 : (size_t)size;
|
||||||
if (allocation_size > MAX_CONNECTION_MEMORY - session->total_allocated_bytes) {
|
if (!protocol_reserve_memory(session, allocation_size)) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Per-connection memory limit exceeded (%llu + %llu > %llu)",
|
log_message(LOG_LEVEL_ERROR, "Per-connection memory limit exceeded (%llu + %llu > %llu)",
|
||||||
(unsigned long long)session->total_allocated_bytes, size,
|
(unsigned long long)atomic_load(&session->total_allocated_bytes), size,
|
||||||
(unsigned long long)MAX_CONNECTION_MEMORY);
|
(unsigned long long)MAX_CONNECTION_MEMORY);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
void* data = malloc(allocation_size);
|
void* data = protocol_alloc_for_session(session, allocation_size);
|
||||||
if (data == NULL)
|
if (data == NULL) {
|
||||||
return NULL;
|
protocol_release_memory_for_session(session, allocation_size);
|
||||||
if (!protocol_receive_n_data(session, data, (size_t)size)) {
|
|
||||||
free(data);
|
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
session->total_allocated_bytes += allocation_size;
|
if (!protocol_receive_n_data(session, data, (size_t)size)) {
|
||||||
log_message(LOG_LEVEL_DEBUG, "Received %lld data", size);
|
free(data);
|
||||||
|
protocol_release_memory_for_session(session, allocation_size);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
log_debug_message(LOG_DEBUG_PROTO, "Received %lld data", size);
|
||||||
Data* result = data_create(data, (size_t)size);
|
Data* result = data_create(data, (size_t)size);
|
||||||
if (!result) {
|
if (!result) {
|
||||||
session->total_allocated_bytes -= allocation_size;
|
protocol_release_memory_for_session(session, allocation_size);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
result->protocol_charge = allocation_size;
|
result->protocol_charge = allocation_size;
|
||||||
@@ -417,28 +489,28 @@ Data* protocol_receive_data(ProtocolSession* session) {
|
|||||||
bool protocol_send_int(ProtocolSession* session, int data) {
|
bool protocol_send_int(ProtocolSession* session, int data) {
|
||||||
if (!protocol_send_n_data(session, &data, sizeof(int)))
|
if (!protocol_send_n_data(session, &data, sizeof(int)))
|
||||||
return false;
|
return false;
|
||||||
log_message(LOG_LEVEL_DEBUG, "Send Int: %d", data);
|
log_debug_message(LOG_DEBUG_PROTO, "Send Int: %d", data);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool protocol_receive_int(ProtocolSession* session, int* data) {
|
bool protocol_receive_int(ProtocolSession* session, int* data) {
|
||||||
if (!protocol_receive_n_data(session, data, sizeof(int)))
|
if (!protocol_receive_n_data(session, data, sizeof(int)))
|
||||||
return false;
|
return false;
|
||||||
log_message(LOG_LEVEL_DEBUG, "Received Int: %d", *data);
|
log_debug_message(LOG_DEBUG_PROTO, "Received Int: %d", *data);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool protocol_send_status(ProtocolSession* session, Status status) {
|
bool protocol_send_status(ProtocolSession* session, Status status) {
|
||||||
if (!protocol_send_n_data(session, &status, sizeof(Status)))
|
if (!protocol_send_n_data(session, &status, sizeof(Status)))
|
||||||
return false;
|
return false;
|
||||||
log_message(LOG_LEVEL_DEBUG, "Send Status: %s", status_to_string(status));
|
log_debug_message(LOG_DEBUG_PROTO, "Send Status: %s", status_to_string(status));
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool protocol_receive_status(ProtocolSession* session, Status* status) {
|
bool protocol_receive_status(ProtocolSession* session, Status* status) {
|
||||||
if (!protocol_receive_n_data(session, status, sizeof(Status)))
|
if (!protocol_receive_n_data(session, status, sizeof(Status)))
|
||||||
return false;
|
return false;
|
||||||
log_message(LOG_LEVEL_DEBUG, "Received Status: %s", status_to_string(*status));
|
log_debug_message(LOG_DEBUG_PROTO, "Received Status: %s", status_to_string(*status));
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+26
-5
@@ -4,20 +4,34 @@
|
|||||||
#include "data.h"
|
#include "data.h"
|
||||||
#include <stdbool.h>
|
#include <stdbool.h>
|
||||||
#include <stddef.h>
|
#include <stddef.h>
|
||||||
|
#include <stdatomic.h>
|
||||||
|
|
||||||
/* Maximum allowed string size for receive_str (64 KB) */
|
/* Maximum allowed string size for receive_str (64 KB) */
|
||||||
#define MAX_STRING_SIZE (64 * 1024)
|
#define MAX_STRING_SIZE (64 * 1024)
|
||||||
|
|
||||||
/* Maximum allowed data payload size for receive_data (100 MB) */
|
/* Maximum uncompressed file payload accepted by the receiver's whole-file
|
||||||
#define MAX_DATA_PAYLOAD_SIZE (100ULL * 1024 * 1024)
|
* paths. A single whole file is charged against the per-connection memory
|
||||||
/* Maximum uncompressed file payload accepted by the receiver. */
|
* reservation (MAX_CONNECTION_MEMORY) and against the server allocation
|
||||||
#define MAX_RECEIVE_FILE_SIZE (64ULL * 1024 * 1024)
|
* ceiling (MAX_SERVER_ALLOC), so this mirrors those 256 MB bounds rather than
|
||||||
|
* the older 64 MB chunk-era cap. Chunk-serialized payloads keep their own
|
||||||
|
* 64 MB cap (MAX_CHUNK_SIZE). */
|
||||||
|
#define MAX_RECEIVE_WHOLE_FILE_SIZE (256ULL * 1024 * 1024)
|
||||||
|
|
||||||
|
/* Maximum allowed data payload size for receive_data (whole-file bound) */
|
||||||
|
#define MAX_DATA_PAYLOAD_SIZE MAX_RECEIVE_WHOLE_FILE_SIZE
|
||||||
|
|
||||||
/* Maximum chunk size (64 MB) — prevents unbounded allocation from the wire */
|
/* Maximum chunk size (64 MB) — prevents unbounded allocation from the wire */
|
||||||
#define MAX_CHUNK_SIZE (64ULL * 1024 * 1024)
|
#define MAX_CHUNK_SIZE (64ULL * 1024 * 1024)
|
||||||
#define MAX_MANIFEST_ENTRIES (1024 * 1024)
|
#define MAX_MANIFEST_ENTRIES (1024 * 1024)
|
||||||
/* Aggregate bytes retained by one received deletion manifest. */
|
/* Aggregate bytes retained by one received deletion manifest. */
|
||||||
#define MAX_MANIFEST_BYTES (16ULL * 1024 * 1024)
|
#define MAX_MANIFEST_BYTES (16ULL * 1024 * 1024)
|
||||||
|
#define DEFAULT_MAX_ALLOC (1ULL * 1024 * 1024 * 1024)
|
||||||
|
/* Server policy ceiling for a client-provided allocation limit. */
|
||||||
|
#define MAX_SERVER_ALLOC (256ULL * 1024 * 1024)
|
||||||
|
/* Bounded cumulative per-connection receive budget. In-flight wire buffers,
|
||||||
|
decompression buffers and queued (not yet written) file payloads for a
|
||||||
|
connection must stay within this ceiling. */
|
||||||
|
#define MAX_CONNECTION_MEMORY (256ULL * 1024 * 1024)
|
||||||
|
|
||||||
typedef struct ssl_st SSL;
|
typedef struct ssl_st SSL;
|
||||||
|
|
||||||
@@ -35,7 +49,9 @@ typedef struct ProtocolSession {
|
|||||||
long long bw_tokens;
|
long long bw_tokens;
|
||||||
long long bw_last_refill_sec;
|
long long bw_last_refill_sec;
|
||||||
long bw_last_refill_nsec;
|
long bw_last_refill_nsec;
|
||||||
unsigned long long total_allocated_bytes;
|
atomic_ullong total_allocated_bytes;
|
||||||
|
bool eight_bit_output;
|
||||||
|
unsigned long long max_alloc;
|
||||||
} ProtocolSession;
|
} ProtocolSession;
|
||||||
|
|
||||||
typedef int Status;
|
typedef int Status;
|
||||||
@@ -65,6 +81,11 @@ void protocol_session_bind(ProtocolSession* session);
|
|||||||
void protocol_session_unbind(void);
|
void protocol_session_unbind(void);
|
||||||
void protocol_session_set_ssl(ProtocolSession* session, SSL* ssl);
|
void protocol_session_set_ssl(ProtocolSession* session, SSL* ssl);
|
||||||
void protocol_session_set_bwlimit(ProtocolSession* session, unsigned long long bytes_per_sec);
|
void protocol_session_set_bwlimit(ProtocolSession* session, unsigned long long bytes_per_sec);
|
||||||
|
void protocol_session_set_max_alloc(ProtocolSession* session, unsigned long long max_alloc);
|
||||||
|
void* protocol_alloc(size_t size);
|
||||||
|
void* protocol_realloc(void* ptr, size_t size);
|
||||||
|
void protocol_session_set_8_bit_output(ProtocolSession* session, bool enabled);
|
||||||
|
void protocol_set_8_bit_output(bool enabled);
|
||||||
bool protocol_send_n_data(ProtocolSession* session, const void* data, size_t data_size);
|
bool protocol_send_n_data(ProtocolSession* session, const void* data, size_t data_size);
|
||||||
bool protocol_receive_n_data(ProtocolSession* session, void* data, size_t data_size);
|
bool protocol_receive_n_data(ProtocolSession* session, void* data, size_t data_size);
|
||||||
bool protocol_send_str(ProtocolSession* session, const char* data);
|
bool protocol_send_str(ProtocolSession* session, const char* data);
|
||||||
|
|||||||
+73
-15
@@ -3,6 +3,7 @@
|
|||||||
#include "utils.h"
|
#include "utils.h"
|
||||||
#include <fcntl.h>
|
#include <fcntl.h>
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
|
#include <stdint.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
#include <sys/socket.h>
|
#include <sys/socket.h>
|
||||||
@@ -15,6 +16,12 @@ typedef struct {
|
|||||||
char* remote_path;
|
char* remote_path;
|
||||||
} RemoteDest;
|
} RemoteDest;
|
||||||
|
|
||||||
|
static void ssh_child_setup_failed(int status_fd) {
|
||||||
|
ssize_t wret = write(status_fd, "x", 1);
|
||||||
|
(void)wret;
|
||||||
|
_exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
static void remote_dest_destroy(RemoteDest* r) {
|
static void remote_dest_destroy(RemoteDest* r) {
|
||||||
free(r->user);
|
free(r->user);
|
||||||
free(r->host);
|
free(r->host);
|
||||||
@@ -68,10 +75,57 @@ static int parse_remote_dest(const char* dest, RemoteDest* r) {
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
Client* client_connect_ssh(const char* destination, int port, const char* server_path) {
|
char* ssh_build_remote_command(const char* server_path, bool old_args) {
|
||||||
|
const char* path = server_path ? server_path : "fastsync-server";
|
||||||
|
const char* suffix = " --stdio";
|
||||||
|
size_t path_len = strlen(path);
|
||||||
|
size_t suffix_len = strlen(suffix);
|
||||||
|
|
||||||
|
if (old_args) {
|
||||||
|
if (path_len > SIZE_MAX - suffix_len - 1)
|
||||||
|
return NULL;
|
||||||
|
char* command = malloc(path_len + suffix_len + 1);
|
||||||
|
if (!command)
|
||||||
|
return NULL;
|
||||||
|
memcpy(command, path, path_len);
|
||||||
|
memcpy(command + path_len, suffix, suffix_len + 1);
|
||||||
|
return command;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Quote the executable as one remote-shell word. This is the default safety boundary. */
|
||||||
|
size_t quote_count = 0;
|
||||||
|
for (const char* p = path; *p; p++)
|
||||||
|
if (*p == '\'')
|
||||||
|
quote_count++;
|
||||||
|
if (path_len > SIZE_MAX - suffix_len - 4 ||
|
||||||
|
quote_count > (SIZE_MAX - path_len - suffix_len - 4) / 4)
|
||||||
|
return NULL;
|
||||||
|
size_t command_len = path_len + quote_count * 4 + suffix_len + 4;
|
||||||
|
char* command = malloc(command_len + 1);
|
||||||
|
if (!command)
|
||||||
|
return NULL;
|
||||||
|
char* out = command;
|
||||||
|
*out++ = '\'';
|
||||||
|
for (const char* p = path; *p; p++) {
|
||||||
|
if (*p == '\'') {
|
||||||
|
memcpy(out, "'\\''", 4);
|
||||||
|
out += 4;
|
||||||
|
} else {
|
||||||
|
*out++ = *p;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
*out++ = '\'';
|
||||||
|
memcpy(out, suffix, suffix_len + 1);
|
||||||
|
return command;
|
||||||
|
}
|
||||||
|
|
||||||
|
Client* client_connect_ssh(const char* destination, int port, const char* server_path,
|
||||||
|
bool old_args) {
|
||||||
RemoteDest r;
|
RemoteDest r;
|
||||||
if (parse_remote_dest(destination, &r) != 0) {
|
if (parse_remote_dest(destination, &r) != 0) {
|
||||||
fprintf(stderr, "Invalid remote destination: %s\n", destination);
|
char* escaped = output_escape(destination, false);
|
||||||
|
fprintf(stderr, "Invalid remote destination: %s\n", escaped ? escaped : "<allocation failed>");
|
||||||
|
free(escaped);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -111,11 +165,12 @@ Client* client_connect_ssh(const char* destination, int port, const char* server
|
|||||||
if (pid == 0) {
|
if (pid == 0) {
|
||||||
close(sv[0]);
|
close(sv[0]);
|
||||||
close(exec_pipe[0]);
|
close(exec_pipe[0]);
|
||||||
fcntl(exec_pipe[1], F_SETFD, FD_CLOEXEC);
|
if (fcntl(exec_pipe[1], F_SETFD, FD_CLOEXEC) < 0)
|
||||||
if (sv[1] != STDIN_FILENO)
|
ssh_child_setup_failed(exec_pipe[1]);
|
||||||
dup2(sv[1], STDIN_FILENO);
|
if (sv[1] != STDIN_FILENO && dup2(sv[1], STDIN_FILENO) < 0)
|
||||||
if (sv[1] != STDOUT_FILENO)
|
ssh_child_setup_failed(exec_pipe[1]);
|
||||||
dup2(sv[1], STDOUT_FILENO);
|
if (sv[1] != STDOUT_FILENO && dup2(sv[1], STDOUT_FILENO) < 0)
|
||||||
|
ssh_child_setup_failed(exec_pipe[1]);
|
||||||
if (sv[1] > 1)
|
if (sv[1] > 1)
|
||||||
close(sv[1]);
|
close(sv[1]);
|
||||||
|
|
||||||
@@ -126,7 +181,7 @@ Client* client_connect_ssh(const char* destination, int port, const char* server
|
|||||||
ssh_user_len = strlen(r.host) + 1;
|
ssh_user_len = strlen(r.host) + 1;
|
||||||
char* ssh_user = malloc(ssh_user_len);
|
char* ssh_user = malloc(ssh_user_len);
|
||||||
if (!ssh_user)
|
if (!ssh_user)
|
||||||
_exit(1);
|
ssh_child_setup_failed(exec_pipe[1]);
|
||||||
if (r.user && r.user[0] != '\0')
|
if (r.user && r.user[0] != '\0')
|
||||||
snprintf(ssh_user, ssh_user_len, "%s@%s", r.user, r.host);
|
snprintf(ssh_user, ssh_user_len, "%s@%s", r.user, r.host);
|
||||||
else
|
else
|
||||||
@@ -135,6 +190,9 @@ Client* client_connect_ssh(const char* destination, int port, const char* server
|
|||||||
char* ssh_argv[16];
|
char* ssh_argv[16];
|
||||||
int ac = 0;
|
int ac = 0;
|
||||||
char port_str[16];
|
char port_str[16];
|
||||||
|
char* remote_command = ssh_build_remote_command(server_path, old_args);
|
||||||
|
if (!remote_command)
|
||||||
|
ssh_child_setup_failed(exec_pipe[1]);
|
||||||
ssh_argv[ac++] = "ssh";
|
ssh_argv[ac++] = "ssh";
|
||||||
ssh_argv[ac++] = "-o";
|
ssh_argv[ac++] = "-o";
|
||||||
ssh_argv[ac++] = "Compression=no";
|
ssh_argv[ac++] = "Compression=no";
|
||||||
@@ -148,14 +206,11 @@ Client* client_connect_ssh(const char* destination, int port, const char* server
|
|||||||
ssh_argv[ac++] = port_str;
|
ssh_argv[ac++] = port_str;
|
||||||
}
|
}
|
||||||
ssh_argv[ac++] = ssh_user;
|
ssh_argv[ac++] = ssh_user;
|
||||||
ssh_argv[ac++] = (char*)(server_path ? server_path : "fastsync-server");
|
ssh_argv[ac++] = remote_command;
|
||||||
ssh_argv[ac++] = "--stdio";
|
|
||||||
ssh_argv[ac] = NULL;
|
ssh_argv[ac] = NULL;
|
||||||
execvp("ssh", ssh_argv);
|
execvp("ssh", ssh_argv);
|
||||||
log_perror("exec of ssh failed");
|
log_perror("exec of ssh failed");
|
||||||
ssize_t wret = write(exec_pipe[1], "x", 1);
|
ssh_child_setup_failed(exec_pipe[1]);
|
||||||
(void)wret;
|
|
||||||
_exit(1);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
close(sv[1]);
|
close(sv[1]);
|
||||||
@@ -165,12 +220,15 @@ Client* client_connect_ssh(const char* destination, int port, const char* server
|
|||||||
ssize_t n = read(exec_pipe[0], &exec_status, 1);
|
ssize_t n = read(exec_pipe[0], &exec_status, 1);
|
||||||
close(exec_pipe[0]);
|
close(exec_pipe[0]);
|
||||||
|
|
||||||
if (n > 0) {
|
if (n != 0) {
|
||||||
close(sv[0]);
|
close(sv[0]);
|
||||||
waitpid(pid, NULL, 0);
|
waitpid(pid, NULL, 0);
|
||||||
remote_dest_destroy(&r);
|
remote_dest_destroy(&r);
|
||||||
|
const char* path = server_path ? server_path : "fastsync-server";
|
||||||
|
char* escaped = output_escape(path, false);
|
||||||
fprintf(stderr, "Error: could not launch '%s --stdio' on remote\n",
|
fprintf(stderr, "Error: could not launch '%s --stdio' on remote\n",
|
||||||
server_path ? server_path : "fastsync-server");
|
escaped ? escaped : "<allocation failed>");
|
||||||
|
free(escaped);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -3,6 +3,8 @@
|
|||||||
|
|
||||||
#include "transport_tcp.h"
|
#include "transport_tcp.h"
|
||||||
|
|
||||||
Client* client_connect_ssh(const char* destination, int port, const char* server_path);
|
Client* client_connect_ssh(const char* destination, int port, const char* server_path,
|
||||||
|
bool old_args);
|
||||||
|
char* ssh_build_remote_command(const char* server_path, bool old_args);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
#include "transport_tcp.h"
|
#include "transport_tcp.h"
|
||||||
#include "log.h"
|
#include "log.h"
|
||||||
#include "protocol.h"
|
#include "protocol.h"
|
||||||
|
#include "utils.h"
|
||||||
#include <arpa/inet.h>
|
#include <arpa/inet.h>
|
||||||
#include <errno.h>
|
#include <errno.h>
|
||||||
#include <netdb.h>
|
#include <netdb.h>
|
||||||
@@ -190,7 +191,10 @@ bool tcp_connect_socket(Client* client, char* host, int port) {
|
|||||||
|
|
||||||
int err = getaddrinfo(host, port_str, &hints, &result);
|
int err = getaddrinfo(host, port_str, &hints, &result);
|
||||||
if (err != 0 || result == NULL) {
|
if (err != 0 || result == NULL) {
|
||||||
fprintf(stderr, "Could not resolve host: %s (%s)\n", host, gai_strerror(err));
|
char* escaped_host = output_escape(host, false);
|
||||||
|
fprintf(stderr, "Could not resolve host: %s (%s)\n",
|
||||||
|
escaped_host ? escaped_host : "<allocation failed>", gai_strerror(err));
|
||||||
|
free(escaped_host);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
#include "log.h"
|
#include "log.h"
|
||||||
#include "protocol.h"
|
#include "protocol.h"
|
||||||
#include "transport_tcp.h"
|
#include "transport_tcp.h"
|
||||||
|
#include "utils.h"
|
||||||
#include <arpa/inet.h>
|
#include <arpa/inet.h>
|
||||||
#include <openssl/err.h>
|
#include <openssl/err.h>
|
||||||
#include <openssl/ssl.h>
|
#include <openssl/ssl.h>
|
||||||
@@ -65,13 +66,19 @@ static SSL_CTX* create_ssl_ctx(bool is_server, const char* cert, const char* key
|
|||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
if (SSL_CTX_use_certificate_file(ctx, cert, SSL_FILETYPE_PEM) <= 0) {
|
if (SSL_CTX_use_certificate_file(ctx, cert, SSL_FILETYPE_PEM) <= 0) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Failed to load certificate: %s", cert);
|
char* escaped = output_escape(cert, false);
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Failed to load certificate: %s",
|
||||||
|
escaped ? escaped : "<allocation failed>");
|
||||||
|
free(escaped);
|
||||||
log_ssl_errors();
|
log_ssl_errors();
|
||||||
SSL_CTX_free(ctx);
|
SSL_CTX_free(ctx);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
if (SSL_CTX_use_PrivateKey_file(ctx, key, SSL_FILETYPE_PEM) <= 0) {
|
if (SSL_CTX_use_PrivateKey_file(ctx, key, SSL_FILETYPE_PEM) <= 0) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Failed to load private key: %s", key);
|
char* escaped = output_escape(key, false);
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Failed to load private key: %s",
|
||||||
|
escaped ? escaped : "<allocation failed>");
|
||||||
|
free(escaped);
|
||||||
log_ssl_errors();
|
log_ssl_errors();
|
||||||
SSL_CTX_free(ctx);
|
SSL_CTX_free(ctx);
|
||||||
return NULL;
|
return NULL;
|
||||||
@@ -85,7 +92,10 @@ static SSL_CTX* create_ssl_ctx(bool is_server, const char* cert, const char* key
|
|||||||
|
|
||||||
if (ca_path) {
|
if (ca_path) {
|
||||||
if (!SSL_CTX_load_verify_locations(ctx, ca_path, NULL)) {
|
if (!SSL_CTX_load_verify_locations(ctx, ca_path, NULL)) {
|
||||||
log_message(LOG_LEVEL_ERROR, "Failed to load CA: %s", ca_path);
|
char* escaped = output_escape(ca_path, false);
|
||||||
|
log_message(LOG_LEVEL_ERROR, "Failed to load CA: %s",
|
||||||
|
escaped ? escaped : "<allocation failed>");
|
||||||
|
free(escaped);
|
||||||
log_ssl_errors();
|
log_ssl_errors();
|
||||||
SSL_CTX_free(ctx);
|
SSL_CTX_free(ctx);
|
||||||
return NULL;
|
return NULL;
|
||||||
|
|||||||
+49
-41
@@ -1,6 +1,6 @@
|
|||||||
#include "utils.h"
|
#include "utils.h"
|
||||||
#include "array_list.h"
|
#include "array_list.h"
|
||||||
#include "libgen.h"
|
#include "log.h"
|
||||||
#include <dirent.h>
|
#include <dirent.h>
|
||||||
#include <errno.h>
|
#include <errno.h>
|
||||||
#include <fcntl.h>
|
#include <fcntl.h>
|
||||||
@@ -82,45 +82,6 @@ static int open_authorized_destination(const char* dest_root) {
|
|||||||
return dirfd;
|
return dirfd;
|
||||||
}
|
}
|
||||||
|
|
||||||
bool mkdir_r(const char* path) {
|
|
||||||
if (!path || *path == '\0')
|
|
||||||
return false;
|
|
||||||
char* duplicate = str_dup(path);
|
|
||||||
if (!duplicate)
|
|
||||||
return false;
|
|
||||||
int dirfd = open(path[0] == '/' ? "/" : ".", O_RDONLY | O_DIRECTORY | O_CLOEXEC | O_NOFOLLOW);
|
|
||||||
if (dirfd < 0) {
|
|
||||||
free(duplicate);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
bool ok = true;
|
|
||||||
char* saveptr = NULL;
|
|
||||||
char* component = strtok_r(duplicate, "/", &saveptr);
|
|
||||||
while (component) {
|
|
||||||
if (strcmp(component, "..") == 0) {
|
|
||||||
ok = false;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
if (strcmp(component, ".") != 0) {
|
|
||||||
int next = openat(dirfd, component, O_RDONLY | O_DIRECTORY | O_CLOEXEC | O_NOFOLLOW);
|
|
||||||
if (next < 0 && errno == ENOENT) {
|
|
||||||
if (mkdirat(dirfd, component, 0755) == 0 || errno == EEXIST)
|
|
||||||
next = openat(dirfd, component, O_RDONLY | O_DIRECTORY | O_CLOEXEC | O_NOFOLLOW);
|
|
||||||
}
|
|
||||||
if (next < 0) {
|
|
||||||
ok = false;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
close(dirfd);
|
|
||||||
dirfd = next;
|
|
||||||
}
|
|
||||||
component = strtok_r(NULL, "/", &saveptr);
|
|
||||||
}
|
|
||||||
close(dirfd);
|
|
||||||
free(duplicate);
|
|
||||||
return ok;
|
|
||||||
}
|
|
||||||
|
|
||||||
char* str_dup(const char* string) {
|
char* str_dup(const char* string) {
|
||||||
if (string == NULL)
|
if (string == NULL)
|
||||||
return NULL;
|
return NULL;
|
||||||
@@ -132,6 +93,32 @@ char* str_dup(const char* string) {
|
|||||||
return new_string;
|
return new_string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
char* output_escape(const char* string, bool eight_bit_output) {
|
||||||
|
if (!string)
|
||||||
|
return NULL;
|
||||||
|
size_t length = strlen(string);
|
||||||
|
if (length > (SIZE_MAX - 1) / 5)
|
||||||
|
return NULL;
|
||||||
|
char* escaped = malloc(length * 5 + 1);
|
||||||
|
if (!escaped)
|
||||||
|
return NULL;
|
||||||
|
size_t out = 0;
|
||||||
|
for (size_t i = 0; i < length; i++) {
|
||||||
|
unsigned char byte = (unsigned char)string[i];
|
||||||
|
if ((byte >= 32 && byte <= 126) || (eight_bit_output && byte >= 128)) {
|
||||||
|
escaped[out++] = (char)byte;
|
||||||
|
} else {
|
||||||
|
escaped[out++] = '\\';
|
||||||
|
escaped[out++] = '#';
|
||||||
|
escaped[out++] = (char)('0' + ((byte >> 6) & 7));
|
||||||
|
escaped[out++] = (char)('0' + ((byte >> 3) & 7));
|
||||||
|
escaped[out++] = (char)('0' + (byte & 7));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
escaped[out] = '\0';
|
||||||
|
return escaped;
|
||||||
|
}
|
||||||
|
|
||||||
/* Match a glob pattern against a string. Supported wildcards:
|
/* Match a glob pattern against a string. Supported wildcards:
|
||||||
* ? matches any single character except '/'.
|
* ? matches any single character except '/'.
|
||||||
* * matches any sequence of characters within one path component (no '/').
|
* * matches any sequence of characters within one path component (no '/').
|
||||||
@@ -187,6 +174,25 @@ bool glob_match(const char* pattern, const char* str) {
|
|||||||
return *str == '\0';
|
return *str == '\0';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool format_human_bytes(unsigned long long bytes, char* buffer, size_t buffer_size) {
|
||||||
|
static const char* const units[] = {"B", "KB", "MB", "GB", "TB", "PB", "EB"};
|
||||||
|
double value = (double)bytes;
|
||||||
|
size_t unit = 0;
|
||||||
|
int written;
|
||||||
|
|
||||||
|
if (!buffer || buffer_size == 0)
|
||||||
|
return false;
|
||||||
|
while (value >= 1024.0 && unit < sizeof(units) / sizeof(units[0]) - 1) {
|
||||||
|
value /= 1024.0;
|
||||||
|
unit++;
|
||||||
|
}
|
||||||
|
if (unit == 0)
|
||||||
|
written = snprintf(buffer, buffer_size, "%llu %s", bytes, units[unit]);
|
||||||
|
else
|
||||||
|
written = snprintf(buffer, buffer_size, "%.1f %s", value, units[unit]);
|
||||||
|
return written >= 0 && (size_t)written < buffer_size;
|
||||||
|
}
|
||||||
|
|
||||||
static bool is_dir_in_manifest(const char* rel_path, ArrayList* manifest) {
|
static bool is_dir_in_manifest(const char* rel_path, ArrayList* manifest) {
|
||||||
size_t len = strlen(rel_path);
|
size_t len = strlen(rel_path);
|
||||||
for (int i = 0; i < manifest->size; i++) {
|
for (int i = 0; i < manifest->size; i++) {
|
||||||
@@ -274,7 +280,9 @@ static bool delete_extras_fd(int dirfd, const char* rel_path, ArrayList* manifes
|
|||||||
} else {
|
} else {
|
||||||
(*deleted_count)++;
|
(*deleted_count)++;
|
||||||
}
|
}
|
||||||
fprintf(stderr, " Deleted: %s\n", child_rel);
|
char* escaped_path = output_escape(child_rel, log_get_8_bit_output());
|
||||||
|
fprintf(stderr, " Deleted: %s\n", escaped_path ? escaped_path : "<allocation failed>");
|
||||||
|
free(escaped_path);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
free(child_rel);
|
free(child_rel);
|
||||||
|
|||||||
+2
-1
@@ -5,8 +5,8 @@
|
|||||||
#include <stddef.h>
|
#include <stddef.h>
|
||||||
#include <stdbool.h>
|
#include <stdbool.h>
|
||||||
|
|
||||||
bool mkdir_r(const char* path);
|
|
||||||
char* str_dup(const char* string);
|
char* str_dup(const char* string);
|
||||||
|
char* output_escape(const char* string, bool eight_bit_output);
|
||||||
char* path_cat(const char* path1, const char* path2);
|
char* path_cat(const char* path1, const char* path2);
|
||||||
bool glob_match(const char* pattern, const char* str);
|
bool glob_match(const char* pattern, const char* str);
|
||||||
bool delete_extras(const char* dest_root, ArrayList* manifest);
|
bool delete_extras(const char* dest_root, ArrayList* manifest);
|
||||||
@@ -17,5 +17,6 @@ bool utils_set_authorized_root(int fd, const char* canonical_path);
|
|||||||
void utils_set_authorized_root_fd(int fd);
|
void utils_set_authorized_root_fd(int fd);
|
||||||
bool has_path_traversal(const char* path);
|
bool has_path_traversal(const char* path);
|
||||||
bool utils_valid_batch_path(const char* path);
|
bool utils_valid_batch_path(const char* path);
|
||||||
|
bool format_human_bytes(unsigned long long bytes, char* buffer, size_t buffer_size);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
@@ -1,4 +1,5 @@
|
|||||||
"""Feature tests: incremental sync, bandwidth limiting, dry run, metadata, filters."""
|
"""Feature tests: incremental sync, bandwidth limiting, dry run, metadata, filters."""
|
||||||
|
import filecmp
|
||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
import sys
|
import sys
|
||||||
@@ -26,6 +27,12 @@ def setup_test_data():
|
|||||||
|
|
||||||
|
|
||||||
class TestDryRun:
|
class TestDryRun:
|
||||||
|
def test_human_readable_dry_run(self):
|
||||||
|
result, dur = run_client(SOURCE_DIR, DEST_DIR, flags=["-h", "--dry-run"])
|
||||||
|
assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}"
|
||||||
|
assert "Total:" in result.stdout
|
||||||
|
assert "KB" in result.stdout
|
||||||
|
|
||||||
def test_dry_run(self):
|
def test_dry_run(self):
|
||||||
clean_dir(DEST_DIR)
|
clean_dir(DEST_DIR)
|
||||||
result, dur = run_client(
|
result, dur = run_client(
|
||||||
@@ -35,6 +42,118 @@ class TestDryRun:
|
|||||||
assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}"
|
assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}"
|
||||||
assert "Dry run:" in result.stdout, f"No dry run output: {result.stdout[:200]}"
|
assert "Dry run:" in result.stdout, f"No dry run output: {result.stdout[:200]}"
|
||||||
|
|
||||||
|
def test_quiet_suppresses_dry_run_output(self):
|
||||||
|
result, dur = run_client(
|
||||||
|
SOURCE_DIR, DEST_DIR,
|
||||||
|
flags=["-q", "-n", "--progress", "--stats"],
|
||||||
|
)
|
||||||
|
assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}"
|
||||||
|
assert result.stdout == ""
|
||||||
|
assert result.stderr == ""
|
||||||
|
|
||||||
|
def test_quiet_preserves_errors(self):
|
||||||
|
result, dur = run_client(
|
||||||
|
SOURCE_DIR, DEST_DIR,
|
||||||
|
flags=["--quiet", "--server-port", "1"],
|
||||||
|
)
|
||||||
|
assert result.returncode != 0
|
||||||
|
assert result.stderr != ""
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("flags", [["-q", "-v"], ["-v", "-q"]])
|
||||||
|
def test_quiet_successful_transfer_and_verbose_order(self, shared_server, flags):
|
||||||
|
clean_dir(DEST_DIR)
|
||||||
|
result, dur = run_client(
|
||||||
|
SOURCE_DIR, DEST_DIR,
|
||||||
|
flags=flags,
|
||||||
|
port=shared_server.port,
|
||||||
|
)
|
||||||
|
assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}"
|
||||||
|
assert result.stdout == ""
|
||||||
|
assert result.stderr == ""
|
||||||
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
||||||
|
mismatches, missing = verify_transfer(SOURCE_DIR, received)
|
||||||
|
assert not missing, f"Missing: {missing}"
|
||||||
|
assert not mismatches, f"Mismatch: {mismatches}"
|
||||||
|
|
||||||
|
|
||||||
|
class TestRemoveSourceFiles:
|
||||||
|
def test_removes_only_transferred_regular_files(self, shared_server):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "remove_source")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "remove_dest")
|
||||||
|
clean_dir(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
with open(os.path.join(source, "one.txt"), "wb") as f:
|
||||||
|
f.write(b"one")
|
||||||
|
with open(os.path.join(source, "two.txt"), "wb") as f:
|
||||||
|
f.write(b"two")
|
||||||
|
os.makedirs(os.path.join(source, "directory"))
|
||||||
|
os.symlink("one.txt", os.path.join(source, "link.txt"))
|
||||||
|
|
||||||
|
result, _ = run_client(source, dest, flags=["--remove-source-files", "-m"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"Remove-source sync failed: {result.stderr[:200]}"
|
||||||
|
assert not os.path.exists(os.path.join(source, "one.txt"))
|
||||||
|
assert not os.path.exists(os.path.join(source, "two.txt"))
|
||||||
|
assert os.path.isdir(os.path.join(source, "directory"))
|
||||||
|
assert os.path.islink(os.path.join(source, "link.txt"))
|
||||||
|
|
||||||
|
def test_single_threaded_removes_transferred_file(self, shared_server):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "remove_single_source")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "remove_single_dest")
|
||||||
|
clean_dir(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
source_file = os.path.join(source, "file.txt")
|
||||||
|
with open(source_file, "wb") as f:
|
||||||
|
f.write(b"single threaded")
|
||||||
|
|
||||||
|
result, _ = run_client(source, dest, flags=["--remove-source-files"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0
|
||||||
|
assert not os.path.exists(source_file)
|
||||||
|
|
||||||
|
def test_dry_run_preserves_source_files(self):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "remove_dry_source")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "remove_dry_dest")
|
||||||
|
clean_dir(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
source_file = os.path.join(source, "file.txt")
|
||||||
|
with open(source_file, "wb") as f:
|
||||||
|
f.write(b"keep")
|
||||||
|
|
||||||
|
result, _ = run_client(source, dest, flags=["--remove-source-files", "--dry-run"])
|
||||||
|
assert result.returncode == 0
|
||||||
|
assert os.path.isfile(source_file)
|
||||||
|
|
||||||
|
def test_failed_connection_preserves_source_files(self):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "remove_failed_source")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "remove_failed_dest")
|
||||||
|
clean_dir(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
source_file = os.path.join(source, "file.txt")
|
||||||
|
with open(source_file, "wb") as f:
|
||||||
|
f.write(b"keep after failure")
|
||||||
|
|
||||||
|
result, _ = run_client(source, dest, flags=["--remove-source-files"], port=1)
|
||||||
|
assert result.returncode != 0
|
||||||
|
assert os.path.isfile(source_file)
|
||||||
|
|
||||||
|
def test_incremental_skip_preserves_source_file(self, shared_server):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "remove_skipped_source")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "remove_skipped_dest")
|
||||||
|
clean_dir(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
source_file = os.path.join(source, "file.txt")
|
||||||
|
with open(source_file, "wb") as f:
|
||||||
|
f.write(b"keep after skip")
|
||||||
|
|
||||||
|
result, _ = run_client(source, dest, port=shared_server.port)
|
||||||
|
assert result.returncode == 0
|
||||||
|
result, _ = run_client(source, dest,
|
||||||
|
flags=["--remove-source-files", "--incremental"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0
|
||||||
|
assert os.path.isfile(source_file)
|
||||||
|
|
||||||
|
|
||||||
class TestArchiveMode:
|
class TestArchiveMode:
|
||||||
def test_archive_mode(self, shared_server):
|
def test_archive_mode(self, shared_server):
|
||||||
@@ -51,6 +170,40 @@ class TestArchiveMode:
|
|||||||
assert not missing, f"Missing: {missing}"
|
assert not missing, f"Missing: {missing}"
|
||||||
assert not mismatches, f"Mismatch: {mismatches}"
|
assert not mismatches, f"Mismatch: {mismatches}"
|
||||||
|
|
||||||
|
def test_archive_implied_options_can_be_negated(self, shared_server):
|
||||||
|
clean_dir(DEST_DIR)
|
||||||
|
result, dur = run_client(
|
||||||
|
SOURCE_DIR, DEST_DIR,
|
||||||
|
flags=["--archive", "--no-compress", "--no-m", "--no-preserve"],
|
||||||
|
port=shared_server.port,
|
||||||
|
)
|
||||||
|
if result.returncode != 0:
|
||||||
|
pytest.fail(f"Exit {result.returncode}: {(result.stderr or result.stdout)[:200]}")
|
||||||
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
||||||
|
mismatches, missing = verify_transfer(SOURCE_DIR, received)
|
||||||
|
assert not missing, f"Missing: {missing}"
|
||||||
|
assert not mismatches, f"Mismatch: {mismatches}"
|
||||||
|
|
||||||
|
|
||||||
|
class TestExecutability:
|
||||||
|
def test_preserves_only_executable_bits(self, shared_server):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "executability_source")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "executability_dest")
|
||||||
|
clean_dir(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
source_file = os.path.join(source, "tool.sh")
|
||||||
|
with open(source_file, "w") as f:
|
||||||
|
f.write("#!/bin/sh\necho test\n")
|
||||||
|
os.chmod(source_file, 0o751)
|
||||||
|
|
||||||
|
result, _ = run_client(source, dest, flags=["-E"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"Executability sync failed: {result.stderr[:200]}"
|
||||||
|
received_file = os.path.join(get_dest_received_dir(dest, source), "tool.sh")
|
||||||
|
received_mode = os.stat(received_file).st_mode
|
||||||
|
assert received_mode & 0o111 == 0o111
|
||||||
|
assert received_mode & 0o600 == 0o600
|
||||||
|
assert received_mode & 0o077 == 0o011
|
||||||
|
|
||||||
|
|
||||||
class TestChmod:
|
class TestChmod:
|
||||||
def test_chmod_applies_to_transferred_files(self, shared_server):
|
def test_chmod_applies_to_transferred_files(self, shared_server):
|
||||||
@@ -68,6 +221,86 @@ class TestChmod:
|
|||||||
assert (os.stat(os.path.join(received, "small.txt")).st_mode & 0o777) == 0o644
|
assert (os.stat(os.path.join(received, "small.txt")).st_mode & 0o777) == 0o644
|
||||||
|
|
||||||
|
|
||||||
|
class TestCompressionChoice:
|
||||||
|
def test_zstd_choice_compresses(self, shared_server):
|
||||||
|
clean_dir(DEST_DIR)
|
||||||
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["--zc", "zstd"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"zstd sync failed: {(result.stderr or result.stdout)[:200]}"
|
||||||
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
||||||
|
mismatches, missing = verify_transfer(SOURCE_DIR, received)
|
||||||
|
assert not missing, f"Missing: {missing}"
|
||||||
|
assert not mismatches, f"Mismatch: {mismatches}"
|
||||||
|
|
||||||
|
def test_none_choice_disables_compression(self, shared_server):
|
||||||
|
clean_dir(DEST_DIR)
|
||||||
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-z", "--compress-choice", "none"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"none sync failed: {(result.stderr or result.stdout)[:200]}"
|
||||||
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
||||||
|
mismatches, missing = verify_transfer(SOURCE_DIR, received)
|
||||||
|
assert not missing, f"Missing: {missing}"
|
||||||
|
assert not mismatches, f"Mismatch: {mismatches}"
|
||||||
|
|
||||||
|
|
||||||
|
class TestSkipCompress:
|
||||||
|
def test_skip_compress_case_insensitive(self, shared_server):
|
||||||
|
clean_dir(DEST_DIR)
|
||||||
|
with open(os.path.join(SOURCE_DIR, "skip-case.TXT"), "wb") as f:
|
||||||
|
f.write((b"skip compression case test\n" * 100))
|
||||||
|
result, _ = run_client(
|
||||||
|
SOURCE_DIR, DEST_DIR,
|
||||||
|
flags=["-c", "--skip-compress=.txt"],
|
||||||
|
port=shared_server.port,
|
||||||
|
)
|
||||||
|
assert result.returncode == 0, f"Skip-compress sync failed: {(result.stderr or result.stdout)[:200]}"
|
||||||
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
||||||
|
with open(os.path.join(received, "skip-case.TXT"), "rb") as f:
|
||||||
|
assert f.read() == b"skip compression case test\n" * 100
|
||||||
|
|
||||||
|
def test_skip_compress_empty_list(self, shared_server):
|
||||||
|
clean_dir(DEST_DIR)
|
||||||
|
result, _ = run_client(
|
||||||
|
SOURCE_DIR, DEST_DIR,
|
||||||
|
flags=["-c", "--skip-compress="],
|
||||||
|
port=shared_server.port,
|
||||||
|
)
|
||||||
|
assert result.returncode == 0, f"Empty skip-compress sync failed: {(result.stderr or result.stdout)[:200]}"
|
||||||
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
||||||
|
mismatches, missing = verify_transfer(SOURCE_DIR, received)
|
||||||
|
assert not missing, f"Missing: {missing}"
|
||||||
|
assert not mismatches, f"Mismatch: {mismatches}"
|
||||||
|
|
||||||
|
def test_skip_compress_incremental_full_fallback(self, shared_server):
|
||||||
|
clean_dir(DEST_DIR)
|
||||||
|
path = os.path.join(SOURCE_DIR, "incremental-skip.TXT")
|
||||||
|
with open(path, "wb") as f:
|
||||||
|
f.write(b"original skipped content\n")
|
||||||
|
flags = ["-c", "-M", "--skip-compress=.txt"]
|
||||||
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=flags, port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"Initial sync failed: {(result.stderr or result.stdout)[:200]}"
|
||||||
|
with open(path, "wb") as f:
|
||||||
|
f.write(b"updated skipped content\n")
|
||||||
|
result, _ = run_client(
|
||||||
|
SOURCE_DIR, DEST_DIR,
|
||||||
|
flags=flags + ["--incremental"],
|
||||||
|
port=shared_server.port,
|
||||||
|
)
|
||||||
|
assert result.returncode == 0, f"Incremental sync failed: {(result.stderr or result.stdout)[:200]}"
|
||||||
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
||||||
|
with open(os.path.join(received, "incremental-skip.TXT"), "rb") as f:
|
||||||
|
assert f.read() == b"updated skipped content\n"
|
||||||
|
|
||||||
|
def test_skip_compress_rejects_chunk_serialization(self, shared_server):
|
||||||
|
result, _ = run_client(
|
||||||
|
SOURCE_DIR, DEST_DIR,
|
||||||
|
flags=["-c", "-s", "--skip-compress=.txt"],
|
||||||
|
port=shared_server.port,
|
||||||
|
)
|
||||||
|
assert result.returncode != 0
|
||||||
|
assert "cannot be combined" in (result.stderr or result.stdout)
|
||||||
|
|
||||||
|
|
||||||
class TestExclude:
|
class TestExclude:
|
||||||
def test_exclude_single(self, shared_server):
|
def test_exclude_single(self, shared_server):
|
||||||
clean_dir(DEST_DIR)
|
clean_dir(DEST_DIR)
|
||||||
@@ -229,6 +462,225 @@ class TestIncremental:
|
|||||||
with open(received_file, "rb") as f:
|
with open(received_file, "rb") as f:
|
||||||
assert f.read() == b"hello world\n"
|
assert f.read() == b"hello world\n"
|
||||||
|
|
||||||
|
def test_size_only_skips_same_size_with_different_mtime(self, shared_server):
|
||||||
|
clean_dir(DEST_DIR)
|
||||||
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0
|
||||||
|
|
||||||
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
||||||
|
received_file = os.path.join(received, "small.txt")
|
||||||
|
with open(received_file, "wb") as f:
|
||||||
|
f.write(b"different!!\n")
|
||||||
|
os.utime(received_file, (time.time() - 3600, time.time() - 3600))
|
||||||
|
|
||||||
|
result, _ = run_client(
|
||||||
|
SOURCE_DIR, DEST_DIR,
|
||||||
|
flags=["-M", "--incremental", "--size-only"],
|
||||||
|
port=shared_server.port,
|
||||||
|
)
|
||||||
|
assert result.returncode == 0, f"Size-only sync failed: {result.stderr[:200]}"
|
||||||
|
with open(received_file, "rb") as f:
|
||||||
|
assert f.read() == b"different!!\n"
|
||||||
|
|
||||||
|
def test_ignore_times_transfers_same_size_and_mtime(self, shared_server):
|
||||||
|
clean_dir(DEST_DIR)
|
||||||
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0
|
||||||
|
|
||||||
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
||||||
|
source_file = os.path.join(SOURCE_DIR, "small.txt")
|
||||||
|
received_file = os.path.join(received, "small.txt")
|
||||||
|
source_stat = os.stat(source_file)
|
||||||
|
with open(received_file, "wb") as f:
|
||||||
|
f.write(b"stale data!\n")
|
||||||
|
os.utime(received_file, (source_stat.st_atime, source_stat.st_mtime))
|
||||||
|
|
||||||
|
result, _ = run_client(SOURCE_DIR, DEST_DIR,
|
||||||
|
flags=["-M", "--incremental", "--ignore-times"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"Ignore-times sync failed: {result.stderr[:200]}"
|
||||||
|
with open(received_file, "rb") as f:
|
||||||
|
assert f.read() == b"hello world\n"
|
||||||
|
|
||||||
|
def test_modify_window_allows_subsecond_mtime_difference(self, shared_server):
|
||||||
|
clean_dir(DEST_DIR)
|
||||||
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0
|
||||||
|
|
||||||
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
||||||
|
source_file = os.path.join(SOURCE_DIR, "small.txt")
|
||||||
|
received_file = os.path.join(received, "small.txt")
|
||||||
|
source_stat = os.stat(source_file)
|
||||||
|
with open(received_file, "wb") as f:
|
||||||
|
f.write(b"modified!!!\n")
|
||||||
|
os.utime(received_file, ns=(source_stat.st_atime_ns,
|
||||||
|
source_stat.st_mtime_ns - 1500000000))
|
||||||
|
|
||||||
|
result, _ = run_client(SOURCE_DIR, DEST_DIR,
|
||||||
|
flags=["-M", "--incremental", "--modify-window=2"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"Modify-window sync failed: {result.stderr[:200]}"
|
||||||
|
with open(received_file, "rb") as f:
|
||||||
|
assert f.read() == b"modified!!!\n"
|
||||||
|
|
||||||
|
def test_whole_file_disables_delta_and_keeps_compression(self, shared_server):
|
||||||
|
clean_dir(DEST_DIR)
|
||||||
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0
|
||||||
|
|
||||||
|
source_file = os.path.join(SOURCE_DIR, "medium.txt")
|
||||||
|
with open(source_file, "wb") as f:
|
||||||
|
f.write(b"whole-file replacement\n" * 5000)
|
||||||
|
|
||||||
|
result, _ = run_client(
|
||||||
|
SOURCE_DIR,
|
||||||
|
DEST_DIR,
|
||||||
|
flags=["-M", "--incremental", "--delta", "-W", "-c"],
|
||||||
|
port=shared_server.port,
|
||||||
|
)
|
||||||
|
assert result.returncode == 0, f"Whole-file sync failed: {(result.stderr or result.stdout)[:200]}"
|
||||||
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
||||||
|
mismatches, missing = verify_transfer(SOURCE_DIR, received)
|
||||||
|
assert not missing, f"Missing: {missing}"
|
||||||
|
assert not mismatches, f"Mismatch: {mismatches}"
|
||||||
|
|
||||||
|
|
||||||
|
class TestUpdate:
|
||||||
|
def test_update_skips_older_destination_and_allows_equal_or_newer_source(self, shared_server):
|
||||||
|
clean_dir(DEST_DIR)
|
||||||
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-u"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0
|
||||||
|
|
||||||
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
||||||
|
source_file = os.path.join(SOURCE_DIR, "small.txt")
|
||||||
|
received_file = os.path.join(received, "small.txt")
|
||||||
|
source_stat = os.stat(source_file)
|
||||||
|
|
||||||
|
with open(received_file, "wb") as f:
|
||||||
|
f.write(b"newer destination\n")
|
||||||
|
os.utime(received_file, ns=(source_stat.st_atime_ns, source_stat.st_mtime_ns + 10_000_000_000))
|
||||||
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-u"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0
|
||||||
|
with open(received_file, "rb") as f:
|
||||||
|
assert f.read() == b"newer destination\n"
|
||||||
|
|
||||||
|
os.utime(received_file, ns=(source_stat.st_atime_ns, source_stat.st_mtime_ns))
|
||||||
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-u"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0
|
||||||
|
with open(received_file, "rb") as f:
|
||||||
|
assert f.read() == b"hello world\n"
|
||||||
|
|
||||||
|
def test_update_skips_unreadable_newer_destination(self, shared_server):
|
||||||
|
clean_dir(DEST_DIR)
|
||||||
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-u"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0
|
||||||
|
|
||||||
|
received_file = os.path.join(get_dest_received_dir(DEST_DIR, SOURCE_DIR), "small.txt")
|
||||||
|
source_stat = os.stat(os.path.join(SOURCE_DIR, "small.txt"))
|
||||||
|
with open(received_file, "wb") as f:
|
||||||
|
f.write(b"protected destination\n")
|
||||||
|
os.utime(received_file, ns=(source_stat.st_atime_ns, source_stat.st_mtime_ns + 10_000_000_000))
|
||||||
|
original_mode = os.stat(received_file).st_mode
|
||||||
|
try:
|
||||||
|
os.chmod(received_file, 0)
|
||||||
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-u"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0
|
||||||
|
os.chmod(received_file, original_mode)
|
||||||
|
with open(received_file, "rb") as f:
|
||||||
|
assert f.read() == b"protected destination\n"
|
||||||
|
finally:
|
||||||
|
os.chmod(received_file, original_mode)
|
||||||
|
|
||||||
|
with open(received_file, "wb") as f:
|
||||||
|
f.write(b"older destination\n")
|
||||||
|
os.utime(received_file, ns=(source_stat.st_atime_ns, source_stat.st_mtime_ns - 10_000_000_000))
|
||||||
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-u"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0
|
||||||
|
with open(received_file, "rb") as f:
|
||||||
|
assert f.read() == b"hello world\n"
|
||||||
|
|
||||||
|
|
||||||
|
class TestExisting:
|
||||||
|
def test_existing_updates_existing_and_skips_new(self, shared_server):
|
||||||
|
clean_dir(DEST_DIR)
|
||||||
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"Initial sync failed: {(result.stderr or result.stdout)[:200]}"
|
||||||
|
|
||||||
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
||||||
|
source_file = os.path.join(SOURCE_DIR, "small.txt")
|
||||||
|
new_source_file = os.path.join(SOURCE_DIR, "new-existing-test.txt")
|
||||||
|
with open(source_file, "wb") as f:
|
||||||
|
f.write(b"updated existing content\n")
|
||||||
|
with open(new_source_file, "wb") as f:
|
||||||
|
f.write(b"this file must not be created\n")
|
||||||
|
|
||||||
|
try:
|
||||||
|
result, _ = run_client(SOURCE_DIR, DEST_DIR,
|
||||||
|
flags=["-M", "--existing"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"--existing sync failed: {(result.stderr or result.stdout)[:200]}"
|
||||||
|
|
||||||
|
with open(os.path.join(received, "small.txt"), "rb") as f:
|
||||||
|
assert f.read() == b"updated existing content\n"
|
||||||
|
assert not os.path.exists(os.path.join(received, "new-existing-test.txt"))
|
||||||
|
finally:
|
||||||
|
os.unlink(new_source_file)
|
||||||
|
with open(source_file, "wb") as f:
|
||||||
|
f.write(b"hello world\n")
|
||||||
|
|
||||||
|
|
||||||
|
class TestIgnoreExisting:
|
||||||
|
def test_ignore_existing_preserves_existing_and_transfers_new(self, shared_server):
|
||||||
|
clean_dir(DEST_DIR)
|
||||||
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, port=shared_server.port)
|
||||||
|
assert result.returncode == 0
|
||||||
|
|
||||||
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
||||||
|
existing_file = os.path.join(received, "small.txt")
|
||||||
|
with open(existing_file, "wb") as f:
|
||||||
|
f.write(b"destination content\n")
|
||||||
|
new_source = os.path.join(SOURCE_DIR, "new.txt")
|
||||||
|
try:
|
||||||
|
with open(new_source, "wb") as f:
|
||||||
|
f.write(b"new file\n")
|
||||||
|
|
||||||
|
result, _ = run_client(SOURCE_DIR, DEST_DIR,
|
||||||
|
flags=["--ignore-existing"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"Sync failed: {(result.stderr or result.stdout)[:200]}"
|
||||||
|
with open(existing_file, "rb") as f:
|
||||||
|
assert f.read() == b"destination content\n"
|
||||||
|
with open(os.path.join(received, "new.txt"), "rb") as f:
|
||||||
|
assert f.read() == b"new file\n"
|
||||||
|
finally:
|
||||||
|
if os.path.lexists(new_source):
|
||||||
|
os.unlink(new_source)
|
||||||
|
|
||||||
|
|
||||||
|
class TestIgnoreExisting:
|
||||||
|
def test_ignore_existing_preserves_existing_and_transfers_new(self, shared_server):
|
||||||
|
clean_dir(DEST_DIR)
|
||||||
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, port=shared_server.port)
|
||||||
|
assert result.returncode == 0
|
||||||
|
|
||||||
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
||||||
|
existing_file = os.path.join(received, "small.txt")
|
||||||
|
with open(existing_file, "wb") as f:
|
||||||
|
f.write(b"destination content\n")
|
||||||
|
new_source = os.path.join(SOURCE_DIR, "new.txt")
|
||||||
|
try:
|
||||||
|
with open(new_source, "wb") as f:
|
||||||
|
f.write(b"new file\n")
|
||||||
|
|
||||||
|
result, _ = run_client(SOURCE_DIR, DEST_DIR,
|
||||||
|
flags=["--ignore-existing"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"Sync failed: {(result.stderr or result.stdout)[:200]}"
|
||||||
|
with open(existing_file, "rb") as f:
|
||||||
|
assert f.read() == b"destination content\n"
|
||||||
|
with open(os.path.join(received, "new.txt"), "rb") as f:
|
||||||
|
assert f.read() == b"new file\n"
|
||||||
|
finally:
|
||||||
|
if os.path.lexists(new_source):
|
||||||
|
os.unlink(new_source)
|
||||||
|
|
||||||
|
|
||||||
class TestDelete:
|
class TestDelete:
|
||||||
def test_delete_removes_extra_files(self, shared_server):
|
def test_delete_removes_extra_files(self, shared_server):
|
||||||
@@ -279,6 +731,75 @@ class TestProgress:
|
|||||||
assert "Sent " in output and "MB" in output, "--progress produced no stable byte marker"
|
assert "Sent " in output and "MB" in output, "--progress produced no stable byte marker"
|
||||||
assert "Done." in output, "--progress did not report completion"
|
assert "Done." in output, "--progress did not report completion"
|
||||||
|
|
||||||
|
def test_human_readable_stats(self, shared_server):
|
||||||
|
clean_dir(DEST_DIR)
|
||||||
|
result, dur = run_client(
|
||||||
|
SOURCE_DIR, DEST_DIR,
|
||||||
|
flags=["-h", "--stats"],
|
||||||
|
port=shared_server.port,
|
||||||
|
)
|
||||||
|
assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}"
|
||||||
|
assert "Stats:" in result.stderr
|
||||||
|
assert "KB" in result.stderr
|
||||||
|
|
||||||
|
def test_human_readable_progress_multithreaded(self, shared_server):
|
||||||
|
clean_dir(DEST_DIR)
|
||||||
|
result, dur = run_client(
|
||||||
|
SOURCE_DIR, DEST_DIR,
|
||||||
|
flags=["-m", "-h", "--progress"],
|
||||||
|
port=shared_server.port,
|
||||||
|
)
|
||||||
|
assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}"
|
||||||
|
output = result.stdout + result.stderr
|
||||||
|
assert "Sent " in output
|
||||||
|
assert "KB" in output
|
||||||
|
assert "Done." in output
|
||||||
|
|
||||||
|
|
||||||
|
class TestInfo:
|
||||||
|
def test_info_copy_reports_transfers(self, shared_server):
|
||||||
|
clean_dir(DEST_DIR)
|
||||||
|
result, _ = run_client(
|
||||||
|
SOURCE_DIR, DEST_DIR,
|
||||||
|
flags=["--info=copy"],
|
||||||
|
port=shared_server.port,
|
||||||
|
)
|
||||||
|
assert result.returncode == 0, f"Info sync failed: {(result.stderr or result.stdout)[:200]}"
|
||||||
|
output = result.stdout + result.stderr
|
||||||
|
assert "[INFO]" in output and "Transferring" in output
|
||||||
|
|
||||||
|
def test_info_stats_reports_multithreaded_transfer(self, shared_server):
|
||||||
|
clean_dir(DEST_DIR)
|
||||||
|
result, _ = run_client(
|
||||||
|
SOURCE_DIR, DEST_DIR,
|
||||||
|
flags=["-m", "--info=stats"],
|
||||||
|
port=shared_server.port,
|
||||||
|
)
|
||||||
|
assert result.returncode == 0, f"Info stats sync failed: {(result.stderr or result.stdout)[:200]}"
|
||||||
|
output = result.stdout + result.stderr
|
||||||
|
assert "[INFO]" in output and "Transfer summary:" in output
|
||||||
|
|
||||||
|
def test_info_rejects_unknown_flag(self):
|
||||||
|
result, _ = run_client(
|
||||||
|
SOURCE_DIR, DEST_DIR,
|
||||||
|
flags=["--info=unknown"],
|
||||||
|
)
|
||||||
|
assert result.returncode != 0
|
||||||
|
assert "unsupported --info flag" in result.stderr
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("flags", [
|
||||||
|
["--info=none", "--verbose"],
|
||||||
|
["--verbose", "--info=none"],
|
||||||
|
])
|
||||||
|
def test_info_none_suppresses_verbose_info(self, shared_server, flags):
|
||||||
|
clean_dir(DEST_DIR)
|
||||||
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=flags, port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"Info sync failed: {(result.stderr or result.stdout)[:200]}"
|
||||||
|
output = result.stdout + result.stderr
|
||||||
|
assert "[INFO]" not in output
|
||||||
|
assert "Transferring" not in output
|
||||||
|
assert "Transfer summary:" not in output
|
||||||
|
|
||||||
|
|
||||||
class TestBandwidthLimit:
|
class TestBandwidthLimit:
|
||||||
def test_bwlimit_runs(self, shared_server):
|
def test_bwlimit_runs(self, shared_server):
|
||||||
@@ -293,3 +814,391 @@ class TestBandwidthLimit:
|
|||||||
mismatches, missing = verify_transfer(SOURCE_DIR, received)
|
mismatches, missing = verify_transfer(SOURCE_DIR, received)
|
||||||
assert not missing, f"Missing: {missing}"
|
assert not missing, f"Missing: {missing}"
|
||||||
assert not mismatches, f"Mismatch: {mismatches}"
|
assert not mismatches, f"Mismatch: {mismatches}"
|
||||||
|
|
||||||
|
|
||||||
|
def _read_file(path):
|
||||||
|
with open(path, "rb") as fh:
|
||||||
|
return fh.read()
|
||||||
|
|
||||||
|
|
||||||
|
class TestRemoveSourceFilesSkips:
|
||||||
|
"""--remove-source-files must not delete sources the receiver skipped
|
||||||
|
(rsync reference behavior)."""
|
||||||
|
|
||||||
|
def test_existing_first_sync_keeps_new_source(self, shared_server):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "remove_rsf_existing_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "remove_rsf_existing_dst")
|
||||||
|
clean_dir(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
with open(os.path.join(source, "only.txt"), "wb") as f:
|
||||||
|
f.write(b"keep me")
|
||||||
|
|
||||||
|
result, _ = run_client(source, dest, flags=["--remove-source-files", "--existing"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"Sync failed: {result.stderr[:200]}"
|
||||||
|
# The file exists only on the source side, so --existing makes the
|
||||||
|
# receiver skip it; the source must therefore not be removed.
|
||||||
|
assert os.path.isfile(os.path.join(source, "only.txt"))
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
assert not os.path.exists(os.path.join(received, "only.txt"))
|
||||||
|
|
||||||
|
def test_ignore_existing_keeps_skipped_source(self, shared_server):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "remove_rsf_ignore_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "remove_rsf_ignore_dst")
|
||||||
|
clean_dir(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
source_file = os.path.join(source, "file.txt")
|
||||||
|
with open(source_file, "wb") as f:
|
||||||
|
f.write(b"payload")
|
||||||
|
|
||||||
|
result, _ = run_client(source, dest, port=shared_server.port)
|
||||||
|
assert result.returncode == 0
|
||||||
|
|
||||||
|
result, _ = run_client(source, dest, flags=["--remove-source-files", "--ignore-existing"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"Sync failed: {result.stderr[:200]}"
|
||||||
|
# Destination already has the file, so the second run is a receiver
|
||||||
|
# skip; the source file must survive.
|
||||||
|
assert os.path.isfile(source_file)
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
assert _read_file(os.path.join(received, "file.txt")) == b"payload"
|
||||||
|
|
||||||
|
def test_update_newer_destination_keeps_source(self, shared_server):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "remove_rsf_update_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "remove_rsf_update_dst")
|
||||||
|
clean_dir(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
source_file = os.path.join(source, "file.txt")
|
||||||
|
with open(source_file, "wb") as f:
|
||||||
|
f.write(b"source payload")
|
||||||
|
|
||||||
|
result, _ = run_client(source, dest, port=shared_server.port)
|
||||||
|
assert result.returncode == 0
|
||||||
|
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
received_file = os.path.join(received, "file.txt")
|
||||||
|
with open(received_file, "wb") as f:
|
||||||
|
f.write(b"newer destination payload")
|
||||||
|
os.utime(received_file, ns=(time.time_ns() + 10**9, time.time_ns() + 10**9))
|
||||||
|
|
||||||
|
result, _ = run_client(source, dest, flags=["--remove-source-files", "--update"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"Sync failed: {result.stderr[:200]}"
|
||||||
|
# --update skips a destination that is newer than the source, so the
|
||||||
|
# source must not be removed.
|
||||||
|
assert os.path.isfile(source_file)
|
||||||
|
assert _read_file(received_file) == b"newer destination payload"
|
||||||
|
|
||||||
|
def test_multithreaded_ignore_existing_keeps_skipped_source(self, shared_server):
|
||||||
|
"""The multithreaded writer path must also report per-file outcomes so a
|
||||||
|
--remove-source-files sender does not delete skipped sources."""
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "remove_rsf_mt_ignore_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "remove_rsf_mt_ignore_dst")
|
||||||
|
clean_dir(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
source_file = os.path.join(source, "file.txt")
|
||||||
|
with open(source_file, "wb") as f:
|
||||||
|
f.write(b"payload")
|
||||||
|
|
||||||
|
result, _ = run_client(source, dest, port=shared_server.port)
|
||||||
|
assert result.returncode == 0
|
||||||
|
|
||||||
|
result, _ = run_client(source, dest,
|
||||||
|
flags=["--remove-source-files", "--ignore-existing", "-m"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"Sync failed: {result.stderr[:200]}"
|
||||||
|
# Destination already has the file, so the receiver (writer thread)
|
||||||
|
# skips it; the source must survive.
|
||||||
|
assert os.path.isfile(source_file)
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
assert _read_file(os.path.join(received, "file.txt")) == b"payload"
|
||||||
|
|
||||||
|
|
||||||
|
class TestBackup:
|
||||||
|
def _sync(self, source, dest, flags, port):
|
||||||
|
return run_client(source, dest, flags=flags, port=port)
|
||||||
|
|
||||||
|
def test_plain_backup_keeps_previous_version(self, shared_server):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "backup_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "backup_dst")
|
||||||
|
clean_dir(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
source_file = os.path.join(source, "f.txt")
|
||||||
|
with open(source_file, "wb") as f:
|
||||||
|
f.write(b"AAAA")
|
||||||
|
|
||||||
|
result, _ = self._sync(source, dest, ["--backup"], shared_server.port)
|
||||||
|
assert result.returncode == 0, f"Backup sync failed: {result.stderr[:200]}"
|
||||||
|
|
||||||
|
with open(source_file, "wb") as f:
|
||||||
|
f.write(b"BBBB")
|
||||||
|
result, _ = self._sync(source, dest, ["--backup"], shared_server.port)
|
||||||
|
assert result.returncode == 0, f"Backup sync failed: {result.stderr[:200]}"
|
||||||
|
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
assert _read_file(os.path.join(received, "f.txt")) == b"BBBB"
|
||||||
|
# rsync default suffix "~" keeps the overwritten version.
|
||||||
|
assert _read_file(os.path.join(received, "f.txt~")) == b"AAAA"
|
||||||
|
|
||||||
|
def test_backup_custom_suffix(self, shared_server):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "backup_suffix_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "backup_suffix_dst")
|
||||||
|
clean_dir(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
source_file = os.path.join(source, "f.txt")
|
||||||
|
with open(source_file, "wb") as f:
|
||||||
|
f.write(b"AAAA")
|
||||||
|
|
||||||
|
flags = ["--backup", "--suffix", ".bak"]
|
||||||
|
result, _ = self._sync(source, dest, flags, shared_server.port)
|
||||||
|
assert result.returncode == 0, f"Backup sync failed: {result.stderr[:200]}"
|
||||||
|
with open(source_file, "wb") as f:
|
||||||
|
f.write(b"BBBB")
|
||||||
|
result, _ = self._sync(source, dest, flags, shared_server.port)
|
||||||
|
assert result.returncode == 0, f"Backup sync failed: {result.stderr[:200]}"
|
||||||
|
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
assert _read_file(os.path.join(received, "f.txt")) == b"BBBB"
|
||||||
|
assert _read_file(os.path.join(received, "f.txt.bak")) == b"AAAA"
|
||||||
|
|
||||||
|
def test_backup_dir_stores_backups_separately(self, shared_server):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "backup_dir_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "backup_dir_dst")
|
||||||
|
clean_dir(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
source_file = os.path.join(source, "f.txt")
|
||||||
|
with open(source_file, "wb") as f:
|
||||||
|
f.write(b"AAAA")
|
||||||
|
|
||||||
|
flags = ["--backup", "--backup-dir", "backups"]
|
||||||
|
result, _ = self._sync(source, dest, flags, shared_server.port)
|
||||||
|
assert result.returncode == 0, f"Backup sync failed: {result.stderr[:200]}"
|
||||||
|
with open(source_file, "wb") as f:
|
||||||
|
f.write(b"BBBB")
|
||||||
|
result, _ = self._sync(source, dest, flags, shared_server.port)
|
||||||
|
assert result.returncode == 0, f"Backup sync failed: {result.stderr[:200]}"
|
||||||
|
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
assert _read_file(os.path.join(received, "f.txt")) == b"BBBB"
|
||||||
|
backup = os.path.join(dest, "backups", os.path.relpath(source_file, os.path.sep))
|
||||||
|
assert _read_file(backup) == b"AAAA"
|
||||||
|
|
||||||
|
|
||||||
|
class TestPartialDir:
|
||||||
|
def test_completed_transfer_installed_in_destination(self, shared_server):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "partial_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "partial_dst")
|
||||||
|
clean_dir(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
source_file = os.path.join(source, "f.txt")
|
||||||
|
with open(source_file, "wb") as f:
|
||||||
|
f.write(b"partial payload")
|
||||||
|
|
||||||
|
result, _ = run_client(source, dest, flags=["--partial", "--partial-dir", ".partial"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"Partial sync failed: {result.stderr[:200]}"
|
||||||
|
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
assert _read_file(os.path.join(received, "f.txt")) == b"partial payload"
|
||||||
|
# A completed transfer must not remain under the partial directory.
|
||||||
|
partial = os.path.join(dest, ".partial", os.path.relpath(source_file, os.path.sep))
|
||||||
|
assert not os.path.exists(partial)
|
||||||
|
|
||||||
|
|
||||||
|
class TestLargeFile:
|
||||||
|
def test_transfer_100mb_file(self, shared_server):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "large_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "large_dst")
|
||||||
|
clean_dir(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
source_file = os.path.join(source, "big.bin")
|
||||||
|
chunk = os.urandom(1024 * 1024)
|
||||||
|
with open(source_file, "wb") as f:
|
||||||
|
for _ in range(100):
|
||||||
|
f.write(chunk)
|
||||||
|
|
||||||
|
result, _ = run_client(source, dest, port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"Large-file sync failed: {result.stderr[:200]}"
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
assert filecmp.cmp(source_file, os.path.join(received, "big.bin"), shallow=False)
|
||||||
|
|
||||||
|
|
||||||
|
def _source_files():
|
||||||
|
"""All source paths (absolute) that a transfer would send right now."""
|
||||||
|
return [
|
||||||
|
os.path.join(root, name)
|
||||||
|
for root, _dirs, names in os.walk(SOURCE_DIR)
|
||||||
|
for name in names
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
class TestListOnly:
|
||||||
|
"""--list-only prints every transfer candidate and changes nothing."""
|
||||||
|
|
||||||
|
def test_list_only_prints_each_file_and_does_not_transfer(self):
|
||||||
|
clean_dir(DEST_DIR)
|
||||||
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["--list-only"])
|
||||||
|
assert result.returncode == 0, f"list-only failed: {result.stderr[:200]}"
|
||||||
|
for full_path in _source_files():
|
||||||
|
assert full_path in result.stdout, f"list-only omitted {full_path}"
|
||||||
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
||||||
|
assert not os.path.exists(received), "list-only wrote to the destination"
|
||||||
|
|
||||||
|
def test_list_only_with_dry_run_does_not_error(self):
|
||||||
|
clean_dir(DEST_DIR)
|
||||||
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["--list-only", "--dry-run"])
|
||||||
|
assert result.returncode == 0, f"list-only -n failed: {result.stderr[:200]}"
|
||||||
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
||||||
|
assert not os.path.exists(received)
|
||||||
|
|
||||||
|
def test_list_only_multithreaded(self):
|
||||||
|
clean_dir(DEST_DIR)
|
||||||
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["--list-only", "-m"])
|
||||||
|
assert result.returncode == 0, f"list-only -m failed: {result.stderr[:200]}"
|
||||||
|
for full_path in _source_files():
|
||||||
|
assert full_path in result.stdout, f"list-only -m omitted {full_path}"
|
||||||
|
received = get_dest_received_dir(DEST_DIR, SOURCE_DIR)
|
||||||
|
assert not os.path.exists(received), "list-only -m wrote to the destination"
|
||||||
|
|
||||||
|
|
||||||
|
class TestItemizeChanges:
|
||||||
|
"""-i/--itemize-changes prints rsync-style lines only for files sent."""
|
||||||
|
|
||||||
|
def test_first_run_prints_sent_lines(self, shared_server):
|
||||||
|
clean_dir(DEST_DIR)
|
||||||
|
result, _ = run_client(SOURCE_DIR, DEST_DIR,
|
||||||
|
flags=["-M", "-i"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"itemize sync failed: {result.stderr[:200]}"
|
||||||
|
sent_lines = {">f+++++++++ " + p for p in _source_files()}
|
||||||
|
assert sent_lines <= set(result.stdout.splitlines()), (
|
||||||
|
f"missing itemize lines; got {result.stdout[:500]}"
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_incremental_second_run_prints_no_line_for_unchanged(self, shared_server):
|
||||||
|
clean_dir(DEST_DIR)
|
||||||
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-M"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}"
|
||||||
|
result, _ = run_client(SOURCE_DIR, DEST_DIR,
|
||||||
|
flags=["-M", "-i", "--incremental"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"incremental itemize failed: {result.stderr[:200]}"
|
||||||
|
itemized = [line for line in result.stdout.splitlines() if line and line[0] in ">.<c"]
|
||||||
|
assert itemized == [], f"unchanged files were itemized: {itemized[:5]}"
|
||||||
|
|
||||||
|
def test_multithreaded_emits_same_itemize_lines(self, shared_server):
|
||||||
|
clean_dir(DEST_DIR)
|
||||||
|
result, _ = run_client(SOURCE_DIR, DEST_DIR,
|
||||||
|
flags=["-M", "-i", "-m"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"itemize -m sync failed: {result.stderr[:200]}"
|
||||||
|
sent_lines = {">f+++++++++ " + p for p in _source_files()}
|
||||||
|
assert sent_lines <= set(result.stdout.splitlines()), (
|
||||||
|
f"missing itemize lines in -m mode; got {result.stdout[:500]}"
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_dry_run_with_itemize_does_not_error(self):
|
||||||
|
clean_dir(DEST_DIR)
|
||||||
|
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-i", "--dry-run"])
|
||||||
|
assert result.returncode == 0, f"dry-run -i failed: {result.stderr[:200]}"
|
||||||
|
|
||||||
|
def test_changed_file_on_second_incremental_run_prints_exactly_one_line(self, shared_server):
|
||||||
|
"""A changed file itemizes exactly once on an incremental rerun while
|
||||||
|
unchanged files print nothing (no double emission)."""
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "itemize_change_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "itemize_change_dst")
|
||||||
|
clean_dir(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
changed = os.path.join(source, "changed.txt")
|
||||||
|
untouched = os.path.join(source, "untouched.txt")
|
||||||
|
with open(changed, "wb") as fh:
|
||||||
|
fh.write(b"original\n")
|
||||||
|
with open(untouched, "wb") as fh:
|
||||||
|
fh.write(b"stable\n")
|
||||||
|
|
||||||
|
result, _ = run_client(source, dest, flags=["-M"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}"
|
||||||
|
|
||||||
|
with open(changed, "wb") as fh:
|
||||||
|
fh.write(b"edited payload\n")
|
||||||
|
|
||||||
|
result, _ = run_client(source, dest,
|
||||||
|
flags=["-M", "-i", "--incremental"],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"incremental itemize failed: {result.stderr[:200]}"
|
||||||
|
itemized = [line for line in result.stdout.splitlines() if line.startswith(">f")]
|
||||||
|
assert itemized == [">f+++++++++ " + changed], (
|
||||||
|
f"expected exactly one itemize line for {changed}, got {itemized}"
|
||||||
|
)
|
||||||
|
received = get_dest_received_dir(dest, source)
|
||||||
|
assert _read_file(os.path.join(received, "changed.txt")) == b"edited payload\n"
|
||||||
|
assert _read_file(os.path.join(received, "untouched.txt")) == b"stable\n"
|
||||||
|
|
||||||
|
|
||||||
|
class TestOutFormat:
|
||||||
|
"""--out-format prints a line per transferred file using the template."""
|
||||||
|
|
||||||
|
def test_out_format_path_and_size(self, shared_server):
|
||||||
|
clean_dir(DEST_DIR)
|
||||||
|
result, _ = run_client(SOURCE_DIR, DEST_DIR,
|
||||||
|
flags=["--out-format=%f %l"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"out-format sync failed: {result.stderr[:200]}"
|
||||||
|
expected = {f"{p} {os.path.getsize(p)}" for p in _source_files()}
|
||||||
|
got = set(result.stdout.splitlines())
|
||||||
|
assert expected <= got, f"out-format lines missing: expected {len(expected)} got {len(got)}"
|
||||||
|
|
||||||
|
def test_out_format_multithreaded_matches_single(self, shared_server):
|
||||||
|
clean_dir(DEST_DIR)
|
||||||
|
result, _ = run_client(SOURCE_DIR, DEST_DIR,
|
||||||
|
flags=["--out-format=%f %l", "-m"], port=shared_server.port)
|
||||||
|
assert result.returncode == 0, f"out-format -m sync failed: {result.stderr[:200]}"
|
||||||
|
expected = {f"{p} {os.path.getsize(p)}" for p in _source_files()}
|
||||||
|
got = set(result.stdout.splitlines())
|
||||||
|
assert expected <= got, f"out-format -m lines missing: {result.stdout[:500]}"
|
||||||
|
|
||||||
|
|
||||||
|
class TestLogFileFormat:
|
||||||
|
"""--log-file plus --log-file-format writes per-file lines to the log."""
|
||||||
|
|
||||||
|
def test_log_file_format_writes_transferred_files(self, shared_server):
|
||||||
|
clean_dir(DEST_DIR)
|
||||||
|
log_path = os.path.join(TEST_DATA_DIR, "itemize_transfer.log")
|
||||||
|
if os.path.exists(log_path):
|
||||||
|
os.unlink(log_path)
|
||||||
|
result, _ = run_client(
|
||||||
|
SOURCE_DIR, DEST_DIR,
|
||||||
|
flags=["--log-file", log_path, "--log-file-format=%f %l"],
|
||||||
|
port=shared_server.port,
|
||||||
|
)
|
||||||
|
assert result.returncode == 0, f"log-file sync failed: {result.stderr[:200]}"
|
||||||
|
assert os.path.exists(log_path), "--log-file created no log"
|
||||||
|
with open(log_path, encoding="utf-8", errors="replace") as fh:
|
||||||
|
content = fh.read()
|
||||||
|
expected = {f"{p} {os.path.getsize(p)}" for p in _source_files()}
|
||||||
|
for line in expected:
|
||||||
|
assert line in content, f"log file missing {line!r}"
|
||||||
|
|
||||||
|
def test_log_file_format_multithreaded_writes_transferred_files(self, shared_server):
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "itemize_log_mt_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "itemize_log_mt_dst")
|
||||||
|
clean_dir(source)
|
||||||
|
clean_dir(dest)
|
||||||
|
files = {"a.txt": b"alpha\n", "b.txt": b"beta\n"}
|
||||||
|
for rel, data in files.items():
|
||||||
|
with open(os.path.join(source, rel), "wb") as fh:
|
||||||
|
fh.write(data)
|
||||||
|
log_path = os.path.join(TEST_DATA_DIR, "itemize_mt.log")
|
||||||
|
if os.path.exists(log_path):
|
||||||
|
os.unlink(log_path)
|
||||||
|
result, _ = run_client(
|
||||||
|
source,
|
||||||
|
dest,
|
||||||
|
flags=["--log-file", log_path, "--log-file-format=%f %l", "-m"],
|
||||||
|
port=shared_server.port,
|
||||||
|
)
|
||||||
|
assert result.returncode == 0, f"log-file -m sync failed: {result.stderr[:200]}"
|
||||||
|
assert os.path.exists(log_path), "--log-file created no log"
|
||||||
|
with open(log_path, encoding="utf-8", errors="replace") as fh:
|
||||||
|
content = fh.read()
|
||||||
|
expected = {f"{os.path.join(source, rel)} {len(data)}" for rel, data in files.items()}
|
||||||
|
for line in expected:
|
||||||
|
assert line in content, f"log file (-m) missing {line!r}"
|
||||||
|
|
||||||
@@ -71,6 +71,11 @@ class TestTCPFlags:
|
|||||||
r = _run_tcp_test("Compression (-c)", shared_server.port, ["-c"])
|
r = _run_tcp_test("Compression (-c)", shared_server.port, ["-c"])
|
||||||
assert r["status"] == "Success", r["error"]
|
assert r["status"] == "Success", r["error"]
|
||||||
|
|
||||||
|
def test_compression_threads(self, shared_server):
|
||||||
|
r = _run_tcp_test("Compression threads (-c --compress-threads=2)", shared_server.port,
|
||||||
|
["-c", "--compress-threads=2"])
|
||||||
|
assert r["status"] == "Success", r["error"]
|
||||||
|
|
||||||
def test_chunk_serialization(self, shared_server):
|
def test_chunk_serialization(self, shared_server):
|
||||||
r = _run_tcp_test("Chunk Serialization (-s)", shared_server.port, ["-s"])
|
r = _run_tcp_test("Chunk Serialization (-s)", shared_server.port, ["-s"])
|
||||||
assert r["status"] == "Success", r["error"]
|
assert r["status"] == "Success", r["error"]
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
#include "test_array_list.h"
|
#include "test_array_list.h"
|
||||||
#include "test_chunk.h"
|
#include "test_chunk.h"
|
||||||
|
#include "test_change_list.h"
|
||||||
#include "test_client_cli.h"
|
#include "test_client_cli.h"
|
||||||
#include "test_compression.h"
|
#include "test_compression.h"
|
||||||
#include "test_config.h"
|
#include "test_config.h"
|
||||||
@@ -40,6 +41,7 @@ int main() {
|
|||||||
RUN_TEST(test_array_list);
|
RUN_TEST(test_array_list);
|
||||||
RUN_TEST(test_shared_utils);
|
RUN_TEST(test_shared_utils);
|
||||||
RUN_TEST(test_chunk);
|
RUN_TEST(test_chunk);
|
||||||
|
RUN_TEST(test_change_list);
|
||||||
RUN_TEST(test_config);
|
RUN_TEST(test_config);
|
||||||
RUN_TEST(test_compression);
|
RUN_TEST(test_compression);
|
||||||
RUN_TEST(test_scanner);
|
RUN_TEST(test_scanner);
|
||||||
|
|||||||
@@ -0,0 +1,101 @@
|
|||||||
|
#include "test_change_list.h"
|
||||||
|
#include "change_list.h"
|
||||||
|
#include "test_utils.h"
|
||||||
|
#include "utils.h"
|
||||||
|
#include <stdlib.h>
|
||||||
|
#include <string.h>
|
||||||
|
#include <time.h>
|
||||||
|
|
||||||
|
static ChangeEvent sample_event(void) {
|
||||||
|
ChangeEvent event;
|
||||||
|
memset(&event, 0, sizeof(event));
|
||||||
|
event.path = "/srv/root/sub/file.txt";
|
||||||
|
event.decision = CHANGE_SENT;
|
||||||
|
event.is_directory = false;
|
||||||
|
event.size = 12345;
|
||||||
|
event.bytes_sent = 999;
|
||||||
|
event.mtime_sec = 1700000000;
|
||||||
|
return event;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_format_tokens() {
|
||||||
|
ChangeEvent event = sample_event();
|
||||||
|
char* line = change_render_format("%f %n %l %b %M %%", &event);
|
||||||
|
EXPECT_NOT_NULL(line);
|
||||||
|
EXPECT_EQ_STR(line, "/srv/root/sub/file.txt file.txt 12345 999 1700000000 %");
|
||||||
|
free(line);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_format_unknown_tokens_preserved() {
|
||||||
|
ChangeEvent event = sample_event();
|
||||||
|
char* line = change_render_format("x%q=%f%z", &event);
|
||||||
|
EXPECT_NOT_NULL(line);
|
||||||
|
EXPECT_EQ_STR(line, "x%q=/srv/root/sub/file.txt%z");
|
||||||
|
free(line);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_format_leaf_name() {
|
||||||
|
ChangeEvent event = sample_event();
|
||||||
|
event.path = "bare.txt";
|
||||||
|
char* line = change_render_format("%n|%f", &event);
|
||||||
|
EXPECT_NOT_NULL(line);
|
||||||
|
EXPECT_EQ_STR(line, "bare.txt|bare.txt");
|
||||||
|
free(line);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_render_itemize_sent_file() {
|
||||||
|
ChangeEvent event = sample_event();
|
||||||
|
char* line = change_render_itemize(&event);
|
||||||
|
EXPECT_NOT_NULL(line);
|
||||||
|
EXPECT_EQ_STR(line, ">f+++++++++ /srv/root/sub/file.txt");
|
||||||
|
free(line);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_render_itemize_up_to_date_is_empty() {
|
||||||
|
ChangeEvent event = sample_event();
|
||||||
|
event.decision = CHANGE_UP_TO_DATE;
|
||||||
|
char* line = change_render_itemize(&event);
|
||||||
|
EXPECT_NOT_NULL(line);
|
||||||
|
EXPECT_EQ_STR(line, "");
|
||||||
|
free(line);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_render_list_line() {
|
||||||
|
char* line = change_render_list_line(0100644, 4096, 1700000000, "/srv/x.txt");
|
||||||
|
EXPECT_NOT_NULL(line);
|
||||||
|
EXPECT_TRUE(strncmp(line, "-rw-r--r--", 10) == 0);
|
||||||
|
EXPECT_TRUE(strstr(line, "4096") != NULL);
|
||||||
|
EXPECT_TRUE(strstr(line, "/srv/x.txt") != NULL);
|
||||||
|
free(line);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_change_list_enabled() {
|
||||||
|
Config* config = config_create();
|
||||||
|
EXPECT_NOT_NULL(config);
|
||||||
|
EXPECT_FALSE(change_list_enabled(config));
|
||||||
|
config->itemize_changes = true;
|
||||||
|
EXPECT_TRUE(change_list_enabled(config));
|
||||||
|
config->itemize_changes = false;
|
||||||
|
config->out_format = str_dup("%f");
|
||||||
|
EXPECT_TRUE(change_list_enabled(config));
|
||||||
|
free(config->out_format);
|
||||||
|
config->out_format = NULL;
|
||||||
|
EXPECT_FALSE(change_list_enabled(config));
|
||||||
|
/* config_delete() closes log_file, so use a throwaway tmpfile. */
|
||||||
|
config->log_file = tmpfile();
|
||||||
|
EXPECT_NOT_NULL(config->log_file);
|
||||||
|
EXPECT_FALSE(change_list_enabled(config)); /* needs a format too */
|
||||||
|
config->log_file_format = str_dup("%n");
|
||||||
|
EXPECT_TRUE(change_list_enabled(config));
|
||||||
|
config_delete(config); /* closes config->log_file */
|
||||||
|
}
|
||||||
|
|
||||||
|
void test_change_list() {
|
||||||
|
test_format_tokens();
|
||||||
|
test_format_unknown_tokens_preserved();
|
||||||
|
test_format_leaf_name();
|
||||||
|
test_render_itemize_sent_file();
|
||||||
|
test_render_itemize_up_to_date_is_empty();
|
||||||
|
test_render_list_line();
|
||||||
|
test_change_list_enabled();
|
||||||
|
}
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
#ifndef TEST_CHANGE_LIST_H
|
||||||
|
#define TEST_CHANGE_LIST_H
|
||||||
|
|
||||||
|
void test_change_list(void);
|
||||||
|
|
||||||
|
#endif
|
||||||
+902
-14
File diff suppressed because it is too large.
Load diff
@@ -55,6 +55,31 @@ static void test_data_compress_decompress_large() {
|
|||||||
data_destroy(decompressed);
|
data_destroy(decompressed);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void test_skip_compress_suffix_matching() {
|
||||||
|
char* suffixes[] = {".ZIP", ".GZ"};
|
||||||
|
EXPECT_TRUE(compression_should_skip_with_suffixes("archive.zip", suffixes, 2));
|
||||||
|
EXPECT_TRUE(compression_should_skip_with_suffixes("backup.TAR.GZ", suffixes, 2));
|
||||||
|
EXPECT_FALSE(compression_should_skip_with_suffixes("notes.txt", suffixes, 2));
|
||||||
|
EXPECT_FALSE(compression_should_skip_with_suffixes("archive.zip", suffixes, 0));
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_data_compress_with_threads_roundtrip() {
|
||||||
|
const size_t size = 8 * 1024 * 1024;
|
||||||
|
Data* input = data_create_empty(size);
|
||||||
|
EXPECT_NOT_NULL(input);
|
||||||
|
for (size_t i = 0; i < size; i++)
|
||||||
|
((char*)input->data)[i] = (char)((i / 4096) % 7);
|
||||||
|
Data* compressed = data_compress_with_threads(input, 3, 2);
|
||||||
|
EXPECT_NOT_NULL(compressed);
|
||||||
|
Data* decompressed = data_decompress(compressed);
|
||||||
|
EXPECT_NOT_NULL(decompressed);
|
||||||
|
EXPECT_EQ_INT((int)decompressed->size, (int)size);
|
||||||
|
EXPECT_EQ_INT(memcmp(decompressed->data, input->data, size), 0);
|
||||||
|
data_destroy(input);
|
||||||
|
data_destroy(compressed);
|
||||||
|
data_destroy(decompressed);
|
||||||
|
}
|
||||||
|
|
||||||
static void test_chunk_compress_decompress_roundtrip() {
|
static void test_chunk_compress_decompress_roundtrip() {
|
||||||
char* path1 = "temp_comp_test_1.txt";
|
char* path1 = "temp_comp_test_1.txt";
|
||||||
char* content1 = "chunk compression test file 1";
|
char* content1 = "chunk compression test file 1";
|
||||||
@@ -115,5 +140,7 @@ static void test_chunk_compress_decompress_roundtrip() {
|
|||||||
void test_compression() {
|
void test_compression() {
|
||||||
test_data_compress_decompress_roundtrip();
|
test_data_compress_decompress_roundtrip();
|
||||||
test_data_compress_decompress_large();
|
test_data_compress_decompress_large();
|
||||||
|
test_skip_compress_suffix_matching();
|
||||||
|
test_data_compress_with_threads_roundtrip();
|
||||||
test_chunk_compress_decompress_roundtrip();
|
test_chunk_compress_decompress_roundtrip();
|
||||||
}
|
}
|
||||||
+142
-4
@@ -95,6 +95,7 @@ static void test_pipeline_sender_lifecycle() {
|
|||||||
EXPECT_EQ_INT(pcs->queue_loader->capacity, 15);
|
EXPECT_EQ_INT(pcs->queue_loader->capacity, 15);
|
||||||
EXPECT_FALSE(pcs->scanner_done);
|
EXPECT_FALSE(pcs->scanner_done);
|
||||||
EXPECT_FALSE(pcs->loader_done);
|
EXPECT_FALSE(pcs->loader_done);
|
||||||
|
EXPECT_EQ_INT((int)pcs->allocation_session.max_alloc, (int)cfg->max_alloc);
|
||||||
|
|
||||||
pipeline_context_sender_destroy(pcs);
|
pipeline_context_sender_destroy(pcs);
|
||||||
}
|
}
|
||||||
@@ -121,11 +122,25 @@ static void test_config_send_receive() {
|
|||||||
send_cfg->receive_root_directory = str_dup("/send/dst");
|
send_cfg->receive_root_directory = str_dup("/send/dst");
|
||||||
send_cfg->save_to_disk = true;
|
send_cfg->save_to_disk = true;
|
||||||
send_cfg->use_multithreading = true;
|
send_cfg->use_multithreading = true;
|
||||||
send_cfg->use_chunk_serialization = true;
|
send_cfg->use_chunk_serialization = false;
|
||||||
send_cfg->use_compression = true;
|
send_cfg->use_compression = true;
|
||||||
send_cfg->use_metadata = true;
|
send_cfg->use_metadata = true;
|
||||||
|
send_cfg->use_executability = true;
|
||||||
|
send_cfg->use_delta = true;
|
||||||
|
send_cfg->whole_file = true;
|
||||||
|
send_cfg->ignore_times = true;
|
||||||
|
send_cfg->size_only = true;
|
||||||
send_cfg->compression_level = 5;
|
send_cfg->compression_level = 5;
|
||||||
send_cfg->chunk_size = 1024;
|
send_cfg->chunk_size = 1024;
|
||||||
|
send_cfg->eight_bit_output = true;
|
||||||
|
send_cfg->modify_window = 4;
|
||||||
|
send_cfg->existing = true;
|
||||||
|
send_cfg->ignore_existing = true;
|
||||||
|
send_cfg->skip_compress_set = true;
|
||||||
|
send_cfg->skip_compress_count = 1;
|
||||||
|
send_cfg->skip_compress_suffixes = calloc(1, sizeof(char*));
|
||||||
|
send_cfg->skip_compress_suffixes[0] = str_dup(".zip");
|
||||||
|
send_cfg->max_alloc = MAX_SERVER_ALLOC + 1;
|
||||||
|
|
||||||
/* Use socketpair for bidirectional communication */
|
/* Use socketpair for bidirectional communication */
|
||||||
int p[2];
|
int p[2];
|
||||||
@@ -154,12 +169,33 @@ static void test_config_send_receive() {
|
|||||||
ok = false;
|
ok = false;
|
||||||
if (!recv_cfg->use_multithreading)
|
if (!recv_cfg->use_multithreading)
|
||||||
ok = false;
|
ok = false;
|
||||||
if (!recv_cfg->use_chunk_serialization)
|
if (recv_cfg->use_chunk_serialization)
|
||||||
ok = false;
|
ok = false;
|
||||||
if (recv_cfg->compression_level != 5)
|
if (recv_cfg->compression_level != 5)
|
||||||
ok = false;
|
ok = false;
|
||||||
if (recv_cfg->chunk_size != 1024)
|
if (recv_cfg->chunk_size != 1024)
|
||||||
ok = false;
|
ok = false;
|
||||||
|
if (!recv_cfg->use_executability)
|
||||||
|
ok = false;
|
||||||
|
if (!recv_cfg->size_only)
|
||||||
|
ok = false;
|
||||||
|
if (!recv_cfg->ignore_times)
|
||||||
|
ok = false;
|
||||||
|
if (!recv_cfg->eight_bit_output)
|
||||||
|
ok = false;
|
||||||
|
if (recv_cfg->use_delta)
|
||||||
|
ok = false;
|
||||||
|
if (recv_cfg->modify_window != 4)
|
||||||
|
ok = false;
|
||||||
|
if (!recv_cfg->existing)
|
||||||
|
ok = false;
|
||||||
|
if (!recv_cfg->ignore_existing)
|
||||||
|
ok = false;
|
||||||
|
if (!recv_cfg->skip_compress_set || recv_cfg->skip_compress_count != 1 ||
|
||||||
|
strcmp(recv_cfg->skip_compress_suffixes[0], ".zip") != 0)
|
||||||
|
ok = false;
|
||||||
|
if (recv_cfg->max_alloc != MAX_SERVER_ALLOC)
|
||||||
|
ok = false;
|
||||||
}
|
}
|
||||||
config_delete(recv_cfg);
|
config_delete(recv_cfg);
|
||||||
close(p[0]);
|
close(p[0]);
|
||||||
@@ -185,11 +221,11 @@ static void test_config_send_receive() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
static void test_config_send_receive_version_mismatch() {
|
static void test_config_send_receive_version_mismatch() {
|
||||||
/* Create a config with a different protocol version */
|
/* A peer using the previous wire format must be rejected. */
|
||||||
Config* cfg = config_create();
|
Config* cfg = config_create();
|
||||||
EXPECT_NOT_NULL(cfg);
|
EXPECT_NOT_NULL(cfg);
|
||||||
free(cfg->version);
|
free(cfg->version);
|
||||||
cfg->version = str_dup("0.0");
|
cfg->version = str_dup("2.3.0");
|
||||||
cfg->send_directory = str_dup("/src");
|
cfg->send_directory = str_dup("/src");
|
||||||
cfg->receive_root_directory = str_dup("/dst");
|
cfg->receive_root_directory = str_dup("/dst");
|
||||||
|
|
||||||
@@ -233,6 +269,8 @@ static void test_config_receive_truncated() {
|
|||||||
/* A valid prefix exercises cleanup after allocated wire strings and a
|
/* A valid prefix exercises cleanup after allocated wire strings and a
|
||||||
* partially received scalar field. */
|
* partially received scalar field. */
|
||||||
EXPECT_TRUE(send_str(p[1], PROTOCOL_VERSION));
|
EXPECT_TRUE(send_str(p[1], PROTOCOL_VERSION));
|
||||||
|
unsigned long long max_alloc = DEFAULT_MAX_ALLOC;
|
||||||
|
EXPECT_TRUE(send_n_data(p[1], &max_alloc, sizeof(max_alloc)));
|
||||||
EXPECT_TRUE(send_str(p[1], "/src"));
|
EXPECT_TRUE(send_str(p[1], "/src"));
|
||||||
EXPECT_TRUE(send_str(p[1], "/dst"));
|
EXPECT_TRUE(send_str(p[1], "/dst"));
|
||||||
EXPECT_TRUE(send_int(p[1], 1));
|
EXPECT_TRUE(send_int(p[1], 1));
|
||||||
@@ -244,6 +282,105 @@ static void test_config_receive_truncated() {
|
|||||||
close(p[1]);
|
close(p[1]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static bool config_string_roundtrip_matches(const Config* send_cfg, Config* recv) {
|
||||||
|
/* The sender serializes NULL strings as "" on the wire. Receivers must
|
||||||
|
canonicalize those empty values back to NULL for the options whose client
|
||||||
|
default is NULL (backup_dir, temp_dir, partial_dir, suffix), while a real
|
||||||
|
non-empty value round-trips unchanged. */
|
||||||
|
const char* fields[4];
|
||||||
|
char* const* recv_fields[4];
|
||||||
|
fields[0] = send_cfg->backup_dir;
|
||||||
|
recv_fields[0] = &recv->backup_dir;
|
||||||
|
fields[1] = send_cfg->temp_dir;
|
||||||
|
recv_fields[1] = &recv->temp_dir;
|
||||||
|
fields[2] = send_cfg->partial_dir;
|
||||||
|
recv_fields[2] = &recv->partial_dir;
|
||||||
|
fields[3] = send_cfg->suffix;
|
||||||
|
recv_fields[3] = &recv->suffix;
|
||||||
|
for (int i = 0; i < 4; i++) {
|
||||||
|
const char* sent = fields[i];
|
||||||
|
const char* got = *recv_fields[i];
|
||||||
|
if (sent == NULL || sent[0] == '\0') {
|
||||||
|
if (got != NULL)
|
||||||
|
return false;
|
||||||
|
} else if (got == NULL || strcmp(sent, got) != 0) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool roundtrip_config_ok(const Config* send_cfg) {
|
||||||
|
int p[2];
|
||||||
|
if (socketpair(AF_UNIX, SOCK_STREAM, 0, p) != 0)
|
||||||
|
return false;
|
||||||
|
pid_t pid = fork();
|
||||||
|
if (pid == 0) {
|
||||||
|
close(p[1]);
|
||||||
|
io_set_fds(p[0], p[0]);
|
||||||
|
Config* recv = config_receive(p[0]);
|
||||||
|
bool ok = recv != NULL;
|
||||||
|
if (ok) {
|
||||||
|
ok = recv->version != NULL && strcmp(recv->version, PROTOCOL_VERSION) == 0;
|
||||||
|
ok = ok && recv->send_directory && recv->receive_root_directory;
|
||||||
|
ok = ok && config_string_roundtrip_matches(send_cfg, recv);
|
||||||
|
}
|
||||||
|
config_delete(recv);
|
||||||
|
close(p[0]);
|
||||||
|
_exit(ok ? 0 : 1);
|
||||||
|
} else {
|
||||||
|
close(p[0]);
|
||||||
|
io_set_fds(p[1], p[1]);
|
||||||
|
bool sent = config_send(p[1], send_cfg);
|
||||||
|
int status;
|
||||||
|
waitpid(pid, &status, 0);
|
||||||
|
close(p[1]);
|
||||||
|
return sent && WIFEXITED(status) && WEXITSTATUS(status) == 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Issue #252: NULL-vs-empty must survive the wire for backup_dir, temp_dir,
|
||||||
|
partial_dir, and suffix. NULL and explicitly-empty client values are both
|
||||||
|
serialized as "" and must be reconstructed as NULL so plain --backup (with
|
||||||
|
no --suffix/--backup-dir) works exactly like the client configured it. */
|
||||||
|
static void test_config_string_null_vs_empty_roundtrip() {
|
||||||
|
if (is_running_under_valgrind())
|
||||||
|
return;
|
||||||
|
|
||||||
|
/* NULL values on the wire must come back as NULL. */
|
||||||
|
Config* a = config_create();
|
||||||
|
EXPECT_NOT_NULL(a);
|
||||||
|
a->send_directory = str_dup("/src");
|
||||||
|
a->receive_root_directory = str_dup("/dst");
|
||||||
|
EXPECT_TRUE(roundtrip_config_ok(a));
|
||||||
|
config_delete(a);
|
||||||
|
|
||||||
|
/* Explicitly empty strings (indistinguishable on the wire from NULL) must
|
||||||
|
be canonicalized to NULL by the receiver. */
|
||||||
|
Config* b = config_create();
|
||||||
|
EXPECT_NOT_NULL(b);
|
||||||
|
b->send_directory = str_dup("/src");
|
||||||
|
b->receive_root_directory = str_dup("/dst");
|
||||||
|
b->backup_dir = str_dup("");
|
||||||
|
b->temp_dir = str_dup("");
|
||||||
|
b->partial_dir = str_dup("");
|
||||||
|
b->suffix = str_dup("");
|
||||||
|
EXPECT_TRUE(roundtrip_config_ok(b));
|
||||||
|
config_delete(b);
|
||||||
|
|
||||||
|
/* Non-empty values must round-trip unchanged. */
|
||||||
|
Config* c = config_create();
|
||||||
|
EXPECT_NOT_NULL(c);
|
||||||
|
c->send_directory = str_dup("/src");
|
||||||
|
c->receive_root_directory = str_dup("/dst");
|
||||||
|
c->backup_dir = str_dup("backups");
|
||||||
|
c->temp_dir = str_dup("/tmp/fast");
|
||||||
|
c->partial_dir = str_dup(".partial");
|
||||||
|
c->suffix = str_dup(".bak");
|
||||||
|
EXPECT_TRUE(roundtrip_config_ok(c));
|
||||||
|
config_delete(c);
|
||||||
|
}
|
||||||
|
|
||||||
static void test_config_is_remote_dest() {
|
static void test_config_is_remote_dest() {
|
||||||
/* Valid SSH-style destinations */
|
/* Valid SSH-style destinations */
|
||||||
EXPECT_TRUE(config_is_remote_dest("user@host:/path"));
|
EXPECT_TRUE(config_is_remote_dest("user@host:/path"));
|
||||||
@@ -277,6 +414,7 @@ void test_config() {
|
|||||||
test_config_send_receive();
|
test_config_send_receive();
|
||||||
test_config_send_receive_version_mismatch();
|
test_config_send_receive_version_mismatch();
|
||||||
test_config_receive_truncated();
|
test_config_receive_truncated();
|
||||||
|
test_config_string_null_vs_empty_roundtrip();
|
||||||
}
|
}
|
||||||
test_config_is_remote_dest();
|
test_config_is_remote_dest();
|
||||||
}
|
}
|
||||||
@@ -343,6 +343,297 @@ static void test_delta_apply_rejects_output_overflow() {
|
|||||||
EXPECT_TRUE(delta_apply(old_data, sizeof(old_data), &delta, 1) == NULL);
|
EXPECT_TRUE(delta_apply(old_data, sizeof(old_data), &delta, 1) == NULL);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* ---------------------------------------------------------------------------
|
||||||
|
* Hash-index lookup differential tests.
|
||||||
|
*
|
||||||
|
* delta_compute buckets signature blocks by their weak checksum. These tests
|
||||||
|
* prove the bucket-indexed candidate lookup is behaviour-identical to the
|
||||||
|
* original per-window linear scan: the emitted instruction stream (types,
|
||||||
|
* lengths, literal bytes and chosen block indices) must match a naive linear
|
||||||
|
* reference exactly, and the delta must reconstruct the new buffer.
|
||||||
|
* ------------------------------------------------------------------------- */
|
||||||
|
|
||||||
|
#define REF_NO_MATCH UINT32_MAX
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
DeltaInstruction* items;
|
||||||
|
uint32_t count;
|
||||||
|
uint32_t cap;
|
||||||
|
} RefDelta;
|
||||||
|
|
||||||
|
static void ref_delta_free(RefDelta* ref) {
|
||||||
|
if (!ref->items)
|
||||||
|
return;
|
||||||
|
for (uint32_t i = 0; i < ref->count; i++)
|
||||||
|
if (ref->items[i].type == DELTA_INSTR_LITERAL)
|
||||||
|
free(ref->items[i].literal.data);
|
||||||
|
free(ref->items);
|
||||||
|
ref->items = NULL;
|
||||||
|
ref->count = 0;
|
||||||
|
ref->cap = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool ref_delta_push(RefDelta* ref, DeltaInstruction instr) {
|
||||||
|
if (ref->count == ref->cap) {
|
||||||
|
uint32_t new_cap = ref->cap ? ref->cap * 2 : 16;
|
||||||
|
DeltaInstruction* tmp = realloc(ref->items, (size_t)new_cap * sizeof(DeltaInstruction));
|
||||||
|
if (!tmp)
|
||||||
|
return false;
|
||||||
|
ref->items = tmp;
|
||||||
|
ref->cap = new_cap;
|
||||||
|
}
|
||||||
|
ref->items[ref->count++] = instr;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool ref_delta_flush_literal(RefDelta* ref, const uint8_t* data, uint64_t start,
|
||||||
|
uint64_t end) {
|
||||||
|
if (start >= end)
|
||||||
|
return true;
|
||||||
|
uint8_t* lit = malloc((size_t)(end - start));
|
||||||
|
if (!lit)
|
||||||
|
return false;
|
||||||
|
memcpy(lit, data + start, (size_t)(end - start));
|
||||||
|
DeltaInstruction instr = {
|
||||||
|
.type = DELTA_INSTR_LITERAL,
|
||||||
|
.literal = {.data = lit, .length = (uint32_t)(end - start)},
|
||||||
|
};
|
||||||
|
return ref_delta_push(ref, instr);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Naive O(windows x blocks) re-implementation of the historical delta_compute
|
||||||
|
* candidate scan: only full windows may match, a candidate needs both the weak
|
||||||
|
* (Adler-32) and strong (xxHash32) checksums to agree, and the lowest matching
|
||||||
|
* block index is selected. */
|
||||||
|
static bool ref_delta_build(RefDelta* ref, const uint8_t* new_data, uint64_t new_size,
|
||||||
|
const DeltaSignature* sig) {
|
||||||
|
uint32_t block_size = sig->block_size;
|
||||||
|
uint64_t i = 0;
|
||||||
|
uint64_t literal_start = 0;
|
||||||
|
bool has_literal = false;
|
||||||
|
|
||||||
|
while (i < new_size) {
|
||||||
|
uint32_t window_len = (uint32_t)((new_size - i < block_size) ? (new_size - i) : block_size);
|
||||||
|
bool full_window = (window_len == block_size);
|
||||||
|
|
||||||
|
uint32_t matched = REF_NO_MATCH;
|
||||||
|
if (full_window) {
|
||||||
|
uint32_t adler = delta_adler32(new_data + i, window_len);
|
||||||
|
for (uint32_t j = 0; j < sig->block_count; j++) {
|
||||||
|
if (sig->blocks[j].adler32 == adler &&
|
||||||
|
delta_xxhash32(new_data + i, window_len) == sig->blocks[j].xxhash) {
|
||||||
|
matched = j;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (matched != REF_NO_MATCH) {
|
||||||
|
if (has_literal) {
|
||||||
|
if (!ref_delta_flush_literal(ref, new_data, literal_start, i))
|
||||||
|
return false;
|
||||||
|
has_literal = false;
|
||||||
|
}
|
||||||
|
DeltaInstruction instr = {
|
||||||
|
.type = DELTA_INSTR_BLOCK_MATCH,
|
||||||
|
.match = {.block_index = matched, .block_offset = 0, .length = window_len},
|
||||||
|
};
|
||||||
|
if (!ref_delta_push(ref, instr))
|
||||||
|
return false;
|
||||||
|
i += window_len;
|
||||||
|
} else {
|
||||||
|
if (!has_literal) {
|
||||||
|
literal_start = i;
|
||||||
|
has_literal = true;
|
||||||
|
}
|
||||||
|
i++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (has_literal && !ref_delta_flush_literal(ref, new_data, literal_start, new_size))
|
||||||
|
return false;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool ref_delta_matches(const RefDelta* ref, const Delta* delta) {
|
||||||
|
if (ref->count != delta->instruction_count)
|
||||||
|
return false;
|
||||||
|
for (uint32_t i = 0; i < ref->count; i++) {
|
||||||
|
const DeltaInstruction* a = &ref->items[i];
|
||||||
|
const DeltaInstruction* b = &delta->instructions[i];
|
||||||
|
if (a->type != b->type)
|
||||||
|
return false;
|
||||||
|
if (a->type == DELTA_INSTR_BLOCK_MATCH) {
|
||||||
|
if (a->match.block_index != b->match.block_index ||
|
||||||
|
a->match.block_offset != b->match.block_offset || a->match.length != b->match.length)
|
||||||
|
return false;
|
||||||
|
} else {
|
||||||
|
if (a->literal.length != b->literal.length ||
|
||||||
|
memcmp(a->literal.data, b->literal.data, a->literal.length) != 0)
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void expect_linear_reference_match(const uint8_t* old_data, uint64_t old_size,
|
||||||
|
const uint8_t* new_data, uint64_t new_size,
|
||||||
|
uint32_t block_size, const char* label) {
|
||||||
|
DeltaSignature* sig = delta_signature_create(old_data, old_size, block_size);
|
||||||
|
if (!sig) {
|
||||||
|
printf(" [FAIL] %s: signature creation failed\n", label);
|
||||||
|
EXPECT_NOT_NULL(sig);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
Delta* delta = delta_compute(new_data, new_size, sig, block_size);
|
||||||
|
if (!delta) {
|
||||||
|
printf(" [FAIL] %s: delta_compute returned NULL\n", label);
|
||||||
|
delta_signature_destroy(sig);
|
||||||
|
EXPECT_NOT_NULL(delta);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
RefDelta ref = {0};
|
||||||
|
bool ok = ref_delta_build(&ref, new_data, new_size, sig);
|
||||||
|
if (ok)
|
||||||
|
ok = ref_delta_matches(&ref, delta);
|
||||||
|
if (!ok) {
|
||||||
|
printf(" [FAIL] %s: instruction stream differs from linear reference "
|
||||||
|
"(linear=%u indexed=%u)\n",
|
||||||
|
label, ref.count, delta->instruction_count);
|
||||||
|
}
|
||||||
|
ref_delta_free(&ref);
|
||||||
|
delta_destroy(delta);
|
||||||
|
delta_signature_destroy(sig);
|
||||||
|
EXPECT_TRUE(ok);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void fill_delta_pattern(uint8_t* buf, uint64_t size, uint32_t seed) {
|
||||||
|
uint32_t x = seed ? seed : 1;
|
||||||
|
for (uint64_t i = 0; i < size; i++) {
|
||||||
|
x ^= x << 13;
|
||||||
|
x ^= x >> 17;
|
||||||
|
x ^= x << 5;
|
||||||
|
buf[i] = (uint8_t)(x >> 24);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_delta_hash_index_matches_linear_reference() {
|
||||||
|
/* Identical file (full block alignment). */
|
||||||
|
uint8_t old_a[32768];
|
||||||
|
uint8_t new_a[32768];
|
||||||
|
fill_delta_pattern(old_a, sizeof(old_a), 42);
|
||||||
|
memcpy(new_a, old_a, sizeof(old_a));
|
||||||
|
expect_linear_reference_match(old_a, sizeof(old_a), new_a, sizeof(new_a), 2048,
|
||||||
|
"identical 32KiB @ 2KiB");
|
||||||
|
|
||||||
|
/* Scattered single-byte edits in the middle of each block. */
|
||||||
|
uint8_t new_b[32768];
|
||||||
|
memcpy(new_b, old_a, sizeof(old_a));
|
||||||
|
for (size_t p = 100; p < sizeof(new_b); p += 4096)
|
||||||
|
new_b[p] ^= 0x5A;
|
||||||
|
expect_linear_reference_match(old_a, sizeof(old_a), new_b, sizeof(new_b), 2048,
|
||||||
|
"32KiB scattered single-byte edits @ 2KiB");
|
||||||
|
|
||||||
|
/* Non-aligned old file (partial final block) with a single edit. */
|
||||||
|
uint8_t old_c[30000];
|
||||||
|
uint8_t new_c[30000];
|
||||||
|
fill_delta_pattern(old_c, sizeof(old_c), 7);
|
||||||
|
memcpy(new_c, old_c, sizeof(old_c));
|
||||||
|
new_c[15000] ^= 0x3C;
|
||||||
|
expect_linear_reference_match(old_c, sizeof(old_c), new_c, sizeof(new_c), 2048,
|
||||||
|
"30KiB partial-tail single edit @ 2KiB");
|
||||||
|
|
||||||
|
/* Growth: appended data after an identical prefix. */
|
||||||
|
uint8_t old_d[24576];
|
||||||
|
uint8_t new_d[34576];
|
||||||
|
fill_delta_pattern(old_d, sizeof(old_d), 11);
|
||||||
|
memcpy(new_d, old_d, sizeof(old_d));
|
||||||
|
fill_delta_pattern(new_d + sizeof(old_d), sizeof(new_d) - sizeof(old_d), 23);
|
||||||
|
expect_linear_reference_match(old_d, sizeof(old_d), new_d, sizeof(new_d), 2048,
|
||||||
|
"24KiB -> 34KiB appended @ 2KiB");
|
||||||
|
|
||||||
|
/* Insertion shifting everything after the edit point (rsync re-sync). */
|
||||||
|
uint8_t old_e[65536];
|
||||||
|
uint8_t new_e[65536 + 3000];
|
||||||
|
fill_delta_pattern(old_e, sizeof(old_e), 99);
|
||||||
|
memcpy(new_e, old_e, 20000);
|
||||||
|
fill_delta_pattern(new_e + 20000, 3000, 101);
|
||||||
|
memcpy(new_e + 23000, old_e + 20000, sizeof(old_e) - 20000);
|
||||||
|
expect_linear_reference_match(old_e, sizeof(old_e), new_e, sizeof(new_e), 2048,
|
||||||
|
"64KiB + 3KiB insertion @ 2KiB");
|
||||||
|
|
||||||
|
/* Deletion shrinking the file. */
|
||||||
|
uint8_t new_f[sizeof(old_e) - 5000];
|
||||||
|
memcpy(new_f, old_e, 30000);
|
||||||
|
memcpy(new_f + 30000, old_e + 35000, sizeof(old_e) - 35000);
|
||||||
|
expect_linear_reference_match(old_e, sizeof(old_e), new_f, sizeof(new_f), 2048,
|
||||||
|
"64KiB - 5KiB deletion @ 2KiB");
|
||||||
|
|
||||||
|
/* Repeated identical blocks must resolve to the lowest block index. */
|
||||||
|
uint8_t old_g[4 * 4096];
|
||||||
|
uint8_t new_g[4 * 4096];
|
||||||
|
for (uint32_t b = 0; b < 4; b++)
|
||||||
|
fill_delta_pattern(old_g + b * 4096, 4096, b % 2 == 0 ? 500 : 501); /* block0==block2 */
|
||||||
|
memcpy(new_g, old_g, sizeof(old_g));
|
||||||
|
new_g[4096 + 5] ^= 0x11; /* edit inside the second (duplicated) chunk */
|
||||||
|
expect_linear_reference_match(old_g, sizeof(old_g), new_g, sizeof(new_g), 4096,
|
||||||
|
"duplicated chunks @ 4KiB");
|
||||||
|
|
||||||
|
/* Block larger than the file: nothing can match, all literal. */
|
||||||
|
uint8_t old_h[1000];
|
||||||
|
uint8_t new_h[1000];
|
||||||
|
fill_delta_pattern(old_h, sizeof(old_h), 3);
|
||||||
|
memcpy(new_h, old_h, sizeof(old_h));
|
||||||
|
expect_linear_reference_match(old_h, sizeof(old_h), new_h, sizeof(new_h), 4096,
|
||||||
|
"1KiB file @ 4KiB block");
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_delta_hash_index_large_mostly_matching() {
|
||||||
|
const uint64_t size = 4ULL * 1024 * 1024;
|
||||||
|
const uint32_t block_size = 8192;
|
||||||
|
|
||||||
|
uint8_t* old_data = malloc((size_t)size);
|
||||||
|
uint8_t* new_data = malloc((size_t)size);
|
||||||
|
EXPECT_TRUE(old_data != NULL && new_data != NULL);
|
||||||
|
|
||||||
|
fill_delta_pattern(old_data, size, 1234);
|
||||||
|
memcpy(new_data, old_data, (size_t)size);
|
||||||
|
|
||||||
|
/* Scattered single-byte changes across the whole buffer. Each change forces
|
||||||
|
* the diff to re-synchronise by walking one byte at a time through the
|
||||||
|
* affected block, which is exactly the case that used to cost O(bytes x
|
||||||
|
* blocks) with the linear scan. */
|
||||||
|
const uint64_t nchanges = 64;
|
||||||
|
for (uint64_t c = 0; c < nchanges; c++) {
|
||||||
|
uint64_t pos = (c * (size / nchanges)) + (c % 17);
|
||||||
|
new_data[pos] ^= (uint8_t)(0xA0 + (c % 16));
|
||||||
|
}
|
||||||
|
|
||||||
|
DeltaSignature* sig = delta_signature_create(old_data, size, block_size);
|
||||||
|
EXPECT_NOT_NULL(sig);
|
||||||
|
EXPECT_EQ_INT((int)sig->block_count, (int)(size / block_size));
|
||||||
|
|
||||||
|
Delta* delta = delta_compute(new_data, size, sig, block_size);
|
||||||
|
EXPECT_NOT_NULL(delta);
|
||||||
|
EXPECT_EQ_INT((int)delta->new_file_size, (int)size);
|
||||||
|
|
||||||
|
uint32_t match_count = 0;
|
||||||
|
for (uint32_t i = 0; i < delta->instruction_count; i++)
|
||||||
|
if (delta->instructions[i].type == DELTA_INSTR_BLOCK_MATCH)
|
||||||
|
match_count++;
|
||||||
|
EXPECT_TRUE(match_count > 0);
|
||||||
|
|
||||||
|
void* result = delta_apply(old_data, size, delta, block_size);
|
||||||
|
EXPECT_NOT_NULL(result);
|
||||||
|
EXPECT_EQ_INT(memcmp(result, new_data, (size_t)size), 0);
|
||||||
|
|
||||||
|
free(result);
|
||||||
|
delta_destroy(delta);
|
||||||
|
delta_signature_destroy(sig);
|
||||||
|
free(old_data);
|
||||||
|
free(new_data);
|
||||||
|
}
|
||||||
|
|
||||||
void test_delta() {
|
void test_delta() {
|
||||||
test_adler32_basic();
|
test_adler32_basic();
|
||||||
test_adler32_different_data();
|
test_adler32_different_data();
|
||||||
@@ -360,4 +651,6 @@ void test_delta() {
|
|||||||
test_is_worthwhile();
|
test_is_worthwhile();
|
||||||
test_large_file_delta();
|
test_large_file_delta();
|
||||||
test_delta_apply_rejects_output_overflow();
|
test_delta_apply_rejects_output_overflow();
|
||||||
|
test_delta_hash_index_matches_linear_reference();
|
||||||
|
test_delta_hash_index_large_mostly_matching();
|
||||||
}
|
}
|
||||||
@@ -1,13 +1,16 @@
|
|||||||
#include "test_file.h"
|
#include "test_file.h"
|
||||||
#include "file.h"
|
#include "file.h"
|
||||||
#include "data.h"
|
#include "data.h"
|
||||||
|
#include "config.h"
|
||||||
#include "utils.h"
|
#include "utils.h"
|
||||||
#include "protocol.h"
|
#include "protocol.h"
|
||||||
#include "test_utils.h"
|
#include "test_utils.h"
|
||||||
|
#include <fcntl.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
#include <sys/stat.h>
|
#include <sys/stat.h>
|
||||||
#include <sys/wait.h>
|
#include <sys/wait.h>
|
||||||
|
#include <time.h>
|
||||||
#include <unistd.h>
|
#include <unistd.h>
|
||||||
|
|
||||||
static void test_file_create() {
|
static void test_file_create() {
|
||||||
@@ -88,6 +91,272 @@ static void test_file_save_to_disk() {
|
|||||||
rmdir("test_save_tmp");
|
rmdir("test_save_tmp");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void test_file_save_to_disk_with_fsync_config() {
|
||||||
|
File* f = file_create("saved_file_fsync.txt");
|
||||||
|
EXPECT_NOT_NULL(f);
|
||||||
|
const char* content = "Save to disk with fsync";
|
||||||
|
f->data->data = malloc(strlen(content));
|
||||||
|
EXPECT_NOT_NULL(f->data->data);
|
||||||
|
memcpy(f->data->data, content, strlen(content));
|
||||||
|
f->data->size = strlen(content);
|
||||||
|
|
||||||
|
Config* config = config_create();
|
||||||
|
EXPECT_NOT_NULL(config);
|
||||||
|
config->use_fsync = true;
|
||||||
|
EXPECT_TRUE(file_save_to_disk("test_save_fsync_tmp", f, config));
|
||||||
|
|
||||||
|
struct stat st;
|
||||||
|
EXPECT_EQ_INT(stat("test_save_fsync_tmp/saved_file_fsync.txt", &st), 0);
|
||||||
|
EXPECT_EQ_INT((int)st.st_size, (int)strlen(content));
|
||||||
|
|
||||||
|
file_destroy(f);
|
||||||
|
config_delete(config);
|
||||||
|
unlink("test_save_fsync_tmp/saved_file_fsync.txt");
|
||||||
|
rmdir("test_save_fsync_tmp");
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_file_save_to_disk_existing() {
|
||||||
|
const char* root = "test_existing_tmp";
|
||||||
|
const char* existing_path = "test_existing_tmp/existing.txt";
|
||||||
|
const char* missing_path = "test_existing_tmp/missing.txt";
|
||||||
|
EXPECT_TRUE(file_write_to_disk(existing_path, "old", 3, false, false));
|
||||||
|
|
||||||
|
Config* cfg = config_create();
|
||||||
|
EXPECT_NOT_NULL(cfg);
|
||||||
|
cfg->existing = true;
|
||||||
|
|
||||||
|
File* existing = file_create("existing.txt");
|
||||||
|
EXPECT_NOT_NULL(existing);
|
||||||
|
existing->data->data = malloc(3);
|
||||||
|
EXPECT_NOT_NULL(existing->data->data);
|
||||||
|
memcpy(existing->data->data, "new", 3);
|
||||||
|
existing->data->size = 3;
|
||||||
|
EXPECT_TRUE(file_save_to_disk(root, existing, cfg));
|
||||||
|
file_destroy(existing);
|
||||||
|
|
||||||
|
File* missing = file_create("missing.txt");
|
||||||
|
EXPECT_NOT_NULL(missing);
|
||||||
|
missing->data->data = malloc(7);
|
||||||
|
EXPECT_NOT_NULL(missing->data->data);
|
||||||
|
memcpy(missing->data->data, "skipped", 7);
|
||||||
|
missing->data->size = 7;
|
||||||
|
EXPECT_TRUE(file_save_to_disk(root, missing, cfg));
|
||||||
|
file_destroy(missing);
|
||||||
|
|
||||||
|
FILE* fp = fopen(existing_path, "rb");
|
||||||
|
char content[4] = {0};
|
||||||
|
EXPECT_NOT_NULL(fp);
|
||||||
|
// cppcheck-suppress knownConditionTrueFalse
|
||||||
|
if (fp) {
|
||||||
|
EXPECT_EQ_INT((int)fread(content, 1, 3, fp), 3);
|
||||||
|
fclose(fp);
|
||||||
|
}
|
||||||
|
EXPECT_EQ_STR(content, "new");
|
||||||
|
EXPECT_EQ_INT(access(missing_path, F_OK), -1);
|
||||||
|
|
||||||
|
config_delete(cfg);
|
||||||
|
unlink(existing_path);
|
||||||
|
rmdir("test_existing_tmp");
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_file_save_to_disk_ignore_existing() {
|
||||||
|
const char* path = "test_ignore_existing_tmp/existing.txt";
|
||||||
|
EXPECT_TRUE(file_write_to_disk(path, "old", 3, false, false));
|
||||||
|
|
||||||
|
File* file = file_create("existing.txt");
|
||||||
|
EXPECT_NOT_NULL(file);
|
||||||
|
file->data->data = malloc(3);
|
||||||
|
EXPECT_NOT_NULL(file->data->data);
|
||||||
|
memcpy(file->data->data, "new", 3);
|
||||||
|
file->data->size = 3;
|
||||||
|
|
||||||
|
Config* config = config_create();
|
||||||
|
EXPECT_NOT_NULL(config);
|
||||||
|
config->ignore_existing = true;
|
||||||
|
EXPECT_TRUE(file_save_to_disk("test_ignore_existing_tmp", file, config));
|
||||||
|
|
||||||
|
FILE* stream = fopen(path, "rb");
|
||||||
|
char content[4] = {0};
|
||||||
|
EXPECT_NOT_NULL(stream);
|
||||||
|
// cppcheck-suppress knownConditionTrueFalse
|
||||||
|
if (stream) {
|
||||||
|
EXPECT_EQ_INT((int)fread(content, 1, 3, stream), 3);
|
||||||
|
fclose(stream);
|
||||||
|
}
|
||||||
|
EXPECT_EQ_STR(content, "old");
|
||||||
|
|
||||||
|
file_destroy(file);
|
||||||
|
config_delete(config);
|
||||||
|
unlink(path);
|
||||||
|
rmdir("test_ignore_existing_tmp");
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_file_save_to_disk_ignore_existing_entry_types() {
|
||||||
|
const char* root = "test_ignore_existing_entries_tmp";
|
||||||
|
const char* directory = "test_ignore_existing_entries_tmp/directory";
|
||||||
|
const char* link = "test_ignore_existing_entries_tmp/link";
|
||||||
|
const char* target = "test_ignore_existing_entries_tmp/target";
|
||||||
|
const char* backup = "test_ignore_existing_entries_tmp/backup.txt~";
|
||||||
|
const char* backup_file = "test_ignore_existing_entries_tmp/backup.txt";
|
||||||
|
Config* config = config_create();
|
||||||
|
File* file = file_create("unused");
|
||||||
|
|
||||||
|
unlink(link);
|
||||||
|
unlink(target);
|
||||||
|
unlink(backup);
|
||||||
|
unlink(backup_file);
|
||||||
|
rmdir(directory);
|
||||||
|
rmdir(root);
|
||||||
|
EXPECT_NOT_NULL(config);
|
||||||
|
EXPECT_NOT_NULL(file);
|
||||||
|
// cppcheck-suppress knownConditionTrueFalse
|
||||||
|
if (!config || !file)
|
||||||
|
return;
|
||||||
|
config->ignore_existing = true;
|
||||||
|
config->backup = true;
|
||||||
|
file->data->data = malloc(3);
|
||||||
|
EXPECT_NOT_NULL(file->data->data);
|
||||||
|
// cppcheck-suppress knownConditionTrueFalse
|
||||||
|
if (!file->data->data) {
|
||||||
|
file_destroy(file);
|
||||||
|
config_delete(config);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
memcpy(file->data->data, "new", 3);
|
||||||
|
file->data->size = 3;
|
||||||
|
|
||||||
|
EXPECT_EQ_INT(mkdir(root, 0755), 0);
|
||||||
|
EXPECT_EQ_INT(mkdir(directory, 0755), 0);
|
||||||
|
EXPECT_TRUE(file_write_to_disk(target, "old", 3, false, false));
|
||||||
|
EXPECT_EQ_INT(symlink("target", link), 0);
|
||||||
|
free(file->path);
|
||||||
|
file->path = str_dup("directory");
|
||||||
|
EXPECT_TRUE(file_save_to_disk(root, file, config));
|
||||||
|
free(file->path);
|
||||||
|
file->path = str_dup("link");
|
||||||
|
EXPECT_TRUE(file_save_to_disk(root, file, config));
|
||||||
|
|
||||||
|
free(file->path);
|
||||||
|
file->path = str_dup("backup.txt");
|
||||||
|
EXPECT_TRUE(file_write_to_disk(backup_file, "old", 3, false, false));
|
||||||
|
EXPECT_TRUE(file_save_to_disk(root, file, config));
|
||||||
|
EXPECT_TRUE(file_path_exists_secure(backup_file));
|
||||||
|
EXPECT_FALSE(file_path_exists_secure(backup));
|
||||||
|
|
||||||
|
file_destroy(file);
|
||||||
|
config_delete(config);
|
||||||
|
unlink(link);
|
||||||
|
unlink(target);
|
||||||
|
unlink(backup_file);
|
||||||
|
rmdir(directory);
|
||||||
|
rmdir(root);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Issue #253: with --partial --partial-dir a completed write must be installed
|
||||||
|
at the real destination rather than left under the partial directory. */
|
||||||
|
static void test_file_save_to_disk_partial_install() {
|
||||||
|
const char* root = "test_partial_install_tmp";
|
||||||
|
const char* dest_file = "test_partial_install_tmp/file.txt";
|
||||||
|
const char* partial_file = "test_partial_install_tmp/.partial/file.txt";
|
||||||
|
unlink(dest_file);
|
||||||
|
unlink(partial_file);
|
||||||
|
rmdir("test_partial_install_tmp/.partial");
|
||||||
|
rmdir(root);
|
||||||
|
|
||||||
|
File* f = file_create("file.txt");
|
||||||
|
EXPECT_NOT_NULL(f);
|
||||||
|
const char* content = "partial-dir content";
|
||||||
|
f->data->data = malloc(strlen(content));
|
||||||
|
EXPECT_NOT_NULL(f->data->data);
|
||||||
|
memcpy(f->data->data, content, strlen(content));
|
||||||
|
f->data->size = strlen(content);
|
||||||
|
|
||||||
|
Config* config = config_create();
|
||||||
|
EXPECT_NOT_NULL(config);
|
||||||
|
config->partial = true;
|
||||||
|
config->partial_dir = str_dup(".partial");
|
||||||
|
|
||||||
|
EXPECT_EQ_INT(file_save_to_disk_full(root, f, config), FILE_SAVE_WRITTEN);
|
||||||
|
|
||||||
|
FILE* fp = fopen(dest_file, "rb");
|
||||||
|
EXPECT_NOT_NULL(fp);
|
||||||
|
// cppcheck-suppress knownConditionTrueFalse
|
||||||
|
if (fp) {
|
||||||
|
char buf[64] = {0};
|
||||||
|
size_t nread = fread(buf, 1, sizeof(buf) - 1, fp);
|
||||||
|
fclose(fp);
|
||||||
|
EXPECT_EQ_INT((int)nread, (int)strlen(content));
|
||||||
|
EXPECT_EQ_INT(memcmp(buf, content, strlen(content)), 0);
|
||||||
|
}
|
||||||
|
/* A completed transfer must not linger under the partial dir. */
|
||||||
|
EXPECT_EQ_INT(access(partial_file, F_OK), -1);
|
||||||
|
|
||||||
|
file_destroy(f);
|
||||||
|
config_delete(config);
|
||||||
|
unlink(dest_file);
|
||||||
|
rmdir(root);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Issue #251: file_save_to_disk_full must distinguish receiver-side skips
|
||||||
|
(--existing/--ignore-existing/--update) from real writes so the sender can
|
||||||
|
decide whether --remove-source-files may unlink its source. */
|
||||||
|
static void test_file_save_to_disk_reports_skips() {
|
||||||
|
const char* root = "test_save_skip_tmp";
|
||||||
|
const char* existing_path = "test_save_skip_tmp/existing.txt";
|
||||||
|
unlink(existing_path);
|
||||||
|
rmdir(root);
|
||||||
|
EXPECT_TRUE(file_write_to_disk(existing_path, "old", 3, false, false));
|
||||||
|
|
||||||
|
Config* cfg = config_create();
|
||||||
|
EXPECT_NOT_NULL(cfg);
|
||||||
|
|
||||||
|
File* new_file = file_create("missing.txt");
|
||||||
|
EXPECT_NOT_NULL(new_file);
|
||||||
|
new_file->data->data = malloc(7);
|
||||||
|
EXPECT_NOT_NULL(new_file->data->data);
|
||||||
|
memcpy(new_file->data->data, "skipped", 7);
|
||||||
|
new_file->data->size = 7;
|
||||||
|
|
||||||
|
/* --existing: destination is missing -> skipped, not an error. */
|
||||||
|
cfg->existing = true;
|
||||||
|
EXPECT_EQ_INT(file_save_to_disk_full(root, new_file, cfg), FILE_SAVE_SKIPPED);
|
||||||
|
cfg->existing = false;
|
||||||
|
|
||||||
|
/* --ignore-existing: destination present -> skipped. */
|
||||||
|
File* present = file_create("existing.txt");
|
||||||
|
EXPECT_NOT_NULL(present);
|
||||||
|
present->data->data = malloc(3);
|
||||||
|
EXPECT_NOT_NULL(present->data->data);
|
||||||
|
memcpy(present->data->data, "new", 3);
|
||||||
|
present->data->size = 3;
|
||||||
|
cfg->ignore_existing = true;
|
||||||
|
EXPECT_EQ_INT(file_save_to_disk_full(root, present, cfg), FILE_SAVE_SKIPPED);
|
||||||
|
cfg->ignore_existing = false;
|
||||||
|
|
||||||
|
/* A normal overwrite of an existing file is a real write. */
|
||||||
|
EXPECT_EQ_INT(file_save_to_disk_full(root, present, cfg), FILE_SAVE_WRITTEN);
|
||||||
|
|
||||||
|
/* --update: a newer destination is skipped. */
|
||||||
|
struct stat st;
|
||||||
|
EXPECT_EQ_INT(stat(existing_path, &st), 0);
|
||||||
|
time_t now = time(NULL);
|
||||||
|
FileMetadata metadata = {.mode = st.st_mode,
|
||||||
|
.uid = st.st_uid,
|
||||||
|
.gid = st.st_gid,
|
||||||
|
.mtime_sec = now - 100,
|
||||||
|
.mtime_nsec = 0};
|
||||||
|
present->metadata = &metadata;
|
||||||
|
cfg->update = true;
|
||||||
|
EXPECT_EQ_INT(file_save_to_disk_full(root, present, cfg), FILE_SAVE_SKIPPED);
|
||||||
|
present->metadata = NULL;
|
||||||
|
|
||||||
|
file_destroy(new_file);
|
||||||
|
file_destroy(present);
|
||||||
|
config_delete(cfg);
|
||||||
|
unlink(existing_path);
|
||||||
|
rmdir(root);
|
||||||
|
}
|
||||||
|
|
||||||
static void test_file_write_to_disk_basic() {
|
static void test_file_write_to_disk_basic() {
|
||||||
const char* content = "Basic file_write_to_disk test";
|
const char* content = "Basic file_write_to_disk test";
|
||||||
EXPECT_TRUE(file_write_to_disk("test_file_write_to_disk_basic.txt", content, strlen(content),
|
EXPECT_TRUE(file_write_to_disk("test_file_write_to_disk_basic.txt", content, strlen(content),
|
||||||
@@ -108,6 +377,17 @@ static void test_file_write_to_disk_basic() {
|
|||||||
unlink("test_file_write_to_disk_basic.txt");
|
unlink("test_file_write_to_disk_basic.txt");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void test_file_write_to_disk_with_fsync() {
|
||||||
|
const char* path = "test_file_write_to_disk_fsync.txt";
|
||||||
|
const char* content = "fsync file content";
|
||||||
|
EXPECT_TRUE(file_to_disk_secure_with_fsync(path, content, strlen(content), false, false, NULL,
|
||||||
|
false, true));
|
||||||
|
struct stat st;
|
||||||
|
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||||
|
EXPECT_EQ_INT((int)st.st_size, (int)strlen(content));
|
||||||
|
unlink(path);
|
||||||
|
}
|
||||||
|
|
||||||
static void test_file_write_to_disk_creates_dirs() {
|
static void test_file_write_to_disk_creates_dirs() {
|
||||||
const char* content = "Nested dir test";
|
const char* content = "Nested dir test";
|
||||||
EXPECT_TRUE(file_write_to_disk("test_nested_tmp/nested/file.txt", content, strlen(content), false,
|
EXPECT_TRUE(file_write_to_disk("test_nested_tmp/nested/file.txt", content, strlen(content), false,
|
||||||
@@ -451,6 +731,161 @@ static void test_file_send_single_calls_metadata_and_path() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void test_inplace_overwrite_clears_special_mode_bits() {
|
||||||
|
const char* root = "test_inplace_tmp";
|
||||||
|
const char* path = "test_inplace_tmp/priv.txt";
|
||||||
|
const char* content = "olddata";
|
||||||
|
unlink(path);
|
||||||
|
rmdir(root);
|
||||||
|
EXPECT_EQ_INT(mkdir(root, 0700), 0);
|
||||||
|
|
||||||
|
/* Create a destination carrying setuid + sticky bits. */
|
||||||
|
int fd = open(path, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0644);
|
||||||
|
EXPECT_TRUE(fd >= 0);
|
||||||
|
// cppcheck-suppress knownConditionTrueFalse
|
||||||
|
if (fd < 0) {
|
||||||
|
rmdir(root);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
EXPECT_EQ_INT((int)write(fd, content, strlen(content)), (int)strlen(content));
|
||||||
|
EXPECT_EQ_INT(fchmod(fd, S_ISUID | S_ISVTX | 0755), 0);
|
||||||
|
EXPECT_EQ_INT(close(fd), 0);
|
||||||
|
|
||||||
|
/* Overwrite in place without metadata: the mode must be normalized to a
|
||||||
|
safe default (0644) and the setuid/sticky bits must be gone. */
|
||||||
|
File* f = file_create("priv.txt");
|
||||||
|
EXPECT_NOT_NULL(f);
|
||||||
|
const char* new_content = "newdata";
|
||||||
|
f->data->data = malloc(strlen(new_content));
|
||||||
|
EXPECT_NOT_NULL(f->data->data);
|
||||||
|
memcpy(f->data->data, new_content, strlen(new_content));
|
||||||
|
f->data->size = strlen(new_content);
|
||||||
|
|
||||||
|
Config* cfg = config_create();
|
||||||
|
EXPECT_NOT_NULL(cfg);
|
||||||
|
cfg->inplace = true;
|
||||||
|
EXPECT_TRUE(file_save_to_disk(root, f, cfg));
|
||||||
|
file_destroy(f);
|
||||||
|
config_delete(cfg);
|
||||||
|
|
||||||
|
struct stat st;
|
||||||
|
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||||
|
EXPECT_EQ_INT((int)(st.st_mode & (S_ISUID | S_ISGID | S_ISVTX)), 0);
|
||||||
|
EXPECT_EQ_INT((int)(st.st_mode & 0777), 0644);
|
||||||
|
FILE* stream = fopen(path, "rb");
|
||||||
|
char buf[16] = {0};
|
||||||
|
EXPECT_NOT_NULL(stream);
|
||||||
|
// cppcheck-suppress knownConditionTrueFalse
|
||||||
|
if (stream) {
|
||||||
|
size_t nread = fread(buf, 1, sizeof(buf) - 1, stream);
|
||||||
|
fclose(stream);
|
||||||
|
EXPECT_EQ_INT((int)nread, (int)strlen(new_content));
|
||||||
|
}
|
||||||
|
EXPECT_EQ_STR(buf, new_content);
|
||||||
|
|
||||||
|
unlink(path);
|
||||||
|
rmdir(root);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_inplace_overwrite_metadata_strips_special_bits() {
|
||||||
|
const char* root = "test_inplace_meta_tmp";
|
||||||
|
const char* path = "test_inplace_meta_tmp/meta.txt";
|
||||||
|
const char* source = "test_inplace_meta_source.txt";
|
||||||
|
unlink(path);
|
||||||
|
unlink(source);
|
||||||
|
rmdir(root);
|
||||||
|
EXPECT_EQ_INT(mkdir(root, 0700), 0);
|
||||||
|
|
||||||
|
/* Existing destination with setuid+sticky set. */
|
||||||
|
int fd = open(path, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0644);
|
||||||
|
EXPECT_TRUE(fd >= 0);
|
||||||
|
// cppcheck-suppress knownConditionTrueFalse
|
||||||
|
if (fd < 0) {
|
||||||
|
rmdir(root);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
EXPECT_EQ_INT((int)write(fd, "olddata", 7), 7);
|
||||||
|
EXPECT_EQ_INT(fchmod(fd, S_ISUID | S_ISVTX | 0755), 0);
|
||||||
|
EXPECT_EQ_INT(close(fd), 0);
|
||||||
|
|
||||||
|
/* Build source metadata carrying a plain executable mode (no specials). */
|
||||||
|
EXPECT_TRUE(file_write_to_disk(source, "source", 6, false, false));
|
||||||
|
EXPECT_EQ_INT(chmod(source, 0755), 0);
|
||||||
|
struct stat source_st;
|
||||||
|
EXPECT_EQ_INT(stat(source, &source_st), 0);
|
||||||
|
|
||||||
|
File* f = file_create("meta.txt");
|
||||||
|
EXPECT_NOT_NULL(f);
|
||||||
|
const char* new_content = "meta";
|
||||||
|
f->data->data = malloc(strlen(new_content));
|
||||||
|
EXPECT_NOT_NULL(f->data->data);
|
||||||
|
memcpy(f->data->data, new_content, strlen(new_content));
|
||||||
|
f->data->size = strlen(new_content);
|
||||||
|
f->metadata = file_metadata_create(&source_st);
|
||||||
|
EXPECT_NOT_NULL(f->metadata);
|
||||||
|
|
||||||
|
Config* cfg = config_create();
|
||||||
|
EXPECT_NOT_NULL(cfg);
|
||||||
|
cfg->inplace = true;
|
||||||
|
EXPECT_TRUE(file_save_to_disk(root, f, cfg));
|
||||||
|
file_destroy(f);
|
||||||
|
config_delete(cfg);
|
||||||
|
unlink(source);
|
||||||
|
|
||||||
|
struct stat st;
|
||||||
|
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||||
|
/* Metadata-derived mode is applied and never includes setuid/setgid/sticky. */
|
||||||
|
EXPECT_EQ_INT((int)(st.st_mode & (S_ISUID | S_ISGID | S_ISVTX)), 0);
|
||||||
|
EXPECT_EQ_INT((int)(st.st_mode & 0777), 0755);
|
||||||
|
|
||||||
|
unlink(path);
|
||||||
|
rmdir(root);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_inplace_overwrite_truncates_shorter_payload() {
|
||||||
|
const char* root = "test_inplace_trunc_tmp";
|
||||||
|
const char* path = "test_inplace_trunc_tmp/big.txt";
|
||||||
|
unlink(path);
|
||||||
|
rmdir(root);
|
||||||
|
EXPECT_EQ_INT(mkdir(root, 0700), 0);
|
||||||
|
|
||||||
|
const char* old_content = "0123456789abcdef"; /* 16 bytes */
|
||||||
|
EXPECT_TRUE(file_write_to_disk(path, old_content, strlen(old_content), false, false));
|
||||||
|
|
||||||
|
File* f = file_create("big.txt");
|
||||||
|
EXPECT_NOT_NULL(f);
|
||||||
|
const char* new_content = "hi";
|
||||||
|
f->data->data = malloc(strlen(new_content));
|
||||||
|
EXPECT_NOT_NULL(f->data->data);
|
||||||
|
memcpy(f->data->data, new_content, strlen(new_content));
|
||||||
|
f->data->size = strlen(new_content);
|
||||||
|
|
||||||
|
Config* cfg = config_create();
|
||||||
|
EXPECT_NOT_NULL(cfg);
|
||||||
|
cfg->inplace = true;
|
||||||
|
EXPECT_TRUE(file_save_to_disk(root, f, cfg));
|
||||||
|
file_destroy(f);
|
||||||
|
config_delete(cfg);
|
||||||
|
|
||||||
|
/* A shorter payload must truncate the file: no stale trailing bytes. */
|
||||||
|
struct stat st;
|
||||||
|
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||||
|
EXPECT_EQ_INT((int)st.st_size, (int)strlen(new_content));
|
||||||
|
FILE* stream = fopen(path, "rb");
|
||||||
|
char buf[32] = {0};
|
||||||
|
EXPECT_NOT_NULL(stream);
|
||||||
|
// cppcheck-suppress knownConditionTrueFalse
|
||||||
|
if (stream) {
|
||||||
|
size_t nread = fread(buf, 1, sizeof(buf) - 1, stream);
|
||||||
|
fclose(stream);
|
||||||
|
EXPECT_EQ_INT((int)nread, (int)strlen(new_content));
|
||||||
|
}
|
||||||
|
EXPECT_EQ_STR(buf, new_content);
|
||||||
|
|
||||||
|
unlink(path);
|
||||||
|
rmdir(root);
|
||||||
|
}
|
||||||
|
|
||||||
void test_file() {
|
void test_file() {
|
||||||
test_file_create();
|
test_file_create();
|
||||||
test_file_destroy_null();
|
test_file_destroy_null();
|
||||||
@@ -458,7 +893,14 @@ void test_file() {
|
|||||||
test_file_load_data();
|
test_file_load_data();
|
||||||
test_file_load_data_missing_file();
|
test_file_load_data_missing_file();
|
||||||
test_file_save_to_disk();
|
test_file_save_to_disk();
|
||||||
|
test_file_save_to_disk_with_fsync_config();
|
||||||
|
test_file_save_to_disk_existing();
|
||||||
|
test_file_save_to_disk_ignore_existing();
|
||||||
|
test_file_save_to_disk_ignore_existing_entry_types();
|
||||||
|
test_file_save_to_disk_partial_install();
|
||||||
|
test_file_save_to_disk_reports_skips();
|
||||||
test_file_write_to_disk_basic();
|
test_file_write_to_disk_basic();
|
||||||
|
test_file_write_to_disk_with_fsync();
|
||||||
test_file_write_to_disk_creates_dirs();
|
test_file_write_to_disk_creates_dirs();
|
||||||
test_file_write_to_disk_does_not_follow_symlink();
|
test_file_write_to_disk_does_not_follow_symlink();
|
||||||
test_file_content_to_buffer();
|
test_file_content_to_buffer();
|
||||||
@@ -476,4 +918,7 @@ void test_file() {
|
|||||||
test_file_send_single_calls_metadata_and_path();
|
test_file_send_single_calls_metadata_and_path();
|
||||||
}
|
}
|
||||||
test_file_metadata_create();
|
test_file_metadata_create();
|
||||||
|
test_inplace_overwrite_clears_special_mode_bits();
|
||||||
|
test_inplace_overwrite_metadata_strips_special_bits();
|
||||||
|
test_inplace_overwrite_truncates_shorter_payload();
|
||||||
}
|
}
|
||||||
@@ -1,6 +1,8 @@
|
|||||||
#include "test_log.h"
|
#include "test_log.h"
|
||||||
#include "log.h"
|
#include "log.h"
|
||||||
#include "test_utils.h"
|
#include "test_utils.h"
|
||||||
|
#include <string.h>
|
||||||
|
#include <unistd.h>
|
||||||
|
|
||||||
/* Test default log level: WARNING and ERROR should print, DEBUG and INFO should not.
|
/* Test default log level: WARNING and ERROR should print, DEBUG and INFO should not.
|
||||||
* We can't easily capture stderr in unit tests, so we verify the functions don't crash
|
* We can't easily capture stderr in unit tests, so we verify the functions don't crash
|
||||||
@@ -90,6 +92,29 @@ static void test_log_filtering() {
|
|||||||
EXPECT_TRUE(true);
|
EXPECT_TRUE(true);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void test_log_stderr_mode_all() {
|
||||||
|
int pipe_fds[2];
|
||||||
|
EXPECT_EQ_INT(pipe(pipe_fds), 0);
|
||||||
|
int saved_stderr = dup(STDERR_FILENO);
|
||||||
|
EXPECT_TRUE(saved_stderr >= 0);
|
||||||
|
EXPECT_TRUE(dup2(pipe_fds[1], STDERR_FILENO) >= 0);
|
||||||
|
close(pipe_fds[1]);
|
||||||
|
|
||||||
|
set_log_level(LOG_LEVEL_WARNING);
|
||||||
|
log_set_stderr_mode(LOG_STDERR_ALL);
|
||||||
|
log_message(LOG_LEVEL_WARNING, "warning routed to stderr");
|
||||||
|
fflush(stderr);
|
||||||
|
|
||||||
|
EXPECT_TRUE(dup2(saved_stderr, STDERR_FILENO) >= 0);
|
||||||
|
close(saved_stderr);
|
||||||
|
char output[128] = {0};
|
||||||
|
ssize_t length = read(pipe_fds[0], output, sizeof(output) - 1);
|
||||||
|
close(pipe_fds[0]);
|
||||||
|
EXPECT_TRUE(length > 0);
|
||||||
|
EXPECT_TRUE(strstr(output, "warning routed to stderr") != NULL);
|
||||||
|
log_set_stderr_mode(LOG_STDERR_ERRORS);
|
||||||
|
}
|
||||||
|
|
||||||
/* Test that log_message handles various format strings */
|
/* Test that log_message handles various format strings */
|
||||||
static void test_log_message_formats() {
|
static void test_log_message_formats() {
|
||||||
set_log_level(LOG_LEVEL_DEBUG);
|
set_log_level(LOG_LEVEL_DEBUG);
|
||||||
@@ -112,5 +137,6 @@ void test_log() {
|
|||||||
test_log_set_level_info();
|
test_log_set_level_info();
|
||||||
test_log_set_level_error();
|
test_log_set_level_error();
|
||||||
test_log_filtering();
|
test_log_filtering();
|
||||||
|
test_log_stderr_mode_all();
|
||||||
test_log_message_formats();
|
test_log_message_formats();
|
||||||
}
|
}
|
||||||
+45
-1
@@ -123,6 +123,18 @@ static void test_metadata_rejects_invalid_values() {
|
|||||||
close(p[1]);
|
close(p[1]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void test_metadata_mtime_window() {
|
||||||
|
EXPECT_TRUE(metadata_mtime_matches(100, 100000000, 101, 600000000, 2));
|
||||||
|
EXPECT_FALSE(metadata_mtime_matches(100, 100000000, 102, 600000000, 2));
|
||||||
|
EXPECT_TRUE(metadata_mtime_matches(100, 100000000, 102, 100000000, 2));
|
||||||
|
EXPECT_TRUE(metadata_mtime_matches(100, 900000000, 102, 100000000, 2));
|
||||||
|
EXPECT_FALSE(metadata_mtime_matches(100, 100000000, 102, 900000000, 2));
|
||||||
|
EXPECT_TRUE(metadata_mtime_matches(100, 900000000, 102, 900000000, 2));
|
||||||
|
EXPECT_FALSE(metadata_mtime_matches(100, 900000000, 101, 100000001, 0));
|
||||||
|
EXPECT_TRUE(metadata_mtime_matches(100, 100000000, 100, 100000001, 0));
|
||||||
|
EXPECT_TRUE(metadata_mtime_matches(100, 100000000, 100, 100000000, 0));
|
||||||
|
}
|
||||||
|
|
||||||
static void test_file_restore_metadata() {
|
static void test_file_restore_metadata() {
|
||||||
const char* path = "temp_meta_restore_test.txt";
|
const char* path = "temp_meta_restore_test.txt";
|
||||||
const char* content = "test content";
|
const char* content = "test content";
|
||||||
@@ -135,7 +147,7 @@ static void test_file_restore_metadata() {
|
|||||||
m.mtime_sec = 1234567890;
|
m.mtime_sec = 1234567890;
|
||||||
m.mtime_nsec = 0;
|
m.mtime_nsec = 0;
|
||||||
|
|
||||||
file_restore_metadata(path, &m);
|
file_restore_metadata(path, &m, false);
|
||||||
|
|
||||||
struct stat st;
|
struct stat st;
|
||||||
EXPECT_EQ_INT(stat(path, &st), 0);
|
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||||
@@ -145,6 +157,35 @@ static void test_file_restore_metadata() {
|
|||||||
unlink(path);
|
unlink(path);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void test_file_restore_executability_only() {
|
||||||
|
const char* path = "temp_exec_restore_test.txt";
|
||||||
|
EXPECT_TRUE(file_write_to_disk(path, "x", 1, false, false));
|
||||||
|
EXPECT_EQ_INT(chmod(path, 0644), 0);
|
||||||
|
|
||||||
|
FileMetadata m = {
|
||||||
|
.mode = 0751, .uid = getuid(), .gid = getgid(), .mtime_sec = 0, .mtime_nsec = 0};
|
||||||
|
file_restore_metadata(path, &m, true);
|
||||||
|
|
||||||
|
struct stat st;
|
||||||
|
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||||
|
EXPECT_EQ_INT(st.st_mode & 0777, 0755);
|
||||||
|
unlink(path);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_directory_restore_executability_only() {
|
||||||
|
const char* path = "temp_exec_restore_test_dir";
|
||||||
|
EXPECT_EQ_INT(mkdir(path, 0700), 0);
|
||||||
|
|
||||||
|
FileMetadata m = {
|
||||||
|
.mode = 0755, .uid = getuid(), .gid = getgid(), .mtime_sec = 0, .mtime_nsec = 0};
|
||||||
|
file_restore_metadata(path, &m, true);
|
||||||
|
|
||||||
|
struct stat st;
|
||||||
|
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||||
|
EXPECT_EQ_INT(st.st_mode & 0777, 0711);
|
||||||
|
rmdir(path);
|
||||||
|
}
|
||||||
|
|
||||||
static void test_chmod_changes() {
|
static void test_chmod_changes() {
|
||||||
mode_t result;
|
mode_t result;
|
||||||
EXPECT_TRUE(chmod_apply(0777, "u=rw,go=r", &result));
|
EXPECT_TRUE(chmod_apply(0777, "u=rw,go=r", &result));
|
||||||
@@ -173,6 +214,9 @@ void test_metadata() {
|
|||||||
test_metadata_send_receive_roundtrip();
|
test_metadata_send_receive_roundtrip();
|
||||||
test_metadata_send_null();
|
test_metadata_send_null();
|
||||||
test_metadata_rejects_invalid_values();
|
test_metadata_rejects_invalid_values();
|
||||||
|
test_metadata_mtime_window();
|
||||||
test_file_restore_metadata();
|
test_file_restore_metadata();
|
||||||
|
test_file_restore_executability_only();
|
||||||
|
test_directory_restore_executability_only();
|
||||||
test_chmod_changes();
|
test_chmod_changes();
|
||||||
}
|
}
|
||||||
@@ -250,6 +250,87 @@ static void test_write_thread_done() {
|
|||||||
config_delete(cfg);
|
config_delete(cfg);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
PipelineContextReceiver* context;
|
||||||
|
File* file;
|
||||||
|
atomic_bool* done;
|
||||||
|
atomic_bool* result;
|
||||||
|
} ByteBudgetEnqueueArg;
|
||||||
|
|
||||||
|
static int byte_budget_enqueue_worker(void* arg) {
|
||||||
|
ByteBudgetEnqueueArg* worker = arg;
|
||||||
|
bool ok = pipeline_context_receiver_enqueue_file(worker->context, worker->file);
|
||||||
|
atomic_store(worker->result, ok);
|
||||||
|
atomic_store(worker->done, true);
|
||||||
|
return thrd_success;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* A receiver must not buffer more decompressed/copied payload bytes ahead of
|
||||||
|
the (slow) disk writer than the configured byte budget: an enqueue that
|
||||||
|
would exceed the budget blocks until the writer releases bytes. */
|
||||||
|
static void test_receiver_enqueue_byte_budget() {
|
||||||
|
Config* cfg = config_create();
|
||||||
|
EXPECT_NOT_NULL(cfg);
|
||||||
|
free(cfg->version);
|
||||||
|
cfg->version = str_dup(PROTOCOL_VERSION);
|
||||||
|
cfg->send_directory = str_dup("/src");
|
||||||
|
cfg->receive_root_directory = str_dup("/dst");
|
||||||
|
cfg->save_to_disk = false;
|
||||||
|
|
||||||
|
Queue* q = queue_create(16, file_destroy);
|
||||||
|
EXPECT_NOT_NULL(q);
|
||||||
|
PipelineContextReceiver* ctx = pipeline_context_receiver_create(cfg, q, -1, NULL);
|
||||||
|
EXPECT_NOT_NULL(ctx);
|
||||||
|
pipeline_context_receiver_set_queue_byte_limit(ctx, 3000);
|
||||||
|
ctx->receiver_done = false;
|
||||||
|
|
||||||
|
File* first = file_create("budget_file_1");
|
||||||
|
EXPECT_NOT_NULL(first);
|
||||||
|
first->data->size = 2000;
|
||||||
|
EXPECT_TRUE(pipeline_context_receiver_enqueue_file(ctx, first));
|
||||||
|
EXPECT_EQ_INT((int)ctx->queued_bytes, 2000);
|
||||||
|
|
||||||
|
/* Second 2000-byte payload would push the pipeline to 4000 > 3000 budget,
|
||||||
|
so the enqueue must block until the first payload is released. */
|
||||||
|
File* second = file_create("budget_file_2");
|
||||||
|
EXPECT_NOT_NULL(second);
|
||||||
|
second->data->size = 2000;
|
||||||
|
atomic_bool done;
|
||||||
|
atomic_bool result;
|
||||||
|
atomic_init(&done, false);
|
||||||
|
atomic_init(&result, false);
|
||||||
|
ByteBudgetEnqueueArg arg = {ctx, second, &done, &result};
|
||||||
|
thrd_t enqueuer;
|
||||||
|
EXPECT_EQ_INT(thrd_create(&enqueuer, byte_budget_enqueue_worker, &arg), thrd_success);
|
||||||
|
|
||||||
|
/* Give a broken (unbounded) implementation every chance to enqueue. */
|
||||||
|
struct timespec wait = {0, 200 * 1000000L};
|
||||||
|
thrd_sleep(&wait, NULL);
|
||||||
|
EXPECT_FALSE(atomic_load(&done));
|
||||||
|
EXPECT_EQ_INT((int)ctx->queued_bytes, 2000); /* budget still honored */
|
||||||
|
|
||||||
|
/* Simulate the disk writer: dequeue + destroy + release the first file. */
|
||||||
|
File* drained = queue_dequeue_multithreaded(q, &ctx->mutex, &ctx->condition_not_empty,
|
||||||
|
&ctx->condition_not_full, &ctx->receiver_done);
|
||||||
|
EXPECT_NOT_NULL(drained);
|
||||||
|
file_destroy(drained);
|
||||||
|
pipeline_context_receiver_note_bytes_released(ctx, 2000);
|
||||||
|
EXPECT_EQ_INT((int)ctx->queued_bytes, 0);
|
||||||
|
|
||||||
|
EXPECT_EQ_INT(thrd_join(enqueuer, NULL), thrd_success);
|
||||||
|
EXPECT_TRUE(atomic_load(&done));
|
||||||
|
EXPECT_TRUE(atomic_load(&result));
|
||||||
|
EXPECT_EQ_INT((int)ctx->queued_bytes, 2000); /* second payload now in flight */
|
||||||
|
|
||||||
|
/* Tear down: the second file is still queued and is freed by queue_destroy. */
|
||||||
|
mtx_destroy(&ctx->mutex);
|
||||||
|
cnd_destroy(&ctx->condition_not_full);
|
||||||
|
cnd_destroy(&ctx->condition_not_empty);
|
||||||
|
free(ctx);
|
||||||
|
queue_destroy(q);
|
||||||
|
config_delete(cfg);
|
||||||
|
}
|
||||||
|
|
||||||
void test_multiprocessing() {
|
void test_multiprocessing() {
|
||||||
test_sender_create_destroy();
|
test_sender_create_destroy();
|
||||||
test_receiver_create_destroy();
|
test_receiver_create_destroy();
|
||||||
@@ -261,4 +342,5 @@ void test_multiprocessing() {
|
|||||||
test_receive_thread_failure_wakes_writer();
|
test_receive_thread_failure_wakes_writer();
|
||||||
}
|
}
|
||||||
test_write_thread_done();
|
test_write_thread_done();
|
||||||
|
test_receiver_enqueue_byte_budget();
|
||||||
}
|
}
|
||||||
@@ -3,6 +3,60 @@
|
|||||||
#include <limits.h>
|
#include <limits.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
#include <unistd.h>
|
#include <unistd.h>
|
||||||
|
#include <threads.h>
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
ProtocolSession* session;
|
||||||
|
bool allocation_allowed;
|
||||||
|
} AllocationWorkerArg;
|
||||||
|
|
||||||
|
static int allocation_worker(void* arg) {
|
||||||
|
AllocationWorkerArg* worker = arg;
|
||||||
|
protocol_session_bind(worker->session);
|
||||||
|
void* allocation = protocol_alloc(8);
|
||||||
|
worker->allocation_allowed = allocation != NULL;
|
||||||
|
free(allocation);
|
||||||
|
protocol_session_unbind();
|
||||||
|
return thrd_success;
|
||||||
|
}
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
ProtocolSession* session;
|
||||||
|
int read_fd;
|
||||||
|
bool released;
|
||||||
|
} AccountingWorkerArg;
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
ProtocolSession* session;
|
||||||
|
atomic_int* ready;
|
||||||
|
atomic_bool* release;
|
||||||
|
bool received;
|
||||||
|
} ConcurrentAccountingWorkerArg;
|
||||||
|
|
||||||
|
static int accounting_worker(void* arg) {
|
||||||
|
AccountingWorkerArg* worker = arg;
|
||||||
|
protocol_session_bind(worker->session);
|
||||||
|
Data* data = protocol_receive_data_limited(worker->session, 8);
|
||||||
|
if (data) {
|
||||||
|
data_destroy(data);
|
||||||
|
worker->released = atomic_load(&worker->session->total_allocated_bytes) == 0;
|
||||||
|
}
|
||||||
|
protocol_session_unbind();
|
||||||
|
return data ? thrd_success : thrd_error;
|
||||||
|
}
|
||||||
|
|
||||||
|
static int concurrent_accounting_worker(void* arg) {
|
||||||
|
ConcurrentAccountingWorkerArg* worker = arg;
|
||||||
|
protocol_session_bind(worker->session);
|
||||||
|
Data* data = protocol_receive_data_limited(worker->session, 8);
|
||||||
|
worker->received = data != NULL;
|
||||||
|
atomic_fetch_add(worker->ready, 1);
|
||||||
|
while (!atomic_load(worker->release))
|
||||||
|
thrd_yield();
|
||||||
|
data_destroy(data);
|
||||||
|
protocol_session_unbind();
|
||||||
|
return thrd_success;
|
||||||
|
}
|
||||||
|
|
||||||
static void test_send_receive_n_data() {
|
static void test_send_receive_n_data() {
|
||||||
int p[2];
|
int p[2];
|
||||||
@@ -187,6 +241,177 @@ static void test_receive_str_truncated() {
|
|||||||
close(p[0]);
|
close(p[0]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void test_max_alloc_rejects_single_buffer() {
|
||||||
|
int p[2];
|
||||||
|
EXPECT_EQ_INT(pipe(p), 0);
|
||||||
|
ProtocolSession session;
|
||||||
|
protocol_session_init(&session, p[0], p[1]);
|
||||||
|
protocol_session_set_max_alloc(&session, 4);
|
||||||
|
protocol_session_bind(&session);
|
||||||
|
char payload[8] = {0};
|
||||||
|
EXPECT_TRUE(write(p[1], &(size_t){sizeof(payload)}, sizeof(size_t)) == sizeof(size_t));
|
||||||
|
EXPECT_NULL(protocol_receive_str(&session));
|
||||||
|
protocol_session_unbind();
|
||||||
|
close(p[0]);
|
||||||
|
close(p[1]);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_explicit_session_max_alloc_cannot_be_bypassed() {
|
||||||
|
int p[2];
|
||||||
|
EXPECT_EQ_INT(pipe(p), 0);
|
||||||
|
ProtocolSession explicit_session;
|
||||||
|
ProtocolSession unrelated_session;
|
||||||
|
protocol_session_init(&explicit_session, p[0], p[1]);
|
||||||
|
protocol_session_init(&unrelated_session, p[0], p[1]);
|
||||||
|
protocol_session_set_max_alloc(&explicit_session, 4);
|
||||||
|
protocol_session_set_max_alloc(&unrelated_session, 64);
|
||||||
|
protocol_session_bind(&unrelated_session);
|
||||||
|
|
||||||
|
unsigned long long size = 8;
|
||||||
|
EXPECT_EQ_INT((int)write(p[1], &size, sizeof(size)), (int)sizeof(size));
|
||||||
|
EXPECT_EQ_INT((int)write(p[1], "12345678", 8), 8);
|
||||||
|
EXPECT_NULL(protocol_receive_data_limited(&explicit_session, 8));
|
||||||
|
EXPECT_EQ_INT((int)atomic_load(&explicit_session.total_allocated_bytes), 0);
|
||||||
|
|
||||||
|
protocol_session_unbind();
|
||||||
|
close(p[0]);
|
||||||
|
close(p[1]);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_max_alloc_allows_configured_buffer() {
|
||||||
|
ProtocolSession session;
|
||||||
|
protocol_session_init(&session, -1, -1);
|
||||||
|
protocol_session_set_max_alloc(&session, 4);
|
||||||
|
protocol_session_bind(&session);
|
||||||
|
void* allowed = protocol_alloc(4);
|
||||||
|
const void* rejected = protocol_alloc(5);
|
||||||
|
EXPECT_NOT_NULL(allowed);
|
||||||
|
EXPECT_NULL(rejected);
|
||||||
|
free(allowed);
|
||||||
|
protocol_session_unbind();
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_max_alloc_is_bound_in_worker_threads() {
|
||||||
|
enum { WORKER_COUNT = 4 };
|
||||||
|
ProtocolSession sessions[WORKER_COUNT];
|
||||||
|
AllocationWorkerArg args[WORKER_COUNT] = {0};
|
||||||
|
thrd_t threads[WORKER_COUNT];
|
||||||
|
for (int i = 0; i < WORKER_COUNT; i++) {
|
||||||
|
protocol_session_init(&sessions[i], -1, -1);
|
||||||
|
protocol_session_set_max_alloc(&sessions[i], 4);
|
||||||
|
args[i].session = &sessions[i];
|
||||||
|
EXPECT_EQ_INT(thrd_create(&threads[i], allocation_worker, &args[i]), thrd_success);
|
||||||
|
}
|
||||||
|
for (int i = 0; i < WORKER_COUNT; i++) {
|
||||||
|
int result;
|
||||||
|
EXPECT_EQ_INT(thrd_join(threads[i], &result), thrd_success);
|
||||||
|
EXPECT_EQ_INT(result, thrd_success);
|
||||||
|
EXPECT_FALSE(args[i].allocation_allowed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_protocol_accounting_is_released_in_worker_threads() {
|
||||||
|
enum { WORKER_COUNT = 4 };
|
||||||
|
ProtocolSession sessions[WORKER_COUNT];
|
||||||
|
AccountingWorkerArg args[WORKER_COUNT] = {0};
|
||||||
|
thrd_t threads[WORKER_COUNT];
|
||||||
|
for (int i = 0; i < WORKER_COUNT; i++) {
|
||||||
|
int p[2];
|
||||||
|
EXPECT_EQ_INT(pipe(p), 0);
|
||||||
|
protocol_session_init(&sessions[i], p[0], p[1]);
|
||||||
|
protocol_session_set_max_alloc(&sessions[i], 64);
|
||||||
|
unsigned long long size = 8;
|
||||||
|
EXPECT_EQ_INT((int)write(p[1], &size, sizeof(size)), (int)sizeof(size));
|
||||||
|
EXPECT_EQ_INT((int)write(p[1], "12345678", 8), 8);
|
||||||
|
close(p[1]);
|
||||||
|
args[i].session = &sessions[i];
|
||||||
|
args[i].read_fd = p[0];
|
||||||
|
EXPECT_EQ_INT(thrd_create(&threads[i], accounting_worker, &args[i]), thrd_success);
|
||||||
|
}
|
||||||
|
for (int i = 0; i < WORKER_COUNT; i++) {
|
||||||
|
int result;
|
||||||
|
EXPECT_EQ_INT(thrd_join(threads[i], &result), thrd_success);
|
||||||
|
EXPECT_EQ_INT(result, thrd_success);
|
||||||
|
EXPECT_TRUE(args[i].released);
|
||||||
|
EXPECT_EQ_INT((int)atomic_load(&sessions[i].total_allocated_bytes), 0);
|
||||||
|
close(args[i].read_fd);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_protocol_accounting_reservation_is_atomic() {
|
||||||
|
enum { WORKER_COUNT = 8 };
|
||||||
|
int p[2];
|
||||||
|
EXPECT_EQ_INT(pipe(p), 0);
|
||||||
|
ProtocolSession session;
|
||||||
|
protocol_session_init(&session, p[0], p[1]);
|
||||||
|
protocol_session_set_max_alloc(&session, 64);
|
||||||
|
const unsigned long long budget_before = MAX_SERVER_ALLOC - 8;
|
||||||
|
atomic_store(&session.total_allocated_bytes, budget_before);
|
||||||
|
|
||||||
|
for (int i = 0; i < WORKER_COUNT; i++) {
|
||||||
|
unsigned long long size = 8;
|
||||||
|
EXPECT_EQ_INT((int)write(p[1], &size, sizeof(size)), (int)sizeof(size));
|
||||||
|
EXPECT_EQ_INT((int)write(p[1], "12345678", 8), 8);
|
||||||
|
}
|
||||||
|
close(p[1]);
|
||||||
|
|
||||||
|
atomic_int ready;
|
||||||
|
atomic_bool release;
|
||||||
|
atomic_init(&ready, 0);
|
||||||
|
atomic_init(&release, false);
|
||||||
|
ConcurrentAccountingWorkerArg args[WORKER_COUNT] = {0};
|
||||||
|
thrd_t threads[WORKER_COUNT];
|
||||||
|
for (int i = 0; i < WORKER_COUNT; i++) {
|
||||||
|
args[i].session = &session;
|
||||||
|
args[i].ready = &ready;
|
||||||
|
args[i].release = &release;
|
||||||
|
EXPECT_EQ_INT(thrd_create(&threads[i], concurrent_accounting_worker, &args[i]), thrd_success);
|
||||||
|
}
|
||||||
|
while (atomic_load(&ready) != WORKER_COUNT)
|
||||||
|
thrd_yield();
|
||||||
|
bool budget_ok = atomic_load(&session.total_allocated_bytes) == budget_before + 8;
|
||||||
|
atomic_store(&release, true);
|
||||||
|
int received = 0;
|
||||||
|
for (int i = 0; i < WORKER_COUNT; i++) {
|
||||||
|
int result;
|
||||||
|
EXPECT_EQ_INT(thrd_join(threads[i], &result), thrd_success);
|
||||||
|
EXPECT_EQ_INT(result, thrd_success);
|
||||||
|
received += args[i].received ? 1 : 0;
|
||||||
|
}
|
||||||
|
EXPECT_EQ_INT(received, 1);
|
||||||
|
EXPECT_TRUE(budget_ok);
|
||||||
|
EXPECT_EQ_INT((int)atomic_load(&session.total_allocated_bytes), (int)budget_before);
|
||||||
|
close(p[0]);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_protocol_string_accounting_is_transient() {
|
||||||
|
int p[2];
|
||||||
|
EXPECT_EQ_INT(pipe(p), 0);
|
||||||
|
ProtocolSession session;
|
||||||
|
protocol_session_init(&session, p[0], p[1]);
|
||||||
|
protocol_session_set_max_alloc(&session, 64);
|
||||||
|
EXPECT_TRUE(protocol_send_str(&session, "temporary"));
|
||||||
|
char* received = protocol_receive_str(&session);
|
||||||
|
EXPECT_NOT_NULL(received);
|
||||||
|
EXPECT_EQ_STR(received, "temporary");
|
||||||
|
EXPECT_EQ_INT((int)atomic_load(&session.total_allocated_bytes), 0);
|
||||||
|
free(received);
|
||||||
|
close(p[0]);
|
||||||
|
close(p[1]);
|
||||||
|
}
|
||||||
|
|
||||||
|
static void test_protocol_accounting_release_does_not_underflow() {
|
||||||
|
ProtocolSession session;
|
||||||
|
protocol_session_init(&session, -1, -1);
|
||||||
|
atomic_store(&session.total_allocated_bytes, 4);
|
||||||
|
protocol_session_bind(&session);
|
||||||
|
protocol_release_memory(8);
|
||||||
|
EXPECT_EQ_INT((int)atomic_load(&session.total_allocated_bytes), 0);
|
||||||
|
protocol_release_memory(1);
|
||||||
|
EXPECT_EQ_INT((int)atomic_load(&session.total_allocated_bytes), 0);
|
||||||
|
protocol_session_unbind();
|
||||||
|
}
|
||||||
|
|
||||||
void test_protocol() {
|
void test_protocol() {
|
||||||
test_send_receive_n_data();
|
test_send_receive_n_data();
|
||||||
test_send_receive_n_data_zero();
|
test_send_receive_n_data_zero();
|
||||||
@@ -198,4 +423,12 @@ void test_protocol() {
|
|||||||
test_send_receive_status();
|
test_send_receive_status();
|
||||||
test_receive_n_data_truncated();
|
test_receive_n_data_truncated();
|
||||||
test_receive_str_truncated();
|
test_receive_str_truncated();
|
||||||
|
test_max_alloc_rejects_single_buffer();
|
||||||
|
test_explicit_session_max_alloc_cannot_be_bypassed();
|
||||||
|
test_max_alloc_allows_configured_buffer();
|
||||||
|
test_max_alloc_is_bound_in_worker_threads();
|
||||||
|
test_protocol_accounting_is_released_in_worker_threads();
|
||||||
|
test_protocol_accounting_reservation_is_atomic();
|
||||||
|
test_protocol_string_accounting_is_transient();
|
||||||
|
test_protocol_accounting_release_does_not_underflow();
|
||||||
}
|
}
|
||||||
@@ -109,6 +109,20 @@ static void test_delta_deserialize_garbage() {
|
|||||||
data_destroy(d);
|
data_destroy(d);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void test_delta_deserialize_respects_max_alloc() {
|
||||||
|
unsigned char serialized[sizeof(uint64_t) + sizeof(uint32_t)] = {0};
|
||||||
|
Data data = {.data = serialized, .size = sizeof(serialized)};
|
||||||
|
ProtocolSession session;
|
||||||
|
protocol_session_init(&session, -1, -1);
|
||||||
|
protocol_session_set_max_alloc(&session, sizeof(Delta) - 1);
|
||||||
|
protocol_session_bind(&session);
|
||||||
|
|
||||||
|
const Delta* result = delta_deserialize(&data);
|
||||||
|
EXPECT_NULL(result);
|
||||||
|
|
||||||
|
protocol_session_unbind();
|
||||||
|
}
|
||||||
|
|
||||||
static void test_delta_signature_deserialize_truncated() {
|
static void test_delta_signature_deserialize_truncated() {
|
||||||
char old_data[4096];
|
char old_data[4096];
|
||||||
for (int i = 0; i < 4096; i++)
|
for (int i = 0; i < 4096; i++)
|
||||||
@@ -206,6 +220,7 @@ void test_robustness() {
|
|||||||
test_delta_deserialize_truncated();
|
test_delta_deserialize_truncated();
|
||||||
test_delta_deserialize_empty();
|
test_delta_deserialize_empty();
|
||||||
test_delta_deserialize_garbage();
|
test_delta_deserialize_garbage();
|
||||||
|
test_delta_deserialize_respects_max_alloc();
|
||||||
test_delta_deserialize_truncated_instructions();
|
test_delta_deserialize_truncated_instructions();
|
||||||
test_delta_signature_deserialize_truncated();
|
test_delta_signature_deserialize_truncated();
|
||||||
test_delta_apply_null();
|
test_delta_apply_null();
|
||||||
|
|||||||
@@ -396,7 +396,7 @@ static void test_parallel_scanner_root_chunks_without_workers() {
|
|||||||
|
|
||||||
ScannerOptions options = {false, 1, NULL, 0, NULL, 0, 0, 0,
|
ScannerOptions options = {false, 1, NULL, 0, NULL, 0, 0, 0,
|
||||||
0, 0, false, false, false, false, false};
|
0, 0, false, false, false, false, false};
|
||||||
ParallelScanner* scanner = parallel_scanner_create_with_options(dir, &options);
|
ParallelScanner* scanner = parallel_scanner_create_with_options(dir, &options, NULL);
|
||||||
EXPECT_NOT_NULL(scanner);
|
EXPECT_NOT_NULL(scanner);
|
||||||
|
|
||||||
int total_files = 0;
|
int total_files = 0;
|
||||||
|
|||||||
@@ -1,14 +1,18 @@
|
|||||||
#include "test_server.h"
|
#include "test_server.h"
|
||||||
#include "config.h"
|
#include "config.h"
|
||||||
|
#include "delta.h"
|
||||||
#include "file.h"
|
#include "file.h"
|
||||||
#include "protocol.h"
|
#include "protocol.h"
|
||||||
#include "test_utils.h"
|
#include "test_utils.h"
|
||||||
#include "utils.h"
|
#include "utils.h"
|
||||||
|
#include <fcntl.h>
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
#include <sys/socket.h>
|
#include <sys/socket.h>
|
||||||
|
#include <sys/stat.h>
|
||||||
#include <sys/wait.h>
|
#include <sys/wait.h>
|
||||||
|
#include <time.h>
|
||||||
#include <unistd.h>
|
#include <unistd.h>
|
||||||
|
|
||||||
#include "receiver.h"
|
#include "receiver.h"
|
||||||
@@ -182,11 +186,286 @@ static void test_receive_manifest_rejects_traversal() {
|
|||||||
config_delete(cfg);
|
config_delete(cfg);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void test_receive_incremental_check_rejects_invalid_nanoseconds() {
|
||||||
|
Config* cfg = config_create();
|
||||||
|
EXPECT_NOT_NULL(cfg);
|
||||||
|
cfg->receive_root_directory = str_dup("/tmp/dst");
|
||||||
|
|
||||||
|
int p[2];
|
||||||
|
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
|
||||||
|
io_set_fds(p[0], p[1]);
|
||||||
|
EXPECT_TRUE(send_str(p[1], "file.txt"));
|
||||||
|
unsigned long long size = 0;
|
||||||
|
long long mtime = 100;
|
||||||
|
long long mtime_nsec = 1000000000LL;
|
||||||
|
EXPECT_TRUE(send_n_data(p[1], &size, sizeof(size)));
|
||||||
|
EXPECT_TRUE(send_n_data(p[1], &mtime, sizeof(mtime)));
|
||||||
|
EXPECT_TRUE(send_n_data(p[1], &mtime_nsec, sizeof(mtime_nsec)));
|
||||||
|
|
||||||
|
bool skipped = false;
|
||||||
|
EXPECT_NULL(receive_incremental_check(p[0], cfg, &skipped));
|
||||||
|
Status status;
|
||||||
|
EXPECT_TRUE(receive_status(p[1], &status));
|
||||||
|
EXPECT_EQ_INT(status, STATUS_ERROR);
|
||||||
|
EXPECT_FALSE(skipped);
|
||||||
|
|
||||||
|
close(p[0]);
|
||||||
|
close(p[1]);
|
||||||
|
config_delete(cfg);
|
||||||
|
}
|
||||||
|
|
||||||
|
static char* make_check_root(const char* tag) {
|
||||||
|
char tmpl[128];
|
||||||
|
snprintf(tmpl, sizeof(tmpl), "/tmp/fastsync_%s_XXXXXX", tag);
|
||||||
|
char* path = str_dup(tmpl);
|
||||||
|
if (!path)
|
||||||
|
return NULL;
|
||||||
|
if (!mkdtemp(path)) {
|
||||||
|
free(path);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
return path;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void write_check_file(const char* dir, const char* name, const char* content) {
|
||||||
|
char path[1024];
|
||||||
|
snprintf(path, sizeof(path), "%s/%s", dir, name);
|
||||||
|
int fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0644);
|
||||||
|
if (fd >= 0) {
|
||||||
|
size_t len = strlen(content);
|
||||||
|
if (write(fd, content, len) != (ssize_t)len) {
|
||||||
|
/* intentionally ignored in tests */
|
||||||
|
}
|
||||||
|
close(fd);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Issue #255: a same-size/mtime match is decided from metadata alone, so the
|
||||||
|
receiver answers STATUS_OK (skip) and never asks for a data body. */
|
||||||
|
static void test_incremental_check_quick_skip_by_mtime() {
|
||||||
|
Config* cfg = config_create();
|
||||||
|
EXPECT_NOT_NULL(cfg);
|
||||||
|
char* root = make_check_root("qskip");
|
||||||
|
EXPECT_NOT_NULL(root);
|
||||||
|
cfg->receive_root_directory = str_dup(root);
|
||||||
|
write_check_file(root, "file.txt", "0123456789abcdef");
|
||||||
|
|
||||||
|
char path[1024];
|
||||||
|
snprintf(path, sizeof(path), "%s/file.txt", root);
|
||||||
|
struct stat st;
|
||||||
|
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||||
|
|
||||||
|
int p[2];
|
||||||
|
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
|
||||||
|
io_set_fds(p[0], p[1]);
|
||||||
|
io_set_bwlimit(0);
|
||||||
|
|
||||||
|
pid_t pid = fork();
|
||||||
|
if (pid == 0) {
|
||||||
|
alarm(30);
|
||||||
|
close(p[1]);
|
||||||
|
io_set_fds(p[0], p[0]);
|
||||||
|
bool skipped = false;
|
||||||
|
File* file = receive_incremental_check(p[0], cfg, &skipped);
|
||||||
|
bool ok = file == NULL && skipped;
|
||||||
|
file_destroy(file);
|
||||||
|
config_delete(cfg);
|
||||||
|
close(p[0]);
|
||||||
|
_exit(ok ? 0 : 1);
|
||||||
|
} else {
|
||||||
|
close(p[0]);
|
||||||
|
io_set_fds(p[1], p[1]);
|
||||||
|
EXPECT_TRUE(send_str(p[1], "file.txt"));
|
||||||
|
unsigned long long size = (unsigned long long)st.st_size;
|
||||||
|
long long mtime = (long long)st.st_mtime;
|
||||||
|
long long mtime_nsec = 0;
|
||||||
|
#ifdef __linux__
|
||||||
|
mtime_nsec = (long long)st.st_mtim.tv_nsec;
|
||||||
|
#endif
|
||||||
|
EXPECT_TRUE(send_n_data(p[1], &size, sizeof(size)));
|
||||||
|
EXPECT_TRUE(send_n_data(p[1], &mtime, sizeof(mtime)));
|
||||||
|
EXPECT_TRUE(send_n_data(p[1], &mtime_nsec, sizeof(mtime_nsec)));
|
||||||
|
Status s;
|
||||||
|
EXPECT_TRUE(receive_status(p[1], &s));
|
||||||
|
EXPECT_EQ_INT(s, STATUS_OK);
|
||||||
|
|
||||||
|
int status;
|
||||||
|
waitpid(pid, &status, 0);
|
||||||
|
close(p[1]);
|
||||||
|
config_delete(cfg);
|
||||||
|
unlink(path);
|
||||||
|
rmdir(root);
|
||||||
|
free(root);
|
||||||
|
EXPECT_TRUE(WIFEXITED(status) && WEXITSTATUS(status) == 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Issue #255: a size mismatch cannot be a skip, so the receiver answers
|
||||||
|
STATUS_NEXT and consumes the full data body that follows. */
|
||||||
|
static void test_incremental_check_size_mismatch_full_transfer() {
|
||||||
|
Config* cfg = config_create();
|
||||||
|
EXPECT_NOT_NULL(cfg);
|
||||||
|
char* root = make_check_root("qnext");
|
||||||
|
EXPECT_NOT_NULL(root);
|
||||||
|
cfg->receive_root_directory = str_dup(root);
|
||||||
|
write_check_file(root, "file.txt", "0123456789abcdef");
|
||||||
|
|
||||||
|
char path[1024];
|
||||||
|
snprintf(path, sizeof(path), "%s/file.txt", root);
|
||||||
|
struct stat st;
|
||||||
|
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||||
|
|
||||||
|
int p[2];
|
||||||
|
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
|
||||||
|
io_set_fds(p[0], p[1]);
|
||||||
|
io_set_bwlimit(0);
|
||||||
|
|
||||||
|
pid_t pid = fork();
|
||||||
|
if (pid == 0) {
|
||||||
|
alarm(30);
|
||||||
|
close(p[1]);
|
||||||
|
io_set_fds(p[0], p[0]);
|
||||||
|
bool skipped = false;
|
||||||
|
File* file = receive_incremental_check(p[0], cfg, &skipped);
|
||||||
|
bool ok = file != NULL && !skipped && file->path != NULL && strcmp(file->path, "file.txt") == 0;
|
||||||
|
file_destroy(file);
|
||||||
|
config_delete(cfg);
|
||||||
|
close(p[0]);
|
||||||
|
_exit(ok ? 0 : 1);
|
||||||
|
} else {
|
||||||
|
close(p[0]);
|
||||||
|
io_set_fds(p[1], p[1]);
|
||||||
|
EXPECT_TRUE(send_str(p[1], "file.txt"));
|
||||||
|
unsigned long long size = (unsigned long long)st.st_size + 1;
|
||||||
|
long long mtime = (long long)st.st_mtime;
|
||||||
|
long long mtime_nsec = 0;
|
||||||
|
#ifdef __linux__
|
||||||
|
mtime_nsec = (long long)st.st_mtim.tv_nsec;
|
||||||
|
#endif
|
||||||
|
EXPECT_TRUE(send_n_data(p[1], &size, sizeof(size)));
|
||||||
|
EXPECT_TRUE(send_n_data(p[1], &mtime, sizeof(mtime)));
|
||||||
|
EXPECT_TRUE(send_n_data(p[1], &mtime_nsec, sizeof(mtime_nsec)));
|
||||||
|
Status s;
|
||||||
|
EXPECT_TRUE(receive_status(p[1], &s));
|
||||||
|
EXPECT_EQ_INT(s, STATUS_NEXT);
|
||||||
|
|
||||||
|
Data* body = data_create_reserve(8);
|
||||||
|
EXPECT_NOT_NULL(body);
|
||||||
|
body->data = malloc(8);
|
||||||
|
EXPECT_NOT_NULL(body->data);
|
||||||
|
memcpy(body->data, "replaced", 8);
|
||||||
|
body->size = 8;
|
||||||
|
EXPECT_TRUE(send_data(p[1], body));
|
||||||
|
data_destroy(body);
|
||||||
|
|
||||||
|
int status;
|
||||||
|
waitpid(pid, &status, 0);
|
||||||
|
close(p[1]);
|
||||||
|
config_delete(cfg);
|
||||||
|
unlink(path);
|
||||||
|
rmdir(root);
|
||||||
|
free(root);
|
||||||
|
EXPECT_TRUE(WIFEXITED(status) && WEXITSTATUS(status) == 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Issue #256: when a received delta claims a result above the whole-file cap,
|
||||||
|
receive_delta_file must mark the operation failed so the caller aborts with
|
||||||
|
STATUS_ERROR instead of emitting STATUS_NEXT and waiting for a body that
|
||||||
|
never arrives. */
|
||||||
|
static void test_incremental_check_delta_oversize_reports_failure() {
|
||||||
|
Config* cfg = config_create();
|
||||||
|
EXPECT_NOT_NULL(cfg);
|
||||||
|
char* root = make_check_root("qdelta");
|
||||||
|
EXPECT_NOT_NULL(root);
|
||||||
|
cfg->receive_root_directory = str_dup(root);
|
||||||
|
cfg->use_delta = true;
|
||||||
|
|
||||||
|
char content[20000];
|
||||||
|
memset(content, 'a', sizeof(content));
|
||||||
|
content[sizeof(content) - 1] = '\0';
|
||||||
|
write_check_file(root, "file.txt", content);
|
||||||
|
|
||||||
|
char path[1024];
|
||||||
|
snprintf(path, sizeof(path), "%s/file.txt", root);
|
||||||
|
struct stat st;
|
||||||
|
EXPECT_EQ_INT(stat(path, &st), 0);
|
||||||
|
EXPECT_EQ_INT((int)st.st_size, 19999);
|
||||||
|
|
||||||
|
int p[2];
|
||||||
|
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
|
||||||
|
io_set_fds(p[0], p[1]);
|
||||||
|
io_set_bwlimit(0);
|
||||||
|
|
||||||
|
pid_t pid = fork();
|
||||||
|
if (pid == 0) {
|
||||||
|
alarm(30);
|
||||||
|
close(p[1]);
|
||||||
|
io_set_fds(p[0], p[0]);
|
||||||
|
bool skipped = false;
|
||||||
|
File* file = receive_incremental_check(p[0], cfg, &skipped);
|
||||||
|
bool ok = file == NULL && !skipped;
|
||||||
|
if (ok)
|
||||||
|
send_status(p[0], STATUS_ERROR); /* mirror the server error path */
|
||||||
|
file_destroy(file);
|
||||||
|
config_delete(cfg);
|
||||||
|
close(p[0]);
|
||||||
|
_exit(ok ? 0 : 1);
|
||||||
|
} else {
|
||||||
|
close(p[0]);
|
||||||
|
io_set_fds(p[1], p[1]);
|
||||||
|
EXPECT_TRUE(send_str(p[1], "file.txt"));
|
||||||
|
unsigned long long size = (unsigned long long)st.st_size;
|
||||||
|
long long mtime = 1; /* different from the file mtime: force a transfer */
|
||||||
|
long long mtime_nsec = 0;
|
||||||
|
EXPECT_TRUE(send_n_data(p[1], &size, sizeof(size)));
|
||||||
|
EXPECT_TRUE(send_n_data(p[1], &mtime, sizeof(mtime)));
|
||||||
|
EXPECT_TRUE(send_n_data(p[1], &mtime_nsec, sizeof(mtime_nsec)));
|
||||||
|
|
||||||
|
Status s;
|
||||||
|
EXPECT_TRUE(receive_status(p[1], &s));
|
||||||
|
EXPECT_EQ_INT(s, STATUS_DELTA_SIGNATURE);
|
||||||
|
Data* sig_data = receive_data(p[1]);
|
||||||
|
EXPECT_NOT_NULL(sig_data);
|
||||||
|
DeltaSignature* sig = delta_signature_deserialize(sig_data);
|
||||||
|
EXPECT_NOT_NULL(sig);
|
||||||
|
delta_signature_destroy(sig);
|
||||||
|
data_destroy(sig_data);
|
||||||
|
|
||||||
|
/* Send a delta whose claimed output size exceeds the whole-file cap. */
|
||||||
|
Delta delta;
|
||||||
|
memset(&delta, 0, sizeof(delta));
|
||||||
|
delta.new_file_size = MAX_RECEIVE_WHOLE_FILE_SIZE + 1;
|
||||||
|
Data* bogus = delta_serialize(&delta);
|
||||||
|
EXPECT_NOT_NULL(bogus);
|
||||||
|
EXPECT_TRUE(send_status(p[1], STATUS_DELTA_DATA));
|
||||||
|
EXPECT_TRUE(bogus != NULL && send_data(p[1], bogus));
|
||||||
|
data_destroy(bogus);
|
||||||
|
|
||||||
|
/* The receiver must answer with an error, never with STATUS_NEXT. */
|
||||||
|
EXPECT_TRUE(receive_status(p[1], &s));
|
||||||
|
EXPECT_EQ_INT(s, STATUS_ERROR);
|
||||||
|
|
||||||
|
int status;
|
||||||
|
waitpid(pid, &status, 0);
|
||||||
|
close(p[1]);
|
||||||
|
config_delete(cfg);
|
||||||
|
unlink(path);
|
||||||
|
rmdir(root);
|
||||||
|
free(root);
|
||||||
|
EXPECT_TRUE(WIFEXITED(status) && WEXITSTATUS(status) == 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
void test_server() {
|
void test_server() {
|
||||||
if (!is_running_under_valgrind()) {
|
if (!is_running_under_valgrind()) {
|
||||||
test_receive_files_finished();
|
test_receive_files_finished();
|
||||||
test_receive_files_single_file();
|
test_receive_files_single_file();
|
||||||
test_receive_files_abort();
|
test_receive_files_abort();
|
||||||
test_receive_manifest_rejects_traversal();
|
test_receive_manifest_rejects_traversal();
|
||||||
|
test_receive_incremental_check_rejects_invalid_nanoseconds();
|
||||||
|
test_incremental_check_quick_skip_by_mtime();
|
||||||
|
test_incremental_check_size_mismatch_full_transfer();
|
||||||
|
test_incremental_check_delta_oversize_reports_failure();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,10 +1,66 @@
|
|||||||
#include "test_shared_utils.h"
|
#include "test_shared_utils.h"
|
||||||
#include "utils.h"
|
#include "utils.h"
|
||||||
|
#include "protocol.h"
|
||||||
#include "test_utils.h"
|
#include "test_utils.h"
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
|
#include <threads.h>
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
bool eight_bit_output;
|
||||||
|
const char* expected;
|
||||||
|
int failed;
|
||||||
|
} EscapeThreadArgs;
|
||||||
|
|
||||||
|
static int escape_thread(void* arg) {
|
||||||
|
EscapeThreadArgs* args = arg;
|
||||||
|
for (int i = 0; i < 1000; i++) {
|
||||||
|
char* escaped = output_escape("x\xc3\xa9\n", args->eight_bit_output);
|
||||||
|
if (!escaped || strcmp(escaped, args->expected) != 0)
|
||||||
|
args->failed = 1;
|
||||||
|
free(escaped);
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
void test_shared_utils() {
|
void test_shared_utils() {
|
||||||
|
char formatted[32];
|
||||||
|
EXPECT_TRUE(format_human_bytes(0, formatted, sizeof(formatted)));
|
||||||
|
EXPECT_EQ_STR(formatted, "0 B");
|
||||||
|
EXPECT_TRUE(format_human_bytes(1024, formatted, sizeof(formatted)));
|
||||||
|
EXPECT_EQ_STR(formatted, "1.0 KB");
|
||||||
|
EXPECT_TRUE(format_human_bytes(1536 * 1024, formatted, sizeof(formatted)));
|
||||||
|
EXPECT_EQ_STR(formatted, "1.5 MB");
|
||||||
|
EXPECT_FALSE(format_human_bytes(1024, formatted, 4));
|
||||||
|
|
||||||
|
char high_bit[] = {'a', (char)0xc3, (char)0xa9, '\n', '\0'};
|
||||||
|
char* escaped = output_escape(high_bit, false);
|
||||||
|
EXPECT_EQ_STR(escaped, "a\\#303\\#251\\#012");
|
||||||
|
free(escaped);
|
||||||
|
escaped = output_escape(high_bit, true);
|
||||||
|
EXPECT_EQ_STR(escaped, "a\xc3\xa9\\#012");
|
||||||
|
free(escaped);
|
||||||
|
|
||||||
|
ProtocolSession safe_session;
|
||||||
|
ProtocolSession eight_bit_session;
|
||||||
|
protocol_session_init(&safe_session, -1, -1);
|
||||||
|
protocol_session_init(&eight_bit_session, -1, -1);
|
||||||
|
protocol_session_set_8_bit_output(&safe_session, false);
|
||||||
|
protocol_session_set_8_bit_output(&eight_bit_session, true);
|
||||||
|
EXPECT_FALSE(safe_session.eight_bit_output);
|
||||||
|
EXPECT_TRUE(eight_bit_session.eight_bit_output);
|
||||||
|
|
||||||
|
EscapeThreadArgs safe_args = {false, "x\\#303\\#251\\#012", 0};
|
||||||
|
EscapeThreadArgs eight_bit_args = {true, "x\xc3\xa9\\#012", 0};
|
||||||
|
thrd_t safe_thread;
|
||||||
|
thrd_t eight_bit_thread;
|
||||||
|
EXPECT_EQ_INT(thrd_create(&safe_thread, escape_thread, &safe_args), thrd_success);
|
||||||
|
EXPECT_EQ_INT(thrd_create(&eight_bit_thread, escape_thread, &eight_bit_args), thrd_success);
|
||||||
|
EXPECT_EQ_INT(thrd_join(safe_thread, NULL), thrd_success);
|
||||||
|
EXPECT_EQ_INT(thrd_join(eight_bit_thread, NULL), thrd_success);
|
||||||
|
EXPECT_FALSE(safe_args.failed);
|
||||||
|
EXPECT_FALSE(eight_bit_args.failed);
|
||||||
|
|
||||||
// Test str_dup
|
// Test str_dup
|
||||||
const char* dup_null = str_dup(NULL);
|
const char* dup_null = str_dup(NULL);
|
||||||
EXPECT_NULL(dup_null);
|
EXPECT_NULL(dup_null);
|
||||||
|
|||||||
+33
-13
@@ -4,34 +4,39 @@
|
|||||||
|
|
||||||
static void test_ssh_connect_invalid_dest_no_colon() {
|
static void test_ssh_connect_invalid_dest_no_colon() {
|
||||||
/* cppcheck-suppress constVariablePointer */
|
/* cppcheck-suppress constVariablePointer */
|
||||||
Client* client = client_connect_ssh("invalid-destination-no-colon", 22, NULL);
|
Client* client = client_connect_ssh("invalid-destination-no-colon", 22, NULL, false);
|
||||||
EXPECT_NULL(client);
|
EXPECT_NULL(client);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void test_ssh_connect_invalid_dest_empty() {
|
static void test_ssh_connect_invalid_dest_empty() {
|
||||||
/* cppcheck-suppress constVariablePointer */
|
/* cppcheck-suppress constVariablePointer */
|
||||||
Client* client = client_connect_ssh("", 22, NULL);
|
Client* client = client_connect_ssh("", 22, NULL, false);
|
||||||
EXPECT_NULL(client);
|
EXPECT_NULL(client);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Test client_connect_ssh with malformed destination (just a colon).
|
/* A child that cannot exec ssh must not be returned as a successful client. */
|
||||||
* parse_remote_dest succeeds, ssh is exec'd and fails, but the function
|
|
||||||
* creates a Client that must be cleaned up. */
|
|
||||||
static void test_ssh_connect_malformed() {
|
static void test_ssh_connect_malformed() {
|
||||||
Client* client = client_connect_ssh(":", 22, NULL);
|
const char* old_path = getenv("PATH");
|
||||||
/* ssh binary exists, so exec succeeds; the function returns a Client.
|
char* saved_path = old_path ? strdup(old_path) : NULL;
|
||||||
* We just verify it doesn't crash and clean up properly. */
|
setenv("PATH", "", 1);
|
||||||
if (client != NULL) {
|
|
||||||
client_disconnect(client);
|
/* cppcheck-suppress constVariablePointer */
|
||||||
client_delete(client);
|
Client* client = client_connect_ssh(":", 22, NULL, false);
|
||||||
|
|
||||||
|
if (saved_path) {
|
||||||
|
setenv("PATH", saved_path, 1);
|
||||||
|
free(saved_path);
|
||||||
|
} else {
|
||||||
|
unsetenv("PATH");
|
||||||
}
|
}
|
||||||
EXPECT_TRUE(true);
|
|
||||||
|
EXPECT_NULL(client);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Test client_connect_ssh with valid format but unreachable host.
|
/* Test client_connect_ssh with valid format but unreachable host.
|
||||||
* The function launches ssh which will fail to connect, returns a Client. */
|
* The function launches ssh which will fail to connect, returns a Client. */
|
||||||
static void test_ssh_connect_unreachable() {
|
static void test_ssh_connect_unreachable() {
|
||||||
Client* client = client_connect_ssh("nonexistent.invalid:/remote/path", 22, NULL);
|
Client* client = client_connect_ssh("nonexistent.invalid:/remote/path", 22, NULL, false);
|
||||||
if (client != NULL) {
|
if (client != NULL) {
|
||||||
client_disconnect(client);
|
client_disconnect(client);
|
||||||
client_delete(client);
|
client_delete(client);
|
||||||
@@ -39,9 +44,24 @@ static void test_ssh_connect_unreachable() {
|
|||||||
EXPECT_TRUE(true);
|
EXPECT_TRUE(true);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void test_ssh_remote_command_argument_modes() {
|
||||||
|
char* command = ssh_build_remote_command("fast sync; touch /tmp/pwned", false);
|
||||||
|
EXPECT_EQ_STR(command, "'fast sync; touch /tmp/pwned' --stdio");
|
||||||
|
free(command);
|
||||||
|
|
||||||
|
command = ssh_build_remote_command("fast'sync", false);
|
||||||
|
EXPECT_EQ_STR(command, "'fast'\\''sync' --stdio");
|
||||||
|
free(command);
|
||||||
|
|
||||||
|
command = ssh_build_remote_command("fast sync; touch /tmp/pwned", true);
|
||||||
|
EXPECT_EQ_STR(command, "fast sync; touch /tmp/pwned --stdio");
|
||||||
|
free(command);
|
||||||
|
}
|
||||||
|
|
||||||
void test_transport_ssh() {
|
void test_transport_ssh() {
|
||||||
test_ssh_connect_invalid_dest_no_colon();
|
test_ssh_connect_invalid_dest_no_colon();
|
||||||
test_ssh_connect_invalid_dest_empty();
|
test_ssh_connect_invalid_dest_empty();
|
||||||
test_ssh_connect_malformed();
|
test_ssh_connect_malformed();
|
||||||
test_ssh_connect_unreachable();
|
test_ssh_connect_unreachable();
|
||||||
|
test_ssh_remote_command_argument_modes();
|
||||||
}
|
}
|
||||||
Reference in new issue
Block a user