29 Commits
Author SHA1 Message Date
TapTap b7e95b8d96 Merge PR #329: quality cycle
CI / lint (push) Successful in 1m52s
CI / parity-fast (push) Skipped
CI / parity-full (push) Successful in 22s
CI / sanitizers (address) (push) Successful in 55s
CI / sanitizers (undefined) (push) Successful in 46s
CI / build-and-test (push) Successful in 1m16s
CI / fuzz-build (push) Successful in 56s
CI / coverage (push) Successful in 46s
CI / valgrind (push) Successful in 2m35s
2026-09-24 03:17:42 +02:00
TapTap 13627e82fc fix: scan-root error propagation, link_target leak, delta guard, meta leak (review)
CI / lint (pull_request) Successful in 1m51s
CI / parity-full (pull_request) Skipped
CI / sanitizers (address) (pull_request) Skipped
CI / sanitizers (undefined) (pull_request) Skipped
CI / fuzz-build (pull_request) Skipped
CI / coverage (pull_request) Skipped
CI / valgrind (pull_request) Skipped
CI / parity-fast (pull_request) Successful in 19s
CI / build-and-test (pull_request) Successful in 54s
2026-09-24 03:12:29 +02:00
TapTap 09ab81120e refactor(scanner): make cur_path const (cppcheck) 2026-09-24 02:45:33 +02:00
TapTap 2a7afbda0a Merge branch 'quality/g325' into quality/refactor 2026-09-24 02:35:33 +02:00
TapTap bab6fcc706 Merge branch 'quality/g324' into quality/refactor 2026-09-24 02:35:33 +02:00
TapTap 6a426cffa8 Merge branch 'quality/g323' into quality/refactor 2026-09-24 02:35:33 +02:00
TapTap b74182c7c1 Merge branch 'quality/g322' into quality/refactor 2026-09-24 02:35:33 +02:00
TapTap f2a8eeaea7 Merge branch 'quality/g321' into quality/refactor 2026-09-24 02:35:33 +02:00
TapTap 18d7d495cf Merge branch 'quality/g319' into quality/refactor 2026-09-24 02:35:33 +02:00
TapTap 59d20e867a refactor(scanner): share file-entry construction; name magic constants (#322, #326) 2026-09-24 02:35:03 +02:00
TapTap cc931790e9 feat(xattr): --fake-super for directories (#319) 2026-09-24 02:30:02 +02:00
TapTap 60776b78fc refactor(server): decompose main into hash-credentials/stdio/daemon entrypoints (#324) 2026-09-24 02:29:17 +02:00
TapTap 52ad0aa512 refactor(delete): decompose delete_walk_fd and missing-args budget walker (#325) 2026-09-24 02:23:47 +02:00
TapTap 759ffea117 refactor(client): share connect/session setup and delete preamble (#323) 2026-09-24 02:23:21 +02:00
TapTap ef37c2b988 refactor(incremental): decompose receive_delta_file with a fail label (#321) 2026-09-24 02:15:28 +02:00
TapTap a14fbb2c10 Merge PR #328: no-wire parity (#317, #318)
CI / lint (push) Successful in 1m55s
CI / parity-fast (push) Skipped
CI / parity-full (push) Successful in 22s
CI / sanitizers (address) (push) Successful in 55s
CI / sanitizers (undefined) (push) Successful in 44s
CI / build-and-test (push) Successful in 1m17s
CI / fuzz-build (push) Successful in 52s
CI / coverage (push) Successful in 46s
CI / valgrind (push) Successful in 2m35s
2026-09-24 01:44:22 +02:00
TapTap 7408686370 test: fix cppcheck style findings in cycle-F tests
CI / lint (pull_request) Successful in 1m53s
CI / parity-full (pull_request) Skipped
CI / sanitizers (address) (pull_request) Skipped
CI / sanitizers (undefined) (pull_request) Skipped
CI / fuzz-build (pull_request) Skipped
CI / coverage (pull_request) Skipped
CI / valgrind (pull_request) Skipped
CI / parity-fast (pull_request) Successful in 17s
CI / build-and-test (pull_request) Successful in 54s
2026-09-24 01:38:56 +02:00
TapTap 90f2fb613f Merge branch 'parity2/f2' into parity/no-wire 2026-09-24 01:25:41 +02:00
TapTap 8843f1e3cf Merge branch 'parity2/f1' into parity/no-wire 2026-09-24 01:25:41 +02:00
TapTap c387beb182 feat(transfer): stream single files and bases above the 256 MiB ceiling (#318) 2026-09-24 01:25:12 +02:00
TapTap 96e02f52c0 fix(delay-updates): unique staging dir and implied --delete-after ordering (#317) 2026-09-24 01:18:13 +02:00
TapTap f7d5dda93b Merge PR #327: wire backlog (2.30.0)
CI / lint (push) Successful in 1m51s
CI / parity-fast (push) Skipped
CI / parity-full (push) Successful in 22s
CI / sanitizers (address) (push) Successful in 54s
CI / sanitizers (undefined) (push) Successful in 43s
CI / build-and-test (push) Successful in 1m18s
CI / fuzz-build (push) Successful in 50s
CI / coverage (push) Successful in 46s
CI / valgrind (push) Successful in 2m39s
2026-09-24 00:37:47 +02:00
TapTap 6d9cdb83ba Merge branch 'wire/h2' into wire/backlog-230
CI / lint (pull_request) Successful in 1m50s
CI / parity-full (pull_request) Skipped
CI / sanitizers (address) (pull_request) Skipped
CI / sanitizers (undefined) (pull_request) Skipped
CI / fuzz-build (pull_request) Skipped
CI / coverage (pull_request) Skipped
CI / valgrind (pull_request) Skipped
CI / parity-fast (pull_request) Successful in 17s
CI / build-and-test (pull_request) Successful in 55s
2026-09-24 00:25:15 +02:00
TapTap 7c748f1f7a Merge branch 'wire/h1' into wire/backlog-230 2026-09-24 00:25:15 +02:00
TapTap 884530c9a2 fix(filter): correct per-directory rule owner coordinate; post-clear ownership and bounds 2026-09-24 00:24:52 +02:00
TapTap 729f3ef8e1 fix(protocol): atomic client-msg flag, sanitize peer messages, flush before terminal, serialization probe 2026-09-23 23:56:31 +02:00
TapTap b414f197af feat(filter): per-directory merge rules at the receiver; implement e/n/w/- modifiers 2026-09-23 22:46:14 +02:00
TapTap 29f8be161c feat(protocol): destination-state itemize for dirs/symlinks; --stats deleted breakdown 2026-09-23 21:52:40 +02:00
TapTap cb2979fdf1 feat(protocol): client-message channel and rsync partial exit 23 (2.30.0) 2026-09-23 20:33:27 +02:00
106 changed files with 6732 additions and 1796 deletions
+33 -2
View File
@@ -6,6 +6,36 @@ run the same version because the handshake is strict.
## [Unreleased]
Wire backlog cycle (protocol 2.29.0 → 2.30.0; config-frame layout unchanged).
- **`--stderr=client` client-message channel (#313):** the client now accepts
`--stderr=client` (and maps the deprecated `--no-msgs2stderr` to it), routing
its own diagnostics over the new bounded `STATUS_CLIENT_MSG` client->server
frame instead of writing them locally; the server writes each received
message to its stderr (respecting the server log destination). `errors`/`all`
behavior is unchanged.
- **Receiver partial failures exit 23 (#320):** a per-entry receiver failure
that does not abort the stream (e.g. an unprivileged `--devices` mknod) now
sends the terminal `STATUS_PARTIAL`; the client exits 23 like rsync and, under
`--remove-source-files`, still removes the sources it successfully
transferred. A clean run stays 0 and a fatal/connection error stays non-23.
- **Directory/symlink destination-state itemize (#314):** when `report_dest_info`
is negotiated (now also for `--progress`), the receiver answers `STATUS_MKDIR`
and `STATUS_SYMLINK` with the entry's pre-transfer destination snapshot
(existence, type, perms/owner/group/time, and whether an existing symlink's
target already matches), and the sender probes every ancestor directory before
the receiver creates it implicitly. A re-run over an unchanged tree no longer
emits per-directory `cd+++++++++` or unchanged-symlink lines, a changed
directory renders rsync's `.d..t......`, and a changed symlink renders
`cLc........` / `.L..t......`. Directory/symlink time comparison uses whole
seconds (rsync's `cmp_time`). The `STATUS_MKDIR` body gains a probe flag and
the `STATUS_DEST_INFO` record gains a symlink-target-match field; the
config-frame layout is unchanged. Differential-tested against rsync 3.4.1.
- **`--stats` deleted per-type breakdown (#316):** `STATUS_STATS` gains
`deleted_reg/dir/link/special`, tallied by the delete observers and rendered
as rsync's `Number of deleted files: X (reg: A, dir: B, link: C, special: D)`.
Differential-tested against rsync 3.4.1 for a mixed-type `--delete` tree.
## [2.29.0] - 2026-09-23
The rsync-parity cycle 2.29 (no wire change; `PROTOCOL_VERSION` stays 2.28.0).
@@ -82,8 +112,9 @@ layout is unchanged (golden length still 886).
wording, and symlink/empty-directory quick-checks.
- `--delete-before`'s phase-0 late-file divergence remains (rsync's pre-scan
fixes the file list before the data pass).
- A single file larger than 256 MiB cannot be streamed in the default path
(a general whole-file limit, not basis-specific).
- A whole-file sender that cannot stream its codec (lz4's one-shot block
format) or a `--append`/delta source above the bound still buffers; the
default zstd/zlib and the uncompressed paths stream (see #318).
- `--stats` byte totals and `--msgs2stderr` stay documented divergences.
### Security
+1 -1
View File
@@ -1,6 +1,6 @@
cmake_minimum_required(VERSION 3.22)
project(FastFileTransfer VERSION 2.29.0)
project(FastFileTransfer VERSION 2.30.0)
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
set(CMAKE_C_STANDARD 11)
+3 -1
View File
@@ -235,7 +235,9 @@
and symlink/empty-dir quick-check feedback.
- **`--delete-before` phase-0 keep-set** (rsync fixes the file list before
the data pass; FastSync keeps its pre-scan snapshot race).
- **>256 MiB single-file streaming** (B4, the general whole-file limit).
- ~~**>256 MiB single-file streaming** (B4, the general whole-file limit).~~
Closed by #318: the whole-file payload, the basis read/verify and the fuzzy
basis are streamed through bounded buffers (lz4/append remain buffered).
- **Wire native-size framing:** lengths are native `size_t` and the protocol
assumes homogeneous word size/endianness — document or move to fixed-width
framing.
+10 -9
View File
@@ -206,9 +206,9 @@ This produces `./build/client` and `./build/server`. `compile_commands.json` is
| `--incremental` | Skip files unchanged since last transfer (size + mtime). Auto-enables `--preserve`. Incompatible with `--chunk-serialization`. |
| `--existing` | Skip files not already present at the destination; update existing files normally. |
| `--ignore-existing` | Skip files that already exist on the receiver; like rsync it does not apply to directories or symlinks. |
| `--compare-dest <dir>` | Extra comparison basis: unchanged files are not transferred (requires/implies `--incremental`; a basis MISS above the 256 MiB whole-file payload bound is refused — see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)) |
| `--copy-dest <dir>` | Like `--compare-dest`, but copies the unchanged file from DIR into the destination (same basis-size caveat; see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)) |
| `--link-dest <dir>` | Like `--copy-dest`, but hard-links the unchanged file from DIR (repeatable; earlier DIRs win; same basis-size caveat; see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)) |
| `--compare-dest <dir>` | Extra comparison basis: unchanged files are not transferred (requires/implies `--incremental`; a basis of any size is supported, streamed in bounded chunks — see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)) |
| `--copy-dest <dir>` | Like `--compare-dest`, but copies the unchanged file from DIR into the destination (the copy is streamed, so a basis of any size works; see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)) |
| `--link-dest <dir>` | Like `--copy-dest`, but hard-links the unchanged file from DIR (repeatable; earlier DIRs win; a basis of any size works; see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)) |
| `--verify-basis` | FastSync-only: require a basis hit (`--compare-dest`/`--copy-dest`/`--link-dest`) to match the source by whole-file digest instead of trusting the size+mtime quick-check (default matches rsync) |
| `--delete` | Delete files on receiver not present in source (default timing: delete-during, matching rsync, so destination space is freed progressively). Scoped to the synchronized directories, so `--files-from` subsets are safe |
| `--delete-before` | Delete extras before the transfer starts (implies `--delete`) |
@@ -300,6 +300,7 @@ transfer is never aborted.
| `FASTSYNC_SOURCE_DIR` | — | Source directory fallback |
| `FASTSYNC_DEST_DIR` | — | Destination directory fallback |
| `FASTSYNC_SAVE_TO_DISK` | `false` | Disk persistence fallback |
| `FASTSYNC_MAX_WHOLE_FILE_SIZE` | `268435456` | Receiver-only test hook: a byte count that lowers the whole-file streaming bound. Payloads above it are streamed through a bounded buffer. Values are clamped to the 256 MiB protocol ceiling, so it can only lower, never raise, the bound. |
## Implementation Details
@@ -580,9 +581,9 @@ remote SSH argv is already built injection-safe.
| `--files-from <file>` | Read the source file list from FILE (paths relative to the source root). |
| `-0, --from0` | Treat entries in `--files-from` files as NUL-delimited instead of newline-delimited. |
| `--delay-updates` | Put updated files into place only at the end of the transfer (the fixed `.fastsync-stage` staging name diverges from rsync; see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)). |
| `--compare-dest <dir>` | Extra comparison basis: unchanged files are not transferred (requires/implies `--incremental`; a basis MISS above the 256 MiB whole-file payload bound is refused — see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)). |
| `--copy-dest <dir>` | Like `--compare-dest`, but copies the unchanged file from DIR into the destination (same basis-size caveat; see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)). |
| `--link-dest <dir>` | Like `--copy-dest`, but hard-links the unchanged file from DIR (repeatable; earlier DIRs win; same basis-size caveat; see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)). |
| `--compare-dest <dir>` | Extra comparison basis: unchanged files are not transferred (requires/implies `--incremental`; a basis of any size is supported, streamed in bounded chunks — see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)). |
| `--copy-dest <dir>` | Like `--compare-dest`, but copies the unchanged file from DIR into the destination (the copy is streamed, so a basis of any size works; see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)). |
| `--link-dest <dir>` | Like `--copy-dest`, but hard-links the unchanged file from DIR (repeatable; earlier DIRs win; a basis of any size works; see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)). |
| `--verify-basis` | FastSync-only: require a basis hit to match the source by whole-file digest instead of trusting the size+mtime quick-check (default matches rsync). |
| `--preallocate` | Allocate destination file space up front (fail-fast on a full disk). |
| `--append` | Resume a shorter destination by appending only its tail (prefix not verified; requires `--incremental`). |
@@ -674,9 +675,9 @@ remote SSH argv is already built injection-safe.
| `--outbuf=MODE` | stdout/stderr buffering: `N` (none/unbuffered), `L` (line-buffered), or `B` (block-buffered, default). |
| `--log-file <path>` | Write log output to a file. |
| `--log-file-format=FORMAT` | Per-file log-line format (requires `--log-file`). |
| `--stderr=MODE` | Route logging to stderr: `errors` or `all`. |
| `--stderr=MODE` | Route logging: `errors` (default), `all`, or `client` (forward the client's diagnostics to the server's stderr over the client-message channel). |
| `--msgs2stderr` | Route all messages to stderr (deprecated spelling of `--stderr=all`). |
| `--no-msgs2stderr` | Select errors-only stderr (deprecated spelling; the default). |
| `--no-msgs2stderr` | Forward the client's diagnostics to the server (deprecated spelling of `--stderr=client`). |
| `-V`, `--version` | Print the FastSync protocol version. |
| `--help` | Print command usage. |
@@ -832,7 +833,7 @@ before the module list, before authentication, and the connecting peer address
## Protocol and Security
FastSync protocol version `2.29.0` is shared by the client and server. The
FastSync protocol version `2.30.0` is shared by the client and server. The
current protocol is sender-driven and includes configuration negotiation,
including the maximum allocation limit, incremental checks, checksums,
manifests, keep-alives, abort handling, per-file remove-source results, and
+49 -36
View File
File diff suppressed because one or more lines are too long
+1
View File
@@ -0,0 +1 @@
top
Symlink
+1
View File
@@ -0,0 +1 @@
b.txt
+1
View File
@@ -0,0 +1 @@
nested
+1
View File
@@ -0,0 +1 @@
top
Symlink
+1
View File
@@ -0,0 +1 @@
b.txt
+1
View File
@@ -0,0 +1 @@
nested
+67 -4
View File
@@ -123,8 +123,15 @@ static char itemize_type_char(const ChangeEvent* event) {
static bool times_match(const Config* config, const ChangeEvent* event) {
if (!event->dest.known || !event->dest.existed)
return false;
if (event->mtime_sec == event->dest.mtime_sec)
if (event->mtime_sec == event->dest.mtime_sec) {
/* A regular file's sub-second mtime IS preserved by the receiver, so an nsec
difference is a real change. A directory or symlink has no preserved
sub-second mtime (rsync's quick-check compares whole seconds there), so a
nanosecond-only difference must not render a spurious `.d..t` / `.L..t`. */
if (event->is_directory || event->is_symlink || event->is_special)
return true;
return event->mtime_nsec == event->dest.mtime_nsec;
}
long long delta = (long long)event->mtime_sec - (long long)event->dest.mtime_sec;
if (delta < 0)
delta = -delta;
@@ -145,6 +152,10 @@ static void itemize_code(const Config* config, const ChangeEvent* event, char co
/* rsync: an existing directory that only has attribute changes carries no
transfer, so the update column is `.` rather than `>`. */
update = '.';
else if (event->is_symlink)
/* rsync: an existing symlink whose target is unchanged is a `.` update
(attributes only); a changed target is `c` (the link value changed). */
update = event->dest.target_matches ? '.' : 'c';
else
update = '>';
code[0] = update;
@@ -155,12 +166,20 @@ static void itemize_code(const Config* config, const ChangeEvent* event, char co
code[11] = '\0';
return;
}
bool size_diff = event->size != event->dest.size;
bool time_diff = !times_match(config, event);
/* rsync's value/checksum column: `c` for a symlink whose target changed (the
link value is the compared content); no destination digest is available for
a regular file. */
bool value_diff = event->is_symlink && !event->dest.target_matches;
/* rsync itemizes size only for regular files: a directory's st_size and a
symlink's target length are not compared. */
bool size_diff = !event->is_directory && !event->is_symlink && !event->is_special &&
event->size != event->dest.size;
/* rsync itemizes the time column only when -t/--times is in effect. */
bool time_diff = config->preserve_times && !times_match(config, event);
bool perms_diff = (event->mode & 07777) != (event->dest.mode & 07777);
bool owner_diff = event->uid != (uid_t)event->dest.uid;
bool group_diff = event->gid != (gid_t)event->dest.gid;
code[2] = '.'; /* checksum: no destination digest available */
code[2] = value_diff ? 'c' : '.';
code[3] = size_diff ? 's' : '.';
code[4] = time_diff ? 't' : '.';
code[5] = (config->preserve_perms && perms_diff) ? 'p' : '.';
@@ -172,6 +191,24 @@ static void itemize_code(const Config* config, const ChangeEvent* event, char co
code[11] = '\0';
}
/* True when the itemized destination entry is unchanged, i.e. rsync would print
* no line at all. Reuses itemize_code so suppression is exactly consistent
* with what would have been rendered: the update column must be `.` and every
* attribute column must be `.`. */
static bool itemize_is_unchanged(const Config* config, const ChangeEvent* event) {
if (!event->dest.known || !event->dest.existed)
return false;
char code[12];
itemize_code(config, event, code);
if (code[0] != '.')
return false;
for (int i = 2; i < 11; i++) {
if (code[i] != '.')
return false;
}
return true;
}
/* rsync %n: the transfer-relative name, with a trailing slash for directories.
* The transfer root is `.` (so `%n` renders `./`), matching rsync's root entry. */
static bool append_name(StrBuf* buf, const ChangeEvent* event) {
@@ -678,6 +715,19 @@ void change_emit_file_sent_bytes(const Config* config, const File* file,
char* name = NULL;
char* path = NULL;
fill_event_from_file(config, file, &event, &name, &path);
if (file->is_symlink) {
/* Output parity (protocol 2.30.0): an unchanged symlink is silent, like
rsync's quick check. The itemize/log stream suppresses it only when every
attribute matches; the name stream suppresses it whenever the link target
is unchanged (rsync names a symlink only when it relinks or creates it). */
bool itemize_output = config->itemize_changes || config->out_format != NULL ||
(config->log_file != NULL && config->log_file_format != NULL);
bool suppress = itemize_output
? itemize_is_unchanged(config, &event)
: (event.dest.known && event.dest.existed && event.dest.target_matches);
if (suppress)
event.decision = CHANGE_UP_TO_DATE;
}
if (name != NULL && path != NULL) {
fill_event_checksum(config, file, &event);
change_emit(config, &event);
@@ -729,6 +779,19 @@ void change_emit_dir_sent(const Config* config, const File* file) {
char* name = NULL;
char* path = NULL;
fill_event_from_file(config, file, &event, &name, &path);
/* Output parity (protocol 2.30.0): suppress a directory rsync would leave
silent. The itemize/log stream suppresses it only when every attribute
matches (`.d.........`); the name stream suppresses any pre-existing
directory (rsync names a directory only when it is created). */
bool itemize_output = config->itemize_changes || config->out_format != NULL ||
(config->log_file != NULL && config->log_file_format != NULL);
bool suppress = itemize_output ? itemize_is_unchanged(config, &event)
: (event.dest.known && event.dest.existed);
/* The transfer root's line is an unconditional FastSync residual (rsync keys
it off the root's own attribute change); keep emitting it. */
bool is_root = event.name != NULL && event.name[0] == '\0';
if (suppress && !is_root)
event.decision = CHANGE_UP_TO_DATE;
if (name != NULL && path != NULL)
change_emit(config, &event);
free(name);
+21 -10
View File
@@ -433,12 +433,10 @@ static int set_stderr_mode(const char* value) {
log_set_stderr_mode(LOG_STDERR_ERRORS);
else if (strcmp(value, "all") == 0 || strcmp(value, "a") == 0)
log_set_stderr_mode(LOG_STDERR_ALL);
else if (strcmp(value, "client") == 0 || strcmp(value, "c") == 0) {
log_message(LOG_LEVEL_ERROR,
"--stderr=client is not supported: FastSync has no client message channel");
return -1;
} else {
log_message(LOG_LEVEL_ERROR, "--stderr must be errors or all");
else if (strcmp(value, "client") == 0 || strcmp(value, "c") == 0)
log_set_stderr_mode(LOG_STDERR_CLIENT);
else {
log_message(LOG_LEVEL_ERROR, "--stderr must be errors, all, or client");
return -1;
}
return 0;
@@ -1353,10 +1351,9 @@ static bool cli_handle_pre_negation(CliParseCtx* ctx) {
return true;
}
/* "--no-msgs2stderr" is the deprecated spelling of --stderr=client (rsync
* 3.4.1). FastSync has no separate client message channel, so the closest
* supported mode is the errors-only default. */
* 3.4.1); the client-message channel now exists, so it maps to `client`. */
if (strcmp(arg, "--no-msgs2stderr") == 0)
return set_stderr_mode("errors") == 0;
return set_stderr_mode("client") == 0;
/* "--no-motd" is a real rsync option name (client-side daemon MOTD display
* suppression), not a negation of a "--motd" flag, so it is handled before
* the generic --no-* negation branch. */
@@ -2629,6 +2626,17 @@ static int cli_finalize_config(Config* config, bool verbose, bool no_delta, bool
bool debug_enabled = verbose || config->debug_level != 0;
set_log_level(config->quiet ? LOG_LEVEL_ERROR
: (debug_enabled ? LOG_LEVEL_DEBUG : LOG_LEVEL_WARNING));
/* rsync parity: --delay-updates implies --delete-after. Every staged file is
published first and only then are extras removed. Normalize onto the
existing delete_after wire bool (no new wire field), overriding any other
explicit timing exactly as rsync does; without --delete there is no
deletion, so no timing is set (and the wire config stays valid). */
if (config->delay_updates && config->use_delete) {
config->delete_before = false;
config->delete_during = false;
config->delete_delay = false;
config->delete_after = true;
}
/* rsync's plain --delete defaults to delete-during (--del): each directory's
extras are removed as that directory is processed, so space is freed
progressively and a tight destination never has to hold the whole old+new
@@ -2810,8 +2818,11 @@ static int cli_finalize_config(Config* config, bool verbose, bool no_delta, bool
* need the pre-transfer destination snapshot (new vs modified and which
* attributes differ), so ask the receiver to report it on every per-file
* check. This is a wire field. */
bool progress_active =
!config->quiet && (config->show_progress || (config->info_level & LOG_INFO_PROGRESS) != 0);
config->report_dest_info = config->itemize_changes || config->out_format != NULL ||
(config->log_file != NULL && config->log_file_format != NULL);
(config->log_file != NULL && config->log_file_format != NULL) ||
progress_active;
/* Wire-stats parity: --stats, --progress/-P, an --out-format token that needs
* a wire counter (%b/%c), or a dry-run --delete need the receiver's
* end-of-transfer STATUS_STATS report. This is a wire field (protocol
+34 -37
View File
@@ -64,16 +64,8 @@ bool add_chunk_to_manifest(ArrayList* manifest, const Chunk* chunk) {
int send_dry_run_manifest(const Config* config) {
int skipped = 0;
ArrayList* missing_dest = NULL;
if (config->delete_missing_args) {
missing_dest = array_list_create(free);
if (!missing_dest)
return -1;
}
if (!files_from_list_check(config, missing_dest, &skipped)) {
if (missing_dest)
array_list_delete(missing_dest);
if (!client_prepare_files_from(config, &missing_dest, &skipped))
return -1;
}
PreparedScanner prepared;
if (!prepare_scanner(config, 0, &prepared)) {
if (missing_dest)
@@ -330,9 +322,11 @@ int send_list_only(const Config* config) {
}
/* Send the delete manifest to the server. Returns 0 on success, -1 on
failure. It carries FOUR sections: the keep-set paths, the protected
excluded prefixes, the --delete-missing-args exact-delete paths, and the
destination-relative directories the sender synchronized this run.
failure. It carries FOUR path sections (keep-set paths, protected excluded
prefixes, --delete-missing-args exact-delete paths, and the destination-
relative directories the sender synchronized this run) followed by the
protocol-2.30.0 per-directory filter-rule block (`per_dir_rules`, the rules
the scan compiled from each directory's merge files).
When --delete-excluded is given `protected` is empty: excluded destination
mirrors are then ordinary extras and are removed. When
--delete-missing-args is active `missing_args` holds the destination mirrors
@@ -346,7 +340,8 @@ int send_list_only(const Config* config) {
frame. A heavily filtered source whose exclusion list is large therefore
fails the run cleanly on the receiver rather than being truncated. */
int send_delete_manifest(int fd, ArrayList* manifest, ArrayList* protected_prefixes,
ArrayList* size_skipped, ArrayList* missing_args, ArrayList* synced_dirs) {
ArrayList* size_skipped, ArrayList* missing_args, ArrayList* synced_dirs,
const FilterRuleList* per_dir_rules) {
if (!send_status(fd, STATUS_MANIFEST))
return -1;
int keep_count = manifest ? manifest->size : 0;
@@ -389,6 +384,11 @@ int send_delete_manifest(int fd, ArrayList* manifest, ArrayList* protected_prefi
if (!send_wire_str(fd, (char*)synced_dirs->items[i]))
return -1;
}
/* Protocol 2.30.0: the receiver-side per-directory filter rules discovered by
the sender's scan, so the whole-tree commit walker can shield a
destination-only entry that matches only a per-directory merge rule. */
if (!delete_filter_dir_rules_send(fd, per_dir_rules))
return -1;
return 0;
}
@@ -409,11 +409,11 @@ int send_delete_manifest(int fd, ArrayList* manifest, ArrayList* protected_prefi
bool send_delete_manifest_early(Client* client, ArrayList* manifest, ArrayList* protected_prefixes,
ArrayList* size_skipped, ArrayList* missing_args,
ArrayList* synced_dirs) {
ArrayList* synced_dirs, const FilterRuleList* per_dir_rules) {
if (!client || !manifest)
return false;
if (send_delete_manifest(client->file_descriptor, manifest, protected_prefixes, size_skipped,
missing_args, synced_dirs) != 0)
missing_args, synced_dirs, per_dir_rules) != 0)
return false;
Status ack;
/* The wait is long (up to an hour) and runs inline on this thread: a helper
@@ -458,35 +458,21 @@ bool send_delete_manifest_early(Client* client, ArrayList* manifest, ArrayList*
int send_dry_run_remote(Config* config) {
int from_skipped = 0;
ArrayList* missing_args = NULL;
if (config->delete_missing_args) {
missing_args = array_list_create(free);
if (!missing_args)
return 1;
}
if (!files_from_list_check(config, missing_args, &from_skipped)) {
if (missing_args)
array_list_delete(missing_args);
if (!client_prepare_files_from(config, &missing_args, &from_skipped))
return 1;
}
if (missing_args)
array_list_delete(missing_args);
/* A live session may follow, so arm graceful abort handling. */
client_set_abort_armed(true);
Client* client = connect_transfer_client(config);
ProtocolSession session;
Client* client = client_connect_and_bind_session(config, &session);
if (!client) {
if (config->transport == TRANSPORT_TCP)
log_message(LOG_LEVEL_ERROR, "could not connect to server%s",
config->use_tls ? " via TLS" : "");
client_set_abort_armed(false);
return 1;
}
ProtocolSession session;
protocol_session_init(&session, client->file_descriptor, client->file_descriptor);
protocol_session_set_io_timeout(&session, config->timeout);
protocol_session_set_ssl(&session, (SSL*)client->ssl);
protocol_session_bind(&session);
int ret = 1;
bool partial = false;
time_t dry_start = time(NULL);
ReceiverStats dry_stats;
memset(&dry_stats, 0, sizeof(dry_stats));
@@ -497,6 +483,7 @@ int send_dry_run_remote(Config* config) {
ArrayList* dry_dirs = NULL;
ArrayList* dry_excluded = NULL;
ArrayList* dry_size_skipped = NULL;
FilterRuleList* dry_per_dir = NULL;
if (!config_send(client->file_descriptor, config))
goto dry_fail;
receive_daemon_motd(client, config);
@@ -509,8 +496,10 @@ int send_dry_run_remote(Config* config) {
dry_manifest = array_list_create(free);
dry_dirs = array_list_create(free);
dry_size_skipped = array_list_create(free);
if (!dry_manifest || !dry_dirs || !dry_size_skipped)
dry_per_dir = filter_rule_list_create();
if (!dry_manifest || !dry_dirs || !dry_size_skipped || !dry_per_dir)
goto dry_fail;
prepared.options.per_dir_rules = dry_per_dir;
if (!config->delete_excluded) {
dry_excluded = array_list_create(free);
if (!dry_excluded)
@@ -612,7 +601,7 @@ int send_dry_run_remote(Config* config) {
bool early_delete = config->use_delete && config_delete_timing_early(config);
if (dry_manifest) {
if (send_delete_manifest(client->file_descriptor, dry_manifest, dry_excluded, dry_size_skipped,
NULL, dry_dirs) != 0)
NULL, dry_dirs, dry_per_dir) != 0)
goto dry_fail;
if (early_delete) {
Status ack;
@@ -642,8 +631,14 @@ int send_dry_run_remote(Config* config) {
if (!receive_status(client->file_descriptor, &status))
goto dry_fail;
}
if (status != STATUS_OK)
/* A per-entry receiver failure is rsync's PARTIAL transfer (exit 23), not a
hard failure: a dry run transfers nothing, but keep the verdict consistent
with the normal path instead of treating it as a protocol error. */
if (status == STATUS_PARTIAL) {
partial = true;
} else if (status != STATUS_OK) {
goto dry_fail;
}
if (!config->quiet) {
if (config->human_readable)
printf("Total: %d files, %s\n", file_count,
@@ -661,7 +656,7 @@ int send_dry_run_remote(Config* config) {
dry_transfer.literal_data = total_bytes;
report_transfer_stats(config, &dry_transfer, dry_start, &dry_stats);
}
ret = io_error ? 1 : 0;
ret = io_error ? 1 : (partial ? 23 : 0);
dry_fail:
if (dry_manifest)
@@ -672,6 +667,8 @@ dry_fail:
array_list_delete(dry_excluded);
if (dry_size_skipped)
array_list_delete(dry_size_skipped);
if (dry_per_dir)
filter_rule_list_free(dry_per_dir);
if (scanner)
directory_scanner_destroy(scanner);
prepared_scanner_destroy(&prepared);
+272 -4
View File
@@ -12,6 +12,7 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <threads.h>
#include <time.h>
/* Surface a server rejection to the user. When the last status exchange
@@ -161,6 +162,9 @@ void report_transfer_stats(const Config* config, const TransferStats* stats, tim
created_breakdown, sizeof(created_breakdown));
unsigned long long created_total =
recv->created_reg + recv->created_dir + recv->created_link + recv->created_special;
char deleted_breakdown[128];
type_breakdown(recv->deleted_reg, recv->deleted_dir, recv->deleted_link, recv->deleted_special,
deleted_breakdown, sizeof(deleted_breakdown));
printf("\n");
if (breakdown[0] != '\0')
printf("Number of files: %llu %s\n", flist_total, breakdown);
@@ -172,7 +176,13 @@ void report_transfer_stats(const Config* config, const TransferStats* stats, tim
printf("Number of created files: %llu %s\n", created_total, created_breakdown);
else
printf("Number of created files: %llu\n", created_total);
printf("Number of deleted files: %llu\n", recv->deleted_files);
/* Protocol 2.30.0: the receiver reports the removed entries split by type, so
this line matches rsync's `Number of deleted files: X (reg: A, dir: B,
link: C, special: D)` (only the non-zero categories are listed). */
if (deleted_breakdown[0] != '\0')
printf("Number of deleted files: %llu %s\n", recv->deleted_files, deleted_breakdown);
else
printf("Number of deleted files: %llu\n", recv->deleted_files);
printf("Number of regular files transferred: %llu\n", stats->transferred_regular);
printf("Total file size: %s bytes\n", total);
printf("Total transferred file size: %s bytes\n", transferred);
@@ -525,9 +535,16 @@ static void client_progress_emit_ancestors(const Config* config, const char* rel
if (dir != NULL)
change_emit_dir_sent(config, dir);
} else {
char* escaped = output_escape(prefix, config->eight_bit_output);
printf("%s/\n", escaped ? escaped : prefix);
free(escaped);
/* --progress/-P names a directory only when it is newly created;
rsync stays silent for a pre-existing directory even when one of
its children changed (protocol 2.30.0 dest-state report). */
const File* dir = progress_dir_lookup(prefix);
bool existed = dir != NULL && dir->dest_state.known && dir->dest_state.existed;
if (!existed) {
char* escaped = output_escape(prefix, config->eight_bit_output);
printf("%s/\n", escaped ? escaped : prefix);
free(escaped);
}
}
g_progress_index++;
} else {
@@ -552,6 +569,122 @@ void client_change_emit_ancestors(const Config* config, const File* file) {
client_progress_emit_ancestors(config, rel);
}
/* Send one STATUS_MKDIR probe (probe=1) for a pre-count directory and cache the
* receiver's pre-transfer destination snapshot in `dir->dest_state`. Returns
* false on a protocol/transport error. */
static bool client_probe_dir_state(int fd, File* dir) {
if (dir == NULL || file_wire_path(dir) == NULL)
return true;
if (!send_status(fd, STATUS_MKDIR) || !send_int(fd, 1) || !send_wire_str(fd, file_wire_path(dir)))
return false;
Status status = STATUS_ERROR;
if (!receive_status(fd, &status) || status != STATUS_DEST_INFO ||
!format_dest_state_receive(fd, &dir->dest_state)) {
log_message(LOG_LEVEL_ERROR, "Directory destination-state probe failed");
return false;
}
return true;
}
/* Output parity (protocol 2.30.0): ask the receiver for each not-yet-probed
* ancestor directory's pre-transfer state BEFORE the entry that first triggers
* it is sent, so the ancestor's -i/--out-format line renders rsync's
* `.d..t......` (existing, attributes changed) versus `cd+++++++++` (created)
* and an unchanged directory is suppressed. The probe is a STATUS_MKDIR frame
* with probe=1 (the receiver reports and creates nothing), so it must run before
* the receiver implicitly creates the parent for the child. Each directory is
* probed at most once; the result is cached in the pre-count File's dest_state.
* Returns false on a protocol/transport error (the caller aborts the transfer). */
bool client_change_probe_ancestors(const Config* config, const File* file, int fd) {
if (config == NULL || file == NULL || fd < 0 || !config->report_dest_info)
return true;
if (!g_progress_dir_index_valid || !g_progress_precount.dir_refs)
return true;
const char* rel = delete_display_path(config, file_wire_path(file));
if (rel == NULL)
return true;
size_t rel_len = strlen(rel);
for (size_t i = 1; i < rel_len; i++) {
if (rel[i] != '/')
continue;
char* prefix = malloc(i + 1);
if (prefix == NULL)
return false;
memcpy(prefix, rel, i);
prefix[i] = '\0';
if (path_index_contains(&g_progress_dir_index, prefix)) {
File* dir = progress_dir_lookup(prefix);
/* The probe path is the same wire path the real STATUS_MKDIR would carry
(the pre-count File's send_path, or its absolute source path for a
plain recursive scan), not the display-relative prefix. */
if (dir != NULL && !dir->dest_state.known && !client_probe_dir_state(fd, dir)) {
free(prefix);
return false;
}
}
free(prefix);
}
return true;
}
/* Mark a directory the data pass already itemized/named so the end-of-transfer
* pending-directory flush does not report it a second time. `file` is a
* transferred directory entry (an empty-directory STATUS_MKDIR). */
void client_change_mark_dir(const Config* config, const File* file) {
if (config == NULL || file == NULL || !g_progress_emitted_valid ||
g_progress_emitted_keys == NULL)
return;
const char* rel = delete_display_path(config, file_wire_path(file));
if (rel == NULL || rel[0] == '\0' || str_hash_set_lookup(&g_progress_emitted, rel))
return;
char* key = str_dup(rel);
if (key == NULL)
return;
if (!array_list_add(g_progress_emitted_keys, key)) {
free(key);
return;
}
str_hash_set_insert_ref(&g_progress_emitted, key);
}
/* Emit the itemize lines for source directories that CHANGED but had no
* transferred child, so no ancestor emission reached them (rsync reports a
* directory whose attributes changed even when its contents did not). Runs at
* the end of the data pass, BEFORE the deferred STATUS_DIR_TIMES apply, so the
* probe still observes each untouched directory's pre-transfer state. Only the
* itemize/out-format/log streams report attribute-only directory changes;
* --progress/-P stays silent for them, matching rsync. Best-effort: a probe
* failure simply stops the flush (the transfer's verdict is unaffected). */
void client_change_emit_pending_dirs(const Config* config, int fd) {
if (config == NULL || fd < 0 || !config->report_dest_info)
return;
bool itemize_output = config->itemize_changes || config->out_format != NULL ||
(config->log_file != NULL && config->log_file_format != NULL);
if (!itemize_output)
return;
if (!g_progress_dir_index_valid || g_progress_precount.dir_refs == NULL ||
!g_progress_emitted_valid || g_progress_emitted_keys == NULL)
return;
for (int i = 0; i < g_progress_precount.dir_refs->size; i++) {
DirRef* ref = (DirRef*)g_progress_precount.dir_refs->items[i];
if (ref == NULL || ref->name == NULL || ref->name[0] == '\0')
continue;
if (str_hash_set_lookup(&g_progress_emitted, ref->name))
continue;
File* dir = ref->file;
if (dir == NULL)
continue;
if (!dir->dest_state.known && !client_probe_dir_state(fd, dir))
return;
change_emit_dir_sent(config, dir);
char* key = str_dup(ref->name);
if (key != NULL && array_list_add(g_progress_emitted_keys, key))
str_hash_set_insert_ref(&g_progress_emitted, key);
else
free(key);
}
}
/* rsync's --info=name/progress line for one entry: transfer-relative name (a
* trailing slash for directories) plus the ` -> target` symlink suffix. */
static char* progress_entry_line(const File* file, const char* rel) {
@@ -644,6 +777,15 @@ void client_progress_file(const Config* config, const File* file) {
void client_progress_name(const Config* config, const File* file) {
if (!g_progress_active || file == NULL)
return;
/* rsync's --progress/-P name stream reports an entry only when it is created
or (for a symlink) actually relinked: a pre-existing directory or an
unchanged symlink is silent (protocol 2.30.0 dest-state report). */
if (file->dest_state.known && file->dest_state.existed) {
if (file->is_dir || (file->is_symlink && file->dest_state.target_matches)) {
g_progress_index++;
return;
}
}
const char* rel = delete_display_path(config, file_wire_path(file));
if (!config->itemize_changes && config->out_format == NULL) {
char* line = progress_entry_line(file, rel ? rel : "");
@@ -1051,3 +1193,129 @@ const char* delete_display_path(const Config* config, const char* path) {
return path;
return utils_strip_transfer_root(path, config->send_directory);
}
/* ---- --stderr=client diagnostic channel (protocol 2.30.0) ----
*
* When the client's --stderr mode is `client`, log_message() hands each of the
* client's own diagnostics to the sink installed here instead of writing them
* locally. The sink QUEUES the text (it may be called from scanner worker
* threads while the sender is streaming) and the sender thread -- the sole
* writer of the protocol stream -- drains the queue over the wire at frame
* boundaries via client_flush_client_messages(). A bounded queue caps the
* memory a chatty run can pin; overflow falls back to local output so a
* diagnostic is never silently dropped. */
#define CLIENT_MSG_MAX_QUEUED 256
#define CLIENT_MSG_MAX_BYTES (256 * 1024)
static mtx_t client_msg_mutex;
static once_flag client_msg_mutex_once = ONCE_FLAG_INIT;
static ArrayList* client_msg_queue = NULL; /* owns char* */
static size_t client_msg_bytes = 0;
/* True only while a live transfer session exists: before the connection is up
(or after it drops) the sink declines so log_message falls back to local
output, matching rsync's documented fallback. Written by the sender thread
(client_messages_activate) and read by scanner worker threads in
client_msg_enqueue, so it must be atomic: the queue itself stays guarded by
client_msg_mutex, but the flag is polled before taking that lock. */
static _Atomic bool client_msg_active = false;
static void client_msg_mutex_init(void) {
mtx_init(&client_msg_mutex, mtx_plain);
}
static bool client_msg_enqueue(const char* message);
/* Install the global log sink for the duration of one transfer. Safe to call
* more than once; the queue is created lazily. */
void client_messages_install(void) {
call_once(&client_msg_mutex_once, client_msg_mutex_init);
mtx_lock(&client_msg_mutex);
if (!client_msg_queue)
client_msg_queue = array_list_create(free);
bool ready = client_msg_queue != NULL;
mtx_unlock(&client_msg_mutex);
/* Only arm the sink once the queue exists; on allocation failure leave the
sink uninstalled so log_message keeps writing locally instead of handing
messages to a sink that would silently drop them. */
if (ready)
log_set_client_msg_sink(client_msg_enqueue);
}
void client_messages_activate(bool active) {
atomic_store(&client_msg_active, active);
}
/* log_message sink: takes ownership (queues) the message when a session is
* live; returns false otherwise so the caller writes it locally. */
static bool client_msg_enqueue(const char* message) {
bool active = atomic_load(&client_msg_active);
if (!message || message[0] == '\0')
return active;
if (!active)
return false;
size_t len = strlen(message);
call_once(&client_msg_mutex_once, client_msg_mutex_init);
mtx_lock(&client_msg_mutex);
bool queued = false;
if (client_msg_queue && (size_t)client_msg_queue->size < CLIENT_MSG_MAX_QUEUED &&
client_msg_bytes + len <= CLIENT_MSG_MAX_BYTES) {
char* copy = str_dup(message);
if (copy) {
if (array_list_add(client_msg_queue, copy)) {
client_msg_bytes += len;
queued = true;
} else {
free(copy);
}
}
}
mtx_unlock(&client_msg_mutex);
return queued;
}
/* Drain the queued diagnostics as STATUS_CLIENT_MSG frames on the sender
* thread. Swaps the queue out under the mutex so a concurrent worker logging
* never blocks on the wire. Must be called at a protocol frame boundary. */
void client_flush_client_messages(int fd) {
if (fd < 0)
return;
call_once(&client_msg_mutex_once, client_msg_mutex_init);
mtx_lock(&client_msg_mutex);
ArrayList* pending = client_msg_queue;
if (pending) {
ArrayList* fresh = array_list_create(free);
if (fresh) {
client_msg_queue = fresh;
} else {
/* No memory for a replacement queue: stop queuing new diagnostics (they
fall back to local output) and drain this batch below so nothing is
silently dropped. */
client_msg_queue = NULL;
log_set_client_msg_sink(NULL);
}
client_msg_bytes = 0;
}
mtx_unlock(&client_msg_mutex);
if (!pending)
return;
for (int i = 0; i < pending->size; i++) {
const char* message = pending->items[i];
if (message && message[0] != '\0' && !send_client_message(fd, message))
break; /* peer is gone; the rest would fail too */
}
array_list_delete(pending);
}
/* Tear down the sink after a transfer and free anything still queued. */
void client_messages_end(void) {
log_set_client_msg_sink(NULL);
atomic_store(&client_msg_active, false);
call_once(&client_msg_mutex_once, client_msg_mutex_init);
mtx_lock(&client_msg_mutex);
ArrayList* pending = client_msg_queue;
client_msg_queue = NULL;
client_msg_bytes = 0;
mtx_unlock(&client_msg_mutex);
if (pending)
array_list_delete(pending);
}
+230 -66
View File
@@ -127,10 +127,59 @@ Client* connect_transfer_client(const Config* config) {
void disconnect_transfer_client(Client* client) {
if (!client)
return;
/* --stderr=client: push any diagnostics logged during the transfer to the
peer before the socket closes; once deactivated, later messages fall back
to local output instead of being lost. */
client_flush_client_messages(client->file_descriptor);
client_messages_activate(false);
client_disconnect(client);
client_delete(client);
}
/* Connect the configured transport and install the per-thread protocol session
* on it: init with the socket fd pair, apply the I/O timeout and (when
* negotiated) the TLS object, then bind it to this thread. Returns the
* connected client, or NULL (after logging the connect failure) when the
* transport could not connect. */
Client* client_connect_and_bind_session(const Config* config, ProtocolSession* session) {
Client* client = connect_transfer_client(config);
if (!client) {
if (config->transport == TRANSPORT_TCP)
log_message(LOG_LEVEL_ERROR, "could not connect to server%s",
config->use_tls ? " via TLS" : "");
return NULL;
}
protocol_session_init(session, client->file_descriptor, client->file_descriptor);
protocol_session_set_io_timeout(session, config->timeout);
protocol_session_set_ssl(session, (SSL*)client->ssl);
protocol_session_bind(session);
return client;
}
/* Shared --files-from/--delete-missing-args preamble: allocate the missing-args
* destination list when the option is set, then validate the --files-from list
* (collecting the destination mirrors of missing entries for the receiver's
* exact-deletion request). On success the caller owns *missing_args_out (NULL
* when the option is off); on failure the list is freed and false is returned. */
bool client_prepare_files_from(const Config* config, ArrayList** missing_args_out,
int* skipped_out) {
ArrayList* missing_args = NULL;
if (config->delete_missing_args) {
missing_args = array_list_create(free);
if (!missing_args)
return false;
}
int skipped = 0;
if (!files_from_list_check(config, missing_args, &skipped)) {
if (missing_args)
array_list_delete(missing_args);
return false;
}
*missing_args_out = missing_args;
*skipped_out = skipped;
return true;
}
/* (finalize_transfer is defined after the SourceFile helpers below.) */
typedef struct SourceFile {
@@ -240,13 +289,28 @@ static void mark_sender_done(PipelineContextSender* context) {
When --remove-source-files is active the receiver acknowledges each data
file it processed, in send order: STATUS_NEXT means the file was written,
STATUS_OK means the file was skipped/unchanged. Skipped sources are marked
so the later removal pass keeps them. */
so the later removal pass keeps them. `partial_out` is set when the receiver
reported STATUS_PARTIAL (a per-entry receiver failure): the transfer is
otherwise complete, so successfully stored sources are still removed and the
caller exits 23 (rsync's partial transfer) instead of a fatal non-zero. */
static bool finalize_transfer(Client* client, const Config* config, ArrayList* remove_sources,
bool* delete_limit_out, ReceiverStats* stats_out) {
bool* delete_limit_out, bool* partial_out, ReceiverStats* stats_out) {
if (delete_limit_out)
*delete_limit_out = false;
if (partial_out)
*partial_out = false;
/* --stderr=client: the receiver consumes frames until it reads
STATUS_FINISHED, after which it no longer reads. Flush every diagnostic
queued during the transfer here -- the last frame boundary at which the
peer is still reading -- so nothing is stranded in the queue. */
client_flush_client_messages(client->file_descriptor);
if (!send_status(client->file_descriptor, STATUS_FINISHED))
return false;
/* Past STATUS_FINISHED the receiver has stopped reading, so any diagnostic
logged from here on (notably the STATUS_PARTIAL warning below) can no
longer be forwarded. Deactivate the channel so those messages fall back
to local output instead of being queued for a closed peer and lost. */
client_messages_activate(false);
/* The receiver emits its optional wire-stats frame (protocol 2.25.0) FIRST,
then any per-file --remove-source-files acks, then the terminal status. */
Status status;
@@ -298,6 +362,16 @@ static bool finalize_transfer(Client* client, const Config* config, ArrayList* r
*delete_limit_out = true;
return true;
}
/* A per-entry receiver failure the receiver chose to continue past is a
rsync PARTIAL transfer: everything else succeeded and the stored sources
may be removed, but the client must exit 23. */
if (status == STATUS_PARTIAL) {
log_message(LOG_LEVEL_WARNING,
"some files could not be transferred (see the server log for details)");
if (partial_out)
*partial_out = true;
return true;
}
if (status != STATUS_OK) {
log_server_rejection("Receiver reported transfer failure");
return false;
@@ -595,14 +669,34 @@ static bool send_file_direct(File* file, int fd, bool use_metadata, int compress
static bool send_directory_entry(const Client* client, File* file, const Config* config) {
if (!file || !file_wire_path(file))
return false;
if (!send_status(client->file_descriptor, STATUS_MKDIR) ||
!send_wire_str(client->file_descriptor, file_wire_path(file)))
int fd = client->file_descriptor;
if (!send_status(fd, STATUS_MKDIR))
return false;
if (config->use_metadata && !metadata_send(client->file_descriptor, file->metadata))
/* Output parity (protocol 2.30.0): when report_dest_info is negotiated every
STATUS_MKDIR body is prefixed with a probe flag (1 = probe only, 0 = a real
create), so the receiver knows whether to expect the metadata/xattr block. */
if (config->report_dest_info && !send_int(fd, 0))
return false;
if (!send_wire_str(fd, file_wire_path(file)))
return false;
if (config->use_metadata && !metadata_send(fd, file->metadata))
return false;
/* Directory xattrs/ACLs (-X/-A) ride the same trailing block as regular files
when the xattr transport was negotiated. */
return !config->use_xattrs || xattr_send(client->file_descriptor, file->xattrs);
if (config->use_xattrs && !xattr_send(fd, file->xattrs))
return false;
/* The receiver answers with the directory's pre-transfer destination state
BEFORE creating it, so the sender can render rsync's `.d..t......` versus
`cd+++++++++` and suppress an unchanged directory. */
if (config->report_dest_info) {
Status status;
if (!receive_status(fd, &status) || status != STATUS_DEST_INFO ||
!format_dest_state_receive(fd, &file->dest_state)) {
log_message(LOG_LEVEL_ERROR, "Unexpected reply to the directory destination-state report");
return false;
}
}
return true;
}
/* P7 Wave D: transmit every captured source directory's metadata in terminal
@@ -658,7 +752,21 @@ static bool send_symlink_entry(const Client* client, File* file, const Config* c
return false;
/* Symlink xattrs/ACLs (-X/-A) ride the same trailing block as regular files
and directories when the xattr transport was negotiated. */
return !config->use_xattrs || xattr_send(fd, file->xattrs);
if (config->use_xattrs && !xattr_send(fd, file->xattrs))
return false;
/* The receiver answers with the symlink's pre-transfer destination state
(including whether the on-disk link target already matches) BEFORE creating
it, so the sender can render rsync's `cLc........` / `.L..t......` and
suppress an unchanged symlink. */
if (config->report_dest_info) {
Status status;
if (!receive_status(fd, &status) || status != STATUS_DEST_INFO ||
!format_dest_state_receive(fd, &file->dest_state)) {
log_message(LOG_LEVEL_ERROR, "Unexpected reply to the symlink destination-state report");
return false;
}
}
return true;
}
// Send a single file directly via sendfile (non-incremental path).
@@ -813,8 +921,28 @@ static bool source_is_regular_file(const File* file) {
return stat(file->path, &st) == 0 && S_ISREG(st.st_mode);
}
/* True when an over-threshold source will be sent by STREAMING from disk rather
* than loaded into memory: either the zero-copy sendfile path (no compression)
* or the sender-side streaming compressor (zstd/zlib, when this file is not on
* the --skip-compress list). Otherwise the loader must materialize it. */
static bool loader_can_stream(const Config* config, const File* file) {
if (!config || !file || !file->data || file->data->size <= STREAM_THRESHOLD)
return false;
if (!config->use_compression)
return true;
if (!compression_stream_compress_supported(compression_get_algo()) ||
config->compression_level <= 0)
return false;
int skip_count = config->skip_compress_set ? config->skip_compress_count : -1;
return !compression_should_skip_with_suffixes(file->path, config->skip_compress_suffixes,
skip_count);
}
static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config,
ArrayList* remove_sources, TransferStats* stats) {
/* --stderr=client: this is a frame boundary, so forward any diagnostics the
scanner/log emitted since the previous chunk before the next frame. */
client_flush_client_messages(client->file_descriptor);
if (config->use_chunk_serialization) {
if (remove_sources) {
for (int i = 0; i < chunk->element_count; i++) {
@@ -842,14 +970,23 @@ static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config,
if (chunk->items[i] == NULL)
continue;
transfer_stats_note_entry(stats, chunk->items[i]);
/* Output parity: probe each entry's ancestor directories' destination
state before emitting its itemize line, exactly as the non-serialized
loop does. Without this, dest_state.known stays false and -i/-P
renders an existing dir/symlink as created instead of `.d..t...` (or
suppressing it). */
if (!client_change_probe_ancestors(config, chunk->items[i], client->file_descriptor))
return -1;
/* The chunk-serialization path emits no --progress name lines, so only
feed -i/--out-format its ancestor directory lines here. */
if (config->itemize_changes || config->out_format != NULL)
client_change_emit_ancestors(config, chunk->items[i]);
if (chunk->items[i]->is_dir)
if (chunk->items[i]->is_dir) {
change_emit_dir_sent(config, chunk->items[i]);
else
client_change_mark_dir(config, chunk->items[i]);
} else {
change_emit_file_sent(config, chunk->items[i]);
}
if (!chunk->items[i]->is_dir)
transfer_stats_note_transferred(stats, chunk->items[i]);
}
@@ -861,6 +998,11 @@ static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config,
if (f == NULL)
continue;
transfer_stats_note_entry(stats, f);
/* Output parity: probe this entry's ancestor directories' destination state
before the entry (or the first child below them) is sent, while the
receiver has not yet created them implicitly. */
if (!client_change_probe_ancestors(config, f, client->file_descriptor))
return -1;
if (f->is_dir) {
/* Explicit directory entry (--dirs): a MKDIR frame carrying the
destination path (and metadata when negotiated). Directories have no
@@ -869,6 +1011,7 @@ static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config,
return -1;
client_change_emit_ancestors(config, f);
change_emit_dir_sent(config, f);
client_change_mark_dir(config, f);
client_progress_name(config, f);
continue;
}
@@ -940,20 +1083,14 @@ static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config,
static int send_chunks_multithreaded(void* pipeline_context) {
PipelineContextSender* context = (PipelineContextSender*)pipeline_context;
time_t start = time(NULL);
Client* client = connect_transfer_client(context->config);
ProtocolSession session;
Client* client = client_connect_and_bind_session(context->config, &session);
if (!client) {
if (context->config->transport == TRANSPORT_TCP)
log_message(LOG_LEVEL_ERROR, "could not connect to server%s",
context->config->use_tls ? " via TLS" : "");
pipeline_cancel(context);
mark_sender_done(context);
return thrd_error;
}
ProtocolSession session;
protocol_session_init(&session, client->file_descriptor, client->file_descriptor);
protocol_session_set_io_timeout(&session, context->config->timeout);
protocol_session_set_ssl(&session, (SSL*)client->ssl);
protocol_session_bind(&session);
client_messages_activate(true);
if (!config_send(client->file_descriptor, context->config)) {
pipeline_cancel(context);
disconnect_transfer_client(client);
@@ -967,7 +1104,7 @@ static int send_chunks_multithreaded(void* pipeline_context) {
and wait for the receiver to delete extras before streaming any data. */
if (!send_delete_manifest_early(client, context->manifest, context->excluded_paths,
context->size_skipped_paths, context->missing_args,
context->synced_dirs)) {
context->synced_dirs, context->per_dir_rules)) {
pipeline_cancel(context);
disconnect_transfer_client(client);
mark_sender_done(context);
@@ -1097,7 +1234,8 @@ static int send_chunks_multithreaded(void* pipeline_context) {
log_message(LOG_LEVEL_WARNING, "IO error encountered -- skipping file deletion");
} else if (send_delete_manifest(client->file_descriptor, context->manifest,
context->excluded_paths, context->size_skipped_paths,
context->missing_args, context->synced_dirs) != 0) {
context->missing_args, context->synced_dirs,
context->per_dir_rules) != 0) {
goto send_fail;
}
} else if (context->config->delete_missing_args && !context->early_delete &&
@@ -1105,7 +1243,7 @@ static int send_chunks_multithreaded(void* pipeline_context) {
/* --delete-missing-args without --delete: no keep-set is built, but the
exact-delete paths still ride the same manifest frame (commit once the
transfer succeeded). */
if (send_delete_manifest(client->file_descriptor, NULL, NULL, NULL, context->missing_args,
if (send_delete_manifest(client->file_descriptor, NULL, NULL, NULL, context->missing_args, NULL,
NULL) != 0)
goto send_fail;
}
@@ -1113,15 +1251,19 @@ static int send_chunks_multithreaded(void* pipeline_context) {
(and all parallel workers) has been joined before scanner_done was set, so
the list is complete and race-free; on an early stop the list may be
incomplete and is deliberately not sent. */
client_change_emit_pending_dirs(context->config, client->file_descriptor);
if (!context->scan_stopped_early &&
!send_dir_times(client, context->config, context->dir_entries))
goto send_fail;
bool delete_limit = false;
bool partial = false;
ReceiverStats recv_stats;
memset(&recv_stats, 0, sizeof(recv_stats));
client_flush_client_messages(client->file_descriptor);
bool ok = finalize_transfer(client, context->config, context->remove_source_files, &delete_limit,
&recv_stats);
&partial, &recv_stats);
context->delete_limit = delete_limit;
context->partial = partial;
if (!ok && context->config->use_delete)
log_message(LOG_LEVEL_ERROR,
"server reported a deletion failure (--delete); see the server log for the reason");
@@ -1215,6 +1357,7 @@ static int scan_directory_multithreaded(void* pipeline_context) {
its protected lists, so the data pass must not append to them again. */
if (!context->early_delete && !context->delete_plans) {
prepared.options.excluded_paths = context->excluded_paths;
prepared.options.per_dir_rules = context->per_dir_rules;
/* The root marker for a full recursive transfer is already in the list; do
not let the scanner append every directory to it. */
if (context->config->files_from_set != NULL)
@@ -1334,7 +1477,7 @@ static int load_files_multithreaded(void* pipeline_context) {
if (!context->config->use_sendfile) {
for (int i = 0; i < chunk->element_count; i++) {
File* f = chunk->items[i];
if (f->data->size > STREAM_THRESHOLD && !context->config->use_compression)
if (loader_can_stream(context->config, f))
continue;
if (!file_load_data(f)) {
log_message(LOG_LEVEL_ERROR, "Failed to load file data");
@@ -1448,6 +1591,8 @@ typedef struct {
ArrayList* synced_dirs;
ArrayList* plan_dirs;
ArrayList* missing_args;
/* Per-directory filter rules compiled by the scan (protocol 2.30.0). */
FilterRuleList* per_dir_rules;
PreparedScanner prepared;
StopCondition stop;
TransferStats transfer_stats;
@@ -1492,9 +1637,11 @@ static bool send_files_prepare(Config* config, SendFilesState* state) {
}
state->size_skipped = array_list_create(free);
state->synced_dirs = array_list_create(free);
if (!state->size_skipped || !state->synced_dirs)
state->per_dir_rules = filter_rule_list_create();
if (!state->size_skipped || !state->synced_dirs || !state->per_dir_rules)
return false;
state->prepared.options.size_skipped_paths = state->size_skipped;
state->prepared.options.per_dir_rules = state->per_dir_rules;
/* Only a --files-from subset confines the extras walk to the directories
the scan synchronized; a full recursive transfer deletes throughout the
receive root, so mark the root itself (the "." sentinel) and let the
@@ -1559,9 +1706,9 @@ static bool send_files_prepare_delete(Config* config, SendFilesState* state) {
log_message(LOG_LEVEL_WARNING, "IO error encountered -- skipping file deletion");
skip_delete = true;
} else {
early_ok =
send_delete_manifest_early(client, early_manifest, state->excluded, state->size_skipped,
state->missing_args, state->synced_dirs);
early_ok = send_delete_manifest_early(client, early_manifest, state->excluded,
state->size_skipped, state->missing_args,
state->synced_dirs, state->per_dir_rules);
}
}
array_list_delete(early_manifest);
@@ -1570,6 +1717,7 @@ static bool send_files_prepare_delete(Config* config, SendFilesState* state) {
state->prepared.options.excluded_paths = NULL;
state->prepared.options.size_skipped_paths = NULL;
state->prepared.options.synced_dirs = NULL;
state->prepared.options.per_dir_rules = NULL;
if (!prescan_ok || (!early_ok && !skip_delete)) {
array_list_delete(prescan_chunks);
return false;
@@ -1599,7 +1747,7 @@ static bool send_files_prepare_delete(Config* config, SendFilesState* state) {
config->files_from_set ? state->synced_dirs : (walk_root ? state->synced_dirs : NULL);
delete_plan_sender_finalize(state->plan_sender, scope, walk_root);
delete_plan_sender_set_config(state->plan_sender, state->excluded, state->size_skipped,
state->missing_args);
state->missing_args, state->per_dir_rules);
if (state->had_scan_io && delete_plan_sender_empty(state->plan_sender)) {
log_message(LOG_LEVEL_ERROR,
"source scan hit an I/O error before finding any file; refusing to delete "
@@ -1623,6 +1771,7 @@ static bool send_files_prepare_delete(Config* config, SendFilesState* state) {
state->prepared.options.size_skipped_paths = NULL;
state->prepared.options.synced_dirs = NULL;
state->prepared.options.plan_dirs = NULL;
state->prepared.options.per_dir_rules = NULL;
if (!prescan_ok || (!plans_ok && !skip_delete))
return false;
} else if (config->use_delete) {
@@ -1717,7 +1866,7 @@ static bool send_files_run(Config* config, SendFilesState* state) {
bool load_ok = true;
for (int i = 0; i < current_chunk->element_count; i++) {
File* f = current_chunk->items[i];
if (f->data->size > STREAM_THRESHOLD && !config->use_compression)
if (loader_can_stream(config, f))
continue;
if (!file_load_data(f)) {
log_message(LOG_LEVEL_ERROR, "Failed to load file data");
@@ -1804,7 +1953,8 @@ static int send_files_finalize(const Config* config, SendFilesState* state) {
modes the deletion already went out with the data, so nothing is
re-sent here. */
if (send_delete_manifest(client->file_descriptor, state->manifest, state->excluded,
state->size_skipped, state->missing_args, state->synced_dirs) != 0) {
state->size_skipped, state->missing_args, state->synced_dirs,
state->per_dir_rules) != 0) {
if (state->manifest) {
array_list_delete(state->manifest);
state->manifest = NULL;
@@ -1817,15 +1967,22 @@ static int send_files_finalize(const Config* config, SendFilesState* state) {
}
}
}
/* Output parity: report changed directories that had no transferred child
before the deferred directory times are applied (so the probe still sees
their pre-transfer state). */
client_change_emit_pending_dirs(config, client->file_descriptor);
/* P7 Wave D: every directory has now been traversed (or the scan stopped
early), so transmit the captured directory times last. The receiver defers
applying them until after its own deletion/publication phase. */
if (!send_dir_times(client, config, state->dir_entries))
return 1;
bool delete_limit = false;
bool partial = false;
ReceiverStats recv_stats;
memset(&recv_stats, 0, sizeof(recv_stats));
bool ok = finalize_transfer(client, config, state->remove_sources, &delete_limit, &recv_stats);
client_flush_client_messages(client->file_descriptor);
bool ok = finalize_transfer(client, config, state->remove_sources, &delete_limit, &partial,
&recv_stats);
if (!ok && config->use_delete)
log_message(LOG_LEVEL_ERROR,
"server reported a deletion failure (--delete); see the server log for the reason");
@@ -1843,12 +2000,12 @@ static int send_files_finalize(const Config* config, SendFilesState* state) {
(double)state->transfer_stats.transferred_file_size / (double)BYTES_PER_MIB);
/* A skipped source entry (--ignore-errors past an unreadable directory, or a
dereferenced symlink with no referent) makes rsync report a partial
transfer (exit 23) even though the rest of the run succeeded. A
--max-delete-capped commit is a successful transfer that rsync reports
transfer (exit 23), as does a receiver per-entry failure (STATUS_PARTIAL).
A --max-delete-capped commit is a successful transfer that rsync reports
with exit code 25. */
if (!ok)
return 1;
if (state->had_scan_io)
if (state->had_scan_io || partial)
return 23;
return delete_limit ? 25 : 0;
}
@@ -1872,6 +2029,8 @@ static void send_files_cleanup(SendFilesState* state) {
array_list_delete(state->plan_dirs);
if (state->missing_args)
array_list_delete(state->missing_args);
if (state->per_dir_rules)
filter_rule_list_free(state->per_dir_rules);
if (state->remove_sources)
array_list_delete(state->remove_sources);
if (state->dir_entries)
@@ -1885,7 +2044,18 @@ static void send_files_cleanup(SendFilesState* state) {
client_set_abort_armed(false);
}
static int send_files_impl(Config* config);
int send_files(Config* config) {
/* Install the --stderr=client sink for the whole run (it only queues while a
session is live) and release its queue on every return path. */
client_messages_install();
int rc = send_files_impl(config);
client_messages_end();
return rc;
}
static int send_files_impl(Config* config) {
if (config->list_only)
return send_list_only(config);
if (config->dry_run)
@@ -1902,35 +2072,21 @@ int send_files(Config* config) {
shielded -- rsync's `-d DIR/ --delete`. */
state.delete_per_dir = config->use_delete && config_delete_timing_per_dir(config);
int skipped = 0;
if (config->delete_missing_args) {
state.missing_args = array_list_create(free);
if (!state.missing_args)
return 1;
}
if (!files_from_list_check(config, state.missing_args, &skipped)) {
if (state.missing_args)
array_list_delete(state.missing_args);
if (!client_prepare_files_from(config, &state.missing_args, &skipped))
return 1;
}
/* From here on a server session may be live, so Ctrl-C/SIGTERM should set the
abort flag (and be forwarded as STATUS_ABORT) instead of terminating. */
client_set_abort_armed(true);
Client* client = connect_transfer_client(config);
ProtocolSession session;
Client* client = client_connect_and_bind_session(config, &session);
if (!client) {
if (config->transport == TRANSPORT_TCP)
log_message(LOG_LEVEL_ERROR, "could not connect to server%s",
config->use_tls ? " via TLS" : "");
if (state.missing_args)
array_list_delete(state.missing_args);
return 1;
}
state.client = client;
ProtocolSession session;
protocol_session_init(&session, client->file_descriptor, client->file_descriptor);
protocol_session_set_io_timeout(&session, config->timeout);
protocol_session_set_ssl(&session, (SSL*)client->ssl);
protocol_session_bind(&session);
client_messages_activate(true);
int ret = 1;
if (!send_files_prepare(config, &state))
@@ -1946,7 +2102,16 @@ send_fail:
return ret;
}
static int send_files_multithreaded_impl(Config* config);
int send_files_multithreaded(Config* config) {
client_messages_install();
int rc = send_files_multithreaded_impl(config);
client_messages_end();
return rc;
}
static int send_files_multithreaded_impl(Config* config) {
if (!config)
return 1;
if (config->list_only)
@@ -1956,16 +2121,8 @@ int send_files_multithreaded(Config* config) {
: send_dry_run_manifest(config);
ArrayList* missing_args = NULL;
int skipped = 0;
if (config->delete_missing_args) {
missing_args = array_list_create(free);
if (!missing_args)
return 1;
}
if (!files_from_list_check(config, missing_args, &skipped)) {
if (missing_args)
array_list_delete(missing_args);
if (!client_prepare_files_from(config, &missing_args, &skipped))
return 1;
}
/* Armed only once a session may go live (see send_files). */
client_set_abort_armed(true);
@@ -1994,6 +2151,8 @@ int send_files_multithreaded(Config* config) {
queue_destroy(q1);
if (q2)
queue_destroy(q2);
if (missing_args)
array_list_delete(missing_args);
return 1;
}
PipelineContextSender* context = pipeline_context_sender_create(config, q1, q2);
@@ -2032,7 +2191,8 @@ int send_files_multithreaded(Config* config) {
confined; only a --files-from subset records concrete directories. */
context->size_skipped_paths = array_list_create(free);
context->synced_dirs = array_list_create(free);
if (!context->size_skipped_paths || !context->synced_dirs) {
context->per_dir_rules = filter_rule_list_create();
if (!context->size_skipped_paths || !context->synced_dirs || !context->per_dir_rules) {
pipeline_context_sender_destroy(context);
return 1;
}
@@ -2061,6 +2221,7 @@ int send_files_multithreaded(Config* config) {
if (context->excluded_paths)
prepared.options.excluded_paths = context->excluded_paths;
prepared.options.size_skipped_paths = context->size_skipped_paths;
prepared.options.per_dir_rules = context->per_dir_rules;
/* The root marker for a full recursive transfer is already in the list;
only a --files-from subset needs the scanner to record directories. */
if (config->files_from_set != NULL)
@@ -2101,7 +2262,8 @@ int send_files_multithreaded(Config* config) {
: (walk_root ? context->synced_dirs : NULL);
delete_plan_sender_finalize(context->delete_plans, scope, walk_root);
delete_plan_sender_set_config(context->delete_plans, context->excluded_paths,
context->size_skipped_paths, context->missing_args);
context->size_skipped_paths, context->missing_args,
context->per_dir_rules);
}
bool empty = per_dir
? (context->delete_plans && delete_plan_sender_empty(context->delete_plans))
@@ -2202,16 +2364,18 @@ int send_files_multithreaded(Config* config) {
mtx_unlock(&context->mutex_scanner);
bool sender_ok = sender_result == thrd_success;
bool delete_limit = context->delete_limit;
bool partial = context->partial;
/* A skipped source entry (--ignore-errors past an unreadable directory, or a
dereferenced symlink with no referent) makes rsync report a partial
transfer (exit 23). A --max-delete-capped commit is a successful transfer
that rsync reports with exit code 25. */
transfer (exit 23), as does a receiver per-entry failure (STATUS_PARTIAL).
A --max-delete-capped commit is a successful transfer that rsync reports
with exit code 25. */
pipeline_context_sender_destroy(context);
client_progress_cleanup();
client_set_abort_armed(false);
if (!sender_ok)
return 1;
if (scan_io)
if (scan_io || partial)
return 23;
return delete_limit ? 25 : 0;
}
+33 -2
View File
@@ -13,6 +13,7 @@
#include "delta.h"
#include "format.h"
#include "log.h"
#include "protocol.h"
#include "scanner.h"
#include <stdatomic.h>
#include <stdbool.h>
@@ -68,14 +69,43 @@ void client_progress_name(const Config* config, const File* file);
/* Emit a transferred entry's ancestor directories (as -i/--out-format change
* lines or --progress name lines) before the entry's own line. */
void client_change_emit_ancestors(const Config* config, const File* file);
/* Output parity (protocol 2.30.0): probe each not-yet-known ancestor directory's
* pre-transfer destination state before the entry that first triggers it is
* sent. Returns false on a protocol/transport error. */
bool client_change_probe_ancestors(const Config* config, const File* file, int fd);
/* Mark a transferred directory entry as already reported, and flush the
* itemize lines for changed directories that had no transferred child. */
void client_change_mark_dir(const Config* config, const File* file);
void client_change_emit_pending_dirs(const Config* config, int fd);
void client_progress_uptodate(const Config* config, const File* file);
void client_progress_prepare(const Config* config, const ArrayList* plan_dirs,
unsigned long long plan_non_dir_count);
bool receive_stats_record(int fd, ReceiverStats* stats, ArrayList* would_delete);
/* --stderr=client diagnostic channel (client_report.c): install the queueing
* log sink for a transfer, mark the session live, flush queued diagnostics over
* the wire at a frame boundary, and tear the sink down. */
void client_messages_install(void);
void client_messages_activate(bool active);
void client_flush_client_messages(int fd);
void client_messages_end(void);
/* client_send.c */
void receive_daemon_motd(Client* client, const Config* config);
Client* connect_transfer_client(const Config* config);
/* Connect the configured transport and install `session` on it: init with the
* socket fd pair, apply the I/O timeout and (when negotiated) the TLS object,
* then bind the session to this thread. Returns the connected client, or NULL
* after logging the connect failure. The caller owns the client and must keep
* `session` alive until it calls protocol_session_unbind(). */
Client* client_connect_and_bind_session(const Config* config, ProtocolSession* session);
/* Shared --files-from/--delete-missing-args preamble for the send entry points:
* when --delete-missing-args is set, allocate the list that
* files_from_list_check fills with the destination mirrors of missing entries;
* then validate the --files-from list. On success returns true and stores the
* (possibly NULL) owned list in *missing_args_out plus the skipped count; on
* failure returns false after freeing the list. */
bool client_prepare_files_from(const Config* config, ArrayList** missing_args_out,
int* skipped_out);
void disconnect_transfer_client(Client* client);
int incremental_check(Client* client, File* file, const Config* config, DeltaSignature** out_sig,
unsigned long long* resume_offset);
@@ -87,9 +117,10 @@ int send_dry_run_manifest(const Config* config);
int send_list_only(const Config* config);
int send_dry_run_remote(Config* config);
int send_delete_manifest(int fd, ArrayList* manifest, ArrayList* protected_prefixes,
ArrayList* size_skipped, ArrayList* missing_args, ArrayList* synced_dirs);
ArrayList* size_skipped, ArrayList* missing_args, ArrayList* synced_dirs,
const FilterRuleList* per_dir_rules);
bool send_delete_manifest_early(Client* client, ArrayList* manifest, ArrayList* protected_prefixes,
ArrayList* size_skipped, ArrayList* missing_args,
ArrayList* synced_dirs);
ArrayList* synced_dirs, const FilterRuleList* per_dir_rules);
#endif
+14 -54
View File
@@ -149,7 +149,7 @@ DirectoryScanner* directory_scanner_create_with_options(const char* root_directo
scanner->options = *options;
if (scanner->options.chunk_size == 0)
scanner->options.chunk_size = DESIRED_CHUNK_SIZE;
scanner->directories = queue_create(100, dir_entry_destroy);
scanner->directories = queue_create(SCANNER_RESULT_QUEUE_CAP, dir_entry_destroy);
if (!scanner->directories) {
free(scanner);
return NULL;
@@ -1026,7 +1026,7 @@ static ScannerAction scanner_process_entry(DirectoryScanner* scanner, ArrayList*
Chunk** out_chunk) {
const char* name = sorted->name;
ScannerEntry* inspected = &sorted->entry;
char* cur_path = inspected->path;
const char* cur_path = inspected->path;
struct stat stats = inspected->stats;
/* --files-from allow-set and the filter layer apply to files and to
@@ -1101,61 +1101,23 @@ static ScannerAction scanner_process_entry(DirectoryScanner* scanner, ArrayList*
free(rel_copy);
return SCANNER_ACTION_CONTINUE;
}
File* file = file_create(cur_path);
if (file == NULL) {
free(rel_copy);
free(inspected->link_target);
inspected->link_target = NULL;
/* Entry construction (data size, -R wire path, special/devices, hardlink
group, metadata, xattrs) is shared with the parallel scanner. */
File* file = NULL;
bool build_failed = false;
ScannerBuildStatus status =
scanner_build_file_entry(&scanner->options, inspected, rel_copy, &file, &build_failed);
free(rel_copy);
rel_copy = NULL;
if (build_failed)
scanner->failed = true;
if (status == SCANNER_BUILD_SKIP)
return SCANNER_ACTION_CONTINUE;
}
if (inspected->is_symlink) {
file->is_symlink = true;
file->symlink_target = inspected->link_target;
inspected->link_target = NULL;
} else {
file->data->size = stats.st_size;
}
if (scanner->relative_mode) {
file->send_path = rel_copy;
rel_copy = NULL;
} else if (scanner->options.relative_prefix) {
file->send_path = scanner_prefix_send_path(scanner->options.relative_prefix, rel_copy);
free(rel_copy);
rel_copy = NULL;
if (!file->send_path) {
file_destroy(file);
scanner->failed = true;
return SCANNER_ACTION_BREAK;
}
}
/* --devices/--specials: a device/FIFO/socket entry marked for preservation
becomes a node to recreate (is_special, no data, rdev captured); an
unrequested non-regular entry is skipped (rsync default). */
ScannerSpecial special =
scanner_prepare_special(scanner->options.preserve_devices, scanner->options.preserve_specials,
scanner->options.copy_devices, file, &stats);
if (special == SCANNER_SPECIAL_SKIP) {
scanner_note_nonreg(&scanner->options, file->path);
free(rel_copy);
file_destroy(file);
return SCANNER_ACTION_CONTINUE;
}
if (scanner->options.hardlinks && S_ISREG(stats.st_mode))
scanner_assign_hardlink(scanner, scanner->options.hardlinks, file, &stats);
if (scanner->options.use_metadata)
file->metadata = file_metadata_create(file->path, &stats, scanner->options.preserve_atimes,
scanner->options.preserve_crtimes);
if (scanner->options.use_metadata && !file->metadata) {
free(rel_copy);
file_destroy(file);
if (status != SCANNER_BUILD_OK) {
scanner->failed = true;
return SCANNER_ACTION_BREAK;
return status == SCANNER_BUILD_FAIL_CONTINUE ? SCANNER_ACTION_CONTINUE : SCANNER_ACTION_BREAK;
}
if (!(file->link_group != 0 && !file->link_first))
scanner_capture_xattrs(scanner, file);
if (!array_list_add(chunk_data, file)) {
free(rel_copy);
file_destroy(file);
scanner->failed = true;
return SCANNER_ACTION_BREAK;
@@ -1163,14 +1125,12 @@ static ScannerAction scanner_process_entry(DirectoryScanner* scanner, ArrayList*
scanner->current_dir_produced = true;
*chunk_data_size += file->data->size;
if (*chunk_data_size > scanner->options.chunk_size) {
free(rel_copy);
Chunk* result = chunk_data_to_chunk(chunk_data);
if (!result)
scanner->failed = true;
*out_chunk = result;
return SCANNER_ACTION_CHUNK;
}
free(rel_copy);
return SCANNER_ACTION_CONTINUE;
}
+12
View File
@@ -19,6 +19,11 @@
* keeps one transfer from spawning an unbounded pool on a very large machine. */
#define MAX_SCANNER_THREADS 256
/* Depth of the scanner's work queues: the sequential scanner's pending-directory
* stack and the parallel scanner's result queue. Bounds memory for a very wide
* or very deep tree while leaving ample headroom for normal scans. */
#define SCANNER_RESULT_QUEUE_CAP 100
typedef struct {
bool use_metadata;
/* Phase 4 metadata capture: -U/--atimes and -N/--crtimes tell the scanner to
@@ -115,6 +120,13 @@ typedef struct {
* directories, exactly like rsync; the receive root is the "." sentinel.
* Guarded by `excluded_mutex`. */
ArrayList* synced_dirs;
/* Per-directory filter-rule sink (optional): when non-NULL the scanner appends
* a deep copy of every rule it reads from a per-directory merge file, each
* carrying its owner directory and no-inherit flag (see filter.h). The delete
* carriers transmit them so the receiver re-derives the per-directory
* protect/risk set for destination-only entries. Guarded by `excluded_mutex`
* like the other sinks. */
FilterRuleList* per_dir_rules;
/* Delete-plan directory sink (optional): when non-NULL the scanner appends
* the destination-relative path of every directory it traverses (except the
* receive root). The per-directory --delete-during/--delete-delay plan
+150 -32
View File
@@ -285,32 +285,34 @@ bool entry_passes_selection(const FileListSet* file_list, const FilterRuleList*
* read xattrs is non-fatal: the file is transferred without them. A symlink
* entry reads the LINK's own xattrs (never the referent's) with the no-follow
* variant; on Linux the VFS refuses xattrs on symlinks, so that yields NULL. */
void scanner_capture_xattrs(const DirectoryScanner* scanner, File* file) {
if (!scanner || !file || !(scanner->options.preserve_xattrs || scanner->options.preserve_acls))
void scanner_capture_xattrs_opts(const ScannerOptions* options, File* file) {
if (!options || !file || !(options->preserve_xattrs || options->preserve_acls))
return;
file->xattrs = file->is_symlink
? xattr_capture_path_nofollow(file->path, scanner->options.preserve_acls)
: xattr_capture_path(file->path, scanner->options.preserve_acls);
file->xattrs = file->is_symlink ? xattr_capture_path_nofollow(file->path, options->preserve_acls)
: xattr_capture_path(file->path, options->preserve_acls);
}
void scanner_capture_xattrs(const DirectoryScanner* scanner, File* file) {
if (!scanner)
return;
scanner_capture_xattrs_opts(&scanner->options, file);
}
/* Apply --hard-links (-H) detection to one regular File. On a sibling (a
* later member of an already-seen source inode) the File keeps the group id
* and the first member's wire path but carries NO data payload (size 0); the
* first member is left untouched (data present, link_first). Allocation
* failure is fatal: the scanner is marked failed. */
void scanner_assign_hardlink(DirectoryScanner* scanner, HardLinkTable* table, File* file,
const struct stat* stats) {
* first member is left untouched (data present, link_first). Returns false on
* allocation failure (the caller marks the scan failed); the File stays usable
* either way. */
bool scanner_assign_hardlink(HardLinkTable* table, File* file, const struct stat* stats) {
if (!table || !file || !stats)
return;
return true;
int gid;
bool is_first;
char* first_path = NULL;
if (!hardlink_table_assign(table, file_wire_path(file), stats->st_dev, stats->st_ino, &gid,
&is_first, &first_path)) {
if (scanner)
scanner->failed = true;
return;
}
&is_first, &first_path))
return false;
file->link_group = gid;
file->link_first = is_first;
if (!is_first) {
@@ -319,6 +321,7 @@ void scanner_assign_hardlink(DirectoryScanner* scanner, HardLinkTable* table, Fi
} else {
free(first_path);
}
return true;
}
/* Phase 4 special/devices decision for one non-regular entry, matching rsync:
@@ -399,6 +402,76 @@ void scanner_note_nonreg(const ScannerOptions* options, const char* fs_path) {
fflush(stdout);
}
/* Construct one non-directory File from an inspected entry. Shared by the
* sequential and parallel scanners so entry construction has a single
* implementation: data size (or carried symlink), -R wire path, special/devices
* classification, hardlink group, metadata and xattr capture all happen here in
* the same order for both. See the declaration for the ownership contract. */
ScannerBuildStatus scanner_build_file_entry(const ScannerOptions* options, ScannerEntry* inspected,
const char* rel, File** out_file, bool* failed) {
*out_file = NULL;
if (failed)
*failed = false;
File* file = file_create(inspected->path);
if (!file) {
/* The File never existed, so drop the not-yet-transferred symlink target
here; the caller's entry teardown would otherwise double-free it. */
free(inspected->link_target);
inspected->link_target = NULL;
return SCANNER_BUILD_FAIL_CONTINUE;
}
if (inspected->is_symlink) {
file->is_symlink = true;
file->symlink_target = inspected->link_target;
inspected->link_target = NULL;
} else {
file->data->size = inspected->stats.st_size;
}
/* -R + --files-from uses the bare transfer-relative path; -R without
--files-from prefixes it. Plain scans keep the source path. */
bool relative_mode = options->relative && options->file_list != NULL;
if (relative_mode) {
file->send_path = str_dup(rel);
} else if (options->relative_prefix) {
file->send_path = scanner_prefix_send_path(options->relative_prefix, rel);
}
if ((relative_mode || options->relative_prefix) && !file->send_path) {
file_destroy(file);
return SCANNER_BUILD_FAIL_BREAK;
}
/* --devices/--specials: a device/FIFO/socket entry marked for preservation
becomes a node to recreate (is_special, no data, rdev captured); an
unrequested non-regular entry is skipped (rsync default). */
ScannerSpecial special =
scanner_prepare_special(options->preserve_devices, options->preserve_specials,
options->copy_devices, file, &inspected->stats);
if (special == SCANNER_SPECIAL_SKIP) {
scanner_note_nonreg(options, file->path);
file_destroy(file);
return SCANNER_BUILD_SKIP;
}
if (options->hardlinks && S_ISREG(inspected->stats.st_mode) &&
!scanner_assign_hardlink(options->hardlinks, file, &inspected->stats)) {
/* Allocation failure is non-fatal to this entry (it is still emitted) but
marks the scan failed, matching the historical inlined behaviour. */
if (failed)
*failed = true;
}
if (options->use_metadata) {
file->metadata = file_metadata_create(file->path, &inspected->stats, options->preserve_atimes,
options->preserve_crtimes);
if (!file->metadata) {
file_destroy(file);
return SCANNER_BUILD_FAIL_BREAK;
}
}
/* A hardlink sibling carries no data, so it carries no xattrs. */
if (!(file->link_group != 0 && !file->link_first))
scanner_capture_xattrs_opts(options, file);
*out_file = file;
return SCANNER_BUILD_OK;
}
/* rsync 3.4.1's `--info=mount` line, emitted when `-xx` drops a mount-point
* directory: `[sender] skipping mount-point dir NAME` (the client is the
* sender). Plain `-x` keeps the empty directory and prints nothing, matching
@@ -443,17 +516,16 @@ void scanner_record_size_skipped(DirectoryScanner* scanner, const char* fs_path)
scanner_record_protected(scanner, fs_path, scanner->options.size_skipped_paths);
}
/* Record a directory the scan synchronized. `fs_path` is its absolute path and
`rel` its path relative to the transfer root ("" for the root); the stored
form matches the wire layout (the bare relative path in -R+--files-from, else
the source path with a leading '/' removed, with "." for the receive root).
Returns false on allocation failure. */
bool scanner_record_synced_dir(const ScannerOptions* options, const char* fs_path, const char* rel,
bool relative_mode) {
if (!options->synced_dirs && !options->plan_dirs)
return true;
if (!file_list_dir_in_scope(options->file_list, rel))
return true;
/* The destination-relative coordinate the receiver's delete walkers match
against for an entry at `fs_path` (with `rel` its path relative to the
transfer root, "" for the root): `relative_prefix + rel` under -R+--relative,
the bare relative path under -R+--files-from, else the source path with a
leading '/' removed, with "." for the receive root. Shared by the
synchronized-directory sink and the mirrored per-directory rule owners so
both live in the same coordinate system. Returns an owned string, or NULL on
allocation failure. */
char* scanner_dest_rel_path(const ScannerOptions* options, const char* fs_path, const char* rel,
bool relative_mode) {
char* prefixed = NULL;
const char* dest;
if (relative_mode) {
@@ -461,7 +533,7 @@ bool scanner_record_synced_dir(const ScannerOptions* options, const char* fs_pat
} else if (options->relative_prefix) {
prefixed = scanner_prefix_send_path(options->relative_prefix, rel);
if (!prefixed)
return false;
return NULL;
dest = prefixed;
} else {
dest = fs_path;
@@ -470,6 +542,24 @@ bool scanner_record_synced_dir(const ScannerOptions* options, const char* fs_pat
dest++;
if (dest[0] == '\0')
dest = ".";
char* out = str_dup(dest);
free(prefixed);
return out;
}
/* Record a directory the scan synchronized. `fs_path` is its absolute path and
`rel` its path relative to the transfer root ("" for the root); the stored
form matches the wire layout (see scanner_dest_rel_path). Returns false on
allocation failure. */
bool scanner_record_synced_dir(const ScannerOptions* options, const char* fs_path, const char* rel,
bool relative_mode) {
if (!options->synced_dirs && !options->plan_dirs)
return true;
if (!file_list_dir_in_scope(options->file_list, rel))
return true;
char* dest = scanner_dest_rel_path(options, fs_path, rel, relative_mode);
if (!dest)
return false;
bool ok = true;
if (options->synced_dirs)
ok = excluded_sink_append(options->synced_dirs, options->excluded_mutex, dest);
@@ -478,7 +568,7 @@ bool scanner_record_synced_dir(const ScannerOptions* options, const char* fs_pat
than deleted as an extra; the receive root (".") is implicit. */
if (ok && options->plan_dirs && strcmp(dest, ".") != 0)
ok = excluded_sink_append(options->plan_dirs, options->excluded_mutex, dest);
free(prefixed);
free(dest);
return ok;
}
@@ -487,7 +577,8 @@ bool scanner_record_synced_dir(const ScannerOptions* options, const char* fs_pat
* fresh list. Returns NULL on allocation/parse failure (message in `err`);
* returns an empty list (and *any_exists=false) when no file exists. */
FilterRuleList* read_dir_filters(const ScannerOptions* options, const char* dir_path,
const char* rel, bool* any_exists, char* err, size_t err_size) {
const char* rel, bool relative_mode, bool* any_exists, char* err,
size_t err_size) {
if (err && err_size > 0)
err[0] = '\0';
const FilterRuleList* base = options->base_filters;
@@ -511,13 +602,40 @@ FilterRuleList* read_dir_filters(const ScannerOptions* options, const char* dir_
}
if (base) {
for (int i = 0; i < base->dir_merge_count; i++) {
if (!filter_file_append(own, dir_path, base->dir_merge_names[i], rel, &opts, &exists, err,
err_size))
if (!filter_dir_merge_append(own, dir_path, &base->dir_merges[i], rel, &opts, &exists, err,
err_size))
goto fail;
if (exists && any_exists)
*any_exists = true;
}
}
/* Mirror the directory's rules into the delete-carrier sink so the receiver
* can reconstruct its per-directory protect/risk set. The mirrored rules
* carry the destination-relative owner coordinate (not the transfer-root-
* relative one the sender's own evaluation uses) so the receiver's delete
* walkers, which match against receive-root-relative paths, find them. */
if (options->per_dir_rules && own->count > 0) {
char* owner = scanner_dest_rel_path(options, dir_path, rel, relative_mode);
if (!owner)
goto fail;
mtx_t* mtx = options->excluded_mutex;
if (mtx)
mtx_lock(mtx);
for (int i = 0; i < own->count; i++) {
FilterRule* copy = filter_rule_clone(own->items[i]);
if (!copy || !filter_rule_set_owner(copy, owner) ||
!filter_rule_list_add(options->per_dir_rules, copy)) {
filter_rule_free(copy);
if (mtx)
mtx_unlock(mtx);
free(owner);
goto fail;
}
}
if (mtx)
mtx_unlock(mtx);
free(owner);
}
return own;
fail:
filter_rule_list_free(own);
@@ -534,7 +652,7 @@ int open_directory_filter_context(DirectoryScanner* scanner, const FilterNode* i
bool any_exists = false;
FilterRuleList* own = read_dir_filters(&scanner->options, scanner->current_path,
scanner->current_rel ? scanner->current_rel : "",
&any_exists, err, sizeof(err));
scanner->relative_mode, &any_exists, err, sizeof(err));
if (!own) {
/* read_dir_filters() leaves `err` set on a parse/allocation failure even
when an earlier merge file in the same directory existed (any_exists true);
+28 -3
View File
@@ -62,6 +62,17 @@ typedef enum {
SCANNER_SPECIAL_SKIP, /* non-regular entry not requested: skip */
} ScannerSpecial;
/* Result of scanner_build_file_entry(). The two failure variants preserve the
* sequential scanner's historical distinction between a failure before the
* File existed (which kept walking the directory) and one afterwards (which cut
* the chunk short); both mark the scan failed. */
typedef enum {
SCANNER_BUILD_OK, /* File built; caller owns it */
SCANNER_BUILD_SKIP, /* non-regular entry not preserved; no File */
SCANNER_BUILD_FAIL_CONTINUE, /* failed before the File existed */
SCANNER_BUILD_FAIL_BREAK, /* failed after the File existed */
} ScannerBuildStatus;
/* scanner_filter.c */
void filter_node_destroy(void* item);
FilterNode* filter_node_alloc(FilterNode* parent, FilterRuleList* own);
@@ -73,14 +84,27 @@ File* scanner_build_dir_file(const char* path, const struct stat* stats,
const ScannerOptions* options);
char* child_rel_path(const char* parent_rel, const char* name);
char* scanner_prefix_send_path(const char* prefix, const char* rel);
char* scanner_dest_rel_path(const ScannerOptions* options, const char* fs_path, const char* rel,
bool relative_mode);
bool entry_passes_selection(const FileListSet* file_list, const FilterRuleList* base,
const FilterNode* node, const char* rel, const char* leaf, bool is_dir,
bool per_dir_filters, bool exclude_filter_files, bool* protect_out);
void scanner_capture_xattrs(const DirectoryScanner* scanner, File* file);
void scanner_assign_hardlink(DirectoryScanner* scanner, HardLinkTable* table, File* file,
const struct stat* stats);
void scanner_capture_xattrs_opts(const ScannerOptions* options, File* file);
bool scanner_assign_hardlink(HardLinkTable* table, File* file, const struct stat* stats);
ScannerSpecial scanner_prepare_special(bool preserve_devices, bool preserve_specials,
bool copy_devices, File* file, const struct stat* stats);
/* Build one non-directory transfer File from an inspected entry. `rel` is the
* entry's transfer-root-relative path (used for the -R wire path); `inspected`
* supplies the on-disk path, stats and (for a carried symlink) the target whose
* ownership transfers to the File. Populates data size, send_path, special-node
* state, hardlink group, metadata and xattrs. On SCANNER_BUILD_OK the caller
* owns *out_file; on SCANNER_BUILD_SKIP it is NULL and the entry is dropped; on
* either failure it is NULL and the caller must mark the scan failed. `*failed`
* additionally reports a non-fatal hardlink-table allocation failure, in which
* case a usable File is still returned. */
ScannerBuildStatus scanner_build_file_entry(const ScannerOptions* options, ScannerEntry* inspected,
const char* rel, File** out_file, bool* failed);
bool excluded_sink_append(ArrayList* list, mtx_t* mtx, const char* rel);
void scanner_note_nonreg(const ScannerOptions* options, const char* fs_path);
void scanner_note_mount(const ScannerOptions* options, const char* fs_path);
@@ -92,7 +116,8 @@ void scanner_record_size_skipped(DirectoryScanner* scanner, const char* fs_path)
bool scanner_record_synced_dir(const ScannerOptions* options, const char* fs_path, const char* rel,
bool relative_mode);
FilterRuleList* read_dir_filters(const ScannerOptions* options, const char* dir_path,
const char* rel, bool* any_exists, char* err, size_t err_size);
const char* rel, bool relative_mode, bool* any_exists, char* err,
size_t err_size);
int open_directory_filter_context(DirectoryScanner* scanner, const FilterNode* inherited);
int scanner_inspect_entry(const ScannerOptions* options, const char* containing_dir,
const char* link_rel, const char* name, ScannerEntry* entry);
+171 -195
View File
@@ -109,7 +109,7 @@ static void parallel_scanner_creation_failed(ParallelScanner* ps) {
/* Initialize result queue and synchronization primitives. Returns true on success. */
static bool parallel_scanner_init(ParallelScanner* ps) {
ps->result_queue = queue_create(100, chunk_destroy);
ps->result_queue = queue_create(SCANNER_RESULT_QUEUE_CAP, chunk_destroy);
if (!ps->result_queue)
return false;
atomic_init(&ps->cancelled, false);
@@ -210,6 +210,157 @@ static Chunk* batch_files(ArrayList* files, unsigned long long chunk_size, Queue
return first;
}
/* Record the delete-protection mirror of a root entry that
* scanner_inspect_entry() skipped (inspection == 0): a dereferenced symlink
* with no referent is a partial-transfer I/O error and a user-selection or size
* prune protects the entry's destination mirror. */
static void scan_root_record_skipped(const ScannerOptions* options, const char* root_directory,
const char* name, const ScannerEntry* inspected,
ParallelScanner* ps) {
if (inspected->referent_error)
ps->io_error = true;
ArrayList* sink = NULL;
if (inspected->excluded)
sink = inspected->size_excluded ? options->size_skipped_paths : options->excluded_paths;
if (!sink)
return;
/* A root-level prune protects the destination mirror of the entry's wire
path: under -R + --files-from that is the bare relative name, otherwise it
is the full source path with a leading '/' removed (matching the
send_path/file_wire_path the scanner hands the sender). */
if (options->relative && options->file_list != NULL) {
if (!excluded_sink_append(sink, options->excluded_mutex, name))
ps->failed = true;
} else if (options->relative_prefix) {
char* wrel = scanner_prefix_send_path(options->relative_prefix, name);
if (!wrel) {
ps->failed = true;
} else {
if (!excluded_sink_append(sink, options->excluded_mutex, wrel))
ps->failed = true;
free(wrel);
}
} else {
char* abs_path = path_cat(root_directory, name);
if (!abs_path) {
ps->failed = true;
} else {
const char* rel = *abs_path == '/' ? abs_path + 1 : abs_path;
if (!excluded_sink_append(sink, options->excluded_mutex, rel))
ps->failed = true;
free(abs_path);
}
}
}
/* Record the delete-protection mirror of a root entry dropped by the
* --files-from allow-set or a filter rule. Returns false only when the -R
* prefix could not be built (the caller must abandon the entry immediately);
* other allocation failures mark the scan failed but let the caller continue to
* the filter-notice step, matching the historical inlined flow. */
static bool scan_root_record_protection(const ScannerOptions* options, const char* rel,
const char* name, const char* cur_path, bool protect,
bool passes, bool use_rel, ParallelScanner* ps) {
if (passes && !protect)
return true;
/* --files-from subset pruning is not a filter exclusion; -R bare-wire-path
exclusions are never recorded (see ScannerOptions.excluded_paths). */
bool files_from_prune = options->file_list && !file_list_affects(options->file_list, rel);
if ((!files_from_prune && !use_rel) || protect) {
const char* rel_path;
char* prefixed = NULL;
if (use_rel) {
/* -R + --files-from: the destination/wire path is the bare relative
name, not the source path. */
rel_path = rel;
} else if (options->relative_prefix) {
prefixed = scanner_prefix_send_path(options->relative_prefix, name);
if (!prefixed)
return false;
rel_path = prefixed;
} else {
rel_path = *cur_path == '/' ? cur_path + 1 : cur_path;
}
if (options->excluded_paths &&
!excluded_sink_append(options->excluded_paths, options->excluded_mutex, rel_path))
ps->failed = true;
free(prefixed);
}
return true;
}
/* Root-level directory node: apply -x/--one-file-system and either emit the
* mount-point directory (plain -x) or queue the directory for a worker. */
static void scan_root_dir(const ScannerOptions* options, const char* cur_path, const char* rel,
const struct stat* st, ArrayList* root_files, ArrayList* subdirs,
dev_t root_dev, ParallelScanner* ps) {
if (!scanner_same_filesystem(options->one_file_system, root_dev, st->st_dev)) {
if (options->one_file_system > 1) {
/* -xx: drop the mount-point directory entirely (rsync) and print the
--info=mount line when enabled. */
scanner_note_mount(options, cur_path);
return;
}
/* -x/--one-file-system: emit the mount-point directory entry (empty) but do
not descend into it (see the sequential scanner for the same rule). */
File* mount = scanner_build_dir_file(cur_path, st, options);
if (!mount) {
ps->failed = true;
return;
}
if (options->relative_prefix) {
mount->send_path = scanner_prefix_send_path(options->relative_prefix, rel);
if (!mount->send_path) {
file_destroy(mount);
ps->failed = true;
return;
}
}
if (!array_list_add(root_files, mount)) {
file_destroy(mount);
ps->failed = true;
}
return;
}
char* dir = str_dup(cur_path);
if (!dir || !array_list_add(subdirs, dir)) {
free(dir);
ps->failed = true;
}
}
/* Build a non-directory root entry through the shared construction path and add
* it to `root_files`. A non-regular entry the options do not preserve is
* dropped by the builder (which prints rsync's nonreg line); an allocation
* failure marks the scan failed. */
static void scan_root_add_non_dir(const ScannerOptions* options, ScannerEntry* inspected,
const char* rel, ArrayList* root_files, ParallelScanner* ps) {
File* file = NULL;
bool failed = false;
ScannerBuildStatus status = scanner_build_file_entry(options, inspected, rel, &file, &failed);
if (failed || status == SCANNER_BUILD_FAIL_CONTINUE || status == SCANNER_BUILD_FAIL_BREAK)
ps->failed = true;
if (status != SCANNER_BUILD_OK)
return;
if (!array_list_add(root_files, file)) {
file_destroy(file);
ps->failed = true;
}
}
/* Regular file or carried symlink at the transfer root. */
static void scan_root_file(const ScannerOptions* options, ScannerEntry* inspected, const char* rel,
ArrayList* root_files, ParallelScanner* ps) {
scan_root_add_non_dir(options, inspected, rel, root_files, ps);
}
/* Device/FIFO/socket at the transfer root: recreated under --devices/--specials,
* otherwise dropped by the shared builder. */
static void scan_root_special(const ScannerOptions* options, ScannerEntry* inspected,
const char* rel, ArrayList* root_files, ParallelScanner* ps) {
scan_root_add_non_dir(options, inspected, rel, root_files, ps);
}
/* Scan one root-directory entry into either the subdirs or files list. */
static void scan_root_entry(const ScannerOptions* options, const FilterNode* root_node,
const char* root_directory, const struct dirent* entry,
@@ -223,51 +374,16 @@ static void scan_root_entry(const ScannerOptions* options, const FilterNode* roo
return;
}
if (inspection == 0) {
if (inspected.referent_error)
ps->io_error = true;
ArrayList* sink = NULL;
if (inspected.excluded)
sink = inspected.size_excluded ? options->size_skipped_paths : options->excluded_paths;
if (sink) {
/* A root-level prune protects the destination mirror of the entry's wire
path: under -R + --files-from that is the bare relative name, otherwise
it is the full source path with a leading '/' removed (matching the
send_path/file_wire_path the scanner hands the sender). */
if (options->relative && options->file_list != NULL) {
if (!excluded_sink_append(sink, options->excluded_mutex, entry->d_name))
ps->failed = true;
} else if (options->relative_prefix) {
char* wrel = scanner_prefix_send_path(options->relative_prefix, entry->d_name);
if (!wrel) {
ps->failed = true;
} else {
if (!excluded_sink_append(sink, options->excluded_mutex, wrel))
ps->failed = true;
free(wrel);
}
} else {
char* abs_path = path_cat(root_directory, entry->d_name);
if (!abs_path) {
ps->failed = true;
} else {
const char* rel = *abs_path == '/' ? abs_path + 1 : abs_path;
if (!excluded_sink_append(sink, options->excluded_mutex, rel))
ps->failed = true;
free(abs_path);
}
}
}
scan_root_record_skipped(options, root_directory, entry->d_name, &inspected, ps);
return;
}
char* cur_path = inspected.path;
struct stat st = inspected.stats;
bool is_dir = inspected.is_directory;
char* rel = str_dup(entry->d_name);
if (!rel) {
free(cur_path);
ps->failed = true;
return;
goto done;
}
bool is_dir = inspected.is_directory;
bool protect = false;
bool passes = entry_passes_selection(options->file_list, options->base_filters, root_node, rel,
entry->d_name, is_dir, options->per_dir_filters,
@@ -275,169 +391,28 @@ static void scan_root_entry(const ScannerOptions* options, const FilterNode* roo
/* -R + --files-from: root-level files keep their bare relative send path. */
bool use_rel = options->relative && options->file_list != NULL;
if (!passes || protect) {
/* --files-from subset pruning is not a filter exclusion; -R bare-wire-path
exclusions are never recorded (see ScannerOptions.excluded_paths). */
bool files_from_prune = options->file_list && !file_list_affects(options->file_list, rel);
if ((!files_from_prune && !use_rel) || protect) {
const char* rel_path;
char* prefixed = NULL;
if (use_rel) {
/* -R + --files-from: the destination/wire path is the bare relative
name, not the source path. */
rel_path = rel;
} else if (options->relative_prefix) {
prefixed = scanner_prefix_send_path(options->relative_prefix, entry->d_name);
if (!prefixed) {
free(rel);
free(cur_path);
ps->failed = true;
return;
}
rel_path = prefixed;
} else {
rel_path = *cur_path == '/' ? cur_path + 1 : cur_path;
}
if (options->excluded_paths &&
!excluded_sink_append(options->excluded_paths, options->excluded_mutex, rel_path))
ps->failed = true;
free(prefixed);
if (!scan_root_record_protection(options, rel, entry->d_name, cur_path, protect, passes,
use_rel, ps)) {
ps->failed = true;
goto done;
}
if (!passes) {
scanner_note_filter(options, entry->d_name);
free(rel);
free(cur_path);
return;
goto done;
}
}
if (is_dir) {
if (!scanner_same_filesystem(options->one_file_system, root_dev, st.st_dev)) {
if (options->one_file_system > 1) {
/* -xx: drop the mount-point directory entirely (rsync) and print the
--info=mount line when enabled. */
scanner_note_mount(options, cur_path);
free(rel);
free(cur_path);
return;
}
/* -x/--one-file-system: emit the mount-point directory entry (empty) but
do not descend into it (see the sequential scanner for the same rule). */
File* mount = file_create(cur_path);
free(cur_path);
if (mount == NULL) {
free(rel);
ps->failed = true;
return;
}
mount->is_dir = true;
if (options->use_metadata) {
mount->metadata = file_metadata_create(mount->path, &st, options->preserve_atimes,
options->preserve_crtimes);
if (!mount->metadata) {
free(rel);
file_destroy(mount);
ps->failed = true;
return;
}
}
if (options->relative_prefix) {
mount->send_path = scanner_prefix_send_path(options->relative_prefix, rel);
if (!mount->send_path) {
free(rel);
file_destroy(mount);
ps->failed = true;
return;
}
}
free(rel);
if (!array_list_add(root_files, mount)) {
file_destroy(mount);
ps->failed = true;
}
return;
}
free(rel);
if (!array_list_add(subdirs, cur_path)) {
free(cur_path);
ps->failed = true;
}
return;
}
File* file = file_create(cur_path);
free(cur_path);
if (!file) {
free(rel);
free(inspected.link_target);
inspected.link_target = NULL;
ps->failed = true;
return;
}
if (inspected.is_symlink) {
file->is_symlink = true;
file->symlink_target = inspected.link_target;
inspected.link_target = NULL;
scan_root_dir(options, cur_path, rel, &inspected.stats, root_files, subdirs, root_dev, ps);
} else if (S_ISCHR(inspected.stats.st_mode) || S_ISBLK(inspected.stats.st_mode) ||
S_ISFIFO(inspected.stats.st_mode) || S_ISSOCK(inspected.stats.st_mode)) {
scan_root_special(options, &inspected, rel, root_files, ps);
} else {
file->data->size = st.st_size;
}
if (use_rel) {
file->send_path = rel;
rel = NULL;
} else if (options->relative_prefix) {
file->send_path = scanner_prefix_send_path(options->relative_prefix, rel);
free(rel);
rel = NULL;
if (!file->send_path) {
file_destroy(file);
ps->failed = true;
return;
}
}
ScannerSpecial special = scanner_prepare_special(
options->preserve_devices, options->preserve_specials, options->copy_devices, file, &st);
if (special == SCANNER_SPECIAL_SKIP) {
scanner_note_nonreg(ps->options, file->path);
free(rel);
file_destroy(file);
return;
}
if (options->hardlinks && S_ISREG(st.st_mode)) {
int gid;
bool is_first;
char* first_path = NULL;
if (!hardlink_table_assign((HardLinkTable*)options->hardlinks, file_wire_path(file), st.st_dev,
st.st_ino, &gid, &is_first, &first_path)) {
ps->failed = true;
} else {
file->link_group = gid;
file->link_first = is_first;
if (!is_first) {
file->hardlink_target = first_path;
file->data->size = 0;
} else {
free(first_path);
}
}
}
if (options->use_metadata)
file->metadata =
file_metadata_create(file->path, &st, options->preserve_atimes, options->preserve_crtimes);
if (options->use_metadata && !file->metadata) {
free(rel);
file_destroy(file);
ps->failed = true;
return;
}
if ((options->preserve_xattrs || options->preserve_acls) &&
!(file->link_group != 0 && !file->link_first))
file->xattrs = file->is_symlink
? xattr_capture_path_nofollow(file->path, options->preserve_acls)
: xattr_capture_path(file->path, options->preserve_acls);
if (!array_list_add(root_files, file)) {
free(rel);
file_destroy(file);
ps->failed = true;
return;
scan_root_file(options, &inspected, rel, root_files, ps);
}
done:
free(rel);
free(cur_path);
free(inspected.link_target);
}
/* Scan the root directory itself, collecting root files and subdirectories.
@@ -589,8 +564,9 @@ ParallelScanner* parallel_scanner_create_with_options(const char* root_directory
{
char err[256];
bool any_exists = false;
FilterRuleList* own =
read_dir_filters(options, root_directory, "", &any_exists, err, sizeof(err));
FilterRuleList* own = read_dir_filters(options, root_directory, "",
options->relative && options->file_list != NULL,
&any_exists, err, sizeof(err));
if (!own) {
/* A parse/allocation failure must fail the scan even when an earlier
merge file in the same directory existed (see the sequential scanner). */
+5 -3
View File
@@ -297,11 +297,13 @@ void print_usage(void) {
printf(" --max-depth <n> Maximum directory depth (0=unlimited)\n");
printf(" -x, --one-file-system Do not cross filesystem boundaries\n");
printf(" --log-file <path>, --log-file=<path> Write log messages to file\n");
printf(" --stderr=MODE Route logging to stderr: errors or all\n");
printf(" --stderr=MODE Route logging: errors (default), all, or client\n");
printf(" (forward the client's diagnostics to the server's\n");
printf(" stderr)\n");
printf(" --msgs2stderr Route all messages to stderr (deprecated spelling of\n");
printf(" --stderr=all)\n");
printf(" --no-msgs2stderr Select errors-only stderr (deprecated spelling; the\n");
printf(" default)\n");
printf(" --no-msgs2stderr Forward the client's diagnostics to the server\n");
printf(" (deprecated spelling of --stderr=client)\n");
printf(" --partial Keep partial files on interrupted transfer\n");
printf(" --partial-dir <dir> Directory for partial files (implies --partial)\n");
printf(" -T, --temp-dir <dir> Scratch dir for temp files before atomic install.\n");
+226 -53
View File
@@ -11,10 +11,13 @@
#include "metadata.h"
#include "protocol.h"
#include "utils.h"
#include <fcntl.h>
#include <limits.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <time.h>
#include <unistd.h>
bool receiver_outcomes_append(ReceiverOutcomes* outcomes, unsigned char code) {
if (!outcomes)
@@ -101,14 +104,35 @@ static void receiver_tally_deleted(const ReceiverSink* sink, size_t deleted) {
sink->stats->deleted_files += deleted;
}
/* Observer for --info=del: record each truly-removed destination-relative path
in the ArrayList passed as the observer context, so the terminal STATUS_STATS
frame can list it. A failed append is best-effort (the deletion already
happened; output is cosmetic). Shared by the single-threaded receiver and
the -m pipeline's deferred commit. */
void receiver_record_deleted_path(void* context, const char* rel_path) {
ArrayList* paths = context;
if (!paths || !rel_path)
/* Observer for --info=del/--stats: record each truly-removed destination-
relative path (when the context carries a path list) and tally it by type
(when it carries a stats record), so the terminal STATUS_STATS frame can list
the paths and render rsync's per-type `Number of deleted files` breakdown. A
failed append is best-effort (the deletion already happened; output is
cosmetic). Shared by the single-threaded receiver and the -m pipeline's
deferred commit. */
void receiver_record_deleted_path(void* context, const char* rel_path, DeleteEntryType type) {
ReceiverDeleteContext* del = context;
if (!del || !rel_path)
return;
if (del->stats) {
switch (type) {
case DELETE_ENTRY_DIR:
del->stats->deleted_dir++;
break;
case DELETE_ENTRY_LINK:
del->stats->deleted_link++;
break;
case DELETE_ENTRY_SPECIAL:
del->stats->deleted_special++;
break;
default:
del->stats->deleted_reg++;
break;
}
}
ArrayList* paths = del->deleted_paths;
if (!paths)
return;
/* Bound the retained list like the keep-set manifest: only MAX_MANIFEST_ENTRIES
paths are ever transmitted in the terminal STATUS_STATS frame, so recording
@@ -120,6 +144,16 @@ void receiver_record_deleted_path(void* context, const char* rel_path) {
free(copy);
}
/* Install the delete observer (and its context) for one commit when the sink
carries a stats record or a path list. Returns NULL when neither is needed,
so the delete engines skip the observer entirely. */
static DeletePathObserver receiver_delete_observer(const ReceiverSink* sink,
ReceiverDeleteContext* del) {
del->stats = sink ? sink->stats : NULL;
del->deleted_paths = sink ? sink->deleted_paths : NULL;
return (del->stats || del->deleted_paths) ? receiver_record_deleted_path : NULL;
}
static bool receiver_process_chunk(Chunk* chunk, const ReceiverSink* sink) {
if (!chunk || !sink || !sink->store_file)
return false;
@@ -277,6 +311,7 @@ static bool status_counts_as_progress(Status status) {
case STATUS_ABORT:
case STATUS_CHECK_BATCH:
case STATUS_DIR_TIMES:
case STATUS_CLIENT_MSG:
return false;
default:
return true;
@@ -306,7 +341,13 @@ static bool receiver_note_status(const struct timespec* session_start,
}
int receiver_process(Config* config, int file_descriptor, const ReceiverSink* sink) {
return receiver_process_pending(config, file_descriptor, sink, NULL, NULL);
return receiver_process_pending_ctx(config, file_descriptor, sink, NULL, NULL, NULL);
}
int receiver_process_pending(Config* config, int file_descriptor, const ReceiverSink* sink,
DeleteManifest** pending_manifest, DeletePlanSession** pending_plans) {
return receiver_process_pending_ctx(config, file_descriptor, sink, pending_manifest,
pending_plans, NULL);
}
/* Per-connection state threaded through the status handlers below. The parked
@@ -327,6 +368,11 @@ typedef struct {
a successful FINISHED it is either committed here or handed to
*pending_plans so the -m caller commits after its disk writer drained. */
DeletePlanSession* plan_session;
/* Observer context for the per-directory delete session, which outlives the
frame handler; must stay alive until the session commits. For a session
handed to the caller (pending_plans) this points at a caller-owned
long-lived context; otherwise it points at an internal stack context. */
ReceiverDeleteContext* delete_ctx;
bool early_delete;
bool per_dir_delete;
bool delete_limit_noted;
@@ -347,6 +393,23 @@ static ReceiverStep receiver_handle_keepalive(ReceiverPendingState* state) {
return RECEIVER_STEP_NEXT;
}
/* rsync --stderr=client: a client diagnostic forwarded over the wire. Read the
* bounded string and log it through the normal destination/level gate. The
* body is peer-controlled text: log_client_message() escapes every
* non-printable byte (newlines, CR, ANSI ESC, ...) before writing, so a hostile
* client cannot forge log lines or inject terminal control sequences. A
* malformed string (over-long or embedded NUL) is a framing error and tears the
* connection down. */
static ReceiverStep receiver_handle_client_msg(ReceiverPendingState* state) {
char* message = receive_str(state->fd);
if (!message)
return RECEIVER_STEP_FAIL;
if (message[0] != '\0')
log_client_message(message);
free(message);
return RECEIVER_STEP_NEXT;
}
static ReceiverStep receiver_handle_abort(ReceiverPendingState* state) {
(void)state;
log_message(LOG_LEVEL_INFO, "Received abort from client, cleaning up");
@@ -383,9 +446,95 @@ static ReceiverStep receiver_handle_check_batch(ReceiverPendingState* state) {
return RECEIVER_STEP_NEXT;
}
/* Probe a destination entry's pre-transfer state for the output-parity
dest-info report (protocol 2.30.0). `wire_path` is the destination-relative
path; `incoming_target` is non-NULL only for a symlink probe, in which case
the on-disk link target is compared with the target the receiver is about to
store (after --munge-links). The final component is never followed and the
parent walk is confined below the receive root. Returns false only on an
allocation/secure-walk failure; a missing entry is reported as existed=false. */
static bool receiver_probe_dest_state(const Config* config, const char* wire_path,
const char* incoming_target, OutputDestState* out) {
memset(out, 0, sizeof(*out));
out->known = true;
if (!config || !wire_path || wire_path[0] == '\0')
return false;
char* full = path_cat(config->receive_root_directory, wire_path);
if (!full)
return false;
char* leaf = NULL;
int parent_fd = file_open_secure_parent(full, &leaf, false);
free(full);
if (parent_fd < 0) {
/* A missing/unreachable parent means the entry cannot exist yet. */
free(leaf);
return true;
}
struct stat st;
if (fstatat(parent_fd, leaf, &st, AT_SYMLINK_NOFOLLOW) == 0) {
out->existed = true;
out->size = (unsigned long long)st.st_size;
out->mtime_sec = (long long)st.st_mtime;
#ifdef __linux__
out->mtime_nsec = st.st_mtim.tv_nsec;
#endif
out->mode = (uint32_t)st.st_mode;
out->uid = (int32_t)st.st_uid;
out->gid = (int32_t)st.st_gid;
if (incoming_target && S_ISLNK(st.st_mode)) {
char target_buf[PATH_MAX];
ssize_t n = readlinkat(parent_fd, leaf, target_buf, sizeof(target_buf) - 1);
if (n >= 0) {
target_buf[n] = '\0';
char* expected =
config->munge_links ? file_symlink_munge(incoming_target) : str_dup(incoming_target);
if (expected) {
out->target_matches = strcmp(target_buf, expected) == 0;
free(expected);
}
}
}
}
close(parent_fd);
free(leaf);
return true;
}
static ReceiverStep receiver_handle_mkdir(ReceiverPendingState* state) {
File* dir = file_receive_directory(state->fd, state->config);
if (!dir || !state->sink->store_file(dir, state->sink->context))
const Config* config = state->config;
int fd = state->fd;
if (config->report_dest_info) {
int probe = 0;
if (!receive_int(fd, &probe) || (probe != 0 && probe != 1))
return RECEIVER_STEP_FAIL;
if (probe) {
/* Probe-only frame: report the destination state and create nothing. */
char* path = receive_wire_str(fd);
if (!path || path[0] == '\0' || (!file_get_trust_sender() && has_path_traversal(path))) {
free(path);
send_status(fd, STATUS_ERROR);
return RECEIVER_STEP_FAIL;
}
OutputDestState info;
bool ok = receiver_probe_dest_state(config, path, NULL, &info);
free(path);
if (!ok || !send_status(fd, STATUS_DEST_INFO) || !format_dest_state_send(fd, &info))
return RECEIVER_STEP_FAIL;
return RECEIVER_STEP_NEXT;
}
}
File* dir = file_receive_directory(fd, config);
if (!dir)
return RECEIVER_STEP_ERROR;
if (config->report_dest_info) {
OutputDestState info;
bool ok = receiver_probe_dest_state(config, file_wire_path(dir), NULL, &info);
if (!ok || !send_status(fd, STATUS_DEST_INFO) || !format_dest_state_send(fd, &info)) {
file_destroy(dir);
return RECEIVER_STEP_FAIL;
}
}
if (!state->sink->store_file(dir, state->sink->context))
return RECEIVER_STEP_ERROR;
return RECEIVER_STEP_NEXT;
}
@@ -404,8 +553,20 @@ static ReceiverStep receiver_handle_hardlink(ReceiverPendingState* state) {
}
static ReceiverStep receiver_handle_symlink(ReceiverPendingState* state) {
File* sym = file_receive_symlink(state->fd, state->config);
if (!sym || !state->sink->store_file(sym, state->sink->context))
const Config* config = state->config;
File* sym = file_receive_symlink(state->fd, config);
if (!sym)
return RECEIVER_STEP_ERROR;
if (config->report_dest_info) {
OutputDestState info;
bool ok = receiver_probe_dest_state(config, file_wire_path(sym), sym->symlink_target, &info);
if (!ok || !send_status(state->fd, STATUS_DEST_INFO) ||
!format_dest_state_send(state->fd, &info)) {
file_destroy(sym);
return RECEIVER_STEP_FAIL;
}
}
if (!state->sink->store_file(sym, state->sink->context))
return RECEIVER_STEP_ERROR;
return RECEIVER_STEP_NEXT;
}
@@ -448,12 +609,11 @@ static ReceiverStep receiver_handle_manifest(ReceiverPendingState* state) {
--max-delete-capped commit still succeeds and the transfer proceeds;
the terminal success frame reports the cap. */
size_t deleted = 0;
DeletePathObserver observer =
(config->report_deletes && sink->deleted_paths) ? receiver_record_deleted_path : NULL;
ReceiverDeleteContext delctx;
DeletePathObserver observer = receiver_delete_observer(sink, &delctx);
DeleteCommitResult deletion =
(config->use_delete || config->delete_missing_args)
? manifest_delete_all_observed(config, manifest, &deleted, observer,
(void*)sink->deleted_paths)
? manifest_delete_all_observed(config, manifest, &deleted, observer, &delctx)
: DELETE_COMMIT_OK;
receiver_tally_deleted(sink, deleted);
delete_manifest_free(manifest);
@@ -496,9 +656,9 @@ static ReceiverStep receiver_handle_delete_plan(ReceiverPendingState* state) {
}
if (!state->plan_session) {
state->plan_session = delete_plan_session_create(config);
if (state->plan_session && config->report_deletes && sink->deleted_paths)
if (state->plan_session && (sink->stats || sink->deleted_paths))
delete_plan_session_set_delete_observer(state->plan_session, receiver_record_deleted_path,
(void*)sink->deleted_paths);
state->delete_ctx);
}
if (!state->plan_session || delete_plan_session_receive(state->plan_session, config, fd) != 0)
return RECEIVER_STEP_FAIL;
@@ -527,6 +687,8 @@ static ReceiverStep receiver_dispatch_status(ReceiverPendingState* state, Status
switch (status) {
case STATUS_KEEPALIVE:
return receiver_handle_keepalive(state);
case STATUS_CLIENT_MSG:
return receiver_handle_client_msg(state);
case STATUS_ABORT:
return receiver_handle_abort(state);
case STATUS_CHECK:
@@ -579,8 +741,10 @@ static void receiver_drop_pending(ReceiverPendingState* state) {
delete-during plan mode (no manifest at all). See the per-frame handlers
above for how the -m receiver defers that commit until its disk writer has
drained. */
int receiver_process_pending(Config* config, int file_descriptor, const ReceiverSink* sink,
DeleteManifest** pending_manifest, DeletePlanSession** pending_plans) {
int receiver_process_pending_ctx(Config* config, int file_descriptor, const ReceiverSink* sink,
DeleteManifest** pending_manifest,
DeletePlanSession** pending_plans,
ReceiverDeleteContext* observer_ctx) {
Status status;
if (!receive_status(file_descriptor, &status))
return -1;
@@ -593,6 +757,13 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver
last_progress = session_start;
if (!receiver_note_status(&session_start, &last_progress, status, file_descriptor, sink))
return -1;
/* A per-directory delete session handed to the caller outlives this stack
frame, so its observer context must be caller-owned (observer_ctx); only
the default inline-commit case may use the stack context. */
ReceiverDeleteContext local_ctx;
ReceiverDeleteContext* delete_ctx = observer_ctx ? observer_ctx : &local_ctx;
delete_ctx->stats = sink ? sink->stats : NULL;
delete_ctx->deleted_paths = sink ? sink->deleted_paths : NULL;
ReceiverPendingState state = {
.config = config,
.fd = file_descriptor,
@@ -601,6 +772,7 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver
.pending_plans = pending_plans,
.deferred_manifest = NULL,
.plan_session = NULL,
.delete_ctx = delete_ctx,
.early_delete = config_delete_timing_early(config),
.per_dir_delete = config_delete_timing_per_dir(config),
.delete_limit_noted = false,
@@ -610,7 +782,7 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver
status == STATUS_KEEPALIVE || status == STATUS_ABORT || status == STATUS_CHECK_BATCH ||
status == STATUS_MKDIR || status == STATUS_MANIFEST || status == STATUS_HARDLINK ||
status == STATUS_SYMLINK || status == STATUS_SPECIAL || status == STATUS_DIR_TIMES ||
status == STATUS_DELETE_PLAN) {
status == STATUS_DELETE_PLAN || status == STATUS_CLIENT_MSG) {
ReceiverStep step = receiver_dispatch_status(&state, status);
if (step == RECEIVER_STEP_FAIL)
goto fail;
@@ -625,25 +797,36 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver
log_message(LOG_LEVEL_ERROR, "Did not receive FINISHED Status");
goto receive_error;
}
/* --delay-updates: publish every staged file BEFORE the deferred delete
commit, matching rsync's --delete-after ordering (all updates land first,
then extras are removed). The single-threaded receiver stores files
synchronously, so every staged file is complete here. The -m receiver
hands both publication and deletion to its caller via
pending_manifest/pending_plans; that caller publishes first, after its disk
writer has drained. */
bool handoff = state.pending_manifest != NULL || state.pending_plans != NULL;
if (!handoff && !config->dry_run && config->delay_updates && config->delay_context) {
if (!delay_updates_publish(config->delay_context, config)) {
send_status(file_descriptor, STATUS_ERROR);
goto fail;
}
}
/* Commit-style (late) deletion: every data frame has been received and the
sender proved the whole tree with STATUS_FINISHED. The single-threaded
receiver stores files synchronously, so everything is on disk here and the
deletion can be committed before the --delay-updates publication in
send_success (the walker skips the staging dir, so staged files are never
treated as extras). The -m receiver passes `pending_manifest` because its
disk writer may still be draining; the caller commits after the writer has
joined so no extra file is removed unless the transfer is known to have
succeeded. */
receiver stores files synchronously, so everything is on disk here (and a
--delay-updates run has already published above). The -m receiver passes
`pending_manifest` because its disk writer may still be draining; the
caller commits after the writer has joined so no extra file is removed
unless the transfer is known to have succeeded. */
if (state.deferred_manifest) {
if (state.pending_manifest) {
*state.pending_manifest = state.deferred_manifest;
state.deferred_manifest = NULL;
} else {
size_t deleted = 0;
DeletePathObserver observer =
(config->report_deletes && sink->deleted_paths) ? receiver_record_deleted_path : NULL;
DeletePathObserver observer = receiver_delete_observer(sink, state.delete_ctx);
DeleteCommitResult deletion = manifest_delete_all_observed(
config, state.deferred_manifest, &deleted, observer, (void*)sink->deleted_paths);
config, state.deferred_manifest, &deleted, observer, state.delete_ctx);
receiver_tally_deleted(sink, deleted);
delete_manifest_free(state.deferred_manifest);
state.deferred_manifest = NULL;
@@ -661,9 +844,9 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver
hands the session to its caller instead, which commits after the disk
writer drained. */
if (state.plan_session) {
if (config->report_deletes && sink->deleted_paths)
if (sink->stats || sink->deleted_paths)
delete_plan_session_set_delete_observer(state.plan_session, receiver_record_deleted_path,
(void*)sink->deleted_paths);
state.delete_ctx);
if (state.pending_plans) {
*state.pending_plans = state.plan_session;
state.plan_session = NULL;
@@ -793,13 +976,14 @@ static void receiver_note_delete_limit(void* context_pointer) {
/* Terminal status for a run. A capped --delete limit wins (rsync exit 25);
otherwise any per-entry failure (for example an unprivileged --devices
mknod) makes the terminal frame non-OK so the client exits non-zero. rsync
reports 23 here; mapping the client's exact exit code to 23 is a separate,
pre-existing concern. A clean run keeps STATUS_OK. */
mknod) makes the terminal frame STATUS_PARTIAL so the client exits 23
(rsync's "partial transfer due to error") while still removing the sources
it successfully transferred under --remove-source-files. A fatal stream
error keeps STATUS_ERROR (a non-23 exit). A clean run keeps STATUS_OK. */
static Status receiver_final_status(bool delete_limit_reached, size_t failed_entries) {
if (delete_limit_reached)
return STATUS_DELETE_LIMIT;
return failed_entries > 0 ? STATUS_ERROR : STATUS_OK;
return failed_entries > 0 ? STATUS_PARTIAL : STATUS_OK;
}
static bool receiver_send_success_frame(int fd, void* context_pointer) {
@@ -817,21 +1001,10 @@ static bool receiver_send_success_frame(int fd, void* context_pointer) {
nothing to publish and no directory times to stamp. */
if (context->config->dry_run)
return receiver_send_final_success(fd, context->config, &context->outcomes, final_status);
/* --delay-updates: the whole protocol stream (including manifest/delete
handling, which ran inside receiver_process) has succeeded and every
staged file was fully written. Publish them atomically now, before the
success/outcome frame tells a --remove-source-files sender it may delete
its sources. */
if (context->config->delay_updates && context->config->delay_context) {
if (!delay_updates_publish(context->config->delay_context, context->config)) {
send_status(fd, STATUS_ERROR);
return false;
}
}
/* P7 Wave D: every child is now written and the delete / --delay-updates
phases have committed, so it is finally safe to stamp directory times.
This runs after the deferred deletion because receiver_process commits it
before calling this success frame. */
/* P7 Wave D: every child is now written and the --delay-updates publication
(done in receiver_process before the delete commit) plus the deferred
deletion have both committed, so it is finally safe to stamp directory
times. */
dir_metadata_list_apply(&context->dir_times, context->config->receive_root_directory,
context->config);
return receiver_send_final_success(fd, context->config, &context->outcomes, final_status);
+25 -4
View File
@@ -55,10 +55,20 @@ typedef struct {
bool receiver_outcomes_append(ReceiverOutcomes* outcomes, unsigned char code);
void receiver_outcomes_destroy(ReceiverOutcomes* outcomes);
/* DeletePathObserver implementation for --info=del: `context` is an ArrayList*
that receives owned copies of every truly-removed destination-relative path.
Shared by the single-threaded receiver and the -m pipeline's deferred commit. */
void receiver_record_deleted_path(void* context, const char* rel_path);
/* Delete observer context: `deleted_paths` (optional) receives owned copies of
every truly-removed destination-relative path for --info=del; `stats`
(optional) receives the per-type `Number of deleted files` tallies for
--stats. Both may be NULL, in which case the observer is a no-op. */
typedef struct {
ReceiverStats* stats;
struct ArrayList* deleted_paths;
} ReceiverDeleteContext;
/* DeletePathObserver implementation: records each truly-removed path (when the
context carries a path list) and tallies it by type (when it carries a stats
record). Shared by the single-threaded receiver and the -m pipeline's
deferred commit. */
void receiver_record_deleted_path(void* context, const char* rel_path, DeleteEntryType type);
/* Send the terminal success frame. `final_status` is usually STATUS_OK, or
STATUS_DELETE_LIMIT when a --max-delete commit was capped. */
@@ -83,6 +93,17 @@ int receiver_process(Config* config, int file_descriptor, const ReceiverSink* si
for either to keep the default behaviour (delete before the success frame). */
int receiver_process_pending(Config* config, int file_descriptor, const ReceiverSink* sink,
DeleteManifest** pending_manifest, DeletePlanSession** pending_plans);
/* receiver_process_pending() with an explicit observer context for a
per-directory delete session that is handed to the caller via
`pending_plans`. The session outlives this call (the -m pipeline commits it
after joining its disk writer), so its observer context must too: pass a
long-lived object such as PipelineContextReceiver.delete_ctx. When
`delete_ctx` is NULL an internal stack context is used, which is only safe
when the session is committed before returning (the default behaviour). */
int receiver_process_pending_ctx(Config* config, int file_descriptor, const ReceiverSink* sink,
DeleteManifest** pending_manifest,
DeletePlanSession** pending_plans,
ReceiverDeleteContext* delete_ctx);
int receiver_receive_files(Config* config, int file_descriptor);
/* ---- Connection time bounds (anti-slowloris) ----
+5 -2
View File
@@ -28,6 +28,8 @@ PipelineContextReceiver* pipeline_context_receiver_create(Config* config, Queue*
context->max_queue_bytes = 0;
context->deferred_manifest = NULL;
context->deferred_plans = NULL;
context->delete_ctx.stats = NULL;
context->delete_ctx.deleted_paths = NULL;
context->delete_limit_reached = false;
context->failed_entries = 0;
memset(&context->stats, 0, sizeof(context->stats));
@@ -205,8 +207,9 @@ int receive_thread(void* pipeline_context) {
&context->stats,
context->would_delete,
context->deleted_paths};
if (receiver_process_pending((Config*)config, file_descriptor, &sink, &context->deferred_manifest,
&context->deferred_plans) != 0) {
if (receiver_process_pending_ctx((Config*)config, file_descriptor, &sink,
&context->deferred_manifest, &context->deferred_plans,
&context->delete_ctx) != 0) {
receiver_thread_fail(context);
protocol_session_unbind();
return thrd_error;
+5
View File
@@ -46,6 +46,11 @@ typedef struct PipelineContextReceiver {
committing while the disk writer may still be draining; server.c commits it
after both threads joined. NULL for every other timing. */
DeletePlanSession* deferred_plans;
/* Observer context for `deferred_plans`. It must outlive the receive thread
(the session is committed by server.c after both threads join), so it lives
here rather than on receiver_process_pending()'s stack; receive_thread
installs it on the session. */
ReceiverDeleteContext delete_ctx;
/* Set by server.c when the deferred delete commit hit the --max-delete
budget; the terminal success frame then carries STATUS_DELETE_LIMIT
(rsync exit 25) while the transfer itself still succeeds. */
+311 -270
View File
@@ -981,18 +981,31 @@ static void server_run_mt_receiver(ServerSession* state) {
thrd_join(writer, &writer_result);
bool transfer_ok = receiver_result == thrd_success && writer_result == thrd_success;
PipelineContextReceiver* context = state->context;
if (transfer_ok && !config->dry_run) {
/* --delay-updates: receive_thread has finished the whole protocol stream
and write_thread has drained its queue, so every staged file is complete.
Publish atomically BEFORE the deferred delete commit, matching rsync's
--delete-after ordering (all updates land first, then extras are
removed). */
if (config->delay_updates && config->delay_context &&
!delay_updates_publish(config->delay_context, config)) {
transfer_ok = false;
}
}
if (transfer_ok && !config->dry_run) {
/* Commit-style (late) deletion: receive_thread handed the keep-set
manifest here instead of deleting while write_thread might still be
draining, so by now every file is on disk and the whole transfer is
known to have succeeded. Remove the extras before publishing a
--delay-updates run; the walker skips the staging directory. A
draining, so by now every file is on disk (and a --delay-updates run has
already published above) and the whole transfer is known to have
succeeded. The walker skips the staging directory. A
server-contacting --dry-run deletes nothing (no manifest is sent). */
if (context->deferred_manifest) {
size_t deleted = 0;
DeletePathObserver observer = config->report_deletes ? receiver_record_deleted_path : NULL;
DeleteCommitResult deletion = manifest_delete_all_observed(
config, context->deferred_manifest, &deleted, observer, (void*)context->deleted_paths);
ReceiverDeleteContext delctx = {&context->stats, context->deleted_paths};
DeletePathObserver observer =
(delctx.stats || delctx.deleted_paths) ? receiver_record_deleted_path : NULL;
DeleteCommitResult deletion = manifest_delete_all_observed(config, context->deferred_manifest,
&deleted, observer, &delctx);
context->stats.deleted_files += deleted;
if (deletion == DELETE_COMMIT_ERROR) {
transfer_ok = false;
@@ -1009,10 +1022,9 @@ static void server_run_mt_receiver(ServerSession* state) {
--delete-during already applied its plans on the receive thread. */
if (context->deferred_plans) {
/* Defence in depth (the enclosing block already excludes dry-run): a
-n run never commits a deletion. */
if (config->report_deletes)
delete_plan_session_set_delete_observer(
context->deferred_plans, receiver_record_deleted_path, (void*)context->deleted_paths);
-n run never commits a deletion. The session's observer context was
installed by receive_thread from context->delete_ctx, which outlives
both threads, so no stack context is needed here. */
DeleteCommitResult deletion =
config->dry_run ? DELETE_COMMIT_OK
: delete_plan_session_commit(context->deferred_plans, config);
@@ -1025,21 +1037,10 @@ static void server_run_mt_receiver(ServerSession* state) {
delete_plan_session_destroy(context->deferred_plans);
context->deferred_plans = NULL;
}
}
if (transfer_ok && !config->dry_run) {
/* --delay-updates: receive_thread has finished the whole protocol stream
(including manifest/delete handling) and write_thread has drained its
queue, so every staged file is complete. Publish atomically before the
success/outcome frame so a --remove-source-files sender only learns of
files that were actually installed. */
if (config->delay_updates && config->delay_context &&
!delay_updates_publish(config->delay_context, config)) {
transfer_ok = false;
}
/* P7 Wave D: all writers have joined and the late deletion (and
--delay-updates publication) has committed above, so it is finally safe
to stamp directory times; a directory's mtime must not be clobbered by
its children or by an extra removal. */
/* P7 Wave D: all writers have joined and the --delay-updates publication
plus the late deletion have committed above, so it is finally safe to
stamp directory times; a directory's mtime must not be clobbered by its
children or by an extra removal. */
if (transfer_ok)
dir_metadata_list_apply(&context->dir_times, config->receive_root_directory, config);
}
@@ -1050,7 +1051,7 @@ static void server_run_mt_receiver(ServerSession* state) {
context->failed_entries, context->failed_entries == 1 ? "y" : "ies");
Status final_status = context->delete_limit_reached
? STATUS_DELETE_LIMIT
: (context->failed_entries > 0 ? STATUS_ERROR : STATUS_OK);
: (context->failed_entries > 0 ? STATUS_PARTIAL : STATUS_OK);
/* Emit the optional wire-stats record first (protocol 2.25.0), then the
success/outcome frame, exactly like the single-threaded receiver. */
if (!receiver_send_stats_frame(state->fd, config, &context->stats, context->would_delete,
@@ -1305,6 +1306,284 @@ static bool daemonize(void) {
return true;
}
/* Apply the process-wide policies shared by the stdio and listener
* entrypoints: logging verbosity, signal handling, the parsed server
* authorization policies, and the socket timeout floor. Runs after CLI
* parsing and after the standalone --hash-credentials tool has been ruled
* out. */
static void configure_server_process(const ServerCliOptions* opts) {
signal(SIGPIPE, SIG_IGN);
if (opts->verbose) {
set_log_level(LOG_LEVEL_DEBUG);
set_log_debug_flags(LOG_DEBUG_ALL);
}
if (opts->tls_ca && !opts->use_tls)
log_message(LOG_LEVEL_WARNING, "--ca has no effect without --tls");
/* Persist the parsed server policies into the process-global policy state
* BEFORE the stdio branch: an SSH-launched `--stdio` server (whose argv came
* from the client via --remote-option and friends) must honor --allow-delete,
* --trust-sender and --client-cn exactly like the standalone listener. */
required_client_cn = opts->client_cn;
allow_delete = opts->allow_delete;
trust_sender = opts->trust_sender;
allow_unauthenticated = opts->allow_unauthenticated;
server_no_super = opts->no_super;
/* --stdio rejects --allow-super at parse time; force it off here as well so
* this process-global policy cannot be re-enabled by a future caller. */
server_allow_super = opts->allow_super && !opts->stdio_mode;
server_iconv_spec = opts->iconv_spec;
install_cleanup_handler(SIGINT);
install_cleanup_handler(SIGTERM);
/* Server-owned socket deadline floor: the client default --timeout=0 would
* otherwise leave accepted sockets without SO_RCVTIMEO/SO_SNDTIMEO and let a
* silent peer hold a connection (and its process slot) forever. */
tcp_set_timeouts(SERVER_IO_TIMEOUT_SEC, SERVER_IO_TIMEOUT_SEC);
}
/* --hash-credentials: standalone offline tool; read user:password lines and
* emit new-format credential-store lines, then exit. Consumes and frees
* opts. */
static int run_hash_credentials_tool(ServerCliOptions* opts) {
uint32_t iters = opts->hash_iterations_set ? opts->hash_iterations : CREDENTIAL_DEFAULT_ITERS;
/* The output is secret material: if it is redirected to a regular file,
* warn when that file is group/other-accessible (the store must be 0600). */
struct stat out_st;
if (fstat(STDOUT_FILENO, &out_st) == 0 && S_ISREG(out_st.st_mode) &&
(out_st.st_mode & (S_IRWXG | S_IRWXO)) != 0)
fprintf(stderr,
"Warning: credential-store output is a group/other-accessible file; restrict it to "
"mode 0600 (chmod 600)\n");
char hash_err[512];
if (credentials_hash_file(opts->hash_credentials_file, iters, stdout, hash_err,
sizeof(hash_err)) != 0) {
fprintf(stderr, "Error: %s\n", hash_err);
server_cli_options_free(opts);
return 1;
}
server_cli_options_free(opts);
return 0;
}
/* SSH --stdio session: the transport is authenticated by sshd outside of
* FastSync, so the single connection is served over STDIN/STDOUT and the
* process exits. The destination root is authorized exactly like the listener
* path. Consumes and frees opts. */
static int run_stdio_server(ServerCliOptions* opts) {
/* SSH authenticates the stdio transport outside of FastSync. */
allow_unauthenticated = true;
if (!configure_authorization(opts->destination_root)) {
char* escaped = output_escape(opts->destination_root, false);
fprintf(stderr, "Error: invalid destination root '%s'\n",
escaped ? escaped : "<allocation failed>");
free(escaped);
server_cli_options_free(opts);
return 1;
}
io_set_fds(STDIN_FILENO, STDOUT_FILENO);
/* handler() does not own the stdio fds: it never closes its descriptor
* argument, so STDIN/STDOUT stay open for this (single-shot) SSH session
* and are released by process exit. */
handler(STDIN_FILENO);
release_authorization();
server_cli_options_free(opts);
return 0;
}
/* Load the daemon config, apply --dparam overrides, resolve the effective
* port/address, and surface the operator-facing module warnings. On failure
* the error is printed and false is returned. */
static bool load_daemon_policy(ServerCliOptions* opts, int* port, const char** bind_address,
char* err, size_t err_size) {
const char* config_path = opts->config_path ? opts->config_path : default_daemon_config_path();
g_daemon_conf = daemon_conf_load(config_path, err, err_size);
if (!g_daemon_conf) {
fprintf(stderr, "Error: %s\n", err);
return false;
}
for (int i = 0; i < opts->dparam_count; i++) {
if (daemon_conf_apply_dparam(g_daemon_conf, opts->dparams[i], err, err_size) != 0) {
fprintf(stderr, "Error: --dparam: %s\n", err);
return false;
}
}
/* Effective port: -p (highest) > --dparam port > config port (default 873). */
if (!opts->port_set)
*port = g_daemon_conf->global.port;
if (!*bind_address)
*bind_address = g_daemon_conf->global.address;
if (g_daemon_conf->module_count == 0)
log_message(LOG_LEVEL_WARNING,
"daemon config has no modules; every connection will be refused");
/* Surface the operator's client-chosen-ownership opt-in prominently: an
opted-in module lets its clients request arbitrary owner ids inside that
module root. */
for (int i = 0; i < g_daemon_conf->module_count; i++) {
if (g_daemon_conf->modules[i].client_owner)
log_message(LOG_LEVEL_WARNING,
"daemon module '%s' allows client-chosen ownership and super-user device "
"activities (`client owner = yes`); clients may request arbitrary owner ids "
"and device nodes within that module root -- pair it with `auth users` "
"unless the module is intentionally open to the network",
g_daemon_conf->modules[i].name);
if (g_daemon_conf->modules[i].max_connections > 0)
log_message(LOG_LEVEL_INFO,
"daemon module '%s': per-module 'max connections' cap = %d (enforced "
"across all connection children)",
g_daemon_conf->modules[i].name, g_daemon_conf->modules[i].max_connections);
}
return true;
}
/* Load the daemon credential store (Wave B) and enforce the fail-closed
* startup check: a module that declares `auth users` without a store (or with
* an empty store) refuses to start rather than serving a module whose
* credentials can never be verified. On failure the error is printed and
* false is returned. */
static bool validate_daemon_credentials(const ServerCliOptions* opts, char* err, size_t err_size) {
/* --password-file and --early-input feed the same store, loaded BEFORE the
* listener forks so every connection child shares one read-only store. */
g_credentials = credentials_load(opts->password_file, opts->early_input_file, err, err_size);
if (!g_credentials) {
fprintf(stderr, "Error: %s\n", err);
return false;
}
bool credential_source_given = opts->password_file != NULL || opts->early_input_file != NULL;
for (int i = 0; i < g_daemon_conf->module_count; i++) {
const DaemonModule* module = &g_daemon_conf->modules[i];
if (module->auth_user_count == 0)
continue;
if (!credential_source_given) {
fprintf(stderr,
"Error: module '%s' declares 'auth users' but no credential store was given "
"(--password-file or --early-input); refusing to start (fail closed)\n",
module->name);
return false;
}
if (credentials_store_size(g_credentials) == 0) {
fprintf(stderr,
"Error: module '%s' declares 'auth users' but the credential store is empty; "
"refusing to start (fail closed)\n",
module->name);
return false;
}
for (int j = 0; j < module->auth_user_count; j++) {
if (!credentials_store_has(g_credentials, module->auth_users[j]))
log_message(LOG_LEVEL_WARNING,
"daemon module '%s': auth user '%s' has no credential store entry; that "
"user can never authenticate",
module->name, module->auth_users[j]);
}
}
return true;
}
/* Create the shared cross-process registry for the per-module / per-source
* caps and the auth lockout. Called in the parent before any accept-loop fork;
* every connection child inherits the mapping. A failure degrades to
* "registry disabled" (the global cap and host ACLs still apply) rather than
* refusing to start. */
static void create_daemon_limits(void) {
g_daemon_limits = daemon_limits_create(
(int)g_daemon_conf->global.max_connections, g_daemon_conf->module_count,
g_daemon_conf->global.max_connections_per_host, g_daemon_conf->global.auth_lockout_threshold,
g_daemon_conf->global.auth_lockout_duration_sec);
if (!g_daemon_limits)
log_message(LOG_LEVEL_WARNING,
"daemon: could not allocate the shared connection registry; per-module / "
"per-host caps and the cross-process auth lockout are disabled (the global "
"'max connections' cap and host ACLs still apply)");
}
/* Bind the listener, apply the daemon caps, set up TLS when requested, detach
* when daemonizing, and run the accept loop. Returns the process exit code. */
static int start_listener(ServerCliOptions* opts, int port, int bind_family,
const char* bind_address) {
ServerBindOptions bind_opts;
bind_opts.bind_address = bind_address;
bind_opts.family = bind_family;
g_server = server_create_ex(port, &bind_opts);
if (!g_server) {
log_message(LOG_LEVEL_ERROR, "Failed to create server");
release_authorization();
return 1;
}
if (g_daemon_conf)
server_set_max_connections(g_server, (unsigned int)g_daemon_conf->global.max_connections);
if (g_daemon_limits)
server_set_limit_registry(g_server, g_daemon_limits);
if (opts->use_tls) {
if (!opts->tls_cert || !opts->tls_key || !opts->tls_ca || !opts->client_cn) {
fprintf(stderr, "Error: --tls requires --cert, --key, --ca, and --client-cn\n");
server_delete(&g_server);
release_authorization();
return 1;
}
tls_global_init();
if (!server_create_tls(g_server, opts->tls_cert, opts->tls_key, opts->tls_ca)) {
log_message(LOG_LEVEL_ERROR, "Failed to set up TLS");
server_delete(&g_server);
release_authorization();
return 1;
}
}
/* Detach after the listening socket (and TLS context) exist so the
* background daemon inherits a fully-bound listener. --no-detach runs in
* the foreground, which is how tests drive the daemon. */
if (opts->daemon_mode && !opts->no_detach) {
if (!daemonize()) {
log_message(LOG_LEVEL_ERROR, "Failed to daemonize");
server_delete(&g_server);
release_authorization();
return 1;
}
}
if (opts->use_tls)
server_listen_tls(g_server, handler);
else
server_listen(g_server, handler);
server_delete(&g_server);
release_authorization();
return 0;
}
/* Listener entrypoint: the daemon (config-driven, possibly detached) and the
* standalone TCP server share the same bind/TLS/listen path. Consumes and
* frees opts. */
static int run_daemon_server(ServerCliOptions* opts) {
int port = opts->port;
const char* bind_address = opts->bind_address;
char cli_err[512];
int exit_code = 0;
if (opts->daemon_mode) {
if (!load_daemon_policy(opts, &port, &bind_address, cli_err, sizeof(cli_err)) ||
!validate_daemon_credentials(opts, cli_err, sizeof(cli_err))) {
exit_code = 1;
goto out;
}
create_daemon_limits();
} else if (!configure_authorization(opts->destination_root)) {
char* escaped = output_escape(opts->destination_root, false);
fprintf(stderr, "Error: invalid destination root '%s'\n",
escaped ? escaped : "<allocation failed>");
free(escaped);
exit_code = 1;
goto out;
}
exit_code = start_listener(opts, port, opts->bind_family, bind_address);
out:
daemon_limits_destroy(g_daemon_limits);
g_daemon_limits = NULL;
daemon_conf_free(g_daemon_conf);
g_daemon_conf = NULL;
credentials_free(g_credentials);
g_credentials = NULL;
server_cli_options_free(opts);
return exit_code;
}
int main(int argc, char* argv[]) {
/* Capture the process umask now, while still single-threaded: the cached
* value is what file_mode_base() uses, and reading it later would race with
@@ -1324,250 +1603,12 @@ int main(int argc, char* argv[]) {
return 1;
}
/* --hash-credentials: standalone offline tool; read user:password lines and
* emit new-format credential-store lines, then exit. */
if (opts.hash_credentials_file) {
uint32_t iters = opts.hash_iterations_set ? opts.hash_iterations : CREDENTIAL_DEFAULT_ITERS;
/* The output is secret material: if it is redirected to a regular file,
* warn when that file is group/other-accessible (the store must be 0600). */
struct stat out_st;
if (fstat(STDOUT_FILENO, &out_st) == 0 && S_ISREG(out_st.st_mode) &&
(out_st.st_mode & (S_IRWXG | S_IRWXO)) != 0)
fprintf(stderr,
"Warning: credential-store output is a group/other-accessible file; restrict it to "
"mode 0600 (chmod 600)\n");
char hash_err[512];
if (credentials_hash_file(opts.hash_credentials_file, iters, stdout, hash_err,
sizeof(hash_err)) != 0) {
fprintf(stderr, "Error: %s\n", hash_err);
server_cli_options_free(&opts);
return 1;
}
server_cli_options_free(&opts);
return 0;
}
if (opts.hash_credentials_file)
return run_hash_credentials_tool(&opts);
int exit_code = 0;
signal(SIGPIPE, SIG_IGN);
if (opts.verbose) {
set_log_level(LOG_LEVEL_DEBUG);
set_log_debug_flags(LOG_DEBUG_ALL);
}
if (opts.tls_ca && !opts.use_tls)
log_message(LOG_LEVEL_WARNING, "--ca has no effect without --tls");
/* Persist the parsed server policies into the process-global policy state
* BEFORE the stdio branch: an SSH-launched `--stdio` server (whose argv came
* from the client via --remote-option and friends) must honor --allow-delete,
* --trust-sender and --client-cn exactly like the standalone listener. */
required_client_cn = opts.client_cn;
allow_delete = opts.allow_delete;
trust_sender = opts.trust_sender;
allow_unauthenticated = opts.allow_unauthenticated;
server_no_super = opts.no_super;
/* --stdio rejects --allow-super at parse time; force it off here as well so
* this process-global policy cannot be re-enabled by a future caller. */
server_allow_super = opts.allow_super && !opts.stdio_mode;
server_iconv_spec = opts.iconv_spec;
install_cleanup_handler(SIGINT);
install_cleanup_handler(SIGTERM);
/* Server-owned socket deadline floor: the client default --timeout=0 would
* otherwise leave accepted sockets without SO_RCVTIMEO/SO_SNDTIMEO and let a
* silent peer hold a connection (and its process slot) forever. */
tcp_set_timeouts(SERVER_IO_TIMEOUT_SEC, SERVER_IO_TIMEOUT_SEC);
if (opts.stdio_mode) {
/* SSH authenticates the stdio transport outside of FastSync. */
allow_unauthenticated = true;
if (!configure_authorization(opts.destination_root)) {
char* escaped = output_escape(opts.destination_root, false);
fprintf(stderr, "Error: invalid destination root '%s'\n",
escaped ? escaped : "<allocation failed>");
free(escaped);
server_cli_options_free(&opts);
return 1;
}
io_set_fds(STDIN_FILENO, STDOUT_FILENO);
/* handler() does not own the stdio fds: it never closes its descriptor
* argument, so STDIN/STDOUT stay open for this (single-shot) SSH session
* and are released by process exit. */
handler(STDIN_FILENO);
release_authorization();
server_cli_options_free(&opts);
return 0;
}
int port = opts.port;
int bind_family = opts.bind_family;
const char* bind_address = opts.bind_address;
if (opts.daemon_mode) {
const char* config_path = opts.config_path ? opts.config_path : default_daemon_config_path();
g_daemon_conf = daemon_conf_load(config_path, cli_err, sizeof(cli_err));
if (!g_daemon_conf) {
server_cli_options_free(&opts);
fprintf(stderr, "Error: %s\n", cli_err);
return 1;
}
for (int i = 0; i < opts.dparam_count; i++) {
if (daemon_conf_apply_dparam(g_daemon_conf, opts.dparams[i], cli_err, sizeof(cli_err)) != 0) {
fprintf(stderr, "Error: --dparam: %s\n", cli_err);
exit_code = 1;
goto out;
}
}
/* Effective port: -p (highest) > --dparam port > config port (default 873). */
if (!opts.port_set)
port = g_daemon_conf->global.port;
if (!bind_address)
bind_address = g_daemon_conf->global.address;
if (g_daemon_conf->module_count == 0)
log_message(LOG_LEVEL_WARNING,
"daemon config has no modules; every connection will be refused");
/* Surface the operator's client-chosen-ownership opt-in prominently: an
opted-in module lets its clients request arbitrary owner ids inside that
module root. */
for (int i = 0; i < g_daemon_conf->module_count; i++) {
if (g_daemon_conf->modules[i].client_owner)
log_message(LOG_LEVEL_WARNING,
"daemon module '%s' allows client-chosen ownership and super-user device "
"activities (`client owner = yes`); clients may request arbitrary owner ids "
"and device nodes within that module root -- pair it with `auth users` "
"unless the module is intentionally open to the network",
g_daemon_conf->modules[i].name);
if (g_daemon_conf->modules[i].max_connections > 0)
log_message(LOG_LEVEL_INFO,
"daemon module '%s': per-module 'max connections' cap = %d (enforced "
"across all connection children)",
g_daemon_conf->modules[i].name, g_daemon_conf->modules[i].max_connections);
}
/* Daemon credential store (Wave B). --password-file and --early-input
* feed the same store, loaded BEFORE the listener forks so every
* connection child shares one read-only store. Fail closed at startup: a
* module that declares `auth users` without a store (or with an empty
* store) refuses to start rather than serving a module whose credentials
* can never be verified. */
g_credentials =
credentials_load(opts.password_file, opts.early_input_file, cli_err, sizeof(cli_err));
if (!g_credentials) {
server_cli_options_free(&opts);
fprintf(stderr, "Error: %s\n", cli_err);
return 1;
}
bool credential_source_given = opts.password_file != NULL || opts.early_input_file != NULL;
for (int i = 0; i < g_daemon_conf->module_count; i++) {
const DaemonModule* module = &g_daemon_conf->modules[i];
if (module->auth_user_count == 0)
continue;
if (!credential_source_given) {
fprintf(stderr,
"Error: module '%s' declares 'auth users' but no credential store was given "
"(--password-file or --early-input); refusing to start (fail closed)\n",
module->name);
server_cli_options_free(&opts);
return 1;
}
if (credentials_store_size(g_credentials) == 0) {
fprintf(stderr,
"Error: module '%s' declares 'auth users' but the credential store is empty; "
"refusing to start (fail closed)\n",
module->name);
server_cli_options_free(&opts);
return 1;
}
for (int j = 0; j < module->auth_user_count; j++) {
if (!credentials_store_has(g_credentials, module->auth_users[j]))
log_message(LOG_LEVEL_WARNING,
"daemon module '%s': auth user '%s' has no credential store entry; that "
"user can never authenticate",
module->name, module->auth_users[j]);
}
}
/* Shared cross-process registry for the per-module / per-source caps and
* the auth lockout. Created HERE in the parent before any accept-loop
* fork; every connection child inherits the mapping. A failure degrades to
* "registry disabled" (the global cap and host ACLs still apply) rather
* than refusing to start. */
g_daemon_limits = daemon_limits_create((int)g_daemon_conf->global.max_connections,
g_daemon_conf->module_count,
g_daemon_conf->global.max_connections_per_host,
g_daemon_conf->global.auth_lockout_threshold,
g_daemon_conf->global.auth_lockout_duration_sec);
if (!g_daemon_limits)
log_message(LOG_LEVEL_WARNING,
"daemon: could not allocate the shared connection registry; per-module / "
"per-host caps and the cross-process auth lockout are disabled (the global "
"'max connections' cap and host ACLs still apply)");
} else {
if (!configure_authorization(opts.destination_root)) {
char* escaped = output_escape(opts.destination_root, false);
fprintf(stderr, "Error: invalid destination root '%s'\n",
escaped ? escaped : "<allocation failed>");
free(escaped);
server_cli_options_free(&opts);
return 1;
}
}
ServerBindOptions bind_opts;
bind_opts.bind_address = bind_address;
bind_opts.family = bind_family;
g_server = server_create_ex(port, &bind_opts);
if (!g_server) {
log_message(LOG_LEVEL_ERROR, "Failed to create server");
release_authorization();
exit_code = 1;
goto out;
}
if (g_daemon_conf)
server_set_max_connections(g_server, (unsigned int)g_daemon_conf->global.max_connections);
if (g_daemon_limits)
server_set_limit_registry(g_server, g_daemon_limits);
if (opts.use_tls) {
if (!opts.tls_cert || !opts.tls_key || !opts.tls_ca || !opts.client_cn) {
fprintf(stderr, "Error: --tls requires --cert, --key, --ca, and --client-cn\n");
server_delete(&g_server);
release_authorization();
exit_code = 1;
goto out;
}
tls_global_init();
if (!server_create_tls(g_server, opts.tls_cert, opts.tls_key, opts.tls_ca)) {
log_message(LOG_LEVEL_ERROR, "Failed to set up TLS");
server_delete(&g_server);
release_authorization();
exit_code = 1;
goto out;
}
}
/* Detach after the listening socket (and TLS context) exist so the
* background daemon inherits a fully-bound listener. --no-detach runs in
* the foreground, which is how tests drive the daemon. */
if (opts.daemon_mode && !opts.no_detach) {
if (!daemonize()) {
log_message(LOG_LEVEL_ERROR, "Failed to daemonize");
server_delete(&g_server);
release_authorization();
exit_code = 1;
goto out;
}
}
if (opts.use_tls)
server_listen_tls(g_server, handler);
else
server_listen(g_server, handler);
server_delete(&g_server);
release_authorization();
out:
daemon_limits_destroy(g_daemon_limits);
g_daemon_limits = NULL;
daemon_conf_free(g_daemon_conf);
g_daemon_conf = NULL;
credentials_free(g_credentials);
g_credentials = NULL;
server_cli_options_free(&opts);
return exit_code;
configure_server_process(&opts);
if (opts.stdio_mode)
return run_stdio_server(&opts);
return run_daemon_server(&opts);
}
#endif
+377
View File
@@ -3,6 +3,7 @@
#include "log.h"
#include "protocol.h"
#include "utils.h"
#include <errno.h>
#include <limits.h>
#include <lz4.h>
#include <stdatomic.h>
@@ -716,3 +717,379 @@ Data* data_decompress_limited(Data* compressed_data, size_t maximum_size) {
Data* data_decompress(Data* compressed_data) {
return data_decompress_limited(compressed_data, MAX_DECOMPRESSED_SIZE);
}
/* ---- streaming decompression ---- */
#define STREAM_DECOMPRESS_OUT_CHUNK (256 * 1024)
struct CompressionStreamDecompressor {
CompressionAlgo algo;
unsigned long long expected_out;
unsigned long long total;
int out_fd;
unsigned char* out_buf;
ZSTD_DCtx* dctx;
z_stream zs;
bool zs_initialized;
bool failed;
};
static bool stream_write_all(int fd, const void* data, size_t size) {
const unsigned char* p = data;
size_t done = 0;
while (done < size) {
ssize_t n = write(fd, p + done, size - done);
if (n < 0 && errno == EINTR)
continue;
if (n <= 0)
return false;
done += (size_t)n;
}
return true;
}
CompressionStreamDecompressor*
compression_stream_decompressor_create(CompressionAlgo algo, unsigned long long expected_out) {
CompressionStreamDecompressor* d = calloc(1, sizeof(*d));
if (!d)
return NULL;
d->algo = algo;
d->expected_out = expected_out;
d->out_fd = -1;
d->out_buf = malloc(STREAM_DECOMPRESS_OUT_CHUNK);
if (!d->out_buf) {
free(d);
return NULL;
}
if (algo == COMPRESSION_ALGO_ZSTD) {
d->dctx = ZSTD_createDCtx();
if (!d->dctx) {
free(d->out_buf);
free(d);
return NULL;
}
} else if (algo == COMPRESSION_ALGO_ZLIB || algo == COMPRESSION_ALGO_ZLIBX) {
if (inflateInit(&d->zs) != Z_OK) {
free(d->out_buf);
free(d);
return NULL;
}
d->zs_initialized = true;
} else if (algo != COMPRESSION_ALGO_NONE) {
/* lz4's block format cannot be decompressed incrementally. */
free(d->out_buf);
free(d);
return NULL;
}
return d;
}
static bool stream_emit(CompressionStreamDecompressor* d, const void* buf, size_t len) {
if (len == 0)
return true;
if (d->expected_out != 0 && (d->total > d->expected_out || len > d->expected_out - d->total)) {
d->failed = true;
return false;
}
if (!stream_write_all(d->out_fd, buf, len)) {
d->failed = true;
return false;
}
d->total += len;
return true;
}
static bool stream_feed_none(CompressionStreamDecompressor* d, const void* in, size_t in_len,
bool* done) {
if (!stream_emit(d, in, in_len))
return false;
/* NONE has no end marker; the caller knows the frame length. */
*done = true;
return true;
}
static bool stream_feed_zstd(CompressionStreamDecompressor* d, const void* in, size_t in_len,
bool* done) {
ZSTD_inBuffer input = {in, in_len, 0};
while (input.pos < input.size) {
ZSTD_outBuffer output = {d->out_buf, STREAM_DECOMPRESS_OUT_CHUNK, 0};
size_t ret = ZSTD_decompressStream(d->dctx, &output, &input);
if (ZSTD_isError(ret)) {
d->failed = true;
return false;
}
if (!stream_emit(d, d->out_buf, output.pos))
return false;
if (ret == 0) {
*done = true;
/* Trailing bytes after a complete frame are malformed; stop consuming. */
if (input.pos < input.size) {
d->failed = true;
return false;
}
return true;
}
}
return true;
}
static bool stream_feed_zlib(CompressionStreamDecompressor* d, const void* in, size_t in_len,
bool* done) {
d->zs.next_in = (Bytef*)in;
d->zs.avail_in = (uInt)in_len;
while (d->zs.avail_in > 0) {
d->zs.next_out = d->out_buf;
d->zs.avail_out = STREAM_DECOMPRESS_OUT_CHUNK;
int rc = inflate(&d->zs, Z_NO_FLUSH);
if (rc != Z_OK && rc != Z_STREAM_END && rc != Z_BUF_ERROR) {
d->failed = true;
return false;
}
size_t produced = STREAM_DECOMPRESS_OUT_CHUNK - d->zs.avail_out;
if (!stream_emit(d, d->out_buf, produced))
return false;
if (rc == Z_STREAM_END) {
*done = true;
return d->zs.avail_in == 0;
}
if (rc == Z_BUF_ERROR && produced == 0) {
/* Need more input. */
break;
}
}
return true;
}
bool compression_stream_decompressor_feed(CompressionStreamDecompressor* d, const void* in,
size_t in_len, int out_fd, bool* done) {
if (!d || d->failed)
return false;
d->out_fd = out_fd;
if (done)
*done = false;
switch (d->algo) {
case COMPRESSION_ALGO_NONE:
return stream_feed_none(d, in, in_len, done);
case COMPRESSION_ALGO_ZSTD:
return stream_feed_zstd(d, in, in_len, done);
case COMPRESSION_ALGO_ZLIB:
case COMPRESSION_ALGO_ZLIBX:
return stream_feed_zlib(d, in, in_len, done);
case COMPRESSION_ALGO_LZ4:
break;
}
d->failed = true;
return false;
}
unsigned long long compression_stream_decompressor_total(const CompressionStreamDecompressor* d) {
return d ? d->total : 0;
}
void compression_stream_decompressor_destroy(CompressionStreamDecompressor* d) {
if (!d)
return;
if (d->dctx)
ZSTD_freeDCtx(d->dctx);
if (d->zs_initialized)
inflateEnd(&d->zs);
free(d->out_buf);
free(d);
}
/* ---- streaming compression ---- */
struct CompressionStreamCompressor {
CompressionAlgo algo;
int level;
ZSTD_CCtx* cctx;
z_stream zs;
bool zs_initialized;
unsigned char* out_buf;
bool failed;
};
bool compression_stream_compress_supported(CompressionAlgo algo) {
return algo == COMPRESSION_ALGO_ZSTD || algo == COMPRESSION_ALGO_ZLIB ||
algo == COMPRESSION_ALGO_ZLIBX;
}
CompressionStreamCompressor* compression_stream_compressor_create(CompressionAlgo algo, int level,
int threads) {
(void)threads;
if (!compression_algo_valid((int)algo) || algo == COMPRESSION_ALGO_LZ4)
return NULL;
CompressionStreamCompressor* c = calloc(1, sizeof(*c));
if (!c)
return NULL;
c->algo = algo;
c->level = level;
c->out_buf = malloc(STREAM_DECOMPRESS_OUT_CHUNK);
if (!c->out_buf) {
free(c);
return NULL;
}
if (algo == COMPRESSION_ALGO_ZSTD) {
c->cctx = ZSTD_createCCtx();
if (!c->cctx) {
free(c->out_buf);
free(c);
return NULL;
}
} else if (algo == COMPRESSION_ALGO_ZLIB || algo == COMPRESSION_ALGO_ZLIBX) {
if (deflateInit(&c->zs, level < 1 ? Z_DEFAULT_COMPRESSION : level) != Z_OK) {
free(c->out_buf);
free(c);
return NULL;
}
c->zs_initialized = true;
}
return c;
}
bool compression_stream_compressor_begin(CompressionStreamCompressor* c,
unsigned long long raw_size, int out_fd) {
if (!c || c->failed)
return false;
unsigned char hdr[1 + 4];
size_t hdr_len = 1;
hdr[0] = (unsigned char)c->algo;
if (c->algo == COMPRESSION_ALGO_ZLIB || c->algo == COMPRESSION_ALGO_ZLIBX) {
uint32_t size32 = raw_size > UINT32_MAX ? UINT32_MAX : (uint32_t)raw_size;
for (int i = 0; i < 4; i++)
hdr[1 + i] = (uint8_t)((size32 >> (8 * i)) & 0xff);
hdr_len = 5;
}
if (c->algo == COMPRESSION_ALGO_ZSTD) {
/* Pledge the source size and force the frame content-size field so the
receiver can decide whether to stream from the frame header alone. */
if (ZSTD_isError(ZSTD_CCtx_setPledgedSrcSize(c->cctx, raw_size)) ||
ZSTD_isError(ZSTD_CCtx_setParameter(c->cctx, ZSTD_c_compressionLevel, c->level)) ||
ZSTD_isError(ZSTD_CCtx_setParameter(c->cctx, ZSTD_c_contentSizeFlag, 1))) {
c->failed = true;
return false;
}
if (ZSTD_isError(ZSTD_CCtx_setParameter(c->cctx, ZSTD_c_checksumFlag, 0))) {
c->failed = true;
return false;
}
}
if (!stream_write_all(out_fd, hdr, hdr_len)) {
c->failed = true;
return false;
}
return true;
}
static bool stream_compress_zlib(CompressionStreamCompressor* c, const void* in, size_t in_len,
int out_fd, int flush) {
c->zs.next_in = (Bytef*)in;
c->zs.avail_in = (uInt)in_len;
do {
c->zs.next_out = c->out_buf;
c->zs.avail_out = STREAM_DECOMPRESS_OUT_CHUNK;
int rc = deflate(&c->zs, flush);
if (rc != Z_OK && rc != Z_STREAM_END && rc != Z_BUF_ERROR) {
c->failed = true;
return false;
}
size_t produced = STREAM_DECOMPRESS_OUT_CHUNK - c->zs.avail_out;
if (!stream_write_all(out_fd, c->out_buf, produced)) {
c->failed = true;
return false;
}
if (rc == Z_STREAM_END)
return true;
if (rc == Z_BUF_ERROR && produced == 0)
break;
} while (c->zs.avail_in > 0 || flush == Z_FINISH);
return true;
}
bool compression_stream_compressor_feed(CompressionStreamCompressor* c, const void* in,
size_t in_len, int out_fd) {
if (!c || c->failed)
return false;
if (c->algo == COMPRESSION_ALGO_NONE)
return stream_write_all(out_fd, in, in_len);
if (c->algo == COMPRESSION_ALGO_ZSTD) {
ZSTD_inBuffer input = {in, in_len, 0};
while (input.pos < input.size) {
ZSTD_outBuffer output = {c->out_buf, STREAM_DECOMPRESS_OUT_CHUNK, 0};
size_t ret = ZSTD_compressStream2(c->cctx, &output, &input, ZSTD_e_continue);
if (ZSTD_isError(ret)) {
c->failed = true;
return false;
}
if (!stream_write_all(out_fd, c->out_buf, output.pos)) {
c->failed = true;
return false;
}
if (output.pos == 0 && input.pos < input.size)
break; /* avoid spinning; zstd buffers the rest internally */
}
return true;
}
return stream_compress_zlib(c, in, in_len, out_fd, Z_NO_FLUSH);
}
bool compression_stream_compressor_finish(CompressionStreamCompressor* c, int out_fd) {
if (!c || c->failed)
return false;
if (c->algo == COMPRESSION_ALGO_NONE)
return true;
if (c->algo == COMPRESSION_ALGO_ZSTD) {
size_t ret;
do {
ZSTD_inBuffer input = {NULL, 0, 0};
ZSTD_outBuffer output = {c->out_buf, STREAM_DECOMPRESS_OUT_CHUNK, 0};
ret = ZSTD_compressStream2(c->cctx, &output, &input, ZSTD_e_end);
if (ZSTD_isError(ret)) {
c->failed = true;
return false;
}
if (!stream_write_all(out_fd, c->out_buf, output.pos)) {
c->failed = true;
return false;
}
} while (ret > 0);
return true;
}
return stream_compress_zlib(c, NULL, 0, out_fd, Z_FINISH);
}
void compression_stream_compressor_destroy(CompressionStreamCompressor* c) {
if (!c)
return;
if (c->cctx)
ZSTD_freeCCtx(c->cctx);
if (c->zs_initialized)
deflateEnd(&c->zs);
free(c->out_buf);
free(c);
}
unsigned long long compression_peek_frame_content_size(const void* buf, size_t len) {
if (!buf || len < 1)
return 0;
const uint8_t* p = buf;
uint8_t codec = p[0];
if (!compression_algo_valid(codec))
return 0;
if (codec == (uint8_t)COMPRESSION_ALGO_NONE)
return len - 1;
if (codec == (uint8_t)COMPRESSION_ALGO_ZSTD) {
if (len < 2)
return 0;
unsigned long long size = ZSTD_getFrameContentSize(p + 1, len - 1);
if (size == ZSTD_CONTENTSIZE_ERROR || size == ZSTD_CONTENTSIZE_UNKNOWN)
return 0;
return size;
}
if (len < 1 + LZ4_SIZE_PREFIX_LEN)
return 0;
uint32_t raw = 0;
for (int i = 0; i < LZ4_SIZE_PREFIX_LEN; i++)
raw |= (uint32_t)p[1 + i] << (8 * i);
return raw;
}
+48
View File
@@ -81,6 +81,54 @@ Data* data_compress_with_threads(Data* data_to_compress, int compression_level,
Data* data_decompress(Data* compressed_data);
bool compression_should_skip_with_suffixes(const char* path, char* const* suffixes, int count);
/* Streaming decompression for a payload too large to hold in memory. The
* caller consumes the frame's leading codec byte (and, for lz4/zlib/zlibx, the
* 4-byte little-endian raw-size prefix) and then feeds the remaining frame
* bytes in bounded chunks; decompressed output is written straight to `out_fd`
* so neither the compressed nor the decompressed image is ever materialized.
* Only zstd (the default), zlib/zlibx and none support streaming; lz4's block
* format is one-shot, so its stream decompressor reports failure and the caller
* falls back (the whole-buffer path keeps its existing bound). */
typedef struct CompressionStreamDecompressor CompressionStreamDecompressor;
CompressionStreamDecompressor*
compression_stream_decompressor_create(CompressionAlgo algo, unsigned long long expected_out);
/* Feed one chunk. Returns false on a malformed frame, an I/O error, or when the
* total output would exceed `expected_out` (when non-zero). *done is set once
* the frame end has been reached. */
bool compression_stream_decompressor_feed(CompressionStreamDecompressor* d, const void* in,
size_t in_len, int out_fd, bool* done);
unsigned long long compression_stream_decompressor_total(const CompressionStreamDecompressor* d);
void compression_stream_decompressor_destroy(CompressionStreamDecompressor* d);
/* Peek the logical (decompressed) size from the leading bytes of a compressed
* frame (codec byte + header), returning 0 when it cannot be determined from
* the supplied prefix. Used to decide whether a frame must take the streaming
* path before its body is read. */
unsigned long long compression_peek_frame_content_size(const void* buf, size_t len);
/* Streaming compression (sender side). Compresses a source in bounded chunks
* into `out_fd` as one self-describing frame (codec byte, the lz4/zlib raw-size
* prefix, then the codec stream), so a whole file can be compressed without
* materializing it in memory. zstd/zlib/zlibx/none are supported; lz4's block
* format is one-shot, so its create() returns NULL and the caller keeps the
* buffered path. `raw_size` is the known source length (used for the zlib
* prefix and, for zstd, the frame content-size field). */
typedef struct CompressionStreamCompressor CompressionStreamCompressor;
/* True when `algo` can be stream-compressed (zstd/zlib/zlibx; lz4's block format
* is one-shot). Used by the sender to decide whether an over-threshold source
* may stay unloaded. */
bool compression_stream_compress_supported(CompressionAlgo algo);
CompressionStreamCompressor* compression_stream_compressor_create(CompressionAlgo algo, int level,
int threads);
bool compression_stream_compressor_begin(CompressionStreamCompressor* c,
unsigned long long raw_size, int out_fd);
bool compression_stream_compressor_feed(CompressionStreamCompressor* c, const void* in,
size_t in_len, int out_fd);
bool compression_stream_compressor_finish(CompressionStreamCompressor* c, int out_fd);
void compression_stream_compressor_destroy(CompressionStreamCompressor* c);
/* Release the calling thread's cached zstd contexts (compressor, decompressor
* and scratch buffer). The cache is thread-local and is also released
* automatically when a worker thread exits (via a C11 tss destructor) and for
+15 -2
View File
@@ -83,7 +83,7 @@ typedef struct {
typedef enum SuperMode { SUPER_MODE_AUTO = 0, SUPER_MODE_ON = 1, SUPER_MODE_OFF = 2 } SuperMode;
/* ===========================================================================
* Config wire-field table (single source of truth for protocol 2.29.0).
* Config wire-field table (single source of truth for protocol 2.30.0).
*
* Every field below crosses the wire. The table is the ONLY place a
* serialized field is named: config.h expands CONFIG_WIRE_FIELDS() to declare
@@ -1084,7 +1084,20 @@ typedef struct Config {
* version must bump; the strict same-version handshake (config_receive rejects a
* mismatched version before parsing anything else) keeps a 2.29 client and a
* 2.28 server from ever reaching that state. */
#define PROTOCOL_VERSION "2.29.0"
/* (11) Client-message channel + partial exit (protocol 2.30.0): the
* config-frame LAYOUT is unchanged (no new config field), but the frame stream
* gains two statuses. STATUS_CLIENT_MSG (client->server) carries a bounded,
* length-prefixed diagnostic string so a client running with --stderr=client
* (rsync's --no-msgs2stderr spelling) can forward its own diagnostics to the
* server's stderr. STATUS_PARTIAL (receiver->client) is the terminal status
* sent instead of STATUS_OK when a per-entry receiver failure (e.g. an
* unprivileged --devices mknod) did not abort the stream; the sender exits 23
* (rsync's partial transfer) and still removes successfully transferred
* --remove-source-files sources. A 2.29 peer that does not know these status
* values would reject them as an unknown status and tear the connection down,
* so the protocol version must bump; the strict same-version handshake keeps a
* 2.30 client and a 2.29 server from ever reaching that state. */
#define PROTOCOL_VERSION "2.30.0"
#define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024)
/* Upper bound on total basis-dir entries (rsync caps --link-dest at 20). */
#define MAX_BASIS_DIRS 64
+118 -36
View File
@@ -8,13 +8,49 @@
#include <errno.h>
#include <fcntl.h>
#include <libgen.h>
#include <stdatomic.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/file.h>
#include <sys/stat.h>
#include <time.h>
#include <unistd.h>
/* Process-wide counter so two staging contexts created in the same process (or
within the same clock tick) can never pick the same name. */
static unsigned long long delay_updates_next_sequence(void) {
static atomic_ullong sequence;
return atomic_fetch_add_explicit(&sequence, 1, memory_order_relaxed);
}
/* Build the per-run staging directory basename: the reserved prefix plus the
pid and an entropy token. A fixed name could collide with a genuine
destination entry; the token makes such a collision vanishingly unlikely and,
if it ever happens, prepare() refuses to touch the existing directory. */
static char* delay_updates_make_staging_name(void) {
unsigned long long entropy = 0;
int fd = open("/dev/urandom", O_RDONLY | O_CLOEXEC);
if (fd >= 0) {
ssize_t got = read(fd, &entropy, sizeof(entropy));
close(fd);
if (got != (ssize_t)sizeof(entropy))
entropy = 0;
}
if (entropy == 0)
entropy = ((unsigned long long)time(NULL) << 20) ^ ((unsigned long long)getpid() << 8) ^
delay_updates_next_sequence();
int length = snprintf(NULL, 0, DELAY_UPDATES_STAGING_DIR ".%ld.%llx", (long)getpid(), entropy);
if (length < 0)
return NULL;
char* name = malloc((size_t)length + 1);
if (!name)
return NULL;
snprintf(name, (size_t)length + 1, DELAY_UPDATES_STAGING_DIR ".%ld.%llx", (long)getpid(),
entropy);
return name;
}
DelayUpdatesContext* delay_updates_context_create(const char* root_directory) {
if (!root_directory)
return NULL;
@@ -26,8 +62,15 @@ DelayUpdatesContext* delay_updates_context_create(const char* root_directory) {
free(context);
return NULL;
}
context->staging_root = path_cat(root_directory, DELAY_UPDATES_STAGING_DIR);
context->staging_name = delay_updates_make_staging_name();
if (!context->staging_name) {
free(context->root_directory);
free(context);
return NULL;
}
context->staging_root = path_cat(root_directory, context->staging_name);
if (!context->staging_root) {
free(context->staging_name);
free(context->root_directory);
free(context);
return NULL;
@@ -39,6 +82,7 @@ DelayUpdatesContext* delay_updates_context_create(const char* root_directory) {
context->lock_fd = -1;
if (mtx_init(&context->mutex, mtx_plain) != thrd_success) {
free(context->staging_root);
free(context->staging_name);
free(context->root_directory);
free(context);
return NULL;
@@ -54,6 +98,7 @@ void delay_updates_context_destroy(DelayUpdatesContext* context) {
close(context->lock_fd);
context->lock_fd = -1;
free(context->staging_root);
free(context->staging_name);
free(context->root_directory);
for (size_t i = 0; i < context->count; i++) {
free(context->entries[i].staged_path);
@@ -125,48 +170,81 @@ bool delay_updates_prepare(DelayUpdatesContext* context) {
return false;
if (context->prepared)
return true;
int fd = file_open_private_dir(context->staging_root);
if (fd < 0) {
/* Create the per-run staging directory with O_EXCL semantics. The name is
unique to this transfer, so if the path already exists it is NOT ours:
either a genuine destination entry that happens to share the name or a
leftover from another session. Refuse rather than wipe it -- the old
fixed-name design could destroy a real destination entry. A crash
leftover is never reused (the next run picks a fresh name). */
char* leaf = NULL;
int parent_fd = file_open_secure_parent(context->staging_root, &leaf, true);
if (parent_fd < 0) {
int saved_errno = errno;
char* escaped = output_escape(context->staging_root, false);
log_message(LOG_LEVEL_ERROR, "could not create --delay-updates staging directory '%s': %s",
escaped ? escaped : "<allocation failed>", strerror(saved_errno));
free(escaped);
free(leaf);
return false;
}
/* Hold an exclusive advisory lock on the staging directory for the whole
transfer. The staging directory name is fixed, so two simultaneous
delayed transfers to the same destination root would otherwise share it
and destroy each other's staged files. The lock makes the second session
fail cleanly instead of corrupting the first. The lock is released when
the context (and its file descriptor) is destroyed. */
int fd = openat(parent_fd, leaf, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
if (fd >= 0) {
close(fd);
close(parent_fd);
char* escaped = output_escape(context->staging_root, false);
log_message(LOG_LEVEL_ERROR,
"--delay-updates staging directory '%s' already exists and is not owned by this "
"transfer; refusing to overwrite it",
escaped ? escaped : "<allocation failed>");
free(escaped);
free(leaf);
return false;
}
if (errno != ENOENT) {
int saved_errno = errno;
close(parent_fd);
char* escaped = output_escape(context->staging_root, false);
log_message(LOG_LEVEL_ERROR, "could not open --delay-updates staging directory '%s': %s",
escaped ? escaped : "<allocation failed>", strerror(saved_errno));
free(escaped);
free(leaf);
return false;
}
if (mkdirat(parent_fd, leaf, 0700) != 0) {
int saved_errno = errno;
close(parent_fd);
char* escaped = output_escape(context->staging_root, false);
log_message(LOG_LEVEL_ERROR, "could not create --delay-updates staging directory '%s': %s",
escaped ? escaped : "<allocation failed>", strerror(saved_errno));
free(escaped);
free(leaf);
return false;
}
fd = openat(parent_fd, leaf, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
close(parent_fd);
free(leaf);
if (fd < 0) {
int saved_errno = errno;
char* escaped = output_escape(context->staging_root, false);
log_message(LOG_LEVEL_ERROR, "could not open --delay-updates staging directory '%s': %s",
escaped ? escaped : "<allocation failed>", strerror(saved_errno));
free(escaped);
return false;
}
/* Keep the exclusive advisory lock as defense in depth: the unique name
already prevents two sessions from sharing a staging directory, but the
lock also catches an improbable same-name collision that raced between the
existence check above and the open. */
if (flock(fd, LOCK_EX | LOCK_NB) != 0) {
int saved_errno = errno;
close(fd);
if (saved_errno == EWOULDBLOCK || saved_errno == EAGAIN) {
char* escaped = output_escape(context->staging_root, false);
log_message(LOG_LEVEL_ERROR,
"another --delay-updates transfer to '%s' is already in progress; refusing to "
"share the staging directory",
escaped ? escaped : "<allocation failed>");
free(escaped);
} else {
log_message(LOG_LEVEL_ERROR, "could not lock --delay-updates staging directory '%s': %s",
context->staging_root, strerror(saved_errno));
}
char* escaped = output_escape(context->staging_root, false);
log_message(LOG_LEVEL_ERROR, "could not lock --delay-updates staging directory '%s': %s",
escaped ? escaped : "<allocation failed>", strerror(saved_errno));
free(escaped);
return false;
}
context->lock_fd = fd;
/* Only now, with exclusive ownership, wipe leftovers from an interrupted
earlier transfer; this can never race with a live session. */
bool ok = delay_wipe_dir_fd(fd);
if (!ok) {
log_message(LOG_LEVEL_ERROR, "could not clear stale --delay-updates staging files under '%s'",
context->staging_root);
close(context->lock_fd);
context->lock_fd = -1;
return false;
}
context->prepared = true;
return true;
}
@@ -264,12 +342,16 @@ static bool delay_publish_entry(DelayUpdatesContext* context, const Config* conf
const StagedFileEntry* entry) {
if (!delay_publish_backup(context, config, entry))
return false;
/* --force: an incoming regular file/symlink may replace a destination
DIRECTORY (possibly non-empty). The immediate-install path handles this in
file_receive; a --delay-updates run stages elsewhere and only discovers the
blocking directory here, so clear it before the rename (rsync's
"could not make way for new regular file" without --force). */
if (config && config->force_delete && file_directory_exists_secure(entry->final_path)) {
/* An incoming regular file/symlink may replace a destination DIRECTORY that
blocks it. rsync removes the blocker recursively when --delete or --force
is active (its generator's "make way" deletion), and a --delay-updates run
stages elsewhere so it only discovers the blocker here. FastSync's
immediate-install path clears it too; without --delete/--force a non-empty
blocker fails the run (rsync's "could not make way for new regular file").
use_delete is gated by the server --allow-delete policy, so a client can
never use this to bypass deletion authorization. */
if (config && (config->force_delete || config->use_delete) &&
file_directory_exists_secure(entry->final_path)) {
if (!file_remove_tree_secure(entry->final_path)) {
char* escaped = output_escape(entry->final_path, false);
log_message(LOG_LEVEL_ERROR, "could not remove destination directory blocking '%s': %s",
+6 -1
View File
@@ -24,6 +24,7 @@ typedef struct {
shared with the publish/cleanup phase that runs after the threads join. */
typedef struct DelayUpdatesContext {
char* root_directory; /* receive root the staging dir lives under */
char* staging_name; /* per-run unique staging dir basename */
char* staging_root; /* root_directory/<staging dir name> */
mtx_t mutex;
StagedFileEntry* entries;
@@ -33,7 +34,11 @@ typedef struct DelayUpdatesContext {
int lock_fd; /* advisory exclusive flock held on the staging dir, or -1 */
} DelayUpdatesContext;
/* Name of the private staging subdirectory created under the receive root. */
/* Reserved prefix for the private staging subdirectory created under the
receive root. The actual directory name is per-run unique (the prefix plus a
pid/entropy token) so it can never clobber a genuine destination entry that
happens to share the name; the bare prefix is still what a --backup-dir must
not collide with. */
#define DELAY_UPDATES_STAGING_DIR ".fastsync-stage"
/* True when `dir` (ignoring a trailing "/") is the reserved staging directory
+313 -192
View File
@@ -32,6 +32,49 @@ static bool keep_is_file(const PathIndex* index, const char* rel_path) {
return path_index_contains(index, rel_path);
}
/* rsync's receiver-side verdict for one candidate extra: the per-directory
* chain first (deepest directory before ancestors), then the command-line base
* rules. Either rule set may be absent. */
FilterAction delete_protect_verdict(const DeleteProtectRules* protect, const char* rel_path,
const char* leaf, bool is_dir) {
if (!protect)
return FILTER_ACTION_NONE;
/* rsync protects its own --backup files from the delete pass: a name ending
in the backup suffix is never an extra. Checked before the filter rules so
an explicit exclude cannot be bypassed (the suffix is always a shield). */
if (protect->backup_suffix && protect->backup_suffix[0] != '\0') {
size_t name_len = strlen(leaf);
size_t suffix_len = strlen(protect->backup_suffix);
if (name_len > suffix_len &&
strcmp(leaf + (name_len - suffix_len), protect->backup_suffix) == 0)
return FILTER_ACTION_PROTECT;
}
FilterAction action = filter_dir_rules_apply_side(protect->dir_rules, rel_path, leaf, is_dir);
if (action != FILTER_ACTION_NONE)
return action;
return filter_rules_apply_side(protect->base_rules, rel_path, leaf, is_dir, FILTER_SIDE_RECEIVER);
}
const char* delete_backup_suffix(const Config* config) {
if (!config || !config->backup || config->ignore_existing)
return NULL;
const char* suffix = config->suffix ? config->suffix : "~";
if (!suffix[0] || strchr(suffix, '/'))
return NULL;
return suffix;
}
/* Classify a removed entry from its st_mode for the per-type delete counters. */
DeleteEntryType delete_entry_type_of_mode(mode_t mode) {
if (S_ISDIR(mode))
return DELETE_ENTRY_DIR;
if (S_ISLNK(mode))
return DELETE_ENTRY_LINK;
if (S_ISREG(mode))
return DELETE_ENTRY_REG;
return DELETE_ENTRY_SPECIAL;
}
/* True when child_rel is, or lies below, a protected entry. A prefix "a"
therefore protects "a" and "a/b/c" but not "ab". Entries with top_level_only
set only protect DIRECT children of the receive root (at_root); nested
@@ -126,6 +169,7 @@ bool delete_dir_entries_collect(int dirfd, DeleteDirEntry** out, size_t* count,
break;
}
entries[used].is_dir = S_ISDIR(st.st_mode);
entries[used].mode = st.st_mode;
used++;
}
closedir(dir);
@@ -180,6 +224,235 @@ typedef struct {
void* observer_context; /* DELETE mode */
} DeleteWalkState;
/* The per-walk invariants threaded unchanged through every recursive descent:
the keep/synchronized-dir indexes, the destination mode and the protection
rules. Bundling them keeps the recursive helpers below to a handful of
positional arguments. */
typedef struct {
const PathIndex* keep;
const PathIndex* dirs;
DeleteWalkState* state;
const DeleteSkipEntry* skips;
int skip_count;
const DeleteProtectRules* protect;
} DeleteWalkContext;
/* Duplicate `path` with rsync's trailing-slash convention, used to report a
removed (or would-be-removed) directory. Returns NULL on allocation
failure. */
static char* with_trailing_slash(const char* path) {
size_t len = strlen(path);
char* copy = malloc(len + 2);
if (!copy)
return NULL;
memcpy(copy, path, len);
copy[len] = '/';
copy[len + 1] = '\0';
return copy;
}
/* Forward declaration: the ordered passes below recurse through the driver. */
static bool delete_walk_fd(int dirfd, const char* rel_path, const DeleteWalkContext* ctx,
bool parent_deletable, bool* all_removed);
/* Descend into the child directory `name` of `dirfd`, walking it as part of the
current operation. Returns false on a genuine open/walk failure; on success
*child_all_removed reports whether the child removed everything it held (so
the caller may rmdir it). */
static bool delete_walk_child(int dirfd, const char* name, const char* child_rel,
const DeleteWalkContext* ctx, bool deletable,
bool* child_all_removed) {
*child_all_removed = false;
int childfd = openat(dirfd, name, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
if (childfd < 0)
return errno == ENOENT;
bool ok = delete_walk_fd(childfd, child_rel, ctx, deletable, child_all_removed);
close(childfd);
return ok;
}
/* Classify every entry up front (the verdict does not depend on processing
order) so the ordered passes below can act on it. Sets shielded[]/is_extra[]
and reports through *local_survives whether anything in this directory stays
in place. Returns false on a path-construction failure. */
static bool delete_walk_classify(const char* rel_path, const DeleteDirEntry* entries, size_t count,
const DeleteWalkContext* ctx, bool deletable, bool at_root,
bool* shielded, bool* is_extra, bool* local_survives) {
bool ok = true;
for (size_t i = 0; i < count; i++) {
char* child_rel = path_cat((char*)rel_path, entries[i].name);
if (!child_rel) {
ok = false;
continue;
}
/* A --delay-updates run keeps its staging directory as a direct child of
the receive root, and basis-dir snapshots live below it too. Their
contents are not manifest entries, so descending into them would delete
every staged / basis file as an "extra". Only the staging name (a
top-level-only prefix) and the basis prefixes are protected: a nested
destination directory that happens to be called .fastsync-stage is
ordinary content. */
if (path_under_skip_prefix(child_rel, at_root, ctx->skips, ctx->skip_count)) {
shielded[i] = true;
*local_survives = true;
} else if (delete_protect_verdict(ctx->protect, child_rel, entries[i].name,
entries[i].is_dir) == FILTER_ACTION_PROTECT) {
/* A first-match protect rule shields the extra; for a directory the whole
subtree is shielded (rsync prunes an excluded directory), so do not
descend. */
shielded[i] = true;
*local_survives = true;
} else if (entries[i].is_dir) {
bool child_synced = ctx->dirs && path_index_contains(ctx->dirs, child_rel);
is_extra[i] = deletable && !child_synced && !keep_is_dir(ctx->keep, child_rel);
if (!is_extra[i])
*local_survives = true;
} else {
is_extra[i] = deletable && !keep_is_file(ctx->keep, child_rel);
if (!is_extra[i])
*local_survives = true;
}
free(child_rel);
}
return ok;
}
/* Pass 1: extraneous subdirectories, descending. Recurses into each and, when
the child removed everything it held, records or removes it and charges the
budget. */
static bool delete_walk_extra_dirs(int dirfd, const char* rel_path, const DeleteDirEntry* entries,
size_t dir_count, const DeleteWalkContext* ctx, bool deletable,
const bool* is_extra, bool* local_survives) {
bool ok = true;
for (size_t i = 0; i < dir_count; i++) {
if (!is_extra[i])
continue;
char* child_rel = path_cat((char*)rel_path, entries[i].name);
if (!child_rel) {
ok = false;
continue;
}
bool child_all_removed = false;
if (!delete_walk_child(dirfd, entries[i].name, child_rel, ctx, deletable, &child_all_removed))
ok = false;
if (child_all_removed && deletable) {
if (ctx->state->mode == DELETE_WALK_MODE_LIST) {
/* Record the directory with rsync's trailing slash. */
char* copy = with_trailing_slash(child_rel);
if (!copy) {
ok = false;
} else if (!array_list_add(ctx->state->out, copy)) {
free(copy);
ok = false;
} else {
(*ctx->state->recorded)++;
}
} else if (ctx->state->budget->deleted >= ctx->state->budget->max_delete) {
ctx->state->budget->limit_hit = true;
ctx->state->budget->skipped++;
*local_survives = true;
} else if (unlinkat(dirfd, entries[i].name, AT_REMOVEDIR) != 0) {
/* ENOENT: already gone (fine). ENOTEMPTY/EEXIST: the directory still
holds entries the walker leaves in place (a protected excluded
prefix, a kept file the manifest protects, a symlink); rsync leaves
such a directory behind, so this is not an error. Only genuine I/O
failures abort the deletion. */
if (errno != ENOENT && errno != ENOTEMPTY && errno != EEXIST)
ok = false;
*local_survives = true;
} else {
ctx->state->budget->deleted++;
/* rsync reports a removed directory with a trailing slash. */
if (ctx->state->observer) {
char* with_slash = with_trailing_slash(child_rel);
if (with_slash) {
ctx->state->observer(ctx->state->observer_context, with_slash, DELETE_ENTRY_DIR);
free(with_slash);
} else {
ctx->state->observer(ctx->state->observer_context, child_rel, DELETE_ENTRY_DIR);
}
}
}
} else {
*local_survives = true;
}
free(child_rel);
}
return ok;
}
/* Pass 2: extraneous files, descending. */
static bool delete_walk_extra_files(int dirfd, const char* rel_path, const DeleteDirEntry* entries,
size_t dir_count, size_t count, const DeleteWalkContext* ctx,
const bool* is_extra, bool* local_survives) {
bool ok = true;
for (size_t i = dir_count; i < count; i++) {
if (!is_extra[i])
continue;
if (ctx->state->mode == DELETE_WALK_MODE_LIST) {
char* child_rel = path_cat((char*)rel_path, entries[i].name);
if (!child_rel) {
ok = false;
continue;
}
char* copy = str_dup(child_rel);
if (!copy || !array_list_add(ctx->state->out, copy)) {
free(copy);
ok = false;
} else {
(*ctx->state->recorded)++;
}
free(child_rel);
} else if (ctx->state->budget->deleted >= ctx->state->budget->max_delete) {
ctx->state->budget->limit_hit = true;
ctx->state->budget->skipped++;
*local_survives = true;
} else if (unlinkat(dirfd, entries[i].name, 0) != 0) {
if (errno != ENOENT)
ok = false;
*local_survives = true;
} else {
ctx->state->budget->deleted++;
char* child_rel = path_cat((char*)rel_path, entries[i].name);
if (child_rel) {
if (ctx->state->observer)
ctx->state->observer(ctx->state->observer_context, child_rel,
delete_entry_type_of_mode(entries[i].mode));
char* escaped_path = output_escape(child_rel, log_get_8_bit_output());
fprintf(stderr, " Deleted: %s\n", escaped_path ? escaped_path : "<allocation failed>");
free(escaped_path);
}
free(child_rel);
}
}
return ok;
}
/* Pass 3: kept subdirectories, ascending (rsync descends into these only after
the parent's own extras have been handled). */
static bool delete_walk_kept_dirs(int dirfd, const char* rel_path, const DeleteDirEntry* entries,
size_t dir_count, const DeleteWalkContext* ctx, bool deletable,
const bool* is_extra, const bool* shielded,
bool* local_survives) {
bool ok = true;
for (size_t i = dir_count; i-- > 0;) {
if (is_extra[i] || shielded[i])
continue;
char* child_rel = path_cat((char*)rel_path, entries[i].name);
if (!child_rel) {
ok = false;
continue;
}
bool child_all_removed = false;
if (!delete_walk_child(dirfd, entries[i].name, child_rel, ctx, deletable, &child_all_removed))
ok = false;
/* A kept/synchronized directory is never removed. */
*local_survives = true;
free(child_rel);
}
return ok;
}
/* Remove the extras directly inside the directory open on `dirfd` (DELETE mode)
or record the paths that WOULD be removed (LIST mode), recursing into every
child directory so kept content below a synchronized prefix is reached.
@@ -194,11 +467,8 @@ typedef struct {
descending name order, then extraneous files, then kept subdirectories in
ascending order) rather than readdir() order, so `--max-delete` leaves the
same survivors and the `--info=del`/dry-run line order matches rsync. */
static bool delete_walk_fd(int dirfd, const char* rel_path, const PathIndex* keep,
const PathIndex* dirs, DeleteWalkState* state,
const DeleteSkipEntry* skips, int skip_count,
const FilterRuleList* protect_rules, bool parent_deletable,
bool* all_removed) {
static bool delete_walk_fd(int dirfd, const char* rel_path, const DeleteWalkContext* ctx,
bool parent_deletable, bool* all_removed) {
DeleteDirEntry* entries = NULL;
size_t count = 0;
bool collect_ok = true;
@@ -217,7 +487,7 @@ static bool delete_walk_fd(int dirfd, const char* rel_path, const PathIndex* kee
/* A directory is deletable when it or ANY ancestor is synchronized; the
`parent_deletable` flag carries that down the recursion so dest-only
directories below a synchronized root are removed wholesale. */
bool deletable = parent_deletable || is_synced_dir(dirs, rel_path);
bool deletable = parent_deletable || is_synced_dir(ctx->dirs, rel_path);
bool at_root = rel_path[0] == '\0';
/* Reproduce rsync's traversal order: extraneous subdirectories in descending
@@ -230,182 +500,18 @@ static bool delete_walk_fd(int dirfd, const char* rel_path, const PathIndex* kee
while (dir_count < count && entries[dir_count].is_dir)
dir_count++;
/* Classify every entry up front (the verdict does not depend on processing
order) so the ordered passes below can act on it. */
for (size_t i = 0; i < count; i++) {
char* child_rel = path_cat((char*)rel_path, entries[i].name);
if (!child_rel) {
operation_ok = false;
continue;
}
/* A --delay-updates run keeps its staging directory as a direct child of
the receive root, and basis-dir snapshots live below it too. Their
contents are not manifest entries, so descending into them would delete
every staged / basis file as an "extra". Only the staging name (a
top-level-only prefix) and the basis prefixes are protected: a nested
destination directory that happens to be called .fastsync-stage is
ordinary content. */
if (path_under_skip_prefix(child_rel, at_root, skips, skip_count)) {
shielded[i] = true;
local_survives = true;
} else if (protect_rules &&
filter_rules_apply_side(protect_rules, child_rel, entries[i].name, entries[i].is_dir,
FILTER_SIDE_RECEIVER) == FILTER_ACTION_PROTECT) {
/* A first-match protect rule shields the extra; for a directory the whole
subtree is shielded (rsync prunes an excluded directory), so do not
descend. */
shielded[i] = true;
local_survives = true;
} else if (entries[i].is_dir) {
bool child_synced = dirs && path_index_contains(dirs, child_rel);
is_extra[i] = deletable && !child_synced && !keep_is_dir(keep, child_rel);
if (!is_extra[i])
local_survives = true;
} else {
is_extra[i] = deletable && !keep_is_file(keep, child_rel);
if (!is_extra[i])
local_survives = true;
}
free(child_rel);
}
/* Pass 1: extraneous subdirectories, descending. */
for (size_t i = 0; i < dir_count; i++) {
if (!is_extra[i])
continue;
char* child_rel = path_cat((char*)rel_path, entries[i].name);
if (!child_rel) {
operation_ok = false;
continue;
}
int childfd = openat(dirfd, entries[i].name, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
bool child_all_removed = false;
if (childfd >= 0) {
if (!delete_walk_fd(childfd, child_rel, keep, dirs, state, skips, skip_count, protect_rules,
deletable, &child_all_removed))
operation_ok = false;
close(childfd);
} else if (errno != ENOENT) {
operation_ok = false;
}
if (child_all_removed && deletable) {
if (state->mode == DELETE_WALK_MODE_LIST) {
/* Record the directory with rsync's trailing slash. */
size_t len = strlen(child_rel);
char* copy = malloc(len + 2);
if (!copy) {
operation_ok = false;
} else {
memcpy(copy, child_rel, len);
copy[len] = '/';
copy[len + 1] = '\0';
if (!array_list_add(state->out, copy)) {
free(copy);
operation_ok = false;
} else {
(*state->recorded)++;
}
}
} else if (state->budget->deleted >= state->budget->max_delete) {
state->budget->limit_hit = true;
state->budget->skipped++;
local_survives = true;
} else if (unlinkat(dirfd, entries[i].name, AT_REMOVEDIR) != 0) {
/* ENOENT: already gone (fine). ENOTEMPTY/EEXIST: the directory still
holds entries the walker leaves in place (a protected excluded
prefix, a kept file the manifest protects, a symlink); rsync leaves
such a directory behind, so this is not an error. Only genuine I/O
failures abort the deletion. */
if (errno != ENOENT && errno != ENOTEMPTY && errno != EEXIST)
operation_ok = false;
local_survives = true;
} else {
state->budget->deleted++;
/* rsync reports a removed directory with a trailing slash. */
if (state->observer) {
size_t len = strlen(child_rel);
char* with_slash = malloc(len + 2);
if (with_slash) {
memcpy(with_slash, child_rel, len);
with_slash[len] = '/';
with_slash[len + 1] = '\0';
state->observer(state->observer_context, with_slash);
free(with_slash);
} else {
state->observer(state->observer_context, child_rel);
}
}
}
} else {
local_survives = true;
}
free(child_rel);
}
/* Pass 2: extraneous files, descending. */
for (size_t i = dir_count; i < count; i++) {
if (!is_extra[i])
continue;
if (state->mode == DELETE_WALK_MODE_LIST) {
char* child_rel = path_cat((char*)rel_path, entries[i].name);
if (!child_rel) {
operation_ok = false;
continue;
}
char* copy = str_dup(child_rel);
if (!copy || !array_list_add(state->out, copy)) {
free(copy);
operation_ok = false;
} else {
(*state->recorded)++;
}
free(child_rel);
} else if (state->budget->deleted >= state->budget->max_delete) {
state->budget->limit_hit = true;
state->budget->skipped++;
local_survives = true;
} else if (unlinkat(dirfd, entries[i].name, 0) != 0) {
if (errno != ENOENT)
operation_ok = false;
local_survives = true;
} else {
state->budget->deleted++;
char* child_rel = path_cat((char*)rel_path, entries[i].name);
if (child_rel) {
if (state->observer)
state->observer(state->observer_context, child_rel);
char* escaped_path = output_escape(child_rel, log_get_8_bit_output());
fprintf(stderr, " Deleted: %s\n", escaped_path ? escaped_path : "<allocation failed>");
free(escaped_path);
}
free(child_rel);
}
}
/* Pass 3: kept subdirectories, ascending (rsync descends into these only
after the parent's own extras have been handled). */
for (size_t i = dir_count; i-- > 0;) {
if (is_extra[i] || shielded[i])
continue;
char* child_rel = path_cat((char*)rel_path, entries[i].name);
if (!child_rel) {
operation_ok = false;
continue;
}
int childfd = openat(dirfd, entries[i].name, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
bool child_all_removed = false;
if (childfd >= 0) {
if (!delete_walk_fd(childfd, child_rel, keep, dirs, state, skips, skip_count, protect_rules,
deletable, &child_all_removed))
operation_ok = false;
close(childfd);
} else if (errno != ENOENT) {
operation_ok = false;
}
/* A kept/synchronized directory is never removed. */
local_survives = true;
free(child_rel);
}
if (!delete_walk_classify(rel_path, entries, count, ctx, deletable, at_root, shielded, is_extra,
&local_survives))
operation_ok = false;
if (!delete_walk_extra_dirs(dirfd, rel_path, entries, dir_count, ctx, deletable, is_extra,
&local_survives))
operation_ok = false;
if (!delete_walk_extra_files(dirfd, rel_path, entries, dir_count, count, ctx, is_extra,
&local_survives))
operation_ok = false;
if (!delete_walk_kept_dirs(dirfd, rel_path, entries, dir_count, ctx, deletable, is_extra,
shielded, &local_survives))
operation_ok = false;
free(shielded);
free(is_extra);
@@ -430,7 +536,7 @@ static int open_destination_root(const char* dest_root) {
bool delete_extras_list(const char* dest_root, const ArrayList* manifest,
const ArrayList* synced_dirs, const DeleteSkipEntry* skips, int skip_count,
const FilterRuleList* protect_rules, ArrayList* out, size_t* count_out) {
const DeleteProtectRules* protect, ArrayList* out, size_t* count_out) {
if (count_out)
*count_out = 0;
if (!manifest || !out)
@@ -460,8 +566,13 @@ bool delete_extras_list(const char* dest_root, const ArrayList* manifest,
.recorded = &recorded,
.observer = NULL,
.observer_context = NULL};
bool ok = delete_walk_fd(rootfd, "", &keep, have_dirs ? &dirs : NULL, &state, skips, skip_count,
protect_rules, false, &all_removed);
DeleteWalkContext ctx = {.keep = &keep,
.dirs = have_dirs ? &dirs : NULL,
.state = &state,
.skips = skips,
.skip_count = skip_count,
.protect = protect};
bool ok = delete_walk_fd(rootfd, "", &ctx, false, &all_removed);
if (close(rootfd) != 0)
ok = false;
path_index_free(&keep);
@@ -475,7 +586,7 @@ bool delete_extras_list(const char* dest_root, const ArrayList* manifest,
DeleteWalkResult delete_extras_limited_observed(const char* dest_root, const ArrayList* manifest,
const ArrayList* synced_dirs, size_t max_delete,
const DeleteSkipEntry* skips, int skip_count,
const FilterRuleList* protect_rules,
const DeleteProtectRules* protect,
size_t* deleted_out, size_t* skipped_out,
DeletePathObserver observer,
void* observer_context) {
@@ -513,8 +624,13 @@ DeleteWalkResult delete_extras_limited_observed(const char* dest_root, const Arr
.recorded = NULL,
.observer = observer,
.observer_context = observer_context};
bool ok = delete_walk_fd(rootfd, "", &keep, have_dirs ? &dirs : NULL, &state, skips, skip_count,
protect_rules, false, &all_removed);
DeleteWalkContext ctx = {.keep = &keep,
.dirs = have_dirs ? &dirs : NULL,
.state = &state,
.skips = skips,
.skip_count = skip_count,
.protect = protect};
bool ok = delete_walk_fd(rootfd, "", &ctx, false, &all_removed);
if (close(rootfd) != 0)
ok = false;
path_index_free(&keep);
@@ -532,11 +648,10 @@ DeleteWalkResult delete_extras_limited_observed(const char* dest_root, const Arr
DeleteWalkResult delete_extras_limited(const char* dest_root, const ArrayList* manifest,
const ArrayList* synced_dirs, size_t max_delete,
const DeleteSkipEntry* skips, int skip_count,
const FilterRuleList* protect_rules, size_t* deleted_out,
const DeleteProtectRules* protect, size_t* deleted_out,
size_t* skipped_out) {
return delete_extras_limited_observed(dest_root, manifest, synced_dirs, max_delete, skips,
skip_count, protect_rules, deleted_out, skipped_out, NULL,
NULL);
skip_count, protect, deleted_out, skipped_out, NULL, NULL);
}
bool delete_extras(const char* dest_root, const ArrayList* manifest) {
@@ -607,7 +722,13 @@ bool delete_skips_build(const Config* config, const ArrayList* protected_paths,
}
int idx = 0;
if (config->delay_updates) {
out->entries[idx].prefix = DELAY_UPDATES_STAGING_DIR;
/* Protect this transfer's actual (per-run unique) staging directory. The
runtime name is only known to the receiver-side context; fall back to the
reserved prefix for a context that was never created (e.g. a dry run). */
const char* staging_name = (config->delay_context && config->delay_context->staging_name)
? config->delay_context->staging_name
: DELAY_UPDATES_STAGING_DIR;
out->entries[idx].prefix = staging_name;
out->entries[idx].top_level_only = true;
idx++;
}
+54 -7
View File
@@ -3,8 +3,10 @@
#include "array_list.h"
#include "config.h"
#include "filter.h"
#include <stdbool.h>
#include <stddef.h>
#include <sys/stat.h>
/* Delete engine.
*
@@ -28,6 +30,34 @@ typedef enum {
DELETE_WALK_ERROR
} DeleteWalkResult;
/* Receiver-side delete-protection rules for one walk. `base_rules` is the
* command-line rule set the config frame carried (owner "" rules); `dir_rules`
* is the received per-directory rule set (rules carrying their owner directory
* and no-inherit flag). Either may be NULL. */
typedef struct {
const FilterRuleList* base_rules;
const FilterRuleList* dir_rules;
/* When non-NULL and non-empty, a destination entry whose name ends with this
suffix is protected from deletion. rsync never treats a --backup file as
an extra, so a backup created at --delay-updates publication (or a
pre-existing one) survives the delete-after pass. */
const char* backup_suffix;
} DeleteProtectRules;
/* rsync's first-match-wins receiver verdict for one candidate extra: the
* per-directory chain is evaluated first (the containing directory's rules,
* then each ancestor's, then the receive root's), then the base rules. Returns
* FILTER_ACTION_PROTECT when the entry is shielded by a receiver-side exclude,
* FILTER_ACTION_RISK when an include explicitly leaves it at risk, or
* FILTER_ACTION_NONE when no rule matched. */
FilterAction delete_protect_verdict(const DeleteProtectRules* protect, const char* rel_path,
const char* leaf, bool is_dir);
/* The backup suffix the delete walker must shield from deletion, or NULL when
--backup is inactive or the configured suffix is unusable (empty, or holding
a path separator). Matches the suffix file_save uses for backups. */
const char* delete_backup_suffix(const Config* config);
/* One protected entry for the delete walker. When top_level_only is true the
prefix is skipped only as a DIRECT child of dest_root (the --delay-updates
staging directory, which must not hide genuine extras inside a nested
@@ -66,6 +96,9 @@ bool path_under_skip_prefix(const char* child_rel, bool at_root, const DeleteSki
typedef struct {
char* name;
bool is_dir;
/* The entry's full st_mode from the AT_SYMLINK_NOFOLLOW stat, so a delete
observer can classify a removed non-directory as reg/link/special. */
mode_t mode;
} DeleteDirEntry;
/* Collect the entries of the directory open on `dirfd` (excluding "." and ".."),
stat'ing each with AT_SYMLINK_NOFOLLOW. On success *out is a malloc'd array of
@@ -96,24 +129,38 @@ int delete_dir_entry_cmp_asc(const void* a, const void* b);
DeleteWalkResult delete_extras_limited(const char* dest_root, const ArrayList* manifest,
const ArrayList* synced_dirs, size_t max_delete,
const DeleteSkipEntry* skips, int skip_count,
const FilterRuleList* protect_rules, size_t* deleted_out,
const DeleteProtectRules* protect, size_t* deleted_out,
size_t* skipped_out);
/* Entry kind of a removed path, reported to the delete observer so the receiver
can build rsync's `--stats` `Number of deleted files` per-type breakdown. The
four categories are a strict partition of every removed entry. */
typedef enum {
DELETE_ENTRY_REG = 0,
DELETE_ENTRY_DIR,
DELETE_ENTRY_LINK,
DELETE_ENTRY_SPECIAL
} DeleteEntryType;
/* Optional per-deletion observer: called for each destination-relative path
actually removed (a file, symlink, or directory), in removal order, so the
receiver can stream rsync's `--info=del`/`--info=remove` lines. */
typedef void (*DeletePathObserver)(void* context, const char* rel_path);
actually removed (a file, symlink, or directory) with its entry kind, in
removal order, so the receiver can stream rsync's `--info=del`/`--info=remove`
lines and tally the per-type `--stats` counters. */
typedef void (*DeletePathObserver)(void* context, const char* rel_path, DeleteEntryType type);
/* Classify a removed entry from its st_mode for the per-type delete counters. */
DeleteEntryType delete_entry_type_of_mode(mode_t mode);
/* `delete_extras_limited_observed` is delete_extras_limited with an optional
* observer; the observer is invoked only for entries truly removed. When
* `protect_rules` is non-NULL its receiver-side verdict is evaluated for every
* `protect` is non-NULL its receiver-side verdict is evaluated for every
* candidate extra: a first-match PROTECT leaves the entry (and, for a
* directory, its whole subtree) in place, while RISK/NONE fall through to the
* ordinary skip-prefix/keep-set logic. */
DeleteWalkResult delete_extras_limited_observed(const char* dest_root, const ArrayList* manifest,
const ArrayList* synced_dirs, size_t max_delete,
const DeleteSkipEntry* skips, int skip_count,
const FilterRuleList* protect_rules,
const DeleteProtectRules* protect,
size_t* deleted_out, size_t* skipped_out,
DeletePathObserver observer,
void* observer_context);
@@ -124,7 +171,7 @@ DeleteWalkResult delete_extras_limited_observed(const char* dest_root, const Arr
strings appended to `out` and receives their count in *count_out. */
bool delete_extras_list(const char* dest_root, const ArrayList* manifest,
const ArrayList* synced_dirs, const DeleteSkipEntry* skips, int skip_count,
const FilterRuleList* protect_rules, ArrayList* out, size_t* count_out);
const DeleteProtectRules* protect, ArrayList* out, size_t* count_out);
bool delete_extras(const char* dest_root, const ArrayList* manifest);
/* Build the delete walk's skip-prefix set from the config's --delay-updates
+181 -139
View File
@@ -19,6 +19,7 @@
#include "data.h"
#include "delay_updates.h"
#include "delete_commit.h"
#include "delete_plan.h"
#include "delta.h"
#include "file.h"
#include "format.h"
@@ -102,6 +103,13 @@ DeleteManifest* receive_manifest_entries(int fd) {
delete_manifest_free(manifest);
return NULL;
}
/* Per-directory filter rules (protocol 2.30.0) follow the manifest sections
* with their own bounded self-describing format. */
if (!delete_filter_dir_rules_receive(fd, &manifest->per_dir_rules)) {
delete_manifest_free(manifest);
send_status(fd, STATUS_ERROR);
return NULL;
}
return manifest;
}
@@ -112,6 +120,7 @@ void delete_manifest_free(DeleteManifest* manifest) {
array_list_delete(manifest->protected);
array_list_delete(manifest->missing);
array_list_delete(manifest->dirs);
filter_rule_list_free(manifest->per_dir_rules);
free(manifest);
}
@@ -160,9 +169,12 @@ static bool delete_extras_budgeted_observed(const Config* config, const DeleteMa
remaining = budget->max_delete - budget->deleted;
size_t deleted = 0;
size_t skipped = 0;
DeleteProtectRules protect = {.base_rules = config->protect_rules,
.dir_rules = manifest->per_dir_rules,
.backup_suffix = delete_backup_suffix(config)};
DeleteWalkResult result = delete_extras_limited_observed(
config->receive_root_directory, manifest->keeps, manifest->dirs, remaining, skips.entries,
skips.count, config->protect_rules, &deleted, &skipped, observer, observer_context);
skips.count, &protect, &deleted, &skipped, observer, observer_context);
delete_skips_free(&skips);
budget->deleted += deleted;
budget->skipped += skipped;
@@ -191,13 +203,13 @@ typedef struct {
const char* prefix;
} PrefixedDeleteObserver;
static void prefixed_delete_observer(void* context, const char* rel) {
static void prefixed_delete_observer(void* context, const char* rel, DeleteEntryType type) {
PrefixedDeleteObserver* prefixed = context;
if (!prefixed->inner || !rel)
return;
char* joined = path_cat((char*)prefixed->prefix, rel);
if (joined) {
prefixed->inner(prefixed->inner_context, joined);
prefixed->inner(prefixed->inner_context, joined, type);
free(joined);
}
}
@@ -215,6 +227,166 @@ static void prefixed_delete_observer(void* context, const char* rel) {
--max-delete budget: once it is exhausted the remaining requests are skipped
and counted. Returns false only on a genuine error (a confinement failure on
a validated path or an I/O error), which fails the run. */
/* How one missing-args request leaves the driver loop. The original walker
`continue`s past an invalid/protected/absent/budget-skipped request (without
breaking) but stops after a request that ran to completion while an error is
pending; NEXT/STOP preserve that control flow exactly. */
typedef enum { MISSING_ARG_NEXT, MISSING_ARG_STOP } MissingArgStep;
/* Remove a NON-empty missing-args directory recursively (--delete/--force in
effect): walk its contents through the budgeted extras walker so every removed
file/dir counts toward --max-delete (rsync parity), then remove the now-empty
directory itself, which costs one more budget unit. A run that hits the cap
leaves the remaining entries in place. The observer is wrapped so the nested
walk reports receive-root-relative paths. Sets the *removed and *ok outputs. */
static void delete_nonempty_missing_dir(const char* full, const char* rel,
DeleteBudgetState* budget, DeletePathObserver observer,
void* observer_context, bool* removed, bool* ok) {
ArrayList* no_keeps = array_list_create(free);
/* Never let an accounting slip (deleted > max_delete) underflow the remaining
budget into SIZE_MAX, which would grant unlimited deletions. */
size_t remaining =
budget->deleted >= budget->max_delete ? 0 : budget->max_delete - budget->deleted;
size_t contents_deleted = 0;
size_t contents_skipped = 0;
PrefixedDeleteObserver nested = {observer, observer_context, rel};
DeleteWalkResult walk =
no_keeps ? delete_extras_limited_observed(full, no_keeps, NULL, remaining, NULL, 0, NULL,
&contents_deleted, &contents_skipped,
observer ? prefixed_delete_observer : NULL,
observer ? &nested : NULL)
: DELETE_WALK_ERROR;
if (no_keeps)
array_list_delete(no_keeps);
budget->deleted += contents_deleted;
budget->skipped += contents_skipped;
if (walk == DELETE_WALK_LIMIT_REACHED) {
budget->limit_hit = true;
} else if (walk != DELETE_WALK_OK) {
*ok = false;
} else if (budget->deleted >= budget->max_delete) {
budget->limit_hit = true;
budget->skipped++;
} else if (file_remove_tree_secure(full)) {
/* The shared `if (removed)` tail charges this directory exactly once;
counting it here too would consume two budget units. */
*removed = true;
} else {
*ok = false;
}
}
/* Remove one missing-args destination mirror. `skips` holds the receiver
artifacts (staging directory, basis snapshots) that stay protected. Returns
MISSING_ARG_STOP when the driver loop must stop (a completed removal left a
genuine error pending) and MISSING_ARG_NEXT otherwise; *ok accumulates the
overall success across the whole run. */
static MissingArgStep delete_one_missing_arg(const Config* config, const char* rel,
const DeleteSkipSet* skips, DeleteBudgetState* budget,
DeletePathObserver observer, void* observer_context,
bool* ok) {
if (!rel || *rel == '\0' || *rel == '/' || has_path_traversal(rel)) {
/* Defensive only: receive_manifest_entries already validated every
section identically, so a controlled peer never reaches this branch. */
log_message(LOG_LEVEL_ERROR, "invalid missing-args delete path");
*ok = false;
return MISSING_ARG_NEXT;
}
bool at_root = strchr(rel, '/') == NULL;
if (path_under_skip_prefix(rel, at_root, skips->entries, skips->count)) {
char* escaped = output_escape(rel, log_get_8_bit_output());
log_message(LOG_LEVEL_WARNING,
"missing-args path '%s' is protected (staging directory or basis snapshot); "
"not deleting",
escaped ? escaped : "<allocation failed>");
free(escaped);
return MISSING_ARG_NEXT;
}
char* full = path_cat(config->receive_root_directory, rel);
if (!full) {
*ok = false;
return MISSING_ARG_NEXT;
}
char* leaf = NULL;
int parent_fd = file_open_secure_parent(full, &leaf, false);
if (parent_fd < 0) {
/* The mirror's parent directory may itself not exist on the destination
(a deeper missing entry whose leading directories were never created).
That is a no-op -- there is nothing to delete -- matching
file_remove_tree_secure's absent-path handling; only a genuine I/O
error (EACCES, a symlink loop, ...) fails the run. */
bool absent = errno == ENOENT || errno == ENOTDIR;
free(full);
free(leaf);
if (!absent)
*ok = false;
return MISSING_ARG_NEXT;
}
struct stat st;
if (fstatat(parent_fd, leaf, &st, AT_SYMLINK_NOFOLLOW) != 0) {
/* Already absent: nothing to delete (a no-op, not a deletion). */
if (errno != ENOENT)
*ok = false;
close(parent_fd);
free(leaf);
free(full);
return MISSING_ARG_NEXT;
}
/* An entry that exists is one deletion: skip it (and count it) when the
shared --max-delete budget is already exhausted. */
if (budget->deleted >= budget->max_delete) {
budget->limit_hit = true;
budget->skipped++;
close(parent_fd);
free(leaf);
free(full);
return MISSING_ARG_NEXT;
}
bool removed = false;
if (S_ISDIR(st.st_mode)) {
if (unlinkat(parent_fd, leaf, AT_REMOVEDIR) == 0) {
removed = true;
} else if (errno == ENOTEMPTY || errno == EEXIST) {
close(parent_fd);
parent_fd = -1;
free(leaf);
leaf = NULL;
if (config->use_delete || config->force_delete) {
delete_nonempty_missing_dir(full, rel, budget, observer, observer_context, &removed, ok);
} else {
char* escaped = output_escape(rel, log_get_8_bit_output());
log_message(LOG_LEVEL_WARNING,
"missing-args destination '%s' is a non-empty directory; use --force or "
"--delete to remove it",
escaped ? escaped : "<allocation failed>");
free(escaped);
}
} else if (errno != ENOENT) {
*ok = false;
}
} else {
if (unlinkat(parent_fd, leaf, 0) == 0) {
removed = true;
} else if (errno != ENOENT) {
*ok = false;
}
}
if (removed) {
budget->deleted++;
if (observer)
observer(observer_context, rel, delete_entry_type_of_mode(st.st_mode));
char* escaped = output_escape(rel, log_get_8_bit_output());
fprintf(stderr, " Deleted: %s\n", escaped ? escaped : "<allocation failed>");
free(escaped);
}
if (parent_fd >= 0)
close(parent_fd);
free(leaf);
free(full);
return *ok ? MISSING_ARG_NEXT : MISSING_ARG_STOP;
}
static bool delete_missing_args_budgeted_observed(const Config* config,
const DeleteManifest* manifest,
DeleteBudgetState* budget,
@@ -234,141 +406,8 @@ static bool delete_missing_args_budgeted_observed(const Config* config,
bool ok = true;
for (int i = 0; i < manifest->missing->size; i++) {
const char* rel = (const char*)manifest->missing->items[i];
if (!rel || *rel == '\0' || *rel == '/' || has_path_traversal(rel)) {
/* Defensive only: receive_manifest_entries already validated every
section identically, so a controlled peer never reaches this branch. */
log_message(LOG_LEVEL_ERROR, "invalid missing-args delete path");
ok = false;
continue;
}
bool at_root = strchr(rel, '/') == NULL;
if (path_under_skip_prefix(rel, at_root, skips.entries, skips.count)) {
char* escaped = output_escape(rel, log_get_8_bit_output());
log_message(LOG_LEVEL_WARNING,
"missing-args path '%s' is protected (staging directory or basis snapshot); "
"not deleting",
escaped ? escaped : "<allocation failed>");
free(escaped);
continue;
}
char* full = path_cat(config->receive_root_directory, rel);
if (!full) {
ok = false;
continue;
}
char* leaf = NULL;
int parent_fd = file_open_secure_parent(full, &leaf, false);
if (parent_fd < 0) {
/* The mirror's parent directory may itself not exist on the destination
(a deeper missing entry whose leading directories were never created).
That is a no-op -- there is nothing to delete -- matching
file_remove_tree_secure's absent-path handling; only a genuine I/O
error (EACCES, a symlink loop, ...) fails the run. */
bool absent = errno == ENOENT || errno == ENOTDIR;
free(full);
free(leaf);
if (!absent)
ok = false;
continue;
}
struct stat st;
if (fstatat(parent_fd, leaf, &st, AT_SYMLINK_NOFOLLOW) != 0) {
/* Already absent: nothing to delete (a no-op, not a deletion). */
if (errno != ENOENT)
ok = false;
close(parent_fd);
free(leaf);
free(full);
continue;
}
/* An entry that exists is one deletion: skip it (and count it) when the
shared --max-delete budget is already exhausted. */
if (budget->deleted >= budget->max_delete) {
budget->limit_hit = true;
budget->skipped++;
close(parent_fd);
free(leaf);
free(full);
continue;
}
bool removed = false;
if (S_ISDIR(st.st_mode)) {
if (unlinkat(parent_fd, leaf, AT_REMOVEDIR) == 0) {
removed = true;
} else if (errno == ENOTEMPTY || errno == EEXIST) {
close(parent_fd);
parent_fd = -1;
free(leaf);
leaf = NULL;
if (config->use_delete || config->force_delete) {
/* Remove the contents entry-by-entry through the budgeted extras
walker so every deleted file/dir counts toward --max-delete (rsync
parity); the now-empty directory itself costs one more. A run that
hits the cap leaves the remaining entries in place. */
ArrayList* no_keeps = array_list_create(free);
/* Never let an accounting slip (deleted > max_delete) underflow the
remaining budget into SIZE_MAX, which would grant unlimited
deletions. */
size_t remaining =
budget->deleted >= budget->max_delete ? 0 : budget->max_delete - budget->deleted;
size_t contents_deleted = 0;
size_t contents_skipped = 0;
PrefixedDeleteObserver nested = {observer, observer_context, rel};
DeleteWalkResult walk =
no_keeps ? delete_extras_limited_observed(full, no_keeps, NULL, remaining, NULL, 0,
NULL, &contents_deleted, &contents_skipped,
observer ? prefixed_delete_observer : NULL,
observer ? &nested : NULL)
: DELETE_WALK_ERROR;
if (no_keeps)
array_list_delete(no_keeps);
budget->deleted += contents_deleted;
budget->skipped += contents_skipped;
if (walk == DELETE_WALK_LIMIT_REACHED) {
budget->limit_hit = true;
} else if (walk != DELETE_WALK_OK) {
ok = false;
} else if (budget->deleted >= budget->max_delete) {
budget->limit_hit = true;
budget->skipped++;
} else if (file_remove_tree_secure(full)) {
/* The shared `if (removed)` tail charges this directory exactly
once; counting it here too would consume two budget units. */
removed = true;
} else {
ok = false;
}
} else {
char* escaped = output_escape(rel, log_get_8_bit_output());
log_message(LOG_LEVEL_WARNING,
"missing-args destination '%s' is a non-empty directory; use --force or "
"--delete to remove it",
escaped ? escaped : "<allocation failed>");
free(escaped);
}
} else if (errno != ENOENT) {
ok = false;
}
} else {
if (unlinkat(parent_fd, leaf, 0) == 0) {
removed = true;
} else if (errno != ENOENT) {
ok = false;
}
}
if (removed) {
budget->deleted++;
if (observer)
observer(observer_context, rel);
char* escaped = output_escape(rel, log_get_8_bit_output());
fprintf(stderr, " Deleted: %s\n", escaped ? escaped : "<allocation failed>");
free(escaped);
}
if (parent_fd >= 0)
close(parent_fd);
free(leaf);
free(full);
if (!ok)
if (delete_one_missing_arg(config, rel, &skips, budget, observer, observer_context, &ok) ==
MISSING_ARG_STOP)
break;
}
delete_skips_free(&skips);
@@ -386,8 +425,11 @@ bool manifest_would_delete_list(const Config* config, const DeleteManifest* mani
DeleteSkipSet skips;
if (!delete_skips_build(config, manifest->protected, NULL, true, &skips))
return false;
DeleteProtectRules protect = {.base_rules = config->protect_rules,
.dir_rules = manifest->per_dir_rules,
.backup_suffix = delete_backup_suffix(config)};
bool ok = delete_extras_list(config->receive_root_directory, manifest->keeps, manifest->dirs,
skips.entries, skips.count, config->protect_rules, out, count_out);
skips.entries, skips.count, &protect, out, count_out);
delete_skips_free(&skips);
return ok;
}
+8
View File
@@ -4,6 +4,7 @@
#include "array_list.h"
#include "config.h"
#include "delete.h"
#include "filter.h"
#include <stdbool.h>
/* Delete-commit module: delete-manifest receive plus the budgeted extras and
@@ -30,6 +31,13 @@ typedef struct DeleteManifest {
leave untransmitted directories and the unlisted parts of listed ones
alone, matching rsync's "delete only in synchronized directories". */
ArrayList* dirs;
/* Per-directory filter rules the sender compiled while scanning (protocol
2.30.0), each carrying its owner directory and no-inherit flag. The
receiver evaluates them (deepest before ancestors, then the command-line
base rules) against every candidate extra so a destination-only entry that
matches ONLY a per-directory `.rsync-filter`/dir-merge rule is shielded.
NULL when the sender transmitted none. */
FilterRuleList* per_dir_rules;
} DeleteManifest;
void delete_manifest_free(DeleteManifest* manifest);
+209 -21
View File
@@ -48,6 +48,7 @@ struct DeletePlanSender {
const ArrayList* protected_prefixes;
const ArrayList* size_skipped;
const ArrayList* missing_args;
const FilterRuleList* per_dir_rules;
size_t entries;
/* Transmitted FILE entries only. The caller's "empty scan" safety guard keys
off this (an I/O error that hid every file must refuse to delete even when
@@ -284,12 +285,14 @@ bool delete_plan_sender_empty(const DeletePlanSender* sender) {
}
void delete_plan_sender_set_config(DeletePlanSender* sender, const ArrayList* protected_prefixes,
const ArrayList* size_skipped, const ArrayList* missing_args) {
const ArrayList* size_skipped, const ArrayList* missing_args,
const FilterRuleList* per_dir_rules) {
if (!sender)
return;
sender->protected_prefixes = protected_prefixes;
sender->size_skipped = size_skipped;
sender->missing_args = missing_args;
sender->per_dir_rules = per_dir_rules;
}
/* True when `dir` is `root` itself or a descendant of it (path-component
@@ -320,6 +323,181 @@ static int send_str_section(int fd, const ArrayList* list) {
return 0;
}
/* The directory a rule belongs to (its owner, or the transfer root for ""). */
static const char* filter_dir_rule_owner(const FilterRule* rule) {
return (rule && rule->owner) ? rule->owner : "";
}
/* Transmit the received-side per-directory filter rules (protocol 2.30.0) as a
* self-describing list of directory groups: a group count, then for each group
* the relative owner directory followed by that directory's rule records (run
* order = the sender's traversal/rule order). Rules of one directory are
* appended to the sink contiguously, so runs reproduce the compilation order.
* Bounded by MAX_FILTER_RULES / MAX_FILTER_BYTES and MAX_PROTECT_PATTERN_LEN so
* the peer never sees a frame it would reject. The sender enforces exactly the
* receiver's limits (including the cumulative owner+pattern byte budget) and
* fails with a clear local error instead of emitting a frame that would abort
* the transfer with STATUS_ERROR. */
bool delete_filter_dir_rules_send(int fd, const FilterRuleList* rules) {
int count = rules ? rules->count : 0;
if (count < 0 || count > MAX_FILTER_RULES) {
log_message(LOG_LEVEL_ERROR, "too many per-directory filter rules: %d (maximum %d)", count,
MAX_FILTER_RULES);
return false;
}
size_t bytes = 0;
for (int i = 0; i < count; i++) {
const FilterRule* rule = rules->items[i];
const char* owner = filter_dir_rule_owner(rule);
size_t owner_len = strlen(owner);
size_t pattern_len = rule && rule->pattern ? strlen(rule->pattern) : 0;
if (!rule || !rule->pattern || pattern_len == 0) {
log_message(LOG_LEVEL_ERROR, "invalid per-directory filter pattern");
return false;
}
if (pattern_len > MAX_PROTECT_PATTERN_LEN) {
log_message(LOG_LEVEL_ERROR,
"per-directory filter pattern exceeds %d bytes (use a shorter pattern)",
MAX_PROTECT_PATTERN_LEN);
return false;
}
if (!(owner_len == 0 || (owner[0] != '/' && !has_path_traversal(owner)))) {
log_message(LOG_LEVEL_ERROR, "invalid per-directory filter owner directory");
return false;
}
if (owner_len + pattern_len > MAX_FILTER_BYTES - bytes) {
log_message(LOG_LEVEL_ERROR, "per-directory filter rules exceed %d bytes", MAX_FILTER_BYTES);
return false;
}
bytes += owner_len + pattern_len;
}
int groups = 0;
for (int i = 0; i < count;) {
const char* owner = filter_dir_rule_owner(rules->items[i]);
groups++;
i++;
while (i < count && strcmp(filter_dir_rule_owner(rules->items[i]), owner) == 0)
i++;
}
if (!send_int(fd, groups))
return false;
for (int i = 0; i < count;) {
const char* owner = filter_dir_rule_owner(rules->items[i]);
int start = i;
i++;
while (i < count && strcmp(filter_dir_rule_owner(rules->items[i]), owner) == 0)
i++;
if (!send_wire_str(fd, owner) || !send_int(fd, i - start))
return false;
for (int j = start; j < i; j++) {
const FilterRule* rule = rules->items[j];
if (!send_int(fd, (int)rule->action) || !send_int(fd, (int)rule->sides) ||
!send_int(fd, rule->anchored ? 1 : 0) || !send_int(fd, rule->dir_only ? 1 : 0) ||
!send_int(fd, rule->negate ? 1 : 0) || !send_int(fd, rule->no_inherit ? 1 : 0) ||
!send_wire_str(fd, rule->pattern))
return false;
}
}
return true;
}
/* Read one wire flag (an int restricted to 0/1). */
static bool receive_flag(int fd, bool* value) {
int raw;
if (!receive_int(fd, &raw) || (raw != 0 && raw != 1))
return false;
*value = raw != 0;
return true;
}
/* Read the per-directory filter block emitted by delete_filter_dir_rules_send.
* Reconstructs a flat FilterRuleList whose rules carry their owner directory;
* `*out` is NULL when the sender transmitted no rules. Every bound is enforced
* (group/rule counts, owner/pattern bytes, pattern length, action/sides domain)
* so a malicious peer can neither overread nor allocate unboundedly. Returns
* false on a malformed frame (the caller signals STATUS_ERROR). */
bool delete_filter_dir_rules_receive(int fd, FilterRuleList** out) {
if (!out)
return false;
*out = NULL;
int groups;
if (!receive_int(fd, &groups) || groups < 0 || groups > MAX_FILTER_RULES)
return false;
if (groups == 0)
return true;
FilterRuleList* list = filter_rule_list_create();
if (!list)
return false;
int total_rules = 0;
size_t bytes = 0;
for (int g = 0; g < groups; g++) {
char* dir = receive_wire_str(fd);
if (!dir)
goto fail;
size_t dir_bytes = strlen(dir);
if (!(dir[0] == '\0' || (dir[0] != '/' && !has_path_traversal(dir))) ||
dir_bytes > MAX_FILTER_BYTES - bytes) {
free(dir);
goto fail;
}
bytes += dir_bytes;
int rule_count;
if (!receive_int(fd, &rule_count) || rule_count < 0 || rule_count > MAX_FILTER_RULES ||
rule_count > MAX_FILTER_RULES - total_rules) {
free(dir);
goto fail;
}
for (int r = 0; r < rule_count; r++) {
int action, sides;
bool anchored, dir_only, negate, no_inherit;
if (!receive_int(fd, &action) ||
(action != FILTER_ACTION_EXCLUDE && action != FILTER_ACTION_INCLUDE) ||
!receive_int(fd, &sides) || sides < (int)FILTER_SIDE_SENDER ||
sides > (int)(FILTER_SIDE_SENDER | FILTER_SIDE_RECEIVER) ||
!receive_flag(fd, &anchored) || !receive_flag(fd, &dir_only) ||
!receive_flag(fd, &negate) || !receive_flag(fd, &no_inherit)) {
free(dir);
goto fail;
}
char* pattern = receive_wire_str(fd);
size_t pattern_bytes = pattern ? strlen(pattern) : 0;
if (!pattern || pattern_bytes == 0 || pattern_bytes > MAX_PROTECT_PATTERN_LEN ||
pattern_bytes > MAX_FILTER_BYTES - bytes) {
free(pattern);
free(dir);
goto fail;
}
bytes += pattern_bytes;
FilterRule* rule = calloc(1, sizeof(FilterRule));
if (!rule) {
free(pattern);
free(dir);
goto fail;
}
rule->action = (FilterAction)action;
rule->sides = (unsigned)sides;
rule->anchored = anchored;
rule->dir_only = dir_only;
rule->negate = negate;
rule->no_inherit = no_inherit;
rule->owner = str_dup(dir);
rule->pattern = pattern;
if (!rule->owner || !filter_rule_list_add(list, rule)) {
filter_rule_free(rule);
free(dir);
goto fail;
}
total_rules++;
}
free(dir);
}
*out = list;
return true;
fail:
filter_rule_list_free(list);
return false;
}
static int send_plan_node(int fd, DeletePlanSender* sender, PlanNode* node) {
if (!send_status(fd, STATUS_DELETE_PLAN))
return -1;
@@ -328,7 +506,8 @@ static int send_plan_node(int fd, DeletePlanSender* sender, PlanNode* node) {
if (!sender->config_sent) {
if (send_str_section(fd, sender->protected_prefixes) != 0 ||
send_str_section(fd, sender->size_skipped) != 0 ||
send_str_section(fd, sender->missing_args) != 0)
send_str_section(fd, sender->missing_args) != 0 ||
!delete_filter_dir_rules_send(fd, sender->per_dir_rules))
return -1;
sender->config_sent = true;
}
@@ -355,7 +534,8 @@ static int send_config_only(int fd, DeletePlanSender* sender) {
return -1;
if (send_str_section(fd, sender->protected_prefixes) != 0 ||
send_str_section(fd, sender->size_skipped) != 0 ||
send_str_section(fd, sender->missing_args) != 0)
send_str_section(fd, sender->missing_args) != 0 ||
!delete_filter_dir_rules_send(fd, sender->per_dir_rules))
return -1;
sender->config_sent = true;
if (!send_int(fd, 0)) /* apply = false */
@@ -472,15 +652,19 @@ struct DeletePlanSession {
ArrayList* protected_prefixes;
ArrayList* size_skipped;
ArrayList* missing;
/* Received per-directory filter rules (protocol 2.30.0), or NULL. Evaluated
deepest-directory-first for every candidate extra so a destination-only
entry matching only a per-directory rule is protected. */
FilterRuleList* per_dir_rules;
ArrayList* deferred;
DeletePathObserver observer;
void* observer_context;
};
/* Report one path the session truly removed (no-op without an observer). */
static void notify_deleted(DeletePlanSession* session, const char* rel) {
static void notify_deleted(DeletePlanSession* session, const char* rel, DeleteEntryType type) {
if (session && session->observer && rel)
session->observer(session->observer_context, rel);
session->observer(session->observer_context, rel, type);
}
/* A removed directory is reported with rsync's trailing slash (`deleting dir/`)
@@ -491,13 +675,13 @@ static void notify_deleted_dir(DeletePlanSession* session, const char* rel) {
size_t len = strlen(rel);
char* with_slash = malloc(len + 2);
if (!with_slash) {
session->observer(session->observer_context, rel);
session->observer(session->observer_context, rel, DELETE_ENTRY_DIR);
return;
}
memcpy(with_slash, rel, len);
with_slash[len] = '/';
with_slash[len + 1] = '\0';
session->observer(session->observer_context, with_slash);
session->observer(session->observer_context, with_slash, DELETE_ENTRY_DIR);
free(with_slash);
}
@@ -532,6 +716,7 @@ void delete_plan_session_destroy(DeletePlanSession* session) {
array_list_delete(session->protected_prefixes);
array_list_delete(session->size_skipped);
array_list_delete(session->missing);
filter_rule_list_free(session->per_dir_rules);
array_list_delete(session->deferred);
free(session);
}
@@ -604,16 +789,19 @@ static int open_plan_dir(const Config* config, const char* dir) {
typedef struct {
DeleteSkipSet set;
/* Receiver-side delete-protection rules received on the config frame (NULL
when the sender sent none). Evaluated per extra so a protect/risk rule is
/* Receiver-side delete-protection rules. `base` is the config-frame
command-line set and `dir` the received per-directory set (both NULL when
the sender sent none). Evaluated per extra so a protect/risk rule is
honored under --delete-during/--delete-delay exactly like the whole-tree
commit walker. */
const FilterRuleList* protect_rules;
DeleteProtectRules protect;
} PlanSkips;
static bool build_plan_skips(const Config* config, const DeletePlanSession* session,
PlanSkips* out) {
out->protect_rules = config->protect_rules;
out->protect.base_rules = config->protect_rules;
out->protect.dir_rules = session->per_dir_rules;
out->protect.backup_suffix = delete_backup_suffix(config);
/* The per-directory plan walk keeps each basis path verbatim (it does not
convert an absolute under-root path to its root-relative form, unlike the
whole-tree commit walk). */
@@ -711,8 +899,8 @@ static bool process_extra_dir(int dirfd, const char* name, const char* child_rel
return errno == ENOTEMPTY || errno == EEXIST;
}
static bool process_extra_file(int dirfd, const char* name, const char* child_rel, bool force_now,
DeletePlanSession* session) {
static bool process_extra_file(int dirfd, const char* name, const char* child_rel, mode_t mode,
bool force_now, DeletePlanSession* session) {
if (session->defer && !force_now) {
return defer_add(session, child_rel);
}
@@ -724,7 +912,7 @@ static bool process_extra_file(int dirfd, const char* name, const char* child_re
session->deleted++;
session->planned++;
log_deleted(child_rel);
notify_deleted(session, child_rel);
notify_deleted(session, child_rel, delete_entry_type_of_mode(mode));
} else if (errno != ENOENT) {
return false;
}
@@ -778,10 +966,8 @@ static bool process_children(int dirfd, const char* dir_rel, const ArrayList* ke
bool is_dir = entries[i].is_dir;
bool in_keep_dirs = is_dir && list_contains_str(keep_dirs, entries[i].name);
bool in_keep_files = !is_dir && list_contains_str(keep_files, entries[i].name);
bool rule_protected =
skips->protect_rules &&
filter_rules_apply_side(skips->protect_rules, child_rel, entries[i].name, is_dir,
FILTER_SIDE_RECEIVER) == FILTER_ACTION_PROTECT;
bool rule_protected = delete_protect_verdict(&skips->protect, child_rel, entries[i].name,
is_dir) == FILTER_ACTION_PROTECT;
if (in_keep_dirs || in_keep_files || rule_protected) {
shielded[i] = true;
local_survives = true;
@@ -828,7 +1014,8 @@ static bool process_children(int dirfd, const char* dir_rel, const ArrayList* ke
operation_ok = false;
continue;
}
if (!process_extra_file(dirfd, entries[i].name, child_rel, force[i] || force_now, session))
if (!process_extra_file(dirfd, entries[i].name, child_rel, entries[i].mode,
force[i] || force_now, session))
operation_ok = false;
free(child_rel);
}
@@ -902,7 +1089,8 @@ int delete_plan_session_receive(DeletePlanSession* session, const Config* config
if (has_config) {
if (session->config_seen || !read_section(fd, session->protected_prefixes, true, &bytes) ||
!read_section(fd, session->size_skipped, true, &bytes) ||
!read_section(fd, session->missing, true, &bytes)) {
!read_section(fd, session->missing, true, &bytes) ||
!delete_filter_dir_rules_receive(fd, &session->per_dir_rules)) {
send_status(fd, STATUS_ERROR);
return -1;
}
@@ -1032,7 +1220,7 @@ static bool apply_deferred_path(DeletePlanSession* session, const Config* config
session->deleted++;
session->planned++;
log_deleted(rel);
notify_deleted(session, rel);
notify_deleted(session, rel, delete_entry_type_of_mode(st.st_mode));
} else if (errno != ENOENT) {
close(parent_fd);
free(leaf);
+15 -1
View File
@@ -25,6 +25,19 @@
* --delete-missing-args exact deletions, the shared --max-delete budget and,
* for --delete-delay, the snapshotted extras. */
/* Per-directory filter-rule block (protocol 2.30.0). The sender compiles the
* source's per-directory merge rules as it scans and streams them so the
* receiver can re-derive the receiver-side protect/risk verdicts for
* destination-only entries. The wire format is a group count, then for each
* directory group its relative owner path followed by that directory's rule
* records (action, sides, anchored, dir-only, negate, no-inherit, pattern).
* All bounds (MAX_FILTER_RULES, MAX_FILTER_BYTES, MAX_PROTECT_PATTERN_LEN) are
* enforced on both sides; a malformed receive frame signals STATUS_ERROR and
* returns false. Receive yields a flat FilterRuleList whose rules carry their
* owner, or NULL when no rules were sent. */
bool delete_filter_dir_rules_send(int fd, const FilterRuleList* rules);
bool delete_filter_dir_rules_receive(int fd, FilterRuleList** out);
/* ---- Sender: plan builder ---- */
typedef struct DeletePlanSender DeletePlanSender;
@@ -53,7 +66,8 @@ bool delete_plan_sender_empty(const DeletePlanSender* sender);
* block is always transmitted by delete_plan_send_root(), on a config-only
* carrier frame when the scope allows no directory plan. */
void delete_plan_sender_set_config(DeletePlanSender* sender, const ArrayList* protected_prefixes,
const ArrayList* size_skipped, const ArrayList* missing_args);
const ArrayList* size_skipped, const ArrayList* missing_args,
const FilterRuleList* per_dir_rules);
/* Send the root plan (even before any data, so root extras are handled like
* rsync's first generator directory), after transmitting the per-run config
* block on its own carrier frame. Returns -1 on I/O error. */
+184
View File
@@ -1,10 +1,12 @@
#include "delta.h"
#include "log.h"
#include "protocol.h"
#include <errno.h>
#include <stdint.h>
#include <limits.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#define XXH_STATIC_LINKING_ONLY
#define XXH_IMPLEMENTATION
@@ -82,6 +84,64 @@ DeltaSignature* delta_signature_create_seeded(const void* old_file_data, uint64_
return sig;
}
/* Bounded read of exactly `len` bytes at `off`; retries on EINTR. */
static bool pread_all(int fd, void* buf, size_t len, uint64_t off) {
uint8_t* p = buf;
size_t done = 0;
while (done < len) {
ssize_t n = pread(fd, p + done, len - done, (off_t)(off + done));
if (n < 0 && errno == EINTR)
continue;
if (n <= 0)
return false;
done += (size_t)n;
}
return true;
}
DeltaSignature* delta_signature_create_fd_seeded(int fd, uint64_t old_file_size,
uint32_t block_size, uint32_t seed) {
if (fd < 0 || old_file_size == 0 || block_size == 0 || block_size > DELTA_BLOCK_SIZE_MAX ||
old_file_size > UINT32_MAX * (uint64_t)block_size)
return NULL;
uint32_t block_count = (uint32_t)((old_file_size + block_size - 1) / block_size);
/* Bound the signature's own memory (block_count * sizeof(DeltaBlockSig)). */
if (block_count == 0 || block_count > MAX_DELTA_BLOCKS)
return NULL;
DeltaSignature* sig = protocol_alloc(sizeof(DeltaSignature));
if (!sig)
return NULL;
sig->file_size = old_file_size;
sig->block_size = block_size;
sig->block_count = block_count;
sig->blocks = protocol_alloc((size_t)block_count * sizeof(DeltaBlockSig));
if (!sig->blocks) {
free(sig);
return NULL;
}
uint8_t* block = malloc(block_size);
if (!block) {
free(sig->blocks);
free(sig);
return NULL;
}
for (uint32_t i = 0; i < block_count; i++) {
uint64_t offset = (uint64_t)i * block_size;
uint32_t len =
(uint32_t)((old_file_size - offset < block_size) ? (old_file_size - offset) : block_size);
if (!pread_all(fd, block, len, offset)) {
free(block);
free(sig->blocks);
free(sig);
return NULL;
}
sig->blocks[i].adler32 = delta_adler32(block, len);
sig->blocks[i].xxhash = delta_xxhash32_seeded(block, len, seed);
}
free(block);
return sig;
}
Data* delta_signature_serialize(const DeltaSignature* sig) {
if (!sig)
return NULL;
@@ -687,6 +747,130 @@ void* delta_apply(const void* old_data, uint64_t old_size, const Delta* delta,
return output;
}
void* delta_apply_fd(int src_fd, uint64_t old_size, const Delta* delta, uint32_t block_size) {
if (!delta || block_size == 0 || block_size > DELTA_BLOCK_SIZE_MAX ||
(delta->instruction_count > 0 && !delta->instructions) || delta->new_file_size == 0 ||
delta->new_file_size > SIZE_MAX)
return NULL;
void* output = protocol_alloc((size_t)delta->new_file_size);
if (!output)
return NULL;
uint8_t* out = (uint8_t*)output;
uint64_t out_pos = 0;
for (uint32_t i = 0; i < delta->instruction_count; i++) {
if (delta->instructions[i].type == DELTA_INSTR_BLOCK_MATCH) {
uint64_t src_offset = (uint64_t)delta->instructions[i].match.block_index * block_size;
if (src_offset > UINT64_MAX - delta->instructions[i].match.block_offset) {
free(output);
return NULL;
}
src_offset += delta->instructions[i].match.block_offset;
uint32_t len = delta->instructions[i].match.length;
if (src_offset > old_size || (uint64_t)len > old_size - src_offset ||
out_pos > delta->new_file_size || (uint64_t)len > delta->new_file_size - out_pos) {
free(output);
return NULL;
}
if (!pread_all(src_fd, out + out_pos, len, src_offset)) {
free(output);
return NULL;
}
out_pos += len;
} else if (delta->instructions[i].type == DELTA_INSTR_LITERAL) {
uint32_t len = delta->instructions[i].literal.length;
if (out_pos > delta->new_file_size || (uint64_t)len > delta->new_file_size - out_pos) {
free(output);
return NULL;
}
memcpy(out + out_pos, delta->instructions[i].literal.data, len);
out_pos += len;
} else {
free(output);
return NULL;
}
}
if (out_pos != delta->new_file_size) {
free(output);
return NULL;
}
return output;
}
bool delta_apply_to_fd(const void* old_data, int src_fd, uint64_t old_size, const Delta* delta,
uint32_t block_size, int dst_fd) {
if (!delta || (old_data == NULL && src_fd < 0) || block_size == 0 ||
block_size > DELTA_BLOCK_SIZE_MAX || (delta->instruction_count > 0 && !delta->instructions))
return false;
const int chunk = 1 << 20;
uint8_t* buf = malloc((size_t)chunk);
if (!buf)
return false;
uint64_t out_pos = 0;
bool ok = true;
for (uint32_t i = 0; i < delta->instruction_count && ok; i++) {
uint64_t src_offset = 0;
uint64_t len = 0;
const uint8_t* lit = NULL;
if (delta->instructions[i].type == DELTA_INSTR_BLOCK_MATCH) {
src_offset = (uint64_t)delta->instructions[i].match.block_index * block_size;
if (src_offset > UINT64_MAX - delta->instructions[i].match.block_offset ||
src_offset + delta->instructions[i].match.block_offset > old_size) {
ok = false;
break;
}
src_offset += delta->instructions[i].match.block_offset;
len = delta->instructions[i].match.length;
if (len > old_size - src_offset) {
ok = false;
break;
}
} else if (delta->instructions[i].type == DELTA_INSTR_LITERAL) {
lit = delta->instructions[i].literal.data;
len = delta->instructions[i].literal.length;
} else {
ok = false;
break;
}
if (out_pos > delta->new_file_size || len > delta->new_file_size - out_pos) {
ok = false;
break;
}
uint64_t done = 0;
while (ok && done < len) {
size_t want = (len - done) < (uint64_t)chunk ? (size_t)(len - done) : (size_t)chunk;
if (lit) {
memcpy(buf, lit + done, want);
} else if (old_data) {
memcpy(buf, (const uint8_t*)old_data + src_offset + done, want);
} else if (!pread_all(src_fd, buf, want, src_offset + done)) {
ok = false;
break;
}
const uint8_t* p = buf;
size_t written = 0;
while (written < want) {
ssize_t n = write(dst_fd, p + written, want - written);
if (n < 0 && errno == EINTR)
continue;
if (n <= 0) {
ok = false;
break;
}
written += (size_t)n;
}
done += want;
}
out_pos += len;
}
free(buf);
return ok && out_pos == delta->new_file_size;
}
void delta_destroy(Delta* delta) {
if (!delta)
return;
+16
View File
@@ -61,6 +61,13 @@ DeltaSignature* delta_signature_create(const void* old_file_data, uint64_t old_f
* identical to the unseeded function. */
DeltaSignature* delta_signature_create_seeded(const void* old_file_data, uint64_t old_file_size,
uint32_t block_size, uint32_t seed);
/* Streaming equivalent of delta_signature_create_seeded: reads the basis blocks
* from `fd` in bounded chunks, so an arbitrarily large basis can be signed
* without materializing it. The signature itself is bounded (MAX_DELTA_BLOCKS
* entries); an over-large basis returns NULL and the caller falls back to a
* whole-file transfer. */
DeltaSignature* delta_signature_create_fd_seeded(int fd, uint64_t old_file_size,
uint32_t block_size, uint32_t seed);
Data* delta_signature_serialize(const DeltaSignature* sig);
DeltaSignature* delta_signature_deserialize(const Data* data);
void delta_signature_destroy(DeltaSignature* sig);
@@ -75,6 +82,15 @@ Delta* delta_compute_seeded(const void* new_file_data, uint64_t new_file_size,
Data* delta_serialize(const Delta* delta);
Delta* delta_deserialize(const Data* data);
void* delta_apply(const void* old_data, uint64_t old_size, const Delta* delta, uint32_t block_size);
/* Streaming equivalent of delta_apply: matched blocks are read from `src_fd` as
* they are emitted, so the basis never has to be resident. */
void* delta_apply_fd(int src_fd, uint64_t old_size, const Delta* delta, uint32_t block_size);
/* Fully streamed reconstruction: matched blocks come from `old_data` (when
* non-NULL) or are read from `src_fd`, and the reconstructed bytes are written
* straight to `dst_fd` in bounded chunks, so a reconstructed file larger than
* memory is never materialized. */
bool delta_apply_to_fd(const void* old_data, int src_fd, uint64_t old_size, const Delta* delta,
uint32_t block_size, int dst_fd);
void delta_destroy(Delta* delta);
bool delta_should_attempt(uint64_t old_size, uint64_t new_size, uint64_t max_file_size);
+261
View File
@@ -16,6 +16,8 @@
#include "data.h"
#include "checksum.h"
#include "chunk.h"
#include "compression.h"
#include "delta.h"
#include "file.h"
#include "file_store.h"
@@ -209,6 +211,7 @@ File* file_create(const char* path) {
file->dir_time_only = false;
file->basis_link = NULL;
file->basis_copy = NULL;
file->data_spool = false;
file->link_group = 0;
file->link_first = false;
file->hardlink_target = NULL;
@@ -238,8 +241,11 @@ void file_destroy(void* item) {
file->send_path = NULL;
free(file->basis_link);
file->basis_link = NULL;
if (file->data_spool && file->basis_copy)
unlink(file->basis_copy);
free(file->basis_copy);
file->basis_copy = NULL;
file->data_spool = false;
free(file->hardlink_target);
file->hardlink_target = NULL;
free(file->symlink_target);
@@ -381,6 +387,261 @@ size_t file_content_to_buffer(File* file) {
return bytes_read;
}
/* ---- Streamed whole-file payload receive ----
*
* A whole-file data frame is a uint64 length followed by that many bytes. When
* the logical payload is at or below the receiver's streaming bound the
* historical charged whole-buffer path is kept (decompressing in one shot for
* `-z`). Above the bound the frame is read in bounded chunks and written into
* a spool temp file next to the destination, decompressing incrementally for
* zstd/zlib (lz4's block format cannot be streamed and keeps the buffered path).
* The spool is then installed by the existing bounded-buffer basis-copy path
* (file_copy_basis_stream_attrs), so the atomic temp+rename store, --partial/
* --partial-dir, --delay-updates, --preallocate and metadata/xattr application
* are all reused unchanged. Only bounded buffers (64 KiB read chunk + the
* decompressor's 256 KiB output window) are ever live. */
#define FILE_PAYLOAD_READ_CHUNK (64 * 1024)
#define FILE_PAYLOAD_PEEK_MAX 32
/* Create a confined spool temp file in the destination's directory. Returns an
* open write fd and an owned absolute path, or -1 (errno set). The parent walk
* creates missing directories exactly as a normal store would. */
static int file_spool_create(const char* dest_path, char** out_spool_path) {
*out_spool_path = NULL;
char* leaf = NULL;
int dirfd = file_open_secure_parent(dest_path, &leaf, true);
free(leaf);
if (dirfd < 0)
return -1;
char name[64];
for (unsigned int i = 0; i < 100; i++) {
snprintf(name, sizeof(name), ".fastsync-spool.%ld.%llu", (long)getpid(), next_temp_sequence());
int fd = openat(dirfd, name, O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC | O_NOFOLLOW, 0600);
if (fd < 0) {
if (errno != EEXIST)
break;
continue;
}
char* copy = str_dup(dest_path);
const char* dir = copy ? dirname(copy) : NULL;
size_t need = dir ? strlen(dir) + 1 + strlen(name) + 1 : 0;
char* full = need ? malloc(need) : NULL;
if (!full) {
free(copy);
close(fd);
unlinkat(dirfd, name, 0);
close(dirfd);
return -1;
}
snprintf(full, need, "%s/%s", dir, name);
free(copy);
close(dirfd);
*out_spool_path = full;
return fd;
}
close(dirfd);
return -1;
}
int file_spool_for_payload(const char* dest_path, char** out_spool_path) {
return file_spool_create(dest_path, out_spool_path);
}
bool file_receive_payload(int fd, bool compress, unsigned long long expected_size,
const char* dest_path, unsigned long long stream_limit, Data** out_buffer,
char** out_spool, unsigned long long* out_size) {
if (out_buffer)
*out_buffer = NULL;
if (out_spool)
*out_spool = NULL;
if (out_size)
*out_size = 0;
if (!out_buffer || !out_spool || !out_size || !dest_path)
return false;
unsigned long long frame_size = 0;
if (!receive_n_data(fd, &frame_size, sizeof(frame_size)))
return false;
/* A frame larger than MAX_DATA_PAYLOAD_SIZE is allowed only on the streamed
path, which never materializes it; the buffered path's
receive_data_alloc/body enforces the historical bound itself. Only a size
unrepresentable on this platform is rejected here. */
if (frame_size > SIZE_MAX) {
log_message(LOG_LEVEL_ERROR, "Data size %llu is not representable", frame_size);
return false;
}
unsigned char prefix[FILE_PAYLOAD_PEEK_MAX];
size_t prefix_len = 0;
bool stream;
if (expected_size != 0) {
/* The check frame already told us the logical size: no need to peek. */
stream = expected_size > stream_limit;
} else if (!compress) {
stream = frame_size > stream_limit;
} else {
/* Non-incremental compressed frame with unknown logical size: peek the
header to decide, so a small compressed frame that expands past the bound
still streams instead of allocating the whole logical image. */
size_t want = frame_size < sizeof(prefix) ? (size_t)frame_size : sizeof(prefix);
if (want > 0 && !receive_n_data(fd, prefix, want))
return false;
prefix_len = want;
unsigned long long logical = compression_peek_frame_content_size(prefix, prefix_len);
stream = logical != 0 ? logical > stream_limit : frame_size > stream_limit;
}
if (!stream) {
Data* frame;
if (prefix_len > 0) {
frame = receive_data_alloc(fd, frame_size);
if (!frame)
return false;
memcpy(frame->data, prefix, prefix_len);
if (frame_size > prefix_len &&
!receive_n_data(fd, (char*)frame->data + prefix_len, (size_t)(frame_size - prefix_len))) {
data_destroy(frame);
return false;
}
} else {
frame = receive_data_body(fd, frame_size);
if (!frame)
return false;
}
if (compress) {
unsigned long long bound =
expected_size != 0 ? expected_size : (unsigned long long)MAX_RECEIVE_WHOLE_FILE_SIZE;
Data* uncompressed = data_decompress_limited(frame, (size_t)bound);
ProtocolSession* owner = frame->owner;
data_destroy(frame);
if (!uncompressed)
return false;
if (expected_size != 0 && uncompressed->size != expected_size) {
data_destroy(uncompressed);
return false;
}
if (!data_charge_session(uncompressed, owner, uncompressed->size)) {
data_destroy(uncompressed);
return false;
}
*out_buffer = uncompressed;
*out_size = uncompressed->size;
return true;
}
*out_buffer = frame;
*out_size = frame->size;
return true;
}
/* Streamed path. */
int spool_fd = file_spool_create(dest_path, out_spool);
if (spool_fd < 0)
return false;
CompressionStreamDecompressor* dec = NULL;
size_t header_len = 0;
unsigned long long learned_size = expected_size;
if (compress) {
unsigned char hdr[1 + 4];
size_t hdr_have = 0;
if (prefix_len >= 1) {
hdr[0] = prefix[0];
hdr_have = 1;
} else {
if (!receive_n_data(fd, hdr, 1))
goto stream_fail;
hdr_have = 1;
}
CompressionAlgo algo = (CompressionAlgo)hdr[0];
if (!compression_algo_valid((int)algo) || algo == COMPRESSION_ALGO_LZ4)
goto stream_fail;
header_len = (algo == COMPRESSION_ALGO_ZLIB || algo == COMPRESSION_ALGO_ZLIBX) ? 5 : 1;
while (hdr_have < header_len) {
size_t need = header_len - hdr_have;
if (prefix_len > hdr_have) {
size_t avail = prefix_len - hdr_have;
size_t take = avail < need ? avail : need;
memcpy(hdr + hdr_have, prefix + hdr_have, take);
hdr_have += take;
} else {
if (!receive_n_data(fd, hdr + hdr_have, need))
goto stream_fail;
hdr_have = header_len;
}
}
if (header_len == 5) {
uint32_t raw = 0;
for (int i = 0; i < 4; i++)
raw |= (uint32_t)hdr[1 + i] << (8 * i);
if (expected_size != 0 && raw != expected_size)
goto stream_fail;
if (learned_size == 0)
learned_size = raw;
}
dec = compression_stream_decompressor_create(algo, learned_size);
if (!dec)
goto stream_fail;
}
if (frame_size < header_len)
goto stream_fail;
{
unsigned long long body_remaining = frame_size - header_len;
if (prefix_len > header_len) {
size_t avail = prefix_len - header_len;
if (compress) {
bool done = false;
if (!compression_stream_decompressor_feed(dec, prefix + header_len, avail, spool_fd, &done))
goto stream_fail;
} else if (!write_all(spool_fd, prefix + header_len, avail)) {
goto stream_fail;
}
body_remaining -= avail;
}
unsigned char buf[FILE_PAYLOAD_READ_CHUNK];
while (body_remaining > 0) {
size_t want = body_remaining < sizeof(buf) ? (size_t)body_remaining : (size_t)sizeof(buf);
if (!receive_n_data(fd, buf, want))
goto stream_fail;
if (compress) {
bool done = false;
if (!compression_stream_decompressor_feed(dec, buf, want, spool_fd, &done))
goto stream_fail;
} else if (!write_all(spool_fd, buf, want)) {
goto stream_fail;
}
body_remaining -= want;
}
}
{
unsigned long long total = compress ? compression_stream_decompressor_total(dec) : frame_size;
if (learned_size != 0 && total != learned_size)
goto stream_fail;
*out_size = total;
}
if (dec)
compression_stream_decompressor_destroy(dec);
if (close(spool_fd) != 0) {
spool_fd = -1;
goto stream_fail;
}
return true;
stream_fail:
if (dec)
compression_stream_decompressor_destroy(dec);
if (spool_fd >= 0)
close(spool_fd);
if (*out_spool) {
unlink(*out_spool);
free(*out_spool);
*out_spool = NULL;
}
return false;
}
/* ---- Secure filesystem primitives ---- */
bool file_path_exists_secure(const char* path) {
+18
View File
@@ -177,6 +177,24 @@ bool file_copy_basis_stream_attrs(const char* path, const char* basis_path,
const FileMetadata* metadata, FileAttrPolicy policy, bool update,
bool use_fsync, const FileXattrList* xattrs, bool fake_super,
const char* temp_dir);
/* Receive one length-prefixed whole-file data frame, streaming the payload
* through a bounded buffer when it (or its known logical size) exceeds
* `stream_limit`. On success exactly one of *out_buffer / *out_spool is set:
* - *out_buffer: the historical charged whole-buffer Data (caller destroys);
* - *out_spool: an owned temp path holding the payload, installed through the
* File's basis_copy field with File.data_spool set so file_destroy unlinks
* it. The destination policy/metadata/atomic-store handling is then the
* existing bounded-buffer basis install (file_copy_basis_stream_attrs).
* `expected_size` (0 = unknown) is the logical size from the check frame;
* `dest_path` locates the spool next to the destination; `compress` selects
* incremental decompression. Returns false on any framing/I/O/size error. */
bool file_receive_payload(int fd, bool compress, unsigned long long expected_size,
const char* dest_path, unsigned long long stream_limit, Data** out_buffer,
char** out_spool, unsigned long long* out_size);
/* Create a confined spool temp file next to `dest_path`; returns an open write
* fd and an owned absolute path (to be installed via File.basis_copy with
* File.data_spool set, and unlinked by file_destroy). */
int file_spool_for_payload(const char* dest_path, char** out_spool_path);
/* Protocol 2.28.0 receiver-stat variants: like the two above but additionally
* report through `dirs_created` (when non-NULL) how many parent directories the
* confined secure walk had to create that lie strictly below `count_floor` (a
+92 -57
View File
@@ -58,36 +58,41 @@ File* file_receive(const Config* config, int file_descriptor) {
file_destroy(file);
return NULL;
}
Data* file_data = receive_data_limited(file_descriptor, MAX_RECEIVE_WHOLE_FILE_SIZE);
if (file_data == NULL) {
bool compress =
config->use_compression && !compression_should_skip_with_suffixes(
file->path, config->skip_compress_suffixes,
config->skip_compress_set ? config->skip_compress_count : -1);
char* dest_path = path_cat(config->receive_root_directory, file->path);
if (!dest_path) {
file_destroy(file);
return NULL;
}
if (config->use_compression &&
!compression_should_skip_with_suffixes(file->path, config->skip_compress_suffixes,
config->skip_compress_set ? config->skip_compress_count
: -1)) {
Data* file_data_uncompressed = data_decompress_limited(file_data, MAX_RECEIVE_WHOLE_FILE_SIZE);
ProtocolSession* owner = file_data->owner;
data_destroy(file_data);
if (file_data_uncompressed == NULL) {
Data* buffer = NULL;
char* spool = NULL;
unsigned long long size = 0;
bool ok = file_receive_payload(file_descriptor, compress, 0, dest_path,
protocol_whole_file_receive_limit(), &buffer, &spool, &size);
free(dest_path);
if (!ok) {
file_destroy(file);
return NULL;
}
if (spool) {
Data* reserved = data_create_reserve((size_t)size);
if (!reserved) {
unlink(spool);
free(spool);
file_destroy(file);
return NULL;
}
if (!data_charge_session(file_data_uncompressed, owner, file_data_uncompressed->size)) {
data_destroy(file_data_uncompressed);
file_destroy(file);
return NULL;
}
if (file_data_uncompressed->size > MAX_FILE_DATA_SIZE) {
data_destroy(file_data_uncompressed);
file_destroy(file);
return NULL;
}
file_data = file_data_uncompressed;
data_destroy(file->data);
file->data = reserved;
file->basis_copy = spool;
file->data_spool = true;
} else {
data_destroy(file->data);
file->data = buffer;
}
data_destroy(file->data);
file->data = file_data;
return file;
}
@@ -97,12 +102,14 @@ bool dir_metadata_should_capture(const Config* config) {
/* Directory metadata is captured when a directory attribute is actually
* requested: -p/--perms (directory modes), -t/--times (directory mtimes,
* unless -O/--omit-dir-times suppresses them), -o/-g (directory ownership),
* or -X/-A (directory xattrs/ACLs). --atimes/-U alone does not pull
* directory metadata (matching the original dir-time bundle). */
* -X/-A (directory xattrs/ACLs), or --fake-super (whose reserved %stat record
* is written on the directory itself, so its metadata must travel).
* --atimes/-U alone does not pull directory metadata (matching the original
* dir-time bundle). */
return config && config->use_metadata &&
(config->preserve_perms || (config->preserve_times && !config->omit_dir_times) ||
config->preserve_owner || config->preserve_group || config->preserve_xattrs ||
config->preserve_acls);
config->preserve_acls || config->fake_super);
}
void dir_time_list_init(DirTimeList* list) {
@@ -200,12 +207,19 @@ void dir_metadata_list_apply(const DirTimeList* list, const char* root_directory
bool apply_times = config->preserve_times && !config->omit_dir_times;
bool apply_mode = config->preserve_perms;
bool apply_xattrs = config->use_xattrs;
bool apply_fake_super = config->fake_super;
/* Ownership is applied through the active identity snapshot (which no-ops
* unless an ownership request is active), and xattrs only when -X/-A was
* negotiated. Times/mode keep their own per-attribute gates. */
bool have_any = apply_times || apply_mode || apply_xattrs || identity_active_enabled();
* unless an ownership request is active), xattrs only when -X/-A was
* negotiated, and the --fake-super record whenever the flag is active.
* Times/mode keep their own per-attribute gates. */
bool have_any =
apply_times || apply_mode || apply_xattrs || apply_fake_super || identity_active_enabled();
if (!have_any)
return;
/* Built once: the --fake-super replay uses it to apply only the recorded
* permission bits (the special bits stay in the record, exactly like the
* regular-file fake-super receiver). */
FileAttrPolicy policy = file_attr_policy_from_config(config);
for (size_t i = 0; i < list->count; i++) {
char* dir_path = path_cat(root_directory, list->paths[i]);
if (!dir_path)
@@ -255,38 +269,59 @@ void dir_metadata_list_apply(const DirTimeList* list, const char* root_directory
free(escaped_path);
}
}
if (apply_mode) {
mode_t dir_mode = list->entries[i].mode;
bool mode_ready = true;
if (config->chmod_spec && *config->chmod_spec &&
!chmod_apply(dir_mode, config->chmod_spec, &dir_mode)) {
/* The final directory mode (after any --chmod) is computed once so the
--fake-super record can carry it even when the on-disk replay is
restricted to the permission bits below. */
mode_t dir_mode = list->entries[i].mode;
bool mode_ready = true;
if (apply_mode && config->chmod_spec && *config->chmod_spec &&
!chmod_apply(dir_mode, config->chmod_spec, &dir_mode)) {
char* escaped_path = output_escape(dir_path, log_get_8_bit_output());
log_message(LOG_LEVEL_WARNING, "Failed to apply --chmod to directory %s",
escaped_path ? escaped_path : "<allocation failed>");
free(escaped_path);
mode_ready = false;
}
/* Under --fake-super the normal fchmod below still applies the mode, but
the fake-super replay that follows narrows the on-disk result to the
recorded permission bits (the full mode, including setuid/setgid/sticky,
lives only in the record). Keeping the normal fchmod first means a
filesystem without xattr support still gets the directory mode rather than
silently losing it. */
if (apply_mode && mode_ready) {
/* rsync -p copies the source directory mode exactly, including
* group/other write and the setgid/sticky bits. Setuid/setgid/sticky
* are super-user activities: when the connection forbade them
* (SUPER_MODE_OFF / --no-super), strip them even under -p. */
mode_t safe_mode = dir_mode & (mode_t)(S_ISUID | S_ISGID | S_ISVTX | 0777);
if (!privilege_super_mode_permitted(config->super_mode))
safe_mode &= ~(mode_t)(S_ISUID | S_ISGID | S_ISVTX);
if (dir_fd < 0) {
char* escaped_path = output_escape(dir_path, log_get_8_bit_output());
log_message(LOG_LEVEL_WARNING, "Failed to apply --chmod to directory %s",
escaped_path ? escaped_path : "<allocation failed>");
log_message(LOG_LEVEL_WARNING, "Failed to open directory %s to set its mode: %s",
escaped_path ? escaped_path : "<allocation failed>", strerror(errno));
free(escaped_path);
} else if (fchmod(dir_fd, safe_mode) != 0) {
char* escaped_path = output_escape(dir_path, log_get_8_bit_output());
log_message(LOG_LEVEL_WARNING, "Failed to set directory mode on %s: %s",
escaped_path ? escaped_path : "<allocation failed>", strerror(errno));
free(escaped_path);
mode_ready = false;
}
if (mode_ready) {
/* rsync -p copies the source directory mode exactly, including
* group/other write and the setgid/sticky bits. Setuid/setgid/sticky
* are super-user activities: when the connection forbade them
* (SUPER_MODE_OFF / --no-super), strip them even under -p. */
mode_t safe_mode = dir_mode & (mode_t)(S_ISUID | S_ISGID | S_ISVTX | 0777);
if (!privilege_super_mode_permitted(config->super_mode))
safe_mode &= ~(mode_t)(S_ISUID | S_ISGID | S_ISVTX);
if (dir_fd < 0) {
char* escaped_path = output_escape(dir_path, log_get_8_bit_output());
log_message(LOG_LEVEL_WARNING, "Failed to open directory %s to set its mode: %s",
escaped_path ? escaped_path : "<allocation failed>", strerror(errno));
free(escaped_path);
} else if (fchmod(dir_fd, safe_mode) != 0) {
char* escaped_path = output_escape(dir_path, log_get_8_bit_output());
log_message(LOG_LEVEL_WARNING, "Failed to set directory mode on %s: %s",
escaped_path ? escaped_path : "<allocation failed>", strerror(errno));
free(escaped_path);
}
}
}
/* --fake-super: park the directory's full stat (rsync 3.4.1's exact
grammar) on the directory ITSELF, then replay only the recorded
permission bits fd-relative. The special bits live only in the record
and the recorded ownership is never real-chowned: the resolved ids are
stored for a later privileged restore, exactly like the file path. Runs
before the xattr apply so a mode change cannot clobber the ACL mask. */
if (apply_fake_super && dir_fd >= 0) {
uint32_t store_uid = 0;
uint32_t store_gid = 0;
identity_resolve_storage_ids((int32_t)list->entries[i].uid, (int32_t)list->entries[i].gid,
&store_uid, &store_gid);
fake_super_store_fd(dir_fd, store_uid, store_gid, (uint32_t)dir_mode, 0, 0);
fake_super_restore_fd(dir_fd, policy);
}
/* xattrs/ACLs last: a mode change can rewrite the ACL mask, so the ACL
xattrs must be (re)applied after fchmod. */
if (apply_xattrs && dir_fd >= 0 && list->xattrs)
+15
View File
@@ -817,6 +817,21 @@ static FileSaveResult file_save_directory_to_disk(const FileSavePlan* plan, bool
} else if (ok && identity_copy_as_active()) {
ok = false;
}
/* --fake-super: park the directory's full stat in rsync's reserved
user.rsync.%stat xattr as soon as the directory exists. This makes even a
direct file_save_to_disk_full() caller -- which never runs the deferred
DirTimeList pass -- produce an rsync-readable fake-super record. The record
carries the full mode/uid/gid; the permission bits are replayed by the
deferred pass (never inline, so a restrictive mode cannot block child
creation) and the recorded ownership is never real-chowned. Best-effort:
fake_super_store_fd() logs and skips a failure, never failing the entry. */
if (ok && plan->config && plan->config->fake_super && file->metadata && dir_fd >= 0) {
uint32_t store_uid = 0;
uint32_t store_gid = 0;
identity_resolve_storage_ids((int32_t)file->metadata->uid, (int32_t)file->metadata->gid,
&store_uid, &store_gid);
fake_super_store_fd(dir_fd, store_uid, store_gid, (uint32_t)file->metadata->mode, 0, 0);
}
/* The final source MODE is deliberately NOT applied inline. A restrictive
source mode (for example 0555) would make the directory unwritable before
its children are created, so a non-root receiver fails each child with
+127 -1
View File
@@ -44,12 +44,138 @@ bool file_send_single_calls(File* file, int file_descriptor, bool use_metadata,
send_path, NULL, -1, 0, false);
}
/* Stream-compress a whole source file into a temp file, then transmit it as the
* normal length-prefixed data frame. Used when the source was too large to
* load (File.data.data == NULL): the source is read in bounded chunks through a
* streaming codec, so neither the raw nor the compressed image is held in
* memory. The compressed length must be known before the frame is sent (the
* wire is length-prefixed), so the stream lands in a private temp file first. */
bool file_send_compressed_stream_with_skip(File* file, int file_descriptor, bool use_metadata,
int compression_level, bool send_path,
char* const* skip_suffixes, int skip_count,
int compression_threads, bool send_xattrs) {
/* The caller has already decided this file compresses; the skip list is part
of the public signature for symmetry with the buffered path. */
(void)skip_suffixes;
(void)skip_count;
if (!file || !file->path || !file->data || file->data->size == 0 || file->data->data != NULL)
return false;
CompressionAlgo algo = compression_get_algo();
CompressionStreamCompressor* compressor =
compression_stream_compressor_create(algo, compression_level, compression_threads);
if (!compressor) {
log_message(LOG_LEVEL_ERROR, "streaming compression is not available for this codec; "
"the source was not loaded for the buffered path");
return false;
}
int src = file_open_for_read(file->path);
if (src < 0) {
compression_stream_compressor_destroy(compressor);
return false;
}
struct stat src_st;
if (fstat(src, &src_st) != 0 || !S_ISREG(src_st.st_mode) ||
(unsigned long long)src_st.st_size < file->data->size) {
close(src);
compression_stream_compressor_destroy(compressor);
return false;
}
const char* tmpdir = getenv("TMPDIR");
if (!tmpdir || tmpdir[0] == '\0')
tmpdir = "/tmp";
size_t tmplen = strlen(tmpdir) + strlen("/fastsync-z-XXXXXX") + 1;
char* tmpl = malloc(tmplen);
if (!tmpl) {
close(src);
compression_stream_compressor_destroy(compressor);
return false;
}
snprintf(tmpl, tmplen, "%s/fastsync-z-XXXXXX", tmpdir);
int tmp_fd = mkstemp(tmpl);
if (tmp_fd < 0) {
log_perror("Could not create compression temp file");
free(tmpl);
close(src);
compression_stream_compressor_destroy(compressor);
return false;
}
unlink(tmpl);
free(tmpl);
bool ok = compression_stream_compressor_begin(compressor, file->data->size, tmp_fd);
unsigned char buf[64 * 1024];
unsigned long long remaining = file->data->size;
while (ok && remaining > 0) {
size_t want = remaining < sizeof(buf) ? (size_t)remaining : sizeof(buf);
ssize_t got = read(src, buf, want);
if (got <= 0) {
ok = false;
break;
}
if (!compression_stream_compressor_feed(compressor, buf, (size_t)got, tmp_fd))
ok = false;
remaining -= (unsigned long long)got;
}
if (ok)
ok = compression_stream_compressor_finish(compressor, tmp_fd);
close(src);
compression_stream_compressor_destroy(compressor);
if (!ok) {
close(tmp_fd);
return false;
}
struct stat tmp_st;
if (fstat(tmp_fd, &tmp_st) != 0 || tmp_st.st_size < 0) {
close(tmp_fd);
return false;
}
unsigned long long compressed_size = (unsigned long long)tmp_st.st_size;
if (lseek(tmp_fd, 0, SEEK_SET) == (off_t)-1) {
close(tmp_fd);
return false;
}
ok = true;
if (send_path && !send_wire_str(file_descriptor, file_wire_path(file)))
ok = false;
if (ok && use_metadata && !metadata_send(file_descriptor, file->metadata))
ok = false;
if (ok && send_xattrs && !xattr_send(file_descriptor, file ? file->xattrs : NULL))
ok = false;
if (ok && !send_n_data(file_descriptor, &compressed_size, sizeof(compressed_size)))
ok = false;
unsigned long long left = compressed_size;
while (ok && left > 0) {
size_t want = left < sizeof(buf) ? (size_t)left : sizeof(buf);
ssize_t got = read(tmp_fd, buf, want);
if (got <= 0 || !send_n_data(file_descriptor, buf, (size_t)got)) {
ok = false;
break;
}
left -= (unsigned long long)got;
}
close(tmp_fd);
return ok;
}
bool file_send_single_calls_with_skip(File* file, int file_descriptor, bool use_metadata,
int compression_level, bool send_path,
char* const* skip_suffixes, int skip_count,
int compression_threads, bool send_xattrs) {
if (!file || !file->path || !file->data || (file->data->size != 0 && !file->data->data))
if (!file || !file->path || !file->data)
return false;
/* A source too large to load is streamed: compression streams through a
* temp file, no compression must have taken the sendfile path instead. */
if (file->data->size != 0 && file->data->data == NULL) {
if (compression_level <= 0 ||
compression_should_skip_with_suffixes(file->path, skip_suffixes, skip_count))
return false;
return file_send_compressed_stream_with_skip(file, file_descriptor, use_metadata,
compression_level, send_path, skip_suffixes,
skip_count, compression_threads, send_xattrs);
}
const Data* data_to_send = file->data;
Data* compressed_data = NULL;
if (compression_level > 0 &&
+6
View File
@@ -13,6 +13,12 @@ bool file_send_single_calls_with_skip(File* file, int file_descriptor, bool use_
int compression_level, bool send_path,
char* const* skip_suffixes, int skip_count,
int compression_threads, bool send_xattrs);
/* Stream-compress an unloaded whole source file into a temp file and send it as
* the usual data frame (see file_send.c). */
bool file_send_compressed_stream_with_skip(File* file, int file_descriptor, bool use_metadata,
int compression_level, bool send_path,
char* const* skip_suffixes, int skip_count,
int compression_threads, bool send_xattrs);
bool file_send_sendfile(File* file, int file_descriptor, bool use_metadata, int compression_level,
bool send_path);
bool file_send_sendfile_with_skip(File* file, int file_descriptor, bool use_metadata,
+5
View File
@@ -62,6 +62,11 @@ typedef struct {
* This lets a basis larger than any whole-file bound materialize without
* buffering it; the source metadata on `metadata` is applied afterwards. */
char* basis_copy;
/* Receiver-only. When set, `basis_copy` points at a receiver-created spool
* temp file holding a STREAMED whole-file payload (rather than a --copy-dest
* basis). file_destroy unlinks it after the install consumes it, so an
* over-limit file leaves no scratch behind. */
bool data_spool;
/* --hard-links (-H), sender + receiver wire state. link_group is a run-local
* id shared by every member of one source inode (0 = not part of a group).
* The FIRST member (link_first == true) carries its data on the wire and is
+337 -94
View File
@@ -21,6 +21,28 @@ void filter_rule_free(FilterRule* rule) {
free(rule);
}
FilterRule* filter_rule_clone(const FilterRule* rule) {
if (!rule)
return NULL;
FilterRule* copy = calloc(1, sizeof(FilterRule));
if (!copy)
return NULL;
copy->action = rule->action;
copy->sides = rule->sides;
copy->anchored = rule->anchored;
copy->dir_only = rule->dir_only;
copy->negate = rule->negate;
copy->perishable = rule->perishable;
copy->no_inherit = rule->no_inherit;
copy->owner = str_dup(rule->owner ? rule->owner : "");
copy->pattern = str_dup(rule->pattern ? rule->pattern : "");
if (!copy->owner || !copy->pattern) {
filter_rule_free(copy);
return NULL;
}
return copy;
}
FilterRuleList* filter_rule_list_create(void) {
return calloc(1, sizeof(FilterRuleList));
}
@@ -48,37 +70,94 @@ void filter_rule_list_free(FilterRuleList* list) {
for (int i = 0; i < list->count; i++)
filter_rule_free(list->items[i]);
for (int i = 0; i < list->dir_merge_count; i++)
free(list->dir_merge_names[i]);
free(list->dir_merge_names);
free(list->dir_merges[i].name);
free(list->dir_merges);
free(list->items);
free(list);
}
/* Append an implicit exclude rule for the merge file itself (rsync's 'e'
* modifier). The rule is owned by the transfer root and matches the basename
* anywhere, exactly like rsync's EXCLUDE_SELF (a dual-sided exclude: it hides
* the file and protects its destination mirror from --delete). */
static bool filter_list_add_exclude_self(FilterRuleList* list, const char* name) {
const char* base = strrchr(name, '/');
base = base ? base + 1 : name;
if (base[0] == '\0')
return true;
FilterRule* rule = calloc(1, sizeof(FilterRule));
if (!rule)
return false;
rule->action = FILTER_ACTION_EXCLUDE;
rule->sides = FILTER_SIDE_SENDER | FILTER_SIDE_RECEIVER;
rule->pattern = str_dup(base);
if (!rule->pattern || !filter_rule_set_owner(rule, "")) {
filter_rule_free(rule);
return false;
}
if (!filter_rule_list_add(list, rule)) {
filter_rule_free(rule);
return false;
}
return true;
}
/* Register a per-directory merge-file basename (for "dir-merge NAME"/": NAME"
* and -F's .rsync-filter). Duplicate names are ignored. */
bool filter_rule_list_add_dir_merge(FilterRuleList* list, const char* name) {
return filter_rule_list_add_dir_merge_ex(list, name, false, false, false, false, false);
}
bool filter_rule_list_add_dir_merge_ex(FilterRuleList* list, const char* name, bool no_prefixes,
bool include, bool word_split, bool no_inherit,
bool exclude_self) {
if (!list || !name || name[0] == '\0')
return false;
for (int i = 0; i < list->dir_merge_count; i++) {
if (strcmp(list->dir_merge_names[i], name) == 0)
if (strcmp(list->dir_merges[i].name, name) == 0) {
/* rsync keeps the first registration (first-wins), but the 'e' modifier
is a list side effect, not a registration field: honor it on the
duplicate path too, adding the implicit exclude-self rule at most
once. */
if (exclude_self && !list->dir_merges[i].exclude_self) {
if (!filter_list_add_exclude_self(list, name))
return false;
list->dir_merges[i].exclude_self = true;
}
return true;
}
}
if (list->dir_merge_count == list->dir_merge_capacity) {
if (list->dir_merge_capacity > INT_MAX / 2)
return false;
int new_cap = list->dir_merge_capacity > 0 ? list->dir_merge_capacity * 2 : 4;
char** grown = realloc(list->dir_merge_names, (size_t)new_cap * sizeof(char*));
FilterDirMerge* grown = realloc(list->dir_merges, (size_t)new_cap * sizeof(*grown));
if (!grown)
return false;
list->dir_merge_names = grown;
list->dir_merges = grown;
list->dir_merge_capacity = new_cap;
}
char* dup = str_dup(name);
if (!dup)
return false;
list->dir_merge_names[list->dir_merge_count++] = dup;
if (exclude_self && !filter_list_add_exclude_self(list, name)) {
free(dup);
return false;
}
FilterDirMerge* entry = &list->dir_merges[list->dir_merge_count];
entry->name = dup;
entry->no_prefixes = no_prefixes;
entry->include = include;
entry->word_split = word_split;
entry->no_inherit = no_inherit;
entry->exclude_self = exclude_self;
list->dir_merge_count++;
return true;
}
static bool set_rule_owner(FilterRule* rule, const char* owner) {
bool filter_rule_set_owner(FilterRule* rule, const char* owner) {
if (!rule)
return false;
char* dup = str_dup(owner ? owner : "");
if (!dup)
return false;
@@ -177,10 +256,11 @@ static bool is_unsupported_modifier_char(char c) {
return c == 'e' || c == 'n' || c == 'w';
}
/* Merge-file modifiers rsync accepts on merge/dir-merge rules: 'e', 'n', 'w'
* and '-' (do not transfer the merge file). */
/* Merge-file modifiers rsync accepts on merge/dir-merge rules: 'e' (exclude
* self), 'n' (no inherit), 'w' (word split), '-' (bare excludes) and '+'
* (bare includes). */
static bool is_merge_modifier_char(char c) {
return c == 'e' || c == 'n' || c == 'w' || c == '-';
return c == 'e' || c == 'n' || c == 'w' || c == '-' || c == '+';
}
/* Characters that count as part of a modifier run for `kind` when deciding
@@ -228,8 +308,10 @@ static char unsupported_modifier_in_token(const char* tok, RuleKind kind) {
* generic syntax error so callers can emit a precise diagnostic. */
static bool parse_rule_syntax(const char* text, RuleKind* kind, unsigned* sides,
bool* sides_explicit, bool* negate, bool* anchored_mod,
bool* perishable, bool* xattr, bool* cvs_inject,
const char** pat_start, size_t* pat_len, char* bad_mod) {
bool* perishable, bool* xattr, bool* cvs_inject, bool* no_prefixes,
bool* include_defaults, bool* word_split, bool* no_inherit,
bool* exclude_self, const char** pat_start, size_t* pat_len,
char* bad_mod) {
const char* p = text;
*sides = FILTER_SIDE_SENDER | FILTER_SIDE_RECEIVER;
*sides_explicit = false;
@@ -238,6 +320,11 @@ static bool parse_rule_syntax(const char* text, RuleKind* kind, unsigned* sides,
*perishable = false;
*xattr = false;
*cvs_inject = false;
*no_prefixes = false;
*include_defaults = false;
*word_split = false;
*no_inherit = false;
*exclude_self = false;
*pat_start = NULL;
*pat_len = 0;
*bad_mod = '\0';
@@ -312,6 +399,21 @@ static bool parse_rule_syntax(const char* text, RuleKind* kind, unsigned* sides,
case 'C':
*cvs_inject = true;
break;
case '-':
*no_prefixes = true;
break;
case '+':
*include_defaults = true;
break;
case 'e':
*exclude_self = true;
break;
case 'n':
*no_inherit = true;
break;
case 'w':
*word_split = true;
break;
default:
break;
}
@@ -347,11 +449,13 @@ FilterRule* filter_rule_parse(const char* line, const FilterParseOptions* opts,
RuleKind kind = RULE_KIND_UNKNOWN;
unsigned sides;
bool sides_explicit, negate, anchored_mod, perishable, xattr, cvs_inject;
bool no_prefixes, include_defaults, word_split, no_inherit, exclude_self;
const char* pat;
size_t pat_len;
char bad_mod;
if (!parse_rule_syntax(p, &kind, &sides, &sides_explicit, &negate, &anchored_mod, &perishable,
&xattr, &cvs_inject, &pat, &pat_len, &bad_mod)) {
&xattr, &cvs_inject, &no_prefixes, &include_defaults, &word_split,
&no_inherit, &exclude_self, &pat, &pat_len, &bad_mod)) {
if (bad_mod != '\0')
filter_set_error(err, err_size, "unsupported filter modifier '%c'", bad_mod);
else
@@ -504,7 +608,7 @@ static bool filter_list_append_cvs(FilterRuleList* list, unsigned sides) {
}
memcpy(rule->pattern, CVS_DEFAULTS[i].pattern, plen);
rule->pattern[plen] = '\0';
if (!set_rule_owner(rule, "")) {
if (!filter_rule_set_owner(rule, "")) {
filter_rule_free(rule);
return false;
}
@@ -522,16 +626,138 @@ static bool filter_list_parse_append_depth(FilterRuleList* list, const char* lin
const FilterParseOptions* opts, const char* base_dir,
int depth, char* err, size_t err_size);
/* Read a merge file and splice its rules into `list`. A relative path is
* resolved below `base_dir` when given, else used as-is (rsync resolves a
* command-line merge file relative to the current directory). */
/* Append one merge-file token/line to `list`, honoring the merge rule's
* no-prefix/include mode. In no-prefix mode the token is a bare pattern whose
* include/exclude default comes from the merge rule (rsync's "-"/"+" merge
* modifiers); otherwise the token is parsed as a full filter rule. */
static bool filter_merge_append_token(FilterRuleList* list, const char* token,
const FilterDirMerge* spec, const FilterParseOptions* opts,
const char* base_dir, int depth, char* err, size_t err_size) {
if (spec->no_prefixes || spec->include) {
size_t tlen = strlen(token);
char* text = malloc(tlen + 3);
if (!text) {
filter_set_error(err, err_size, "memory allocation failed");
return false;
}
text[0] = spec->include ? '+' : '-';
text[1] = ' ';
memcpy(text + 2, token, tlen + 1);
bool ok = filter_list_parse_append_depth(list, text, opts, base_dir, depth + 1, err, err_size);
free(text);
return ok;
}
return filter_list_parse_append_depth(list, token, opts, base_dir, depth + 1, err, err_size);
}
/* Read a merge file's tokens/lines into `list` for `spec`. A `w` merge rule
* word-splits on whitespace (turning comments off); otherwise lines are parsed
* and whole-line `#` comments skipped. When `owner_rel` is non-NULL the newly
* added rules are owned by that directory; a no-inherit spec marks them so they
* apply only there. Returns false on parse/allocation failure. */
static bool filter_merge_read(FilterRuleList* list, FILE* fp, const char* display_path,
const FilterDirMerge* spec, const FilterParseOptions* opts,
const char* base_dir, const char* owner_rel, int depth, char* err,
size_t err_size) {
/* Lowest list index this read is responsible for. A "clear"/"!" inside the
* file resets list->count to 0 (freeing the caller's earlier rules too), so
* the base must follow it down: otherwise post-clear rules sit below the
* original count and never receive an owner (nor no-inherit) and are missed
* by the rollback. */
int floor = list->count;
char* line = NULL;
size_t cap = 0;
bool ok = true;
while (ok) {
ssize_t n = utils_getdelim_bounded(fp, &line, &cap, '\n', UTILS_MAX_LINE_LEN);
if (n < 0) {
if (errno == EFBIG)
filter_set_error(err, err_size, "line in %s exceeds %d bytes", display_path,
(int)UTILS_MAX_LINE_LEN);
else
filter_set_error(err, err_size, "error reading %s: %s", display_path, strerror(errno));
ok = false;
break;
}
if (n == 0)
break;
if (spec->word_split) {
/* Whitespace-separated tokens; newlines are ordinary separators and
* comments are disabled. */
const char* s = line;
while (*s) {
while (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r')
s++;
if (*s == '\0')
break;
const char* start = s;
while (*s != '\0' && *s != ' ' && *s != '\t' && *s != '\n' && *s != '\r')
s++;
size_t tlen = (size_t)(s - start);
char* token = malloc(tlen + 1);
if (!token) {
filter_set_error(err, err_size, "memory allocation failed");
ok = false;
break;
}
memcpy(token, start, tlen);
token[tlen] = '\0';
if (!filter_merge_append_token(list, token, spec, opts, base_dir, depth, err, err_size))
ok = false;
if (list->count < floor)
floor = list->count; /* a "clear" reset the list below this read's base */
free(token);
}
} else {
const char* lp = line;
while (*lp == ' ' || *lp == '\t')
lp++;
if (*lp == '\0' || *lp == '\n' || *lp == '\r' || *lp == '#')
continue;
if (!filter_merge_append_token(list, lp, spec, opts, base_dir, depth, err, err_size))
ok = false;
if (list->count < floor)
floor = list->count; /* a "clear" reset the list below this read's base */
}
}
free(line);
if (!ok) {
/* Drop every live rule this read is responsible for. After a "clear" that
* base is 0, so the post-clear rules are freed too instead of leaking. */
for (int i = floor; i < list->count; i++)
filter_rule_free(list->items[i]);
list->count = floor;
return false;
}
for (int i = floor; i < list->count; i++) {
FilterRule* rule = list->items[i];
if (spec->no_inherit)
rule->no_inherit = true;
if (owner_rel && !filter_rule_set_owner(rule, owner_rel)) {
filter_set_error(err, err_size, "memory allocation failed");
for (int j = floor; j < list->count; j++)
filter_rule_free(list->items[j]);
list->count = floor;
return false;
}
}
return true;
}
/* Read a single-instance merge file and splice its rules into `list`. A
* relative path is resolved below `base_dir` when given, else used as-is (rsync
* resolves a command-line merge file relative to the current directory). */
static bool filter_list_merge_file(FilterRuleList* list, const char* name,
const FilterParseOptions* opts, const char* base_dir, int depth,
char* err, size_t err_size) {
const FilterDirMerge* spec, const FilterParseOptions* opts,
const char* base_dir, int depth, char* err, size_t err_size) {
if (name[0] == '\0') {
filter_set_error(err, err_size, "merge requires a filename");
return false;
}
if (spec->exclude_self && !filter_list_add_exclude_self(list, name)) {
filter_set_error(err, err_size, "memory allocation failed");
return false;
}
char* path =
(base_dir && base_dir[0] && name[0] != '/') ? path_cat(base_dir, name) : str_dup(name);
if (!path) {
@@ -544,29 +770,7 @@ static bool filter_list_merge_file(FilterRuleList* list, const char* name,
free(path);
return false;
}
char* line = NULL;
size_t cap = 0;
bool ok = true;
while (true) {
ssize_t n = utils_getdelim_bounded(fp, &line, &cap, '\n', UTILS_MAX_LINE_LEN);
if (n < 0) {
filter_set_error(err, err_size, "error reading merge file '%s'", path);
ok = false;
break;
}
if (n == 0)
break;
const char* lp = line;
while (*lp == ' ' || *lp == '\t')
lp++;
if (*lp == '\0' || *lp == '\n' || *lp == '\r' || *lp == '#')
continue;
if (!filter_list_parse_append_depth(list, lp, opts, base_dir, depth + 1, err, err_size)) {
ok = false;
break;
}
}
free(line);
bool ok = filter_merge_read(list, fp, path, spec, opts, base_dir, NULL, depth, err, err_size);
fclose(fp);
free(path);
return ok;
@@ -590,11 +794,13 @@ static bool filter_list_parse_append_depth(FilterRuleList* list, const char* lin
RuleKind kind = RULE_KIND_UNKNOWN;
unsigned sides;
bool sides_explicit, negate, anchored_mod, perishable, xattr, cvs_inject;
bool no_prefixes, include_defaults, word_split, no_inherit, exclude_self;
const char* pat;
size_t pat_len;
char bad_mod;
if (!parse_rule_syntax(p, &kind, &sides, &sides_explicit, &negate, &anchored_mod, &perishable,
&xattr, &cvs_inject, &pat, &pat_len, &bad_mod)) {
&xattr, &cvs_inject, &no_prefixes, &include_defaults, &word_split,
&no_inherit, &exclude_self, &pat, &pat_len, &bad_mod)) {
if (bad_mod != '\0')
filter_set_error(err, err_size, "unsupported filter modifier '%c': %s", bad_mod, p);
else
@@ -640,7 +846,15 @@ static bool filter_list_parse_append_depth(FilterRuleList* list, const char* lin
}
memcpy(name, pat, pat_len);
name[pat_len] = '\0';
bool ok = filter_list_merge_file(list, name, opts, base_dir, depth, err, err_size);
/* A single-instance merge has no inheritance, so 'n' is meaningless; the
* other merge modifiers still shape how the file is read. */
FilterDirMerge spec = {.name = name,
.no_prefixes = no_prefixes,
.include = include_defaults,
.word_split = word_split,
.no_inherit = false,
.exclude_self = exclude_self};
bool ok = filter_list_merge_file(list, name, &spec, opts, base_dir, depth, err, err_size);
free(name);
return ok;
}
@@ -656,7 +870,8 @@ static bool filter_list_parse_append_depth(FilterRuleList* list, const char* lin
}
memcpy(name, pat, pat_len);
name[pat_len] = '\0';
bool ok = filter_rule_list_add_dir_merge(list, name);
bool ok = filter_rule_list_add_dir_merge_ex(list, name, no_prefixes, include_defaults,
word_split, no_inherit, exclude_self);
free(name);
if (!ok) {
filter_set_error(err, err_size, "memory allocation failed");
@@ -717,27 +932,30 @@ FilterRuleList* filter_base_build(const char* const* rule_texts, int rule_count,
/* Undo the rules and dir-merge registrations that one merge file appended,
* leaving the caller's earlier content intact. A "clear" rule inside the file
* frees every rule, including the caller's; clamp to the surviving count so
* those already-freed rules are never resurrected and freed a second time. */
* those already-freed rules are never resurrected and freed a second time.
* (filter_merge_read() has already rolled its own range back by the time this
* runs, so on a post-clear failure `list->count` is below `rules_before` and
* this is a no-op for the rules.) */
static void filter_file_rollback(FilterRuleList* list, int rules_before, int dir_merges_before) {
int first = rules_before < list->count ? rules_before : list->count;
for (int i = first; i < list->count; i++)
filter_rule_free(list->items[i]);
list->count = first;
for (int i = dir_merges_before; i < list->dir_merge_count; i++)
free(list->dir_merge_names[i]);
free(list->dir_merges[i].name);
list->dir_merge_count = dir_merges_before;
}
bool filter_file_append(FilterRuleList* list, const char* dir_path, const char* name,
const char* owner_rel, const FilterParseOptions* opts, bool* exists,
char* err, size_t err_size) {
bool filter_dir_merge_append(FilterRuleList* list, const char* dir_path, const FilterDirMerge* spec,
const char* owner_rel, const FilterParseOptions* opts, bool* exists,
char* err, size_t err_size) {
if (err && err_size > 0)
err[0] = '\0';
if (exists)
*exists = false;
if (!list)
if (!list || !spec || !spec->name)
return false;
char* filter_path = path_cat(dir_path, name);
char* filter_path = path_cat(dir_path, spec->name);
if (!filter_path) {
filter_set_error(err, err_size, "memory allocation failed");
return false;
@@ -748,7 +966,7 @@ bool filter_file_append(FilterRuleList* list, const char* dir_path, const char*
if (errno == ENOENT || errno == ENOTDIR)
return true;
char* escaped_dir = output_escape(dir_path, log_get_8_bit_output());
log_message(LOG_LEVEL_WARNING, "Could not read %s in %s: %s", name,
log_message(LOG_LEVEL_WARNING, "Could not read %s in %s: %s", spec->name,
escaped_dir ? escaped_dir : "<allocation failed>", strerror(errno));
free(escaped_dir);
return true;
@@ -757,51 +975,25 @@ bool filter_file_append(FilterRuleList* list, const char* dir_path, const char*
*exists = true;
int rules_before = list->count;
int dir_merges_before = list->dir_merge_count;
char* line = NULL;
size_t line_cap = 0;
bool ok = true;
while (true) {
ssize_t n = utils_getdelim_bounded(fp, &line, &line_cap, '\n', UTILS_MAX_LINE_LEN);
if (n < 0) {
if (errno == EFBIG) {
filter_set_error(err, err_size, "line in %s exceeds %d bytes", name,
(int)UTILS_MAX_LINE_LEN);
} else {
filter_set_error(err, err_size, "error reading %s: %s", name, strerror(errno));
}
ok = false;
break;
}
if (n == 0)
break;
const char* p = line;
while (*p == ' ' || *p == '\t')
p++;
if (*p == '\0' || *p == '\n' || *p == '\r' || *p == '#')
continue;
/* Merge files inside a per-directory file resolve relative to that
directory. */
if (!filter_list_parse_append_depth(list, p, opts, dir_path, 0, err, err_size)) {
ok = false;
break;
}
}
free(line);
/* Merge files inside a per-directory file resolve relative to that
directory. */
bool ok =
filter_merge_read(list, fp, spec->name, spec, opts, dir_path, owner_rel, 0, err, err_size);
fclose(fp);
if (!ok) {
filter_file_rollback(list, rules_before, dir_merges_before);
return false;
}
for (int i = rules_before; i < list->count; i++) {
if (!set_rule_owner(list->items[i], owner_rel)) {
filter_set_error(err, err_size, "memory allocation failed");
filter_file_rollback(list, rules_before, dir_merges_before);
return false;
}
}
return true;
}
bool filter_file_append(FilterRuleList* list, const char* dir_path, const char* name,
const char* owner_rel, const FilterParseOptions* opts, bool* exists,
char* err, size_t err_size) {
FilterDirMerge spec = {.name = (char*)name};
return filter_dir_merge_append(list, dir_path, &spec, owner_rel, opts, exists, err, err_size);
}
FilterRuleList* filter_file_read_named(const char* dir_path, const char* name,
const char* owner_rel, const FilterParseOptions* opts,
bool* exists, char* err, size_t err_size) {
@@ -843,11 +1035,16 @@ static FilterAction rule_matches(const FilterRule* rule, const char* rel_path, c
return FILTER_ACTION_NONE;
if (!(rule->sides & side))
return FILTER_ACTION_NONE;
/* A rule applies only to entries below its owner directory. */
/* A rule applies only to entries below its owner directory. The receive
* root's destination-relative coordinate may be written as "." (the
* synced-directory sentinel), which is the same scope as the empty owner. */
const char* owner = rule->owner;
if (owner && strcmp(owner, ".") == 0)
owner = "";
const char* rel2 = rel_path;
if (rule->owner && rule->owner[0] != '\0') {
size_t owner_len = strlen(rule->owner);
if (strncmp(rule->owner, rel_path, owner_len) != 0)
if (owner && owner[0] != '\0') {
size_t owner_len = strlen(owner);
if (strncmp(owner, rel_path, owner_len) != 0)
return FILTER_ACTION_NONE;
if (rel_path[owner_len] != '/')
return FILTER_ACTION_NONE;
@@ -855,6 +1052,10 @@ static FilterAction rule_matches(const FilterRule* rule, const char* rel_path, c
}
if (rel2[0] == '\0')
return FILTER_ACTION_NONE;
/* A no-inherit rule ('n' on its dir-merge) applies only to direct children of
* its owner directory, never to deeper entries. */
if (rule->no_inherit && strchr(rel2, '/') != NULL)
return FILTER_ACTION_NONE;
bool matched;
if (rule->dir_only && !is_dir)
matched = false;
@@ -884,3 +1085,45 @@ FilterAction filter_rules_apply_side(const FilterRuleList* list, const char* rel
}
return FILTER_ACTION_NONE;
}
FilterAction filter_dir_rules_apply_side(const FilterRuleList* dir_rules, const char* rel_path,
const char* leaf, bool is_dir) {
if (!dir_rules || !rel_path)
return FILTER_ACTION_NONE;
size_t len = strlen(rel_path);
if (len == 0)
return FILTER_ACTION_NONE;
/* The containing directory of rel_path is the prefix before its final '/'. */
size_t owner_len = 0;
for (size_t i = 0; i < len; i++) {
if (rel_path[i] == '/')
owner_len = i;
}
for (;;) {
for (int i = 0; i < dir_rules->count; i++) {
const FilterRule* rule = dir_rules->items[i];
const char* rule_owner = rule && rule->owner ? rule->owner : "";
/* "." is the receive root's coordinate (see rule_matches). */
if (strcmp(rule_owner, ".") == 0)
rule_owner = "";
size_t rule_owner_len = strlen(rule_owner);
if (rule_owner_len != owner_len)
continue;
if (owner_len != 0 && memcmp(rule_owner, rel_path, owner_len) != 0)
continue;
FilterAction action = rule_matches(rule, rel_path, leaf, is_dir, FILTER_SIDE_RECEIVER);
if (action != FILTER_ACTION_NONE)
return action;
}
if (owner_len == 0)
break;
/* Move to the parent directory: the last '/' before owner_len. */
size_t parent = 0;
for (size_t j = 0; j < owner_len; j++) {
if (rel_path[j] == '/')
parent = j;
}
owner_len = parent;
}
return FILTER_ACTION_NONE;
}
+61 -11
View File
@@ -25,11 +25,12 @@
* Modifiers: '/' absolute anchor, '!' negate match, 'C' inject CVS defaults,
* 's' sender side, 'r' receiver side, 'p' perishable. The rsync 'x'
* (xattr-name) modifier is not implemented and is rejected explicitly
* everywhere. The merge-file modifiers 'e' (exclude the merge file itself),
* 'n' (do not inherit the merge file), 'w' (word-split the merge file) and '-'
* (do not transfer the merge file) are accepted and consumed only on merge/
* dir-merge rules (rejected on every other rule, matching rsync); their
* semantics are not implemented and they are otherwise ignored.
* everywhere. The merge-only modifiers are accepted only on merge/dir-merge
* rules (rejected on every other rule, matching rsync): 'e' excludes the merge
* file itself, 'n' makes the merged rules non-inheriting (they apply only to
* the directory that holds the merge file), 'w' word-splits the merge file on
* whitespace instead of lines, and '-' reads the merge file as a list of bare
* exclude patterns with no rule prefixes.
* A trailing '/' makes a pattern match directories only. A leading '/' anchors
* the pattern to its owner directory.
*/
@@ -55,18 +56,32 @@ typedef struct {
bool dir_only; /* pattern had a trailing '/': matches directories only */
bool negate; /* '!' modifier: match succeeds when the pattern does not */
bool perishable; /* 'p' modifier (ignored in deleted directories) */
bool no_inherit; /* 'n' on the owning dir-merge: applies only in `owner` */
char* owner; /* owning directory rel path ("" == transfer root) */
char* pattern; /* cleaned glob pattern (no leading '/', no trailing '/') */
} FilterRule;
/* One per-directory merge-file registration ("dir-merge NAME"/": NAME", "merge
* NAME"/". NAME" and -F's .rsync-filter) together with the merge-only modifiers
* parsed from the rule. The scanner reads `name` in every directory it
* traverses and applies `no_prefixes`/`include`/`word_split`/`no_inherit`/
* `exclude_self` while merging the file's rules. */
typedef struct {
char* name;
bool no_prefixes; /* '-' : file holds only bare exclude patterns */
bool include; /* '+' : file holds only bare include patterns */
bool word_split; /* 'w' : split the file on whitespace, not lines */
bool no_inherit; /* 'n' : the merged rules do not inherit below their dir */
bool exclude_self; /* 'e' : exclude the merge file itself from the transfer */
} FilterDirMerge;
typedef struct FilterRuleList {
FilterRule** items; /* owned array of rule pointers */
int count;
int capacity;
/* Per-directory merge-file basenames registered by "dir-merge NAME"/": NAME"
* or by -F (.rsync-filter). Owned strings; the scanner reads each name in
* every directory it traverses. */
char** dir_merge_names;
/* Per-directory merge-file registrations. Owned; the scanner reads each
* name in every directory it traverses. */
FilterDirMerge* dir_merges;
int dir_merge_count;
int dir_merge_capacity;
} FilterRuleList;
@@ -83,13 +98,28 @@ typedef struct {
FilterRule* filter_rule_parse(const char* line, const FilterParseOptions* opts, char* err,
size_t err_size);
void filter_rule_free(FilterRule* rule);
/* Deep-copy a rule (owned pattern/owner). Returns NULL on allocation failure. */
FilterRule* filter_rule_clone(const FilterRule* rule);
/* Replace a rule's owner directory (owned copy of `owner`, "" for the transfer
* root). Returns false on allocation failure, leaving the rule unchanged.
* Used to re-express a mirrored per-directory rule in the receiver's
* destination-relative coordinate system. */
bool filter_rule_set_owner(FilterRule* rule, const char* owner);
FilterRuleList* filter_rule_list_create(void);
/* Append a fully-parsed rule (takes ownership). Returns false on OOM. */
bool filter_rule_list_add(FilterRuleList* list, FilterRule* rule);
/* Register a per-directory merge-file basename (idempotent). Returns false on
* OOM. Used by the scanner to read custom "dir-merge" files. */
/* Register a per-directory merge-file basename (idempotent, no modifiers).
* Returns false on OOM. Used by the scanner to read custom "dir-merge" files. */
bool filter_rule_list_add_dir_merge(FilterRuleList* list, const char* name);
/* Register a per-directory merge file with its merge-only modifiers. On a
* duplicate name the existing registration is kept (rsync's first wins) and true
* is returned. When `exclude_self` is set an implicit exclude rule for the
* merge file's basename is appended to the list at this position, matching
* rsync's `e` modifier. Returns false on OOM. */
bool filter_rule_list_add_dir_merge_ex(FilterRuleList* list, const char* name, bool no_prefixes,
bool include, bool word_split, bool no_inherit,
bool exclude_self);
/* Parse `line` and append it. Handles "clear"/"!" (resets the list), "merge
* FILE"/". FILE" (splices the file's rules) and "dir-merge NAME"/": NAME"
* (registers a per-directory filename). Returns false and fills `err` on bad
@@ -122,6 +152,15 @@ bool filter_file_append(FilterRuleList* list, const char* dir_path, const char*
const char* owner_rel, const FilterParseOptions* opts, bool* exists,
char* err, size_t err_size);
/* Append a per-directory merge file honoring its merge-only modifiers: `-`
* reads every (word-split, when `w`) token as a bare exclude, `+` as a bare
* include, and `n` marks each read rule non-inheriting. A plain name behaves
* like filter_file_append. A missing file yields *exists=false with no error;
* returns false only on a parse/allocation failure (message in `err`). */
bool filter_dir_merge_append(FilterRuleList* list, const char* dir_path, const FilterDirMerge* spec,
const char* owner_rel, const FilterParseOptions* opts, bool* exists,
char* err, size_t err_size);
/* filter_file_read_named with the default ".rsync-filter" name. */
FilterRuleList* filter_file_read(const char* dir_path, const char* owner_rel, bool* exists,
char* err, size_t err_size);
@@ -135,4 +174,15 @@ FilterRuleList* filter_file_read(const char* dir_path, const char* owner_rel, bo
FilterAction filter_rules_apply_side(const FilterRuleList* list, const char* rel_path,
const char* leaf, bool is_dir, unsigned side);
/* Evaluate a received per-directory rule set for the receiver side, using
* rsync's per-directory-before-ancestors order: the entry's containing
* directory's rules are tried first, then each ancestor's, then the receive
* root's. `dir_rules` is a flat list whose rules carry `owner`; a rule applies
* only when `owner` is exactly the directory being examined (a no-inherit rule
* therefore applies only to that directory's direct children). Returns the
* first matching rule's receiver verdict (PROTECT/RISK) or FILTER_ACTION_NONE.
* The caller evaluates the command-line base rules after this chain. */
FilterAction filter_dir_rules_apply_side(const FilterRuleList* dir_rules, const char* rel_path,
const char* leaf, bool is_dir);
#endif
+16 -6
View File
@@ -62,14 +62,16 @@ bool format_dest_state_send(int fd, const OutputDestState* state) {
if (!state)
return false;
int32_t has_old = state->existed ? 1 : 0;
int32_t target_matches = state->target_matches ? 1 : 0;
uint64_t size = (uint64_t)state->size;
int64_t mtime = (int64_t)state->mtime_sec;
int64_t mtime_nsec = state->mtime_nsec;
uint32_t mode = state->mode;
int32_t uid = state->uid;
int32_t gid = state->gid;
return send_n_data(fd, &has_old, sizeof(has_old)) && send_n_data(fd, &size, sizeof(size)) &&
send_n_data(fd, &mtime, sizeof(mtime)) &&
return send_n_data(fd, &has_old, sizeof(has_old)) &&
send_n_data(fd, &target_matches, sizeof(target_matches)) &&
send_n_data(fd, &size, sizeof(size)) && send_n_data(fd, &mtime, sizeof(mtime)) &&
send_n_data(fd, &mtime_nsec, sizeof(mtime_nsec)) && send_n_data(fd, &mode, sizeof(mode)) &&
send_n_data(fd, &uid, sizeof(uid)) && send_n_data(fd, &gid, sizeof(gid));
}
@@ -78,14 +80,16 @@ bool format_dest_state_receive(int fd, OutputDestState* state) {
if (!state)
return false;
int32_t has_old = 0;
int32_t target_matches = 0;
uint64_t size = 0;
int64_t mtime = 0;
int64_t mtime_nsec = 0;
uint32_t mode = 0;
int32_t uid = 0;
int32_t gid = 0;
if (!receive_n_data(fd, &has_old, sizeof(has_old)) || !receive_n_data(fd, &size, sizeof(size)) ||
!receive_n_data(fd, &mtime, sizeof(mtime)) ||
if (!receive_n_data(fd, &has_old, sizeof(has_old)) ||
!receive_n_data(fd, &target_matches, sizeof(target_matches)) ||
!receive_n_data(fd, &size, sizeof(size)) || !receive_n_data(fd, &mtime, sizeof(mtime)) ||
!receive_n_data(fd, &mtime_nsec, sizeof(mtime_nsec)) ||
!receive_n_data(fd, &mode, sizeof(mode)) || !receive_n_data(fd, &uid, sizeof(uid)) ||
!receive_n_data(fd, &gid, sizeof(gid)))
@@ -93,6 +97,7 @@ bool format_dest_state_receive(int fd, OutputDestState* state) {
memset(state, 0, sizeof(*state));
state->known = true;
state->existed = has_old != 0;
state->target_matches = target_matches != 0;
state->size = size;
state->mtime_sec = mtime;
state->mtime_nsec = mtime_nsec;
@@ -105,9 +110,10 @@ bool format_dest_state_receive(int fd, OutputDestState* state) {
bool format_stats_send(int fd, const ReceiverStats* stats) {
if (!stats)
return false;
unsigned long long fields[8] = {
unsigned long long fields[12] = {
stats->matched_data, stats->deleted_files, stats->would_delete_count, stats->literal_bytes,
stats->created_reg, stats->created_dir, stats->created_link, stats->created_special,
stats->deleted_reg, stats->deleted_dir, stats->deleted_link, stats->deleted_special,
};
return send_n_data(fd, fields, sizeof(fields));
}
@@ -115,7 +121,7 @@ bool format_stats_send(int fd, const ReceiverStats* stats) {
bool format_stats_receive(int fd, ReceiverStats* stats) {
if (!stats)
return false;
unsigned long long fields[8] = {0};
unsigned long long fields[12] = {0};
if (!receive_n_data(fd, fields, sizeof(fields)))
return false;
memset(stats, 0, sizeof(*stats));
@@ -127,5 +133,9 @@ bool format_stats_receive(int fd, ReceiverStats* stats) {
stats->created_dir = fields[5];
stats->created_link = fields[6];
stats->created_special = fields[7];
stats->deleted_reg = fields[8];
stats->deleted_dir = fields[9];
stats->deleted_link = fields[10];
stats->deleted_special = fields[11];
return true;
}
+25 -6
View File
@@ -17,10 +17,18 @@
/* Pre-transfer destination snapshot, reported by the receiver when the wire
* config carries report_dest_info. `known` distinguishes "no report was
* requested/received" from "the destination did not exist" (`existed == false`
* with `known == true`). */
* with `known == true`).
*
* `target_matches` is meaningful only for a symlink destination (protocol
* 2.30.0): the receiver compares its on-disk link target with the incoming
* target and reports whether they are equal, so the sender can render rsync's
* `cLc........` (target changed) versus `.L..t......` (attributes only) and
* suppress an unchanged symlink's line entirely. It is always false for every
* other entry kind. */
typedef struct {
bool known;
bool existed;
bool target_matches;
unsigned long long size;
long long mtime_sec;
long long mtime_nsec;
@@ -57,17 +65,24 @@ bool format_dest_state_send(int fd, const OutputDestState* state);
bool format_dest_state_receive(int fd, OutputDestState* state);
/* End-of-transfer receiver counters reported through STATUS_STATS (protocol
* 2.25.0, extended in 2.28.0) when the wire config carries report_stats.
* `would_delete_count` is the number of destination-relative paths the receiver
* would have deleted in a -n/--dry-run --delete run; that many wire strings
* immediately follow the fixed record (sent/read by the caller).
* 2.25.0, extended in 2.28.0 and 2.30.0) when the wire config carries
* report_stats. `would_delete_count` is the number of destination-relative
* paths the receiver would have deleted in a -n/--dry-run --delete run; that
* many wire strings immediately follow the fixed record (sent/read by the
* caller).
*
* Protocol 2.28.0 adds the receiver-observed counters the sender cannot see:
* `literal_bytes` is the file data the receiver actually stored literally
* (whole files plus the literal fragments of a delta) and the four `created_*`
* counters split the destination entries the receiver newly created by type,
* reproducing rsync's `Number of created files` breakdown and an exact
* `Literal data` for a delta run. */
* `Literal data` for a delta run.
*
* Protocol 2.30.0 appends the four `deleted_*` counters: the same reg/dir/link/
* special split for the entries the receiver ACTUALLY removed, so `--stats` can
* render rsync's `Number of deleted files: X (reg: A, dir: B, link: C,
* special: D)` parenthetical. The scalar `deleted_files` stays the authoritative
* total (the breakdown is a strict partition of it). */
typedef struct {
unsigned long long matched_data;
unsigned long long deleted_files;
@@ -77,6 +92,10 @@ typedef struct {
unsigned long long created_dir;
unsigned long long created_link;
unsigned long long created_special;
unsigned long long deleted_reg;
unsigned long long deleted_dir;
unsigned long long deleted_link;
unsigned long long deleted_special;
} ReceiverStats;
/* Fixed-width STATUS_STATS counter record. The status frame and the optional
+1 -1
View File
@@ -276,7 +276,7 @@ static bool identity_wire_map_valid(const IdentityMap* map) {
}
if (map->to < IDENTITY_CURRENT)
return false;
if (map->to_name && strlen(map->to_name) > 255)
if (map->to_name && strlen(map->to_name) > IDENTITY_MAX_NAME_LEN)
return false;
return true;
}
+5
View File
@@ -6,6 +6,11 @@
#include <stdint.h>
#include <sys/types.h>
/* Maximum length of a receiver-resolved identity name in a FROM:TO map's TO
* field. Bounded so a malicious/huge name can never cross the wire (see
* identity_wire_map_valid). */
#define IDENTITY_MAX_NAME_LEN 255
/*
* Identity mapping: --numeric-ids / --usermap / --groupmap / --chown / --copy-as.
*
+313 -295
View File
@@ -44,239 +44,239 @@ bool receive_file_xattrs(File* file, int fd, const Config* config) {
return true;
}
static File* receive_delta_file(int fd, const Config* config, const char* check_path,
void* old_data, unsigned long long old_size, bool* failed) {
if (!old_data) {
free(old_data); /* defensive: old_data is always non-NULL today */
*failed = true;
return NULL;
}
static bool receive_file_payload_into(File* file, int fd, const Config* config,
const char* dest_path, unsigned long long expected_size);
DeltaSignature* sig = delta_signature_create_seeded(old_data, old_size, config->delta_block_size,
(uint32_t)config->checksum_seed);
if (!sig) {
free(old_data);
*failed = true;
return NULL;
}
/* Receive a STATUS_DELTA_DATA response: the sender's delta against the basis we
signed. Deserializes, decompresses and applies the delta (in memory or
through a spool temp file), then receives the metadata/xattr block and
installs the reconstructed payload. Takes ownership of `old_data` and `sig`,
releasing both on every path. */
static File* receive_delta_data_branch(int fd, const Config* config, const char* check_path,
void* old_data, unsigned long long old_size, int basis_fd,
DeltaSignature* sig, bool* failed) {
Data* raw_delta = NULL;
Delta* delta = NULL;
void* new_data = NULL;
char* spool = NULL;
File* file = NULL;
Data* sig_data = delta_signature_serialize(sig);
if (!sig_data) {
delta_signature_destroy(sig);
free(old_data);
*failed = true;
return NULL;
}
raw_delta = receive_data_limited(fd, MAX_RECEIVE_WHOLE_FILE_SIZE);
if (!raw_delta)
goto fail;
bool sig_sent = send_status(fd, STATUS_DELTA_SIGNATURE) && send_data(fd, sig_data);
data_destroy(sig_data);
if (!sig_sent) {
delta_signature_destroy(sig);
free(old_data);
*failed = true;
return NULL;
}
Status resp;
if (!receive_status(fd, &resp)) {
delta_signature_destroy(sig);
free(old_data);
*failed = true;
return NULL;
}
if (resp == STATUS_DELTA_DATA) {
Data* delta_data = receive_data_limited(fd, MAX_RECEIVE_WHOLE_FILE_SIZE);
if (!delta_data) {
delta_signature_destroy(sig);
free(old_data);
*failed = true;
return NULL;
}
Data* raw_delta = delta_data;
if (config->use_compression &&
!compression_should_skip_with_suffixes(
check_path, config->skip_compress_suffixes,
config->skip_compress_set ? config->skip_compress_count : -1)) {
ProtocolSession* owner = delta_data->owner;
raw_delta = data_decompress_limited(delta_data, MAX_RECEIVE_WHOLE_FILE_SIZE);
data_destroy(delta_data);
if (!raw_delta) {
free(old_data);
delta_signature_destroy(sig);
*failed = true;
return NULL;
}
/* Charge the decompressed delta to the connection budget (the paired
wire buffer's charge was just released). */
if (!data_charge_session(raw_delta, owner, raw_delta->size)) {
data_destroy(raw_delta);
free(old_data);
delta_signature_destroy(sig);
*failed = true;
return NULL;
}
}
Delta* delta = delta_deserialize(raw_delta);
if (config->use_compression &&
!compression_should_skip_with_suffixes(check_path, config->skip_compress_suffixes,
config->skip_compress_set ? config->skip_compress_count
: -1)) {
ProtocolSession* owner = raw_delta->owner;
Data* decompressed = data_decompress_limited(raw_delta, MAX_RECEIVE_WHOLE_FILE_SIZE);
data_destroy(raw_delta);
if (!delta) {
free(old_data);
delta_signature_destroy(sig);
*failed = true;
return NULL;
}
raw_delta = decompressed;
if (!raw_delta)
goto fail;
/* Charge the decompressed delta to the connection budget (the paired
wire buffer's charge was just released). */
if (!data_charge_session(raw_delta, owner, raw_delta->size))
goto fail;
}
uint64_t new_size = delta->new_file_size;
if (new_size > MAX_RECEIVE_WHOLE_FILE_SIZE || new_size > SIZE_MAX) {
delta_destroy(delta);
free(old_data);
delta_signature_destroy(sig);
delta = delta_deserialize(raw_delta);
data_destroy(raw_delta);
raw_delta = NULL;
if (!delta)
goto fail;
uint64_t new_size = delta->new_file_size;
if (new_size > SIZE_MAX) {
send_status(fd, STATUS_ERROR);
goto fail;
}
/* Wire-stats tally: bytes taken straight from the basis file (matched
delta blocks) and bytes shipped literally (protocol 2.28.0). Computed
before the delta is destroyed. */
unsigned long long matched = 0;
unsigned long long literal = 0;
for (uint32_t k = 0; k < delta->instruction_count; k++) {
if (delta->instructions[k].type == DELTA_INSTR_BLOCK_MATCH)
matched += delta->instructions[k].match.length;
else if (delta->instructions[k].type == DELTA_INSTR_LITERAL)
literal += delta->instructions[k].literal.length;
}
/* A reconstructed file above the streaming bound is written into a spool
temp file through delta_apply_to_fd; a smaller one keeps the historical
in-memory reconstruction. */
if (new_size > protocol_whole_file_receive_limit()) {
char* dest_path = path_cat(config->receive_root_directory, check_path);
int spool_fd = dest_path ? file_spool_for_payload(dest_path, &spool) : -1;
free(dest_path);
if (spool_fd < 0) {
send_status(fd, STATUS_ERROR);
*failed = true;
return NULL;
goto fail;
}
/* Wire-stats tally: bytes taken straight from the basis file (matched
delta blocks) and bytes shipped literally (protocol 2.28.0). Computed
before the delta is destroyed. */
unsigned long long matched = 0;
unsigned long long literal = 0;
for (uint32_t k = 0; k < delta->instruction_count; k++) {
if (delta->instructions[k].type == DELTA_INSTR_BLOCK_MATCH)
matched += delta->instructions[k].match.length;
else if (delta->instructions[k].type == DELTA_INSTR_LITERAL)
literal += delta->instructions[k].literal.length;
}
void* new_data = delta_apply(old_data, old_size, delta, config->delta_block_size);
bool applied =
delta_apply_to_fd(old_data, basis_fd, old_size, delta, config->delta_block_size, spool_fd);
if (close(spool_fd) != 0)
applied = false;
delta_destroy(delta);
if (!new_data) {
free(old_data);
delta_signature_destroy(sig);
*failed = true;
return NULL;
}
File* file = file_create(check_path);
if (!file) {
free(new_data);
free(old_data);
delta_signature_destroy(sig);
*failed = true;
return NULL;
}
file->matched_bytes = matched;
file->literal_bytes = literal;
if (config->use_metadata) {
int meta_ok = 1;
file->metadata = metadata_receive(fd, &meta_ok);
if (!meta_ok) {
file_destroy(file);
free(new_data);
free(old_data);
delta_signature_destroy(sig);
*failed = true;
return NULL;
}
}
if (!receive_file_xattrs(file, fd, config)) {
file_destroy(file);
free(new_data);
free(old_data);
delta_signature_destroy(sig);
*failed = true;
return NULL;
}
Data* replacement = data_create(new_data, (size_t)new_size);
if (replacement == NULL) {
file_destroy(file);
free(old_data);
delta_signature_destroy(sig);
delta = NULL;
if (!applied) {
send_status(fd, STATUS_ERROR);
*failed = true;
return NULL;
goto fail;
}
} else {
new_data = old_data ? delta_apply(old_data, old_size, delta, config->delta_block_size)
: delta_apply_fd(basis_fd, old_size, delta, config->delta_block_size);
delta_destroy(delta);
delta = NULL;
if (!new_data)
goto fail;
}
file = file_create(check_path);
if (!file)
goto fail;
file->matched_bytes = matched;
file->literal_bytes = literal;
if (config->use_metadata) {
int meta_ok = 1;
file->metadata = metadata_receive(fd, &meta_ok);
if (!meta_ok)
goto fail;
}
if (!receive_file_xattrs(file, fd, config))
goto fail;
if (spool) {
Data* reserved = data_create_reserve((size_t)new_size);
if (reserved == NULL) {
send_status(fd, STATUS_ERROR);
goto fail;
}
data_destroy(file->data);
file->data = reserved;
file->basis_copy = spool;
spool = NULL; /* ownership moved into file->basis_copy */
file->data_spool = true;
} else {
Data* replacement = data_create(new_data, (size_t)new_size);
new_data = NULL; /* data_create owns, and frees, the buffer on failure */
if (replacement == NULL) {
send_status(fd, STATUS_ERROR);
goto fail;
}
data_destroy(file->data);
file->data = replacement;
free(old_data);
delta_signature_destroy(sig);
return file;
}
if (resp == STATUS_NEXT) {
delta_signature_destroy(sig);
free(old_data);
free(old_data);
delta_signature_destroy(sig);
return file;
File* file = file_create(check_path);
if (!file) {
*failed = true;
return NULL;
}
if (config->use_metadata) {
int meta_ok = 1;
file->metadata = metadata_receive(fd, &meta_ok);
if (!meta_ok) {
file_destroy(file);
*failed = true;
return NULL;
}
}
if (!receive_file_xattrs(file, fd, config)) {
file_destroy(file);
*failed = true;
return NULL;
}
Data* file_data = receive_data_limited(fd, MAX_RECEIVE_WHOLE_FILE_SIZE);
if (file_data == NULL) {
file_destroy(file);
*failed = true;
return NULL;
}
if (config->use_compression &&
!compression_should_skip_with_suffixes(
file->path, config->skip_compress_suffixes,
config->skip_compress_set ? config->skip_compress_count : -1)) {
Data* uncompressed = data_decompress_limited(file_data, MAX_RECEIVE_WHOLE_FILE_SIZE);
ProtocolSession* owner = file_data->owner;
data_destroy(file_data);
if (uncompressed == NULL) {
file_destroy(file);
*failed = true;
return NULL;
}
if (!data_charge_session(uncompressed, owner, uncompressed->size)) {
data_destroy(uncompressed);
file_destroy(file);
send_status(fd, STATUS_ERROR);
*failed = true;
return NULL;
}
if (uncompressed->size > MAX_FILE_DATA_SIZE) {
data_destroy(uncompressed);
file_destroy(file);
send_status(fd, STATUS_ERROR);
*failed = true;
return NULL;
}
file_data = uncompressed;
}
data_destroy(file->data);
file->data = file_data;
return file;
fail:
free(new_data);
if (spool) {
unlink(spool);
free(spool);
}
file_destroy(file);
delta_destroy(delta);
data_destroy(raw_delta);
free(old_data);
delta_signature_destroy(sig);
*failed = true;
return NULL;
}
/* Receive a STATUS_NEXT response: the sender declined the delta and will send
the whole file. Releases the basis signature and snapshot, then receives the
metadata/xattr block and the full payload. Takes ownership of `old_data` and
`sig`, releasing both immediately. */
static File* receive_next_branch(int fd, const Config* config, const char* check_path,
unsigned long long expected_size, void* old_data,
DeltaSignature* sig, bool* failed) {
delta_signature_destroy(sig);
free(old_data);
File* file = file_create(check_path);
if (!file)
goto fail;
if (config->use_metadata) {
int meta_ok = 1;
file->metadata = metadata_receive(fd, &meta_ok);
if (!meta_ok)
goto fail;
}
if (!receive_file_xattrs(file, fd, config))
goto fail;
char* dest_path = path_cat(config->receive_root_directory, check_path);
if (!dest_path)
goto fail;
bool payload_ok = receive_file_payload_into(file, fd, config, dest_path, expected_size);
free(dest_path);
if (!payload_ok)
goto fail;
return file;
fail:
file_destroy(file);
*failed = true;
return NULL;
}
/* Delta handshake dispatcher: sign the basis, ship the signature, then hand the
response off to the matching branch helper. Takes ownership of `old_data`
(and, once created, `sig`); sets `*failed` on every error path. */
static File* receive_delta_file(int fd, const Config* config, const char* check_path,
void* old_data, unsigned long long old_size,
unsigned long long expected_size, int basis_fd, bool* failed) {
/* The basis is either an in-memory snapshot (the destination file, bounded) or
* a confined descriptor (a --fuzzy sibling, possibly larger than memory) that
* is signed/applied in bounded chunks. */
Data* sig_data = NULL;
Status resp = STATUS_ERROR;
bool sig_sent = false;
/* A delta check needs at least one basis source: the in-memory destination
* snapshot or a confined basis descriptor. */
if (!old_data && basis_fd < 0) {
*failed = true;
return NULL;
}
DeltaSignature* sig =
old_data ? delta_signature_create_seeded(old_data, old_size, config->delta_block_size,
(uint32_t)config->checksum_seed)
: delta_signature_create_fd_seeded(basis_fd, old_size, config->delta_block_size,
(uint32_t)config->checksum_seed);
if (!sig)
goto fail;
sig_data = delta_signature_serialize(sig);
if (!sig_data)
goto fail;
sig_sent = send_status(fd, STATUS_DELTA_SIGNATURE) && send_data(fd, sig_data);
data_destroy(sig_data);
sig_data = NULL;
if (!sig_sent || !receive_status(fd, &resp))
goto fail;
if (resp == STATUS_DELTA_DATA)
return receive_delta_data_branch(fd, config, check_path, old_data, old_size, basis_fd, sig,
failed);
if (resp == STATUS_NEXT)
return receive_next_branch(fd, config, check_path, expected_size, old_data, sig, failed);
send_status(fd, STATUS_ERROR);
fail:
data_destroy(sig_data);
delta_signature_destroy(sig);
free(old_data);
*failed = true;
return NULL;
}
@@ -640,28 +640,29 @@ static bool fuzzy_candidate_better(const FuzzyCandidate* cand, const FuzzyCandid
return strcmp(cand->name, best->name) < 0;
}
/* Search the destination directory that will contain `check_path` for a
* similar regular file usable as a --fuzzy delta basis and return its full
* content in a malloc'd (protocol_alloc) buffer. Returns NULL (with *out_size
* = 0) when no candidate qualifies, which means the caller performs the normal
* whole-file transfer. */
static void* fuzzy_basis_find_and_load(const Config* config, const char* check_path,
unsigned long long check_size, time_t check_mtime,
long check_mtime_nsec, unsigned long long* out_size) {
/* Search the destination directory that will contain `check_path` for a similar
* regular file usable as a --fuzzy delta basis and return an open, confined
* read descriptor to it (with *out_size set). Returns -1 (with *out_size 0)
* when no candidate qualifies, which means the caller performs the normal
* whole-file transfer. The basis is signed/applied by streaming its descriptor,
* so no whole-basis buffer is ever needed and its size is not capped. */
static int fuzzy_basis_find_and_open(const Config* config, const char* check_path,
unsigned long long check_size, time_t check_mtime,
long check_mtime_nsec, unsigned long long* out_size) {
*out_size = 0;
if (!config || !config->receive_root_directory || !config->fuzzy || !config->use_delta ||
!check_path || check_size > MAX_RECEIVE_WHOLE_FILE_SIZE)
return NULL;
!check_path)
return -1;
char* full_path = path_cat(config->receive_root_directory, check_path);
if (!full_path)
return NULL;
return -1;
char* leaf = NULL;
int dir_fd = file_open_secure_parent(full_path, &leaf, false);
if (dir_fd < 0 || !leaf) {
free(leaf);
free(full_path);
return NULL;
return -1;
}
size_t target_len = strlen(leaf);
/* A target basename longer than FUZZY_NAME_LIMIT can never pass the name gate
@@ -670,7 +671,7 @@ static void* fuzzy_basis_find_and_load(const Config* config, const char* check_p
close(dir_fd);
free(leaf);
free(full_path);
return NULL;
return -1;
}
int scanfd = dup(dir_fd);
@@ -678,7 +679,7 @@ static void* fuzzy_basis_find_and_load(const Config* config, const char* check_p
close(dir_fd);
free(leaf);
free(full_path);
return NULL;
return -1;
}
DIR* dir = fdopendir(scanfd);
if (!dir) {
@@ -686,7 +687,7 @@ static void* fuzzy_basis_find_and_load(const Config* config, const char* check_p
close(dir_fd);
free(leaf);
free(full_path);
return NULL;
return -1;
}
/* The weighted-distance scratch row is allocated once per scan (not once per
@@ -697,7 +698,7 @@ static void* fuzzy_basis_find_and_load(const Config* config, const char* check_p
close(dir_fd);
free(leaf);
free(full_path);
return NULL;
return -1;
}
int fname_suf_len = 0;
const char* fname_suf = fuzzy_find_suffix(leaf, (int)target_len, &fname_suf_len);
@@ -727,7 +728,7 @@ static void* fuzzy_basis_find_and_load(const Config* config, const char* check_p
if (fstatat(dir_fd, name, &st, AT_SYMLINK_NOFOLLOW) != 0 || !S_ISREG(st.st_mode))
continue;
unsigned long long cand_size = (unsigned long long)st.st_size;
if (cand_size == 0 || cand_size > MAX_RECEIVE_WHOLE_FILE_SIZE)
if (cand_size == 0)
continue;
long cand_nsec = 0;
#ifdef __linux__
@@ -771,7 +772,7 @@ static void* fuzzy_basis_find_and_load(const Config* config, const char* check_p
if (exact.name[0])
best = exact;
void* basis = NULL;
int basis_fd = -1;
if (best.name[0]) {
/* O_NONBLOCK: a name raced to a FIFO between the fstatat gate and this open
would otherwise block the receive thread forever on open(2); with it the
@@ -781,29 +782,55 @@ static void* fuzzy_basis_find_and_load(const Config* config, const char* check_p
if (fd >= 0) {
struct stat st;
if (fstat(fd, &st) == 0 && S_ISREG(st.st_mode) &&
(unsigned long long)st.st_size == best.size && best.size <= SIZE_MAX) {
basis = protocol_alloc((size_t)best.size);
if (basis) {
size_t got = 0;
while (got < (size_t)best.size) {
ssize_t n = read(fd, (char*)basis + got, (size_t)best.size - got);
if (n <= 0) {
free(basis);
basis = NULL;
break;
}
got += (size_t)n;
}
}
(unsigned long long)st.st_size == best.size) {
basis_fd = fd;
} else {
close(fd);
}
close(fd);
}
}
close(dir_fd);
free(full_path);
if (basis)
if (basis_fd >= 0)
*out_size = best.size;
return basis;
return basis_fd;
}
/* Receive one whole-file data frame into `file`. A payload at or below the
* receiver's streaming bound keeps the historical charged whole-buffer path; a
* larger one is streamed into a spool temp file (decompressing incrementally)
* and installed through the File's basis_copy field. `expected_size` is the
* logical size from the check frame (0 when unknown, e.g. the non-incremental
* path). */
static bool receive_file_payload_into(File* file, int fd, const Config* config,
const char* dest_path, unsigned long long expected_size) {
bool compress =
config->use_compression && !compression_should_skip_with_suffixes(
file->path, config->skip_compress_suffixes,
config->skip_compress_set ? config->skip_compress_count : -1);
Data* buffer = NULL;
char* spool = NULL;
unsigned long long size = 0;
if (!file_receive_payload(fd, compress, expected_size, dest_path,
protocol_whole_file_receive_limit(), &buffer, &spool, &size)) {
return false;
}
if (spool) {
Data* reserved = data_create_reserve((size_t)size);
if (!reserved) {
unlink(spool);
free(spool);
return false;
}
data_destroy(file->data);
file->data = reserved;
file->basis_copy = spool;
file->data_spool = true;
} else {
data_destroy(file->data);
file->data = buffer;
}
return true;
}
/* Read the remainder of a full-file transfer after the receiver has already
@@ -811,7 +838,8 @@ static void* fuzzy_basis_find_and_load(const Config* config, const char* check_p
* data frame, and return an owned File. Shared by the plain full-transfer path
* and the --append-verify prefix-mismatch fallback (a clean full transfer
* instead of a corrupt prefix+tail blend). */
static File* receive_full_file(int fd, const Config* config, const char* path) {
static File* receive_full_file(int fd, const Config* config, const char* path,
unsigned long long expected_size) {
File* file = file_create(path);
if (!file)
return NULL;
@@ -827,36 +855,17 @@ static File* receive_full_file(int fd, const Config* config, const char* path) {
file_destroy(file);
return NULL;
}
Data* file_data = receive_data_limited(fd, MAX_RECEIVE_WHOLE_FILE_SIZE);
if (file_data == NULL) {
char* dest_path = path_cat(config->receive_root_directory, path);
if (!dest_path) {
file_destroy(file);
return NULL;
}
if (config->use_compression &&
!compression_should_skip_with_suffixes(file->path, config->skip_compress_suffixes,
config->skip_compress_set ? config->skip_compress_count
: -1)) {
Data* uncompressed = data_decompress_limited(file_data, MAX_RECEIVE_WHOLE_FILE_SIZE);
ProtocolSession* owner = file_data->owner;
data_destroy(file_data);
if (uncompressed == NULL) {
file_destroy(file);
return NULL;
}
if (!data_charge_session(uncompressed, owner, uncompressed->size)) {
data_destroy(uncompressed);
file_destroy(file);
return NULL;
}
if (uncompressed->size > MAX_FILE_DATA_SIZE) {
data_destroy(uncompressed);
file_destroy(file);
return NULL;
}
file_data = uncompressed;
bool ok = receive_file_payload_into(file, fd, config, dest_path, expected_size);
free(dest_path);
if (!ok) {
file_destroy(file);
return NULL;
}
data_destroy(file->data);
file->data = file_data;
return file;
}
@@ -974,13 +983,12 @@ static IncrementalCheckOutcome incremental_check_receive_request(IncrementalChec
return INCREMENTAL_ERROR;
}
/* A basis-configured run may materialize a file larger than the whole-file
payload bound: a basis hit is streamed from the basis path (bounded
buffers), so the check size is not itself an allocation. Every other
path (delta/append/full) still applies MAX_RECEIVE_WHOLE_FILE_SIZE, and a
miss simply falls through to the normal transfer with its own bound. */
if (!config_has_basis(config) && state->check_size > MAX_RECEIVE_WHOLE_FILE_SIZE) {
send_error_detail(fd, "check size exceeds receiver limit");
/* No file-size refusal: a whole-file payload larger than the historical
whole-file bound is streamed through a bounded buffer (see
file_receive_payload). Only a size that cannot be represented on this
platform is rejected. */
if (state->check_size > SIZE_MAX) {
send_error_detail(fd, "check size is not representable");
return INCREMENTAL_ERROR;
}
@@ -1411,7 +1419,7 @@ static IncrementalCheckOutcome incremental_check_try_append_resume(IncrementalCh
close(state->old_fd);
state->old_fd = -1;
}
*out_file = receive_full_file(fd, config, check_path);
*out_file = receive_full_file(fd, config, check_path, check_size);
return INCREMENTAL_FILE;
}
@@ -1428,20 +1436,24 @@ static IncrementalCheckOutcome incremental_check_try_append_resume(IncrementalCh
if (config->use_metadata) {
int meta_ok = 1;
meta = metadata_receive(fd, &meta_ok);
if (!meta_ok)
if (!meta_ok) {
file_metadata_destroy(meta);
return INCREMENTAL_ERROR;
}
}
if (config->use_xattrs) {
int xok = 0;
append_xattrs = xattr_receive(fd, &xok, config->preserve_acls);
if (!xok) {
xattr_list_free(append_xattrs);
file_metadata_destroy(meta);
return INCREMENTAL_ERROR;
}
}
Data* tail = receive_data_limited(fd, MAX_RECEIVE_WHOLE_FILE_SIZE);
if (tail == NULL) {
xattr_list_free(append_xattrs);
file_metadata_destroy(meta);
return INCREMENTAL_ERROR;
}
if (config->use_compression &&
@@ -1453,16 +1465,19 @@ static IncrementalCheckOutcome incremental_check_try_append_resume(IncrementalCh
data_destroy(tail);
if (uncompressed == NULL) {
xattr_list_free(append_xattrs);
file_metadata_destroy(meta);
return INCREMENTAL_ERROR;
}
if (!data_charge_session(uncompressed, owner, uncompressed->size)) {
data_destroy(uncompressed);
xattr_list_free(append_xattrs);
file_metadata_destroy(meta);
return INCREMENTAL_ERROR;
}
if (uncompressed->size > MAX_FILE_DATA_SIZE) {
data_destroy(uncompressed);
xattr_list_free(append_xattrs);
file_metadata_destroy(meta);
return INCREMENTAL_ERROR;
}
tail = uncompressed;
@@ -1475,6 +1490,7 @@ static IncrementalCheckOutcome incremental_check_try_append_resume(IncrementalCh
send_status(fd, STATUS_ERROR);
data_destroy(tail);
xattr_list_free(append_xattrs);
file_metadata_destroy(meta);
return INCREMENTAL_ERROR;
}
size_t full_size = (size_t)check_size;
@@ -1482,6 +1498,7 @@ static IncrementalCheckOutcome incremental_check_try_append_resume(IncrementalCh
if (!full) {
data_destroy(tail);
xattr_list_free(append_xattrs);
file_metadata_destroy(meta);
return INCREMENTAL_ERROR;
}
if (old_size > 0 && state->old_data)
@@ -1496,6 +1513,7 @@ static IncrementalCheckOutcome incremental_check_try_append_resume(IncrementalCh
if (!file) {
free(full);
xattr_list_free(append_xattrs);
file_metadata_destroy(meta);
return INCREMENTAL_ERROR;
}
file->metadata = meta;
@@ -1516,8 +1534,9 @@ static IncrementalCheckOutcome incremental_check_try_delta(IncrementalCheckState
bool try_delta, File** out_file) {
if (try_delta && state->old_data != NULL) {
bool delta_failed = false;
File* delta_file = receive_delta_file(state->fd, state->config, state->check_path,
state->old_data, state->old_size, &delta_failed);
File* delta_file =
receive_delta_file(state->fd, state->config, state->check_path, state->old_data,
state->old_size, state->check_size, -1, &delta_failed);
state->old_data = NULL; /* receive_delta_file consumes the snapshot on every path */
if (delta_file) {
*out_file = delta_file;
@@ -1540,14 +1559,14 @@ static IncrementalCheckOutcome incremental_check_try_fuzzy(IncrementalCheckState
if (!config->fuzzy || !config->use_delta)
return INCREMENTAL_CONTINUE;
unsigned long long fuzzy_size = 0;
void* fuzzy_basis = fuzzy_basis_find_and_load(config, state->check_path, state->check_size,
(time_t)state->check_mtime,
(long)state->check_mtime_nsec, &fuzzy_size);
if (fuzzy_basis != NULL) {
int fuzzy_fd = fuzzy_basis_find_and_open(config, state->check_path, state->check_size,
(time_t)state->check_mtime,
(long)state->check_mtime_nsec, &fuzzy_size);
if (fuzzy_fd >= 0) {
bool fuzzy_failed = false;
File* fuzzy_file = receive_delta_file(state->fd, config, state->check_path, fuzzy_basis,
fuzzy_size, &fuzzy_failed);
fuzzy_basis = NULL; /* receive_delta_file consumes the buffer on every path */
File* fuzzy_file = receive_delta_file(state->fd, config, state->check_path, NULL, fuzzy_size,
state->check_size, fuzzy_fd, &fuzzy_failed);
close(fuzzy_fd);
if (fuzzy_file) {
*out_file = fuzzy_file;
return INCREMENTAL_FILE;
@@ -1555,7 +1574,6 @@ static IncrementalCheckOutcome incremental_check_try_fuzzy(IncrementalCheckState
if (fuzzy_failed)
return INCREMENTAL_ERROR;
}
free(fuzzy_basis);
return INCREMENTAL_CONTINUE;
}
@@ -1567,7 +1585,7 @@ static File* incremental_check_receive_full(IncrementalCheckState* state) {
close(state->old_fd);
state->old_fd = -1;
}
return receive_full_file(state->fd, state->config, state->check_path);
return receive_full_file(state->fd, state->config, state->check_path, state->check_size);
}
/* Core implementation. `would_transfer` (may be NULL) is set true only on the
+87 -20
View File
@@ -1,4 +1,5 @@
#include "log.h"
#include "utils.h"
#include <errno.h>
#include <stdbool.h>
#include <stdarg.h>
@@ -16,6 +17,7 @@ static bool info_flags_explicit = false;
static FILE* log_fp = NULL;
static _Thread_local bool eight_bit_output;
static LogStderrMode stderr_mode = LOG_STDERR_ERRORS;
static LogClientMsgSink client_msg_sink = NULL;
/* Serializes access to log_fp and makes each emitted line atomic: the
* timestamp prefix, formatted body, and trailing newline are written as one
@@ -77,6 +79,51 @@ LogStderrMode log_get_stderr_mode(void) {
return stderr_mode;
}
void log_set_client_msg_sink(LogClientMsgSink sink) {
client_msg_sink = sink;
}
LogClientMsgSink log_get_client_msg_sink(void) {
return client_msg_sink;
}
/* Format just the message body (no prefix/newline) into a freshly allocated
* buffer. Shared by log_message (which may hand the body to a client-message
* sink) and log_client_message. Returns NULL on allocation/format failure. */
static char* format_log_body(const char* format, va_list args) {
va_list copy;
va_copy(copy, args);
int body_len = vsnprintf(NULL, 0, format, copy);
va_end(copy);
if (body_len < 0)
return NULL;
char* body = malloc((size_t)body_len + 1);
if (!body)
return NULL;
vsnprintf(body, (size_t)body_len + 1, format, args);
return body;
}
/* Assemble a complete log line (prefix + body + newline) from an already
* formatted body. Returns NULL on allocation failure. */
static char* format_log_line_from_body(LogLevel log_level, const struct tm* t, const char* body) {
char prefix[64];
int prefix_len = snprintf(
prefix, sizeof(prefix), "%04d-%02d-%02d %02d:%02d:%02d [%s]: ", t->tm_year + 1900,
t->tm_mon + 1, t->tm_mday, t->tm_hour, t->tm_min, t->tm_sec, log_level_strings[log_level]);
if (prefix_len < 0 || prefix_len >= (int)sizeof(prefix))
return NULL;
size_t body_len = strlen(body);
char* line = malloc((size_t)prefix_len + body_len + 2); /* body + '\n' + NUL */
if (!line)
return NULL;
memcpy(line, prefix, (size_t)prefix_len);
memcpy(line + prefix_len, body, body_len);
line[(size_t)prefix_len + body_len] = '\n';
line[(size_t)prefix_len + body_len + 1] = '\0';
return line;
}
/* Format one complete log line (timestamp prefix + body + newline) into a
* freshly allocated buffer. This is pure CPU/malloc work and must happen
* OUTSIDE the log mutex: the mutex only guards the log_fp pointer, so a
@@ -84,26 +131,11 @@ LogStderrMode log_get_stderr_mode(void) {
* on allocation/formatting failure. */
static char* format_log_line(LogLevel log_level, const struct tm* t, const char* format,
va_list args) {
char prefix[64];
int prefix_len = snprintf(
prefix, sizeof(prefix), "%04d-%02d-%02d %02d:%02d:%02d [%s]: ", t->tm_year + 1900,
t->tm_mon + 1, t->tm_mday, t->tm_hour, t->tm_min, t->tm_sec, log_level_strings[log_level]);
if (prefix_len < 0 || prefix_len >= (int)sizeof(prefix))
char* body = format_log_body(format, args);
if (!body)
return NULL;
va_list copy;
va_copy(copy, args);
int body_len = vsnprintf(NULL, 0, format, copy);
va_end(copy);
if (body_len < 0)
return NULL;
size_t total = (size_t)prefix_len + (size_t)body_len;
char* line = malloc(total + 2); /* body bytes + '\n' + NUL */
if (!line)
return NULL;
memcpy(line, prefix, (size_t)prefix_len);
vsnprintf(line + prefix_len, (size_t)body_len + 1, format, args);
line[total] = '\n';
line[total + 1] = '\0';
char* line = format_log_line_from_body(log_level, t, body);
free(body);
return line;
}
@@ -121,6 +153,26 @@ static void emit_log_line(FILE* console, const char* line) {
mtx_unlock(&log_mutex);
}
void log_client_message(const char* message) {
if (!message)
return;
/* The body is peer-controlled: escape every non-printable byte (newlines,
CR, ANSI ESC, ...) so a hostile client cannot forge log lines or inject
terminal control sequences. output_escape() is the codebase's canonical
escaper and leaves printable text untouched. */
char* escaped = output_escape(message, log_get_8_bit_output());
if (!escaped)
return;
/* Route through the ordinary log level / destination gate (log_message):
this respects --log-file, the configured stderr mode and the level
threshold instead of always writing to stderr. The wire body carries no
severity, so the forwarded diagnostic is emitted as a warning -- the
lowest level the default gate admits, which keeps the peer's messages
visible without bypassing --quiet. */
log_message(LOG_LEVEL_WARNING, "%s", escaped);
free(escaped);
}
void log_message(LogLevel log_level, const char* format, ...) {
if (log_level < current_log_level)
return;
@@ -138,8 +190,23 @@ void log_message(LogLevel log_level, const char* format, ...) {
va_list args;
va_start(args, format);
char* line = format_log_line(log_level, &t, format, args);
char* body = format_log_body(format, args);
va_end(args);
if (!body)
return;
/* LOG_STDERR_CLIENT: hand the diagnostic to the client-message channel. A
sink that takes ownership suppresses the local write; otherwise (no sink
yet, or the peer connection is not up) fall through to local output so the
diagnostic is never lost. */
if (stderr_mode == LOG_STDERR_CLIENT) {
LogClientMsgSink sink = client_msg_sink;
if (sink && sink(body)) {
free(body);
return;
}
}
char* line = format_log_line_from_body(log_level, &t, body);
free(body);
if (!line)
return;
emit_log_line(dest_io, line);
+20 -1
View File
@@ -15,7 +15,11 @@
#endif
typedef enum { LOG_LEVEL_DEBUG, LOG_LEVEL_INFO, LOG_LEVEL_WARNING, LOG_LEVEL_ERROR } LogLevel;
typedef enum { LOG_STDERR_ERRORS, LOG_STDERR_ALL } LogStderrMode;
/* --stderr=MODE destinations. ERRORS keeps errors on stderr and everything
* else on stdout; ALL sends every message to stderr; CLIENT routes the client's
* own diagnostics over the protocol stream to the peer's stderr (rsync's
* --stderr=client / --no-msgs2stderr). */
typedef enum { LOG_STDERR_ERRORS, LOG_STDERR_ALL, LOG_STDERR_CLIENT } LogStderrMode;
typedef enum {
LOG_DEBUG_IO = 1u << 0,
@@ -86,5 +90,20 @@ void log_set_8_bit_output(bool enabled);
bool log_get_8_bit_output(void);
void log_set_stderr_mode(LogStderrMode mode);
LogStderrMode log_get_stderr_mode(void);
/* Write a message a peer forwarded over the client-message channel to this
* process's stderr (and log file), with the standard log prefix. Used by the
* server side of rsync's --stderr=client. */
void log_client_message(const char* message);
/* Sink for LOG_STDERR_CLIENT. log_message() passes the un-prefixed message
* body to the installed sink; a `true` return means the sink took ownership
* (e.g. queued it for protocol transmission) and the message must NOT also be
* written locally. A `false` return (or a NULL sink) makes log_message fall
* back to the normal local destination, so a diagnostic emitted before the peer
* connection exists is never lost (rsync's documented fallback). The sink may
* be called from any thread and must be tolerant of that. */
typedef bool (*LogClientMsgSink)(const char* message);
void log_set_client_msg_sink(LogClientMsgSink sink);
LogClientMsgSink log_get_client_msg_sink(void);
#endif
+4
View File
@@ -34,6 +34,7 @@ PipelineContextSender* pipeline_context_sender_create(Config* config, Queue* que
context->synced_dirs = NULL;
context->plan_dirs = NULL;
context->missing_args = NULL;
context->per_dir_rules = NULL;
context->scan_had_io_error = false;
context->remove_source_files = NULL;
context->early_delete = false;
@@ -53,6 +54,7 @@ PipelineContextSender* pipeline_context_sender_create(Config* config, Queue* que
context->dir_entries_mutex_init = false;
atomic_init(&context->dir_count, 0);
context->delete_limit = false;
context->partial = false;
int init = 0;
if (config->use_metadata) {
context->dir_entries = array_list_create(file_destroy);
@@ -209,6 +211,8 @@ void pipeline_context_sender_destroy(PipelineContextSender* context) {
array_list_delete(context->plan_dirs);
if (context->missing_args)
array_list_delete(context->missing_args);
if (context->per_dir_rules)
filter_rule_list_free(context->per_dir_rules);
if (context->remove_source_files)
array_list_delete(context->remove_source_files);
if (context->dir_entries)
+11
View File
@@ -67,6 +67,12 @@ typedef struct {
them in the manifest frame's third section and the receiver deletes each as
an explicit request. */
ArrayList* missing_args;
/* Per-directory filter rules the source scan compiled (protocol 2.30.0),
sent with the delete manifest/plan config so the receiver can re-derive the
per-directory protect/risk set. NULL when --delete is off. Populated by
the scanner (parallel workers append under mutex_scanner) or the early
pre-scan. */
FilterRuleList* per_dir_rules;
/* A source I/O error (unreadable directory) was recorded during the scan.
Set by the pre-scan (before the threads start) or by the scanner thread
under mutex_scanner; the caller turns it into a non-zero exit when
@@ -134,6 +140,11 @@ typedef struct {
deletion (STATUS_DELETE_LIMIT): the transfer succeeded and the process must
exit 25 like rsync. Read by the caller after the sender thread is joined. */
bool delete_limit;
/* Set by the sender thread when the receiver reported STATUS_PARTIAL (a
per-entry receiver failure that did not abort the stream): the transfer
otherwise succeeded, successfully stored --remove-source-files sources were
removed, and the process must exit 23 like rsync. Read after join. */
bool partial;
} PipelineContextSender;
/* `config` is borrowed and must outlive the context: destroy does NOT free it,
+74 -16
View File
@@ -38,6 +38,27 @@ static atomic_ullong io_bytes_read = 0;
static unsigned long long global_bwlimit(void);
/* Runtime whole-file receive bound (see protocol.h). Resolved once; an
* override can only LOWER the ceiling, never raise it above the protocol
* constant, so the wire/security bound is unchanged. A parse failure or a
* non-positive value leaves the default in place. */
unsigned long long protocol_whole_file_receive_limit(void) {
static atomic_ullong cached = 0;
unsigned long long value = atomic_load_explicit(&cached, memory_order_relaxed);
if (value != 0)
return value;
value = MAX_RECEIVE_WHOLE_FILE_SIZE;
const char* env = getenv("FASTSYNC_MAX_WHOLE_FILE_SIZE");
if (env && env[0] != '\0') {
char* end = NULL;
unsigned long long parsed = strtoull(env, &end, 10);
if (end && *end == '\0' && parsed > 0 && parsed < value)
value = parsed;
}
atomic_store_explicit(&cached, value, memory_order_relaxed);
return value;
}
/* ------------------------------------------------------------------------- *
* Transport vtable implementations.
*
@@ -664,6 +685,10 @@ static const char* status_to_string(Status status) {
return "DELETE_LIMIT";
case STATUS_DEST_INFO:
return "DEST_INFO";
case STATUS_CLIENT_MSG:
return "CLIENT_MSG";
case STATUS_PARTIAL:
return "PARTIAL";
default:
return "UNKNOWN";
}
@@ -672,10 +697,10 @@ static const char* status_to_string(Status status) {
/* Reject a raw wire status outside the known enum range before it is handed to
* callers, so an unknown/corrupt frame fails as a protocol error instead of
* being silently interpreted as an unexpected-but-valid verdict. STATUS_OK is
* the first enumerator and STATUS_STATS the last, so the range check accepts
* the first enumerator and STATUS_PARTIAL the last, so the range check accepts
* every status the protocol defines. */
static bool status_is_valid(Status status) {
return status >= STATUS_OK && status <= STATUS_STATS;
return status >= STATUS_OK && status <= STATUS_PARTIAL;
}
/* Shared string send/receive implementation. `redact` selects whether the
@@ -765,17 +790,11 @@ bool protocol_send_data(ProtocolSession* session, const Data* data) {
return true;
}
Data* protocol_receive_data_limited(ProtocolSession* session, unsigned long long maximum_size) {
Data* protocol_receive_data_alloc(ProtocolSession* session, unsigned long long size) {
if (!session)
return NULL;
unsigned long long size = 0;
if (!protocol_receive_n_data(session, &size, sizeof(unsigned long long)))
if (size > MAX_DATA_PAYLOAD_SIZE)
return NULL;
if (size > MAX_DATA_PAYLOAD_SIZE || size > maximum_size) {
log_message(LOG_LEVEL_ERROR, "Data size %llu exceeds maximum %llu", size,
(unsigned long long)MAX_DATA_PAYLOAD_SIZE);
return NULL;
}
if (size > SIZE_MAX)
return NULL;
size_t allocation_size = size == 0 ? 1 : (size_t)size;
@@ -790,12 +809,6 @@ Data* protocol_receive_data_limited(ProtocolSession* session, unsigned long long
protocol_release_memory_for_session(session, allocation_size);
return NULL;
}
if (!protocol_receive_n_data(session, data, (size_t)size)) {
free(data);
protocol_release_memory_for_session(session, allocation_size);
return NULL;
}
log_debug_message(LOG_DEBUG_PROTO, "Received %llu data", size);
Data* result = data_create(data, (size_t)size);
if (!result) {
protocol_release_memory_for_session(session, allocation_size);
@@ -806,6 +819,32 @@ Data* protocol_receive_data_limited(ProtocolSession* session, unsigned long long
return result;
}
Data* protocol_receive_data_body(ProtocolSession* session, unsigned long long size) {
Data* result = protocol_receive_data_alloc(session, size);
if (!result)
return NULL;
if (!protocol_receive_n_data(session, result->data, (size_t)size)) {
data_destroy(result);
return NULL;
}
log_debug_message(LOG_DEBUG_PROTO, "Received %llu data", size);
return result;
}
Data* protocol_receive_data_limited(ProtocolSession* session, unsigned long long maximum_size) {
if (!session)
return NULL;
unsigned long long size = 0;
if (!protocol_receive_n_data(session, &size, sizeof(unsigned long long)))
return NULL;
if (size > MAX_DATA_PAYLOAD_SIZE || size > maximum_size) {
log_message(LOG_LEVEL_ERROR, "Data size %llu exceeds maximum %llu", size,
(unsigned long long)MAX_DATA_PAYLOAD_SIZE);
return NULL;
}
return protocol_receive_data_body(session, size);
}
bool protocol_send_int(ProtocolSession* session, int data) {
if (!protocol_send_n_data(session, &data, sizeof(int)))
return false;
@@ -1110,6 +1149,12 @@ Data* receive_data(int fd) {
Data* receive_data_limited(int fd, unsigned long long maximum_size) {
return protocol_receive_data_limited(legacy_session(fd, -1), maximum_size);
}
Data* receive_data_body(int fd, unsigned long long size) {
return protocol_receive_data_body(legacy_session(fd, -1), size);
}
Data* receive_data_alloc(int fd, unsigned long long size) {
return protocol_receive_data_alloc(legacy_session(fd, -1), size);
}
bool send_int(int fd, int data) {
return protocol_send_int(legacy_session(-1, fd), data);
}
@@ -1144,6 +1189,19 @@ bool send_error_detail(int fd, const char* message) {
return send_status(fd, STATUS_ERROR_DETAIL) && send_str(fd, message);
}
bool send_client_message(int fd, const char* message) {
if (!message)
message = "";
char bounded[MAX_CLIENT_MSG_BYTES + 1];
size_t len = strlen(message);
if (len > MAX_CLIENT_MSG_BYTES) {
memcpy(bounded, message, MAX_CLIENT_MSG_BYTES);
bounded[MAX_CLIENT_MSG_BYTES] = '\0';
message = bounded;
}
return send_status(fd, STATUS_CLIENT_MSG) && send_str(fd, message);
}
const char* protocol_last_error(void) {
return io_error_detail;
}
+76 -15
View File
@@ -15,6 +15,12 @@
* this for a rejection and the detail frame stays a small, fixed bound. */
#define MAX_ERROR_DETAIL_BYTES 4096
/* Hard cap on a client diagnostic forwarded over the STATUS_CLIENT_MSG channel
* (protocol 2.30.0, rsync's --stderr=client). The body is reused from the
* bounded-string wire helper and sliced to this many bytes before it is sent,
* so a peer can never be made to retain more than this per message. */
#define MAX_CLIENT_MSG_BYTES 4096
/* Maximum uncompressed file payload accepted by the receiver's whole-file
* paths. A single whole file is charged against the per-connection memory
* reservation (MAX_CONNECTION_MEMORY) and against the server allocation
@@ -26,6 +32,16 @@
/* Maximum allowed data payload size for receive_data (whole-file bound) */
#define MAX_DATA_PAYLOAD_SIZE MAX_RECEIVE_WHOLE_FILE_SIZE
/* Runtime whole-file receive bound. It defaults to MAX_RECEIVE_WHOLE_FILE_SIZE
* and exists so the test suite can lower the ceiling (via the
* FASTSYNC_MAX_WHOLE_FILE_SIZE environment variable, a byte count) and exercise
* the streaming path with a small, fast transfer. A payload at or below the
* bound keeps the historical whole-buffer path; a larger one is streamed
* through a bounded buffer. The value is resolved once per process and never
* exceeds the compile-time ceiling, so a malicious environment cannot raise it
* beyond the protocol limit. */
unsigned long long protocol_whole_file_receive_limit(void);
/* Maximum chunk size (64 MB) — prevents unbounded allocation from the wire */
#define MAX_CHUNK_SIZE (64ULL * 1024 * 1024)
/* Files larger than this are not kept fully in memory while loading: the
@@ -123,8 +139,13 @@ enum NET_STATUS {
STATUS_KEEPALIVE,
STATUS_ABORT,
STATUS_CHECK_BATCH,
/* An explicit directory entry (--dirs): the sender transmits only the path;
* the receiver creates the directory below the receive root. */
/* An explicit directory entry (--dirs / an empty source directory): the sender
* transmits the path and, when metadata/xattrs are negotiated, their blocks;
* the receiver creates the directory below the receive root. Protocol 2.30.0
* inserts an int32 probe flag right after the status when report_dest_info is
* negotiated: probe=1 is a report-only frame (path only; the receiver answers
* STATUS_DEST_INFO and creates nothing), probe=0 is a real create that is
* answered with the directory's pre-transfer state before it is created. */
STATUS_MKDIR,
/* --append / --append-verify tail resume. STATUS_APPEND is sent by the
* receiver after a per-file STATUS_CHECK when the existing destination file
@@ -207,16 +228,22 @@ enum NET_STATUS {
* limit stopped deletions"). Appended after STATUS_DRY_RUN_TRANSFER so no
* existing status is renumbered. */
STATUS_DELETE_LIMIT,
/* Destination-state report for output parity (protocol 2.23.0). When the
* wire config carries report_dest_info=true, the receiver answers every
* per-file STATUS_CHECK request with STATUS_DEST_INFO FIRST, followed by a
* fixed record describing the pre-transfer destination entry
* (int32 has_old; uint64 size; int64 mtime; int64 mtime_nsec; uint32 mode;
* int32 uid; int32 gid). The ordinary STATUS_OK/STATUS_NEXT/... verdict
* follows, so the sender can render rsync-accurate -i/--out-format columns
* (new vs modified, and which of size/time/perms/owner/group differ) without
* changing the transfer decision itself. Appended after
* STATUS_DELETE_LIMIT so no existing status is renumbered. */
/* Destination-state report for output parity (protocol 2.23.0; extended to
* directories/symlinks in 2.30.0). When the wire config carries
* report_dest_info=true, the receiver answers every per-file STATUS_CHECK
* request with STATUS_DEST_INFO FIRST, followed by a fixed record describing
* the pre-transfer destination entry (int32 has_old; int32 target_matches;
* uint64 size; int64 mtime; int64 mtime_nsec; uint32 mode; int32 uid;
* int32 gid). The ordinary STATUS_OK/STATUS_NEXT/... verdict follows, so the
* sender can render rsync-accurate -i/--out-format columns (new vs modified,
* and which of size/time/perms/owner/group differ) without changing the
* transfer decision itself. Protocol 2.30.0 also uses this record for
* STATUS_MKDIR and STATUS_SYMLINK: the sender consumes it into the entry's
* dest_state before emitting its change line, and target_matches reports
* whether an existing symlink's on-disk target already equals the incoming
* one (so the sender can render `cLc........` vs `.L..t......` and suppress
* an unchanged symlink). Appended after STATUS_DELETE_LIMIT so no existing
* status is renumbered. */
STATUS_DEST_INFO,
/* Per-directory delete plan (protocol 2.24.0). The sender of a
* --delete-during/--delete-delay transfer streams one frame per source
@@ -238,9 +265,29 @@ enum NET_STATUS {
* config carries report_stats=true, the receiver sends this status once,
* immediately before its terminal success status, followed by a fixed stats
* record (see format_stats_send/receive in format.h) and, when the run is a
* --dry-run with --delete, the would-delete path list. Appended after
* STATUS_DELETE_PLAN so no existing status is renumbered. */
STATUS_STATS
* --dry-run with --delete, the would-delete path list. Protocol 2.30.0
* appends the four deleted_reg/dir/link/special counters to that record, so
* --stats can render rsync's `Number of deleted files` per-type breakdown.
* Appended after STATUS_DELETE_PLAN so no existing status is renumbered. */
STATUS_STATS,
/* Client diagnostic channel (protocol 2.30.0, rsync's --stderr=client /
* --no-msgs2stderr). When the client's --stderr mode is `client`, the
* client forwards its own diagnostics over this client->server frame
* (STATUS_CLIENT_MSG followed by a bounded length-prefixed string, capped at
* MAX_CLIENT_MSG_BYTES) instead of writing them to its local stderr. The
* receiver reads the string and writes it to the server's stderr (respecting
* the server log destination). Appended after STATUS_STATS so no existing
* status is renumbered. */
STATUS_CLIENT_MSG,
/* Receiver-side partial transfer (protocol 2.30.0). Sent by the receiver as
* the terminal status INSTEAD of STATUS_OK when one or more entries failed
* per-entry without aborting the stream (currently a --devices mknod
* EPERM/EACCES). The transfer otherwise succeeded and every successfully
* stored file was acknowledged, so the sender may still remove
* --remove-source-files sources; the sender maps this to rsync's exit code
* 23 ("partial transfer due to error"), distinct from a fatal STATUS_ERROR.
* Appended after STATUS_CLIENT_MSG so no existing status is renumbered. */
STATUS_PARTIAL
};
void io_set_fds(int read_fd, int write_fd);
@@ -333,6 +380,15 @@ char* receive_str_redacted(int file_descriptor);
bool send_data(int file_descriptor, const Data* data);
Data* receive_data(int file_descriptor);
Data* receive_data_limited(int file_descriptor, unsigned long long maximum_size);
/* Read exactly `size` bytes as a charged Data body. The length-prefixed
* receive_data_limited() reads the header itself; this variant is for callers
* that must inspect the declared size (and possibly stream the body instead)
* before allocating. `size` must already be within MAX_DATA_PAYLOAD_SIZE. */
Data* receive_data_body(int file_descriptor, unsigned long long size);
/* Allocate (and charge) a `size`-byte Data body without reading it; the caller
* fills `result->data` itself. Used when a frame's leading bytes must be
* inspected before the rest of the body is read. */
Data* receive_data_alloc(int file_descriptor, unsigned long long size);
bool send_int(int file_descriptor, int data);
bool receive_int(int file_descriptor, int* data);
bool send_status(int file_descriptor, Status status);
@@ -341,6 +397,11 @@ bool receive_status(int file_descriptor, Status* status);
* length-prefixed string. Over-long messages are sliced and NULL is treated
* as "". Returns false if the status or the string could not be sent. */
bool send_error_detail(int file_descriptor, const char* message);
/* Send STATUS_CLIENT_MSG followed by a bounded (<= MAX_CLIENT_MSG_BYTES)
* length-prefixed string carrying a client diagnostic. Over-long messages are
* sliced and NULL is treated as "". Returns false if the status or the string
* could not be sent. */
bool send_client_message(int file_descriptor, const char* message);
/* Human-readable reason captured from the most recent STATUS_ERROR_DETAIL
* received on this thread, or "" when the last status was a bare STATUS_ERROR
* (or no detail was seen). Thread-local, and valid until the next non-keepalive
+1 -1
View File
@@ -134,7 +134,7 @@ Server* server_create_ex(int port, const ServerBindOptions* bind_opts) {
server->file_descriptor = file_descriptor;
server->ssl_ctx = NULL;
server->max_connections = 100;
server->max_connections = SERVER_DEFAULT_MAX_CONNECTIONS;
server->active_connections = 0;
server->limit_registry = NULL;
+4
View File
@@ -11,6 +11,10 @@
* stored here so the transport layer does not depend on daemon config. */
struct DaemonLimitRegistry;
/* Connection cap applied by server_create_ex() until the daemon's configured
* `max connections` overrides it via server_set_max_connections(). */
#define SERVER_DEFAULT_MAX_CONNECTIONS 100
typedef struct Server {
struct sockaddr_storage address;
unsigned int address_length;
+2 -2
View File
@@ -450,7 +450,7 @@ void fake_super_store_fd(int fd, uint32_t uid, uint32_t gid, uint32_t mode, uint
if (len <= 0 || (size_t)len >= sizeof(record))
return;
if (fsetxattr(fd, FAKESUPER_XATTR, record, (size_t)len, 0) != 0) {
log_message(LOG_LEVEL_WARNING, "--fake-super: could not store %s on destination file: %s",
log_message(LOG_LEVEL_WARNING, "--fake-super: could not store %s on destination entry: %s",
FAKESUPER_XATTR, strerror(errno));
}
}
@@ -550,7 +550,7 @@ bool fake_super_restore_fd(int fd, FileAttrPolicy policy) {
} else if (metadata_mode_for_policy((mode_t)(ul_mode & 0777U), cur.st_mode, policy, &want)) {
if (fchmod(fd, want) != 0)
log_message(LOG_LEVEL_WARNING,
"--fake-super: could not restore mode on destination file: %s",
"--fake-super: could not restore mode on destination entry: %s",
strerror(errno));
}
}
+10 -6
View File
@@ -140,21 +140,25 @@ bool xattr_apply_path_nofollow(int parent_fd, const char* leaf, const FileXattrL
/* --fake-super: write the source uid/gid/mode/rdev record into the reserved
* FAKESUPER_XATTR on `fd`, using rsync 3.4.1's exact grammar (see the key
* comment above). `mode` is the full st_mode including its S_IFMT bits.
* Best-effort (logged, never fatal). Only meaningful when metadata was
* transmitted so the values exist. */
* `fd` may be a regular file, a faked char/block device (written as a regular
* file), or a DIRECTORY: rsync stores a directory's faked mode/uid/gid in the
* reserved xattr on the directory itself. Best-effort (logged, never fatal).
* Only meaningful when metadata was transmitted so the values exist. */
void fake_super_store_fd(int fd, uint32_t uid, uint32_t gid, uint32_t mode, uint32_t rdev_major,
uint32_t rdev_minor);
/* --fake-super replay: parse the FAKESUPER_XATTR record previously written on
* `fd` by fake_super_store_fd and re-apply the recorded permission bits
* fd-relative. The recorded uid/gid are deliberately NOT chowned for real:
* --fake-super only RECORDS ownership (the caller stores the resolved mapping
* via identity_resolve_storage_ids), it never performs a real chown. The
* fd-relative. `fd` may be a regular file, a faked device, or a DIRECTORY;
* fgetxattr/fchmod work identically on a directory descriptor. The recorded
* uid/gid are deliberately NOT chowned for real: --fake-super only RECORDS
* ownership (the caller stores the resolved mapping via
* identity_resolve_storage_ids), it never performs a real chown. The
* recorded rdev is retained for a later privileged restore but is not acted on
* here. Best-effort: absence of the xattr or a malformed record is a silent
* no-op that never fails the transfer. The MODE leg is applied only when
* policy.perms||policy.executability, and the recorded special bits
* (setuid/setgid/sticky) are NOT applied to the real file -- exactly like
* (setuid/setgid/sticky) are NOT applied to the real entry -- exactly like
* rsync's fake-super receiver, which stores the full mode in the xattr but
* strips the special bits on disk. mtime is not part of the record; the normal
* metadata path carries it (policy.times) exactly as rsync sets the file's own
@@ -0,0 +1 @@
hello
@@ -0,0 +1 @@
x
@@ -0,0 +1 @@
y
+1
View File
@@ -0,0 +1 @@
a.txt
+1
View File
@@ -0,0 +1 @@
b.txt
@@ -0,0 +1 @@
world
@@ -0,0 +1 @@
deep
+1
View File
@@ -0,0 +1 @@
../a.txt
+1
View File
@@ -0,0 +1 @@
hello
+1
View File
@@ -0,0 +1 @@
b.txt
+1
View File
@@ -0,0 +1 @@
world
+1
View File
@@ -0,0 +1 @@
deep
+1
View File
@@ -0,0 +1 @@
../a.txt
@@ -0,0 +1 @@
hello
@@ -0,0 +1 @@
world
+1
View File
@@ -0,0 +1 @@
hello
+1
View File
@@ -0,0 +1 @@
world
@@ -0,0 +1 @@
hello
@@ -0,0 +1 @@
world
+1
View File
@@ -0,0 +1 @@
hello
+1
View File
@@ -0,0 +1 @@
world
+6 -2
View File
@@ -34,7 +34,7 @@ class ServerManager:
self._proc = None
self._port = None
def start(self, extra_args=None):
def start(self, extra_args=None, env=None):
self.stop()
self._port = _find_free_port()
# Plain TCP is intentionally explicit in the server; integration tests
@@ -42,7 +42,11 @@ class ServerManager:
cmd = SERVER_CMD + ["-p", str(self._port), "--allow-unauthenticated"]
if extra_args:
cmd += extra_args
self._proc = subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
proc_env = dict(os.environ)
if env:
proc_env.update(env)
self._proc = subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
env=proc_env)
_wait_for_port(self._port, timeout=5)
def stop(self):
@@ -17,6 +17,7 @@ Run locally::
python3 -m pytest tests/integration/test_differential_parity.py -n 4 --dist=load -m parity
"""
import os
import re
import shutil
import sys
import warnings
@@ -105,6 +106,15 @@ def seed_backup(_src, rroot, froot):
_mk(os.path.join(root, "a.txt"), b"OLD-CONTENT\n", _OLD_MTIME)
def seed_delay_updates(_src, rroot, froot):
"""A changed file plus an extra, so --delay-updates (and its implied
--delete-after) has both a publication and a deletion to order."""
for root in (rroot, froot):
_mk(os.path.join(root, "a.txt"), b"OLD-CONTENT\n", _OLD_MTIME)
_mk(os.path.join(root, "extra.txt"), b"extra\n", _OLD_MTIME)
_mk(os.path.join(root, "extradir", "z.txt"), b"z\n", _OLD_MTIME)
def seed_size_only(_src, rroot, froot):
for root in (rroot, froot):
_mk(os.path.join(root, "a.txt"), b"XXXXXXXXXXX\n", _OLD_MTIME)
@@ -127,6 +137,66 @@ def seed_filter_protect(_src, rroot, froot):
_mk(os.path.join(root, "sub", "other2.txt"), b"nested dest-only other\n", _OLD_MTIME)
def seed_perdir_protect(_src, rroot, froot):
"""Per-directory `.rsync-filter` carrying `P` rules on the source and both
destinations, plus destination-only extras. The `.log` extras must survive
--delete under every timing while the other extras go; the source's
`.rsync-filter` (which FastSync carries to the receiver) and the seeded
destination one (which rsync's receiver reads) are byte-identical."""
for root in (_src, rroot, froot):
_mk(os.path.join(root, ".rsync-filter"), b"P extra.log\nP nested.log\n")
for root in (rroot, froot):
_mk(os.path.join(root, "extra.log"), b"dest-only protected\n", _OLD_MTIME)
_mk(os.path.join(root, "other.txt"), b"dest-only deleted\n", _OLD_MTIME)
_mk(os.path.join(root, "sub", "nested.log"), b"nested protected\n", _OLD_MTIME)
_mk(os.path.join(root, "sub", "other2.txt"), b"nested deleted\n", _OLD_MTIME)
def seed_perdir_exclude(_src, rroot, froot):
"""Per-directory unqualified exclude (`-`): dual-sided, so it protects the
matching destination-only extra (and is opted back in by
--delete-excluded)."""
for root in (_src, rroot, froot):
_mk(os.path.join(root, ".rsync-filter"), b"- extra.log\n")
for root in (rroot, froot):
_mk(os.path.join(root, "extra.log"), b"dest-only excluded\n", _OLD_MTIME)
_mk(os.path.join(root, "other.txt"), b"dest-only deleted\n", _OLD_MTIME)
def seed_perdir_subdir_protect(_src, rroot, froot):
"""A SUBDIRECTORY-owned `.rsync-filter` (its owner is not the transfer root):
the receiver must re-derive the `P` rules in the destination-relative
coordinate system, otherwise the destination-only nested extras are wrongly
deleted (silent data loss). A root-level extra is included so a too-broad
rule would over-protect. The file is seeded on both destinations because
rsync's receiver reads the per-directory file locally for delete-during."""
for root in (_src, rroot, froot):
_mk(os.path.join(root, "sub", ".rsync-filter"), b"P nested.log\nP extra.log\n")
for root in (rroot, froot):
_mk(os.path.join(root, "sub", "nested.log"), b"dest-only protected\n", _OLD_MTIME)
_mk(os.path.join(root, "sub", "extra.log"), b"dest-only protected 2\n", _OLD_MTIME)
_mk(os.path.join(root, "sub", "other.txt"), b"dest-only deleted\n", _OLD_MTIME)
_mk(os.path.join(root, "root_extra.txt"), b"root dest-only deleted\n", _OLD_MTIME)
def seed_perdir_subdir_exclude(_src, rroot, froot):
"""A SUBDIRECTORY-owned unqualified exclude (`-`): dual-sided, so it protects
the matching destination-only nested extra under plain --delete and is opted
back in by --delete-excluded."""
for root in (_src, rroot, froot):
_mk(os.path.join(root, "sub", ".rsync-filter"), b"- nested.log\n")
for root in (rroot, froot):
_mk(os.path.join(root, "sub", "nested.log"), b"dest-only excluded\n", _OLD_MTIME)
_mk(os.path.join(root, "sub", "other.txt"), b"dest-only deleted\n", _OLD_MTIME)
def _seed_rules(content):
def seed(_src, _rroot, _froot):
_mk(os.path.join(_src, ".rules"), content)
return seed
def seed_max_delete(_src, rroot, froot):
for root in (rroot, froot):
_mk(os.path.join(root, "extra1.txt"), b"e1\n", _OLD_MTIME)
@@ -244,6 +314,14 @@ _CASES = [
H.Case("delete_commit", "basic", ["-a", "--delete-after"], seed=seed_extras,
fastsync_flags=["-a", "--delete-commit"], server_args=DELETE,
ref="FastSync-only --delete-commit == rsync --delete-after"),
# #317: --delay-updates stages under a per-run unique name and publishes
# every update before the implied --delete-after removes extras.
H.Case("delay_updates", "basic", ["-a", "--delay-updates"],
seed=seed_delay_updates, ref="--delay-updates stages then publishes"),
H.Case("delay_updates_delete", "basic",
["-a", "--delay-updates", "--delete"], seed=seed_delay_updates,
server_args=DELETE, ci=True,
ref="--delay-updates implies --delete-after (publish before delete)"),
H.Case("delete_excluded", "filters",
["-a", "--delete", "--delete-excluded", "--exclude=*.log"],
seed=seed_delete_excluded, server_args=DELETE, ref="--delete-excluded"),
@@ -271,6 +349,81 @@ _CASES = [
["-a", "--delete-after", "--filter=P *.log"],
seed=seed_filter_protect, server_args=DELETE, ci=True,
ref="--filter P/--protect under the whole-tree --delete-after commit"),
# Per-directory merge rules (#315): the receiver must re-derive the
# protect/risk verdict from the carried per-directory rules, so a
# destination-only entry matching ONLY a per-directory rule is shielded.
H.Case("filter_perdir_protect", "filters",
["-a", "-F", "--delete"],
seed=seed_perdir_protect, server_args=DELETE, ci=True,
ref="-F per-directory P rule under the default --delete timing"),
H.Case("filter_perdir_protect_during", "filters",
["-a", "-F", "--delete-during"],
seed=seed_perdir_protect, server_args=DELETE, ci=True,
ref="-F per-directory P rule under --delete-during"),
H.Case("filter_perdir_protect_delay", "filters",
["-a", "-F", "--delete-delay"],
seed=seed_perdir_protect, server_args=DELETE, ci=True,
ref="-F per-directory P rule under --delete-delay"),
H.Case("filter_perdir_protect_before", "filters",
["-a", "-F", "--delete-before"],
seed=seed_perdir_protect, server_args=DELETE, ci=True,
ref="-F per-directory P rule under the whole-tree --delete-before commit"),
H.Case("filter_perdir_protect_after", "filters",
["-a", "-F", "--delete-after"],
seed=seed_perdir_protect, server_args=DELETE, ci=True,
ref="-F per-directory P rule under the whole-tree --delete-after commit"),
H.Case("filter_perdir_exclude_protect", "filters",
["-a", "-F", "--delete"],
seed=seed_perdir_exclude, server_args=DELETE, ci=True,
ref="-F per-directory exclude protects its destination mirror"),
H.Case("filter_perdir_exclude_deleted", "filters",
["-a", "-F", "--delete", "--delete-excluded"],
seed=seed_perdir_exclude, server_args=DELETE, ci=True,
ref="-F per-directory exclude under --delete-excluded is at risk"),
# #316: a rule owned by a SUBDIRECTORY (not the transfer root) must be
# re-expressed in the receiver's destination-relative coordinate system, or
# the dest-only extras it protects are silently deleted.
H.Case("filter_perdir_subdir_protect", "filters",
["-a", "-F", "--delete"],
seed=seed_perdir_subdir_protect, server_args=DELETE, ci=True,
ref="-F subdirectory-owned P rule under the default --delete timing"),
H.Case("filter_perdir_subdir_protect_during", "filters",
["-a", "-F", "--delete-during"],
seed=seed_perdir_subdir_protect, server_args=DELETE, ci=True,
ref="-F subdirectory-owned P rule under --delete-during"),
H.Case("filter_perdir_subdir_protect_delay", "filters",
["-a", "-F", "--delete-delay"],
seed=seed_perdir_subdir_protect, server_args=DELETE, ci=True,
ref="-F subdirectory-owned P rule under --delete-delay"),
H.Case("filter_perdir_subdir_protect_before", "filters",
["-a", "-F", "--delete-before"],
seed=seed_perdir_subdir_protect, server_args=DELETE, ci=True,
ref="-F subdirectory-owned P rule under the whole-tree --delete-before commit"),
H.Case("filter_perdir_subdir_protect_after", "filters",
["-a", "-F", "--delete-after"],
seed=seed_perdir_subdir_protect, server_args=DELETE, ci=True,
ref="-F subdirectory-owned P rule under the whole-tree --delete-after commit"),
H.Case("filter_perdir_subdir_exclude_protect", "filters",
["-a", "-F", "--delete"],
seed=seed_perdir_subdir_exclude, server_args=DELETE, ci=True,
ref="-F subdirectory-owned exclude protects its destination mirror"),
H.Case("filter_perdir_subdir_exclude_deleted", "filters",
["-a", "-F", "--delete", "--delete-excluded"],
seed=seed_perdir_subdir_exclude, server_args=DELETE, ci=True,
ref="-F subdirectory-owned exclude under --delete-excluded is at risk"),
# Merge-file modifiers (#315): e/n/w/- semantics match rsync 3.4.1.
H.Case("dir_merge_e", "filters", ["-a", "--filter=:e .rules"],
seed=_seed_rules(b"- *.log\n"), ci=True,
ref="dir-merge,e excludes the merge file itself"),
H.Case("dir_merge_n", "filters", ["-a", "--filter=:n .rules"],
seed=_seed_rules(b"- *.log\n"), ci=True,
ref="dir-merge,n does not inherit into subdirectories"),
H.Case("dir_merge_dash", "filters", ["-a", "--filter=:- .rules"],
seed=_seed_rules(b"*.log\n*.bin\n"), ci=True,
ref="dir-merge,- reads the file as bare exclude patterns"),
H.Case("dir_merge_w", "filters", ["-a", "--filter=:-w .rules"],
seed=_seed_rules(b"*.log *.bin\n"), ci=True,
ref="dir-merge,w word-splits bare patterns on whitespace"),
# --- relative / dirs --------------------------------------------------
H.Case("relative_general", "basic", ["-a", "-R"], layout=H.MIRROR_ABS,
@@ -666,6 +819,157 @@ def test_added_and_deleted_between_runs(parity_server_factory):
_run_and_check(case_id, result)
def _seed_dest_tree(src, root):
"""Copy `src`'s tree into `root` (the transfer mirror), preserving symlinks
and directory mtimes, so a second differential run starts from an existing
destination exactly like a seeded rsync run."""
os.makedirs(root, exist_ok=True)
for dirpath, dirnames, filenames in os.walk(src):
rel = os.path.relpath(dirpath, src)
for name in dirnames:
s = os.path.join(dirpath, name)
d = os.path.join(root, rel, name) if rel != "." else os.path.join(root, name)
if os.path.islink(s):
continue
os.makedirs(d, exist_ok=True)
for name in filenames:
s = os.path.join(dirpath, name)
d = os.path.join(root, rel, name) if rel != "." else os.path.join(root, name)
os.makedirs(os.path.dirname(d), exist_ok=True)
if os.path.islink(s):
if os.path.lexists(d):
os.remove(d)
os.symlink(os.readlink(s), d)
else:
shutil.copy2(s, d)
if rel != ".":
os.utime(os.path.join(root, rel), None)
os.utime(root, None)
# A full rsync itemize code (11 columns) followed by the name. H._ITEMIZE_RE
# only matches created (`+`) entries, so the changed-attribute codes this test
# asserts need their own matcher.
_ITEMIZE_LINE_RE = re.compile(r"^[<>ch.*][fdLDS].{9} ")
def _itemize_dir_link_lines(text):
"""The itemize lines for directory and symlink entries, excluding the
transfer-root `./` line (FastSync emits it unconditionally; a documented
residual)."""
out = []
for line in (text or "").splitlines():
line = line.rstrip()
if not line or not _ITEMIZE_LINE_RE.match(line):
continue
name = line.rsplit(" ", 1)[-1]
if name == "./":
continue
if name.endswith("/") or " -> " in line:
out.append(line)
return sorted(out)
@requires_rsync
@parity
def test_itemize_rerun_dirs_symlinks_matches_rsync(parity_server_factory):
"""#314: a re-run reports directory/symlink destination state like rsync.
On an unchanged tree FastSync emits no per-directory `cd+++++++++` (or
symlink) lines, and after a changed directory mtime / symlink target it
renders rsync's `.d..t......` / `cLc........` instead of `cd`/`cL`."""
case_id = "itemize_rerun_dirs_symlinks"
src = os.path.join(TEST_DATA_DIR, "parity_itemds_src")
rdst = os.path.join(TEST_DATA_DIR, "parity_itemds_rdst")
fdst = os.path.join(TEST_DATA_DIR, "parity_itemds_fdst")
clean_dir(src)
_mk(os.path.join(src, "sub", "b.txt"), b"nested\n")
os.makedirs(os.path.join(src, "emptydir"), exist_ok=True)
os.symlink("a.txt", os.path.join(src, "link"))
_mk(os.path.join(src, "a.txt"), b"top\n")
clean_dir(rdst)
clean_dir(fdst)
server = parity_server_factory(SUPER)
rroot = rdst
froot = get_dest_received_dir(fdst, src)
_seed_dest_tree(src, rroot)
_seed_dest_tree(src, froot)
# Unchanged re-run: no directory or symlink itemize lines from either tool.
rs = H.run_rsync(src, rdst, ["-a", "-i"])
fs, _ = H.run_fastsync(src, fdst, ["-a", "-i", "--incremental"], server.port)
assert rs.returncode == 0, rs.stderr
assert fs.returncode == 0, fs.stderr
assert _itemize_dir_link_lines(rs.stdout) == []
fast_unchanged = _itemize_dir_link_lines(fs.stdout)
assert fast_unchanged == [], f"unchanged re-run itemized dirs/links: {fast_unchanged}"
# Change the directory mtime and the symlink target, then re-run.
_pin(os.path.join(src, "sub"), _OLD_MTIME)
os.remove(os.path.join(src, "link"))
os.symlink("b.txt", os.path.join(src, "link"))
rs = H.run_rsync(src, rdst, ["-a", "-i"])
fs, _ = H.run_fastsync(src, fdst, ["-a", "-i", "--incremental"], server.port)
assert rs.returncode == 0, rs.stderr
assert fs.returncode == 0, fs.stderr
expected = _itemize_dir_link_lines(rs.stdout)
actual = _itemize_dir_link_lines(fs.stdout)
assert actual == expected, f"rsync={rs.stdout!r} fastsync={fs.stdout!r}"
assert any(line.endswith(" sub/") and line.startswith(".d..t") for line in actual), actual
assert any(line.startswith("cLc") and " -> b.txt" in line for line in actual), actual
def _deleted_breakdown_line(text):
for line in (text or "").splitlines():
if line.startswith("Number of deleted files:"):
return " ".join(line.split())
return ""
@requires_rsync
@parity
def test_stats_deleted_breakdown_matches_rsync(parity_server_factory):
"""#316: `--stats` renders rsync's per-type `Number of deleted files`
breakdown for removed regular files, directories, symlinks and a special."""
case_id = "stats_deleted_breakdown"
src = os.path.join(TEST_DATA_DIR, "parity_delbd_src")
rdst = os.path.join(TEST_DATA_DIR, "parity_delbd_rdst")
fdst = os.path.join(TEST_DATA_DIR, "parity_delbd_fdst")
clean_dir(src)
_mk(os.path.join(src, "keep.txt"), b"keep\n")
server = parity_server_factory(DELETE)
def seed(_src, rroot, froot):
for root in (rroot, froot):
_mk(os.path.join(root, "extra1.txt"), b"e1\n", _OLD_MTIME)
_mk(os.path.join(root, "extradir", "inside.txt"), b"e2\n", _OLD_MTIME)
os.makedirs(os.path.join(root, "extradir"), exist_ok=True)
link = os.path.join(root, "extralink")
if not os.path.lexists(link):
os.symlink("keep.txt", link)
fifo = os.path.join(root, "extrafifo")
if not os.path.exists(fifo):
os.mkfifo(fifo)
def extra(_src, _rroot, _froot, rs, fs):
rs_line = _deleted_breakdown_line(rs.stdout)
fs_line = _deleted_breakdown_line(fs.stdout)
if not rs_line:
return ["rsync printed no deleted-files line"]
if rs_line != fs_line:
return [f"deleted breakdown rsync={rs_line!r} fastsync={fs_line!r}"]
if "reg:" not in rs_line or "dir:" not in rs_line or \
"link:" not in rs_line or "special:" not in rs_line:
return [f"breakdown missing a category: {rs_line!r}"]
return []
result = H.run_differential(
src, rdst, fdst, ["-a", "--delete", "--stats"],
["-a", "--delete", "--stats", "--incremental"], server,
seed=seed, extra_check=extra)
_run_and_check(case_id, result, ref="--stats deleted per-type breakdown")
@requires_rsync
@parity
def test_one_file_system(parity_server_factory):
+1 -1
View File
@@ -36,7 +36,7 @@ from common import ( # noqa: E402
verify_transfer,
)
PROTOCOL_VERSION = b"2.29.0"
PROTOCOL_VERSION = b"2.30.0"
STATUS_MANIFEST = 5
STATUS_OK = 0
+258 -15
View File
@@ -203,9 +203,9 @@ class TestDeviceSpecial:
flags=["--devices"], port=port)
finally:
out, err = _stop_captured_server(server)
assert result.returncode != 0, (
f"a failed device mknod must be a transfer error like rsync (got exit 0): "
f"{(out + err)[:300]}"
assert result.returncode == 23, (
f"a failed device mknod must exit 23 (rsync partial transfer), got "
f"{result.returncode}: {(out + err)[:300]}"
)
received = get_dest_received_dir(DEVICE_DEST, DEVICE_SOURCE)
assert not os.path.lexists(os.path.join(received, "chardev")), (
@@ -215,6 +215,38 @@ class TestDeviceSpecial:
f"receiver did not log the device creation error: out={out!r} err={err!r}"
)
@pytest.mark.skipif(os.geteuid() != 0, reason="requires root to create device nodes")
def test_devices_nonroot_partial_removes_transferred_sources(self):
"""rsync parity for a partial receiver run under --remove-source-files:
the successfully transferred regular source is still removed, the
un-creatable device source is kept, and the client exits 23 (verified
against rsync 3.4.1: it removes ok.txt/ok2.txt, keeps the device, and
exits 23)."""
if os.geteuid() != 0 or shutil.which("setpriv") is None:
pytest.skip("requires root + setpriv to run the receiver unprivileged")
self._setup()
os.mknod(os.path.join(DEVICE_SOURCE, "chardev"), stat.S_IFCHR | 0o666,
os.makedev(1, 3))
os.makedirs(DEVICE_DEST, exist_ok=True)
os.chmod(DEVICE_DEST, 0o777)
server, port = _start_captured_server(
prefix=["setpriv", "--reuid=65534", "--regid=65534", "--clear-groups"])
try:
result, _ = run_client(DEVICE_SOURCE, DEVICE_DEST,
flags=["--devices", "--remove-source-files"], port=port)
finally:
out, err = _stop_captured_server(server)
assert result.returncode == 23, (
f"a partial receiver run must exit 23, got {result.returncode}: "
f"{(out + err)[:300]}"
)
assert not os.path.exists(os.path.join(DEVICE_SOURCE, "plain.txt")), (
"a successfully transferred source must be removed even on a partial run"
)
assert os.path.exists(os.path.join(DEVICE_SOURCE, "chardev")), (
"the source device that failed to materialize must be kept"
)
@pytest.mark.skipif(os.geteuid() != 0, reason="requires root to create device nodes")
def test_devices_recreates_real_char_device(self, shared_server):
"""Root-only: a source char device node is recreated on the destination
@@ -334,6 +366,57 @@ def setup_test_data():
shutil.rmtree(DEST_DIR, ignore_errors=True)
class TestClientStderrChannel:
"""--stderr=client: the client's own diagnostics go to the peer's stderr."""
@pytest.mark.ci
def test_client_diagnostic_reaches_server_stderr(self):
"""A client-side warning emitted during the transfer is forwarded over
the STATUS_CLIENT_MSG channel and printed on the server's stderr, not the
client's. A dangling symlink under -L is the deterministic trigger."""
source = os.path.join(TEST_DATA_DIR, "client_msg_src")
dest = os.path.join(TEST_DATA_DIR, "client_msg_dst")
clean_dir(source)
clean_dir(dest)
with open(os.path.join(source, "plain.txt"), "wb") as f:
f.write(b"payload\n")
os.symlink("no-such-referent", os.path.join(source, "dangling"))
server, port = _start_captured_server()
try:
result, _ = run_client(source, dest, flags=["-L", "--stderr=client"],
port=port)
finally:
out, err = _stop_captured_server(server)
assert "symlink has no referent" in (out + err), (
f"client diagnostic did not reach the server stderr: out={out!r} err={err!r}"
)
assert "symlink has no referent" not in (result.stderr or ""), (
f"client diagnostic must not also be written locally: {result.stderr!r}"
)
@pytest.mark.ci
def test_no_msgs2stderr_alias_uses_client_channel(self):
"""--no-msgs2stderr is rsync's spelling of --stderr=client and now
forwards the client's diagnostics to the server too."""
source = os.path.join(TEST_DATA_DIR, "client_msg_alias_src")
dest = os.path.join(TEST_DATA_DIR, "client_msg_alias_dst")
clean_dir(source)
clean_dir(dest)
with open(os.path.join(source, "plain.txt"), "wb") as f:
f.write(b"payload\n")
os.symlink("no-such-referent", os.path.join(source, "dangling"))
server, port = _start_captured_server()
try:
result, _ = run_client(source, dest, flags=["-L", "--no-msgs2stderr"],
port=port)
finally:
out, err = _stop_captured_server(server)
assert "symlink has no referent" in (out + err), (
f"--no-msgs2stderr did not route to the server: out={out!r} err={err!r}"
)
assert "symlink has no referent" not in (result.stderr or "")
class TestDryRun:
def test_trust_sender_transfer_completes(self, shared_server):
"""--trust-sender is a receiver-local policy (never sent to the peer).
@@ -2810,6 +2893,37 @@ class TestItemizeChanges:
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-i", "--dry-run"])
assert result.returncode == 0, f"dry-run -i failed: {result.stderr[:200]}"
def test_chunk_serialization_probes_ancestor_dir_state(self, shared_server):
"""#314: --chunk-serialization + -i must probe ancestor directory state
so a pre-existing directory with a changed mtime itemizes as an
attribute change (`.d..t......`) instead of being rendered as created
(`cd+++++++++`)."""
source = os.path.join(TEST_DATA_DIR, "itemize_chunk_serial_src")
dest = os.path.join(TEST_DATA_DIR, "itemize_chunk_serial_dst")
clean_dir(source)
clean_dir(dest)
subdir = os.path.join(source, "sub")
os.makedirs(subdir)
with open(os.path.join(subdir, "file.txt"), "wb") as fh:
fh.write(b"payload\n")
result, _ = run_client(source, dest, flags=["--preserve"], port=shared_server.port)
assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}"
# Change only the source directory's mtime; its contents stay identical
# so only the directory's time attribute differs on the rerun.
os.utime(subdir, (1_000_000_000, 1_000_000_000))
result, _ = run_client(source, dest,
flags=["--preserve", "-i", "--chunk-serialization"],
port=shared_server.port)
assert result.returncode == 0, f"chunk-serialization -i failed: {result.stderr[:200]}"
dir_lines = [line for line in result.stdout.splitlines() if line.endswith(" sub/")]
assert dir_lines == [".d..t...... sub/"], (
f"expected an attribute-change dir line, got {dir_lines!r}; "
f"full stdout={result.stdout!r}"
)
def test_changed_file_on_second_incremental_run_prints_exactly_one_line(self, shared_server):
"""A changed file itemizes exactly once on an incremental rerun while
unchanged files print nothing (no double emission)."""
@@ -2975,12 +3089,12 @@ class TestDelayUpdates:
"staging directory left behind after a successful delayed transfer"
@pytest.mark.skipif(shutil.which("rsync") is None, reason="rsync not installed")
def test_delay_updates_staging_name_collision_residual(self):
"""Documented residual (RSYNC_COMPAT.md `--delay-updates` row): FastSync
uses a fixed `.fastsync-stage` staging name and wipes a pre-existing tree
of that name at the start of a delayed run (crash-leftover cleanup),
even without `--delete`; rsync leaves a genuine destination entry of that
name untouched. Pins the divergence that keeps the row Divergent."""
def test_delay_updates_staging_name_collision_preserved(self):
"""rsync parity (RSYNC_COMPAT.md `--delay-updates` row): the receiver
stages under a per-run unique name, so a genuine pre-existing
destination entry named like the reserved staging prefix (`.fastsync-
stage`) is never wiped -- even without `--delete`. rsync likewise
leaves a real destination entry of its own temp name untouched."""
source = self._make_source("delay_collide_src")
rdst = os.path.join(TEST_DATA_DIR, "delay_collide_rdst")
fdst = os.path.join(TEST_DATA_DIR, "delay_collide_fdst")
@@ -3006,8 +3120,11 @@ class TestDelayUpdates:
result, _ = run_client(source, fdst, flags=["--delay-updates"],
port=server.port)
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
assert not os.path.exists(os.path.join(fdst, self.STAGING)), \
"FastSync did not wipe the reserved staging name (residual changed)"
assert _read_file(os.path.join(fdst, self.STAGING, "keepme.txt")) == b"genuine user data\n", \
"FastSync destroyed a genuine destination entry named like the staging prefix"
# The per-run staging directory itself is removed after a clean run.
leftovers = [n for n in os.listdir(fdst) if n.startswith(self.STAGING + ".")]
assert leftovers == [], f"per-run staging directories left behind: {leftovers}"
@pytest.mark.parametrize("mt", [False, True])
def test_delay_updates_incremental_rerun_no_leftovers(self, shared_server, mt):
@@ -3047,10 +3164,11 @@ class TestDelayUpdates:
@pytest.mark.parametrize("mt", [False, True])
def test_delete_with_delay_updates(self, mt):
"""--delete runs before publication, so the delete walker must not treat
the staging directory as a set of extras: a changed file must still be
published after genuine extras are removed. Uses its own server started
with --allow-delete (the shared session server refuses deletion)."""
"""rsync parity: --delay-updates implies --delete-after, so every staged
update is published first and the genuine extras are removed only after
that (the delete walker must never treat the staging directory as a set
of extras). Uses its own server started with --allow-delete (the shared
session server refuses deletion)."""
source = os.path.join(TEST_DATA_DIR, "delay_delete_src")
dest = os.path.join(TEST_DATA_DIR, "delay_delete_dst")
clean_dir(source)
@@ -3080,6 +3198,65 @@ class TestDelayUpdates:
assert not os.path.exists(os.path.join(received, "extra.txt")), \
"genuine extra file was not deleted"
assert not os.path.isdir(os.path.join(dest, self.STAGING))
assert [n for n in os.listdir(dest) if n.startswith(self.STAGING + ".")] == []
@pytest.mark.parametrize("mt", [False, True])
def test_delay_updates_delete_keeps_backup(self, mt):
"""The --backup/--delay-updates interplay: the old destination file is
moved aside at publication, and that backup survives the implied
--delete-after pass (rsync never treats a backup file as an extra)."""
source = os.path.join(TEST_DATA_DIR, "delay_bak_del_src")
dest = os.path.join(TEST_DATA_DIR, "delay_bak_del_dst")
clean_dir(source)
clean_dir(dest)
with open(os.path.join(source, "f.txt"), "wb") as fh:
fh.write(b"NEW")
received = get_dest_received_dir(dest, source)
os.makedirs(received, exist_ok=True)
with open(os.path.join(received, "f.txt"), "wb") as fh:
fh.write(b"OLD")
os.utime(os.path.join(received, "f.txt"), (1_500_000_000, 1_500_000_000))
# A pre-existing backup-looking extra must also be shielded.
with open(os.path.join(received, "stale.txt~"), "wb") as fh:
fh.write(b"stale backup")
with ServerManager() as server:
server.start(extra_args=["--allow-delete"])
flags = ["--delete", "--backup", "--delay-updates"] + (["--threads"] if mt else [])
result, _ = run_client(source, dest, flags=flags, port=server.port)
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
assert _read_file(os.path.join(received, "f.txt")) == b"NEW"
assert _read_file(os.path.join(received, "f.txt~")) == b"OLD", \
"the publication backup was removed by the delete-after pass"
assert os.path.exists(os.path.join(received, "stale.txt~")), \
"a pre-existing backup-suffixed entry was deleted"
@pytest.mark.parametrize("mt", [False, True])
def test_delay_updates_failed_run_leaves_no_staged_files(self, shared_server, mt):
"""A run that fails before publication installs nothing and removes the
per-run staging directory (no staged leftovers)."""
source = os.path.join(TEST_DATA_DIR, "delay_fail_src")
dest = os.path.join(TEST_DATA_DIR, "delay_fail_dst")
clean_dir(source)
clean_dir(dest)
with open(os.path.join(source, "top.txt"), "wb") as fh:
fh.write(b"top\n")
os.makedirs(os.path.join(source, "sub"))
with open(os.path.join(source, "sub", "deep.txt"), "wb") as fh:
fh.write(b"deep\n")
# Plant a regular file where the "sub" directory must be created so the
# nested publish fails (the top-level file still publishes first).
received = get_dest_received_dir(dest, source)
os.makedirs(received)
with open(os.path.join(received, "sub"), "wb") as fh:
fh.write(b"blocker")
flags = ["--delay-updates"] + (["--threads"] if mt else [])
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
assert result.returncode != 0, "a blocked nested publish must fail the run"
assert not os.path.lexists(os.path.join(received, "sub", "deep.txt")), \
"a staged file appeared despite the failed run"
assert [n for n in os.listdir(dest) if n.startswith(self.STAGING + ".")] == [], \
"the per-run staging directory survived a failed run"
def test_delay_updates_rejects_reserved_backup_dir(self):
"""--backup-dir equal to the internal staging name must be rejected so
@@ -6865,6 +7042,72 @@ class TestExtendedAttributes:
f"is not interoperable: ours={rec!r} rsync={out_rec!r}"
)
@pytest.mark.ci
def test_fake_super_directory_rsync_interop(self, shared_server):
"""#319: --fake-super fakes DIRECTORIES too. A recursive -a
--fake-super run must write rsync 3.4.1's `user.rsync.%stat` record on
the directory itself (full mode with S_IFDIR + special bits, rdev 0,0,
uid:gid), replay only the permission bits on disk, and real rsync must
read the tree and re-emit the identical record."""
rsync = shutil.which("rsync")
if rsync is None:
pytest.skip("rsync not installed")
source, dest = self._source_and_dest("fakesuper_dir_interop")
sub = os.path.join(source, "subdir")
os.makedirs(sub)
with open(os.path.join(sub, "f.txt"), "wb") as fh:
fh.write(b"dir interop\n")
if not _xattr_supported(sub):
pytest.skip("filesystem does not support user xattrs")
# A special bit (setgid) is exactly what a fake-super record exists to
# carry: rsync only re-emits a directory record when there is something
# it cannot represent on disk (a special bit, or a mode it would widen
# to keep the owner's rwx). Skip cleanly when the filesystem drops it.
os.chmod(sub, 0o2751)
if stat.S_IMODE(os.stat(sub).st_mode) & 0o7000 == 0:
pytest.skip("filesystem drops directory special bits")
uid = os.stat(sub).st_uid
result, _ = run_client(source, dest, flags=["-a", "--fake-super"],
port=shared_server.port)
assert result.returncode == 0, \
f"-a --fake-super dir sync failed: {(result.stderr or result.stdout)[:300]}"
received = get_dest_received_dir(dest, source)
dst_sub = os.path.join(received, "subdir")
assert os.path.isdir(dst_sub), "the directory entry was not transferred"
rec = os.getxattr(dst_sub, "user.rsync.%stat").decode()
fields = rec.split()
assert len(fields) == 3, f"unexpected rsync fake-super record {rec!r}"
mode_field, rdev_field, owner_field = fields
assert rdev_field == "0,0", f"directory rdev must be 0,0, got {rdev_field!r}"
assert int(mode_field, 8) & 0o170000 == stat.S_IFDIR, (
f"recorded mode {mode_field!r} must carry S_IFDIR"
)
assert int(mode_field, 8) & 0o7777 == 0o2751, (
f"recorded mode {mode_field!r} must carry the full source mode 02751"
)
assert owner_field.split(":")[0] == str(uid), \
f"recorded uid {owner_field!r} != source uid {uid}"
# Permission bits only on disk: the setgid bit stays in the record.
assert stat.S_IMODE(os.stat(dst_sub).st_mode) == 0o751, (
"the directory's special bits must not be installed on disk"
)
# Real rsync reads FastSync's directory record and re-emits it verbatim.
out = os.path.join(TEST_DATA_DIR, "fakesuper_dir_interop_rsync")
clean_dir(out)
rs = subprocess.run([rsync, "-aX", "--fake-super", received + "/", out + "/"],
capture_output=True, text=True, timeout=120)
assert rs.returncode == 0, (
f"rsync could not read FastSync's fake-super directory tree: {rs.stderr[:300]}"
)
out_rec = os.getxattr(os.path.join(out, "subdir"), "user.rsync.%stat").decode()
assert out_rec == rec, (
"rsync re-emitted a different directory fake-super record; FastSync's "
f"grammar is not interoperable: ours={rec!r} rsync={out_rec!r}"
)
@pytest.mark.ci
def test_directory_xattrs_preserved(self, shared_server):
"""#286.3: -aX must preserve user.* xattrs on DIRECTORIES, not just files."""
+222
View File
@@ -0,0 +1,222 @@
"""#318: whole-file streaming above the receiver's 256 MiB ceiling.
The receiver's historical whole-file bound (``MAX_RECEIVE_WHOLE_FILE_SIZE``,
256 MiB) refused any single-file payload above it. The transfer engine now
streams such a payload (and the basis read/verify/hash) through a bounded buffer
and spools it to a temp file, so arbitrarily large single files transfer without
being materialized in memory.
To exercise the streaming path deterministically and quickly, these tests lower
the receiver bound with the test-only ``FASTSYNC_MAX_WHOLE_FILE_SIZE`` hook (it
can only lower, never raise, the protocol ceiling) and transfer a file a few
times larger than the lowered bound. A real >256 MiB transfer is covered once,
unmarked, so it runs in the full suite but not the fast PR gate.
"""
import hashlib
import os
import random
import shutil
import sys
import pytest
sys.path.insert(0, os.path.dirname(__file__))
from common import ( # noqa: E402
ServerManager,
TEST_DATA_DIR,
clean_dir,
get_dest_received_dir,
run_client,
)
LOW_BOUND = 1024 * 1024
FILE_SIZE = 3 * 1024 * 1024
OLD_MTIME = 1_500_000_000
@pytest.fixture(scope="module")
def small_bound_server():
"""A server whose whole-file streaming bound is 1 MiB."""
server = ServerManager()
server.start(extra_args=["--allow-super"],
env={"FASTSYNC_MAX_WHOLE_FILE_SIZE": str(LOW_BOUND)})
yield server
server.stop()
def _payload(n):
rng = random.Random(0xC0FFEE)
return rng.randbytes(n)
def _write(path, data, mtime=None):
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "wb") as fh:
fh.write(data)
if mtime is not None:
os.utime(path, (mtime, mtime))
def _resolved(dest, source, rel):
return os.path.join(get_dest_received_dir(dest, source), rel)
def _no_spool_leftovers(dest):
leftovers = []
for root, _dirs, files in os.walk(dest):
leftovers += [os.path.join(root, f) for f in files if ".fastsync-spool." in f]
return leftovers
class TestStreamedWholeFile:
"""A file above the (lowered) bound transfers correctly in every mode."""
@pytest.mark.parametrize(
"flags",
[
["-a"],
["-a", "--incremental"],
["-a", "-z"],
["-a", "--incremental", "-z"],
["-a", "--threads", "--incremental"],
["-a", "--inplace"],
["-a", "--partial"],
],
)
def test_above_bound_transfers(self, small_bound_server, flags):
tag = "_".join(f.strip("-") for f in flags) or "default"
source = os.path.join(TEST_DATA_DIR, f"stream_src_{tag}")
dest = os.path.join(TEST_DATA_DIR, f"stream_dst_{tag}")
clean_dir(source)
clean_dir(dest)
data = _payload(FILE_SIZE)
_write(os.path.join(source, "big.bin"), data, OLD_MTIME)
if "--inplace" in flags:
# --inplace only matters when the destination already exists.
_write(_resolved(dest, source, "big.bin"), b"stale", OLD_MTIME)
result, _ = run_client(source, dest, flags=flags, port=small_bound_server.port)
assert result.returncode == 0, (result.stderr or result.stdout)[:400]
got = os.path.join(get_dest_received_dir(dest, source), "big.bin")
assert os.path.exists(got), "streamed file was not written"
with open(got, "rb") as fh:
assert fh.read() == data, "streamed file content mismatch"
assert _no_spool_leftovers(dest) == [], "a spool temp file leaked"
class TestStreamedBasis:
"""A basis above the bound is streamed, not refused (compare/copy/link)."""
def _seed(self, dest, source, data):
clean_dir(source)
clean_dir(dest)
_write(os.path.join(source, "big.bin"), data, OLD_MTIME)
# FastSync resolves a relative basis DIR against the destination and
# appends the transfer-relative name.
_write(os.path.join(dest, "basis", "big.bin"), data, OLD_MTIME)
def test_compare_dest_above_bound(self, small_bound_server):
source = os.path.join(TEST_DATA_DIR, "sbasis_cmp_src")
dest = os.path.join(TEST_DATA_DIR, "sbasis_cmp_dst")
data = _payload(FILE_SIZE)
self._seed(dest, source, data)
result, _ = run_client(source, dest,
flags=["-a", "--compare-dest=basis", "--incremental"],
port=small_bound_server.port)
assert result.returncode == 0, (result.stderr or result.stdout)[:400]
# compare-dest never copies: an already-present destination stays sparse.
assert not os.path.exists(_resolved(dest, source, "big.bin"))
def test_copy_dest_above_bound(self, small_bound_server):
source = os.path.join(TEST_DATA_DIR, "sbasis_cpy_src")
dest = os.path.join(TEST_DATA_DIR, "sbasis_cpy_dst")
data = _payload(FILE_SIZE)
self._seed(dest, source, data)
result, _ = run_client(source, dest,
flags=["-a", "--copy-dest=basis", "--incremental"],
port=small_bound_server.port)
assert result.returncode == 0, (result.stderr or result.stdout)[:400]
got = _resolved(dest, source, "big.bin")
assert os.path.exists(got)
with open(got, "rb") as fh:
assert fh.read() == data
assert os.stat(got).st_ino != os.stat(os.path.join(dest, "basis", "big.bin")).st_ino
assert _no_spool_leftovers(dest) == []
def test_link_dest_above_bound(self, small_bound_server):
source = os.path.join(TEST_DATA_DIR, "sbasis_lnk_src")
dest = os.path.join(TEST_DATA_DIR, "sbasis_lnk_dst")
data = _payload(FILE_SIZE)
self._seed(dest, source, data)
result, _ = run_client(source, dest,
flags=["-a", "--link-dest=basis", "--incremental"],
port=small_bound_server.port)
assert result.returncode == 0, (result.stderr or result.stdout)[:400]
got = _resolved(dest, source, "big.bin")
assert os.path.exists(got)
with open(got, "rb") as fh:
assert fh.read() == data
assert os.stat(got).st_ino == os.stat(os.path.join(dest, "basis", "big.bin")).st_ino
class TestFuzzyAboveBound:
"""-y/--fuzzy reuses a basis above the bound by streaming its signature."""
def test_fuzzy_oversized_sibling(self, small_bound_server):
source = os.path.join(TEST_DATA_DIR, "sfuzzy_src")
dest = os.path.join(TEST_DATA_DIR, "sfuzzy_dst")
clean_dir(source)
clean_dir(dest)
base = _payload(FILE_SIZE)
sibling = bytearray(base)
sibling[FILE_SIZE // 2:FILE_SIZE // 2 + 4096] = bytes(
(b + 1) % 256 for b in sibling[FILE_SIZE // 2:FILE_SIZE // 2 + 4096])
_write(os.path.join(source, "report_v2.txt"), base)
_write(os.path.join(get_dest_received_dir(dest, source), "report_v1.txt"), bytes(sibling))
result, _ = run_client(
source, dest,
flags=["-a", "--incremental", "--delta", "--fuzzy", "--stats"],
port=small_bound_server.port)
assert result.returncode == 0, (result.stderr or result.stdout)[:400]
got = _resolved(dest, source, "report_v2.txt")
with open(got, "rb") as fh:
assert fh.read() == base, "fuzzy reconstruction mismatch"
assert _no_spool_leftovers(dest) == []
class TestRealLargeFile:
"""A real >256 MiB transfer, run only in the full (non-PR-gate) suite."""
def test_real_300mib_transfer(self, shared_server):
source = os.path.join(TEST_DATA_DIR, "real_large_src")
dest = os.path.join(TEST_DATA_DIR, "real_large_dst")
clean_dir(source)
clean_dir(dest)
n = 300 * 1024 * 1024
# Deterministic, compressible pattern written in bounded chunks.
chunk = bytes(range(256)) * 4096
digest = hashlib.sha256()
with open(os.path.join(source, "big.bin"), "wb") as fh:
written = 0
while written < n:
piece = chunk[: min(len(chunk), n - written)]
fh.write(piece)
digest.update(piece)
written += len(piece)
result, _ = run_client(source, dest, flags=["-a", "--incremental"],
port=shared_server.port)
assert result.returncode == 0, (result.stderr or result.stdout)[:400]
got = os.path.join(get_dest_received_dir(dest, source), "big.bin")
assert os.path.getsize(got) == n
got_digest = hashlib.sha256()
with open(got, "rb") as fh:
while True:
block = fh.read(1 << 20)
if not block:
break
got_digest.update(block)
assert got_digest.hexdigest() == digest.hexdigest()
shutil.rmtree(source, ignore_errors=True)
shutil.rmtree(dest, ignore_errors=True)
+75
View File
@@ -536,3 +536,78 @@ class TestFilterProtect:
assert "extra.log" not in result.stdout, result.stdout
assert os.path.exists(os.path.join(received, "extra.log"))
assert os.path.exists(os.path.join(received, "other.txt"))
@pytest.mark.ci
def test_perdir_protect_dest_only_matches_rsync(self):
"""#315: a `P` rule inside a per-directory `.rsync-filter` is carried to
the receiver, so a destination-only extra matching ONLY that rule is
shielded under --delete. Both roots carry the same filter file (rsync's
receiver reads the destination one; FastSync carries the source's)."""
source = os.path.join(TEST_DATA_DIR, "fpdp_src")
dest = os.path.join(TEST_DATA_DIR, "fpdp_dst")
rdst = os.path.join(TEST_DATA_DIR, "fpdp_rdst")
clean_dir(source)
_write(os.path.join(source, "keep.txt"), b"keep\n")
_write(os.path.join(source, ".rsync-filter"), b"P extra.log\n")
clean_dir(rdst)
_write(os.path.join(rdst, ".rsync-filter"), b"P extra.log\n")
_write(os.path.join(rdst, "extra.log"), b"extra\n")
_write(os.path.join(rdst, "other.txt"), b"other\n")
rsync_result = _rsync(["-aF", "--delete", source + "/", rdst + "/"])
assert rsync_result.returncode == 0, rsync_result.stderr
assert os.path.exists(os.path.join(rdst, "extra.log")), "rsync did not protect extra.log"
assert not os.path.exists(os.path.join(rdst, "other.txt"))
clean_dir(dest)
received = get_dest_received_dir(dest, source)
os.makedirs(received, exist_ok=True)
_write(os.path.join(received, ".rsync-filter"), b"P extra.log\n")
_write(os.path.join(received, "extra.log"), b"extra\n")
_write(os.path.join(received, "other.txt"), b"other\n")
with ServerManager() as server:
server.start(extra_args=["--allow-delete"])
result, _ = run_client(source, dest, flags=["-aF", "--delete"], port=server.port)
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
assert os.path.exists(os.path.join(received, "extra.log")), (
"FastSync must protect a destination-only per-directory P match like rsync")
assert not os.path.exists(os.path.join(received, "other.txt"))
@requires_rsync
@pytest.mark.ci
def test_perdir_protect_dry_run_enumeration(self, shared_server):
"""#315: the -n/--dry-run would-delete enumeration also honors the
carried per-directory rules, matching rsync's `*deleting` set: a
destination-only entry matching only a `.rsync-filter` P rule is not
reported (nor removed)."""
source = os.path.join(TEST_DATA_DIR, "fpdp_nd_src")
dest = os.path.join(TEST_DATA_DIR, "fpdp_nd_dst")
rdst = os.path.join(TEST_DATA_DIR, "fpdp_nd_rdst")
clean_dir(source)
_write(os.path.join(source, "keep.txt"), b"keep\n")
_write(os.path.join(source, ".rsync-filter"), b"P extra.log\n")
received = get_dest_received_dir(dest, source)
clean_dir(rdst)
clean_dir(received)
for root in (rdst, received):
_write(os.path.join(root, "keep.txt"), b"keep\n")
_write(os.path.join(root, ".rsync-filter"), b"P extra.log\n")
_write(os.path.join(root, "extra.log"), b"extra\n")
_write(os.path.join(root, "other.txt"), b"other\n")
rsync_result = _rsync(["-an", "-i", "-F", "--delete", source + "/", rdst + "/"])
assert rsync_result.returncode == 0, rsync_result.stderr
rsync_del = sorted(l for l in rsync_result.stdout.splitlines()
if l.startswith("*deleting"))
assert rsync_del == ["*deleting other.txt"], f"unexpected rsync set: {rsync_del}"
with ServerManager() as server:
server.start(extra_args=["--allow-delete"])
result, _ = run_client(source, dest, flags=["-aF", "-n", "-i", "--delete"],
port=server.port)
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
fs_del = sorted(l for l in (result.stdout or "").splitlines()
if l.startswith("*deleting"))
assert fs_del == rsync_del, f"rsync={rsync_del}\nfastsync={fs_del}"
assert os.path.exists(os.path.join(received, "extra.log"))
assert os.path.exists(os.path.join(received, "other.txt"))
+3 -3
View File
@@ -133,14 +133,14 @@ def _seed_protocol_source(source):
class TestProtocol:
@pytest.mark.ci
def test_protocol_current_version_accepted(self, shared_server):
"""--protocol=2.29.0 (the current PROTOCOL_VERSION) is accepted and the
"""--protocol=2.30.0 (the current PROTOCOL_VERSION) is accepted and the
transfer completes normally."""
source = os.path.join(TEST_DATA_DIR, "proto_ok_src")
dest = os.path.join(TEST_DATA_DIR, "proto_ok_dst")
shutil.rmtree(dest, ignore_errors=True)
os.makedirs(dest)
_seed_protocol_source(source)
result, _ = run_client(source, dest, flags=["--protocol=2.29.0"],
result, _ = run_client(source, dest, flags=["--protocol=2.30.0"],
port=shared_server.port)
assert result.returncode == 0, \
f"--protocol current run failed: {(result.stderr or result.stdout)[:400]}"
@@ -157,7 +157,7 @@ class TestProtocol:
shutil.rmtree(dest, ignore_errors=True)
os.makedirs(dest)
_seed_protocol_source(source)
for bad in ("2.28.0", "2.27.0", "2.26.0", "2.25.0", "2.24.0", "2.23.0", "2.22.0", "2.21.0", "2.20.0",
for bad in ("2.29.0", "2.28.0", "2.27.0", "2.26.0", "2.25.0", "2.24.0", "2.23.0", "2.22.0", "2.21.0", "2.20.0",
"2.19.0", "2.18.0", "2.17.0", "2.15.0", "2.16.0", "216", "31"):
result, _ = run_client(source, dest, flags=[f"--protocol={bad}"],
port=shared_server.port)
+75 -10
View File
@@ -352,7 +352,7 @@ static void test_parse_args_protocol_accept_current() {
Config* cfg = valid_client_config();
EXPECT_NOT_NULL(cfg);
char* argv_equals[] = {"fastsync", "--source-dir", "/src",
"--dest-dir", "/dst", "--protocol=2.29.0"};
"--dest-dir", "/dst", "--protocol=2.30.0"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 6, argv_equals, positional_args, &positional_count), 0);
@@ -362,7 +362,7 @@ static void test_parse_args_protocol_accept_current() {
cfg = valid_client_config();
EXPECT_NOT_NULL(cfg);
char* argv_space[] = {"fastsync", "--source-dir", "/src", "--dest-dir",
"/dst", "--protocol", "2.29.0"};
"/dst", "--protocol", "2.30.0"};
positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 7, argv_space, positional_args, &positional_count), 0);
EXPECT_EQ_STR(cfg->version, PROTOCOL_VERSION);
@@ -372,10 +372,10 @@ static void test_parse_args_protocol_accept_current() {
/* Any --protocol value other than the current PROTOCOL_VERSION must end in
* failure (parse_args simply stores it; validate_config rejects it up front). */
static void test_parse_args_protocol_rejects_other_versions() {
static const char* const bad_versions[] = {"2.17", "2.16", "2.15.0", "2.16.0", "2.17.0",
"2.18.0", "2.19.0", "2.20.0", "2.21.0", "2.22.0",
"2.23.0", "2.24.0", "2.25.0", "2.26.0", "2.27.0",
"2.28.0", "216", "31", "abc", ""};
static const char* const bad_versions[] = {
"2.17", "2.16", "2.15.0", "2.16.0", "2.17.0", "2.18.0", "2.19.0",
"2.20.0", "2.21.0", "2.22.0", "2.23.0", "2.24.0", "2.25.0", "2.26.0",
"2.27.0", "2.28.0", "2.29.0", "216", "31", "abc", ""};
for (size_t i = 0; i < sizeof(bad_versions) / sizeof(bad_versions[0]); i++) {
Config* cfg = valid_client_config();
EXPECT_NOT_NULL(cfg);
@@ -2323,9 +2323,9 @@ static void test_parse_args_8_bit_output() {
}
static void test_parse_args_stderr_modes() {
static const char* const modes[] = {"errors", "all", "e", "a"};
static const LogStderrMode expected[] = {LOG_STDERR_ERRORS, LOG_STDERR_ALL, LOG_STDERR_ERRORS,
LOG_STDERR_ALL};
static const char* const modes[] = {"errors", "all", "client", "e", "a", "c"};
static const LogStderrMode expected[] = {LOG_STDERR_ERRORS, LOG_STDERR_ALL, LOG_STDERR_CLIENT,
LOG_STDERR_ERRORS, LOG_STDERR_ALL, LOG_STDERR_CLIENT};
for (size_t i = 0; i < sizeof(modes) / sizeof(modes[0]); i++) {
Config* cfg = config_create();
char option[32];
@@ -2340,8 +2340,29 @@ static void test_parse_args_stderr_modes() {
log_set_stderr_mode(LOG_STDERR_ERRORS);
}
/* rsync's deprecated --msgs2stderr / --no-msgs2stderr spellings map to
* --stderr=all and --stderr=client respectively; the client-message channel
* that `client` needs now exists (protocol 2.30.0). */
static void test_parse_args_msgs2stderr_aliases() {
Config* cfg = config_create();
char* argv_all[] = {"fastsync", "--msgs2stderr", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, argv_all, positional_args, &positional_count), 0);
EXPECT_EQ_INT(log_get_stderr_mode(), LOG_STDERR_ALL);
config_delete(cfg);
cfg = config_create();
char* argv_client[] = {"fastsync", "--no-msgs2stderr", "/src", "/dst"};
positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, argv_client, positional_args, &positional_count), 0);
EXPECT_EQ_INT(log_get_stderr_mode(), LOG_STDERR_CLIENT);
config_delete(cfg);
log_set_stderr_mode(LOG_STDERR_ERRORS);
}
static void test_parse_args_rejects_unsupported_stderr_modes() {
static const char* const modes[] = {"client", "c", "invalid"};
static const char* const modes[] = {"invalid", "x", ""};
for (size_t i = 0; i < sizeof(modes) / sizeof(modes[0]); i++) {
Config* cfg = config_create();
char option[32];
@@ -2956,6 +2977,48 @@ static void test_parse_args_delay_updates() {
config_delete(cfg);
}
/* rsync parity: --delay-updates implies --delete-after when --delete is
active (all updates publish first, then extras are removed). An explicit
other timing is overridden; without --delete no timing is set. */
static void test_parse_args_delay_updates_implies_delete_after() {
Config* cfg = config_create();
char* argv[] = {"fastsync", "--delay-updates", "--delete", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0);
EXPECT_TRUE(cfg->use_delete);
EXPECT_TRUE(cfg->delete_after);
EXPECT_FALSE(cfg->delete_before);
EXPECT_FALSE(cfg->delete_during);
EXPECT_FALSE(cfg->delete_delay);
cfg->send_directory = str_dup("/src");
cfg->receive_root_directory = str_dup("/dst");
EXPECT_TRUE(validate_config(cfg));
config_delete(cfg);
/* An explicit conflicting timing is normalized to delete-after. */
cfg = config_create();
char* argv_before[] = {"fastsync", "--delay-updates", "--delete-before", "/src", "/dst"};
positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 5, argv_before, positional_args, &positional_count), 0);
EXPECT_TRUE(cfg->use_delete);
EXPECT_TRUE(cfg->delete_after);
EXPECT_FALSE(cfg->delete_before);
config_delete(cfg);
/* Without --delete there is no deletion, so no timing is selected. */
cfg = config_create();
char* argv_alone[] = {"fastsync", "--delay-updates", "/src", "/dst"};
positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, argv_alone, positional_args, &positional_count), 0);
EXPECT_FALSE(cfg->use_delete);
EXPECT_FALSE(cfg->delete_after);
EXPECT_FALSE(cfg->delete_before);
EXPECT_FALSE(cfg->delete_during);
EXPECT_FALSE(cfg->delete_delay);
config_delete(cfg);
}
/* rsync rejects --delay-updates with --inplace; FastSync must too. */
static void test_validate_config_delay_updates_rejects_inplace() {
Config* cfg = valid_client_config();
@@ -5242,6 +5305,7 @@ void test_client_cli() {
test_parse_args_ignore_times();
test_parse_args_8_bit_output();
test_parse_args_stderr_modes();
test_parse_args_msgs2stderr_aliases();
test_parse_args_rejects_unsupported_stderr_modes();
test_parse_args_secluded_args();
test_parse_args_chunk_serialization_long_form();
@@ -5290,6 +5354,7 @@ void test_client_cli() {
test_parse_args_checksum_seed();
test_parse_args_temp_dir();
test_parse_args_delay_updates();
test_parse_args_delay_updates_implies_delete_after();
test_validate_config_delay_updates_rejects_inplace();
test_validate_config_delay_updates_rejects_reserved_backup_dir();
test_parse_args_files_from();
+8 -5
View File
@@ -2924,7 +2924,7 @@ static void golden_config_populate(Config* c) {
array_list_add(c->filters, str_dup("- /sub/dir/"));
}
/* The pinned golden frame (protocol 2.29.0). The values below are the only
/* The pinned golden frame (protocol 2.30.0). The values below are the only
* thing that ties the generated table to the historical wire format; update
* them ONLY with a PROTOCOL_VERSION bump and a documented reason. The 2.24.0
* delete-plan wave changed only the version string; 2.25.0 appended the
@@ -2936,10 +2936,13 @@ static void golden_config_populate(Config* c) {
* (project decision), so the frame grew by one int to 886 bytes. The 2.29.0
* symlink-xattr wave changes only the version string: the config-frame layout
* is unchanged (use_xattrs already crosses the wire); the STATUS_SYMLINK frame
* body grows instead. The byte-exact values are recomputed for the merged
* layout. */
* body grows instead. The 2.30.0 client-message/partial wave changes only the
* version string: the config-frame layout is unchanged (the new
* STATUS_CLIENT_MSG and STATUS_PARTIAL statuses are not part of this frame), so
* the length stays 886 and only the hash moves. The byte-exact values are
* recomputed for the merged layout. */
#define GOLDEN_WIRE_LEN 886
#define GOLDEN_WIRE_HASH 17827864270611927842ULL
#define GOLDEN_WIRE_HASH 4169866417069573876ULL
static unsigned long long fnv1a_64(const unsigned char* buf, size_t len) {
unsigned long long h = 1469598103934665603ULL;
@@ -3021,7 +3024,7 @@ static unsigned long long capture_wire_hash(const Config* cfg, size_t* out_len)
return h;
}
/* Byte-for-byte wire compatibility guard (protocol 2.29.0). The expected hash
/* Byte-for-byte wire compatibility guard (protocol 2.30.0). The expected hash
* pins the pre-X-macro byte stream; the refactor MUST NOT change it. */
static void test_config_wire_golden() {
if (is_running_under_valgrind())
+67 -5
View File
@@ -87,8 +87,10 @@ static void test_delay_updates_no_final_before_publish() {
const char* final_path = "test_delay_tmp/sub/file.txt";
/* Before publication the final destination must not contain the file. */
EXPECT_FALSE(file_path_exists_secure(final_path));
/* The complete staged copy must live inside the staging tree. */
char* staged = path_cat("test_delay_tmp/.fastsync-stage", "/sub/file.txt");
/* The complete staged copy must live inside the per-run staging tree. */
EXPECT_NOT_NULL(cfg->delay_context->staging_name);
EXPECT_EQ_INT(strncmp(cfg->delay_context->staging_name, ".fastsync-stage.", 16), 0);
char* staged = path_cat(cfg->delay_context->staging_root, "/sub/file.txt");
EXPECT_NOT_NULL(staged);
// cppcheck-suppress knownConditionTrueFalse
if (staged) {
@@ -125,6 +127,8 @@ static void test_delay_updates_publish_installs_files() {
const char* final_path = "test_delay_pub_tmp/sub/file.txt";
EXPECT_FALSE(file_path_exists_secure(final_path));
char* staging_root = str_dup(cfg->delay_context->staging_root);
EXPECT_NOT_NULL(staging_root);
EXPECT_TRUE(delay_updates_publish(cfg->delay_context, cfg));
/* After a successful publish the file is installed and staging is gone. */
char* content = read_all(final_path);
@@ -134,7 +138,8 @@ static void test_delay_updates_publish_installs_files() {
EXPECT_EQ_STR(content, "published payload");
free(content);
}
EXPECT_FALSE(file_path_exists_secure("test_delay_pub_tmp/.fastsync-stage"));
EXPECT_FALSE(file_path_exists_secure(staging_root));
free(staging_root);
out:
file_destroy(f);
@@ -158,11 +163,17 @@ static void test_delay_updates_cleanup_removes_staged() {
goto out;
EXPECT_EQ_INT(file_save_to_disk_full(root, f, cfg), FILE_SAVE_WRITTEN);
EXPECT_TRUE(file_path_exists_secure("test_delay_clean_tmp/.fastsync-stage/sub/file.txt"));
char* staged_file = path_cat(cfg->delay_context->staging_root, "/sub/file.txt");
char* staging_root = str_dup(cfg->delay_context->staging_root);
EXPECT_NOT_NULL(staged_file);
EXPECT_NOT_NULL(staging_root);
EXPECT_TRUE(file_path_exists_secure(staged_file));
delay_updates_cleanup(cfg->delay_context);
EXPECT_FALSE(file_path_exists_secure("test_delay_clean_tmp/.fastsync-stage"));
EXPECT_FALSE(file_path_exists_secure(staging_root));
EXPECT_FALSE(file_path_exists_secure("test_delay_clean_tmp/sub/file.txt"));
free(staged_file);
free(staging_root);
out:
file_destroy(f);
@@ -274,6 +285,55 @@ out:
remove_tree(root);
}
/* Every context picks its own staging directory name, so two delayed
transfers to the same root can never share (and corrupt) a staging tree. */
static void test_delay_updates_unique_staging_name() {
DelayUpdatesContext* first = delay_updates_context_create("test_delay_uniq_tmp");
DelayUpdatesContext* second = delay_updates_context_create("test_delay_uniq_tmp");
EXPECT_NOT_NULL(first);
EXPECT_NOT_NULL(second);
/* cppcheck-suppress knownConditionTrueFalse -- the EXPECT_NOT_NULL checks above return on NULL */
if (first && second) {
EXPECT_EQ_INT(strncmp(first->staging_name, ".fastsync-stage.", 16), 0);
EXPECT_EQ_INT(strncmp(second->staging_name, ".fastsync-stage.", 16), 0);
EXPECT_TRUE(strcmp(first->staging_name, second->staging_name) != 0);
EXPECT_TRUE(strcmp(first->staging_root, second->staging_root) != 0);
}
delay_updates_context_destroy(first);
delay_updates_context_destroy(second);
}
/* A pre-existing destination entry at the exact (random) staging path is not
ours: prepare() must refuse rather than wipe it. */
static void test_delay_updates_prepare_refuses_non_owned_collision() {
const char* root = "test_delay_collide_tmp";
remove_tree(root);
DelayUpdatesContext* context = delay_updates_context_create(root);
EXPECT_NOT_NULL(context);
// cppcheck-suppress knownConditionTrueFalse
if (!context)
return;
/* Plant a genuine directory with user data at the exact staging path. */
EXPECT_TRUE(file_ensure_directory_secure(context->staging_root));
char* inner = path_cat(context->staging_root, "keepme.txt");
EXPECT_NOT_NULL(inner);
// cppcheck-suppress knownConditionTrueFalse
if (inner) {
EXPECT_TRUE(file_write_to_disk(inner, "genuine", 7, false, false));
EXPECT_FALSE(delay_updates_prepare(context));
char* content = read_all(inner);
EXPECT_NOT_NULL(content);
// cppcheck-suppress knownConditionTrueFalse
if (content) {
EXPECT_EQ_STR(content, "genuine");
free(content);
}
free(inner);
}
delay_updates_context_destroy(context);
remove_tree(root);
}
/* The reserved staging name must be recognizable for validation, including
with a trailing slash. */
static void test_delay_updates_reserved_name_helper() {
@@ -288,6 +348,8 @@ static void test_delay_updates_reserved_name_helper() {
void test_delay_updates() {
test_delay_updates_reserved_name_helper();
test_delay_updates_unique_staging_name();
test_delay_updates_prepare_refuses_non_owned_collision();
test_delay_updates_no_final_before_publish();
test_delay_updates_publish_installs_files();
test_delay_updates_cleanup_removes_staged();
+207
View File
@@ -11,8 +11,23 @@
#include <string.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <threads.h>
#include <unistd.h>
/* Discards everything written to `fd` until EOF, so a sender that regresses to
* emitting an over-budget frame does not block forever on a full socket. */
typedef struct {
int fd;
} DrainArg;
static int drain_fd_thread(void* arg) {
DrainArg* drain = arg;
char buffer[8192];
while (read(drain->fd, buffer, sizeof(buffer)) > 0)
;
return 0;
}
/* Send one STATUS_DELETE_PLAN body (the leading status is consumed by the
* caller/receiver entry point) describing `dir` with no kept children. */
static void send_plan_frame(int fd, const char* dir) {
@@ -225,6 +240,7 @@ static void send_config_only_frame(int fd, const char* missing_path) {
EXPECT_TRUE(send_int(fd, 0)); /* size-skipped */
EXPECT_TRUE(send_int(fd, 1)); /* missing args */
EXPECT_TRUE(send_wire_str(fd, missing_path));
EXPECT_TRUE(send_int(fd, 0)); /* per-directory filter-rule block is empty */
EXPECT_TRUE(send_int(fd, 0)); /* apply = false */
EXPECT_TRUE(send_wire_str(fd, "."));
EXPECT_TRUE(send_int(fd, 0));
@@ -265,10 +281,201 @@ static void test_config_only_frame_applies_missing_args(void) {
config_delete(config);
}
/* The per-directory filter-rule block (protocol 2.30.0) must be bounded on
* receive: every count, the action/sides domain, the owner-directory syntax and
* the pattern length are validated so a hostile peer can neither overread nor
* allocate unboundedly. It also round-trips a valid group faithfully. */
static void test_filter_dir_rules_receive_bounds(void) {
int p[2];
FilterRuleList* out = NULL;
/* Group count beyond the cap is rejected. */
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
EXPECT_TRUE(send_int(p[1], MAX_FILTER_RULES + 1));
EXPECT_FALSE(delete_filter_dir_rules_receive(p[0], &out));
EXPECT_NULL(out);
close(p[0]);
close(p[1]);
/* A negative group count is rejected. */
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
EXPECT_TRUE(send_int(p[1], -1));
EXPECT_FALSE(delete_filter_dir_rules_receive(p[0], &out));
EXPECT_NULL(out);
close(p[0]);
close(p[1]);
/* An empty block is valid and yields NULL. */
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
EXPECT_TRUE(send_int(p[1], 0));
EXPECT_TRUE(delete_filter_dir_rules_receive(p[0], &out));
EXPECT_NULL(out);
close(p[0]);
close(p[1]);
/* An unknown action is a protocol error. */
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
EXPECT_TRUE(send_int(p[1], 1));
EXPECT_TRUE(send_wire_str(p[1], ""));
EXPECT_TRUE(send_int(p[1], 1));
EXPECT_TRUE(send_int(p[1], 999));
EXPECT_FALSE(delete_filter_dir_rules_receive(p[0], &out));
EXPECT_NULL(out);
close(p[0]);
close(p[1]);
/* An absolute owner directory is rejected (confinement). */
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
EXPECT_TRUE(send_int(p[1], 1));
EXPECT_TRUE(send_wire_str(p[1], "/etc"));
EXPECT_TRUE(send_int(p[1], 0));
EXPECT_FALSE(delete_filter_dir_rules_receive(p[0], &out));
EXPECT_NULL(out);
close(p[0]);
close(p[1]);
/* An over-long pattern is rejected before allocation. */
{
char* big = malloc(MAX_PROTECT_PATTERN_LEN + 2);
EXPECT_NOT_NULL(big);
memset(big, 'a', MAX_PROTECT_PATTERN_LEN + 1);
big[MAX_PROTECT_PATTERN_LEN + 1] = '\0';
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
EXPECT_TRUE(send_int(p[1], 1));
EXPECT_TRUE(send_wire_str(p[1], ""));
EXPECT_TRUE(send_int(p[1], 1));
EXPECT_TRUE(send_int(p[1], (int)FILTER_ACTION_EXCLUDE));
EXPECT_TRUE(send_int(p[1], (int)(FILTER_SIDE_SENDER | FILTER_SIDE_RECEIVER)));
EXPECT_TRUE(send_int(p[1], 0));
EXPECT_TRUE(send_int(p[1], 0));
EXPECT_TRUE(send_int(p[1], 0));
EXPECT_TRUE(send_int(p[1], 0));
EXPECT_TRUE(send_wire_str(p[1], big));
EXPECT_FALSE(delete_filter_dir_rules_receive(p[0], &out));
EXPECT_NULL(out);
close(p[0]);
close(p[1]);
free(big);
}
/* A valid group round-trips its owner, no-inherit flag and pattern. */
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
EXPECT_TRUE(send_int(p[1], 1));
EXPECT_TRUE(send_wire_str(p[1], "sub"));
EXPECT_TRUE(send_int(p[1], 1));
EXPECT_TRUE(send_int(p[1], (int)FILTER_ACTION_EXCLUDE));
EXPECT_TRUE(send_int(p[1], (int)(FILTER_SIDE_SENDER | FILTER_SIDE_RECEIVER)));
EXPECT_TRUE(send_int(p[1], 0));
EXPECT_TRUE(send_int(p[1], 0));
EXPECT_TRUE(send_int(p[1], 0));
EXPECT_TRUE(send_int(p[1], 1)); /* no_inherit */
EXPECT_TRUE(send_wire_str(p[1], "*.log"));
EXPECT_TRUE(delete_filter_dir_rules_receive(p[0], &out));
EXPECT_NOT_NULL(out);
EXPECT_EQ_INT(out->count, 1);
EXPECT_EQ_STR(out->items[0]->owner, "sub");
EXPECT_EQ_STR(out->items[0]->pattern, "*.log");
EXPECT_TRUE(out->items[0]->no_inherit);
filter_rule_list_free(out);
close(p[0]);
close(p[1]);
}
/* The per-directory rule sender enforces exactly the receiver's limits: an
* over-long pattern and an over-budget owner+pattern total are rejected locally
* with a clear error instead of emitting a frame the peer would abort the
* transfer on. A valid block still round-trips. */
static void test_filter_dir_rules_send_bounds(void) {
int p[2];
/* An over-long pattern is rejected before anything is written. */
{
FilterRuleList* list = filter_rule_list_create();
EXPECT_NOT_NULL(list);
FilterRule* rule = calloc(1, sizeof(FilterRule));
EXPECT_NOT_NULL(rule);
rule->action = FILTER_ACTION_EXCLUDE;
rule->sides = FILTER_SIDE_SENDER | FILTER_SIDE_RECEIVER;
rule->owner = str_dup("sub");
rule->pattern = malloc(MAX_PROTECT_PATTERN_LEN + 2);
EXPECT_NOT_NULL(rule->owner);
EXPECT_NOT_NULL(rule->pattern);
memset(rule->pattern, 'a', MAX_PROTECT_PATTERN_LEN + 1);
rule->pattern[MAX_PROTECT_PATTERN_LEN + 1] = '\0';
EXPECT_TRUE(filter_rule_list_add(list, rule));
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
EXPECT_FALSE(delete_filter_dir_rules_send(p[1], list));
close(p[0]);
close(p[1]);
filter_rule_list_free(list);
}
/* A cumulative owner+pattern total over MAX_FILTER_BYTES is rejected. */
{
FilterRuleList* list = filter_rule_list_create();
EXPECT_NOT_NULL(list);
int per = MAX_PROTECT_PATTERN_LEN;
int need = MAX_FILTER_BYTES / per + 1;
EXPECT_TRUE(need < MAX_FILTER_RULES);
for (int i = 0; i < need; i++) {
FilterRule* rule = calloc(1, sizeof(FilterRule));
EXPECT_NOT_NULL(rule);
rule->action = FILTER_ACTION_EXCLUDE;
rule->sides = FILTER_SIDE_SENDER | FILTER_SIDE_RECEIVER;
rule->owner = str_dup("");
rule->pattern = malloc((size_t)per + 1);
EXPECT_NOT_NULL(rule->owner);
EXPECT_NOT_NULL(rule->pattern);
memset(rule->pattern, 'b', (size_t)per);
rule->pattern[per] = '\0';
EXPECT_TRUE(filter_rule_list_add(list, rule));
}
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
DrainArg drain = {p[0]};
thrd_t drainer;
EXPECT_EQ_INT(thrd_create(&drainer, drain_fd_thread, &drain), thrd_success);
bool sent = delete_filter_dir_rules_send(p[1], list);
close(p[1]);
thrd_join(drainer, NULL);
EXPECT_FALSE(sent);
close(p[0]);
filter_rule_list_free(list);
}
/* A valid block still round-trips through send -> receive. */
{
FilterRuleList* list = filter_rule_list_create();
EXPECT_NOT_NULL(list);
FilterRule* rule = calloc(1, sizeof(FilterRule));
EXPECT_NOT_NULL(rule);
rule->action = FILTER_ACTION_EXCLUDE;
rule->sides = FILTER_SIDE_SENDER | FILTER_SIDE_RECEIVER;
rule->owner = str_dup("sub");
rule->pattern = str_dup("*.log");
EXPECT_NOT_NULL(rule->owner);
EXPECT_NOT_NULL(rule->pattern);
EXPECT_TRUE(filter_rule_list_add(list, rule));
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
EXPECT_TRUE(delete_filter_dir_rules_send(p[1], list));
FilterRuleList* out = NULL;
EXPECT_TRUE(delete_filter_dir_rules_receive(p[0], &out));
EXPECT_NOT_NULL(out);
EXPECT_EQ_INT(out->count, 1);
EXPECT_EQ_STR(out->items[0]->owner, "sub");
EXPECT_EQ_STR(out->items[0]->pattern, "*.log");
filter_rule_list_free(out);
close(p[0]);
close(p[1]);
filter_rule_list_free(list);
}
}
void test_delete_plan(void) {
test_delete_delay_refilled_dir_removed_recursively();
test_delete_delay_removed_file_counted();
test_delete_delay_max_delete_bounds_actual();
test_delete_delay_actual_removal_charges_budget();
test_config_only_frame_applies_missing_args();
test_filter_dir_rules_receive_bounds();
test_filter_dir_rules_send_bounds();
}
+167
View File
@@ -5,6 +5,8 @@
#include "file.h"
#include "file_receive.h"
#include "data.h"
#include "compression.h"
#include "delta.h"
#include "config.h"
#include "charset.h"
#include "utils.h"
@@ -2465,7 +2467,172 @@ static void test_manifest_would_delete_protects_absolute_basis() {
free(extra);
}
/* #318: a whole-file payload above the streaming bound must be written to a
* spool temp file in bounded chunks, not materialized in memory. Exercises the
* raw and zstd-compressed paths and asserts the exact bytes land in the spool. */
static void test_file_receive_payload_streams(void) {
const char* dir = "test_file_stream_tmp";
char dest_path[512];
snprintf(dest_path, sizeof(dest_path), "%s/out.bin", dir);
mkdir(dir, 0777);
const unsigned long long stream_limit = 4096;
size_t size = 20000;
unsigned char* payload = malloc(size);
EXPECT_NOT_NULL(payload);
for (size_t i = 0; i < size; i++)
payload[i] = (unsigned char)((i * 7 + 3) & 0xff);
/* Raw (uncompressed) streamed payload. */
{
int p[2];
EXPECT_EQ_INT(pipe(p), 0);
unsigned long long hdr = size;
EXPECT_TRUE(send_n_data(p[1], &hdr, sizeof(hdr)));
EXPECT_TRUE(send_n_data(p[1], payload, size));
Data* buffer = NULL;
char* spool = NULL;
unsigned long long out_size = 0;
EXPECT_TRUE(file_receive_payload(p[0], false, size, dest_path, stream_limit, &buffer, &spool,
&out_size));
EXPECT_NULL(buffer);
EXPECT_NOT_NULL(spool);
EXPECT_TRUE(out_size == size);
/* cppcheck-suppress knownConditionTrueFalse -- EXPECT_NOT_NULL/EXPECT_TRUE above returns on
* failure */
if (spool) {
FILE* fh = fopen(spool, "rb");
EXPECT_NOT_NULL(fh);
/* cppcheck-suppress knownConditionTrueFalse -- EXPECT_NOT_NULL/EXPECT_TRUE above returns on
* failure */
if (fh) {
unsigned char* got = malloc(size);
EXPECT_TRUE(fread(got, 1, size, fh) == size);
EXPECT_EQ_INT(memcmp(got, payload, size), 0);
free(got);
fclose(fh);
}
unlink(spool);
free(spool);
}
close(p[0]);
close(p[1]);
}
/* zstd-compressed payload whose logical size exceeds the bound: the frame is
* decompressed incrementally straight into the spool. */
{
unsigned char* copy = malloc(size);
EXPECT_NOT_NULL(copy);
memcpy(copy, payload, size);
Data* raw = data_create(copy, size); /* data_create takes ownership of copy */
Data* compressed = data_compress_codec(raw, COMPRESSION_ALGO_ZSTD, 3, 0);
data_destroy(raw);
EXPECT_NOT_NULL(compressed);
/* cppcheck-suppress knownConditionTrueFalse -- EXPECT_NOT_NULL/EXPECT_TRUE above returns on
* failure */
if (compressed) {
int p[2];
EXPECT_EQ_INT(pipe(p), 0);
EXPECT_TRUE(send_data(p[1], compressed));
Data* buffer = NULL;
char* spool = NULL;
unsigned long long out_size = 0;
EXPECT_TRUE(file_receive_payload(p[0], true, size, dest_path, stream_limit, &buffer, &spool,
&out_size));
EXPECT_NULL(buffer);
EXPECT_NOT_NULL(spool);
EXPECT_TRUE(out_size == size);
/* cppcheck-suppress knownConditionTrueFalse -- EXPECT_NOT_NULL/EXPECT_TRUE above returns on
* failure */
if (spool) {
FILE* fh = fopen(spool, "rb");
EXPECT_NOT_NULL(fh);
/* cppcheck-suppress knownConditionTrueFalse -- EXPECT_NOT_NULL/EXPECT_TRUE above returns on
* failure */
if (fh) {
unsigned char* got = malloc(size);
EXPECT_TRUE(fread(got, 1, size, fh) == size);
EXPECT_EQ_INT(memcmp(got, payload, size), 0);
free(got);
fclose(fh);
}
unlink(spool);
free(spool);
}
close(p[0]);
close(p[1]);
data_destroy(compressed);
}
}
free(payload);
unlink(dest_path);
rmdir(dir);
}
/* #318: the fd-based delta helpers must match the in-memory ones and stream the
* reconstruction to a descriptor without allocating the whole output. */
static void test_delta_stream_helpers(void) {
const unsigned char basis[] = {0x11, 0x22, 0x33, 0x44};
const char* basis_path = "test_file_delta_basis.bin";
int bfd = open(basis_path, O_RDWR | O_CREAT | O_TRUNC, 0600);
EXPECT_TRUE(bfd >= 0);
EXPECT_TRUE(write(bfd, basis, sizeof(basis)) == (ssize_t)sizeof(basis));
EXPECT_TRUE(lseek(bfd, 0, SEEK_SET) == 0);
DeltaSignature* fd_sig = delta_signature_create_fd_seeded(bfd, sizeof(basis), 4, 0);
DeltaSignature* mem_sig = delta_signature_create_seeded(basis, sizeof(basis), 4, 0);
EXPECT_NOT_NULL(fd_sig);
EXPECT_NOT_NULL(mem_sig);
/* cppcheck-suppress knownConditionTrueFalse -- EXPECT_NOT_NULL/EXPECT_TRUE above returns on
* failure */
if (fd_sig && mem_sig) {
EXPECT_TRUE(fd_sig->block_count == mem_sig->block_count);
for (uint32_t i = 0; i < fd_sig->block_count; i++) {
EXPECT_TRUE(fd_sig->blocks[i].adler32 == mem_sig->blocks[i].adler32);
EXPECT_TRUE(fd_sig->blocks[i].xxhash == mem_sig->blocks[i].xxhash);
}
}
delta_signature_destroy(fd_sig);
delta_signature_destroy(mem_sig);
DeltaInstruction instrs[2];
instrs[0].type = DELTA_INSTR_BLOCK_MATCH;
instrs[0].match.block_index = 0;
instrs[0].match.block_offset = 0;
instrs[0].match.length = 4;
instrs[1].type = DELTA_INSTR_LITERAL;
instrs[1].literal.data = (uint8_t*)"XY";
instrs[1].literal.length = 2;
Delta delta;
delta.new_file_size = 6;
delta.instruction_count = 2;
delta.instructions = instrs;
delta.delta_size = 0;
int out[2];
EXPECT_EQ_INT(pipe(out), 0);
EXPECT_TRUE(delta_apply_to_fd(NULL, bfd, sizeof(basis), &delta, 4, out[1]));
close(out[1]);
unsigned char got[6] = {0};
size_t total = 0;
while (total < sizeof(got)) {
ssize_t n = read(out[0], got + total, sizeof(got) - total);
if (n <= 0)
break;
total += (size_t)n;
}
EXPECT_TRUE(total == sizeof(got));
EXPECT_TRUE(memcmp(got, "\x11\x22\x33\x44XY", 6) == 0);
close(out[0]);
close(bfd);
unlink(basis_path);
}
void test_file() {
test_file_receive_payload_streams();
test_delta_stream_helpers();
test_file_create();
test_file_special_rdev_valid();
test_file_destroy_null();
+226 -29
View File
@@ -1,6 +1,7 @@
#include "test_filter.h"
#include "filter.h"
#include "test_utils.h"
#include "utils.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
@@ -64,44 +65,47 @@ static void test_filter_list_rejects_unsupported_modifiers() {
}
/* rsync accepts the merge-file modifiers e/n/w/- on merge and dir-merge rules.
* They must be consumed so they never leak into the merge filename. */
* They must be consumed so they never leak into the merge filename, and their
* semantics (exclude-self, no-inherit, word-split, no-prefixes) must be
* applied while the file is read. */
static void test_filter_list_accepts_merge_modifiers() {
char tmpl[] = "/tmp/fastsync_filter_mmod_XXXXXX";
EXPECT_TRUE(mkdtemp(tmpl) != NULL);
char path[512];
snprintf(path, sizeof(path), "%s/rules", tmpl);
FILE* fp = fopen(path, "w");
char prefixed[512];
char bare[512];
char words[512];
snprintf(prefixed, sizeof(prefixed), "%s/prefixed", tmpl);
snprintf(bare, sizeof(bare), "%s/bare", tmpl);
snprintf(words, sizeof(words), "%s/words", tmpl);
FILE* fp = fopen(prefixed, "w");
EXPECT_NOT_NULL(fp);
fputs("- *.tmp\n", fp);
fclose(fp);
fp = fopen(bare, "w");
EXPECT_NOT_NULL(fp);
fputs("*.log\n*.tmp\n", fp);
fclose(fp);
fp = fopen(words, "w");
EXPECT_NOT_NULL(fp);
fputs("*.log *.tmp\n", fp);
fclose(fp);
/* merge with e/n/w/- consumes the modifiers and reads the right file. */
static const char* const fmts[] = {
"merge,e %s", "merge,n %s", "merge,w %s", "merge,- %s", ".e %s", ".- %s",
};
for (size_t i = 0; i < sizeof(fmts) / sizeof(fmts[0]); i++) {
FilterRuleList* list = filter_rule_list_create();
EXPECT_NOT_NULL(list);
char rule[600];
char err[256] = "";
snprintf(rule, sizeof(rule), fmts[i], path);
bool ok = filter_rule_list_parse_append(list, rule, NULL, NULL, err, sizeof(err));
if (!ok)
printf(" merge rule '%s' errored: %s\n", rule, err);
EXPECT_TRUE(ok);
EXPECT_EQ_INT(list->count, 1);
EXPECT_EQ_STR(list->items[0]->pattern, "*.tmp");
filter_rule_list_free(list);
}
/* dir-merge with e/n/w/- registers the basename without the modifiers. */
/* dir-merge with e/n/w/- registers the basename without the modifiers and
* records the modifier flags; 'e' appends an exclude-self rule. */
static const struct {
const char* rule;
const char* want;
bool no_prefixes;
bool word_split;
bool no_inherit;
bool exclude_self;
} drules[] = {
{"dir-merge,e .rules", ".rules"}, {"dir-merge,n .rules", ".rules"},
{"dir-merge,w .rules", ".rules"}, {"dir-merge,- .rules", ".rules"},
{":e .rules", ".rules"}, {":- .rules", ".rules"},
{"dir-merge,e .rules", ".rules", false, false, false, true},
{"dir-merge,n .rules", ".rules", false, false, true, false},
{"dir-merge,w .rules", ".rules", false, true, false, false},
{"dir-merge,- .rules", ".rules", true, false, false, false},
{":e .rules", ".rules", false, false, false, true},
{":- .rules", ".rules", true, false, false, false},
};
for (size_t i = 0; i < sizeof(drules) / sizeof(drules[0]); i++) {
FilterRuleList* list = filter_rule_list_create();
@@ -112,11 +116,78 @@ static void test_filter_list_accepts_merge_modifiers() {
printf(" dir-merge rule '%s' errored: %s\n", drules[i].rule, err);
EXPECT_TRUE(ok);
EXPECT_EQ_INT(list->dir_merge_count, 1);
EXPECT_EQ_STR(list->dir_merge_names[0], drules[i].want);
EXPECT_EQ_STR(list->dir_merges[0].name, drules[i].want);
EXPECT_EQ_INT(list->dir_merges[0].no_prefixes, drules[i].no_prefixes);
EXPECT_EQ_INT(list->dir_merges[0].word_split, drules[i].word_split);
EXPECT_EQ_INT(list->dir_merges[0].no_inherit, drules[i].no_inherit);
EXPECT_EQ_INT(list->dir_merges[0].exclude_self, drules[i].exclude_self);
if (drules[i].exclude_self) {
EXPECT_EQ_INT(list->count, 1);
EXPECT_EQ_STR(list->items[0]->pattern, ".rules");
EXPECT_EQ_INT(list->items[0]->action, FILTER_ACTION_EXCLUDE);
} else {
EXPECT_EQ_INT(list->count, 0);
}
filter_rule_list_free(list);
}
unlink(path);
/* merge,n reads the file normally; no-inherit is meaningless for a single
* merge so the rule is not marked. */
{
FilterRuleList* list = filter_rule_list_create();
char err[256] = "";
char rule[600];
snprintf(rule, sizeof(rule), "merge,n %s", prefixed);
EXPECT_TRUE(filter_rule_list_parse_append(list, rule, NULL, NULL, err, sizeof(err)));
EXPECT_EQ_INT(list->count, 1);
EXPECT_EQ_STR(list->items[0]->pattern, "*.tmp");
EXPECT_FALSE(list->items[0]->no_inherit);
filter_rule_list_free(list);
}
/* merge,e adds an implicit exclude for the merge file's basename. */
{
FilterRuleList* list = filter_rule_list_create();
char err[256] = "";
char rule[600];
snprintf(rule, sizeof(rule), "merge,e %s", prefixed);
EXPECT_TRUE(filter_rule_list_parse_append(list, rule, NULL, NULL, err, sizeof(err)));
EXPECT_EQ_INT(list->count, 2);
EXPECT_EQ_STR(list->items[0]->pattern, "prefixed");
EXPECT_EQ_INT(list->items[0]->action, FILTER_ACTION_EXCLUDE);
EXPECT_EQ_STR(list->items[1]->pattern, "*.tmp");
filter_rule_list_free(list);
}
/* merge,- reads the file as bare exclude patterns with no prefix parsing. */
{
FilterRuleList* list = filter_rule_list_create();
char err[256] = "";
char rule[600];
snprintf(rule, sizeof(rule), "merge,- %s", bare);
EXPECT_TRUE(filter_rule_list_parse_append(list, rule, NULL, NULL, err, sizeof(err)));
EXPECT_EQ_INT(list->count, 2);
EXPECT_EQ_STR(list->items[0]->pattern, "*.log");
EXPECT_EQ_STR(list->items[1]->pattern, "*.tmp");
filter_rule_list_free(list);
}
/* merge,-w word-splits bare patterns on whitespace. */
{
FilterRuleList* list = filter_rule_list_create();
char err[256] = "";
char rule[600];
snprintf(rule, sizeof(rule), "merge,w- %s", words);
EXPECT_TRUE(filter_rule_list_parse_append(list, rule, NULL, NULL, err, sizeof(err)));
EXPECT_EQ_INT(list->count, 2);
EXPECT_EQ_STR(list->items[0]->pattern, "*.log");
EXPECT_EQ_STR(list->items[1]->pattern, "*.tmp");
filter_rule_list_free(list);
}
unlink(prefixed);
unlink(bare);
unlink(words);
rmdir(tmpl);
}
@@ -187,6 +258,59 @@ static void test_filter_list_accepts_supported_rules_and_modifiers() {
}
}
/* A "clear"/"!" inside a merge file resets the list to empty. Rules read after
* it must still be owned by the merge file's directory (and marked no-inherit
* when the dir-merge says so). The base index must follow the clear down: when
* it was captured before the clear, post-clear rules sat below it and were left
* globally owned by "" (and unmarked). */
static void test_filter_merge_clear_then_owner() {
char tmpl[] = "/tmp/fastsync_filter_clear_XXXXXX";
EXPECT_TRUE(mkdtemp(tmpl) != NULL);
char path[512];
snprintf(path, sizeof(path), "%s/.rsync-filter", tmpl);
FILE* fp = fopen(path, "w");
EXPECT_NOT_NULL(fp);
fputs("- *.tmp\n!\nP *.log\n", fp);
fclose(fp);
FilterRuleList* list = filter_rule_list_create();
EXPECT_NOT_NULL(list);
char err[256] = "";
/* A pre-existing rule that the in-file clear must discard. */
EXPECT_TRUE(filter_rule_list_parse_append(list, "- keep.txt", NULL, NULL, err, sizeof(err)));
EXPECT_EQ_INT(list->count, 1);
FilterDirMerge spec = {.name = ".rsync-filter", .no_inherit = true};
bool exists = false;
EXPECT_TRUE(filter_dir_merge_append(list, tmpl, &spec, "sub", NULL, &exists, err, sizeof(err)));
EXPECT_TRUE(exists);
/* Only the post-clear rule survives, owned by "sub" and no-inherit. */
EXPECT_EQ_INT(list->count, 1);
EXPECT_EQ_STR(list->items[0]->pattern, "*.log");
EXPECT_EQ_STR(list->items[0]->owner, "sub");
EXPECT_TRUE(list->items[0]->no_inherit);
filter_rule_list_free(list);
/* A parse failure after the clear must roll the list back to the post-clear
* base (empty here), freeing the post-clear rule rather than retaining it. */
fp = fopen(path, "w");
EXPECT_NOT_NULL(fp);
fputs("- *.tmp\n!\nP *.log\n-e bogus\n", fp);
fclose(fp);
list = filter_rule_list_create();
EXPECT_NOT_NULL(list);
EXPECT_TRUE(filter_rule_list_parse_append(list, "- keep.txt", NULL, NULL, err, sizeof(err)));
EXPECT_EQ_INT(list->count, 1);
exists = false;
EXPECT_FALSE(filter_dir_merge_append(list, tmpl, &spec, "sub", NULL, &exists, err, sizeof(err)));
EXPECT_TRUE(exists);
EXPECT_EQ_INT(list->count, 0);
filter_rule_list_free(list);
unlink(path);
rmdir(tmpl);
}
static void test_filter_list_merge_file_still_supported() {
char tmpl[] = "/tmp/fastsync_filter_XXXXXX";
EXPECT_TRUE(mkdtemp(tmpl) != NULL);
@@ -283,12 +407,85 @@ static void test_filter_rules_apply_supported_modifiers() {
}
}
static FilterRule* chain_rule(const char* owner, const char* pattern, FilterAction action,
bool no_inherit) {
FilterRule* rule = calloc(1, sizeof(FilterRule));
if (!rule)
return NULL;
rule->action = action;
rule->sides = FILTER_SIDE_SENDER | FILTER_SIDE_RECEIVER;
rule->owner = str_dup(owner);
rule->pattern = str_dup(pattern);
rule->no_inherit = no_inherit;
if (!rule->owner || !rule->pattern) {
filter_rule_free(rule);
return NULL;
}
return rule;
}
/* The receiver's per-directory chain: a containing directory's rules win over
* an ancestor's (deepest-first), root rules are inherited, and a no-inherit
* rule applies only to its own directory's direct children. */
static void test_filter_dir_rules_chain(void) {
FilterRuleList* list = filter_rule_list_create();
EXPECT_NOT_NULL(list);
FilterRule* root_log = chain_rule("", "*.log", FILTER_ACTION_EXCLUDE, false);
FilterRule* sub_keep = chain_rule("sub", "keep.log", FILTER_ACTION_INCLUDE, false);
FilterRule* sub_tmp = chain_rule("sub", "*.tmp", FILTER_ACTION_EXCLUDE, true);
EXPECT_NOT_NULL(root_log);
EXPECT_NOT_NULL(sub_keep);
EXPECT_NOT_NULL(sub_tmp);
EXPECT_TRUE(filter_rule_list_add(list, root_log));
EXPECT_TRUE(filter_rule_list_add(list, sub_keep));
EXPECT_TRUE(filter_rule_list_add(list, sub_tmp));
/* Root rule inherited by every directory (leaf match). */
EXPECT_EQ_INT(filter_dir_rules_apply_side(list, "a.log", "a.log", false), FILTER_ACTION_PROTECT);
EXPECT_EQ_INT(filter_dir_rules_apply_side(list, "sub/a.log", "a.log", false),
FILTER_ACTION_PROTECT);
/* The deeper include overrides the inherited root exclude. */
EXPECT_EQ_INT(filter_dir_rules_apply_side(list, "sub/keep.log", "keep.log", false),
FILTER_ACTION_RISK);
/* No-inherit applies directly in its owner... */
EXPECT_EQ_INT(filter_dir_rules_apply_side(list, "sub/x.tmp", "x.tmp", false),
FILTER_ACTION_PROTECT);
/* ...but not below it. */
EXPECT_EQ_INT(filter_dir_rules_apply_side(list, "sub/deep/x.tmp", "x.tmp", false),
FILTER_ACTION_NONE);
/* No matching rule. */
EXPECT_EQ_INT(filter_dir_rules_apply_side(list, "sub/deep/plain.txt", "plain.txt", false),
FILTER_ACTION_NONE);
filter_rule_list_free(list);
}
static void test_filter_rule_clone_copies_every_field(void) {
char err[128] = "";
FilterRule* original = filter_rule_parse("-!p /a/*.o", NULL, err, sizeof(err));
EXPECT_NOT_NULL(original);
FilterRule* copy = filter_rule_clone(original);
EXPECT_NOT_NULL(copy);
EXPECT_TRUE(copy != original);
EXPECT_EQ_INT(copy->action, original->action);
EXPECT_EQ_INT(copy->sides, original->sides);
EXPECT_EQ_INT(copy->anchored, original->anchored);
EXPECT_EQ_INT(copy->dir_only, original->dir_only);
EXPECT_EQ_INT(copy->negate, original->negate);
EXPECT_EQ_INT(copy->perishable, original->perishable);
EXPECT_EQ_STR(copy->pattern, original->pattern);
filter_rule_free(original);
filter_rule_free(copy);
}
void test_filter() {
test_filter_list_rejects_xattr_modifier();
test_filter_list_rejects_unsupported_modifiers();
test_filter_list_accepts_merge_modifiers();
test_filter_list_accepts_supported_rules_and_modifiers();
test_filter_list_merge_file_still_supported();
test_filter_merge_clear_then_owner();
test_filter_rule_parse_rejects_unsupported_and_keeps_supported();
test_filter_rules_apply_supported_modifiers();
test_filter_dir_rules_chain();
test_filter_rule_clone_copies_every_field();
}
+10
View File
@@ -64,6 +64,7 @@ static void test_dest_state_roundtrip() {
memset(&out, 0, sizeof(out));
out.known = true;
out.existed = true;
out.target_matches = true;
out.size = 123456789ULL;
out.mtime_sec = 1700000000;
out.mtime_nsec = 123456789;
@@ -76,6 +77,7 @@ static void test_dest_state_roundtrip() {
EXPECT_TRUE(format_dest_state_receive(fds[1], &in));
EXPECT_TRUE(in.known);
EXPECT_TRUE(in.existed);
EXPECT_TRUE(in.target_matches);
EXPECT_TRUE(in.size == out.size);
EXPECT_TRUE(in.mtime_sec == out.mtime_sec);
EXPECT_TRUE(in.mtime_nsec == out.mtime_nsec);
@@ -103,6 +105,10 @@ static void test_stats_roundtrip() {
out.created_dir = 4;
out.created_link = 2;
out.created_special = 1;
out.deleted_reg = 9;
out.deleted_dir = 6;
out.deleted_link = 3;
out.deleted_special = 2;
ReceiverStats in;
memset(&in, 0, sizeof(in));
EXPECT_TRUE(format_stats_send(fds[0], &out));
@@ -115,6 +121,10 @@ static void test_stats_roundtrip() {
EXPECT_TRUE(in.created_dir == out.created_dir);
EXPECT_TRUE(in.created_link == out.created_link);
EXPECT_TRUE(in.created_special == out.created_special);
EXPECT_TRUE(in.deleted_reg == out.deleted_reg);
EXPECT_TRUE(in.deleted_dir == out.deleted_dir);
EXPECT_TRUE(in.deleted_link == out.deleted_link);
EXPECT_TRUE(in.deleted_special == out.deleted_special);
close(fds[0]);
close(fds[1]);
}

Some files were not shown because too many files have changed in this diff Show More