Compare commits
29
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b7e95b8d96 | ||
|
|
13627e82fc | ||
|
|
09ab81120e | ||
|
|
2a7afbda0a | ||
|
|
bab6fcc706 | ||
|
|
6a426cffa8 | ||
|
|
b74182c7c1 | ||
|
|
f2a8eeaea7 | ||
|
|
18d7d495cf | ||
|
|
59d20e867a | ||
|
|
cc931790e9 | ||
|
|
60776b78fc | ||
|
|
52ad0aa512 | ||
|
|
759ffea117 | ||
|
|
ef37c2b988 | ||
|
|
a14fbb2c10 | ||
|
|
7408686370 | ||
|
|
90f2fb613f | ||
|
|
8843f1e3cf | ||
|
|
c387beb182 | ||
|
|
96e02f52c0 | ||
|
|
f7d5dda93b | ||
|
|
6d9cdb83ba | ||
|
|
7c748f1f7a | ||
|
|
884530c9a2 | ||
|
|
729f3ef8e1 | ||
|
|
b414f197af | ||
|
|
29f8be161c | ||
|
|
cb2979fdf1 |
+33
-2
@@ -6,6 +6,36 @@ run the same version because the handshake is strict.
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
Wire backlog cycle (protocol 2.29.0 → 2.30.0; config-frame layout unchanged).
|
||||
|
||||
- **`--stderr=client` client-message channel (#313):** the client now accepts
|
||||
`--stderr=client` (and maps the deprecated `--no-msgs2stderr` to it), routing
|
||||
its own diagnostics over the new bounded `STATUS_CLIENT_MSG` client->server
|
||||
frame instead of writing them locally; the server writes each received
|
||||
message to its stderr (respecting the server log destination). `errors`/`all`
|
||||
behavior is unchanged.
|
||||
- **Receiver partial failures exit 23 (#320):** a per-entry receiver failure
|
||||
that does not abort the stream (e.g. an unprivileged `--devices` mknod) now
|
||||
sends the terminal `STATUS_PARTIAL`; the client exits 23 like rsync and, under
|
||||
`--remove-source-files`, still removes the sources it successfully
|
||||
transferred. A clean run stays 0 and a fatal/connection error stays non-23.
|
||||
- **Directory/symlink destination-state itemize (#314):** when `report_dest_info`
|
||||
is negotiated (now also for `--progress`), the receiver answers `STATUS_MKDIR`
|
||||
and `STATUS_SYMLINK` with the entry's pre-transfer destination snapshot
|
||||
(existence, type, perms/owner/group/time, and whether an existing symlink's
|
||||
target already matches), and the sender probes every ancestor directory before
|
||||
the receiver creates it implicitly. A re-run over an unchanged tree no longer
|
||||
emits per-directory `cd+++++++++` or unchanged-symlink lines, a changed
|
||||
directory renders rsync's `.d..t......`, and a changed symlink renders
|
||||
`cLc........` / `.L..t......`. Directory/symlink time comparison uses whole
|
||||
seconds (rsync's `cmp_time`). The `STATUS_MKDIR` body gains a probe flag and
|
||||
the `STATUS_DEST_INFO` record gains a symlink-target-match field; the
|
||||
config-frame layout is unchanged. Differential-tested against rsync 3.4.1.
|
||||
- **`--stats` deleted per-type breakdown (#316):** `STATUS_STATS` gains
|
||||
`deleted_reg/dir/link/special`, tallied by the delete observers and rendered
|
||||
as rsync's `Number of deleted files: X (reg: A, dir: B, link: C, special: D)`.
|
||||
Differential-tested against rsync 3.4.1 for a mixed-type `--delete` tree.
|
||||
|
||||
## [2.29.0] - 2026-09-23
|
||||
|
||||
The rsync-parity cycle 2.29 (no wire change; `PROTOCOL_VERSION` stays 2.28.0).
|
||||
@@ -82,8 +112,9 @@ layout is unchanged (golden length still 886).
|
||||
wording, and symlink/empty-directory quick-checks.
|
||||
- `--delete-before`'s phase-0 late-file divergence remains (rsync's pre-scan
|
||||
fixes the file list before the data pass).
|
||||
- A single file larger than 256 MiB cannot be streamed in the default path
|
||||
(a general whole-file limit, not basis-specific).
|
||||
- A whole-file sender that cannot stream its codec (lz4's one-shot block
|
||||
format) or a `--append`/delta source above the bound still buffers; the
|
||||
default zstd/zlib and the uncompressed paths stream (see #318).
|
||||
- `--stats` byte totals and `--msgs2stderr` stay documented divergences.
|
||||
|
||||
### Security
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
cmake_minimum_required(VERSION 3.22)
|
||||
|
||||
project(FastFileTransfer VERSION 2.29.0)
|
||||
project(FastFileTransfer VERSION 2.30.0)
|
||||
|
||||
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
|
||||
set(CMAKE_C_STANDARD 11)
|
||||
|
||||
+3
-1
@@ -235,7 +235,9 @@
|
||||
and symlink/empty-dir quick-check feedback.
|
||||
- **`--delete-before` phase-0 keep-set** (rsync fixes the file list before
|
||||
the data pass; FastSync keeps its pre-scan snapshot race).
|
||||
- **>256 MiB single-file streaming** (B4, the general whole-file limit).
|
||||
- ~~**>256 MiB single-file streaming** (B4, the general whole-file limit).~~
|
||||
Closed by #318: the whole-file payload, the basis read/verify and the fuzzy
|
||||
basis are streamed through bounded buffers (lz4/append remain buffered).
|
||||
- **Wire native-size framing:** lengths are native `size_t` and the protocol
|
||||
assumes homogeneous word size/endianness — document or move to fixed-width
|
||||
framing.
|
||||
|
||||
@@ -206,9 +206,9 @@ This produces `./build/client` and `./build/server`. `compile_commands.json` is
|
||||
| `--incremental` | Skip files unchanged since last transfer (size + mtime). Auto-enables `--preserve`. Incompatible with `--chunk-serialization`. |
|
||||
| `--existing` | Skip files not already present at the destination; update existing files normally. |
|
||||
| `--ignore-existing` | Skip files that already exist on the receiver; like rsync it does not apply to directories or symlinks. |
|
||||
| `--compare-dest <dir>` | Extra comparison basis: unchanged files are not transferred (requires/implies `--incremental`; a basis MISS above the 256 MiB whole-file payload bound is refused — see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)) |
|
||||
| `--copy-dest <dir>` | Like `--compare-dest`, but copies the unchanged file from DIR into the destination (same basis-size caveat; see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)) |
|
||||
| `--link-dest <dir>` | Like `--copy-dest`, but hard-links the unchanged file from DIR (repeatable; earlier DIRs win; same basis-size caveat; see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)) |
|
||||
| `--compare-dest <dir>` | Extra comparison basis: unchanged files are not transferred (requires/implies `--incremental`; a basis of any size is supported, streamed in bounded chunks — see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)) |
|
||||
| `--copy-dest <dir>` | Like `--compare-dest`, but copies the unchanged file from DIR into the destination (the copy is streamed, so a basis of any size works; see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)) |
|
||||
| `--link-dest <dir>` | Like `--copy-dest`, but hard-links the unchanged file from DIR (repeatable; earlier DIRs win; a basis of any size works; see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)) |
|
||||
| `--verify-basis` | FastSync-only: require a basis hit (`--compare-dest`/`--copy-dest`/`--link-dest`) to match the source by whole-file digest instead of trusting the size+mtime quick-check (default matches rsync) |
|
||||
| `--delete` | Delete files on receiver not present in source (default timing: delete-during, matching rsync, so destination space is freed progressively). Scoped to the synchronized directories, so `--files-from` subsets are safe |
|
||||
| `--delete-before` | Delete extras before the transfer starts (implies `--delete`) |
|
||||
@@ -300,6 +300,7 @@ transfer is never aborted.
|
||||
| `FASTSYNC_SOURCE_DIR` | — | Source directory fallback |
|
||||
| `FASTSYNC_DEST_DIR` | — | Destination directory fallback |
|
||||
| `FASTSYNC_SAVE_TO_DISK` | `false` | Disk persistence fallback |
|
||||
| `FASTSYNC_MAX_WHOLE_FILE_SIZE` | `268435456` | Receiver-only test hook: a byte count that lowers the whole-file streaming bound. Payloads above it are streamed through a bounded buffer. Values are clamped to the 256 MiB protocol ceiling, so it can only lower, never raise, the bound. |
|
||||
|
||||
## Implementation Details
|
||||
|
||||
@@ -580,9 +581,9 @@ remote SSH argv is already built injection-safe.
|
||||
| `--files-from <file>` | Read the source file list from FILE (paths relative to the source root). |
|
||||
| `-0, --from0` | Treat entries in `--files-from` files as NUL-delimited instead of newline-delimited. |
|
||||
| `--delay-updates` | Put updated files into place only at the end of the transfer (the fixed `.fastsync-stage` staging name diverges from rsync; see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)). |
|
||||
| `--compare-dest <dir>` | Extra comparison basis: unchanged files are not transferred (requires/implies `--incremental`; a basis MISS above the 256 MiB whole-file payload bound is refused — see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)). |
|
||||
| `--copy-dest <dir>` | Like `--compare-dest`, but copies the unchanged file from DIR into the destination (same basis-size caveat; see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)). |
|
||||
| `--link-dest <dir>` | Like `--copy-dest`, but hard-links the unchanged file from DIR (repeatable; earlier DIRs win; same basis-size caveat; see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)). |
|
||||
| `--compare-dest <dir>` | Extra comparison basis: unchanged files are not transferred (requires/implies `--incremental`; a basis of any size is supported, streamed in bounded chunks — see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)). |
|
||||
| `--copy-dest <dir>` | Like `--compare-dest`, but copies the unchanged file from DIR into the destination (the copy is streamed, so a basis of any size works; see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)). |
|
||||
| `--link-dest <dir>` | Like `--copy-dest`, but hard-links the unchanged file from DIR (repeatable; earlier DIRs win; a basis of any size works; see [`RSYNC_COMPAT.md`](RSYNC_COMPAT.md)). |
|
||||
| `--verify-basis` | FastSync-only: require a basis hit to match the source by whole-file digest instead of trusting the size+mtime quick-check (default matches rsync). |
|
||||
| `--preallocate` | Allocate destination file space up front (fail-fast on a full disk). |
|
||||
| `--append` | Resume a shorter destination by appending only its tail (prefix not verified; requires `--incremental`). |
|
||||
@@ -674,9 +675,9 @@ remote SSH argv is already built injection-safe.
|
||||
| `--outbuf=MODE` | stdout/stderr buffering: `N` (none/unbuffered), `L` (line-buffered), or `B` (block-buffered, default). |
|
||||
| `--log-file <path>` | Write log output to a file. |
|
||||
| `--log-file-format=FORMAT` | Per-file log-line format (requires `--log-file`). |
|
||||
| `--stderr=MODE` | Route logging to stderr: `errors` or `all`. |
|
||||
| `--stderr=MODE` | Route logging: `errors` (default), `all`, or `client` (forward the client's diagnostics to the server's stderr over the client-message channel). |
|
||||
| `--msgs2stderr` | Route all messages to stderr (deprecated spelling of `--stderr=all`). |
|
||||
| `--no-msgs2stderr` | Select errors-only stderr (deprecated spelling; the default). |
|
||||
| `--no-msgs2stderr` | Forward the client's diagnostics to the server (deprecated spelling of `--stderr=client`). |
|
||||
| `-V`, `--version` | Print the FastSync protocol version. |
|
||||
| `--help` | Print command usage. |
|
||||
|
||||
@@ -832,7 +833,7 @@ before the module list, before authentication, and the connecting peer address
|
||||
|
||||
## Protocol and Security
|
||||
|
||||
FastSync protocol version `2.29.0` is shared by the client and server. The
|
||||
FastSync protocol version `2.30.0` is shared by the client and server. The
|
||||
current protocol is sender-driven and includes configuration negotiation,
|
||||
including the maximum allocation limit, incremental checks, checksums,
|
||||
manifests, keep-alives, abort handling, per-file remove-source results, and
|
||||
|
||||
+49
-36
File diff suppressed because one or more lines are too long
@@ -0,0 +1 @@
|
||||
nested
|
||||
@@ -0,0 +1 @@
|
||||
nested
|
||||
@@ -123,8 +123,15 @@ static char itemize_type_char(const ChangeEvent* event) {
|
||||
static bool times_match(const Config* config, const ChangeEvent* event) {
|
||||
if (!event->dest.known || !event->dest.existed)
|
||||
return false;
|
||||
if (event->mtime_sec == event->dest.mtime_sec)
|
||||
if (event->mtime_sec == event->dest.mtime_sec) {
|
||||
/* A regular file's sub-second mtime IS preserved by the receiver, so an nsec
|
||||
difference is a real change. A directory or symlink has no preserved
|
||||
sub-second mtime (rsync's quick-check compares whole seconds there), so a
|
||||
nanosecond-only difference must not render a spurious `.d..t` / `.L..t`. */
|
||||
if (event->is_directory || event->is_symlink || event->is_special)
|
||||
return true;
|
||||
return event->mtime_nsec == event->dest.mtime_nsec;
|
||||
}
|
||||
long long delta = (long long)event->mtime_sec - (long long)event->dest.mtime_sec;
|
||||
if (delta < 0)
|
||||
delta = -delta;
|
||||
@@ -145,6 +152,10 @@ static void itemize_code(const Config* config, const ChangeEvent* event, char co
|
||||
/* rsync: an existing directory that only has attribute changes carries no
|
||||
transfer, so the update column is `.` rather than `>`. */
|
||||
update = '.';
|
||||
else if (event->is_symlink)
|
||||
/* rsync: an existing symlink whose target is unchanged is a `.` update
|
||||
(attributes only); a changed target is `c` (the link value changed). */
|
||||
update = event->dest.target_matches ? '.' : 'c';
|
||||
else
|
||||
update = '>';
|
||||
code[0] = update;
|
||||
@@ -155,12 +166,20 @@ static void itemize_code(const Config* config, const ChangeEvent* event, char co
|
||||
code[11] = '\0';
|
||||
return;
|
||||
}
|
||||
bool size_diff = event->size != event->dest.size;
|
||||
bool time_diff = !times_match(config, event);
|
||||
/* rsync's value/checksum column: `c` for a symlink whose target changed (the
|
||||
link value is the compared content); no destination digest is available for
|
||||
a regular file. */
|
||||
bool value_diff = event->is_symlink && !event->dest.target_matches;
|
||||
/* rsync itemizes size only for regular files: a directory's st_size and a
|
||||
symlink's target length are not compared. */
|
||||
bool size_diff = !event->is_directory && !event->is_symlink && !event->is_special &&
|
||||
event->size != event->dest.size;
|
||||
/* rsync itemizes the time column only when -t/--times is in effect. */
|
||||
bool time_diff = config->preserve_times && !times_match(config, event);
|
||||
bool perms_diff = (event->mode & 07777) != (event->dest.mode & 07777);
|
||||
bool owner_diff = event->uid != (uid_t)event->dest.uid;
|
||||
bool group_diff = event->gid != (gid_t)event->dest.gid;
|
||||
code[2] = '.'; /* checksum: no destination digest available */
|
||||
code[2] = value_diff ? 'c' : '.';
|
||||
code[3] = size_diff ? 's' : '.';
|
||||
code[4] = time_diff ? 't' : '.';
|
||||
code[5] = (config->preserve_perms && perms_diff) ? 'p' : '.';
|
||||
@@ -172,6 +191,24 @@ static void itemize_code(const Config* config, const ChangeEvent* event, char co
|
||||
code[11] = '\0';
|
||||
}
|
||||
|
||||
/* True when the itemized destination entry is unchanged, i.e. rsync would print
|
||||
* no line at all. Reuses itemize_code so suppression is exactly consistent
|
||||
* with what would have been rendered: the update column must be `.` and every
|
||||
* attribute column must be `.`. */
|
||||
static bool itemize_is_unchanged(const Config* config, const ChangeEvent* event) {
|
||||
if (!event->dest.known || !event->dest.existed)
|
||||
return false;
|
||||
char code[12];
|
||||
itemize_code(config, event, code);
|
||||
if (code[0] != '.')
|
||||
return false;
|
||||
for (int i = 2; i < 11; i++) {
|
||||
if (code[i] != '.')
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/* rsync %n: the transfer-relative name, with a trailing slash for directories.
|
||||
* The transfer root is `.` (so `%n` renders `./`), matching rsync's root entry. */
|
||||
static bool append_name(StrBuf* buf, const ChangeEvent* event) {
|
||||
@@ -678,6 +715,19 @@ void change_emit_file_sent_bytes(const Config* config, const File* file,
|
||||
char* name = NULL;
|
||||
char* path = NULL;
|
||||
fill_event_from_file(config, file, &event, &name, &path);
|
||||
if (file->is_symlink) {
|
||||
/* Output parity (protocol 2.30.0): an unchanged symlink is silent, like
|
||||
rsync's quick check. The itemize/log stream suppresses it only when every
|
||||
attribute matches; the name stream suppresses it whenever the link target
|
||||
is unchanged (rsync names a symlink only when it relinks or creates it). */
|
||||
bool itemize_output = config->itemize_changes || config->out_format != NULL ||
|
||||
(config->log_file != NULL && config->log_file_format != NULL);
|
||||
bool suppress = itemize_output
|
||||
? itemize_is_unchanged(config, &event)
|
||||
: (event.dest.known && event.dest.existed && event.dest.target_matches);
|
||||
if (suppress)
|
||||
event.decision = CHANGE_UP_TO_DATE;
|
||||
}
|
||||
if (name != NULL && path != NULL) {
|
||||
fill_event_checksum(config, file, &event);
|
||||
change_emit(config, &event);
|
||||
@@ -729,6 +779,19 @@ void change_emit_dir_sent(const Config* config, const File* file) {
|
||||
char* name = NULL;
|
||||
char* path = NULL;
|
||||
fill_event_from_file(config, file, &event, &name, &path);
|
||||
/* Output parity (protocol 2.30.0): suppress a directory rsync would leave
|
||||
silent. The itemize/log stream suppresses it only when every attribute
|
||||
matches (`.d.........`); the name stream suppresses any pre-existing
|
||||
directory (rsync names a directory only when it is created). */
|
||||
bool itemize_output = config->itemize_changes || config->out_format != NULL ||
|
||||
(config->log_file != NULL && config->log_file_format != NULL);
|
||||
bool suppress = itemize_output ? itemize_is_unchanged(config, &event)
|
||||
: (event.dest.known && event.dest.existed);
|
||||
/* The transfer root's line is an unconditional FastSync residual (rsync keys
|
||||
it off the root's own attribute change); keep emitting it. */
|
||||
bool is_root = event.name != NULL && event.name[0] == '\0';
|
||||
if (suppress && !is_root)
|
||||
event.decision = CHANGE_UP_TO_DATE;
|
||||
if (name != NULL && path != NULL)
|
||||
change_emit(config, &event);
|
||||
free(name);
|
||||
|
||||
+21
-10
@@ -433,12 +433,10 @@ static int set_stderr_mode(const char* value) {
|
||||
log_set_stderr_mode(LOG_STDERR_ERRORS);
|
||||
else if (strcmp(value, "all") == 0 || strcmp(value, "a") == 0)
|
||||
log_set_stderr_mode(LOG_STDERR_ALL);
|
||||
else if (strcmp(value, "client") == 0 || strcmp(value, "c") == 0) {
|
||||
log_message(LOG_LEVEL_ERROR,
|
||||
"--stderr=client is not supported: FastSync has no client message channel");
|
||||
return -1;
|
||||
} else {
|
||||
log_message(LOG_LEVEL_ERROR, "--stderr must be errors or all");
|
||||
else if (strcmp(value, "client") == 0 || strcmp(value, "c") == 0)
|
||||
log_set_stderr_mode(LOG_STDERR_CLIENT);
|
||||
else {
|
||||
log_message(LOG_LEVEL_ERROR, "--stderr must be errors, all, or client");
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
@@ -1353,10 +1351,9 @@ static bool cli_handle_pre_negation(CliParseCtx* ctx) {
|
||||
return true;
|
||||
}
|
||||
/* "--no-msgs2stderr" is the deprecated spelling of --stderr=client (rsync
|
||||
* 3.4.1). FastSync has no separate client message channel, so the closest
|
||||
* supported mode is the errors-only default. */
|
||||
* 3.4.1); the client-message channel now exists, so it maps to `client`. */
|
||||
if (strcmp(arg, "--no-msgs2stderr") == 0)
|
||||
return set_stderr_mode("errors") == 0;
|
||||
return set_stderr_mode("client") == 0;
|
||||
/* "--no-motd" is a real rsync option name (client-side daemon MOTD display
|
||||
* suppression), not a negation of a "--motd" flag, so it is handled before
|
||||
* the generic --no-* negation branch. */
|
||||
@@ -2629,6 +2626,17 @@ static int cli_finalize_config(Config* config, bool verbose, bool no_delta, bool
|
||||
bool debug_enabled = verbose || config->debug_level != 0;
|
||||
set_log_level(config->quiet ? LOG_LEVEL_ERROR
|
||||
: (debug_enabled ? LOG_LEVEL_DEBUG : LOG_LEVEL_WARNING));
|
||||
/* rsync parity: --delay-updates implies --delete-after. Every staged file is
|
||||
published first and only then are extras removed. Normalize onto the
|
||||
existing delete_after wire bool (no new wire field), overriding any other
|
||||
explicit timing exactly as rsync does; without --delete there is no
|
||||
deletion, so no timing is set (and the wire config stays valid). */
|
||||
if (config->delay_updates && config->use_delete) {
|
||||
config->delete_before = false;
|
||||
config->delete_during = false;
|
||||
config->delete_delay = false;
|
||||
config->delete_after = true;
|
||||
}
|
||||
/* rsync's plain --delete defaults to delete-during (--del): each directory's
|
||||
extras are removed as that directory is processed, so space is freed
|
||||
progressively and a tight destination never has to hold the whole old+new
|
||||
@@ -2810,8 +2818,11 @@ static int cli_finalize_config(Config* config, bool verbose, bool no_delta, bool
|
||||
* need the pre-transfer destination snapshot (new vs modified and which
|
||||
* attributes differ), so ask the receiver to report it on every per-file
|
||||
* check. This is a wire field. */
|
||||
bool progress_active =
|
||||
!config->quiet && (config->show_progress || (config->info_level & LOG_INFO_PROGRESS) != 0);
|
||||
config->report_dest_info = config->itemize_changes || config->out_format != NULL ||
|
||||
(config->log_file != NULL && config->log_file_format != NULL);
|
||||
(config->log_file != NULL && config->log_file_format != NULL) ||
|
||||
progress_active;
|
||||
/* Wire-stats parity: --stats, --progress/-P, an --out-format token that needs
|
||||
* a wire counter (%b/%c), or a dry-run --delete need the receiver's
|
||||
* end-of-transfer STATUS_STATS report. This is a wire field (protocol
|
||||
|
||||
@@ -64,16 +64,8 @@ bool add_chunk_to_manifest(ArrayList* manifest, const Chunk* chunk) {
|
||||
int send_dry_run_manifest(const Config* config) {
|
||||
int skipped = 0;
|
||||
ArrayList* missing_dest = NULL;
|
||||
if (config->delete_missing_args) {
|
||||
missing_dest = array_list_create(free);
|
||||
if (!missing_dest)
|
||||
if (!client_prepare_files_from(config, &missing_dest, &skipped))
|
||||
return -1;
|
||||
}
|
||||
if (!files_from_list_check(config, missing_dest, &skipped)) {
|
||||
if (missing_dest)
|
||||
array_list_delete(missing_dest);
|
||||
return -1;
|
||||
}
|
||||
PreparedScanner prepared;
|
||||
if (!prepare_scanner(config, 0, &prepared)) {
|
||||
if (missing_dest)
|
||||
@@ -330,9 +322,11 @@ int send_list_only(const Config* config) {
|
||||
}
|
||||
|
||||
/* Send the delete manifest to the server. Returns 0 on success, -1 on
|
||||
failure. It carries FOUR sections: the keep-set paths, the protected
|
||||
excluded prefixes, the --delete-missing-args exact-delete paths, and the
|
||||
destination-relative directories the sender synchronized this run.
|
||||
failure. It carries FOUR path sections (keep-set paths, protected excluded
|
||||
prefixes, --delete-missing-args exact-delete paths, and the destination-
|
||||
relative directories the sender synchronized this run) followed by the
|
||||
protocol-2.30.0 per-directory filter-rule block (`per_dir_rules`, the rules
|
||||
the scan compiled from each directory's merge files).
|
||||
When --delete-excluded is given `protected` is empty: excluded destination
|
||||
mirrors are then ordinary extras and are removed. When
|
||||
--delete-missing-args is active `missing_args` holds the destination mirrors
|
||||
@@ -346,7 +340,8 @@ int send_list_only(const Config* config) {
|
||||
frame. A heavily filtered source whose exclusion list is large therefore
|
||||
fails the run cleanly on the receiver rather than being truncated. */
|
||||
int send_delete_manifest(int fd, ArrayList* manifest, ArrayList* protected_prefixes,
|
||||
ArrayList* size_skipped, ArrayList* missing_args, ArrayList* synced_dirs) {
|
||||
ArrayList* size_skipped, ArrayList* missing_args, ArrayList* synced_dirs,
|
||||
const FilterRuleList* per_dir_rules) {
|
||||
if (!send_status(fd, STATUS_MANIFEST))
|
||||
return -1;
|
||||
int keep_count = manifest ? manifest->size : 0;
|
||||
@@ -389,6 +384,11 @@ int send_delete_manifest(int fd, ArrayList* manifest, ArrayList* protected_prefi
|
||||
if (!send_wire_str(fd, (char*)synced_dirs->items[i]))
|
||||
return -1;
|
||||
}
|
||||
/* Protocol 2.30.0: the receiver-side per-directory filter rules discovered by
|
||||
the sender's scan, so the whole-tree commit walker can shield a
|
||||
destination-only entry that matches only a per-directory merge rule. */
|
||||
if (!delete_filter_dir_rules_send(fd, per_dir_rules))
|
||||
return -1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -409,11 +409,11 @@ int send_delete_manifest(int fd, ArrayList* manifest, ArrayList* protected_prefi
|
||||
|
||||
bool send_delete_manifest_early(Client* client, ArrayList* manifest, ArrayList* protected_prefixes,
|
||||
ArrayList* size_skipped, ArrayList* missing_args,
|
||||
ArrayList* synced_dirs) {
|
||||
ArrayList* synced_dirs, const FilterRuleList* per_dir_rules) {
|
||||
if (!client || !manifest)
|
||||
return false;
|
||||
if (send_delete_manifest(client->file_descriptor, manifest, protected_prefixes, size_skipped,
|
||||
missing_args, synced_dirs) != 0)
|
||||
missing_args, synced_dirs, per_dir_rules) != 0)
|
||||
return false;
|
||||
Status ack;
|
||||
/* The wait is long (up to an hour) and runs inline on this thread: a helper
|
||||
@@ -458,35 +458,21 @@ bool send_delete_manifest_early(Client* client, ArrayList* manifest, ArrayList*
|
||||
int send_dry_run_remote(Config* config) {
|
||||
int from_skipped = 0;
|
||||
ArrayList* missing_args = NULL;
|
||||
if (config->delete_missing_args) {
|
||||
missing_args = array_list_create(free);
|
||||
if (!missing_args)
|
||||
if (!client_prepare_files_from(config, &missing_args, &from_skipped))
|
||||
return 1;
|
||||
}
|
||||
if (!files_from_list_check(config, missing_args, &from_skipped)) {
|
||||
if (missing_args)
|
||||
array_list_delete(missing_args);
|
||||
return 1;
|
||||
}
|
||||
if (missing_args)
|
||||
array_list_delete(missing_args);
|
||||
/* A live session may follow, so arm graceful abort handling. */
|
||||
client_set_abort_armed(true);
|
||||
Client* client = connect_transfer_client(config);
|
||||
ProtocolSession session;
|
||||
Client* client = client_connect_and_bind_session(config, &session);
|
||||
if (!client) {
|
||||
if (config->transport == TRANSPORT_TCP)
|
||||
log_message(LOG_LEVEL_ERROR, "could not connect to server%s",
|
||||
config->use_tls ? " via TLS" : "");
|
||||
client_set_abort_armed(false);
|
||||
return 1;
|
||||
}
|
||||
ProtocolSession session;
|
||||
protocol_session_init(&session, client->file_descriptor, client->file_descriptor);
|
||||
protocol_session_set_io_timeout(&session, config->timeout);
|
||||
protocol_session_set_ssl(&session, (SSL*)client->ssl);
|
||||
protocol_session_bind(&session);
|
||||
|
||||
int ret = 1;
|
||||
bool partial = false;
|
||||
time_t dry_start = time(NULL);
|
||||
ReceiverStats dry_stats;
|
||||
memset(&dry_stats, 0, sizeof(dry_stats));
|
||||
@@ -497,6 +483,7 @@ int send_dry_run_remote(Config* config) {
|
||||
ArrayList* dry_dirs = NULL;
|
||||
ArrayList* dry_excluded = NULL;
|
||||
ArrayList* dry_size_skipped = NULL;
|
||||
FilterRuleList* dry_per_dir = NULL;
|
||||
if (!config_send(client->file_descriptor, config))
|
||||
goto dry_fail;
|
||||
receive_daemon_motd(client, config);
|
||||
@@ -509,8 +496,10 @@ int send_dry_run_remote(Config* config) {
|
||||
dry_manifest = array_list_create(free);
|
||||
dry_dirs = array_list_create(free);
|
||||
dry_size_skipped = array_list_create(free);
|
||||
if (!dry_manifest || !dry_dirs || !dry_size_skipped)
|
||||
dry_per_dir = filter_rule_list_create();
|
||||
if (!dry_manifest || !dry_dirs || !dry_size_skipped || !dry_per_dir)
|
||||
goto dry_fail;
|
||||
prepared.options.per_dir_rules = dry_per_dir;
|
||||
if (!config->delete_excluded) {
|
||||
dry_excluded = array_list_create(free);
|
||||
if (!dry_excluded)
|
||||
@@ -612,7 +601,7 @@ int send_dry_run_remote(Config* config) {
|
||||
bool early_delete = config->use_delete && config_delete_timing_early(config);
|
||||
if (dry_manifest) {
|
||||
if (send_delete_manifest(client->file_descriptor, dry_manifest, dry_excluded, dry_size_skipped,
|
||||
NULL, dry_dirs) != 0)
|
||||
NULL, dry_dirs, dry_per_dir) != 0)
|
||||
goto dry_fail;
|
||||
if (early_delete) {
|
||||
Status ack;
|
||||
@@ -642,8 +631,14 @@ int send_dry_run_remote(Config* config) {
|
||||
if (!receive_status(client->file_descriptor, &status))
|
||||
goto dry_fail;
|
||||
}
|
||||
if (status != STATUS_OK)
|
||||
/* A per-entry receiver failure is rsync's PARTIAL transfer (exit 23), not a
|
||||
hard failure: a dry run transfers nothing, but keep the verdict consistent
|
||||
with the normal path instead of treating it as a protocol error. */
|
||||
if (status == STATUS_PARTIAL) {
|
||||
partial = true;
|
||||
} else if (status != STATUS_OK) {
|
||||
goto dry_fail;
|
||||
}
|
||||
if (!config->quiet) {
|
||||
if (config->human_readable)
|
||||
printf("Total: %d files, %s\n", file_count,
|
||||
@@ -661,7 +656,7 @@ int send_dry_run_remote(Config* config) {
|
||||
dry_transfer.literal_data = total_bytes;
|
||||
report_transfer_stats(config, &dry_transfer, dry_start, &dry_stats);
|
||||
}
|
||||
ret = io_error ? 1 : 0;
|
||||
ret = io_error ? 1 : (partial ? 23 : 0);
|
||||
|
||||
dry_fail:
|
||||
if (dry_manifest)
|
||||
@@ -672,6 +667,8 @@ dry_fail:
|
||||
array_list_delete(dry_excluded);
|
||||
if (dry_size_skipped)
|
||||
array_list_delete(dry_size_skipped);
|
||||
if (dry_per_dir)
|
||||
filter_rule_list_free(dry_per_dir);
|
||||
if (scanner)
|
||||
directory_scanner_destroy(scanner);
|
||||
prepared_scanner_destroy(&prepared);
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <threads.h>
|
||||
#include <time.h>
|
||||
|
||||
/* Surface a server rejection to the user. When the last status exchange
|
||||
@@ -161,6 +162,9 @@ void report_transfer_stats(const Config* config, const TransferStats* stats, tim
|
||||
created_breakdown, sizeof(created_breakdown));
|
||||
unsigned long long created_total =
|
||||
recv->created_reg + recv->created_dir + recv->created_link + recv->created_special;
|
||||
char deleted_breakdown[128];
|
||||
type_breakdown(recv->deleted_reg, recv->deleted_dir, recv->deleted_link, recv->deleted_special,
|
||||
deleted_breakdown, sizeof(deleted_breakdown));
|
||||
printf("\n");
|
||||
if (breakdown[0] != '\0')
|
||||
printf("Number of files: %llu %s\n", flist_total, breakdown);
|
||||
@@ -172,6 +176,12 @@ void report_transfer_stats(const Config* config, const TransferStats* stats, tim
|
||||
printf("Number of created files: %llu %s\n", created_total, created_breakdown);
|
||||
else
|
||||
printf("Number of created files: %llu\n", created_total);
|
||||
/* Protocol 2.30.0: the receiver reports the removed entries split by type, so
|
||||
this line matches rsync's `Number of deleted files: X (reg: A, dir: B,
|
||||
link: C, special: D)` (only the non-zero categories are listed). */
|
||||
if (deleted_breakdown[0] != '\0')
|
||||
printf("Number of deleted files: %llu %s\n", recv->deleted_files, deleted_breakdown);
|
||||
else
|
||||
printf("Number of deleted files: %llu\n", recv->deleted_files);
|
||||
printf("Number of regular files transferred: %llu\n", stats->transferred_regular);
|
||||
printf("Total file size: %s bytes\n", total);
|
||||
@@ -525,10 +535,17 @@ static void client_progress_emit_ancestors(const Config* config, const char* rel
|
||||
if (dir != NULL)
|
||||
change_emit_dir_sent(config, dir);
|
||||
} else {
|
||||
/* --progress/-P names a directory only when it is newly created;
|
||||
rsync stays silent for a pre-existing directory even when one of
|
||||
its children changed (protocol 2.30.0 dest-state report). */
|
||||
const File* dir = progress_dir_lookup(prefix);
|
||||
bool existed = dir != NULL && dir->dest_state.known && dir->dest_state.existed;
|
||||
if (!existed) {
|
||||
char* escaped = output_escape(prefix, config->eight_bit_output);
|
||||
printf("%s/\n", escaped ? escaped : prefix);
|
||||
free(escaped);
|
||||
}
|
||||
}
|
||||
g_progress_index++;
|
||||
} else {
|
||||
free(key);
|
||||
@@ -552,6 +569,122 @@ void client_change_emit_ancestors(const Config* config, const File* file) {
|
||||
client_progress_emit_ancestors(config, rel);
|
||||
}
|
||||
|
||||
/* Send one STATUS_MKDIR probe (probe=1) for a pre-count directory and cache the
|
||||
* receiver's pre-transfer destination snapshot in `dir->dest_state`. Returns
|
||||
* false on a protocol/transport error. */
|
||||
static bool client_probe_dir_state(int fd, File* dir) {
|
||||
if (dir == NULL || file_wire_path(dir) == NULL)
|
||||
return true;
|
||||
if (!send_status(fd, STATUS_MKDIR) || !send_int(fd, 1) || !send_wire_str(fd, file_wire_path(dir)))
|
||||
return false;
|
||||
Status status = STATUS_ERROR;
|
||||
if (!receive_status(fd, &status) || status != STATUS_DEST_INFO ||
|
||||
!format_dest_state_receive(fd, &dir->dest_state)) {
|
||||
log_message(LOG_LEVEL_ERROR, "Directory destination-state probe failed");
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Output parity (protocol 2.30.0): ask the receiver for each not-yet-probed
|
||||
* ancestor directory's pre-transfer state BEFORE the entry that first triggers
|
||||
* it is sent, so the ancestor's -i/--out-format line renders rsync's
|
||||
* `.d..t......` (existing, attributes changed) versus `cd+++++++++` (created)
|
||||
* and an unchanged directory is suppressed. The probe is a STATUS_MKDIR frame
|
||||
* with probe=1 (the receiver reports and creates nothing), so it must run before
|
||||
* the receiver implicitly creates the parent for the child. Each directory is
|
||||
* probed at most once; the result is cached in the pre-count File's dest_state.
|
||||
* Returns false on a protocol/transport error (the caller aborts the transfer). */
|
||||
bool client_change_probe_ancestors(const Config* config, const File* file, int fd) {
|
||||
if (config == NULL || file == NULL || fd < 0 || !config->report_dest_info)
|
||||
return true;
|
||||
if (!g_progress_dir_index_valid || !g_progress_precount.dir_refs)
|
||||
return true;
|
||||
const char* rel = delete_display_path(config, file_wire_path(file));
|
||||
if (rel == NULL)
|
||||
return true;
|
||||
size_t rel_len = strlen(rel);
|
||||
for (size_t i = 1; i < rel_len; i++) {
|
||||
if (rel[i] != '/')
|
||||
continue;
|
||||
char* prefix = malloc(i + 1);
|
||||
if (prefix == NULL)
|
||||
return false;
|
||||
memcpy(prefix, rel, i);
|
||||
prefix[i] = '\0';
|
||||
if (path_index_contains(&g_progress_dir_index, prefix)) {
|
||||
File* dir = progress_dir_lookup(prefix);
|
||||
/* The probe path is the same wire path the real STATUS_MKDIR would carry
|
||||
(the pre-count File's send_path, or its absolute source path for a
|
||||
plain recursive scan), not the display-relative prefix. */
|
||||
if (dir != NULL && !dir->dest_state.known && !client_probe_dir_state(fd, dir)) {
|
||||
free(prefix);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
free(prefix);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Mark a directory the data pass already itemized/named so the end-of-transfer
|
||||
* pending-directory flush does not report it a second time. `file` is a
|
||||
* transferred directory entry (an empty-directory STATUS_MKDIR). */
|
||||
void client_change_mark_dir(const Config* config, const File* file) {
|
||||
if (config == NULL || file == NULL || !g_progress_emitted_valid ||
|
||||
g_progress_emitted_keys == NULL)
|
||||
return;
|
||||
const char* rel = delete_display_path(config, file_wire_path(file));
|
||||
if (rel == NULL || rel[0] == '\0' || str_hash_set_lookup(&g_progress_emitted, rel))
|
||||
return;
|
||||
char* key = str_dup(rel);
|
||||
if (key == NULL)
|
||||
return;
|
||||
if (!array_list_add(g_progress_emitted_keys, key)) {
|
||||
free(key);
|
||||
return;
|
||||
}
|
||||
str_hash_set_insert_ref(&g_progress_emitted, key);
|
||||
}
|
||||
|
||||
/* Emit the itemize lines for source directories that CHANGED but had no
|
||||
* transferred child, so no ancestor emission reached them (rsync reports a
|
||||
* directory whose attributes changed even when its contents did not). Runs at
|
||||
* the end of the data pass, BEFORE the deferred STATUS_DIR_TIMES apply, so the
|
||||
* probe still observes each untouched directory's pre-transfer state. Only the
|
||||
* itemize/out-format/log streams report attribute-only directory changes;
|
||||
* --progress/-P stays silent for them, matching rsync. Best-effort: a probe
|
||||
* failure simply stops the flush (the transfer's verdict is unaffected). */
|
||||
void client_change_emit_pending_dirs(const Config* config, int fd) {
|
||||
if (config == NULL || fd < 0 || !config->report_dest_info)
|
||||
return;
|
||||
bool itemize_output = config->itemize_changes || config->out_format != NULL ||
|
||||
(config->log_file != NULL && config->log_file_format != NULL);
|
||||
if (!itemize_output)
|
||||
return;
|
||||
if (!g_progress_dir_index_valid || g_progress_precount.dir_refs == NULL ||
|
||||
!g_progress_emitted_valid || g_progress_emitted_keys == NULL)
|
||||
return;
|
||||
for (int i = 0; i < g_progress_precount.dir_refs->size; i++) {
|
||||
DirRef* ref = (DirRef*)g_progress_precount.dir_refs->items[i];
|
||||
if (ref == NULL || ref->name == NULL || ref->name[0] == '\0')
|
||||
continue;
|
||||
if (str_hash_set_lookup(&g_progress_emitted, ref->name))
|
||||
continue;
|
||||
File* dir = ref->file;
|
||||
if (dir == NULL)
|
||||
continue;
|
||||
if (!dir->dest_state.known && !client_probe_dir_state(fd, dir))
|
||||
return;
|
||||
change_emit_dir_sent(config, dir);
|
||||
char* key = str_dup(ref->name);
|
||||
if (key != NULL && array_list_add(g_progress_emitted_keys, key))
|
||||
str_hash_set_insert_ref(&g_progress_emitted, key);
|
||||
else
|
||||
free(key);
|
||||
}
|
||||
}
|
||||
|
||||
/* rsync's --info=name/progress line for one entry: transfer-relative name (a
|
||||
* trailing slash for directories) plus the ` -> target` symlink suffix. */
|
||||
static char* progress_entry_line(const File* file, const char* rel) {
|
||||
@@ -644,6 +777,15 @@ void client_progress_file(const Config* config, const File* file) {
|
||||
void client_progress_name(const Config* config, const File* file) {
|
||||
if (!g_progress_active || file == NULL)
|
||||
return;
|
||||
/* rsync's --progress/-P name stream reports an entry only when it is created
|
||||
or (for a symlink) actually relinked: a pre-existing directory or an
|
||||
unchanged symlink is silent (protocol 2.30.0 dest-state report). */
|
||||
if (file->dest_state.known && file->dest_state.existed) {
|
||||
if (file->is_dir || (file->is_symlink && file->dest_state.target_matches)) {
|
||||
g_progress_index++;
|
||||
return;
|
||||
}
|
||||
}
|
||||
const char* rel = delete_display_path(config, file_wire_path(file));
|
||||
if (!config->itemize_changes && config->out_format == NULL) {
|
||||
char* line = progress_entry_line(file, rel ? rel : "");
|
||||
@@ -1051,3 +1193,129 @@ const char* delete_display_path(const Config* config, const char* path) {
|
||||
return path;
|
||||
return utils_strip_transfer_root(path, config->send_directory);
|
||||
}
|
||||
|
||||
/* ---- --stderr=client diagnostic channel (protocol 2.30.0) ----
|
||||
*
|
||||
* When the client's --stderr mode is `client`, log_message() hands each of the
|
||||
* client's own diagnostics to the sink installed here instead of writing them
|
||||
* locally. The sink QUEUES the text (it may be called from scanner worker
|
||||
* threads while the sender is streaming) and the sender thread -- the sole
|
||||
* writer of the protocol stream -- drains the queue over the wire at frame
|
||||
* boundaries via client_flush_client_messages(). A bounded queue caps the
|
||||
* memory a chatty run can pin; overflow falls back to local output so a
|
||||
* diagnostic is never silently dropped. */
|
||||
#define CLIENT_MSG_MAX_QUEUED 256
|
||||
#define CLIENT_MSG_MAX_BYTES (256 * 1024)
|
||||
|
||||
static mtx_t client_msg_mutex;
|
||||
static once_flag client_msg_mutex_once = ONCE_FLAG_INIT;
|
||||
static ArrayList* client_msg_queue = NULL; /* owns char* */
|
||||
static size_t client_msg_bytes = 0;
|
||||
/* True only while a live transfer session exists: before the connection is up
|
||||
(or after it drops) the sink declines so log_message falls back to local
|
||||
output, matching rsync's documented fallback. Written by the sender thread
|
||||
(client_messages_activate) and read by scanner worker threads in
|
||||
client_msg_enqueue, so it must be atomic: the queue itself stays guarded by
|
||||
client_msg_mutex, but the flag is polled before taking that lock. */
|
||||
static _Atomic bool client_msg_active = false;
|
||||
|
||||
static void client_msg_mutex_init(void) {
|
||||
mtx_init(&client_msg_mutex, mtx_plain);
|
||||
}
|
||||
|
||||
static bool client_msg_enqueue(const char* message);
|
||||
|
||||
/* Install the global log sink for the duration of one transfer. Safe to call
|
||||
* more than once; the queue is created lazily. */
|
||||
void client_messages_install(void) {
|
||||
call_once(&client_msg_mutex_once, client_msg_mutex_init);
|
||||
mtx_lock(&client_msg_mutex);
|
||||
if (!client_msg_queue)
|
||||
client_msg_queue = array_list_create(free);
|
||||
bool ready = client_msg_queue != NULL;
|
||||
mtx_unlock(&client_msg_mutex);
|
||||
/* Only arm the sink once the queue exists; on allocation failure leave the
|
||||
sink uninstalled so log_message keeps writing locally instead of handing
|
||||
messages to a sink that would silently drop them. */
|
||||
if (ready)
|
||||
log_set_client_msg_sink(client_msg_enqueue);
|
||||
}
|
||||
|
||||
void client_messages_activate(bool active) {
|
||||
atomic_store(&client_msg_active, active);
|
||||
}
|
||||
|
||||
/* log_message sink: takes ownership (queues) the message when a session is
|
||||
* live; returns false otherwise so the caller writes it locally. */
|
||||
static bool client_msg_enqueue(const char* message) {
|
||||
bool active = atomic_load(&client_msg_active);
|
||||
if (!message || message[0] == '\0')
|
||||
return active;
|
||||
if (!active)
|
||||
return false;
|
||||
size_t len = strlen(message);
|
||||
call_once(&client_msg_mutex_once, client_msg_mutex_init);
|
||||
mtx_lock(&client_msg_mutex);
|
||||
bool queued = false;
|
||||
if (client_msg_queue && (size_t)client_msg_queue->size < CLIENT_MSG_MAX_QUEUED &&
|
||||
client_msg_bytes + len <= CLIENT_MSG_MAX_BYTES) {
|
||||
char* copy = str_dup(message);
|
||||
if (copy) {
|
||||
if (array_list_add(client_msg_queue, copy)) {
|
||||
client_msg_bytes += len;
|
||||
queued = true;
|
||||
} else {
|
||||
free(copy);
|
||||
}
|
||||
}
|
||||
}
|
||||
mtx_unlock(&client_msg_mutex);
|
||||
return queued;
|
||||
}
|
||||
|
||||
/* Drain the queued diagnostics as STATUS_CLIENT_MSG frames on the sender
|
||||
* thread. Swaps the queue out under the mutex so a concurrent worker logging
|
||||
* never blocks on the wire. Must be called at a protocol frame boundary. */
|
||||
void client_flush_client_messages(int fd) {
|
||||
if (fd < 0)
|
||||
return;
|
||||
call_once(&client_msg_mutex_once, client_msg_mutex_init);
|
||||
mtx_lock(&client_msg_mutex);
|
||||
ArrayList* pending = client_msg_queue;
|
||||
if (pending) {
|
||||
ArrayList* fresh = array_list_create(free);
|
||||
if (fresh) {
|
||||
client_msg_queue = fresh;
|
||||
} else {
|
||||
/* No memory for a replacement queue: stop queuing new diagnostics (they
|
||||
fall back to local output) and drain this batch below so nothing is
|
||||
silently dropped. */
|
||||
client_msg_queue = NULL;
|
||||
log_set_client_msg_sink(NULL);
|
||||
}
|
||||
client_msg_bytes = 0;
|
||||
}
|
||||
mtx_unlock(&client_msg_mutex);
|
||||
if (!pending)
|
||||
return;
|
||||
for (int i = 0; i < pending->size; i++) {
|
||||
const char* message = pending->items[i];
|
||||
if (message && message[0] != '\0' && !send_client_message(fd, message))
|
||||
break; /* peer is gone; the rest would fail too */
|
||||
}
|
||||
array_list_delete(pending);
|
||||
}
|
||||
|
||||
/* Tear down the sink after a transfer and free anything still queued. */
|
||||
void client_messages_end(void) {
|
||||
log_set_client_msg_sink(NULL);
|
||||
atomic_store(&client_msg_active, false);
|
||||
call_once(&client_msg_mutex_once, client_msg_mutex_init);
|
||||
mtx_lock(&client_msg_mutex);
|
||||
ArrayList* pending = client_msg_queue;
|
||||
client_msg_queue = NULL;
|
||||
client_msg_bytes = 0;
|
||||
mtx_unlock(&client_msg_mutex);
|
||||
if (pending)
|
||||
array_list_delete(pending);
|
||||
}
|
||||
|
||||
+230
-66
@@ -127,10 +127,59 @@ Client* connect_transfer_client(const Config* config) {
|
||||
void disconnect_transfer_client(Client* client) {
|
||||
if (!client)
|
||||
return;
|
||||
/* --stderr=client: push any diagnostics logged during the transfer to the
|
||||
peer before the socket closes; once deactivated, later messages fall back
|
||||
to local output instead of being lost. */
|
||||
client_flush_client_messages(client->file_descriptor);
|
||||
client_messages_activate(false);
|
||||
client_disconnect(client);
|
||||
client_delete(client);
|
||||
}
|
||||
|
||||
/* Connect the configured transport and install the per-thread protocol session
|
||||
* on it: init with the socket fd pair, apply the I/O timeout and (when
|
||||
* negotiated) the TLS object, then bind it to this thread. Returns the
|
||||
* connected client, or NULL (after logging the connect failure) when the
|
||||
* transport could not connect. */
|
||||
Client* client_connect_and_bind_session(const Config* config, ProtocolSession* session) {
|
||||
Client* client = connect_transfer_client(config);
|
||||
if (!client) {
|
||||
if (config->transport == TRANSPORT_TCP)
|
||||
log_message(LOG_LEVEL_ERROR, "could not connect to server%s",
|
||||
config->use_tls ? " via TLS" : "");
|
||||
return NULL;
|
||||
}
|
||||
protocol_session_init(session, client->file_descriptor, client->file_descriptor);
|
||||
protocol_session_set_io_timeout(session, config->timeout);
|
||||
protocol_session_set_ssl(session, (SSL*)client->ssl);
|
||||
protocol_session_bind(session);
|
||||
return client;
|
||||
}
|
||||
|
||||
/* Shared --files-from/--delete-missing-args preamble: allocate the missing-args
|
||||
* destination list when the option is set, then validate the --files-from list
|
||||
* (collecting the destination mirrors of missing entries for the receiver's
|
||||
* exact-deletion request). On success the caller owns *missing_args_out (NULL
|
||||
* when the option is off); on failure the list is freed and false is returned. */
|
||||
bool client_prepare_files_from(const Config* config, ArrayList** missing_args_out,
|
||||
int* skipped_out) {
|
||||
ArrayList* missing_args = NULL;
|
||||
if (config->delete_missing_args) {
|
||||
missing_args = array_list_create(free);
|
||||
if (!missing_args)
|
||||
return false;
|
||||
}
|
||||
int skipped = 0;
|
||||
if (!files_from_list_check(config, missing_args, &skipped)) {
|
||||
if (missing_args)
|
||||
array_list_delete(missing_args);
|
||||
return false;
|
||||
}
|
||||
*missing_args_out = missing_args;
|
||||
*skipped_out = skipped;
|
||||
return true;
|
||||
}
|
||||
|
||||
/* (finalize_transfer is defined after the SourceFile helpers below.) */
|
||||
|
||||
typedef struct SourceFile {
|
||||
@@ -240,13 +289,28 @@ static void mark_sender_done(PipelineContextSender* context) {
|
||||
When --remove-source-files is active the receiver acknowledges each data
|
||||
file it processed, in send order: STATUS_NEXT means the file was written,
|
||||
STATUS_OK means the file was skipped/unchanged. Skipped sources are marked
|
||||
so the later removal pass keeps them. */
|
||||
so the later removal pass keeps them. `partial_out` is set when the receiver
|
||||
reported STATUS_PARTIAL (a per-entry receiver failure): the transfer is
|
||||
otherwise complete, so successfully stored sources are still removed and the
|
||||
caller exits 23 (rsync's partial transfer) instead of a fatal non-zero. */
|
||||
static bool finalize_transfer(Client* client, const Config* config, ArrayList* remove_sources,
|
||||
bool* delete_limit_out, ReceiverStats* stats_out) {
|
||||
bool* delete_limit_out, bool* partial_out, ReceiverStats* stats_out) {
|
||||
if (delete_limit_out)
|
||||
*delete_limit_out = false;
|
||||
if (partial_out)
|
||||
*partial_out = false;
|
||||
/* --stderr=client: the receiver consumes frames until it reads
|
||||
STATUS_FINISHED, after which it no longer reads. Flush every diagnostic
|
||||
queued during the transfer here -- the last frame boundary at which the
|
||||
peer is still reading -- so nothing is stranded in the queue. */
|
||||
client_flush_client_messages(client->file_descriptor);
|
||||
if (!send_status(client->file_descriptor, STATUS_FINISHED))
|
||||
return false;
|
||||
/* Past STATUS_FINISHED the receiver has stopped reading, so any diagnostic
|
||||
logged from here on (notably the STATUS_PARTIAL warning below) can no
|
||||
longer be forwarded. Deactivate the channel so those messages fall back
|
||||
to local output instead of being queued for a closed peer and lost. */
|
||||
client_messages_activate(false);
|
||||
/* The receiver emits its optional wire-stats frame (protocol 2.25.0) FIRST,
|
||||
then any per-file --remove-source-files acks, then the terminal status. */
|
||||
Status status;
|
||||
@@ -298,6 +362,16 @@ static bool finalize_transfer(Client* client, const Config* config, ArrayList* r
|
||||
*delete_limit_out = true;
|
||||
return true;
|
||||
}
|
||||
/* A per-entry receiver failure the receiver chose to continue past is a
|
||||
rsync PARTIAL transfer: everything else succeeded and the stored sources
|
||||
may be removed, but the client must exit 23. */
|
||||
if (status == STATUS_PARTIAL) {
|
||||
log_message(LOG_LEVEL_WARNING,
|
||||
"some files could not be transferred (see the server log for details)");
|
||||
if (partial_out)
|
||||
*partial_out = true;
|
||||
return true;
|
||||
}
|
||||
if (status != STATUS_OK) {
|
||||
log_server_rejection("Receiver reported transfer failure");
|
||||
return false;
|
||||
@@ -595,14 +669,34 @@ static bool send_file_direct(File* file, int fd, bool use_metadata, int compress
|
||||
static bool send_directory_entry(const Client* client, File* file, const Config* config) {
|
||||
if (!file || !file_wire_path(file))
|
||||
return false;
|
||||
if (!send_status(client->file_descriptor, STATUS_MKDIR) ||
|
||||
!send_wire_str(client->file_descriptor, file_wire_path(file)))
|
||||
int fd = client->file_descriptor;
|
||||
if (!send_status(fd, STATUS_MKDIR))
|
||||
return false;
|
||||
if (config->use_metadata && !metadata_send(client->file_descriptor, file->metadata))
|
||||
/* Output parity (protocol 2.30.0): when report_dest_info is negotiated every
|
||||
STATUS_MKDIR body is prefixed with a probe flag (1 = probe only, 0 = a real
|
||||
create), so the receiver knows whether to expect the metadata/xattr block. */
|
||||
if (config->report_dest_info && !send_int(fd, 0))
|
||||
return false;
|
||||
if (!send_wire_str(fd, file_wire_path(file)))
|
||||
return false;
|
||||
if (config->use_metadata && !metadata_send(fd, file->metadata))
|
||||
return false;
|
||||
/* Directory xattrs/ACLs (-X/-A) ride the same trailing block as regular files
|
||||
when the xattr transport was negotiated. */
|
||||
return !config->use_xattrs || xattr_send(client->file_descriptor, file->xattrs);
|
||||
if (config->use_xattrs && !xattr_send(fd, file->xattrs))
|
||||
return false;
|
||||
/* The receiver answers with the directory's pre-transfer destination state
|
||||
BEFORE creating it, so the sender can render rsync's `.d..t......` versus
|
||||
`cd+++++++++` and suppress an unchanged directory. */
|
||||
if (config->report_dest_info) {
|
||||
Status status;
|
||||
if (!receive_status(fd, &status) || status != STATUS_DEST_INFO ||
|
||||
!format_dest_state_receive(fd, &file->dest_state)) {
|
||||
log_message(LOG_LEVEL_ERROR, "Unexpected reply to the directory destination-state report");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/* P7 Wave D: transmit every captured source directory's metadata in terminal
|
||||
@@ -658,7 +752,21 @@ static bool send_symlink_entry(const Client* client, File* file, const Config* c
|
||||
return false;
|
||||
/* Symlink xattrs/ACLs (-X/-A) ride the same trailing block as regular files
|
||||
and directories when the xattr transport was negotiated. */
|
||||
return !config->use_xattrs || xattr_send(fd, file->xattrs);
|
||||
if (config->use_xattrs && !xattr_send(fd, file->xattrs))
|
||||
return false;
|
||||
/* The receiver answers with the symlink's pre-transfer destination state
|
||||
(including whether the on-disk link target already matches) BEFORE creating
|
||||
it, so the sender can render rsync's `cLc........` / `.L..t......` and
|
||||
suppress an unchanged symlink. */
|
||||
if (config->report_dest_info) {
|
||||
Status status;
|
||||
if (!receive_status(fd, &status) || status != STATUS_DEST_INFO ||
|
||||
!format_dest_state_receive(fd, &file->dest_state)) {
|
||||
log_message(LOG_LEVEL_ERROR, "Unexpected reply to the symlink destination-state report");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// Send a single file directly via sendfile (non-incremental path).
|
||||
@@ -813,8 +921,28 @@ static bool source_is_regular_file(const File* file) {
|
||||
return stat(file->path, &st) == 0 && S_ISREG(st.st_mode);
|
||||
}
|
||||
|
||||
/* True when an over-threshold source will be sent by STREAMING from disk rather
|
||||
* than loaded into memory: either the zero-copy sendfile path (no compression)
|
||||
* or the sender-side streaming compressor (zstd/zlib, when this file is not on
|
||||
* the --skip-compress list). Otherwise the loader must materialize it. */
|
||||
static bool loader_can_stream(const Config* config, const File* file) {
|
||||
if (!config || !file || !file->data || file->data->size <= STREAM_THRESHOLD)
|
||||
return false;
|
||||
if (!config->use_compression)
|
||||
return true;
|
||||
if (!compression_stream_compress_supported(compression_get_algo()) ||
|
||||
config->compression_level <= 0)
|
||||
return false;
|
||||
int skip_count = config->skip_compress_set ? config->skip_compress_count : -1;
|
||||
return !compression_should_skip_with_suffixes(file->path, config->skip_compress_suffixes,
|
||||
skip_count);
|
||||
}
|
||||
|
||||
static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config,
|
||||
ArrayList* remove_sources, TransferStats* stats) {
|
||||
/* --stderr=client: this is a frame boundary, so forward any diagnostics the
|
||||
scanner/log emitted since the previous chunk before the next frame. */
|
||||
client_flush_client_messages(client->file_descriptor);
|
||||
if (config->use_chunk_serialization) {
|
||||
if (remove_sources) {
|
||||
for (int i = 0; i < chunk->element_count; i++) {
|
||||
@@ -842,14 +970,23 @@ static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config,
|
||||
if (chunk->items[i] == NULL)
|
||||
continue;
|
||||
transfer_stats_note_entry(stats, chunk->items[i]);
|
||||
/* Output parity: probe each entry's ancestor directories' destination
|
||||
state before emitting its itemize line, exactly as the non-serialized
|
||||
loop does. Without this, dest_state.known stays false and -i/-P
|
||||
renders an existing dir/symlink as created instead of `.d..t...` (or
|
||||
suppressing it). */
|
||||
if (!client_change_probe_ancestors(config, chunk->items[i], client->file_descriptor))
|
||||
return -1;
|
||||
/* The chunk-serialization path emits no --progress name lines, so only
|
||||
feed -i/--out-format its ancestor directory lines here. */
|
||||
if (config->itemize_changes || config->out_format != NULL)
|
||||
client_change_emit_ancestors(config, chunk->items[i]);
|
||||
if (chunk->items[i]->is_dir)
|
||||
if (chunk->items[i]->is_dir) {
|
||||
change_emit_dir_sent(config, chunk->items[i]);
|
||||
else
|
||||
client_change_mark_dir(config, chunk->items[i]);
|
||||
} else {
|
||||
change_emit_file_sent(config, chunk->items[i]);
|
||||
}
|
||||
if (!chunk->items[i]->is_dir)
|
||||
transfer_stats_note_transferred(stats, chunk->items[i]);
|
||||
}
|
||||
@@ -861,6 +998,11 @@ static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config,
|
||||
if (f == NULL)
|
||||
continue;
|
||||
transfer_stats_note_entry(stats, f);
|
||||
/* Output parity: probe this entry's ancestor directories' destination state
|
||||
before the entry (or the first child below them) is sent, while the
|
||||
receiver has not yet created them implicitly. */
|
||||
if (!client_change_probe_ancestors(config, f, client->file_descriptor))
|
||||
return -1;
|
||||
if (f->is_dir) {
|
||||
/* Explicit directory entry (--dirs): a MKDIR frame carrying the
|
||||
destination path (and metadata when negotiated). Directories have no
|
||||
@@ -869,6 +1011,7 @@ static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config,
|
||||
return -1;
|
||||
client_change_emit_ancestors(config, f);
|
||||
change_emit_dir_sent(config, f);
|
||||
client_change_mark_dir(config, f);
|
||||
client_progress_name(config, f);
|
||||
continue;
|
||||
}
|
||||
@@ -940,20 +1083,14 @@ static int send_chunk_with_removal(Client* client, Chunk* chunk, Config* config,
|
||||
static int send_chunks_multithreaded(void* pipeline_context) {
|
||||
PipelineContextSender* context = (PipelineContextSender*)pipeline_context;
|
||||
time_t start = time(NULL);
|
||||
Client* client = connect_transfer_client(context->config);
|
||||
ProtocolSession session;
|
||||
Client* client = client_connect_and_bind_session(context->config, &session);
|
||||
if (!client) {
|
||||
if (context->config->transport == TRANSPORT_TCP)
|
||||
log_message(LOG_LEVEL_ERROR, "could not connect to server%s",
|
||||
context->config->use_tls ? " via TLS" : "");
|
||||
pipeline_cancel(context);
|
||||
mark_sender_done(context);
|
||||
return thrd_error;
|
||||
}
|
||||
ProtocolSession session;
|
||||
protocol_session_init(&session, client->file_descriptor, client->file_descriptor);
|
||||
protocol_session_set_io_timeout(&session, context->config->timeout);
|
||||
protocol_session_set_ssl(&session, (SSL*)client->ssl);
|
||||
protocol_session_bind(&session);
|
||||
client_messages_activate(true);
|
||||
if (!config_send(client->file_descriptor, context->config)) {
|
||||
pipeline_cancel(context);
|
||||
disconnect_transfer_client(client);
|
||||
@@ -967,7 +1104,7 @@ static int send_chunks_multithreaded(void* pipeline_context) {
|
||||
and wait for the receiver to delete extras before streaming any data. */
|
||||
if (!send_delete_manifest_early(client, context->manifest, context->excluded_paths,
|
||||
context->size_skipped_paths, context->missing_args,
|
||||
context->synced_dirs)) {
|
||||
context->synced_dirs, context->per_dir_rules)) {
|
||||
pipeline_cancel(context);
|
||||
disconnect_transfer_client(client);
|
||||
mark_sender_done(context);
|
||||
@@ -1097,7 +1234,8 @@ static int send_chunks_multithreaded(void* pipeline_context) {
|
||||
log_message(LOG_LEVEL_WARNING, "IO error encountered -- skipping file deletion");
|
||||
} else if (send_delete_manifest(client->file_descriptor, context->manifest,
|
||||
context->excluded_paths, context->size_skipped_paths,
|
||||
context->missing_args, context->synced_dirs) != 0) {
|
||||
context->missing_args, context->synced_dirs,
|
||||
context->per_dir_rules) != 0) {
|
||||
goto send_fail;
|
||||
}
|
||||
} else if (context->config->delete_missing_args && !context->early_delete &&
|
||||
@@ -1105,7 +1243,7 @@ static int send_chunks_multithreaded(void* pipeline_context) {
|
||||
/* --delete-missing-args without --delete: no keep-set is built, but the
|
||||
exact-delete paths still ride the same manifest frame (commit once the
|
||||
transfer succeeded). */
|
||||
if (send_delete_manifest(client->file_descriptor, NULL, NULL, NULL, context->missing_args,
|
||||
if (send_delete_manifest(client->file_descriptor, NULL, NULL, NULL, context->missing_args, NULL,
|
||||
NULL) != 0)
|
||||
goto send_fail;
|
||||
}
|
||||
@@ -1113,15 +1251,19 @@ static int send_chunks_multithreaded(void* pipeline_context) {
|
||||
(and all parallel workers) has been joined before scanner_done was set, so
|
||||
the list is complete and race-free; on an early stop the list may be
|
||||
incomplete and is deliberately not sent. */
|
||||
client_change_emit_pending_dirs(context->config, client->file_descriptor);
|
||||
if (!context->scan_stopped_early &&
|
||||
!send_dir_times(client, context->config, context->dir_entries))
|
||||
goto send_fail;
|
||||
bool delete_limit = false;
|
||||
bool partial = false;
|
||||
ReceiverStats recv_stats;
|
||||
memset(&recv_stats, 0, sizeof(recv_stats));
|
||||
client_flush_client_messages(client->file_descriptor);
|
||||
bool ok = finalize_transfer(client, context->config, context->remove_source_files, &delete_limit,
|
||||
&recv_stats);
|
||||
&partial, &recv_stats);
|
||||
context->delete_limit = delete_limit;
|
||||
context->partial = partial;
|
||||
if (!ok && context->config->use_delete)
|
||||
log_message(LOG_LEVEL_ERROR,
|
||||
"server reported a deletion failure (--delete); see the server log for the reason");
|
||||
@@ -1215,6 +1357,7 @@ static int scan_directory_multithreaded(void* pipeline_context) {
|
||||
its protected lists, so the data pass must not append to them again. */
|
||||
if (!context->early_delete && !context->delete_plans) {
|
||||
prepared.options.excluded_paths = context->excluded_paths;
|
||||
prepared.options.per_dir_rules = context->per_dir_rules;
|
||||
/* The root marker for a full recursive transfer is already in the list; do
|
||||
not let the scanner append every directory to it. */
|
||||
if (context->config->files_from_set != NULL)
|
||||
@@ -1334,7 +1477,7 @@ static int load_files_multithreaded(void* pipeline_context) {
|
||||
if (!context->config->use_sendfile) {
|
||||
for (int i = 0; i < chunk->element_count; i++) {
|
||||
File* f = chunk->items[i];
|
||||
if (f->data->size > STREAM_THRESHOLD && !context->config->use_compression)
|
||||
if (loader_can_stream(context->config, f))
|
||||
continue;
|
||||
if (!file_load_data(f)) {
|
||||
log_message(LOG_LEVEL_ERROR, "Failed to load file data");
|
||||
@@ -1448,6 +1591,8 @@ typedef struct {
|
||||
ArrayList* synced_dirs;
|
||||
ArrayList* plan_dirs;
|
||||
ArrayList* missing_args;
|
||||
/* Per-directory filter rules compiled by the scan (protocol 2.30.0). */
|
||||
FilterRuleList* per_dir_rules;
|
||||
PreparedScanner prepared;
|
||||
StopCondition stop;
|
||||
TransferStats transfer_stats;
|
||||
@@ -1492,9 +1637,11 @@ static bool send_files_prepare(Config* config, SendFilesState* state) {
|
||||
}
|
||||
state->size_skipped = array_list_create(free);
|
||||
state->synced_dirs = array_list_create(free);
|
||||
if (!state->size_skipped || !state->synced_dirs)
|
||||
state->per_dir_rules = filter_rule_list_create();
|
||||
if (!state->size_skipped || !state->synced_dirs || !state->per_dir_rules)
|
||||
return false;
|
||||
state->prepared.options.size_skipped_paths = state->size_skipped;
|
||||
state->prepared.options.per_dir_rules = state->per_dir_rules;
|
||||
/* Only a --files-from subset confines the extras walk to the directories
|
||||
the scan synchronized; a full recursive transfer deletes throughout the
|
||||
receive root, so mark the root itself (the "." sentinel) and let the
|
||||
@@ -1559,9 +1706,9 @@ static bool send_files_prepare_delete(Config* config, SendFilesState* state) {
|
||||
log_message(LOG_LEVEL_WARNING, "IO error encountered -- skipping file deletion");
|
||||
skip_delete = true;
|
||||
} else {
|
||||
early_ok =
|
||||
send_delete_manifest_early(client, early_manifest, state->excluded, state->size_skipped,
|
||||
state->missing_args, state->synced_dirs);
|
||||
early_ok = send_delete_manifest_early(client, early_manifest, state->excluded,
|
||||
state->size_skipped, state->missing_args,
|
||||
state->synced_dirs, state->per_dir_rules);
|
||||
}
|
||||
}
|
||||
array_list_delete(early_manifest);
|
||||
@@ -1570,6 +1717,7 @@ static bool send_files_prepare_delete(Config* config, SendFilesState* state) {
|
||||
state->prepared.options.excluded_paths = NULL;
|
||||
state->prepared.options.size_skipped_paths = NULL;
|
||||
state->prepared.options.synced_dirs = NULL;
|
||||
state->prepared.options.per_dir_rules = NULL;
|
||||
if (!prescan_ok || (!early_ok && !skip_delete)) {
|
||||
array_list_delete(prescan_chunks);
|
||||
return false;
|
||||
@@ -1599,7 +1747,7 @@ static bool send_files_prepare_delete(Config* config, SendFilesState* state) {
|
||||
config->files_from_set ? state->synced_dirs : (walk_root ? state->synced_dirs : NULL);
|
||||
delete_plan_sender_finalize(state->plan_sender, scope, walk_root);
|
||||
delete_plan_sender_set_config(state->plan_sender, state->excluded, state->size_skipped,
|
||||
state->missing_args);
|
||||
state->missing_args, state->per_dir_rules);
|
||||
if (state->had_scan_io && delete_plan_sender_empty(state->plan_sender)) {
|
||||
log_message(LOG_LEVEL_ERROR,
|
||||
"source scan hit an I/O error before finding any file; refusing to delete "
|
||||
@@ -1623,6 +1771,7 @@ static bool send_files_prepare_delete(Config* config, SendFilesState* state) {
|
||||
state->prepared.options.size_skipped_paths = NULL;
|
||||
state->prepared.options.synced_dirs = NULL;
|
||||
state->prepared.options.plan_dirs = NULL;
|
||||
state->prepared.options.per_dir_rules = NULL;
|
||||
if (!prescan_ok || (!plans_ok && !skip_delete))
|
||||
return false;
|
||||
} else if (config->use_delete) {
|
||||
@@ -1717,7 +1866,7 @@ static bool send_files_run(Config* config, SendFilesState* state) {
|
||||
bool load_ok = true;
|
||||
for (int i = 0; i < current_chunk->element_count; i++) {
|
||||
File* f = current_chunk->items[i];
|
||||
if (f->data->size > STREAM_THRESHOLD && !config->use_compression)
|
||||
if (loader_can_stream(config, f))
|
||||
continue;
|
||||
if (!file_load_data(f)) {
|
||||
log_message(LOG_LEVEL_ERROR, "Failed to load file data");
|
||||
@@ -1804,7 +1953,8 @@ static int send_files_finalize(const Config* config, SendFilesState* state) {
|
||||
modes the deletion already went out with the data, so nothing is
|
||||
re-sent here. */
|
||||
if (send_delete_manifest(client->file_descriptor, state->manifest, state->excluded,
|
||||
state->size_skipped, state->missing_args, state->synced_dirs) != 0) {
|
||||
state->size_skipped, state->missing_args, state->synced_dirs,
|
||||
state->per_dir_rules) != 0) {
|
||||
if (state->manifest) {
|
||||
array_list_delete(state->manifest);
|
||||
state->manifest = NULL;
|
||||
@@ -1817,15 +1967,22 @@ static int send_files_finalize(const Config* config, SendFilesState* state) {
|
||||
}
|
||||
}
|
||||
}
|
||||
/* Output parity: report changed directories that had no transferred child
|
||||
before the deferred directory times are applied (so the probe still sees
|
||||
their pre-transfer state). */
|
||||
client_change_emit_pending_dirs(config, client->file_descriptor);
|
||||
/* P7 Wave D: every directory has now been traversed (or the scan stopped
|
||||
early), so transmit the captured directory times last. The receiver defers
|
||||
applying them until after its own deletion/publication phase. */
|
||||
if (!send_dir_times(client, config, state->dir_entries))
|
||||
return 1;
|
||||
bool delete_limit = false;
|
||||
bool partial = false;
|
||||
ReceiverStats recv_stats;
|
||||
memset(&recv_stats, 0, sizeof(recv_stats));
|
||||
bool ok = finalize_transfer(client, config, state->remove_sources, &delete_limit, &recv_stats);
|
||||
client_flush_client_messages(client->file_descriptor);
|
||||
bool ok = finalize_transfer(client, config, state->remove_sources, &delete_limit, &partial,
|
||||
&recv_stats);
|
||||
if (!ok && config->use_delete)
|
||||
log_message(LOG_LEVEL_ERROR,
|
||||
"server reported a deletion failure (--delete); see the server log for the reason");
|
||||
@@ -1843,12 +2000,12 @@ static int send_files_finalize(const Config* config, SendFilesState* state) {
|
||||
(double)state->transfer_stats.transferred_file_size / (double)BYTES_PER_MIB);
|
||||
/* A skipped source entry (--ignore-errors past an unreadable directory, or a
|
||||
dereferenced symlink with no referent) makes rsync report a partial
|
||||
transfer (exit 23) even though the rest of the run succeeded. A
|
||||
--max-delete-capped commit is a successful transfer that rsync reports
|
||||
transfer (exit 23), as does a receiver per-entry failure (STATUS_PARTIAL).
|
||||
A --max-delete-capped commit is a successful transfer that rsync reports
|
||||
with exit code 25. */
|
||||
if (!ok)
|
||||
return 1;
|
||||
if (state->had_scan_io)
|
||||
if (state->had_scan_io || partial)
|
||||
return 23;
|
||||
return delete_limit ? 25 : 0;
|
||||
}
|
||||
@@ -1872,6 +2029,8 @@ static void send_files_cleanup(SendFilesState* state) {
|
||||
array_list_delete(state->plan_dirs);
|
||||
if (state->missing_args)
|
||||
array_list_delete(state->missing_args);
|
||||
if (state->per_dir_rules)
|
||||
filter_rule_list_free(state->per_dir_rules);
|
||||
if (state->remove_sources)
|
||||
array_list_delete(state->remove_sources);
|
||||
if (state->dir_entries)
|
||||
@@ -1885,7 +2044,18 @@ static void send_files_cleanup(SendFilesState* state) {
|
||||
client_set_abort_armed(false);
|
||||
}
|
||||
|
||||
static int send_files_impl(Config* config);
|
||||
|
||||
int send_files(Config* config) {
|
||||
/* Install the --stderr=client sink for the whole run (it only queues while a
|
||||
session is live) and release its queue on every return path. */
|
||||
client_messages_install();
|
||||
int rc = send_files_impl(config);
|
||||
client_messages_end();
|
||||
return rc;
|
||||
}
|
||||
|
||||
static int send_files_impl(Config* config) {
|
||||
if (config->list_only)
|
||||
return send_list_only(config);
|
||||
if (config->dry_run)
|
||||
@@ -1902,35 +2072,21 @@ int send_files(Config* config) {
|
||||
shielded -- rsync's `-d DIR/ --delete`. */
|
||||
state.delete_per_dir = config->use_delete && config_delete_timing_per_dir(config);
|
||||
int skipped = 0;
|
||||
if (config->delete_missing_args) {
|
||||
state.missing_args = array_list_create(free);
|
||||
if (!state.missing_args)
|
||||
if (!client_prepare_files_from(config, &state.missing_args, &skipped))
|
||||
return 1;
|
||||
}
|
||||
if (!files_from_list_check(config, state.missing_args, &skipped)) {
|
||||
if (state.missing_args)
|
||||
array_list_delete(state.missing_args);
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* From here on a server session may be live, so Ctrl-C/SIGTERM should set the
|
||||
abort flag (and be forwarded as STATUS_ABORT) instead of terminating. */
|
||||
client_set_abort_armed(true);
|
||||
Client* client = connect_transfer_client(config);
|
||||
ProtocolSession session;
|
||||
Client* client = client_connect_and_bind_session(config, &session);
|
||||
if (!client) {
|
||||
if (config->transport == TRANSPORT_TCP)
|
||||
log_message(LOG_LEVEL_ERROR, "could not connect to server%s",
|
||||
config->use_tls ? " via TLS" : "");
|
||||
if (state.missing_args)
|
||||
array_list_delete(state.missing_args);
|
||||
return 1;
|
||||
}
|
||||
state.client = client;
|
||||
ProtocolSession session;
|
||||
protocol_session_init(&session, client->file_descriptor, client->file_descriptor);
|
||||
protocol_session_set_io_timeout(&session, config->timeout);
|
||||
protocol_session_set_ssl(&session, (SSL*)client->ssl);
|
||||
protocol_session_bind(&session);
|
||||
client_messages_activate(true);
|
||||
|
||||
int ret = 1;
|
||||
if (!send_files_prepare(config, &state))
|
||||
@@ -1946,7 +2102,16 @@ send_fail:
|
||||
return ret;
|
||||
}
|
||||
|
||||
static int send_files_multithreaded_impl(Config* config);
|
||||
|
||||
int send_files_multithreaded(Config* config) {
|
||||
client_messages_install();
|
||||
int rc = send_files_multithreaded_impl(config);
|
||||
client_messages_end();
|
||||
return rc;
|
||||
}
|
||||
|
||||
static int send_files_multithreaded_impl(Config* config) {
|
||||
if (!config)
|
||||
return 1;
|
||||
if (config->list_only)
|
||||
@@ -1956,16 +2121,8 @@ int send_files_multithreaded(Config* config) {
|
||||
: send_dry_run_manifest(config);
|
||||
ArrayList* missing_args = NULL;
|
||||
int skipped = 0;
|
||||
if (config->delete_missing_args) {
|
||||
missing_args = array_list_create(free);
|
||||
if (!missing_args)
|
||||
if (!client_prepare_files_from(config, &missing_args, &skipped))
|
||||
return 1;
|
||||
}
|
||||
if (!files_from_list_check(config, missing_args, &skipped)) {
|
||||
if (missing_args)
|
||||
array_list_delete(missing_args);
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* Armed only once a session may go live (see send_files). */
|
||||
client_set_abort_armed(true);
|
||||
@@ -1994,6 +2151,8 @@ int send_files_multithreaded(Config* config) {
|
||||
queue_destroy(q1);
|
||||
if (q2)
|
||||
queue_destroy(q2);
|
||||
if (missing_args)
|
||||
array_list_delete(missing_args);
|
||||
return 1;
|
||||
}
|
||||
PipelineContextSender* context = pipeline_context_sender_create(config, q1, q2);
|
||||
@@ -2032,7 +2191,8 @@ int send_files_multithreaded(Config* config) {
|
||||
confined; only a --files-from subset records concrete directories. */
|
||||
context->size_skipped_paths = array_list_create(free);
|
||||
context->synced_dirs = array_list_create(free);
|
||||
if (!context->size_skipped_paths || !context->synced_dirs) {
|
||||
context->per_dir_rules = filter_rule_list_create();
|
||||
if (!context->size_skipped_paths || !context->synced_dirs || !context->per_dir_rules) {
|
||||
pipeline_context_sender_destroy(context);
|
||||
return 1;
|
||||
}
|
||||
@@ -2061,6 +2221,7 @@ int send_files_multithreaded(Config* config) {
|
||||
if (context->excluded_paths)
|
||||
prepared.options.excluded_paths = context->excluded_paths;
|
||||
prepared.options.size_skipped_paths = context->size_skipped_paths;
|
||||
prepared.options.per_dir_rules = context->per_dir_rules;
|
||||
/* The root marker for a full recursive transfer is already in the list;
|
||||
only a --files-from subset needs the scanner to record directories. */
|
||||
if (config->files_from_set != NULL)
|
||||
@@ -2101,7 +2262,8 @@ int send_files_multithreaded(Config* config) {
|
||||
: (walk_root ? context->synced_dirs : NULL);
|
||||
delete_plan_sender_finalize(context->delete_plans, scope, walk_root);
|
||||
delete_plan_sender_set_config(context->delete_plans, context->excluded_paths,
|
||||
context->size_skipped_paths, context->missing_args);
|
||||
context->size_skipped_paths, context->missing_args,
|
||||
context->per_dir_rules);
|
||||
}
|
||||
bool empty = per_dir
|
||||
? (context->delete_plans && delete_plan_sender_empty(context->delete_plans))
|
||||
@@ -2202,16 +2364,18 @@ int send_files_multithreaded(Config* config) {
|
||||
mtx_unlock(&context->mutex_scanner);
|
||||
bool sender_ok = sender_result == thrd_success;
|
||||
bool delete_limit = context->delete_limit;
|
||||
bool partial = context->partial;
|
||||
/* A skipped source entry (--ignore-errors past an unreadable directory, or a
|
||||
dereferenced symlink with no referent) makes rsync report a partial
|
||||
transfer (exit 23). A --max-delete-capped commit is a successful transfer
|
||||
that rsync reports with exit code 25. */
|
||||
transfer (exit 23), as does a receiver per-entry failure (STATUS_PARTIAL).
|
||||
A --max-delete-capped commit is a successful transfer that rsync reports
|
||||
with exit code 25. */
|
||||
pipeline_context_sender_destroy(context);
|
||||
client_progress_cleanup();
|
||||
client_set_abort_armed(false);
|
||||
if (!sender_ok)
|
||||
return 1;
|
||||
if (scan_io)
|
||||
if (scan_io || partial)
|
||||
return 23;
|
||||
return delete_limit ? 25 : 0;
|
||||
}
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
#include "delta.h"
|
||||
#include "format.h"
|
||||
#include "log.h"
|
||||
#include "protocol.h"
|
||||
#include "scanner.h"
|
||||
#include <stdatomic.h>
|
||||
#include <stdbool.h>
|
||||
@@ -68,14 +69,43 @@ void client_progress_name(const Config* config, const File* file);
|
||||
/* Emit a transferred entry's ancestor directories (as -i/--out-format change
|
||||
* lines or --progress name lines) before the entry's own line. */
|
||||
void client_change_emit_ancestors(const Config* config, const File* file);
|
||||
/* Output parity (protocol 2.30.0): probe each not-yet-known ancestor directory's
|
||||
* pre-transfer destination state before the entry that first triggers it is
|
||||
* sent. Returns false on a protocol/transport error. */
|
||||
bool client_change_probe_ancestors(const Config* config, const File* file, int fd);
|
||||
/* Mark a transferred directory entry as already reported, and flush the
|
||||
* itemize lines for changed directories that had no transferred child. */
|
||||
void client_change_mark_dir(const Config* config, const File* file);
|
||||
void client_change_emit_pending_dirs(const Config* config, int fd);
|
||||
void client_progress_uptodate(const Config* config, const File* file);
|
||||
void client_progress_prepare(const Config* config, const ArrayList* plan_dirs,
|
||||
unsigned long long plan_non_dir_count);
|
||||
bool receive_stats_record(int fd, ReceiverStats* stats, ArrayList* would_delete);
|
||||
/* --stderr=client diagnostic channel (client_report.c): install the queueing
|
||||
* log sink for a transfer, mark the session live, flush queued diagnostics over
|
||||
* the wire at a frame boundary, and tear the sink down. */
|
||||
void client_messages_install(void);
|
||||
void client_messages_activate(bool active);
|
||||
void client_flush_client_messages(int fd);
|
||||
void client_messages_end(void);
|
||||
|
||||
/* client_send.c */
|
||||
void receive_daemon_motd(Client* client, const Config* config);
|
||||
Client* connect_transfer_client(const Config* config);
|
||||
/* Connect the configured transport and install `session` on it: init with the
|
||||
* socket fd pair, apply the I/O timeout and (when negotiated) the TLS object,
|
||||
* then bind the session to this thread. Returns the connected client, or NULL
|
||||
* after logging the connect failure. The caller owns the client and must keep
|
||||
* `session` alive until it calls protocol_session_unbind(). */
|
||||
Client* client_connect_and_bind_session(const Config* config, ProtocolSession* session);
|
||||
/* Shared --files-from/--delete-missing-args preamble for the send entry points:
|
||||
* when --delete-missing-args is set, allocate the list that
|
||||
* files_from_list_check fills with the destination mirrors of missing entries;
|
||||
* then validate the --files-from list. On success returns true and stores the
|
||||
* (possibly NULL) owned list in *missing_args_out plus the skipped count; on
|
||||
* failure returns false after freeing the list. */
|
||||
bool client_prepare_files_from(const Config* config, ArrayList** missing_args_out,
|
||||
int* skipped_out);
|
||||
void disconnect_transfer_client(Client* client);
|
||||
int incremental_check(Client* client, File* file, const Config* config, DeltaSignature** out_sig,
|
||||
unsigned long long* resume_offset);
|
||||
@@ -87,9 +117,10 @@ int send_dry_run_manifest(const Config* config);
|
||||
int send_list_only(const Config* config);
|
||||
int send_dry_run_remote(Config* config);
|
||||
int send_delete_manifest(int fd, ArrayList* manifest, ArrayList* protected_prefixes,
|
||||
ArrayList* size_skipped, ArrayList* missing_args, ArrayList* synced_dirs);
|
||||
ArrayList* size_skipped, ArrayList* missing_args, ArrayList* synced_dirs,
|
||||
const FilterRuleList* per_dir_rules);
|
||||
bool send_delete_manifest_early(Client* client, ArrayList* manifest, ArrayList* protected_prefixes,
|
||||
ArrayList* size_skipped, ArrayList* missing_args,
|
||||
ArrayList* synced_dirs);
|
||||
ArrayList* synced_dirs, const FilterRuleList* per_dir_rules);
|
||||
|
||||
#endif
|
||||
|
||||
+12
-52
@@ -149,7 +149,7 @@ DirectoryScanner* directory_scanner_create_with_options(const char* root_directo
|
||||
scanner->options = *options;
|
||||
if (scanner->options.chunk_size == 0)
|
||||
scanner->options.chunk_size = DESIRED_CHUNK_SIZE;
|
||||
scanner->directories = queue_create(100, dir_entry_destroy);
|
||||
scanner->directories = queue_create(SCANNER_RESULT_QUEUE_CAP, dir_entry_destroy);
|
||||
if (!scanner->directories) {
|
||||
free(scanner);
|
||||
return NULL;
|
||||
@@ -1026,7 +1026,7 @@ static ScannerAction scanner_process_entry(DirectoryScanner* scanner, ArrayList*
|
||||
Chunk** out_chunk) {
|
||||
const char* name = sorted->name;
|
||||
ScannerEntry* inspected = &sorted->entry;
|
||||
char* cur_path = inspected->path;
|
||||
const char* cur_path = inspected->path;
|
||||
struct stat stats = inspected->stats;
|
||||
|
||||
/* --files-from allow-set and the filter layer apply to files and to
|
||||
@@ -1101,61 +1101,23 @@ static ScannerAction scanner_process_entry(DirectoryScanner* scanner, ArrayList*
|
||||
free(rel_copy);
|
||||
return SCANNER_ACTION_CONTINUE;
|
||||
}
|
||||
File* file = file_create(cur_path);
|
||||
if (file == NULL) {
|
||||
free(rel_copy);
|
||||
free(inspected->link_target);
|
||||
inspected->link_target = NULL;
|
||||
scanner->failed = true;
|
||||
return SCANNER_ACTION_CONTINUE;
|
||||
}
|
||||
if (inspected->is_symlink) {
|
||||
file->is_symlink = true;
|
||||
file->symlink_target = inspected->link_target;
|
||||
inspected->link_target = NULL;
|
||||
} else {
|
||||
file->data->size = stats.st_size;
|
||||
}
|
||||
if (scanner->relative_mode) {
|
||||
file->send_path = rel_copy;
|
||||
rel_copy = NULL;
|
||||
} else if (scanner->options.relative_prefix) {
|
||||
file->send_path = scanner_prefix_send_path(scanner->options.relative_prefix, rel_copy);
|
||||
/* Entry construction (data size, -R wire path, special/devices, hardlink
|
||||
group, metadata, xattrs) is shared with the parallel scanner. */
|
||||
File* file = NULL;
|
||||
bool build_failed = false;
|
||||
ScannerBuildStatus status =
|
||||
scanner_build_file_entry(&scanner->options, inspected, rel_copy, &file, &build_failed);
|
||||
free(rel_copy);
|
||||
rel_copy = NULL;
|
||||
if (!file->send_path) {
|
||||
file_destroy(file);
|
||||
if (build_failed)
|
||||
scanner->failed = true;
|
||||
return SCANNER_ACTION_BREAK;
|
||||
}
|
||||
}
|
||||
/* --devices/--specials: a device/FIFO/socket entry marked for preservation
|
||||
becomes a node to recreate (is_special, no data, rdev captured); an
|
||||
unrequested non-regular entry is skipped (rsync default). */
|
||||
ScannerSpecial special =
|
||||
scanner_prepare_special(scanner->options.preserve_devices, scanner->options.preserve_specials,
|
||||
scanner->options.copy_devices, file, &stats);
|
||||
if (special == SCANNER_SPECIAL_SKIP) {
|
||||
scanner_note_nonreg(&scanner->options, file->path);
|
||||
free(rel_copy);
|
||||
file_destroy(file);
|
||||
if (status == SCANNER_BUILD_SKIP)
|
||||
return SCANNER_ACTION_CONTINUE;
|
||||
}
|
||||
if (scanner->options.hardlinks && S_ISREG(stats.st_mode))
|
||||
scanner_assign_hardlink(scanner, scanner->options.hardlinks, file, &stats);
|
||||
if (scanner->options.use_metadata)
|
||||
file->metadata = file_metadata_create(file->path, &stats, scanner->options.preserve_atimes,
|
||||
scanner->options.preserve_crtimes);
|
||||
if (scanner->options.use_metadata && !file->metadata) {
|
||||
free(rel_copy);
|
||||
file_destroy(file);
|
||||
if (status != SCANNER_BUILD_OK) {
|
||||
scanner->failed = true;
|
||||
return SCANNER_ACTION_BREAK;
|
||||
return status == SCANNER_BUILD_FAIL_CONTINUE ? SCANNER_ACTION_CONTINUE : SCANNER_ACTION_BREAK;
|
||||
}
|
||||
if (!(file->link_group != 0 && !file->link_first))
|
||||
scanner_capture_xattrs(scanner, file);
|
||||
if (!array_list_add(chunk_data, file)) {
|
||||
free(rel_copy);
|
||||
file_destroy(file);
|
||||
scanner->failed = true;
|
||||
return SCANNER_ACTION_BREAK;
|
||||
@@ -1163,14 +1125,12 @@ static ScannerAction scanner_process_entry(DirectoryScanner* scanner, ArrayList*
|
||||
scanner->current_dir_produced = true;
|
||||
*chunk_data_size += file->data->size;
|
||||
if (*chunk_data_size > scanner->options.chunk_size) {
|
||||
free(rel_copy);
|
||||
Chunk* result = chunk_data_to_chunk(chunk_data);
|
||||
if (!result)
|
||||
scanner->failed = true;
|
||||
*out_chunk = result;
|
||||
return SCANNER_ACTION_CHUNK;
|
||||
}
|
||||
free(rel_copy);
|
||||
return SCANNER_ACTION_CONTINUE;
|
||||
}
|
||||
|
||||
|
||||
@@ -19,6 +19,11 @@
|
||||
* keeps one transfer from spawning an unbounded pool on a very large machine. */
|
||||
#define MAX_SCANNER_THREADS 256
|
||||
|
||||
/* Depth of the scanner's work queues: the sequential scanner's pending-directory
|
||||
* stack and the parallel scanner's result queue. Bounds memory for a very wide
|
||||
* or very deep tree while leaving ample headroom for normal scans. */
|
||||
#define SCANNER_RESULT_QUEUE_CAP 100
|
||||
|
||||
typedef struct {
|
||||
bool use_metadata;
|
||||
/* Phase 4 metadata capture: -U/--atimes and -N/--crtimes tell the scanner to
|
||||
@@ -115,6 +120,13 @@ typedef struct {
|
||||
* directories, exactly like rsync; the receive root is the "." sentinel.
|
||||
* Guarded by `excluded_mutex`. */
|
||||
ArrayList* synced_dirs;
|
||||
/* Per-directory filter-rule sink (optional): when non-NULL the scanner appends
|
||||
* a deep copy of every rule it reads from a per-directory merge file, each
|
||||
* carrying its owner directory and no-inherit flag (see filter.h). The delete
|
||||
* carriers transmit them so the receiver re-derives the per-directory
|
||||
* protect/risk set for destination-only entries. Guarded by `excluded_mutex`
|
||||
* like the other sinks. */
|
||||
FilterRuleList* per_dir_rules;
|
||||
/* Delete-plan directory sink (optional): when non-NULL the scanner appends
|
||||
* the destination-relative path of every directory it traverses (except the
|
||||
* receive root). The per-directory --delete-during/--delete-delay plan
|
||||
|
||||
+148
-30
@@ -285,32 +285,34 @@ bool entry_passes_selection(const FileListSet* file_list, const FilterRuleList*
|
||||
* read xattrs is non-fatal: the file is transferred without them. A symlink
|
||||
* entry reads the LINK's own xattrs (never the referent's) with the no-follow
|
||||
* variant; on Linux the VFS refuses xattrs on symlinks, so that yields NULL. */
|
||||
void scanner_capture_xattrs(const DirectoryScanner* scanner, File* file) {
|
||||
if (!scanner || !file || !(scanner->options.preserve_xattrs || scanner->options.preserve_acls))
|
||||
void scanner_capture_xattrs_opts(const ScannerOptions* options, File* file) {
|
||||
if (!options || !file || !(options->preserve_xattrs || options->preserve_acls))
|
||||
return;
|
||||
file->xattrs = file->is_symlink
|
||||
? xattr_capture_path_nofollow(file->path, scanner->options.preserve_acls)
|
||||
: xattr_capture_path(file->path, scanner->options.preserve_acls);
|
||||
file->xattrs = file->is_symlink ? xattr_capture_path_nofollow(file->path, options->preserve_acls)
|
||||
: xattr_capture_path(file->path, options->preserve_acls);
|
||||
}
|
||||
|
||||
void scanner_capture_xattrs(const DirectoryScanner* scanner, File* file) {
|
||||
if (!scanner)
|
||||
return;
|
||||
scanner_capture_xattrs_opts(&scanner->options, file);
|
||||
}
|
||||
|
||||
/* Apply --hard-links (-H) detection to one regular File. On a sibling (a
|
||||
* later member of an already-seen source inode) the File keeps the group id
|
||||
* and the first member's wire path but carries NO data payload (size 0); the
|
||||
* first member is left untouched (data present, link_first). Allocation
|
||||
* failure is fatal: the scanner is marked failed. */
|
||||
void scanner_assign_hardlink(DirectoryScanner* scanner, HardLinkTable* table, File* file,
|
||||
const struct stat* stats) {
|
||||
* first member is left untouched (data present, link_first). Returns false on
|
||||
* allocation failure (the caller marks the scan failed); the File stays usable
|
||||
* either way. */
|
||||
bool scanner_assign_hardlink(HardLinkTable* table, File* file, const struct stat* stats) {
|
||||
if (!table || !file || !stats)
|
||||
return;
|
||||
return true;
|
||||
int gid;
|
||||
bool is_first;
|
||||
char* first_path = NULL;
|
||||
if (!hardlink_table_assign(table, file_wire_path(file), stats->st_dev, stats->st_ino, &gid,
|
||||
&is_first, &first_path)) {
|
||||
if (scanner)
|
||||
scanner->failed = true;
|
||||
return;
|
||||
}
|
||||
&is_first, &first_path))
|
||||
return false;
|
||||
file->link_group = gid;
|
||||
file->link_first = is_first;
|
||||
if (!is_first) {
|
||||
@@ -319,6 +321,7 @@ void scanner_assign_hardlink(DirectoryScanner* scanner, HardLinkTable* table, Fi
|
||||
} else {
|
||||
free(first_path);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Phase 4 special/devices decision for one non-regular entry, matching rsync:
|
||||
@@ -399,6 +402,76 @@ void scanner_note_nonreg(const ScannerOptions* options, const char* fs_path) {
|
||||
fflush(stdout);
|
||||
}
|
||||
|
||||
/* Construct one non-directory File from an inspected entry. Shared by the
|
||||
* sequential and parallel scanners so entry construction has a single
|
||||
* implementation: data size (or carried symlink), -R wire path, special/devices
|
||||
* classification, hardlink group, metadata and xattr capture all happen here in
|
||||
* the same order for both. See the declaration for the ownership contract. */
|
||||
ScannerBuildStatus scanner_build_file_entry(const ScannerOptions* options, ScannerEntry* inspected,
|
||||
const char* rel, File** out_file, bool* failed) {
|
||||
*out_file = NULL;
|
||||
if (failed)
|
||||
*failed = false;
|
||||
File* file = file_create(inspected->path);
|
||||
if (!file) {
|
||||
/* The File never existed, so drop the not-yet-transferred symlink target
|
||||
here; the caller's entry teardown would otherwise double-free it. */
|
||||
free(inspected->link_target);
|
||||
inspected->link_target = NULL;
|
||||
return SCANNER_BUILD_FAIL_CONTINUE;
|
||||
}
|
||||
if (inspected->is_symlink) {
|
||||
file->is_symlink = true;
|
||||
file->symlink_target = inspected->link_target;
|
||||
inspected->link_target = NULL;
|
||||
} else {
|
||||
file->data->size = inspected->stats.st_size;
|
||||
}
|
||||
/* -R + --files-from uses the bare transfer-relative path; -R without
|
||||
--files-from prefixes it. Plain scans keep the source path. */
|
||||
bool relative_mode = options->relative && options->file_list != NULL;
|
||||
if (relative_mode) {
|
||||
file->send_path = str_dup(rel);
|
||||
} else if (options->relative_prefix) {
|
||||
file->send_path = scanner_prefix_send_path(options->relative_prefix, rel);
|
||||
}
|
||||
if ((relative_mode || options->relative_prefix) && !file->send_path) {
|
||||
file_destroy(file);
|
||||
return SCANNER_BUILD_FAIL_BREAK;
|
||||
}
|
||||
/* --devices/--specials: a device/FIFO/socket entry marked for preservation
|
||||
becomes a node to recreate (is_special, no data, rdev captured); an
|
||||
unrequested non-regular entry is skipped (rsync default). */
|
||||
ScannerSpecial special =
|
||||
scanner_prepare_special(options->preserve_devices, options->preserve_specials,
|
||||
options->copy_devices, file, &inspected->stats);
|
||||
if (special == SCANNER_SPECIAL_SKIP) {
|
||||
scanner_note_nonreg(options, file->path);
|
||||
file_destroy(file);
|
||||
return SCANNER_BUILD_SKIP;
|
||||
}
|
||||
if (options->hardlinks && S_ISREG(inspected->stats.st_mode) &&
|
||||
!scanner_assign_hardlink(options->hardlinks, file, &inspected->stats)) {
|
||||
/* Allocation failure is non-fatal to this entry (it is still emitted) but
|
||||
marks the scan failed, matching the historical inlined behaviour. */
|
||||
if (failed)
|
||||
*failed = true;
|
||||
}
|
||||
if (options->use_metadata) {
|
||||
file->metadata = file_metadata_create(file->path, &inspected->stats, options->preserve_atimes,
|
||||
options->preserve_crtimes);
|
||||
if (!file->metadata) {
|
||||
file_destroy(file);
|
||||
return SCANNER_BUILD_FAIL_BREAK;
|
||||
}
|
||||
}
|
||||
/* A hardlink sibling carries no data, so it carries no xattrs. */
|
||||
if (!(file->link_group != 0 && !file->link_first))
|
||||
scanner_capture_xattrs_opts(options, file);
|
||||
*out_file = file;
|
||||
return SCANNER_BUILD_OK;
|
||||
}
|
||||
|
||||
/* rsync 3.4.1's `--info=mount` line, emitted when `-xx` drops a mount-point
|
||||
* directory: `[sender] skipping mount-point dir NAME` (the client is the
|
||||
* sender). Plain `-x` keeps the empty directory and prints nothing, matching
|
||||
@@ -443,17 +516,16 @@ void scanner_record_size_skipped(DirectoryScanner* scanner, const char* fs_path)
|
||||
scanner_record_protected(scanner, fs_path, scanner->options.size_skipped_paths);
|
||||
}
|
||||
|
||||
/* Record a directory the scan synchronized. `fs_path` is its absolute path and
|
||||
`rel` its path relative to the transfer root ("" for the root); the stored
|
||||
form matches the wire layout (the bare relative path in -R+--files-from, else
|
||||
the source path with a leading '/' removed, with "." for the receive root).
|
||||
Returns false on allocation failure. */
|
||||
bool scanner_record_synced_dir(const ScannerOptions* options, const char* fs_path, const char* rel,
|
||||
/* The destination-relative coordinate the receiver's delete walkers match
|
||||
against for an entry at `fs_path` (with `rel` its path relative to the
|
||||
transfer root, "" for the root): `relative_prefix + rel` under -R+--relative,
|
||||
the bare relative path under -R+--files-from, else the source path with a
|
||||
leading '/' removed, with "." for the receive root. Shared by the
|
||||
synchronized-directory sink and the mirrored per-directory rule owners so
|
||||
both live in the same coordinate system. Returns an owned string, or NULL on
|
||||
allocation failure. */
|
||||
char* scanner_dest_rel_path(const ScannerOptions* options, const char* fs_path, const char* rel,
|
||||
bool relative_mode) {
|
||||
if (!options->synced_dirs && !options->plan_dirs)
|
||||
return true;
|
||||
if (!file_list_dir_in_scope(options->file_list, rel))
|
||||
return true;
|
||||
char* prefixed = NULL;
|
||||
const char* dest;
|
||||
if (relative_mode) {
|
||||
@@ -461,7 +533,7 @@ bool scanner_record_synced_dir(const ScannerOptions* options, const char* fs_pat
|
||||
} else if (options->relative_prefix) {
|
||||
prefixed = scanner_prefix_send_path(options->relative_prefix, rel);
|
||||
if (!prefixed)
|
||||
return false;
|
||||
return NULL;
|
||||
dest = prefixed;
|
||||
} else {
|
||||
dest = fs_path;
|
||||
@@ -470,6 +542,24 @@ bool scanner_record_synced_dir(const ScannerOptions* options, const char* fs_pat
|
||||
dest++;
|
||||
if (dest[0] == '\0')
|
||||
dest = ".";
|
||||
char* out = str_dup(dest);
|
||||
free(prefixed);
|
||||
return out;
|
||||
}
|
||||
|
||||
/* Record a directory the scan synchronized. `fs_path` is its absolute path and
|
||||
`rel` its path relative to the transfer root ("" for the root); the stored
|
||||
form matches the wire layout (see scanner_dest_rel_path). Returns false on
|
||||
allocation failure. */
|
||||
bool scanner_record_synced_dir(const ScannerOptions* options, const char* fs_path, const char* rel,
|
||||
bool relative_mode) {
|
||||
if (!options->synced_dirs && !options->plan_dirs)
|
||||
return true;
|
||||
if (!file_list_dir_in_scope(options->file_list, rel))
|
||||
return true;
|
||||
char* dest = scanner_dest_rel_path(options, fs_path, rel, relative_mode);
|
||||
if (!dest)
|
||||
return false;
|
||||
bool ok = true;
|
||||
if (options->synced_dirs)
|
||||
ok = excluded_sink_append(options->synced_dirs, options->excluded_mutex, dest);
|
||||
@@ -478,7 +568,7 @@ bool scanner_record_synced_dir(const ScannerOptions* options, const char* fs_pat
|
||||
than deleted as an extra; the receive root (".") is implicit. */
|
||||
if (ok && options->plan_dirs && strcmp(dest, ".") != 0)
|
||||
ok = excluded_sink_append(options->plan_dirs, options->excluded_mutex, dest);
|
||||
free(prefixed);
|
||||
free(dest);
|
||||
return ok;
|
||||
}
|
||||
|
||||
@@ -487,7 +577,8 @@ bool scanner_record_synced_dir(const ScannerOptions* options, const char* fs_pat
|
||||
* fresh list. Returns NULL on allocation/parse failure (message in `err`);
|
||||
* returns an empty list (and *any_exists=false) when no file exists. */
|
||||
FilterRuleList* read_dir_filters(const ScannerOptions* options, const char* dir_path,
|
||||
const char* rel, bool* any_exists, char* err, size_t err_size) {
|
||||
const char* rel, bool relative_mode, bool* any_exists, char* err,
|
||||
size_t err_size) {
|
||||
if (err && err_size > 0)
|
||||
err[0] = '\0';
|
||||
const FilterRuleList* base = options->base_filters;
|
||||
@@ -511,13 +602,40 @@ FilterRuleList* read_dir_filters(const ScannerOptions* options, const char* dir_
|
||||
}
|
||||
if (base) {
|
||||
for (int i = 0; i < base->dir_merge_count; i++) {
|
||||
if (!filter_file_append(own, dir_path, base->dir_merge_names[i], rel, &opts, &exists, err,
|
||||
if (!filter_dir_merge_append(own, dir_path, &base->dir_merges[i], rel, &opts, &exists, err,
|
||||
err_size))
|
||||
goto fail;
|
||||
if (exists && any_exists)
|
||||
*any_exists = true;
|
||||
}
|
||||
}
|
||||
/* Mirror the directory's rules into the delete-carrier sink so the receiver
|
||||
* can reconstruct its per-directory protect/risk set. The mirrored rules
|
||||
* carry the destination-relative owner coordinate (not the transfer-root-
|
||||
* relative one the sender's own evaluation uses) so the receiver's delete
|
||||
* walkers, which match against receive-root-relative paths, find them. */
|
||||
if (options->per_dir_rules && own->count > 0) {
|
||||
char* owner = scanner_dest_rel_path(options, dir_path, rel, relative_mode);
|
||||
if (!owner)
|
||||
goto fail;
|
||||
mtx_t* mtx = options->excluded_mutex;
|
||||
if (mtx)
|
||||
mtx_lock(mtx);
|
||||
for (int i = 0; i < own->count; i++) {
|
||||
FilterRule* copy = filter_rule_clone(own->items[i]);
|
||||
if (!copy || !filter_rule_set_owner(copy, owner) ||
|
||||
!filter_rule_list_add(options->per_dir_rules, copy)) {
|
||||
filter_rule_free(copy);
|
||||
if (mtx)
|
||||
mtx_unlock(mtx);
|
||||
free(owner);
|
||||
goto fail;
|
||||
}
|
||||
}
|
||||
if (mtx)
|
||||
mtx_unlock(mtx);
|
||||
free(owner);
|
||||
}
|
||||
return own;
|
||||
fail:
|
||||
filter_rule_list_free(own);
|
||||
@@ -534,7 +652,7 @@ int open_directory_filter_context(DirectoryScanner* scanner, const FilterNode* i
|
||||
bool any_exists = false;
|
||||
FilterRuleList* own = read_dir_filters(&scanner->options, scanner->current_path,
|
||||
scanner->current_rel ? scanner->current_rel : "",
|
||||
&any_exists, err, sizeof(err));
|
||||
scanner->relative_mode, &any_exists, err, sizeof(err));
|
||||
if (!own) {
|
||||
/* read_dir_filters() leaves `err` set on a parse/allocation failure even
|
||||
when an earlier merge file in the same directory existed (any_exists true);
|
||||
|
||||
@@ -62,6 +62,17 @@ typedef enum {
|
||||
SCANNER_SPECIAL_SKIP, /* non-regular entry not requested: skip */
|
||||
} ScannerSpecial;
|
||||
|
||||
/* Result of scanner_build_file_entry(). The two failure variants preserve the
|
||||
* sequential scanner's historical distinction between a failure before the
|
||||
* File existed (which kept walking the directory) and one afterwards (which cut
|
||||
* the chunk short); both mark the scan failed. */
|
||||
typedef enum {
|
||||
SCANNER_BUILD_OK, /* File built; caller owns it */
|
||||
SCANNER_BUILD_SKIP, /* non-regular entry not preserved; no File */
|
||||
SCANNER_BUILD_FAIL_CONTINUE, /* failed before the File existed */
|
||||
SCANNER_BUILD_FAIL_BREAK, /* failed after the File existed */
|
||||
} ScannerBuildStatus;
|
||||
|
||||
/* scanner_filter.c */
|
||||
void filter_node_destroy(void* item);
|
||||
FilterNode* filter_node_alloc(FilterNode* parent, FilterRuleList* own);
|
||||
@@ -73,14 +84,27 @@ File* scanner_build_dir_file(const char* path, const struct stat* stats,
|
||||
const ScannerOptions* options);
|
||||
char* child_rel_path(const char* parent_rel, const char* name);
|
||||
char* scanner_prefix_send_path(const char* prefix, const char* rel);
|
||||
char* scanner_dest_rel_path(const ScannerOptions* options, const char* fs_path, const char* rel,
|
||||
bool relative_mode);
|
||||
bool entry_passes_selection(const FileListSet* file_list, const FilterRuleList* base,
|
||||
const FilterNode* node, const char* rel, const char* leaf, bool is_dir,
|
||||
bool per_dir_filters, bool exclude_filter_files, bool* protect_out);
|
||||
void scanner_capture_xattrs(const DirectoryScanner* scanner, File* file);
|
||||
void scanner_assign_hardlink(DirectoryScanner* scanner, HardLinkTable* table, File* file,
|
||||
const struct stat* stats);
|
||||
void scanner_capture_xattrs_opts(const ScannerOptions* options, File* file);
|
||||
bool scanner_assign_hardlink(HardLinkTable* table, File* file, const struct stat* stats);
|
||||
ScannerSpecial scanner_prepare_special(bool preserve_devices, bool preserve_specials,
|
||||
bool copy_devices, File* file, const struct stat* stats);
|
||||
/* Build one non-directory transfer File from an inspected entry. `rel` is the
|
||||
* entry's transfer-root-relative path (used for the -R wire path); `inspected`
|
||||
* supplies the on-disk path, stats and (for a carried symlink) the target whose
|
||||
* ownership transfers to the File. Populates data size, send_path, special-node
|
||||
* state, hardlink group, metadata and xattrs. On SCANNER_BUILD_OK the caller
|
||||
* owns *out_file; on SCANNER_BUILD_SKIP it is NULL and the entry is dropped; on
|
||||
* either failure it is NULL and the caller must mark the scan failed. `*failed`
|
||||
* additionally reports a non-fatal hardlink-table allocation failure, in which
|
||||
* case a usable File is still returned. */
|
||||
ScannerBuildStatus scanner_build_file_entry(const ScannerOptions* options, ScannerEntry* inspected,
|
||||
const char* rel, File** out_file, bool* failed);
|
||||
bool excluded_sink_append(ArrayList* list, mtx_t* mtx, const char* rel);
|
||||
void scanner_note_nonreg(const ScannerOptions* options, const char* fs_path);
|
||||
void scanner_note_mount(const ScannerOptions* options, const char* fs_path);
|
||||
@@ -92,7 +116,8 @@ void scanner_record_size_skipped(DirectoryScanner* scanner, const char* fs_path)
|
||||
bool scanner_record_synced_dir(const ScannerOptions* options, const char* fs_path, const char* rel,
|
||||
bool relative_mode);
|
||||
FilterRuleList* read_dir_filters(const ScannerOptions* options, const char* dir_path,
|
||||
const char* rel, bool* any_exists, char* err, size_t err_size);
|
||||
const char* rel, bool relative_mode, bool* any_exists, char* err,
|
||||
size_t err_size);
|
||||
int open_directory_filter_context(DirectoryScanner* scanner, const FilterNode* inherited);
|
||||
int scanner_inspect_entry(const ScannerOptions* options, const char* containing_dir,
|
||||
const char* link_rel, const char* name, ScannerEntry* entry);
|
||||
|
||||
+170
-194
@@ -109,7 +109,7 @@ static void parallel_scanner_creation_failed(ParallelScanner* ps) {
|
||||
|
||||
/* Initialize result queue and synchronization primitives. Returns true on success. */
|
||||
static bool parallel_scanner_init(ParallelScanner* ps) {
|
||||
ps->result_queue = queue_create(100, chunk_destroy);
|
||||
ps->result_queue = queue_create(SCANNER_RESULT_QUEUE_CAP, chunk_destroy);
|
||||
if (!ps->result_queue)
|
||||
return false;
|
||||
atomic_init(&ps->cancelled, false);
|
||||
@@ -210,6 +210,157 @@ static Chunk* batch_files(ArrayList* files, unsigned long long chunk_size, Queue
|
||||
return first;
|
||||
}
|
||||
|
||||
/* Record the delete-protection mirror of a root entry that
|
||||
* scanner_inspect_entry() skipped (inspection == 0): a dereferenced symlink
|
||||
* with no referent is a partial-transfer I/O error and a user-selection or size
|
||||
* prune protects the entry's destination mirror. */
|
||||
static void scan_root_record_skipped(const ScannerOptions* options, const char* root_directory,
|
||||
const char* name, const ScannerEntry* inspected,
|
||||
ParallelScanner* ps) {
|
||||
if (inspected->referent_error)
|
||||
ps->io_error = true;
|
||||
ArrayList* sink = NULL;
|
||||
if (inspected->excluded)
|
||||
sink = inspected->size_excluded ? options->size_skipped_paths : options->excluded_paths;
|
||||
if (!sink)
|
||||
return;
|
||||
/* A root-level prune protects the destination mirror of the entry's wire
|
||||
path: under -R + --files-from that is the bare relative name, otherwise it
|
||||
is the full source path with a leading '/' removed (matching the
|
||||
send_path/file_wire_path the scanner hands the sender). */
|
||||
if (options->relative && options->file_list != NULL) {
|
||||
if (!excluded_sink_append(sink, options->excluded_mutex, name))
|
||||
ps->failed = true;
|
||||
} else if (options->relative_prefix) {
|
||||
char* wrel = scanner_prefix_send_path(options->relative_prefix, name);
|
||||
if (!wrel) {
|
||||
ps->failed = true;
|
||||
} else {
|
||||
if (!excluded_sink_append(sink, options->excluded_mutex, wrel))
|
||||
ps->failed = true;
|
||||
free(wrel);
|
||||
}
|
||||
} else {
|
||||
char* abs_path = path_cat(root_directory, name);
|
||||
if (!abs_path) {
|
||||
ps->failed = true;
|
||||
} else {
|
||||
const char* rel = *abs_path == '/' ? abs_path + 1 : abs_path;
|
||||
if (!excluded_sink_append(sink, options->excluded_mutex, rel))
|
||||
ps->failed = true;
|
||||
free(abs_path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Record the delete-protection mirror of a root entry dropped by the
|
||||
* --files-from allow-set or a filter rule. Returns false only when the -R
|
||||
* prefix could not be built (the caller must abandon the entry immediately);
|
||||
* other allocation failures mark the scan failed but let the caller continue to
|
||||
* the filter-notice step, matching the historical inlined flow. */
|
||||
static bool scan_root_record_protection(const ScannerOptions* options, const char* rel,
|
||||
const char* name, const char* cur_path, bool protect,
|
||||
bool passes, bool use_rel, ParallelScanner* ps) {
|
||||
if (passes && !protect)
|
||||
return true;
|
||||
/* --files-from subset pruning is not a filter exclusion; -R bare-wire-path
|
||||
exclusions are never recorded (see ScannerOptions.excluded_paths). */
|
||||
bool files_from_prune = options->file_list && !file_list_affects(options->file_list, rel);
|
||||
if ((!files_from_prune && !use_rel) || protect) {
|
||||
const char* rel_path;
|
||||
char* prefixed = NULL;
|
||||
if (use_rel) {
|
||||
/* -R + --files-from: the destination/wire path is the bare relative
|
||||
name, not the source path. */
|
||||
rel_path = rel;
|
||||
} else if (options->relative_prefix) {
|
||||
prefixed = scanner_prefix_send_path(options->relative_prefix, name);
|
||||
if (!prefixed)
|
||||
return false;
|
||||
rel_path = prefixed;
|
||||
} else {
|
||||
rel_path = *cur_path == '/' ? cur_path + 1 : cur_path;
|
||||
}
|
||||
if (options->excluded_paths &&
|
||||
!excluded_sink_append(options->excluded_paths, options->excluded_mutex, rel_path))
|
||||
ps->failed = true;
|
||||
free(prefixed);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Root-level directory node: apply -x/--one-file-system and either emit the
|
||||
* mount-point directory (plain -x) or queue the directory for a worker. */
|
||||
static void scan_root_dir(const ScannerOptions* options, const char* cur_path, const char* rel,
|
||||
const struct stat* st, ArrayList* root_files, ArrayList* subdirs,
|
||||
dev_t root_dev, ParallelScanner* ps) {
|
||||
if (!scanner_same_filesystem(options->one_file_system, root_dev, st->st_dev)) {
|
||||
if (options->one_file_system > 1) {
|
||||
/* -xx: drop the mount-point directory entirely (rsync) and print the
|
||||
--info=mount line when enabled. */
|
||||
scanner_note_mount(options, cur_path);
|
||||
return;
|
||||
}
|
||||
/* -x/--one-file-system: emit the mount-point directory entry (empty) but do
|
||||
not descend into it (see the sequential scanner for the same rule). */
|
||||
File* mount = scanner_build_dir_file(cur_path, st, options);
|
||||
if (!mount) {
|
||||
ps->failed = true;
|
||||
return;
|
||||
}
|
||||
if (options->relative_prefix) {
|
||||
mount->send_path = scanner_prefix_send_path(options->relative_prefix, rel);
|
||||
if (!mount->send_path) {
|
||||
file_destroy(mount);
|
||||
ps->failed = true;
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (!array_list_add(root_files, mount)) {
|
||||
file_destroy(mount);
|
||||
ps->failed = true;
|
||||
}
|
||||
return;
|
||||
}
|
||||
char* dir = str_dup(cur_path);
|
||||
if (!dir || !array_list_add(subdirs, dir)) {
|
||||
free(dir);
|
||||
ps->failed = true;
|
||||
}
|
||||
}
|
||||
|
||||
/* Build a non-directory root entry through the shared construction path and add
|
||||
* it to `root_files`. A non-regular entry the options do not preserve is
|
||||
* dropped by the builder (which prints rsync's nonreg line); an allocation
|
||||
* failure marks the scan failed. */
|
||||
static void scan_root_add_non_dir(const ScannerOptions* options, ScannerEntry* inspected,
|
||||
const char* rel, ArrayList* root_files, ParallelScanner* ps) {
|
||||
File* file = NULL;
|
||||
bool failed = false;
|
||||
ScannerBuildStatus status = scanner_build_file_entry(options, inspected, rel, &file, &failed);
|
||||
if (failed || status == SCANNER_BUILD_FAIL_CONTINUE || status == SCANNER_BUILD_FAIL_BREAK)
|
||||
ps->failed = true;
|
||||
if (status != SCANNER_BUILD_OK)
|
||||
return;
|
||||
if (!array_list_add(root_files, file)) {
|
||||
file_destroy(file);
|
||||
ps->failed = true;
|
||||
}
|
||||
}
|
||||
|
||||
/* Regular file or carried symlink at the transfer root. */
|
||||
static void scan_root_file(const ScannerOptions* options, ScannerEntry* inspected, const char* rel,
|
||||
ArrayList* root_files, ParallelScanner* ps) {
|
||||
scan_root_add_non_dir(options, inspected, rel, root_files, ps);
|
||||
}
|
||||
|
||||
/* Device/FIFO/socket at the transfer root: recreated under --devices/--specials,
|
||||
* otherwise dropped by the shared builder. */
|
||||
static void scan_root_special(const ScannerOptions* options, ScannerEntry* inspected,
|
||||
const char* rel, ArrayList* root_files, ParallelScanner* ps) {
|
||||
scan_root_add_non_dir(options, inspected, rel, root_files, ps);
|
||||
}
|
||||
|
||||
/* Scan one root-directory entry into either the subdirs or files list. */
|
||||
static void scan_root_entry(const ScannerOptions* options, const FilterNode* root_node,
|
||||
const char* root_directory, const struct dirent* entry,
|
||||
@@ -223,51 +374,16 @@ static void scan_root_entry(const ScannerOptions* options, const FilterNode* roo
|
||||
return;
|
||||
}
|
||||
if (inspection == 0) {
|
||||
if (inspected.referent_error)
|
||||
ps->io_error = true;
|
||||
ArrayList* sink = NULL;
|
||||
if (inspected.excluded)
|
||||
sink = inspected.size_excluded ? options->size_skipped_paths : options->excluded_paths;
|
||||
if (sink) {
|
||||
/* A root-level prune protects the destination mirror of the entry's wire
|
||||
path: under -R + --files-from that is the bare relative name, otherwise
|
||||
it is the full source path with a leading '/' removed (matching the
|
||||
send_path/file_wire_path the scanner hands the sender). */
|
||||
if (options->relative && options->file_list != NULL) {
|
||||
if (!excluded_sink_append(sink, options->excluded_mutex, entry->d_name))
|
||||
ps->failed = true;
|
||||
} else if (options->relative_prefix) {
|
||||
char* wrel = scanner_prefix_send_path(options->relative_prefix, entry->d_name);
|
||||
if (!wrel) {
|
||||
ps->failed = true;
|
||||
} else {
|
||||
if (!excluded_sink_append(sink, options->excluded_mutex, wrel))
|
||||
ps->failed = true;
|
||||
free(wrel);
|
||||
}
|
||||
} else {
|
||||
char* abs_path = path_cat(root_directory, entry->d_name);
|
||||
if (!abs_path) {
|
||||
ps->failed = true;
|
||||
} else {
|
||||
const char* rel = *abs_path == '/' ? abs_path + 1 : abs_path;
|
||||
if (!excluded_sink_append(sink, options->excluded_mutex, rel))
|
||||
ps->failed = true;
|
||||
free(abs_path);
|
||||
}
|
||||
}
|
||||
}
|
||||
scan_root_record_skipped(options, root_directory, entry->d_name, &inspected, ps);
|
||||
return;
|
||||
}
|
||||
char* cur_path = inspected.path;
|
||||
struct stat st = inspected.stats;
|
||||
bool is_dir = inspected.is_directory;
|
||||
char* rel = str_dup(entry->d_name);
|
||||
if (!rel) {
|
||||
free(cur_path);
|
||||
ps->failed = true;
|
||||
return;
|
||||
goto done;
|
||||
}
|
||||
bool is_dir = inspected.is_directory;
|
||||
bool protect = false;
|
||||
bool passes = entry_passes_selection(options->file_list, options->base_filters, root_node, rel,
|
||||
entry->d_name, is_dir, options->per_dir_filters,
|
||||
@@ -275,169 +391,28 @@ static void scan_root_entry(const ScannerOptions* options, const FilterNode* roo
|
||||
/* -R + --files-from: root-level files keep their bare relative send path. */
|
||||
bool use_rel = options->relative && options->file_list != NULL;
|
||||
if (!passes || protect) {
|
||||
/* --files-from subset pruning is not a filter exclusion; -R bare-wire-path
|
||||
exclusions are never recorded (see ScannerOptions.excluded_paths). */
|
||||
bool files_from_prune = options->file_list && !file_list_affects(options->file_list, rel);
|
||||
if ((!files_from_prune && !use_rel) || protect) {
|
||||
const char* rel_path;
|
||||
char* prefixed = NULL;
|
||||
if (use_rel) {
|
||||
/* -R + --files-from: the destination/wire path is the bare relative
|
||||
name, not the source path. */
|
||||
rel_path = rel;
|
||||
} else if (options->relative_prefix) {
|
||||
prefixed = scanner_prefix_send_path(options->relative_prefix, entry->d_name);
|
||||
if (!prefixed) {
|
||||
free(rel);
|
||||
free(cur_path);
|
||||
if (!scan_root_record_protection(options, rel, entry->d_name, cur_path, protect, passes,
|
||||
use_rel, ps)) {
|
||||
ps->failed = true;
|
||||
return;
|
||||
}
|
||||
rel_path = prefixed;
|
||||
} else {
|
||||
rel_path = *cur_path == '/' ? cur_path + 1 : cur_path;
|
||||
}
|
||||
if (options->excluded_paths &&
|
||||
!excluded_sink_append(options->excluded_paths, options->excluded_mutex, rel_path))
|
||||
ps->failed = true;
|
||||
free(prefixed);
|
||||
goto done;
|
||||
}
|
||||
if (!passes) {
|
||||
scanner_note_filter(options, entry->d_name);
|
||||
free(rel);
|
||||
free(cur_path);
|
||||
return;
|
||||
goto done;
|
||||
}
|
||||
}
|
||||
if (is_dir) {
|
||||
if (!scanner_same_filesystem(options->one_file_system, root_dev, st.st_dev)) {
|
||||
if (options->one_file_system > 1) {
|
||||
/* -xx: drop the mount-point directory entirely (rsync) and print the
|
||||
--info=mount line when enabled. */
|
||||
scanner_note_mount(options, cur_path);
|
||||
scan_root_dir(options, cur_path, rel, &inspected.stats, root_files, subdirs, root_dev, ps);
|
||||
} else if (S_ISCHR(inspected.stats.st_mode) || S_ISBLK(inspected.stats.st_mode) ||
|
||||
S_ISFIFO(inspected.stats.st_mode) || S_ISSOCK(inspected.stats.st_mode)) {
|
||||
scan_root_special(options, &inspected, rel, root_files, ps);
|
||||
} else {
|
||||
scan_root_file(options, &inspected, rel, root_files, ps);
|
||||
}
|
||||
done:
|
||||
free(rel);
|
||||
free(cur_path);
|
||||
return;
|
||||
}
|
||||
/* -x/--one-file-system: emit the mount-point directory entry (empty) but
|
||||
do not descend into it (see the sequential scanner for the same rule). */
|
||||
File* mount = file_create(cur_path);
|
||||
free(cur_path);
|
||||
if (mount == NULL) {
|
||||
free(rel);
|
||||
ps->failed = true;
|
||||
return;
|
||||
}
|
||||
mount->is_dir = true;
|
||||
if (options->use_metadata) {
|
||||
mount->metadata = file_metadata_create(mount->path, &st, options->preserve_atimes,
|
||||
options->preserve_crtimes);
|
||||
if (!mount->metadata) {
|
||||
free(rel);
|
||||
file_destroy(mount);
|
||||
ps->failed = true;
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (options->relative_prefix) {
|
||||
mount->send_path = scanner_prefix_send_path(options->relative_prefix, rel);
|
||||
if (!mount->send_path) {
|
||||
free(rel);
|
||||
file_destroy(mount);
|
||||
ps->failed = true;
|
||||
return;
|
||||
}
|
||||
}
|
||||
free(rel);
|
||||
if (!array_list_add(root_files, mount)) {
|
||||
file_destroy(mount);
|
||||
ps->failed = true;
|
||||
}
|
||||
return;
|
||||
}
|
||||
free(rel);
|
||||
if (!array_list_add(subdirs, cur_path)) {
|
||||
free(cur_path);
|
||||
ps->failed = true;
|
||||
}
|
||||
return;
|
||||
}
|
||||
File* file = file_create(cur_path);
|
||||
free(cur_path);
|
||||
if (!file) {
|
||||
free(rel);
|
||||
free(inspected.link_target);
|
||||
inspected.link_target = NULL;
|
||||
ps->failed = true;
|
||||
return;
|
||||
}
|
||||
if (inspected.is_symlink) {
|
||||
file->is_symlink = true;
|
||||
file->symlink_target = inspected.link_target;
|
||||
inspected.link_target = NULL;
|
||||
} else {
|
||||
file->data->size = st.st_size;
|
||||
}
|
||||
if (use_rel) {
|
||||
file->send_path = rel;
|
||||
rel = NULL;
|
||||
} else if (options->relative_prefix) {
|
||||
file->send_path = scanner_prefix_send_path(options->relative_prefix, rel);
|
||||
free(rel);
|
||||
rel = NULL;
|
||||
if (!file->send_path) {
|
||||
file_destroy(file);
|
||||
ps->failed = true;
|
||||
return;
|
||||
}
|
||||
}
|
||||
ScannerSpecial special = scanner_prepare_special(
|
||||
options->preserve_devices, options->preserve_specials, options->copy_devices, file, &st);
|
||||
if (special == SCANNER_SPECIAL_SKIP) {
|
||||
scanner_note_nonreg(ps->options, file->path);
|
||||
free(rel);
|
||||
file_destroy(file);
|
||||
return;
|
||||
}
|
||||
if (options->hardlinks && S_ISREG(st.st_mode)) {
|
||||
int gid;
|
||||
bool is_first;
|
||||
char* first_path = NULL;
|
||||
if (!hardlink_table_assign((HardLinkTable*)options->hardlinks, file_wire_path(file), st.st_dev,
|
||||
st.st_ino, &gid, &is_first, &first_path)) {
|
||||
ps->failed = true;
|
||||
} else {
|
||||
file->link_group = gid;
|
||||
file->link_first = is_first;
|
||||
if (!is_first) {
|
||||
file->hardlink_target = first_path;
|
||||
file->data->size = 0;
|
||||
} else {
|
||||
free(first_path);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (options->use_metadata)
|
||||
file->metadata =
|
||||
file_metadata_create(file->path, &st, options->preserve_atimes, options->preserve_crtimes);
|
||||
if (options->use_metadata && !file->metadata) {
|
||||
free(rel);
|
||||
file_destroy(file);
|
||||
ps->failed = true;
|
||||
return;
|
||||
}
|
||||
if ((options->preserve_xattrs || options->preserve_acls) &&
|
||||
!(file->link_group != 0 && !file->link_first))
|
||||
file->xattrs = file->is_symlink
|
||||
? xattr_capture_path_nofollow(file->path, options->preserve_acls)
|
||||
: xattr_capture_path(file->path, options->preserve_acls);
|
||||
if (!array_list_add(root_files, file)) {
|
||||
free(rel);
|
||||
file_destroy(file);
|
||||
ps->failed = true;
|
||||
return;
|
||||
}
|
||||
free(rel);
|
||||
}
|
||||
|
||||
/* Scan the root directory itself, collecting root files and subdirectories.
|
||||
@@ -589,8 +564,9 @@ ParallelScanner* parallel_scanner_create_with_options(const char* root_directory
|
||||
{
|
||||
char err[256];
|
||||
bool any_exists = false;
|
||||
FilterRuleList* own =
|
||||
read_dir_filters(options, root_directory, "", &any_exists, err, sizeof(err));
|
||||
FilterRuleList* own = read_dir_filters(options, root_directory, "",
|
||||
options->relative && options->file_list != NULL,
|
||||
&any_exists, err, sizeof(err));
|
||||
if (!own) {
|
||||
/* A parse/allocation failure must fail the scan even when an earlier
|
||||
merge file in the same directory existed (see the sequential scanner). */
|
||||
|
||||
+5
-3
@@ -297,11 +297,13 @@ void print_usage(void) {
|
||||
printf(" --max-depth <n> Maximum directory depth (0=unlimited)\n");
|
||||
printf(" -x, --one-file-system Do not cross filesystem boundaries\n");
|
||||
printf(" --log-file <path>, --log-file=<path> Write log messages to file\n");
|
||||
printf(" --stderr=MODE Route logging to stderr: errors or all\n");
|
||||
printf(" --stderr=MODE Route logging: errors (default), all, or client\n");
|
||||
printf(" (forward the client's diagnostics to the server's\n");
|
||||
printf(" stderr)\n");
|
||||
printf(" --msgs2stderr Route all messages to stderr (deprecated spelling of\n");
|
||||
printf(" --stderr=all)\n");
|
||||
printf(" --no-msgs2stderr Select errors-only stderr (deprecated spelling; the\n");
|
||||
printf(" default)\n");
|
||||
printf(" --no-msgs2stderr Forward the client's diagnostics to the server\n");
|
||||
printf(" (deprecated spelling of --stderr=client)\n");
|
||||
printf(" --partial Keep partial files on interrupted transfer\n");
|
||||
printf(" --partial-dir <dir> Directory for partial files (implies --partial)\n");
|
||||
printf(" -T, --temp-dir <dir> Scratch dir for temp files before atomic install.\n");
|
||||
|
||||
+226
-53
@@ -11,10 +11,13 @@
|
||||
#include "metadata.h"
|
||||
#include "protocol.h"
|
||||
#include "utils.h"
|
||||
#include <fcntl.h>
|
||||
#include <limits.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/stat.h>
|
||||
#include <time.h>
|
||||
#include <unistd.h>
|
||||
|
||||
bool receiver_outcomes_append(ReceiverOutcomes* outcomes, unsigned char code) {
|
||||
if (!outcomes)
|
||||
@@ -101,14 +104,35 @@ static void receiver_tally_deleted(const ReceiverSink* sink, size_t deleted) {
|
||||
sink->stats->deleted_files += deleted;
|
||||
}
|
||||
|
||||
/* Observer for --info=del: record each truly-removed destination-relative path
|
||||
in the ArrayList passed as the observer context, so the terminal STATUS_STATS
|
||||
frame can list it. A failed append is best-effort (the deletion already
|
||||
happened; output is cosmetic). Shared by the single-threaded receiver and
|
||||
the -m pipeline's deferred commit. */
|
||||
void receiver_record_deleted_path(void* context, const char* rel_path) {
|
||||
ArrayList* paths = context;
|
||||
if (!paths || !rel_path)
|
||||
/* Observer for --info=del/--stats: record each truly-removed destination-
|
||||
relative path (when the context carries a path list) and tally it by type
|
||||
(when it carries a stats record), so the terminal STATUS_STATS frame can list
|
||||
the paths and render rsync's per-type `Number of deleted files` breakdown. A
|
||||
failed append is best-effort (the deletion already happened; output is
|
||||
cosmetic). Shared by the single-threaded receiver and the -m pipeline's
|
||||
deferred commit. */
|
||||
void receiver_record_deleted_path(void* context, const char* rel_path, DeleteEntryType type) {
|
||||
ReceiverDeleteContext* del = context;
|
||||
if (!del || !rel_path)
|
||||
return;
|
||||
if (del->stats) {
|
||||
switch (type) {
|
||||
case DELETE_ENTRY_DIR:
|
||||
del->stats->deleted_dir++;
|
||||
break;
|
||||
case DELETE_ENTRY_LINK:
|
||||
del->stats->deleted_link++;
|
||||
break;
|
||||
case DELETE_ENTRY_SPECIAL:
|
||||
del->stats->deleted_special++;
|
||||
break;
|
||||
default:
|
||||
del->stats->deleted_reg++;
|
||||
break;
|
||||
}
|
||||
}
|
||||
ArrayList* paths = del->deleted_paths;
|
||||
if (!paths)
|
||||
return;
|
||||
/* Bound the retained list like the keep-set manifest: only MAX_MANIFEST_ENTRIES
|
||||
paths are ever transmitted in the terminal STATUS_STATS frame, so recording
|
||||
@@ -120,6 +144,16 @@ void receiver_record_deleted_path(void* context, const char* rel_path) {
|
||||
free(copy);
|
||||
}
|
||||
|
||||
/* Install the delete observer (and its context) for one commit when the sink
|
||||
carries a stats record or a path list. Returns NULL when neither is needed,
|
||||
so the delete engines skip the observer entirely. */
|
||||
static DeletePathObserver receiver_delete_observer(const ReceiverSink* sink,
|
||||
ReceiverDeleteContext* del) {
|
||||
del->stats = sink ? sink->stats : NULL;
|
||||
del->deleted_paths = sink ? sink->deleted_paths : NULL;
|
||||
return (del->stats || del->deleted_paths) ? receiver_record_deleted_path : NULL;
|
||||
}
|
||||
|
||||
static bool receiver_process_chunk(Chunk* chunk, const ReceiverSink* sink) {
|
||||
if (!chunk || !sink || !sink->store_file)
|
||||
return false;
|
||||
@@ -277,6 +311,7 @@ static bool status_counts_as_progress(Status status) {
|
||||
case STATUS_ABORT:
|
||||
case STATUS_CHECK_BATCH:
|
||||
case STATUS_DIR_TIMES:
|
||||
case STATUS_CLIENT_MSG:
|
||||
return false;
|
||||
default:
|
||||
return true;
|
||||
@@ -306,7 +341,13 @@ static bool receiver_note_status(const struct timespec* session_start,
|
||||
}
|
||||
|
||||
int receiver_process(Config* config, int file_descriptor, const ReceiverSink* sink) {
|
||||
return receiver_process_pending(config, file_descriptor, sink, NULL, NULL);
|
||||
return receiver_process_pending_ctx(config, file_descriptor, sink, NULL, NULL, NULL);
|
||||
}
|
||||
|
||||
int receiver_process_pending(Config* config, int file_descriptor, const ReceiverSink* sink,
|
||||
DeleteManifest** pending_manifest, DeletePlanSession** pending_plans) {
|
||||
return receiver_process_pending_ctx(config, file_descriptor, sink, pending_manifest,
|
||||
pending_plans, NULL);
|
||||
}
|
||||
|
||||
/* Per-connection state threaded through the status handlers below. The parked
|
||||
@@ -327,6 +368,11 @@ typedef struct {
|
||||
a successful FINISHED it is either committed here or handed to
|
||||
*pending_plans so the -m caller commits after its disk writer drained. */
|
||||
DeletePlanSession* plan_session;
|
||||
/* Observer context for the per-directory delete session, which outlives the
|
||||
frame handler; must stay alive until the session commits. For a session
|
||||
handed to the caller (pending_plans) this points at a caller-owned
|
||||
long-lived context; otherwise it points at an internal stack context. */
|
||||
ReceiverDeleteContext* delete_ctx;
|
||||
bool early_delete;
|
||||
bool per_dir_delete;
|
||||
bool delete_limit_noted;
|
||||
@@ -347,6 +393,23 @@ static ReceiverStep receiver_handle_keepalive(ReceiverPendingState* state) {
|
||||
return RECEIVER_STEP_NEXT;
|
||||
}
|
||||
|
||||
/* rsync --stderr=client: a client diagnostic forwarded over the wire. Read the
|
||||
* bounded string and log it through the normal destination/level gate. The
|
||||
* body is peer-controlled text: log_client_message() escapes every
|
||||
* non-printable byte (newlines, CR, ANSI ESC, ...) before writing, so a hostile
|
||||
* client cannot forge log lines or inject terminal control sequences. A
|
||||
* malformed string (over-long or embedded NUL) is a framing error and tears the
|
||||
* connection down. */
|
||||
static ReceiverStep receiver_handle_client_msg(ReceiverPendingState* state) {
|
||||
char* message = receive_str(state->fd);
|
||||
if (!message)
|
||||
return RECEIVER_STEP_FAIL;
|
||||
if (message[0] != '\0')
|
||||
log_client_message(message);
|
||||
free(message);
|
||||
return RECEIVER_STEP_NEXT;
|
||||
}
|
||||
|
||||
static ReceiverStep receiver_handle_abort(ReceiverPendingState* state) {
|
||||
(void)state;
|
||||
log_message(LOG_LEVEL_INFO, "Received abort from client, cleaning up");
|
||||
@@ -383,9 +446,95 @@ static ReceiverStep receiver_handle_check_batch(ReceiverPendingState* state) {
|
||||
return RECEIVER_STEP_NEXT;
|
||||
}
|
||||
|
||||
/* Probe a destination entry's pre-transfer state for the output-parity
|
||||
dest-info report (protocol 2.30.0). `wire_path` is the destination-relative
|
||||
path; `incoming_target` is non-NULL only for a symlink probe, in which case
|
||||
the on-disk link target is compared with the target the receiver is about to
|
||||
store (after --munge-links). The final component is never followed and the
|
||||
parent walk is confined below the receive root. Returns false only on an
|
||||
allocation/secure-walk failure; a missing entry is reported as existed=false. */
|
||||
static bool receiver_probe_dest_state(const Config* config, const char* wire_path,
|
||||
const char* incoming_target, OutputDestState* out) {
|
||||
memset(out, 0, sizeof(*out));
|
||||
out->known = true;
|
||||
if (!config || !wire_path || wire_path[0] == '\0')
|
||||
return false;
|
||||
char* full = path_cat(config->receive_root_directory, wire_path);
|
||||
if (!full)
|
||||
return false;
|
||||
char* leaf = NULL;
|
||||
int parent_fd = file_open_secure_parent(full, &leaf, false);
|
||||
free(full);
|
||||
if (parent_fd < 0) {
|
||||
/* A missing/unreachable parent means the entry cannot exist yet. */
|
||||
free(leaf);
|
||||
return true;
|
||||
}
|
||||
struct stat st;
|
||||
if (fstatat(parent_fd, leaf, &st, AT_SYMLINK_NOFOLLOW) == 0) {
|
||||
out->existed = true;
|
||||
out->size = (unsigned long long)st.st_size;
|
||||
out->mtime_sec = (long long)st.st_mtime;
|
||||
#ifdef __linux__
|
||||
out->mtime_nsec = st.st_mtim.tv_nsec;
|
||||
#endif
|
||||
out->mode = (uint32_t)st.st_mode;
|
||||
out->uid = (int32_t)st.st_uid;
|
||||
out->gid = (int32_t)st.st_gid;
|
||||
if (incoming_target && S_ISLNK(st.st_mode)) {
|
||||
char target_buf[PATH_MAX];
|
||||
ssize_t n = readlinkat(parent_fd, leaf, target_buf, sizeof(target_buf) - 1);
|
||||
if (n >= 0) {
|
||||
target_buf[n] = '\0';
|
||||
char* expected =
|
||||
config->munge_links ? file_symlink_munge(incoming_target) : str_dup(incoming_target);
|
||||
if (expected) {
|
||||
out->target_matches = strcmp(target_buf, expected) == 0;
|
||||
free(expected);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
close(parent_fd);
|
||||
free(leaf);
|
||||
return true;
|
||||
}
|
||||
|
||||
static ReceiverStep receiver_handle_mkdir(ReceiverPendingState* state) {
|
||||
File* dir = file_receive_directory(state->fd, state->config);
|
||||
if (!dir || !state->sink->store_file(dir, state->sink->context))
|
||||
const Config* config = state->config;
|
||||
int fd = state->fd;
|
||||
if (config->report_dest_info) {
|
||||
int probe = 0;
|
||||
if (!receive_int(fd, &probe) || (probe != 0 && probe != 1))
|
||||
return RECEIVER_STEP_FAIL;
|
||||
if (probe) {
|
||||
/* Probe-only frame: report the destination state and create nothing. */
|
||||
char* path = receive_wire_str(fd);
|
||||
if (!path || path[0] == '\0' || (!file_get_trust_sender() && has_path_traversal(path))) {
|
||||
free(path);
|
||||
send_status(fd, STATUS_ERROR);
|
||||
return RECEIVER_STEP_FAIL;
|
||||
}
|
||||
OutputDestState info;
|
||||
bool ok = receiver_probe_dest_state(config, path, NULL, &info);
|
||||
free(path);
|
||||
if (!ok || !send_status(fd, STATUS_DEST_INFO) || !format_dest_state_send(fd, &info))
|
||||
return RECEIVER_STEP_FAIL;
|
||||
return RECEIVER_STEP_NEXT;
|
||||
}
|
||||
}
|
||||
File* dir = file_receive_directory(fd, config);
|
||||
if (!dir)
|
||||
return RECEIVER_STEP_ERROR;
|
||||
if (config->report_dest_info) {
|
||||
OutputDestState info;
|
||||
bool ok = receiver_probe_dest_state(config, file_wire_path(dir), NULL, &info);
|
||||
if (!ok || !send_status(fd, STATUS_DEST_INFO) || !format_dest_state_send(fd, &info)) {
|
||||
file_destroy(dir);
|
||||
return RECEIVER_STEP_FAIL;
|
||||
}
|
||||
}
|
||||
if (!state->sink->store_file(dir, state->sink->context))
|
||||
return RECEIVER_STEP_ERROR;
|
||||
return RECEIVER_STEP_NEXT;
|
||||
}
|
||||
@@ -404,8 +553,20 @@ static ReceiverStep receiver_handle_hardlink(ReceiverPendingState* state) {
|
||||
}
|
||||
|
||||
static ReceiverStep receiver_handle_symlink(ReceiverPendingState* state) {
|
||||
File* sym = file_receive_symlink(state->fd, state->config);
|
||||
if (!sym || !state->sink->store_file(sym, state->sink->context))
|
||||
const Config* config = state->config;
|
||||
File* sym = file_receive_symlink(state->fd, config);
|
||||
if (!sym)
|
||||
return RECEIVER_STEP_ERROR;
|
||||
if (config->report_dest_info) {
|
||||
OutputDestState info;
|
||||
bool ok = receiver_probe_dest_state(config, file_wire_path(sym), sym->symlink_target, &info);
|
||||
if (!ok || !send_status(state->fd, STATUS_DEST_INFO) ||
|
||||
!format_dest_state_send(state->fd, &info)) {
|
||||
file_destroy(sym);
|
||||
return RECEIVER_STEP_FAIL;
|
||||
}
|
||||
}
|
||||
if (!state->sink->store_file(sym, state->sink->context))
|
||||
return RECEIVER_STEP_ERROR;
|
||||
return RECEIVER_STEP_NEXT;
|
||||
}
|
||||
@@ -448,12 +609,11 @@ static ReceiverStep receiver_handle_manifest(ReceiverPendingState* state) {
|
||||
--max-delete-capped commit still succeeds and the transfer proceeds;
|
||||
the terminal success frame reports the cap. */
|
||||
size_t deleted = 0;
|
||||
DeletePathObserver observer =
|
||||
(config->report_deletes && sink->deleted_paths) ? receiver_record_deleted_path : NULL;
|
||||
ReceiverDeleteContext delctx;
|
||||
DeletePathObserver observer = receiver_delete_observer(sink, &delctx);
|
||||
DeleteCommitResult deletion =
|
||||
(config->use_delete || config->delete_missing_args)
|
||||
? manifest_delete_all_observed(config, manifest, &deleted, observer,
|
||||
(void*)sink->deleted_paths)
|
||||
? manifest_delete_all_observed(config, manifest, &deleted, observer, &delctx)
|
||||
: DELETE_COMMIT_OK;
|
||||
receiver_tally_deleted(sink, deleted);
|
||||
delete_manifest_free(manifest);
|
||||
@@ -496,9 +656,9 @@ static ReceiverStep receiver_handle_delete_plan(ReceiverPendingState* state) {
|
||||
}
|
||||
if (!state->plan_session) {
|
||||
state->plan_session = delete_plan_session_create(config);
|
||||
if (state->plan_session && config->report_deletes && sink->deleted_paths)
|
||||
if (state->plan_session && (sink->stats || sink->deleted_paths))
|
||||
delete_plan_session_set_delete_observer(state->plan_session, receiver_record_deleted_path,
|
||||
(void*)sink->deleted_paths);
|
||||
state->delete_ctx);
|
||||
}
|
||||
if (!state->plan_session || delete_plan_session_receive(state->plan_session, config, fd) != 0)
|
||||
return RECEIVER_STEP_FAIL;
|
||||
@@ -527,6 +687,8 @@ static ReceiverStep receiver_dispatch_status(ReceiverPendingState* state, Status
|
||||
switch (status) {
|
||||
case STATUS_KEEPALIVE:
|
||||
return receiver_handle_keepalive(state);
|
||||
case STATUS_CLIENT_MSG:
|
||||
return receiver_handle_client_msg(state);
|
||||
case STATUS_ABORT:
|
||||
return receiver_handle_abort(state);
|
||||
case STATUS_CHECK:
|
||||
@@ -579,8 +741,10 @@ static void receiver_drop_pending(ReceiverPendingState* state) {
|
||||
delete-during plan mode (no manifest at all). See the per-frame handlers
|
||||
above for how the -m receiver defers that commit until its disk writer has
|
||||
drained. */
|
||||
int receiver_process_pending(Config* config, int file_descriptor, const ReceiverSink* sink,
|
||||
DeleteManifest** pending_manifest, DeletePlanSession** pending_plans) {
|
||||
int receiver_process_pending_ctx(Config* config, int file_descriptor, const ReceiverSink* sink,
|
||||
DeleteManifest** pending_manifest,
|
||||
DeletePlanSession** pending_plans,
|
||||
ReceiverDeleteContext* observer_ctx) {
|
||||
Status status;
|
||||
if (!receive_status(file_descriptor, &status))
|
||||
return -1;
|
||||
@@ -593,6 +757,13 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver
|
||||
last_progress = session_start;
|
||||
if (!receiver_note_status(&session_start, &last_progress, status, file_descriptor, sink))
|
||||
return -1;
|
||||
/* A per-directory delete session handed to the caller outlives this stack
|
||||
frame, so its observer context must be caller-owned (observer_ctx); only
|
||||
the default inline-commit case may use the stack context. */
|
||||
ReceiverDeleteContext local_ctx;
|
||||
ReceiverDeleteContext* delete_ctx = observer_ctx ? observer_ctx : &local_ctx;
|
||||
delete_ctx->stats = sink ? sink->stats : NULL;
|
||||
delete_ctx->deleted_paths = sink ? sink->deleted_paths : NULL;
|
||||
ReceiverPendingState state = {
|
||||
.config = config,
|
||||
.fd = file_descriptor,
|
||||
@@ -601,6 +772,7 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver
|
||||
.pending_plans = pending_plans,
|
||||
.deferred_manifest = NULL,
|
||||
.plan_session = NULL,
|
||||
.delete_ctx = delete_ctx,
|
||||
.early_delete = config_delete_timing_early(config),
|
||||
.per_dir_delete = config_delete_timing_per_dir(config),
|
||||
.delete_limit_noted = false,
|
||||
@@ -610,7 +782,7 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver
|
||||
status == STATUS_KEEPALIVE || status == STATUS_ABORT || status == STATUS_CHECK_BATCH ||
|
||||
status == STATUS_MKDIR || status == STATUS_MANIFEST || status == STATUS_HARDLINK ||
|
||||
status == STATUS_SYMLINK || status == STATUS_SPECIAL || status == STATUS_DIR_TIMES ||
|
||||
status == STATUS_DELETE_PLAN) {
|
||||
status == STATUS_DELETE_PLAN || status == STATUS_CLIENT_MSG) {
|
||||
ReceiverStep step = receiver_dispatch_status(&state, status);
|
||||
if (step == RECEIVER_STEP_FAIL)
|
||||
goto fail;
|
||||
@@ -625,25 +797,36 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver
|
||||
log_message(LOG_LEVEL_ERROR, "Did not receive FINISHED Status");
|
||||
goto receive_error;
|
||||
}
|
||||
/* --delay-updates: publish every staged file BEFORE the deferred delete
|
||||
commit, matching rsync's --delete-after ordering (all updates land first,
|
||||
then extras are removed). The single-threaded receiver stores files
|
||||
synchronously, so every staged file is complete here. The -m receiver
|
||||
hands both publication and deletion to its caller via
|
||||
pending_manifest/pending_plans; that caller publishes first, after its disk
|
||||
writer has drained. */
|
||||
bool handoff = state.pending_manifest != NULL || state.pending_plans != NULL;
|
||||
if (!handoff && !config->dry_run && config->delay_updates && config->delay_context) {
|
||||
if (!delay_updates_publish(config->delay_context, config)) {
|
||||
send_status(file_descriptor, STATUS_ERROR);
|
||||
goto fail;
|
||||
}
|
||||
}
|
||||
/* Commit-style (late) deletion: every data frame has been received and the
|
||||
sender proved the whole tree with STATUS_FINISHED. The single-threaded
|
||||
receiver stores files synchronously, so everything is on disk here and the
|
||||
deletion can be committed before the --delay-updates publication in
|
||||
send_success (the walker skips the staging dir, so staged files are never
|
||||
treated as extras). The -m receiver passes `pending_manifest` because its
|
||||
disk writer may still be draining; the caller commits after the writer has
|
||||
joined so no extra file is removed unless the transfer is known to have
|
||||
succeeded. */
|
||||
receiver stores files synchronously, so everything is on disk here (and a
|
||||
--delay-updates run has already published above). The -m receiver passes
|
||||
`pending_manifest` because its disk writer may still be draining; the
|
||||
caller commits after the writer has joined so no extra file is removed
|
||||
unless the transfer is known to have succeeded. */
|
||||
if (state.deferred_manifest) {
|
||||
if (state.pending_manifest) {
|
||||
*state.pending_manifest = state.deferred_manifest;
|
||||
state.deferred_manifest = NULL;
|
||||
} else {
|
||||
size_t deleted = 0;
|
||||
DeletePathObserver observer =
|
||||
(config->report_deletes && sink->deleted_paths) ? receiver_record_deleted_path : NULL;
|
||||
DeletePathObserver observer = receiver_delete_observer(sink, state.delete_ctx);
|
||||
DeleteCommitResult deletion = manifest_delete_all_observed(
|
||||
config, state.deferred_manifest, &deleted, observer, (void*)sink->deleted_paths);
|
||||
config, state.deferred_manifest, &deleted, observer, state.delete_ctx);
|
||||
receiver_tally_deleted(sink, deleted);
|
||||
delete_manifest_free(state.deferred_manifest);
|
||||
state.deferred_manifest = NULL;
|
||||
@@ -661,9 +844,9 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver
|
||||
hands the session to its caller instead, which commits after the disk
|
||||
writer drained. */
|
||||
if (state.plan_session) {
|
||||
if (config->report_deletes && sink->deleted_paths)
|
||||
if (sink->stats || sink->deleted_paths)
|
||||
delete_plan_session_set_delete_observer(state.plan_session, receiver_record_deleted_path,
|
||||
(void*)sink->deleted_paths);
|
||||
state.delete_ctx);
|
||||
if (state.pending_plans) {
|
||||
*state.pending_plans = state.plan_session;
|
||||
state.plan_session = NULL;
|
||||
@@ -793,13 +976,14 @@ static void receiver_note_delete_limit(void* context_pointer) {
|
||||
|
||||
/* Terminal status for a run. A capped --delete limit wins (rsync exit 25);
|
||||
otherwise any per-entry failure (for example an unprivileged --devices
|
||||
mknod) makes the terminal frame non-OK so the client exits non-zero. rsync
|
||||
reports 23 here; mapping the client's exact exit code to 23 is a separate,
|
||||
pre-existing concern. A clean run keeps STATUS_OK. */
|
||||
mknod) makes the terminal frame STATUS_PARTIAL so the client exits 23
|
||||
(rsync's "partial transfer due to error") while still removing the sources
|
||||
it successfully transferred under --remove-source-files. A fatal stream
|
||||
error keeps STATUS_ERROR (a non-23 exit). A clean run keeps STATUS_OK. */
|
||||
static Status receiver_final_status(bool delete_limit_reached, size_t failed_entries) {
|
||||
if (delete_limit_reached)
|
||||
return STATUS_DELETE_LIMIT;
|
||||
return failed_entries > 0 ? STATUS_ERROR : STATUS_OK;
|
||||
return failed_entries > 0 ? STATUS_PARTIAL : STATUS_OK;
|
||||
}
|
||||
|
||||
static bool receiver_send_success_frame(int fd, void* context_pointer) {
|
||||
@@ -817,21 +1001,10 @@ static bool receiver_send_success_frame(int fd, void* context_pointer) {
|
||||
nothing to publish and no directory times to stamp. */
|
||||
if (context->config->dry_run)
|
||||
return receiver_send_final_success(fd, context->config, &context->outcomes, final_status);
|
||||
/* --delay-updates: the whole protocol stream (including manifest/delete
|
||||
handling, which ran inside receiver_process) has succeeded and every
|
||||
staged file was fully written. Publish them atomically now, before the
|
||||
success/outcome frame tells a --remove-source-files sender it may delete
|
||||
its sources. */
|
||||
if (context->config->delay_updates && context->config->delay_context) {
|
||||
if (!delay_updates_publish(context->config->delay_context, context->config)) {
|
||||
send_status(fd, STATUS_ERROR);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
/* P7 Wave D: every child is now written and the delete / --delay-updates
|
||||
phases have committed, so it is finally safe to stamp directory times.
|
||||
This runs after the deferred deletion because receiver_process commits it
|
||||
before calling this success frame. */
|
||||
/* P7 Wave D: every child is now written and the --delay-updates publication
|
||||
(done in receiver_process before the delete commit) plus the deferred
|
||||
deletion have both committed, so it is finally safe to stamp directory
|
||||
times. */
|
||||
dir_metadata_list_apply(&context->dir_times, context->config->receive_root_directory,
|
||||
context->config);
|
||||
return receiver_send_final_success(fd, context->config, &context->outcomes, final_status);
|
||||
|
||||
+25
-4
@@ -55,10 +55,20 @@ typedef struct {
|
||||
bool receiver_outcomes_append(ReceiverOutcomes* outcomes, unsigned char code);
|
||||
void receiver_outcomes_destroy(ReceiverOutcomes* outcomes);
|
||||
|
||||
/* DeletePathObserver implementation for --info=del: `context` is an ArrayList*
|
||||
that receives owned copies of every truly-removed destination-relative path.
|
||||
Shared by the single-threaded receiver and the -m pipeline's deferred commit. */
|
||||
void receiver_record_deleted_path(void* context, const char* rel_path);
|
||||
/* Delete observer context: `deleted_paths` (optional) receives owned copies of
|
||||
every truly-removed destination-relative path for --info=del; `stats`
|
||||
(optional) receives the per-type `Number of deleted files` tallies for
|
||||
--stats. Both may be NULL, in which case the observer is a no-op. */
|
||||
typedef struct {
|
||||
ReceiverStats* stats;
|
||||
struct ArrayList* deleted_paths;
|
||||
} ReceiverDeleteContext;
|
||||
|
||||
/* DeletePathObserver implementation: records each truly-removed path (when the
|
||||
context carries a path list) and tallies it by type (when it carries a stats
|
||||
record). Shared by the single-threaded receiver and the -m pipeline's
|
||||
deferred commit. */
|
||||
void receiver_record_deleted_path(void* context, const char* rel_path, DeleteEntryType type);
|
||||
|
||||
/* Send the terminal success frame. `final_status` is usually STATUS_OK, or
|
||||
STATUS_DELETE_LIMIT when a --max-delete commit was capped. */
|
||||
@@ -83,6 +93,17 @@ int receiver_process(Config* config, int file_descriptor, const ReceiverSink* si
|
||||
for either to keep the default behaviour (delete before the success frame). */
|
||||
int receiver_process_pending(Config* config, int file_descriptor, const ReceiverSink* sink,
|
||||
DeleteManifest** pending_manifest, DeletePlanSession** pending_plans);
|
||||
/* receiver_process_pending() with an explicit observer context for a
|
||||
per-directory delete session that is handed to the caller via
|
||||
`pending_plans`. The session outlives this call (the -m pipeline commits it
|
||||
after joining its disk writer), so its observer context must too: pass a
|
||||
long-lived object such as PipelineContextReceiver.delete_ctx. When
|
||||
`delete_ctx` is NULL an internal stack context is used, which is only safe
|
||||
when the session is committed before returning (the default behaviour). */
|
||||
int receiver_process_pending_ctx(Config* config, int file_descriptor, const ReceiverSink* sink,
|
||||
DeleteManifest** pending_manifest,
|
||||
DeletePlanSession** pending_plans,
|
||||
ReceiverDeleteContext* delete_ctx);
|
||||
int receiver_receive_files(Config* config, int file_descriptor);
|
||||
|
||||
/* ---- Connection time bounds (anti-slowloris) ----
|
||||
|
||||
@@ -28,6 +28,8 @@ PipelineContextReceiver* pipeline_context_receiver_create(Config* config, Queue*
|
||||
context->max_queue_bytes = 0;
|
||||
context->deferred_manifest = NULL;
|
||||
context->deferred_plans = NULL;
|
||||
context->delete_ctx.stats = NULL;
|
||||
context->delete_ctx.deleted_paths = NULL;
|
||||
context->delete_limit_reached = false;
|
||||
context->failed_entries = 0;
|
||||
memset(&context->stats, 0, sizeof(context->stats));
|
||||
@@ -205,8 +207,9 @@ int receive_thread(void* pipeline_context) {
|
||||
&context->stats,
|
||||
context->would_delete,
|
||||
context->deleted_paths};
|
||||
if (receiver_process_pending((Config*)config, file_descriptor, &sink, &context->deferred_manifest,
|
||||
&context->deferred_plans) != 0) {
|
||||
if (receiver_process_pending_ctx((Config*)config, file_descriptor, &sink,
|
||||
&context->deferred_manifest, &context->deferred_plans,
|
||||
&context->delete_ctx) != 0) {
|
||||
receiver_thread_fail(context);
|
||||
protocol_session_unbind();
|
||||
return thrd_error;
|
||||
|
||||
@@ -46,6 +46,11 @@ typedef struct PipelineContextReceiver {
|
||||
committing while the disk writer may still be draining; server.c commits it
|
||||
after both threads joined. NULL for every other timing. */
|
||||
DeletePlanSession* deferred_plans;
|
||||
/* Observer context for `deferred_plans`. It must outlive the receive thread
|
||||
(the session is committed by server.c after both threads join), so it lives
|
||||
here rather than on receiver_process_pending()'s stack; receive_thread
|
||||
installs it on the session. */
|
||||
ReceiverDeleteContext delete_ctx;
|
||||
/* Set by server.c when the deferred delete commit hit the --max-delete
|
||||
budget; the terminal success frame then carries STATUS_DELETE_LIMIT
|
||||
(rsync exit 25) while the transfer itself still succeeds. */
|
||||
|
||||
+192
-151
@@ -981,18 +981,31 @@ static void server_run_mt_receiver(ServerSession* state) {
|
||||
thrd_join(writer, &writer_result);
|
||||
bool transfer_ok = receiver_result == thrd_success && writer_result == thrd_success;
|
||||
PipelineContextReceiver* context = state->context;
|
||||
if (transfer_ok && !config->dry_run) {
|
||||
/* --delay-updates: receive_thread has finished the whole protocol stream
|
||||
and write_thread has drained its queue, so every staged file is complete.
|
||||
Publish atomically BEFORE the deferred delete commit, matching rsync's
|
||||
--delete-after ordering (all updates land first, then extras are
|
||||
removed). */
|
||||
if (config->delay_updates && config->delay_context &&
|
||||
!delay_updates_publish(config->delay_context, config)) {
|
||||
transfer_ok = false;
|
||||
}
|
||||
}
|
||||
if (transfer_ok && !config->dry_run) {
|
||||
/* Commit-style (late) deletion: receive_thread handed the keep-set
|
||||
manifest here instead of deleting while write_thread might still be
|
||||
draining, so by now every file is on disk and the whole transfer is
|
||||
known to have succeeded. Remove the extras before publishing a
|
||||
--delay-updates run; the walker skips the staging directory. A
|
||||
draining, so by now every file is on disk (and a --delay-updates run has
|
||||
already published above) and the whole transfer is known to have
|
||||
succeeded. The walker skips the staging directory. A
|
||||
server-contacting --dry-run deletes nothing (no manifest is sent). */
|
||||
if (context->deferred_manifest) {
|
||||
size_t deleted = 0;
|
||||
DeletePathObserver observer = config->report_deletes ? receiver_record_deleted_path : NULL;
|
||||
DeleteCommitResult deletion = manifest_delete_all_observed(
|
||||
config, context->deferred_manifest, &deleted, observer, (void*)context->deleted_paths);
|
||||
ReceiverDeleteContext delctx = {&context->stats, context->deleted_paths};
|
||||
DeletePathObserver observer =
|
||||
(delctx.stats || delctx.deleted_paths) ? receiver_record_deleted_path : NULL;
|
||||
DeleteCommitResult deletion = manifest_delete_all_observed(config, context->deferred_manifest,
|
||||
&deleted, observer, &delctx);
|
||||
context->stats.deleted_files += deleted;
|
||||
if (deletion == DELETE_COMMIT_ERROR) {
|
||||
transfer_ok = false;
|
||||
@@ -1009,10 +1022,9 @@ static void server_run_mt_receiver(ServerSession* state) {
|
||||
--delete-during already applied its plans on the receive thread. */
|
||||
if (context->deferred_plans) {
|
||||
/* Defence in depth (the enclosing block already excludes dry-run): a
|
||||
-n run never commits a deletion. */
|
||||
if (config->report_deletes)
|
||||
delete_plan_session_set_delete_observer(
|
||||
context->deferred_plans, receiver_record_deleted_path, (void*)context->deleted_paths);
|
||||
-n run never commits a deletion. The session's observer context was
|
||||
installed by receive_thread from context->delete_ctx, which outlives
|
||||
both threads, so no stack context is needed here. */
|
||||
DeleteCommitResult deletion =
|
||||
config->dry_run ? DELETE_COMMIT_OK
|
||||
: delete_plan_session_commit(context->deferred_plans, config);
|
||||
@@ -1025,21 +1037,10 @@ static void server_run_mt_receiver(ServerSession* state) {
|
||||
delete_plan_session_destroy(context->deferred_plans);
|
||||
context->deferred_plans = NULL;
|
||||
}
|
||||
}
|
||||
if (transfer_ok && !config->dry_run) {
|
||||
/* --delay-updates: receive_thread has finished the whole protocol stream
|
||||
(including manifest/delete handling) and write_thread has drained its
|
||||
queue, so every staged file is complete. Publish atomically before the
|
||||
success/outcome frame so a --remove-source-files sender only learns of
|
||||
files that were actually installed. */
|
||||
if (config->delay_updates && config->delay_context &&
|
||||
!delay_updates_publish(config->delay_context, config)) {
|
||||
transfer_ok = false;
|
||||
}
|
||||
/* P7 Wave D: all writers have joined and the late deletion (and
|
||||
--delay-updates publication) has committed above, so it is finally safe
|
||||
to stamp directory times; a directory's mtime must not be clobbered by
|
||||
its children or by an extra removal. */
|
||||
/* P7 Wave D: all writers have joined and the --delay-updates publication
|
||||
plus the late deletion have committed above, so it is finally safe to
|
||||
stamp directory times; a directory's mtime must not be clobbered by its
|
||||
children or by an extra removal. */
|
||||
if (transfer_ok)
|
||||
dir_metadata_list_apply(&context->dir_times, config->receive_root_directory, config);
|
||||
}
|
||||
@@ -1050,7 +1051,7 @@ static void server_run_mt_receiver(ServerSession* state) {
|
||||
context->failed_entries, context->failed_entries == 1 ? "y" : "ies");
|
||||
Status final_status = context->delete_limit_reached
|
||||
? STATUS_DELETE_LIMIT
|
||||
: (context->failed_entries > 0 ? STATUS_ERROR : STATUS_OK);
|
||||
: (context->failed_entries > 0 ? STATUS_PARTIAL : STATUS_OK);
|
||||
/* Emit the optional wire-stats record first (protocol 2.25.0), then the
|
||||
success/outcome frame, exactly like the single-threaded receiver. */
|
||||
if (!receiver_send_stats_frame(state->fd, config, &context->stats, context->would_delete,
|
||||
@@ -1305,29 +1306,45 @@ static bool daemonize(void) {
|
||||
return true;
|
||||
}
|
||||
|
||||
int main(int argc, char* argv[]) {
|
||||
/* Capture the process umask now, while still single-threaded: the cached
|
||||
* value is what file_mode_base() uses, and reading it later would race with
|
||||
* receiver threads creating files. */
|
||||
file_umask_capture();
|
||||
ServerCliOptions opts;
|
||||
char cli_err[512];
|
||||
int parse_result = server_cli_parse(argc, argv, &opts, cli_err, sizeof(cli_err));
|
||||
if (parse_result == 1) {
|
||||
print_server_usage();
|
||||
return 0;
|
||||
/* Apply the process-wide policies shared by the stdio and listener
|
||||
* entrypoints: logging verbosity, signal handling, the parsed server
|
||||
* authorization policies, and the socket timeout floor. Runs after CLI
|
||||
* parsing and after the standalone --hash-credentials tool has been ruled
|
||||
* out. */
|
||||
static void configure_server_process(const ServerCliOptions* opts) {
|
||||
signal(SIGPIPE, SIG_IGN);
|
||||
if (opts->verbose) {
|
||||
set_log_level(LOG_LEVEL_DEBUG);
|
||||
set_log_debug_flags(LOG_DEBUG_ALL);
|
||||
}
|
||||
if (parse_result < 0) {
|
||||
server_cli_options_free(&opts);
|
||||
fprintf(stderr, "Error: %s\n", cli_err);
|
||||
print_server_usage();
|
||||
return 1;
|
||||
if (opts->tls_ca && !opts->use_tls)
|
||||
log_message(LOG_LEVEL_WARNING, "--ca has no effect without --tls");
|
||||
/* Persist the parsed server policies into the process-global policy state
|
||||
* BEFORE the stdio branch: an SSH-launched `--stdio` server (whose argv came
|
||||
* from the client via --remote-option and friends) must honor --allow-delete,
|
||||
* --trust-sender and --client-cn exactly like the standalone listener. */
|
||||
required_client_cn = opts->client_cn;
|
||||
allow_delete = opts->allow_delete;
|
||||
trust_sender = opts->trust_sender;
|
||||
allow_unauthenticated = opts->allow_unauthenticated;
|
||||
server_no_super = opts->no_super;
|
||||
/* --stdio rejects --allow-super at parse time; force it off here as well so
|
||||
* this process-global policy cannot be re-enabled by a future caller. */
|
||||
server_allow_super = opts->allow_super && !opts->stdio_mode;
|
||||
server_iconv_spec = opts->iconv_spec;
|
||||
install_cleanup_handler(SIGINT);
|
||||
install_cleanup_handler(SIGTERM);
|
||||
/* Server-owned socket deadline floor: the client default --timeout=0 would
|
||||
* otherwise leave accepted sockets without SO_RCVTIMEO/SO_SNDTIMEO and let a
|
||||
* silent peer hold a connection (and its process slot) forever. */
|
||||
tcp_set_timeouts(SERVER_IO_TIMEOUT_SEC, SERVER_IO_TIMEOUT_SEC);
|
||||
}
|
||||
|
||||
/* --hash-credentials: standalone offline tool; read user:password lines and
|
||||
* emit new-format credential-store lines, then exit. */
|
||||
if (opts.hash_credentials_file) {
|
||||
uint32_t iters = opts.hash_iterations_set ? opts.hash_iterations : CREDENTIAL_DEFAULT_ITERS;
|
||||
* emit new-format credential-store lines, then exit. Consumes and frees
|
||||
* opts. */
|
||||
static int run_hash_credentials_tool(ServerCliOptions* opts) {
|
||||
uint32_t iters = opts->hash_iterations_set ? opts->hash_iterations : CREDENTIAL_DEFAULT_ITERS;
|
||||
/* The output is secret material: if it is redirected to a regular file,
|
||||
* warn when that file is group/other-accessible (the store must be 0600). */
|
||||
struct stat out_st;
|
||||
@@ -1337,53 +1354,29 @@ int main(int argc, char* argv[]) {
|
||||
"Warning: credential-store output is a group/other-accessible file; restrict it to "
|
||||
"mode 0600 (chmod 600)\n");
|
||||
char hash_err[512];
|
||||
if (credentials_hash_file(opts.hash_credentials_file, iters, stdout, hash_err,
|
||||
if (credentials_hash_file(opts->hash_credentials_file, iters, stdout, hash_err,
|
||||
sizeof(hash_err)) != 0) {
|
||||
fprintf(stderr, "Error: %s\n", hash_err);
|
||||
server_cli_options_free(&opts);
|
||||
server_cli_options_free(opts);
|
||||
return 1;
|
||||
}
|
||||
server_cli_options_free(&opts);
|
||||
server_cli_options_free(opts);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int exit_code = 0;
|
||||
signal(SIGPIPE, SIG_IGN);
|
||||
if (opts.verbose) {
|
||||
set_log_level(LOG_LEVEL_DEBUG);
|
||||
set_log_debug_flags(LOG_DEBUG_ALL);
|
||||
}
|
||||
if (opts.tls_ca && !opts.use_tls)
|
||||
log_message(LOG_LEVEL_WARNING, "--ca has no effect without --tls");
|
||||
/* Persist the parsed server policies into the process-global policy state
|
||||
* BEFORE the stdio branch: an SSH-launched `--stdio` server (whose argv came
|
||||
* from the client via --remote-option and friends) must honor --allow-delete,
|
||||
* --trust-sender and --client-cn exactly like the standalone listener. */
|
||||
required_client_cn = opts.client_cn;
|
||||
allow_delete = opts.allow_delete;
|
||||
trust_sender = opts.trust_sender;
|
||||
allow_unauthenticated = opts.allow_unauthenticated;
|
||||
server_no_super = opts.no_super;
|
||||
/* --stdio rejects --allow-super at parse time; force it off here as well so
|
||||
* this process-global policy cannot be re-enabled by a future caller. */
|
||||
server_allow_super = opts.allow_super && !opts.stdio_mode;
|
||||
server_iconv_spec = opts.iconv_spec;
|
||||
install_cleanup_handler(SIGINT);
|
||||
install_cleanup_handler(SIGTERM);
|
||||
/* Server-owned socket deadline floor: the client default --timeout=0 would
|
||||
* otherwise leave accepted sockets without SO_RCVTIMEO/SO_SNDTIMEO and let a
|
||||
* silent peer hold a connection (and its process slot) forever. */
|
||||
tcp_set_timeouts(SERVER_IO_TIMEOUT_SEC, SERVER_IO_TIMEOUT_SEC);
|
||||
|
||||
if (opts.stdio_mode) {
|
||||
/* SSH --stdio session: the transport is authenticated by sshd outside of
|
||||
* FastSync, so the single connection is served over STDIN/STDOUT and the
|
||||
* process exits. The destination root is authorized exactly like the listener
|
||||
* path. Consumes and frees opts. */
|
||||
static int run_stdio_server(ServerCliOptions* opts) {
|
||||
/* SSH authenticates the stdio transport outside of FastSync. */
|
||||
allow_unauthenticated = true;
|
||||
if (!configure_authorization(opts.destination_root)) {
|
||||
char* escaped = output_escape(opts.destination_root, false);
|
||||
if (!configure_authorization(opts->destination_root)) {
|
||||
char* escaped = output_escape(opts->destination_root, false);
|
||||
fprintf(stderr, "Error: invalid destination root '%s'\n",
|
||||
escaped ? escaped : "<allocation failed>");
|
||||
free(escaped);
|
||||
server_cli_options_free(&opts);
|
||||
server_cli_options_free(opts);
|
||||
return 1;
|
||||
}
|
||||
io_set_fds(STDIN_FILENO, STDOUT_FILENO);
|
||||
@@ -1392,34 +1385,32 @@ int main(int argc, char* argv[]) {
|
||||
* and are released by process exit. */
|
||||
handler(STDIN_FILENO);
|
||||
release_authorization();
|
||||
server_cli_options_free(&opts);
|
||||
server_cli_options_free(opts);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int port = opts.port;
|
||||
int bind_family = opts.bind_family;
|
||||
const char* bind_address = opts.bind_address;
|
||||
|
||||
if (opts.daemon_mode) {
|
||||
const char* config_path = opts.config_path ? opts.config_path : default_daemon_config_path();
|
||||
g_daemon_conf = daemon_conf_load(config_path, cli_err, sizeof(cli_err));
|
||||
/* Load the daemon config, apply --dparam overrides, resolve the effective
|
||||
* port/address, and surface the operator-facing module warnings. On failure
|
||||
* the error is printed and false is returned. */
|
||||
static bool load_daemon_policy(ServerCliOptions* opts, int* port, const char** bind_address,
|
||||
char* err, size_t err_size) {
|
||||
const char* config_path = opts->config_path ? opts->config_path : default_daemon_config_path();
|
||||
g_daemon_conf = daemon_conf_load(config_path, err, err_size);
|
||||
if (!g_daemon_conf) {
|
||||
server_cli_options_free(&opts);
|
||||
fprintf(stderr, "Error: %s\n", cli_err);
|
||||
return 1;
|
||||
fprintf(stderr, "Error: %s\n", err);
|
||||
return false;
|
||||
}
|
||||
for (int i = 0; i < opts.dparam_count; i++) {
|
||||
if (daemon_conf_apply_dparam(g_daemon_conf, opts.dparams[i], cli_err, sizeof(cli_err)) != 0) {
|
||||
fprintf(stderr, "Error: --dparam: %s\n", cli_err);
|
||||
exit_code = 1;
|
||||
goto out;
|
||||
for (int i = 0; i < opts->dparam_count; i++) {
|
||||
if (daemon_conf_apply_dparam(g_daemon_conf, opts->dparams[i], err, err_size) != 0) {
|
||||
fprintf(stderr, "Error: --dparam: %s\n", err);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
/* Effective port: -p (highest) > --dparam port > config port (default 873). */
|
||||
if (!opts.port_set)
|
||||
port = g_daemon_conf->global.port;
|
||||
if (!bind_address)
|
||||
bind_address = g_daemon_conf->global.address;
|
||||
if (!opts->port_set)
|
||||
*port = g_daemon_conf->global.port;
|
||||
if (!*bind_address)
|
||||
*bind_address = g_daemon_conf->global.address;
|
||||
if (g_daemon_conf->module_count == 0)
|
||||
log_message(LOG_LEVEL_WARNING,
|
||||
"daemon config has no modules; every connection will be refused");
|
||||
@@ -1440,20 +1431,23 @@ int main(int argc, char* argv[]) {
|
||||
"across all connection children)",
|
||||
g_daemon_conf->modules[i].name, g_daemon_conf->modules[i].max_connections);
|
||||
}
|
||||
/* Daemon credential store (Wave B). --password-file and --early-input
|
||||
* feed the same store, loaded BEFORE the listener forks so every
|
||||
* connection child shares one read-only store. Fail closed at startup: a
|
||||
* module that declares `auth users` without a store (or with an empty
|
||||
* store) refuses to start rather than serving a module whose credentials
|
||||
* can never be verified. */
|
||||
g_credentials =
|
||||
credentials_load(opts.password_file, opts.early_input_file, cli_err, sizeof(cli_err));
|
||||
if (!g_credentials) {
|
||||
server_cli_options_free(&opts);
|
||||
fprintf(stderr, "Error: %s\n", cli_err);
|
||||
return 1;
|
||||
return true;
|
||||
}
|
||||
bool credential_source_given = opts.password_file != NULL || opts.early_input_file != NULL;
|
||||
|
||||
/* Load the daemon credential store (Wave B) and enforce the fail-closed
|
||||
* startup check: a module that declares `auth users` without a store (or with
|
||||
* an empty store) refuses to start rather than serving a module whose
|
||||
* credentials can never be verified. On failure the error is printed and
|
||||
* false is returned. */
|
||||
static bool validate_daemon_credentials(const ServerCliOptions* opts, char* err, size_t err_size) {
|
||||
/* --password-file and --early-input feed the same store, loaded BEFORE the
|
||||
* listener forks so every connection child shares one read-only store. */
|
||||
g_credentials = credentials_load(opts->password_file, opts->early_input_file, err, err_size);
|
||||
if (!g_credentials) {
|
||||
fprintf(stderr, "Error: %s\n", err);
|
||||
return false;
|
||||
}
|
||||
bool credential_source_given = opts->password_file != NULL || opts->early_input_file != NULL;
|
||||
for (int i = 0; i < g_daemon_conf->module_count; i++) {
|
||||
const DaemonModule* module = &g_daemon_conf->modules[i];
|
||||
if (module->auth_user_count == 0)
|
||||
@@ -1463,16 +1457,14 @@ int main(int argc, char* argv[]) {
|
||||
"Error: module '%s' declares 'auth users' but no credential store was given "
|
||||
"(--password-file or --early-input); refusing to start (fail closed)\n",
|
||||
module->name);
|
||||
server_cli_options_free(&opts);
|
||||
return 1;
|
||||
return false;
|
||||
}
|
||||
if (credentials_store_size(g_credentials) == 0) {
|
||||
fprintf(stderr,
|
||||
"Error: module '%s' declares 'auth users' but the credential store is empty; "
|
||||
"refusing to start (fail closed)\n",
|
||||
module->name);
|
||||
server_cli_options_free(&opts);
|
||||
return 1;
|
||||
return false;
|
||||
}
|
||||
for (int j = 0; j < module->auth_user_count; j++) {
|
||||
if (!credentials_store_has(g_credentials, module->auth_users[j]))
|
||||
@@ -1482,32 +1474,30 @@ int main(int argc, char* argv[]) {
|
||||
module->name, module->auth_users[j]);
|
||||
}
|
||||
}
|
||||
/* Shared cross-process registry for the per-module / per-source caps and
|
||||
* the auth lockout. Created HERE in the parent before any accept-loop
|
||||
* fork; every connection child inherits the mapping. A failure degrades to
|
||||
* "registry disabled" (the global cap and host ACLs still apply) rather
|
||||
* than refusing to start. */
|
||||
g_daemon_limits = daemon_limits_create((int)g_daemon_conf->global.max_connections,
|
||||
g_daemon_conf->module_count,
|
||||
g_daemon_conf->global.max_connections_per_host,
|
||||
g_daemon_conf->global.auth_lockout_threshold,
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Create the shared cross-process registry for the per-module / per-source
|
||||
* caps and the auth lockout. Called in the parent before any accept-loop fork;
|
||||
* every connection child inherits the mapping. A failure degrades to
|
||||
* "registry disabled" (the global cap and host ACLs still apply) rather than
|
||||
* refusing to start. */
|
||||
static void create_daemon_limits(void) {
|
||||
g_daemon_limits = daemon_limits_create(
|
||||
(int)g_daemon_conf->global.max_connections, g_daemon_conf->module_count,
|
||||
g_daemon_conf->global.max_connections_per_host, g_daemon_conf->global.auth_lockout_threshold,
|
||||
g_daemon_conf->global.auth_lockout_duration_sec);
|
||||
if (!g_daemon_limits)
|
||||
log_message(LOG_LEVEL_WARNING,
|
||||
"daemon: could not allocate the shared connection registry; per-module / "
|
||||
"per-host caps and the cross-process auth lockout are disabled (the global "
|
||||
"'max connections' cap and host ACLs still apply)");
|
||||
} else {
|
||||
if (!configure_authorization(opts.destination_root)) {
|
||||
char* escaped = output_escape(opts.destination_root, false);
|
||||
fprintf(stderr, "Error: invalid destination root '%s'\n",
|
||||
escaped ? escaped : "<allocation failed>");
|
||||
free(escaped);
|
||||
server_cli_options_free(&opts);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
/* Bind the listener, apply the daemon caps, set up TLS when requested, detach
|
||||
* when daemonizing, and run the accept loop. Returns the process exit code. */
|
||||
static int start_listener(ServerCliOptions* opts, int port, int bind_family,
|
||||
const char* bind_address) {
|
||||
ServerBindOptions bind_opts;
|
||||
bind_opts.bind_address = bind_address;
|
||||
bind_opts.family = bind_family;
|
||||
@@ -1515,50 +1505,73 @@ int main(int argc, char* argv[]) {
|
||||
if (!g_server) {
|
||||
log_message(LOG_LEVEL_ERROR, "Failed to create server");
|
||||
release_authorization();
|
||||
exit_code = 1;
|
||||
goto out;
|
||||
return 1;
|
||||
}
|
||||
if (g_daemon_conf)
|
||||
server_set_max_connections(g_server, (unsigned int)g_daemon_conf->global.max_connections);
|
||||
if (g_daemon_limits)
|
||||
server_set_limit_registry(g_server, g_daemon_limits);
|
||||
if (opts.use_tls) {
|
||||
if (!opts.tls_cert || !opts.tls_key || !opts.tls_ca || !opts.client_cn) {
|
||||
if (opts->use_tls) {
|
||||
if (!opts->tls_cert || !opts->tls_key || !opts->tls_ca || !opts->client_cn) {
|
||||
fprintf(stderr, "Error: --tls requires --cert, --key, --ca, and --client-cn\n");
|
||||
server_delete(&g_server);
|
||||
release_authorization();
|
||||
exit_code = 1;
|
||||
goto out;
|
||||
return 1;
|
||||
}
|
||||
tls_global_init();
|
||||
if (!server_create_tls(g_server, opts.tls_cert, opts.tls_key, opts.tls_ca)) {
|
||||
if (!server_create_tls(g_server, opts->tls_cert, opts->tls_key, opts->tls_ca)) {
|
||||
log_message(LOG_LEVEL_ERROR, "Failed to set up TLS");
|
||||
server_delete(&g_server);
|
||||
release_authorization();
|
||||
exit_code = 1;
|
||||
goto out;
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
/* Detach after the listening socket (and TLS context) exist so the
|
||||
* background daemon inherits a fully-bound listener. --no-detach runs in
|
||||
* the foreground, which is how tests drive the daemon. */
|
||||
if (opts.daemon_mode && !opts.no_detach) {
|
||||
if (opts->daemon_mode && !opts->no_detach) {
|
||||
if (!daemonize()) {
|
||||
log_message(LOG_LEVEL_ERROR, "Failed to daemonize");
|
||||
server_delete(&g_server);
|
||||
release_authorization();
|
||||
exit_code = 1;
|
||||
goto out;
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (opts.use_tls)
|
||||
if (opts->use_tls)
|
||||
server_listen_tls(g_server, handler);
|
||||
else
|
||||
server_listen(g_server, handler);
|
||||
server_delete(&g_server);
|
||||
release_authorization();
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Listener entrypoint: the daemon (config-driven, possibly detached) and the
|
||||
* standalone TCP server share the same bind/TLS/listen path. Consumes and
|
||||
* frees opts. */
|
||||
static int run_daemon_server(ServerCliOptions* opts) {
|
||||
int port = opts->port;
|
||||
const char* bind_address = opts->bind_address;
|
||||
char cli_err[512];
|
||||
int exit_code = 0;
|
||||
|
||||
if (opts->daemon_mode) {
|
||||
if (!load_daemon_policy(opts, &port, &bind_address, cli_err, sizeof(cli_err)) ||
|
||||
!validate_daemon_credentials(opts, cli_err, sizeof(cli_err))) {
|
||||
exit_code = 1;
|
||||
goto out;
|
||||
}
|
||||
create_daemon_limits();
|
||||
} else if (!configure_authorization(opts->destination_root)) {
|
||||
char* escaped = output_escape(opts->destination_root, false);
|
||||
fprintf(stderr, "Error: invalid destination root '%s'\n",
|
||||
escaped ? escaped : "<allocation failed>");
|
||||
free(escaped);
|
||||
exit_code = 1;
|
||||
goto out;
|
||||
}
|
||||
|
||||
exit_code = start_listener(opts, port, opts->bind_family, bind_address);
|
||||
|
||||
out:
|
||||
daemon_limits_destroy(g_daemon_limits);
|
||||
@@ -1567,7 +1580,35 @@ out:
|
||||
g_daemon_conf = NULL;
|
||||
credentials_free(g_credentials);
|
||||
g_credentials = NULL;
|
||||
server_cli_options_free(&opts);
|
||||
server_cli_options_free(opts);
|
||||
return exit_code;
|
||||
}
|
||||
|
||||
int main(int argc, char* argv[]) {
|
||||
/* Capture the process umask now, while still single-threaded: the cached
|
||||
* value is what file_mode_base() uses, and reading it later would race with
|
||||
* receiver threads creating files. */
|
||||
file_umask_capture();
|
||||
ServerCliOptions opts;
|
||||
char cli_err[512];
|
||||
int parse_result = server_cli_parse(argc, argv, &opts, cli_err, sizeof(cli_err));
|
||||
if (parse_result == 1) {
|
||||
print_server_usage();
|
||||
return 0;
|
||||
}
|
||||
if (parse_result < 0) {
|
||||
server_cli_options_free(&opts);
|
||||
fprintf(stderr, "Error: %s\n", cli_err);
|
||||
print_server_usage();
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (opts.hash_credentials_file)
|
||||
return run_hash_credentials_tool(&opts);
|
||||
|
||||
configure_server_process(&opts);
|
||||
if (opts.stdio_mode)
|
||||
return run_stdio_server(&opts);
|
||||
return run_daemon_server(&opts);
|
||||
}
|
||||
#endif
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
#include "log.h"
|
||||
#include "protocol.h"
|
||||
#include "utils.h"
|
||||
#include <errno.h>
|
||||
#include <limits.h>
|
||||
#include <lz4.h>
|
||||
#include <stdatomic.h>
|
||||
@@ -716,3 +717,379 @@ Data* data_decompress_limited(Data* compressed_data, size_t maximum_size) {
|
||||
Data* data_decompress(Data* compressed_data) {
|
||||
return data_decompress_limited(compressed_data, MAX_DECOMPRESSED_SIZE);
|
||||
}
|
||||
|
||||
/* ---- streaming decompression ---- */
|
||||
|
||||
#define STREAM_DECOMPRESS_OUT_CHUNK (256 * 1024)
|
||||
|
||||
struct CompressionStreamDecompressor {
|
||||
CompressionAlgo algo;
|
||||
unsigned long long expected_out;
|
||||
unsigned long long total;
|
||||
int out_fd;
|
||||
unsigned char* out_buf;
|
||||
ZSTD_DCtx* dctx;
|
||||
z_stream zs;
|
||||
bool zs_initialized;
|
||||
bool failed;
|
||||
};
|
||||
|
||||
static bool stream_write_all(int fd, const void* data, size_t size) {
|
||||
const unsigned char* p = data;
|
||||
size_t done = 0;
|
||||
while (done < size) {
|
||||
ssize_t n = write(fd, p + done, size - done);
|
||||
if (n < 0 && errno == EINTR)
|
||||
continue;
|
||||
if (n <= 0)
|
||||
return false;
|
||||
done += (size_t)n;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
CompressionStreamDecompressor*
|
||||
compression_stream_decompressor_create(CompressionAlgo algo, unsigned long long expected_out) {
|
||||
CompressionStreamDecompressor* d = calloc(1, sizeof(*d));
|
||||
if (!d)
|
||||
return NULL;
|
||||
d->algo = algo;
|
||||
d->expected_out = expected_out;
|
||||
d->out_fd = -1;
|
||||
d->out_buf = malloc(STREAM_DECOMPRESS_OUT_CHUNK);
|
||||
if (!d->out_buf) {
|
||||
free(d);
|
||||
return NULL;
|
||||
}
|
||||
if (algo == COMPRESSION_ALGO_ZSTD) {
|
||||
d->dctx = ZSTD_createDCtx();
|
||||
if (!d->dctx) {
|
||||
free(d->out_buf);
|
||||
free(d);
|
||||
return NULL;
|
||||
}
|
||||
} else if (algo == COMPRESSION_ALGO_ZLIB || algo == COMPRESSION_ALGO_ZLIBX) {
|
||||
if (inflateInit(&d->zs) != Z_OK) {
|
||||
free(d->out_buf);
|
||||
free(d);
|
||||
return NULL;
|
||||
}
|
||||
d->zs_initialized = true;
|
||||
} else if (algo != COMPRESSION_ALGO_NONE) {
|
||||
/* lz4's block format cannot be decompressed incrementally. */
|
||||
free(d->out_buf);
|
||||
free(d);
|
||||
return NULL;
|
||||
}
|
||||
return d;
|
||||
}
|
||||
|
||||
static bool stream_emit(CompressionStreamDecompressor* d, const void* buf, size_t len) {
|
||||
if (len == 0)
|
||||
return true;
|
||||
if (d->expected_out != 0 && (d->total > d->expected_out || len > d->expected_out - d->total)) {
|
||||
d->failed = true;
|
||||
return false;
|
||||
}
|
||||
if (!stream_write_all(d->out_fd, buf, len)) {
|
||||
d->failed = true;
|
||||
return false;
|
||||
}
|
||||
d->total += len;
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool stream_feed_none(CompressionStreamDecompressor* d, const void* in, size_t in_len,
|
||||
bool* done) {
|
||||
if (!stream_emit(d, in, in_len))
|
||||
return false;
|
||||
/* NONE has no end marker; the caller knows the frame length. */
|
||||
*done = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool stream_feed_zstd(CompressionStreamDecompressor* d, const void* in, size_t in_len,
|
||||
bool* done) {
|
||||
ZSTD_inBuffer input = {in, in_len, 0};
|
||||
while (input.pos < input.size) {
|
||||
ZSTD_outBuffer output = {d->out_buf, STREAM_DECOMPRESS_OUT_CHUNK, 0};
|
||||
size_t ret = ZSTD_decompressStream(d->dctx, &output, &input);
|
||||
if (ZSTD_isError(ret)) {
|
||||
d->failed = true;
|
||||
return false;
|
||||
}
|
||||
if (!stream_emit(d, d->out_buf, output.pos))
|
||||
return false;
|
||||
if (ret == 0) {
|
||||
*done = true;
|
||||
/* Trailing bytes after a complete frame are malformed; stop consuming. */
|
||||
if (input.pos < input.size) {
|
||||
d->failed = true;
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool stream_feed_zlib(CompressionStreamDecompressor* d, const void* in, size_t in_len,
|
||||
bool* done) {
|
||||
d->zs.next_in = (Bytef*)in;
|
||||
d->zs.avail_in = (uInt)in_len;
|
||||
while (d->zs.avail_in > 0) {
|
||||
d->zs.next_out = d->out_buf;
|
||||
d->zs.avail_out = STREAM_DECOMPRESS_OUT_CHUNK;
|
||||
int rc = inflate(&d->zs, Z_NO_FLUSH);
|
||||
if (rc != Z_OK && rc != Z_STREAM_END && rc != Z_BUF_ERROR) {
|
||||
d->failed = true;
|
||||
return false;
|
||||
}
|
||||
size_t produced = STREAM_DECOMPRESS_OUT_CHUNK - d->zs.avail_out;
|
||||
if (!stream_emit(d, d->out_buf, produced))
|
||||
return false;
|
||||
if (rc == Z_STREAM_END) {
|
||||
*done = true;
|
||||
return d->zs.avail_in == 0;
|
||||
}
|
||||
if (rc == Z_BUF_ERROR && produced == 0) {
|
||||
/* Need more input. */
|
||||
break;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool compression_stream_decompressor_feed(CompressionStreamDecompressor* d, const void* in,
|
||||
size_t in_len, int out_fd, bool* done) {
|
||||
if (!d || d->failed)
|
||||
return false;
|
||||
d->out_fd = out_fd;
|
||||
if (done)
|
||||
*done = false;
|
||||
switch (d->algo) {
|
||||
case COMPRESSION_ALGO_NONE:
|
||||
return stream_feed_none(d, in, in_len, done);
|
||||
case COMPRESSION_ALGO_ZSTD:
|
||||
return stream_feed_zstd(d, in, in_len, done);
|
||||
case COMPRESSION_ALGO_ZLIB:
|
||||
case COMPRESSION_ALGO_ZLIBX:
|
||||
return stream_feed_zlib(d, in, in_len, done);
|
||||
case COMPRESSION_ALGO_LZ4:
|
||||
break;
|
||||
}
|
||||
d->failed = true;
|
||||
return false;
|
||||
}
|
||||
|
||||
unsigned long long compression_stream_decompressor_total(const CompressionStreamDecompressor* d) {
|
||||
return d ? d->total : 0;
|
||||
}
|
||||
|
||||
void compression_stream_decompressor_destroy(CompressionStreamDecompressor* d) {
|
||||
if (!d)
|
||||
return;
|
||||
if (d->dctx)
|
||||
ZSTD_freeDCtx(d->dctx);
|
||||
if (d->zs_initialized)
|
||||
inflateEnd(&d->zs);
|
||||
free(d->out_buf);
|
||||
free(d);
|
||||
}
|
||||
|
||||
/* ---- streaming compression ---- */
|
||||
|
||||
struct CompressionStreamCompressor {
|
||||
CompressionAlgo algo;
|
||||
int level;
|
||||
ZSTD_CCtx* cctx;
|
||||
z_stream zs;
|
||||
bool zs_initialized;
|
||||
unsigned char* out_buf;
|
||||
bool failed;
|
||||
};
|
||||
|
||||
bool compression_stream_compress_supported(CompressionAlgo algo) {
|
||||
return algo == COMPRESSION_ALGO_ZSTD || algo == COMPRESSION_ALGO_ZLIB ||
|
||||
algo == COMPRESSION_ALGO_ZLIBX;
|
||||
}
|
||||
|
||||
CompressionStreamCompressor* compression_stream_compressor_create(CompressionAlgo algo, int level,
|
||||
int threads) {
|
||||
(void)threads;
|
||||
if (!compression_algo_valid((int)algo) || algo == COMPRESSION_ALGO_LZ4)
|
||||
return NULL;
|
||||
CompressionStreamCompressor* c = calloc(1, sizeof(*c));
|
||||
if (!c)
|
||||
return NULL;
|
||||
c->algo = algo;
|
||||
c->level = level;
|
||||
c->out_buf = malloc(STREAM_DECOMPRESS_OUT_CHUNK);
|
||||
if (!c->out_buf) {
|
||||
free(c);
|
||||
return NULL;
|
||||
}
|
||||
if (algo == COMPRESSION_ALGO_ZSTD) {
|
||||
c->cctx = ZSTD_createCCtx();
|
||||
if (!c->cctx) {
|
||||
free(c->out_buf);
|
||||
free(c);
|
||||
return NULL;
|
||||
}
|
||||
} else if (algo == COMPRESSION_ALGO_ZLIB || algo == COMPRESSION_ALGO_ZLIBX) {
|
||||
if (deflateInit(&c->zs, level < 1 ? Z_DEFAULT_COMPRESSION : level) != Z_OK) {
|
||||
free(c->out_buf);
|
||||
free(c);
|
||||
return NULL;
|
||||
}
|
||||
c->zs_initialized = true;
|
||||
}
|
||||
return c;
|
||||
}
|
||||
|
||||
bool compression_stream_compressor_begin(CompressionStreamCompressor* c,
|
||||
unsigned long long raw_size, int out_fd) {
|
||||
if (!c || c->failed)
|
||||
return false;
|
||||
unsigned char hdr[1 + 4];
|
||||
size_t hdr_len = 1;
|
||||
hdr[0] = (unsigned char)c->algo;
|
||||
if (c->algo == COMPRESSION_ALGO_ZLIB || c->algo == COMPRESSION_ALGO_ZLIBX) {
|
||||
uint32_t size32 = raw_size > UINT32_MAX ? UINT32_MAX : (uint32_t)raw_size;
|
||||
for (int i = 0; i < 4; i++)
|
||||
hdr[1 + i] = (uint8_t)((size32 >> (8 * i)) & 0xff);
|
||||
hdr_len = 5;
|
||||
}
|
||||
if (c->algo == COMPRESSION_ALGO_ZSTD) {
|
||||
/* Pledge the source size and force the frame content-size field so the
|
||||
receiver can decide whether to stream from the frame header alone. */
|
||||
if (ZSTD_isError(ZSTD_CCtx_setPledgedSrcSize(c->cctx, raw_size)) ||
|
||||
ZSTD_isError(ZSTD_CCtx_setParameter(c->cctx, ZSTD_c_compressionLevel, c->level)) ||
|
||||
ZSTD_isError(ZSTD_CCtx_setParameter(c->cctx, ZSTD_c_contentSizeFlag, 1))) {
|
||||
c->failed = true;
|
||||
return false;
|
||||
}
|
||||
if (ZSTD_isError(ZSTD_CCtx_setParameter(c->cctx, ZSTD_c_checksumFlag, 0))) {
|
||||
c->failed = true;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if (!stream_write_all(out_fd, hdr, hdr_len)) {
|
||||
c->failed = true;
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool stream_compress_zlib(CompressionStreamCompressor* c, const void* in, size_t in_len,
|
||||
int out_fd, int flush) {
|
||||
c->zs.next_in = (Bytef*)in;
|
||||
c->zs.avail_in = (uInt)in_len;
|
||||
do {
|
||||
c->zs.next_out = c->out_buf;
|
||||
c->zs.avail_out = STREAM_DECOMPRESS_OUT_CHUNK;
|
||||
int rc = deflate(&c->zs, flush);
|
||||
if (rc != Z_OK && rc != Z_STREAM_END && rc != Z_BUF_ERROR) {
|
||||
c->failed = true;
|
||||
return false;
|
||||
}
|
||||
size_t produced = STREAM_DECOMPRESS_OUT_CHUNK - c->zs.avail_out;
|
||||
if (!stream_write_all(out_fd, c->out_buf, produced)) {
|
||||
c->failed = true;
|
||||
return false;
|
||||
}
|
||||
if (rc == Z_STREAM_END)
|
||||
return true;
|
||||
if (rc == Z_BUF_ERROR && produced == 0)
|
||||
break;
|
||||
} while (c->zs.avail_in > 0 || flush == Z_FINISH);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool compression_stream_compressor_feed(CompressionStreamCompressor* c, const void* in,
|
||||
size_t in_len, int out_fd) {
|
||||
if (!c || c->failed)
|
||||
return false;
|
||||
if (c->algo == COMPRESSION_ALGO_NONE)
|
||||
return stream_write_all(out_fd, in, in_len);
|
||||
if (c->algo == COMPRESSION_ALGO_ZSTD) {
|
||||
ZSTD_inBuffer input = {in, in_len, 0};
|
||||
while (input.pos < input.size) {
|
||||
ZSTD_outBuffer output = {c->out_buf, STREAM_DECOMPRESS_OUT_CHUNK, 0};
|
||||
size_t ret = ZSTD_compressStream2(c->cctx, &output, &input, ZSTD_e_continue);
|
||||
if (ZSTD_isError(ret)) {
|
||||
c->failed = true;
|
||||
return false;
|
||||
}
|
||||
if (!stream_write_all(out_fd, c->out_buf, output.pos)) {
|
||||
c->failed = true;
|
||||
return false;
|
||||
}
|
||||
if (output.pos == 0 && input.pos < input.size)
|
||||
break; /* avoid spinning; zstd buffers the rest internally */
|
||||
}
|
||||
return true;
|
||||
}
|
||||
return stream_compress_zlib(c, in, in_len, out_fd, Z_NO_FLUSH);
|
||||
}
|
||||
|
||||
bool compression_stream_compressor_finish(CompressionStreamCompressor* c, int out_fd) {
|
||||
if (!c || c->failed)
|
||||
return false;
|
||||
if (c->algo == COMPRESSION_ALGO_NONE)
|
||||
return true;
|
||||
if (c->algo == COMPRESSION_ALGO_ZSTD) {
|
||||
size_t ret;
|
||||
do {
|
||||
ZSTD_inBuffer input = {NULL, 0, 0};
|
||||
ZSTD_outBuffer output = {c->out_buf, STREAM_DECOMPRESS_OUT_CHUNK, 0};
|
||||
ret = ZSTD_compressStream2(c->cctx, &output, &input, ZSTD_e_end);
|
||||
if (ZSTD_isError(ret)) {
|
||||
c->failed = true;
|
||||
return false;
|
||||
}
|
||||
if (!stream_write_all(out_fd, c->out_buf, output.pos)) {
|
||||
c->failed = true;
|
||||
return false;
|
||||
}
|
||||
} while (ret > 0);
|
||||
return true;
|
||||
}
|
||||
return stream_compress_zlib(c, NULL, 0, out_fd, Z_FINISH);
|
||||
}
|
||||
|
||||
void compression_stream_compressor_destroy(CompressionStreamCompressor* c) {
|
||||
if (!c)
|
||||
return;
|
||||
if (c->cctx)
|
||||
ZSTD_freeCCtx(c->cctx);
|
||||
if (c->zs_initialized)
|
||||
deflateEnd(&c->zs);
|
||||
free(c->out_buf);
|
||||
free(c);
|
||||
}
|
||||
|
||||
unsigned long long compression_peek_frame_content_size(const void* buf, size_t len) {
|
||||
if (!buf || len < 1)
|
||||
return 0;
|
||||
const uint8_t* p = buf;
|
||||
uint8_t codec = p[0];
|
||||
if (!compression_algo_valid(codec))
|
||||
return 0;
|
||||
if (codec == (uint8_t)COMPRESSION_ALGO_NONE)
|
||||
return len - 1;
|
||||
if (codec == (uint8_t)COMPRESSION_ALGO_ZSTD) {
|
||||
if (len < 2)
|
||||
return 0;
|
||||
unsigned long long size = ZSTD_getFrameContentSize(p + 1, len - 1);
|
||||
if (size == ZSTD_CONTENTSIZE_ERROR || size == ZSTD_CONTENTSIZE_UNKNOWN)
|
||||
return 0;
|
||||
return size;
|
||||
}
|
||||
if (len < 1 + LZ4_SIZE_PREFIX_LEN)
|
||||
return 0;
|
||||
uint32_t raw = 0;
|
||||
for (int i = 0; i < LZ4_SIZE_PREFIX_LEN; i++)
|
||||
raw |= (uint32_t)p[1 + i] << (8 * i);
|
||||
return raw;
|
||||
}
|
||||
|
||||
@@ -81,6 +81,54 @@ Data* data_compress_with_threads(Data* data_to_compress, int compression_level,
|
||||
Data* data_decompress(Data* compressed_data);
|
||||
bool compression_should_skip_with_suffixes(const char* path, char* const* suffixes, int count);
|
||||
|
||||
/* Streaming decompression for a payload too large to hold in memory. The
|
||||
* caller consumes the frame's leading codec byte (and, for lz4/zlib/zlibx, the
|
||||
* 4-byte little-endian raw-size prefix) and then feeds the remaining frame
|
||||
* bytes in bounded chunks; decompressed output is written straight to `out_fd`
|
||||
* so neither the compressed nor the decompressed image is ever materialized.
|
||||
* Only zstd (the default), zlib/zlibx and none support streaming; lz4's block
|
||||
* format is one-shot, so its stream decompressor reports failure and the caller
|
||||
* falls back (the whole-buffer path keeps its existing bound). */
|
||||
typedef struct CompressionStreamDecompressor CompressionStreamDecompressor;
|
||||
|
||||
CompressionStreamDecompressor*
|
||||
compression_stream_decompressor_create(CompressionAlgo algo, unsigned long long expected_out);
|
||||
/* Feed one chunk. Returns false on a malformed frame, an I/O error, or when the
|
||||
* total output would exceed `expected_out` (when non-zero). *done is set once
|
||||
* the frame end has been reached. */
|
||||
bool compression_stream_decompressor_feed(CompressionStreamDecompressor* d, const void* in,
|
||||
size_t in_len, int out_fd, bool* done);
|
||||
unsigned long long compression_stream_decompressor_total(const CompressionStreamDecompressor* d);
|
||||
void compression_stream_decompressor_destroy(CompressionStreamDecompressor* d);
|
||||
|
||||
/* Peek the logical (decompressed) size from the leading bytes of a compressed
|
||||
* frame (codec byte + header), returning 0 when it cannot be determined from
|
||||
* the supplied prefix. Used to decide whether a frame must take the streaming
|
||||
* path before its body is read. */
|
||||
unsigned long long compression_peek_frame_content_size(const void* buf, size_t len);
|
||||
|
||||
/* Streaming compression (sender side). Compresses a source in bounded chunks
|
||||
* into `out_fd` as one self-describing frame (codec byte, the lz4/zlib raw-size
|
||||
* prefix, then the codec stream), so a whole file can be compressed without
|
||||
* materializing it in memory. zstd/zlib/zlibx/none are supported; lz4's block
|
||||
* format is one-shot, so its create() returns NULL and the caller keeps the
|
||||
* buffered path. `raw_size` is the known source length (used for the zlib
|
||||
* prefix and, for zstd, the frame content-size field). */
|
||||
typedef struct CompressionStreamCompressor CompressionStreamCompressor;
|
||||
|
||||
/* True when `algo` can be stream-compressed (zstd/zlib/zlibx; lz4's block format
|
||||
* is one-shot). Used by the sender to decide whether an over-threshold source
|
||||
* may stay unloaded. */
|
||||
bool compression_stream_compress_supported(CompressionAlgo algo);
|
||||
CompressionStreamCompressor* compression_stream_compressor_create(CompressionAlgo algo, int level,
|
||||
int threads);
|
||||
bool compression_stream_compressor_begin(CompressionStreamCompressor* c,
|
||||
unsigned long long raw_size, int out_fd);
|
||||
bool compression_stream_compressor_feed(CompressionStreamCompressor* c, const void* in,
|
||||
size_t in_len, int out_fd);
|
||||
bool compression_stream_compressor_finish(CompressionStreamCompressor* c, int out_fd);
|
||||
void compression_stream_compressor_destroy(CompressionStreamCompressor* c);
|
||||
|
||||
/* Release the calling thread's cached zstd contexts (compressor, decompressor
|
||||
* and scratch buffer). The cache is thread-local and is also released
|
||||
* automatically when a worker thread exits (via a C11 tss destructor) and for
|
||||
|
||||
+15
-2
@@ -83,7 +83,7 @@ typedef struct {
|
||||
typedef enum SuperMode { SUPER_MODE_AUTO = 0, SUPER_MODE_ON = 1, SUPER_MODE_OFF = 2 } SuperMode;
|
||||
|
||||
/* ===========================================================================
|
||||
* Config wire-field table (single source of truth for protocol 2.29.0).
|
||||
* Config wire-field table (single source of truth for protocol 2.30.0).
|
||||
*
|
||||
* Every field below crosses the wire. The table is the ONLY place a
|
||||
* serialized field is named: config.h expands CONFIG_WIRE_FIELDS() to declare
|
||||
@@ -1084,7 +1084,20 @@ typedef struct Config {
|
||||
* version must bump; the strict same-version handshake (config_receive rejects a
|
||||
* mismatched version before parsing anything else) keeps a 2.29 client and a
|
||||
* 2.28 server from ever reaching that state. */
|
||||
#define PROTOCOL_VERSION "2.29.0"
|
||||
/* (11) Client-message channel + partial exit (protocol 2.30.0): the
|
||||
* config-frame LAYOUT is unchanged (no new config field), but the frame stream
|
||||
* gains two statuses. STATUS_CLIENT_MSG (client->server) carries a bounded,
|
||||
* length-prefixed diagnostic string so a client running with --stderr=client
|
||||
* (rsync's --no-msgs2stderr spelling) can forward its own diagnostics to the
|
||||
* server's stderr. STATUS_PARTIAL (receiver->client) is the terminal status
|
||||
* sent instead of STATUS_OK when a per-entry receiver failure (e.g. an
|
||||
* unprivileged --devices mknod) did not abort the stream; the sender exits 23
|
||||
* (rsync's partial transfer) and still removes successfully transferred
|
||||
* --remove-source-files sources. A 2.29 peer that does not know these status
|
||||
* values would reject them as an unknown status and tear the connection down,
|
||||
* so the protocol version must bump; the strict same-version handshake keeps a
|
||||
* 2.30 client and a 2.29 server from ever reaching that state. */
|
||||
#define PROTOCOL_VERSION "2.30.0"
|
||||
#define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024)
|
||||
/* Upper bound on total basis-dir entries (rsync caps --link-dest at 20). */
|
||||
#define MAX_BASIS_DIRS 64
|
||||
|
||||
+116
-34
@@ -8,13 +8,49 @@
|
||||
#include <errno.h>
|
||||
#include <fcntl.h>
|
||||
#include <libgen.h>
|
||||
#include <stdatomic.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/file.h>
|
||||
#include <sys/stat.h>
|
||||
#include <time.h>
|
||||
#include <unistd.h>
|
||||
|
||||
/* Process-wide counter so two staging contexts created in the same process (or
|
||||
within the same clock tick) can never pick the same name. */
|
||||
static unsigned long long delay_updates_next_sequence(void) {
|
||||
static atomic_ullong sequence;
|
||||
return atomic_fetch_add_explicit(&sequence, 1, memory_order_relaxed);
|
||||
}
|
||||
|
||||
/* Build the per-run staging directory basename: the reserved prefix plus the
|
||||
pid and an entropy token. A fixed name could collide with a genuine
|
||||
destination entry; the token makes such a collision vanishingly unlikely and,
|
||||
if it ever happens, prepare() refuses to touch the existing directory. */
|
||||
static char* delay_updates_make_staging_name(void) {
|
||||
unsigned long long entropy = 0;
|
||||
int fd = open("/dev/urandom", O_RDONLY | O_CLOEXEC);
|
||||
if (fd >= 0) {
|
||||
ssize_t got = read(fd, &entropy, sizeof(entropy));
|
||||
close(fd);
|
||||
if (got != (ssize_t)sizeof(entropy))
|
||||
entropy = 0;
|
||||
}
|
||||
if (entropy == 0)
|
||||
entropy = ((unsigned long long)time(NULL) << 20) ^ ((unsigned long long)getpid() << 8) ^
|
||||
delay_updates_next_sequence();
|
||||
int length = snprintf(NULL, 0, DELAY_UPDATES_STAGING_DIR ".%ld.%llx", (long)getpid(), entropy);
|
||||
if (length < 0)
|
||||
return NULL;
|
||||
char* name = malloc((size_t)length + 1);
|
||||
if (!name)
|
||||
return NULL;
|
||||
snprintf(name, (size_t)length + 1, DELAY_UPDATES_STAGING_DIR ".%ld.%llx", (long)getpid(),
|
||||
entropy);
|
||||
return name;
|
||||
}
|
||||
|
||||
DelayUpdatesContext* delay_updates_context_create(const char* root_directory) {
|
||||
if (!root_directory)
|
||||
return NULL;
|
||||
@@ -26,8 +62,15 @@ DelayUpdatesContext* delay_updates_context_create(const char* root_directory) {
|
||||
free(context);
|
||||
return NULL;
|
||||
}
|
||||
context->staging_root = path_cat(root_directory, DELAY_UPDATES_STAGING_DIR);
|
||||
context->staging_name = delay_updates_make_staging_name();
|
||||
if (!context->staging_name) {
|
||||
free(context->root_directory);
|
||||
free(context);
|
||||
return NULL;
|
||||
}
|
||||
context->staging_root = path_cat(root_directory, context->staging_name);
|
||||
if (!context->staging_root) {
|
||||
free(context->staging_name);
|
||||
free(context->root_directory);
|
||||
free(context);
|
||||
return NULL;
|
||||
@@ -39,6 +82,7 @@ DelayUpdatesContext* delay_updates_context_create(const char* root_directory) {
|
||||
context->lock_fd = -1;
|
||||
if (mtx_init(&context->mutex, mtx_plain) != thrd_success) {
|
||||
free(context->staging_root);
|
||||
free(context->staging_name);
|
||||
free(context->root_directory);
|
||||
free(context);
|
||||
return NULL;
|
||||
@@ -54,6 +98,7 @@ void delay_updates_context_destroy(DelayUpdatesContext* context) {
|
||||
close(context->lock_fd);
|
||||
context->lock_fd = -1;
|
||||
free(context->staging_root);
|
||||
free(context->staging_name);
|
||||
free(context->root_directory);
|
||||
for (size_t i = 0; i < context->count; i++) {
|
||||
free(context->entries[i].staged_path);
|
||||
@@ -125,48 +170,81 @@ bool delay_updates_prepare(DelayUpdatesContext* context) {
|
||||
return false;
|
||||
if (context->prepared)
|
||||
return true;
|
||||
int fd = file_open_private_dir(context->staging_root);
|
||||
if (fd < 0) {
|
||||
/* Create the per-run staging directory with O_EXCL semantics. The name is
|
||||
unique to this transfer, so if the path already exists it is NOT ours:
|
||||
either a genuine destination entry that happens to share the name or a
|
||||
leftover from another session. Refuse rather than wipe it -- the old
|
||||
fixed-name design could destroy a real destination entry. A crash
|
||||
leftover is never reused (the next run picks a fresh name). */
|
||||
char* leaf = NULL;
|
||||
int parent_fd = file_open_secure_parent(context->staging_root, &leaf, true);
|
||||
if (parent_fd < 0) {
|
||||
int saved_errno = errno;
|
||||
char* escaped = output_escape(context->staging_root, false);
|
||||
log_message(LOG_LEVEL_ERROR, "could not create --delay-updates staging directory '%s': %s",
|
||||
escaped ? escaped : "<allocation failed>", strerror(saved_errno));
|
||||
free(escaped);
|
||||
free(leaf);
|
||||
return false;
|
||||
}
|
||||
/* Hold an exclusive advisory lock on the staging directory for the whole
|
||||
transfer. The staging directory name is fixed, so two simultaneous
|
||||
delayed transfers to the same destination root would otherwise share it
|
||||
and destroy each other's staged files. The lock makes the second session
|
||||
fail cleanly instead of corrupting the first. The lock is released when
|
||||
the context (and its file descriptor) is destroyed. */
|
||||
int fd = openat(parent_fd, leaf, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||
if (fd >= 0) {
|
||||
close(fd);
|
||||
close(parent_fd);
|
||||
char* escaped = output_escape(context->staging_root, false);
|
||||
log_message(LOG_LEVEL_ERROR,
|
||||
"--delay-updates staging directory '%s' already exists and is not owned by this "
|
||||
"transfer; refusing to overwrite it",
|
||||
escaped ? escaped : "<allocation failed>");
|
||||
free(escaped);
|
||||
free(leaf);
|
||||
return false;
|
||||
}
|
||||
if (errno != ENOENT) {
|
||||
int saved_errno = errno;
|
||||
close(parent_fd);
|
||||
char* escaped = output_escape(context->staging_root, false);
|
||||
log_message(LOG_LEVEL_ERROR, "could not open --delay-updates staging directory '%s': %s",
|
||||
escaped ? escaped : "<allocation failed>", strerror(saved_errno));
|
||||
free(escaped);
|
||||
free(leaf);
|
||||
return false;
|
||||
}
|
||||
if (mkdirat(parent_fd, leaf, 0700) != 0) {
|
||||
int saved_errno = errno;
|
||||
close(parent_fd);
|
||||
char* escaped = output_escape(context->staging_root, false);
|
||||
log_message(LOG_LEVEL_ERROR, "could not create --delay-updates staging directory '%s': %s",
|
||||
escaped ? escaped : "<allocation failed>", strerror(saved_errno));
|
||||
free(escaped);
|
||||
free(leaf);
|
||||
return false;
|
||||
}
|
||||
fd = openat(parent_fd, leaf, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||
close(parent_fd);
|
||||
free(leaf);
|
||||
if (fd < 0) {
|
||||
int saved_errno = errno;
|
||||
char* escaped = output_escape(context->staging_root, false);
|
||||
log_message(LOG_LEVEL_ERROR, "could not open --delay-updates staging directory '%s': %s",
|
||||
escaped ? escaped : "<allocation failed>", strerror(saved_errno));
|
||||
free(escaped);
|
||||
return false;
|
||||
}
|
||||
/* Keep the exclusive advisory lock as defense in depth: the unique name
|
||||
already prevents two sessions from sharing a staging directory, but the
|
||||
lock also catches an improbable same-name collision that raced between the
|
||||
existence check above and the open. */
|
||||
if (flock(fd, LOCK_EX | LOCK_NB) != 0) {
|
||||
int saved_errno = errno;
|
||||
close(fd);
|
||||
if (saved_errno == EWOULDBLOCK || saved_errno == EAGAIN) {
|
||||
char* escaped = output_escape(context->staging_root, false);
|
||||
log_message(LOG_LEVEL_ERROR,
|
||||
"another --delay-updates transfer to '%s' is already in progress; refusing to "
|
||||
"share the staging directory",
|
||||
escaped ? escaped : "<allocation failed>");
|
||||
free(escaped);
|
||||
} else {
|
||||
log_message(LOG_LEVEL_ERROR, "could not lock --delay-updates staging directory '%s': %s",
|
||||
context->staging_root, strerror(saved_errno));
|
||||
}
|
||||
escaped ? escaped : "<allocation failed>", strerror(saved_errno));
|
||||
free(escaped);
|
||||
return false;
|
||||
}
|
||||
context->lock_fd = fd;
|
||||
/* Only now, with exclusive ownership, wipe leftovers from an interrupted
|
||||
earlier transfer; this can never race with a live session. */
|
||||
bool ok = delay_wipe_dir_fd(fd);
|
||||
if (!ok) {
|
||||
log_message(LOG_LEVEL_ERROR, "could not clear stale --delay-updates staging files under '%s'",
|
||||
context->staging_root);
|
||||
close(context->lock_fd);
|
||||
context->lock_fd = -1;
|
||||
return false;
|
||||
}
|
||||
context->prepared = true;
|
||||
return true;
|
||||
}
|
||||
@@ -264,12 +342,16 @@ static bool delay_publish_entry(DelayUpdatesContext* context, const Config* conf
|
||||
const StagedFileEntry* entry) {
|
||||
if (!delay_publish_backup(context, config, entry))
|
||||
return false;
|
||||
/* --force: an incoming regular file/symlink may replace a destination
|
||||
DIRECTORY (possibly non-empty). The immediate-install path handles this in
|
||||
file_receive; a --delay-updates run stages elsewhere and only discovers the
|
||||
blocking directory here, so clear it before the rename (rsync's
|
||||
"could not make way for new regular file" without --force). */
|
||||
if (config && config->force_delete && file_directory_exists_secure(entry->final_path)) {
|
||||
/* An incoming regular file/symlink may replace a destination DIRECTORY that
|
||||
blocks it. rsync removes the blocker recursively when --delete or --force
|
||||
is active (its generator's "make way" deletion), and a --delay-updates run
|
||||
stages elsewhere so it only discovers the blocker here. FastSync's
|
||||
immediate-install path clears it too; without --delete/--force a non-empty
|
||||
blocker fails the run (rsync's "could not make way for new regular file").
|
||||
use_delete is gated by the server --allow-delete policy, so a client can
|
||||
never use this to bypass deletion authorization. */
|
||||
if (config && (config->force_delete || config->use_delete) &&
|
||||
file_directory_exists_secure(entry->final_path)) {
|
||||
if (!file_remove_tree_secure(entry->final_path)) {
|
||||
char* escaped = output_escape(entry->final_path, false);
|
||||
log_message(LOG_LEVEL_ERROR, "could not remove destination directory blocking '%s': %s",
|
||||
|
||||
@@ -24,6 +24,7 @@ typedef struct {
|
||||
shared with the publish/cleanup phase that runs after the threads join. */
|
||||
typedef struct DelayUpdatesContext {
|
||||
char* root_directory; /* receive root the staging dir lives under */
|
||||
char* staging_name; /* per-run unique staging dir basename */
|
||||
char* staging_root; /* root_directory/<staging dir name> */
|
||||
mtx_t mutex;
|
||||
StagedFileEntry* entries;
|
||||
@@ -33,7 +34,11 @@ typedef struct DelayUpdatesContext {
|
||||
int lock_fd; /* advisory exclusive flock held on the staging dir, or -1 */
|
||||
} DelayUpdatesContext;
|
||||
|
||||
/* Name of the private staging subdirectory created under the receive root. */
|
||||
/* Reserved prefix for the private staging subdirectory created under the
|
||||
receive root. The actual directory name is per-run unique (the prefix plus a
|
||||
pid/entropy token) so it can never clobber a genuine destination entry that
|
||||
happens to share the name; the bare prefix is still what a --backup-dir must
|
||||
not collide with. */
|
||||
#define DELAY_UPDATES_STAGING_DIR ".fastsync-stage"
|
||||
|
||||
/* True when `dir` (ignoring a trailing "/") is the reserved staging directory
|
||||
|
||||
+309
-188
@@ -32,6 +32,49 @@ static bool keep_is_file(const PathIndex* index, const char* rel_path) {
|
||||
return path_index_contains(index, rel_path);
|
||||
}
|
||||
|
||||
/* rsync's receiver-side verdict for one candidate extra: the per-directory
|
||||
* chain first (deepest directory before ancestors), then the command-line base
|
||||
* rules. Either rule set may be absent. */
|
||||
FilterAction delete_protect_verdict(const DeleteProtectRules* protect, const char* rel_path,
|
||||
const char* leaf, bool is_dir) {
|
||||
if (!protect)
|
||||
return FILTER_ACTION_NONE;
|
||||
/* rsync protects its own --backup files from the delete pass: a name ending
|
||||
in the backup suffix is never an extra. Checked before the filter rules so
|
||||
an explicit exclude cannot be bypassed (the suffix is always a shield). */
|
||||
if (protect->backup_suffix && protect->backup_suffix[0] != '\0') {
|
||||
size_t name_len = strlen(leaf);
|
||||
size_t suffix_len = strlen(protect->backup_suffix);
|
||||
if (name_len > suffix_len &&
|
||||
strcmp(leaf + (name_len - suffix_len), protect->backup_suffix) == 0)
|
||||
return FILTER_ACTION_PROTECT;
|
||||
}
|
||||
FilterAction action = filter_dir_rules_apply_side(protect->dir_rules, rel_path, leaf, is_dir);
|
||||
if (action != FILTER_ACTION_NONE)
|
||||
return action;
|
||||
return filter_rules_apply_side(protect->base_rules, rel_path, leaf, is_dir, FILTER_SIDE_RECEIVER);
|
||||
}
|
||||
|
||||
const char* delete_backup_suffix(const Config* config) {
|
||||
if (!config || !config->backup || config->ignore_existing)
|
||||
return NULL;
|
||||
const char* suffix = config->suffix ? config->suffix : "~";
|
||||
if (!suffix[0] || strchr(suffix, '/'))
|
||||
return NULL;
|
||||
return suffix;
|
||||
}
|
||||
|
||||
/* Classify a removed entry from its st_mode for the per-type delete counters. */
|
||||
DeleteEntryType delete_entry_type_of_mode(mode_t mode) {
|
||||
if (S_ISDIR(mode))
|
||||
return DELETE_ENTRY_DIR;
|
||||
if (S_ISLNK(mode))
|
||||
return DELETE_ENTRY_LINK;
|
||||
if (S_ISREG(mode))
|
||||
return DELETE_ENTRY_REG;
|
||||
return DELETE_ENTRY_SPECIAL;
|
||||
}
|
||||
|
||||
/* True when child_rel is, or lies below, a protected entry. A prefix "a"
|
||||
therefore protects "a" and "a/b/c" but not "ab". Entries with top_level_only
|
||||
set only protect DIRECT children of the receive root (at_root); nested
|
||||
@@ -126,6 +169,7 @@ bool delete_dir_entries_collect(int dirfd, DeleteDirEntry** out, size_t* count,
|
||||
break;
|
||||
}
|
||||
entries[used].is_dir = S_ISDIR(st.st_mode);
|
||||
entries[used].mode = st.st_mode;
|
||||
used++;
|
||||
}
|
||||
closedir(dir);
|
||||
@@ -180,6 +224,235 @@ typedef struct {
|
||||
void* observer_context; /* DELETE mode */
|
||||
} DeleteWalkState;
|
||||
|
||||
/* The per-walk invariants threaded unchanged through every recursive descent:
|
||||
the keep/synchronized-dir indexes, the destination mode and the protection
|
||||
rules. Bundling them keeps the recursive helpers below to a handful of
|
||||
positional arguments. */
|
||||
typedef struct {
|
||||
const PathIndex* keep;
|
||||
const PathIndex* dirs;
|
||||
DeleteWalkState* state;
|
||||
const DeleteSkipEntry* skips;
|
||||
int skip_count;
|
||||
const DeleteProtectRules* protect;
|
||||
} DeleteWalkContext;
|
||||
|
||||
/* Duplicate `path` with rsync's trailing-slash convention, used to report a
|
||||
removed (or would-be-removed) directory. Returns NULL on allocation
|
||||
failure. */
|
||||
static char* with_trailing_slash(const char* path) {
|
||||
size_t len = strlen(path);
|
||||
char* copy = malloc(len + 2);
|
||||
if (!copy)
|
||||
return NULL;
|
||||
memcpy(copy, path, len);
|
||||
copy[len] = '/';
|
||||
copy[len + 1] = '\0';
|
||||
return copy;
|
||||
}
|
||||
|
||||
/* Forward declaration: the ordered passes below recurse through the driver. */
|
||||
static bool delete_walk_fd(int dirfd, const char* rel_path, const DeleteWalkContext* ctx,
|
||||
bool parent_deletable, bool* all_removed);
|
||||
|
||||
/* Descend into the child directory `name` of `dirfd`, walking it as part of the
|
||||
current operation. Returns false on a genuine open/walk failure; on success
|
||||
*child_all_removed reports whether the child removed everything it held (so
|
||||
the caller may rmdir it). */
|
||||
static bool delete_walk_child(int dirfd, const char* name, const char* child_rel,
|
||||
const DeleteWalkContext* ctx, bool deletable,
|
||||
bool* child_all_removed) {
|
||||
*child_all_removed = false;
|
||||
int childfd = openat(dirfd, name, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||
if (childfd < 0)
|
||||
return errno == ENOENT;
|
||||
bool ok = delete_walk_fd(childfd, child_rel, ctx, deletable, child_all_removed);
|
||||
close(childfd);
|
||||
return ok;
|
||||
}
|
||||
|
||||
/* Classify every entry up front (the verdict does not depend on processing
|
||||
order) so the ordered passes below can act on it. Sets shielded[]/is_extra[]
|
||||
and reports through *local_survives whether anything in this directory stays
|
||||
in place. Returns false on a path-construction failure. */
|
||||
static bool delete_walk_classify(const char* rel_path, const DeleteDirEntry* entries, size_t count,
|
||||
const DeleteWalkContext* ctx, bool deletable, bool at_root,
|
||||
bool* shielded, bool* is_extra, bool* local_survives) {
|
||||
bool ok = true;
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
char* child_rel = path_cat((char*)rel_path, entries[i].name);
|
||||
if (!child_rel) {
|
||||
ok = false;
|
||||
continue;
|
||||
}
|
||||
/* A --delay-updates run keeps its staging directory as a direct child of
|
||||
the receive root, and basis-dir snapshots live below it too. Their
|
||||
contents are not manifest entries, so descending into them would delete
|
||||
every staged / basis file as an "extra". Only the staging name (a
|
||||
top-level-only prefix) and the basis prefixes are protected: a nested
|
||||
destination directory that happens to be called .fastsync-stage is
|
||||
ordinary content. */
|
||||
if (path_under_skip_prefix(child_rel, at_root, ctx->skips, ctx->skip_count)) {
|
||||
shielded[i] = true;
|
||||
*local_survives = true;
|
||||
} else if (delete_protect_verdict(ctx->protect, child_rel, entries[i].name,
|
||||
entries[i].is_dir) == FILTER_ACTION_PROTECT) {
|
||||
/* A first-match protect rule shields the extra; for a directory the whole
|
||||
subtree is shielded (rsync prunes an excluded directory), so do not
|
||||
descend. */
|
||||
shielded[i] = true;
|
||||
*local_survives = true;
|
||||
} else if (entries[i].is_dir) {
|
||||
bool child_synced = ctx->dirs && path_index_contains(ctx->dirs, child_rel);
|
||||
is_extra[i] = deletable && !child_synced && !keep_is_dir(ctx->keep, child_rel);
|
||||
if (!is_extra[i])
|
||||
*local_survives = true;
|
||||
} else {
|
||||
is_extra[i] = deletable && !keep_is_file(ctx->keep, child_rel);
|
||||
if (!is_extra[i])
|
||||
*local_survives = true;
|
||||
}
|
||||
free(child_rel);
|
||||
}
|
||||
return ok;
|
||||
}
|
||||
|
||||
/* Pass 1: extraneous subdirectories, descending. Recurses into each and, when
|
||||
the child removed everything it held, records or removes it and charges the
|
||||
budget. */
|
||||
static bool delete_walk_extra_dirs(int dirfd, const char* rel_path, const DeleteDirEntry* entries,
|
||||
size_t dir_count, const DeleteWalkContext* ctx, bool deletable,
|
||||
const bool* is_extra, bool* local_survives) {
|
||||
bool ok = true;
|
||||
for (size_t i = 0; i < dir_count; i++) {
|
||||
if (!is_extra[i])
|
||||
continue;
|
||||
char* child_rel = path_cat((char*)rel_path, entries[i].name);
|
||||
if (!child_rel) {
|
||||
ok = false;
|
||||
continue;
|
||||
}
|
||||
bool child_all_removed = false;
|
||||
if (!delete_walk_child(dirfd, entries[i].name, child_rel, ctx, deletable, &child_all_removed))
|
||||
ok = false;
|
||||
if (child_all_removed && deletable) {
|
||||
if (ctx->state->mode == DELETE_WALK_MODE_LIST) {
|
||||
/* Record the directory with rsync's trailing slash. */
|
||||
char* copy = with_trailing_slash(child_rel);
|
||||
if (!copy) {
|
||||
ok = false;
|
||||
} else if (!array_list_add(ctx->state->out, copy)) {
|
||||
free(copy);
|
||||
ok = false;
|
||||
} else {
|
||||
(*ctx->state->recorded)++;
|
||||
}
|
||||
} else if (ctx->state->budget->deleted >= ctx->state->budget->max_delete) {
|
||||
ctx->state->budget->limit_hit = true;
|
||||
ctx->state->budget->skipped++;
|
||||
*local_survives = true;
|
||||
} else if (unlinkat(dirfd, entries[i].name, AT_REMOVEDIR) != 0) {
|
||||
/* ENOENT: already gone (fine). ENOTEMPTY/EEXIST: the directory still
|
||||
holds entries the walker leaves in place (a protected excluded
|
||||
prefix, a kept file the manifest protects, a symlink); rsync leaves
|
||||
such a directory behind, so this is not an error. Only genuine I/O
|
||||
failures abort the deletion. */
|
||||
if (errno != ENOENT && errno != ENOTEMPTY && errno != EEXIST)
|
||||
ok = false;
|
||||
*local_survives = true;
|
||||
} else {
|
||||
ctx->state->budget->deleted++;
|
||||
/* rsync reports a removed directory with a trailing slash. */
|
||||
if (ctx->state->observer) {
|
||||
char* with_slash = with_trailing_slash(child_rel);
|
||||
if (with_slash) {
|
||||
ctx->state->observer(ctx->state->observer_context, with_slash, DELETE_ENTRY_DIR);
|
||||
free(with_slash);
|
||||
} else {
|
||||
ctx->state->observer(ctx->state->observer_context, child_rel, DELETE_ENTRY_DIR);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
*local_survives = true;
|
||||
}
|
||||
free(child_rel);
|
||||
}
|
||||
return ok;
|
||||
}
|
||||
|
||||
/* Pass 2: extraneous files, descending. */
|
||||
static bool delete_walk_extra_files(int dirfd, const char* rel_path, const DeleteDirEntry* entries,
|
||||
size_t dir_count, size_t count, const DeleteWalkContext* ctx,
|
||||
const bool* is_extra, bool* local_survives) {
|
||||
bool ok = true;
|
||||
for (size_t i = dir_count; i < count; i++) {
|
||||
if (!is_extra[i])
|
||||
continue;
|
||||
if (ctx->state->mode == DELETE_WALK_MODE_LIST) {
|
||||
char* child_rel = path_cat((char*)rel_path, entries[i].name);
|
||||
if (!child_rel) {
|
||||
ok = false;
|
||||
continue;
|
||||
}
|
||||
char* copy = str_dup(child_rel);
|
||||
if (!copy || !array_list_add(ctx->state->out, copy)) {
|
||||
free(copy);
|
||||
ok = false;
|
||||
} else {
|
||||
(*ctx->state->recorded)++;
|
||||
}
|
||||
free(child_rel);
|
||||
} else if (ctx->state->budget->deleted >= ctx->state->budget->max_delete) {
|
||||
ctx->state->budget->limit_hit = true;
|
||||
ctx->state->budget->skipped++;
|
||||
*local_survives = true;
|
||||
} else if (unlinkat(dirfd, entries[i].name, 0) != 0) {
|
||||
if (errno != ENOENT)
|
||||
ok = false;
|
||||
*local_survives = true;
|
||||
} else {
|
||||
ctx->state->budget->deleted++;
|
||||
char* child_rel = path_cat((char*)rel_path, entries[i].name);
|
||||
if (child_rel) {
|
||||
if (ctx->state->observer)
|
||||
ctx->state->observer(ctx->state->observer_context, child_rel,
|
||||
delete_entry_type_of_mode(entries[i].mode));
|
||||
char* escaped_path = output_escape(child_rel, log_get_8_bit_output());
|
||||
fprintf(stderr, " Deleted: %s\n", escaped_path ? escaped_path : "<allocation failed>");
|
||||
free(escaped_path);
|
||||
}
|
||||
free(child_rel);
|
||||
}
|
||||
}
|
||||
return ok;
|
||||
}
|
||||
|
||||
/* Pass 3: kept subdirectories, ascending (rsync descends into these only after
|
||||
the parent's own extras have been handled). */
|
||||
static bool delete_walk_kept_dirs(int dirfd, const char* rel_path, const DeleteDirEntry* entries,
|
||||
size_t dir_count, const DeleteWalkContext* ctx, bool deletable,
|
||||
const bool* is_extra, const bool* shielded,
|
||||
bool* local_survives) {
|
||||
bool ok = true;
|
||||
for (size_t i = dir_count; i-- > 0;) {
|
||||
if (is_extra[i] || shielded[i])
|
||||
continue;
|
||||
char* child_rel = path_cat((char*)rel_path, entries[i].name);
|
||||
if (!child_rel) {
|
||||
ok = false;
|
||||
continue;
|
||||
}
|
||||
bool child_all_removed = false;
|
||||
if (!delete_walk_child(dirfd, entries[i].name, child_rel, ctx, deletable, &child_all_removed))
|
||||
ok = false;
|
||||
/* A kept/synchronized directory is never removed. */
|
||||
*local_survives = true;
|
||||
free(child_rel);
|
||||
}
|
||||
return ok;
|
||||
}
|
||||
|
||||
/* Remove the extras directly inside the directory open on `dirfd` (DELETE mode)
|
||||
or record the paths that WOULD be removed (LIST mode), recursing into every
|
||||
child directory so kept content below a synchronized prefix is reached.
|
||||
@@ -194,11 +467,8 @@ typedef struct {
|
||||
descending name order, then extraneous files, then kept subdirectories in
|
||||
ascending order) rather than readdir() order, so `--max-delete` leaves the
|
||||
same survivors and the `--info=del`/dry-run line order matches rsync. */
|
||||
static bool delete_walk_fd(int dirfd, const char* rel_path, const PathIndex* keep,
|
||||
const PathIndex* dirs, DeleteWalkState* state,
|
||||
const DeleteSkipEntry* skips, int skip_count,
|
||||
const FilterRuleList* protect_rules, bool parent_deletable,
|
||||
bool* all_removed) {
|
||||
static bool delete_walk_fd(int dirfd, const char* rel_path, const DeleteWalkContext* ctx,
|
||||
bool parent_deletable, bool* all_removed) {
|
||||
DeleteDirEntry* entries = NULL;
|
||||
size_t count = 0;
|
||||
bool collect_ok = true;
|
||||
@@ -217,7 +487,7 @@ static bool delete_walk_fd(int dirfd, const char* rel_path, const PathIndex* kee
|
||||
/* A directory is deletable when it or ANY ancestor is synchronized; the
|
||||
`parent_deletable` flag carries that down the recursion so dest-only
|
||||
directories below a synchronized root are removed wholesale. */
|
||||
bool deletable = parent_deletable || is_synced_dir(dirs, rel_path);
|
||||
bool deletable = parent_deletable || is_synced_dir(ctx->dirs, rel_path);
|
||||
bool at_root = rel_path[0] == '\0';
|
||||
|
||||
/* Reproduce rsync's traversal order: extraneous subdirectories in descending
|
||||
@@ -230,182 +500,18 @@ static bool delete_walk_fd(int dirfd, const char* rel_path, const PathIndex* kee
|
||||
while (dir_count < count && entries[dir_count].is_dir)
|
||||
dir_count++;
|
||||
|
||||
/* Classify every entry up front (the verdict does not depend on processing
|
||||
order) so the ordered passes below can act on it. */
|
||||
for (size_t i = 0; i < count; i++) {
|
||||
char* child_rel = path_cat((char*)rel_path, entries[i].name);
|
||||
if (!child_rel) {
|
||||
if (!delete_walk_classify(rel_path, entries, count, ctx, deletable, at_root, shielded, is_extra,
|
||||
&local_survives))
|
||||
operation_ok = false;
|
||||
continue;
|
||||
}
|
||||
/* A --delay-updates run keeps its staging directory as a direct child of
|
||||
the receive root, and basis-dir snapshots live below it too. Their
|
||||
contents are not manifest entries, so descending into them would delete
|
||||
every staged / basis file as an "extra". Only the staging name (a
|
||||
top-level-only prefix) and the basis prefixes are protected: a nested
|
||||
destination directory that happens to be called .fastsync-stage is
|
||||
ordinary content. */
|
||||
if (path_under_skip_prefix(child_rel, at_root, skips, skip_count)) {
|
||||
shielded[i] = true;
|
||||
local_survives = true;
|
||||
} else if (protect_rules &&
|
||||
filter_rules_apply_side(protect_rules, child_rel, entries[i].name, entries[i].is_dir,
|
||||
FILTER_SIDE_RECEIVER) == FILTER_ACTION_PROTECT) {
|
||||
/* A first-match protect rule shields the extra; for a directory the whole
|
||||
subtree is shielded (rsync prunes an excluded directory), so do not
|
||||
descend. */
|
||||
shielded[i] = true;
|
||||
local_survives = true;
|
||||
} else if (entries[i].is_dir) {
|
||||
bool child_synced = dirs && path_index_contains(dirs, child_rel);
|
||||
is_extra[i] = deletable && !child_synced && !keep_is_dir(keep, child_rel);
|
||||
if (!is_extra[i])
|
||||
local_survives = true;
|
||||
} else {
|
||||
is_extra[i] = deletable && !keep_is_file(keep, child_rel);
|
||||
if (!is_extra[i])
|
||||
local_survives = true;
|
||||
}
|
||||
free(child_rel);
|
||||
}
|
||||
|
||||
/* Pass 1: extraneous subdirectories, descending. */
|
||||
for (size_t i = 0; i < dir_count; i++) {
|
||||
if (!is_extra[i])
|
||||
continue;
|
||||
char* child_rel = path_cat((char*)rel_path, entries[i].name);
|
||||
if (!child_rel) {
|
||||
if (!delete_walk_extra_dirs(dirfd, rel_path, entries, dir_count, ctx, deletable, is_extra,
|
||||
&local_survives))
|
||||
operation_ok = false;
|
||||
continue;
|
||||
}
|
||||
int childfd = openat(dirfd, entries[i].name, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||
bool child_all_removed = false;
|
||||
if (childfd >= 0) {
|
||||
if (!delete_walk_fd(childfd, child_rel, keep, dirs, state, skips, skip_count, protect_rules,
|
||||
deletable, &child_all_removed))
|
||||
if (!delete_walk_extra_files(dirfd, rel_path, entries, dir_count, count, ctx, is_extra,
|
||||
&local_survives))
|
||||
operation_ok = false;
|
||||
close(childfd);
|
||||
} else if (errno != ENOENT) {
|
||||
if (!delete_walk_kept_dirs(dirfd, rel_path, entries, dir_count, ctx, deletable, is_extra,
|
||||
shielded, &local_survives))
|
||||
operation_ok = false;
|
||||
}
|
||||
if (child_all_removed && deletable) {
|
||||
if (state->mode == DELETE_WALK_MODE_LIST) {
|
||||
/* Record the directory with rsync's trailing slash. */
|
||||
size_t len = strlen(child_rel);
|
||||
char* copy = malloc(len + 2);
|
||||
if (!copy) {
|
||||
operation_ok = false;
|
||||
} else {
|
||||
memcpy(copy, child_rel, len);
|
||||
copy[len] = '/';
|
||||
copy[len + 1] = '\0';
|
||||
if (!array_list_add(state->out, copy)) {
|
||||
free(copy);
|
||||
operation_ok = false;
|
||||
} else {
|
||||
(*state->recorded)++;
|
||||
}
|
||||
}
|
||||
} else if (state->budget->deleted >= state->budget->max_delete) {
|
||||
state->budget->limit_hit = true;
|
||||
state->budget->skipped++;
|
||||
local_survives = true;
|
||||
} else if (unlinkat(dirfd, entries[i].name, AT_REMOVEDIR) != 0) {
|
||||
/* ENOENT: already gone (fine). ENOTEMPTY/EEXIST: the directory still
|
||||
holds entries the walker leaves in place (a protected excluded
|
||||
prefix, a kept file the manifest protects, a symlink); rsync leaves
|
||||
such a directory behind, so this is not an error. Only genuine I/O
|
||||
failures abort the deletion. */
|
||||
if (errno != ENOENT && errno != ENOTEMPTY && errno != EEXIST)
|
||||
operation_ok = false;
|
||||
local_survives = true;
|
||||
} else {
|
||||
state->budget->deleted++;
|
||||
/* rsync reports a removed directory with a trailing slash. */
|
||||
if (state->observer) {
|
||||
size_t len = strlen(child_rel);
|
||||
char* with_slash = malloc(len + 2);
|
||||
if (with_slash) {
|
||||
memcpy(with_slash, child_rel, len);
|
||||
with_slash[len] = '/';
|
||||
with_slash[len + 1] = '\0';
|
||||
state->observer(state->observer_context, with_slash);
|
||||
free(with_slash);
|
||||
} else {
|
||||
state->observer(state->observer_context, child_rel);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
local_survives = true;
|
||||
}
|
||||
free(child_rel);
|
||||
}
|
||||
|
||||
/* Pass 2: extraneous files, descending. */
|
||||
for (size_t i = dir_count; i < count; i++) {
|
||||
if (!is_extra[i])
|
||||
continue;
|
||||
if (state->mode == DELETE_WALK_MODE_LIST) {
|
||||
char* child_rel = path_cat((char*)rel_path, entries[i].name);
|
||||
if (!child_rel) {
|
||||
operation_ok = false;
|
||||
continue;
|
||||
}
|
||||
char* copy = str_dup(child_rel);
|
||||
if (!copy || !array_list_add(state->out, copy)) {
|
||||
free(copy);
|
||||
operation_ok = false;
|
||||
} else {
|
||||
(*state->recorded)++;
|
||||
}
|
||||
free(child_rel);
|
||||
} else if (state->budget->deleted >= state->budget->max_delete) {
|
||||
state->budget->limit_hit = true;
|
||||
state->budget->skipped++;
|
||||
local_survives = true;
|
||||
} else if (unlinkat(dirfd, entries[i].name, 0) != 0) {
|
||||
if (errno != ENOENT)
|
||||
operation_ok = false;
|
||||
local_survives = true;
|
||||
} else {
|
||||
state->budget->deleted++;
|
||||
char* child_rel = path_cat((char*)rel_path, entries[i].name);
|
||||
if (child_rel) {
|
||||
if (state->observer)
|
||||
state->observer(state->observer_context, child_rel);
|
||||
char* escaped_path = output_escape(child_rel, log_get_8_bit_output());
|
||||
fprintf(stderr, " Deleted: %s\n", escaped_path ? escaped_path : "<allocation failed>");
|
||||
free(escaped_path);
|
||||
}
|
||||
free(child_rel);
|
||||
}
|
||||
}
|
||||
|
||||
/* Pass 3: kept subdirectories, ascending (rsync descends into these only
|
||||
after the parent's own extras have been handled). */
|
||||
for (size_t i = dir_count; i-- > 0;) {
|
||||
if (is_extra[i] || shielded[i])
|
||||
continue;
|
||||
char* child_rel = path_cat((char*)rel_path, entries[i].name);
|
||||
if (!child_rel) {
|
||||
operation_ok = false;
|
||||
continue;
|
||||
}
|
||||
int childfd = openat(dirfd, entries[i].name, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC);
|
||||
bool child_all_removed = false;
|
||||
if (childfd >= 0) {
|
||||
if (!delete_walk_fd(childfd, child_rel, keep, dirs, state, skips, skip_count, protect_rules,
|
||||
deletable, &child_all_removed))
|
||||
operation_ok = false;
|
||||
close(childfd);
|
||||
} else if (errno != ENOENT) {
|
||||
operation_ok = false;
|
||||
}
|
||||
/* A kept/synchronized directory is never removed. */
|
||||
local_survives = true;
|
||||
free(child_rel);
|
||||
}
|
||||
|
||||
free(shielded);
|
||||
free(is_extra);
|
||||
@@ -430,7 +536,7 @@ static int open_destination_root(const char* dest_root) {
|
||||
|
||||
bool delete_extras_list(const char* dest_root, const ArrayList* manifest,
|
||||
const ArrayList* synced_dirs, const DeleteSkipEntry* skips, int skip_count,
|
||||
const FilterRuleList* protect_rules, ArrayList* out, size_t* count_out) {
|
||||
const DeleteProtectRules* protect, ArrayList* out, size_t* count_out) {
|
||||
if (count_out)
|
||||
*count_out = 0;
|
||||
if (!manifest || !out)
|
||||
@@ -460,8 +566,13 @@ bool delete_extras_list(const char* dest_root, const ArrayList* manifest,
|
||||
.recorded = &recorded,
|
||||
.observer = NULL,
|
||||
.observer_context = NULL};
|
||||
bool ok = delete_walk_fd(rootfd, "", &keep, have_dirs ? &dirs : NULL, &state, skips, skip_count,
|
||||
protect_rules, false, &all_removed);
|
||||
DeleteWalkContext ctx = {.keep = &keep,
|
||||
.dirs = have_dirs ? &dirs : NULL,
|
||||
.state = &state,
|
||||
.skips = skips,
|
||||
.skip_count = skip_count,
|
||||
.protect = protect};
|
||||
bool ok = delete_walk_fd(rootfd, "", &ctx, false, &all_removed);
|
||||
if (close(rootfd) != 0)
|
||||
ok = false;
|
||||
path_index_free(&keep);
|
||||
@@ -475,7 +586,7 @@ bool delete_extras_list(const char* dest_root, const ArrayList* manifest,
|
||||
DeleteWalkResult delete_extras_limited_observed(const char* dest_root, const ArrayList* manifest,
|
||||
const ArrayList* synced_dirs, size_t max_delete,
|
||||
const DeleteSkipEntry* skips, int skip_count,
|
||||
const FilterRuleList* protect_rules,
|
||||
const DeleteProtectRules* protect,
|
||||
size_t* deleted_out, size_t* skipped_out,
|
||||
DeletePathObserver observer,
|
||||
void* observer_context) {
|
||||
@@ -513,8 +624,13 @@ DeleteWalkResult delete_extras_limited_observed(const char* dest_root, const Arr
|
||||
.recorded = NULL,
|
||||
.observer = observer,
|
||||
.observer_context = observer_context};
|
||||
bool ok = delete_walk_fd(rootfd, "", &keep, have_dirs ? &dirs : NULL, &state, skips, skip_count,
|
||||
protect_rules, false, &all_removed);
|
||||
DeleteWalkContext ctx = {.keep = &keep,
|
||||
.dirs = have_dirs ? &dirs : NULL,
|
||||
.state = &state,
|
||||
.skips = skips,
|
||||
.skip_count = skip_count,
|
||||
.protect = protect};
|
||||
bool ok = delete_walk_fd(rootfd, "", &ctx, false, &all_removed);
|
||||
if (close(rootfd) != 0)
|
||||
ok = false;
|
||||
path_index_free(&keep);
|
||||
@@ -532,11 +648,10 @@ DeleteWalkResult delete_extras_limited_observed(const char* dest_root, const Arr
|
||||
DeleteWalkResult delete_extras_limited(const char* dest_root, const ArrayList* manifest,
|
||||
const ArrayList* synced_dirs, size_t max_delete,
|
||||
const DeleteSkipEntry* skips, int skip_count,
|
||||
const FilterRuleList* protect_rules, size_t* deleted_out,
|
||||
const DeleteProtectRules* protect, size_t* deleted_out,
|
||||
size_t* skipped_out) {
|
||||
return delete_extras_limited_observed(dest_root, manifest, synced_dirs, max_delete, skips,
|
||||
skip_count, protect_rules, deleted_out, skipped_out, NULL,
|
||||
NULL);
|
||||
skip_count, protect, deleted_out, skipped_out, NULL, NULL);
|
||||
}
|
||||
|
||||
bool delete_extras(const char* dest_root, const ArrayList* manifest) {
|
||||
@@ -607,7 +722,13 @@ bool delete_skips_build(const Config* config, const ArrayList* protected_paths,
|
||||
}
|
||||
int idx = 0;
|
||||
if (config->delay_updates) {
|
||||
out->entries[idx].prefix = DELAY_UPDATES_STAGING_DIR;
|
||||
/* Protect this transfer's actual (per-run unique) staging directory. The
|
||||
runtime name is only known to the receiver-side context; fall back to the
|
||||
reserved prefix for a context that was never created (e.g. a dry run). */
|
||||
const char* staging_name = (config->delay_context && config->delay_context->staging_name)
|
||||
? config->delay_context->staging_name
|
||||
: DELAY_UPDATES_STAGING_DIR;
|
||||
out->entries[idx].prefix = staging_name;
|
||||
out->entries[idx].top_level_only = true;
|
||||
idx++;
|
||||
}
|
||||
|
||||
+54
-7
@@ -3,8 +3,10 @@
|
||||
|
||||
#include "array_list.h"
|
||||
#include "config.h"
|
||||
#include "filter.h"
|
||||
#include <stdbool.h>
|
||||
#include <stddef.h>
|
||||
#include <sys/stat.h>
|
||||
|
||||
/* Delete engine.
|
||||
*
|
||||
@@ -28,6 +30,34 @@ typedef enum {
|
||||
DELETE_WALK_ERROR
|
||||
} DeleteWalkResult;
|
||||
|
||||
/* Receiver-side delete-protection rules for one walk. `base_rules` is the
|
||||
* command-line rule set the config frame carried (owner "" rules); `dir_rules`
|
||||
* is the received per-directory rule set (rules carrying their owner directory
|
||||
* and no-inherit flag). Either may be NULL. */
|
||||
typedef struct {
|
||||
const FilterRuleList* base_rules;
|
||||
const FilterRuleList* dir_rules;
|
||||
/* When non-NULL and non-empty, a destination entry whose name ends with this
|
||||
suffix is protected from deletion. rsync never treats a --backup file as
|
||||
an extra, so a backup created at --delay-updates publication (or a
|
||||
pre-existing one) survives the delete-after pass. */
|
||||
const char* backup_suffix;
|
||||
} DeleteProtectRules;
|
||||
|
||||
/* rsync's first-match-wins receiver verdict for one candidate extra: the
|
||||
* per-directory chain is evaluated first (the containing directory's rules,
|
||||
* then each ancestor's, then the receive root's), then the base rules. Returns
|
||||
* FILTER_ACTION_PROTECT when the entry is shielded by a receiver-side exclude,
|
||||
* FILTER_ACTION_RISK when an include explicitly leaves it at risk, or
|
||||
* FILTER_ACTION_NONE when no rule matched. */
|
||||
FilterAction delete_protect_verdict(const DeleteProtectRules* protect, const char* rel_path,
|
||||
const char* leaf, bool is_dir);
|
||||
|
||||
/* The backup suffix the delete walker must shield from deletion, or NULL when
|
||||
--backup is inactive or the configured suffix is unusable (empty, or holding
|
||||
a path separator). Matches the suffix file_save uses for backups. */
|
||||
const char* delete_backup_suffix(const Config* config);
|
||||
|
||||
/* One protected entry for the delete walker. When top_level_only is true the
|
||||
prefix is skipped only as a DIRECT child of dest_root (the --delay-updates
|
||||
staging directory, which must not hide genuine extras inside a nested
|
||||
@@ -66,6 +96,9 @@ bool path_under_skip_prefix(const char* child_rel, bool at_root, const DeleteSki
|
||||
typedef struct {
|
||||
char* name;
|
||||
bool is_dir;
|
||||
/* The entry's full st_mode from the AT_SYMLINK_NOFOLLOW stat, so a delete
|
||||
observer can classify a removed non-directory as reg/link/special. */
|
||||
mode_t mode;
|
||||
} DeleteDirEntry;
|
||||
/* Collect the entries of the directory open on `dirfd` (excluding "." and ".."),
|
||||
stat'ing each with AT_SYMLINK_NOFOLLOW. On success *out is a malloc'd array of
|
||||
@@ -96,24 +129,38 @@ int delete_dir_entry_cmp_asc(const void* a, const void* b);
|
||||
DeleteWalkResult delete_extras_limited(const char* dest_root, const ArrayList* manifest,
|
||||
const ArrayList* synced_dirs, size_t max_delete,
|
||||
const DeleteSkipEntry* skips, int skip_count,
|
||||
const FilterRuleList* protect_rules, size_t* deleted_out,
|
||||
const DeleteProtectRules* protect, size_t* deleted_out,
|
||||
size_t* skipped_out);
|
||||
|
||||
/* Entry kind of a removed path, reported to the delete observer so the receiver
|
||||
can build rsync's `--stats` `Number of deleted files` per-type breakdown. The
|
||||
four categories are a strict partition of every removed entry. */
|
||||
typedef enum {
|
||||
DELETE_ENTRY_REG = 0,
|
||||
DELETE_ENTRY_DIR,
|
||||
DELETE_ENTRY_LINK,
|
||||
DELETE_ENTRY_SPECIAL
|
||||
} DeleteEntryType;
|
||||
|
||||
/* Optional per-deletion observer: called for each destination-relative path
|
||||
actually removed (a file, symlink, or directory), in removal order, so the
|
||||
receiver can stream rsync's `--info=del`/`--info=remove` lines. */
|
||||
typedef void (*DeletePathObserver)(void* context, const char* rel_path);
|
||||
actually removed (a file, symlink, or directory) with its entry kind, in
|
||||
removal order, so the receiver can stream rsync's `--info=del`/`--info=remove`
|
||||
lines and tally the per-type `--stats` counters. */
|
||||
typedef void (*DeletePathObserver)(void* context, const char* rel_path, DeleteEntryType type);
|
||||
|
||||
/* Classify a removed entry from its st_mode for the per-type delete counters. */
|
||||
DeleteEntryType delete_entry_type_of_mode(mode_t mode);
|
||||
|
||||
/* `delete_extras_limited_observed` is delete_extras_limited with an optional
|
||||
* observer; the observer is invoked only for entries truly removed. When
|
||||
* `protect_rules` is non-NULL its receiver-side verdict is evaluated for every
|
||||
* `protect` is non-NULL its receiver-side verdict is evaluated for every
|
||||
* candidate extra: a first-match PROTECT leaves the entry (and, for a
|
||||
* directory, its whole subtree) in place, while RISK/NONE fall through to the
|
||||
* ordinary skip-prefix/keep-set logic. */
|
||||
DeleteWalkResult delete_extras_limited_observed(const char* dest_root, const ArrayList* manifest,
|
||||
const ArrayList* synced_dirs, size_t max_delete,
|
||||
const DeleteSkipEntry* skips, int skip_count,
|
||||
const FilterRuleList* protect_rules,
|
||||
const DeleteProtectRules* protect,
|
||||
size_t* deleted_out, size_t* skipped_out,
|
||||
DeletePathObserver observer,
|
||||
void* observer_context);
|
||||
@@ -124,7 +171,7 @@ DeleteWalkResult delete_extras_limited_observed(const char* dest_root, const Arr
|
||||
strings appended to `out` and receives their count in *count_out. */
|
||||
bool delete_extras_list(const char* dest_root, const ArrayList* manifest,
|
||||
const ArrayList* synced_dirs, const DeleteSkipEntry* skips, int skip_count,
|
||||
const FilterRuleList* protect_rules, ArrayList* out, size_t* count_out);
|
||||
const DeleteProtectRules* protect, ArrayList* out, size_t* count_out);
|
||||
bool delete_extras(const char* dest_root, const ArrayList* manifest);
|
||||
|
||||
/* Build the delete walk's skip-prefix set from the config's --delay-updates
|
||||
|
||||
+181
-139
@@ -19,6 +19,7 @@
|
||||
#include "data.h"
|
||||
#include "delay_updates.h"
|
||||
#include "delete_commit.h"
|
||||
#include "delete_plan.h"
|
||||
#include "delta.h"
|
||||
#include "file.h"
|
||||
#include "format.h"
|
||||
@@ -102,6 +103,13 @@ DeleteManifest* receive_manifest_entries(int fd) {
|
||||
delete_manifest_free(manifest);
|
||||
return NULL;
|
||||
}
|
||||
/* Per-directory filter rules (protocol 2.30.0) follow the manifest sections
|
||||
* with their own bounded self-describing format. */
|
||||
if (!delete_filter_dir_rules_receive(fd, &manifest->per_dir_rules)) {
|
||||
delete_manifest_free(manifest);
|
||||
send_status(fd, STATUS_ERROR);
|
||||
return NULL;
|
||||
}
|
||||
return manifest;
|
||||
}
|
||||
|
||||
@@ -112,6 +120,7 @@ void delete_manifest_free(DeleteManifest* manifest) {
|
||||
array_list_delete(manifest->protected);
|
||||
array_list_delete(manifest->missing);
|
||||
array_list_delete(manifest->dirs);
|
||||
filter_rule_list_free(manifest->per_dir_rules);
|
||||
free(manifest);
|
||||
}
|
||||
|
||||
@@ -160,9 +169,12 @@ static bool delete_extras_budgeted_observed(const Config* config, const DeleteMa
|
||||
remaining = budget->max_delete - budget->deleted;
|
||||
size_t deleted = 0;
|
||||
size_t skipped = 0;
|
||||
DeleteProtectRules protect = {.base_rules = config->protect_rules,
|
||||
.dir_rules = manifest->per_dir_rules,
|
||||
.backup_suffix = delete_backup_suffix(config)};
|
||||
DeleteWalkResult result = delete_extras_limited_observed(
|
||||
config->receive_root_directory, manifest->keeps, manifest->dirs, remaining, skips.entries,
|
||||
skips.count, config->protect_rules, &deleted, &skipped, observer, observer_context);
|
||||
skips.count, &protect, &deleted, &skipped, observer, observer_context);
|
||||
delete_skips_free(&skips);
|
||||
budget->deleted += deleted;
|
||||
budget->skipped += skipped;
|
||||
@@ -191,13 +203,13 @@ typedef struct {
|
||||
const char* prefix;
|
||||
} PrefixedDeleteObserver;
|
||||
|
||||
static void prefixed_delete_observer(void* context, const char* rel) {
|
||||
static void prefixed_delete_observer(void* context, const char* rel, DeleteEntryType type) {
|
||||
PrefixedDeleteObserver* prefixed = context;
|
||||
if (!prefixed->inner || !rel)
|
||||
return;
|
||||
char* joined = path_cat((char*)prefixed->prefix, rel);
|
||||
if (joined) {
|
||||
prefixed->inner(prefixed->inner_context, joined);
|
||||
prefixed->inner(prefixed->inner_context, joined, type);
|
||||
free(joined);
|
||||
}
|
||||
}
|
||||
@@ -215,6 +227,166 @@ static void prefixed_delete_observer(void* context, const char* rel) {
|
||||
--max-delete budget: once it is exhausted the remaining requests are skipped
|
||||
and counted. Returns false only on a genuine error (a confinement failure on
|
||||
a validated path or an I/O error), which fails the run. */
|
||||
|
||||
/* How one missing-args request leaves the driver loop. The original walker
|
||||
`continue`s past an invalid/protected/absent/budget-skipped request (without
|
||||
breaking) but stops after a request that ran to completion while an error is
|
||||
pending; NEXT/STOP preserve that control flow exactly. */
|
||||
typedef enum { MISSING_ARG_NEXT, MISSING_ARG_STOP } MissingArgStep;
|
||||
|
||||
/* Remove a NON-empty missing-args directory recursively (--delete/--force in
|
||||
effect): walk its contents through the budgeted extras walker so every removed
|
||||
file/dir counts toward --max-delete (rsync parity), then remove the now-empty
|
||||
directory itself, which costs one more budget unit. A run that hits the cap
|
||||
leaves the remaining entries in place. The observer is wrapped so the nested
|
||||
walk reports receive-root-relative paths. Sets the *removed and *ok outputs. */
|
||||
static void delete_nonempty_missing_dir(const char* full, const char* rel,
|
||||
DeleteBudgetState* budget, DeletePathObserver observer,
|
||||
void* observer_context, bool* removed, bool* ok) {
|
||||
ArrayList* no_keeps = array_list_create(free);
|
||||
/* Never let an accounting slip (deleted > max_delete) underflow the remaining
|
||||
budget into SIZE_MAX, which would grant unlimited deletions. */
|
||||
size_t remaining =
|
||||
budget->deleted >= budget->max_delete ? 0 : budget->max_delete - budget->deleted;
|
||||
size_t contents_deleted = 0;
|
||||
size_t contents_skipped = 0;
|
||||
PrefixedDeleteObserver nested = {observer, observer_context, rel};
|
||||
DeleteWalkResult walk =
|
||||
no_keeps ? delete_extras_limited_observed(full, no_keeps, NULL, remaining, NULL, 0, NULL,
|
||||
&contents_deleted, &contents_skipped,
|
||||
observer ? prefixed_delete_observer : NULL,
|
||||
observer ? &nested : NULL)
|
||||
: DELETE_WALK_ERROR;
|
||||
if (no_keeps)
|
||||
array_list_delete(no_keeps);
|
||||
budget->deleted += contents_deleted;
|
||||
budget->skipped += contents_skipped;
|
||||
if (walk == DELETE_WALK_LIMIT_REACHED) {
|
||||
budget->limit_hit = true;
|
||||
} else if (walk != DELETE_WALK_OK) {
|
||||
*ok = false;
|
||||
} else if (budget->deleted >= budget->max_delete) {
|
||||
budget->limit_hit = true;
|
||||
budget->skipped++;
|
||||
} else if (file_remove_tree_secure(full)) {
|
||||
/* The shared `if (removed)` tail charges this directory exactly once;
|
||||
counting it here too would consume two budget units. */
|
||||
*removed = true;
|
||||
} else {
|
||||
*ok = false;
|
||||
}
|
||||
}
|
||||
|
||||
/* Remove one missing-args destination mirror. `skips` holds the receiver
|
||||
artifacts (staging directory, basis snapshots) that stay protected. Returns
|
||||
MISSING_ARG_STOP when the driver loop must stop (a completed removal left a
|
||||
genuine error pending) and MISSING_ARG_NEXT otherwise; *ok accumulates the
|
||||
overall success across the whole run. */
|
||||
static MissingArgStep delete_one_missing_arg(const Config* config, const char* rel,
|
||||
const DeleteSkipSet* skips, DeleteBudgetState* budget,
|
||||
DeletePathObserver observer, void* observer_context,
|
||||
bool* ok) {
|
||||
if (!rel || *rel == '\0' || *rel == '/' || has_path_traversal(rel)) {
|
||||
/* Defensive only: receive_manifest_entries already validated every
|
||||
section identically, so a controlled peer never reaches this branch. */
|
||||
log_message(LOG_LEVEL_ERROR, "invalid missing-args delete path");
|
||||
*ok = false;
|
||||
return MISSING_ARG_NEXT;
|
||||
}
|
||||
bool at_root = strchr(rel, '/') == NULL;
|
||||
if (path_under_skip_prefix(rel, at_root, skips->entries, skips->count)) {
|
||||
char* escaped = output_escape(rel, log_get_8_bit_output());
|
||||
log_message(LOG_LEVEL_WARNING,
|
||||
"missing-args path '%s' is protected (staging directory or basis snapshot); "
|
||||
"not deleting",
|
||||
escaped ? escaped : "<allocation failed>");
|
||||
free(escaped);
|
||||
return MISSING_ARG_NEXT;
|
||||
}
|
||||
char* full = path_cat(config->receive_root_directory, rel);
|
||||
if (!full) {
|
||||
*ok = false;
|
||||
return MISSING_ARG_NEXT;
|
||||
}
|
||||
char* leaf = NULL;
|
||||
int parent_fd = file_open_secure_parent(full, &leaf, false);
|
||||
if (parent_fd < 0) {
|
||||
/* The mirror's parent directory may itself not exist on the destination
|
||||
(a deeper missing entry whose leading directories were never created).
|
||||
That is a no-op -- there is nothing to delete -- matching
|
||||
file_remove_tree_secure's absent-path handling; only a genuine I/O
|
||||
error (EACCES, a symlink loop, ...) fails the run. */
|
||||
bool absent = errno == ENOENT || errno == ENOTDIR;
|
||||
free(full);
|
||||
free(leaf);
|
||||
if (!absent)
|
||||
*ok = false;
|
||||
return MISSING_ARG_NEXT;
|
||||
}
|
||||
struct stat st;
|
||||
if (fstatat(parent_fd, leaf, &st, AT_SYMLINK_NOFOLLOW) != 0) {
|
||||
/* Already absent: nothing to delete (a no-op, not a deletion). */
|
||||
if (errno != ENOENT)
|
||||
*ok = false;
|
||||
close(parent_fd);
|
||||
free(leaf);
|
||||
free(full);
|
||||
return MISSING_ARG_NEXT;
|
||||
}
|
||||
/* An entry that exists is one deletion: skip it (and count it) when the
|
||||
shared --max-delete budget is already exhausted. */
|
||||
if (budget->deleted >= budget->max_delete) {
|
||||
budget->limit_hit = true;
|
||||
budget->skipped++;
|
||||
close(parent_fd);
|
||||
free(leaf);
|
||||
free(full);
|
||||
return MISSING_ARG_NEXT;
|
||||
}
|
||||
bool removed = false;
|
||||
if (S_ISDIR(st.st_mode)) {
|
||||
if (unlinkat(parent_fd, leaf, AT_REMOVEDIR) == 0) {
|
||||
removed = true;
|
||||
} else if (errno == ENOTEMPTY || errno == EEXIST) {
|
||||
close(parent_fd);
|
||||
parent_fd = -1;
|
||||
free(leaf);
|
||||
leaf = NULL;
|
||||
if (config->use_delete || config->force_delete) {
|
||||
delete_nonempty_missing_dir(full, rel, budget, observer, observer_context, &removed, ok);
|
||||
} else {
|
||||
char* escaped = output_escape(rel, log_get_8_bit_output());
|
||||
log_message(LOG_LEVEL_WARNING,
|
||||
"missing-args destination '%s' is a non-empty directory; use --force or "
|
||||
"--delete to remove it",
|
||||
escaped ? escaped : "<allocation failed>");
|
||||
free(escaped);
|
||||
}
|
||||
} else if (errno != ENOENT) {
|
||||
*ok = false;
|
||||
}
|
||||
} else {
|
||||
if (unlinkat(parent_fd, leaf, 0) == 0) {
|
||||
removed = true;
|
||||
} else if (errno != ENOENT) {
|
||||
*ok = false;
|
||||
}
|
||||
}
|
||||
if (removed) {
|
||||
budget->deleted++;
|
||||
if (observer)
|
||||
observer(observer_context, rel, delete_entry_type_of_mode(st.st_mode));
|
||||
char* escaped = output_escape(rel, log_get_8_bit_output());
|
||||
fprintf(stderr, " Deleted: %s\n", escaped ? escaped : "<allocation failed>");
|
||||
free(escaped);
|
||||
}
|
||||
if (parent_fd >= 0)
|
||||
close(parent_fd);
|
||||
free(leaf);
|
||||
free(full);
|
||||
return *ok ? MISSING_ARG_NEXT : MISSING_ARG_STOP;
|
||||
}
|
||||
|
||||
static bool delete_missing_args_budgeted_observed(const Config* config,
|
||||
const DeleteManifest* manifest,
|
||||
DeleteBudgetState* budget,
|
||||
@@ -234,141 +406,8 @@ static bool delete_missing_args_budgeted_observed(const Config* config,
|
||||
bool ok = true;
|
||||
for (int i = 0; i < manifest->missing->size; i++) {
|
||||
const char* rel = (const char*)manifest->missing->items[i];
|
||||
if (!rel || *rel == '\0' || *rel == '/' || has_path_traversal(rel)) {
|
||||
/* Defensive only: receive_manifest_entries already validated every
|
||||
section identically, so a controlled peer never reaches this branch. */
|
||||
log_message(LOG_LEVEL_ERROR, "invalid missing-args delete path");
|
||||
ok = false;
|
||||
continue;
|
||||
}
|
||||
bool at_root = strchr(rel, '/') == NULL;
|
||||
if (path_under_skip_prefix(rel, at_root, skips.entries, skips.count)) {
|
||||
char* escaped = output_escape(rel, log_get_8_bit_output());
|
||||
log_message(LOG_LEVEL_WARNING,
|
||||
"missing-args path '%s' is protected (staging directory or basis snapshot); "
|
||||
"not deleting",
|
||||
escaped ? escaped : "<allocation failed>");
|
||||
free(escaped);
|
||||
continue;
|
||||
}
|
||||
char* full = path_cat(config->receive_root_directory, rel);
|
||||
if (!full) {
|
||||
ok = false;
|
||||
continue;
|
||||
}
|
||||
char* leaf = NULL;
|
||||
int parent_fd = file_open_secure_parent(full, &leaf, false);
|
||||
if (parent_fd < 0) {
|
||||
/* The mirror's parent directory may itself not exist on the destination
|
||||
(a deeper missing entry whose leading directories were never created).
|
||||
That is a no-op -- there is nothing to delete -- matching
|
||||
file_remove_tree_secure's absent-path handling; only a genuine I/O
|
||||
error (EACCES, a symlink loop, ...) fails the run. */
|
||||
bool absent = errno == ENOENT || errno == ENOTDIR;
|
||||
free(full);
|
||||
free(leaf);
|
||||
if (!absent)
|
||||
ok = false;
|
||||
continue;
|
||||
}
|
||||
struct stat st;
|
||||
if (fstatat(parent_fd, leaf, &st, AT_SYMLINK_NOFOLLOW) != 0) {
|
||||
/* Already absent: nothing to delete (a no-op, not a deletion). */
|
||||
if (errno != ENOENT)
|
||||
ok = false;
|
||||
close(parent_fd);
|
||||
free(leaf);
|
||||
free(full);
|
||||
continue;
|
||||
}
|
||||
/* An entry that exists is one deletion: skip it (and count it) when the
|
||||
shared --max-delete budget is already exhausted. */
|
||||
if (budget->deleted >= budget->max_delete) {
|
||||
budget->limit_hit = true;
|
||||
budget->skipped++;
|
||||
close(parent_fd);
|
||||
free(leaf);
|
||||
free(full);
|
||||
continue;
|
||||
}
|
||||
bool removed = false;
|
||||
if (S_ISDIR(st.st_mode)) {
|
||||
if (unlinkat(parent_fd, leaf, AT_REMOVEDIR) == 0) {
|
||||
removed = true;
|
||||
} else if (errno == ENOTEMPTY || errno == EEXIST) {
|
||||
close(parent_fd);
|
||||
parent_fd = -1;
|
||||
free(leaf);
|
||||
leaf = NULL;
|
||||
if (config->use_delete || config->force_delete) {
|
||||
/* Remove the contents entry-by-entry through the budgeted extras
|
||||
walker so every deleted file/dir counts toward --max-delete (rsync
|
||||
parity); the now-empty directory itself costs one more. A run that
|
||||
hits the cap leaves the remaining entries in place. */
|
||||
ArrayList* no_keeps = array_list_create(free);
|
||||
/* Never let an accounting slip (deleted > max_delete) underflow the
|
||||
remaining budget into SIZE_MAX, which would grant unlimited
|
||||
deletions. */
|
||||
size_t remaining =
|
||||
budget->deleted >= budget->max_delete ? 0 : budget->max_delete - budget->deleted;
|
||||
size_t contents_deleted = 0;
|
||||
size_t contents_skipped = 0;
|
||||
PrefixedDeleteObserver nested = {observer, observer_context, rel};
|
||||
DeleteWalkResult walk =
|
||||
no_keeps ? delete_extras_limited_observed(full, no_keeps, NULL, remaining, NULL, 0,
|
||||
NULL, &contents_deleted, &contents_skipped,
|
||||
observer ? prefixed_delete_observer : NULL,
|
||||
observer ? &nested : NULL)
|
||||
: DELETE_WALK_ERROR;
|
||||
if (no_keeps)
|
||||
array_list_delete(no_keeps);
|
||||
budget->deleted += contents_deleted;
|
||||
budget->skipped += contents_skipped;
|
||||
if (walk == DELETE_WALK_LIMIT_REACHED) {
|
||||
budget->limit_hit = true;
|
||||
} else if (walk != DELETE_WALK_OK) {
|
||||
ok = false;
|
||||
} else if (budget->deleted >= budget->max_delete) {
|
||||
budget->limit_hit = true;
|
||||
budget->skipped++;
|
||||
} else if (file_remove_tree_secure(full)) {
|
||||
/* The shared `if (removed)` tail charges this directory exactly
|
||||
once; counting it here too would consume two budget units. */
|
||||
removed = true;
|
||||
} else {
|
||||
ok = false;
|
||||
}
|
||||
} else {
|
||||
char* escaped = output_escape(rel, log_get_8_bit_output());
|
||||
log_message(LOG_LEVEL_WARNING,
|
||||
"missing-args destination '%s' is a non-empty directory; use --force or "
|
||||
"--delete to remove it",
|
||||
escaped ? escaped : "<allocation failed>");
|
||||
free(escaped);
|
||||
}
|
||||
} else if (errno != ENOENT) {
|
||||
ok = false;
|
||||
}
|
||||
} else {
|
||||
if (unlinkat(parent_fd, leaf, 0) == 0) {
|
||||
removed = true;
|
||||
} else if (errno != ENOENT) {
|
||||
ok = false;
|
||||
}
|
||||
}
|
||||
if (removed) {
|
||||
budget->deleted++;
|
||||
if (observer)
|
||||
observer(observer_context, rel);
|
||||
char* escaped = output_escape(rel, log_get_8_bit_output());
|
||||
fprintf(stderr, " Deleted: %s\n", escaped ? escaped : "<allocation failed>");
|
||||
free(escaped);
|
||||
}
|
||||
if (parent_fd >= 0)
|
||||
close(parent_fd);
|
||||
free(leaf);
|
||||
free(full);
|
||||
if (!ok)
|
||||
if (delete_one_missing_arg(config, rel, &skips, budget, observer, observer_context, &ok) ==
|
||||
MISSING_ARG_STOP)
|
||||
break;
|
||||
}
|
||||
delete_skips_free(&skips);
|
||||
@@ -386,8 +425,11 @@ bool manifest_would_delete_list(const Config* config, const DeleteManifest* mani
|
||||
DeleteSkipSet skips;
|
||||
if (!delete_skips_build(config, manifest->protected, NULL, true, &skips))
|
||||
return false;
|
||||
DeleteProtectRules protect = {.base_rules = config->protect_rules,
|
||||
.dir_rules = manifest->per_dir_rules,
|
||||
.backup_suffix = delete_backup_suffix(config)};
|
||||
bool ok = delete_extras_list(config->receive_root_directory, manifest->keeps, manifest->dirs,
|
||||
skips.entries, skips.count, config->protect_rules, out, count_out);
|
||||
skips.entries, skips.count, &protect, out, count_out);
|
||||
delete_skips_free(&skips);
|
||||
return ok;
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
#include "array_list.h"
|
||||
#include "config.h"
|
||||
#include "delete.h"
|
||||
#include "filter.h"
|
||||
#include <stdbool.h>
|
||||
|
||||
/* Delete-commit module: delete-manifest receive plus the budgeted extras and
|
||||
@@ -30,6 +31,13 @@ typedef struct DeleteManifest {
|
||||
leave untransmitted directories and the unlisted parts of listed ones
|
||||
alone, matching rsync's "delete only in synchronized directories". */
|
||||
ArrayList* dirs;
|
||||
/* Per-directory filter rules the sender compiled while scanning (protocol
|
||||
2.30.0), each carrying its owner directory and no-inherit flag. The
|
||||
receiver evaluates them (deepest before ancestors, then the command-line
|
||||
base rules) against every candidate extra so a destination-only entry that
|
||||
matches ONLY a per-directory `.rsync-filter`/dir-merge rule is shielded.
|
||||
NULL when the sender transmitted none. */
|
||||
FilterRuleList* per_dir_rules;
|
||||
} DeleteManifest;
|
||||
|
||||
void delete_manifest_free(DeleteManifest* manifest);
|
||||
|
||||
+209
-21
@@ -48,6 +48,7 @@ struct DeletePlanSender {
|
||||
const ArrayList* protected_prefixes;
|
||||
const ArrayList* size_skipped;
|
||||
const ArrayList* missing_args;
|
||||
const FilterRuleList* per_dir_rules;
|
||||
size_t entries;
|
||||
/* Transmitted FILE entries only. The caller's "empty scan" safety guard keys
|
||||
off this (an I/O error that hid every file must refuse to delete even when
|
||||
@@ -284,12 +285,14 @@ bool delete_plan_sender_empty(const DeletePlanSender* sender) {
|
||||
}
|
||||
|
||||
void delete_plan_sender_set_config(DeletePlanSender* sender, const ArrayList* protected_prefixes,
|
||||
const ArrayList* size_skipped, const ArrayList* missing_args) {
|
||||
const ArrayList* size_skipped, const ArrayList* missing_args,
|
||||
const FilterRuleList* per_dir_rules) {
|
||||
if (!sender)
|
||||
return;
|
||||
sender->protected_prefixes = protected_prefixes;
|
||||
sender->size_skipped = size_skipped;
|
||||
sender->missing_args = missing_args;
|
||||
sender->per_dir_rules = per_dir_rules;
|
||||
}
|
||||
|
||||
/* True when `dir` is `root` itself or a descendant of it (path-component
|
||||
@@ -320,6 +323,181 @@ static int send_str_section(int fd, const ArrayList* list) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* The directory a rule belongs to (its owner, or the transfer root for ""). */
|
||||
static const char* filter_dir_rule_owner(const FilterRule* rule) {
|
||||
return (rule && rule->owner) ? rule->owner : "";
|
||||
}
|
||||
|
||||
/* Transmit the received-side per-directory filter rules (protocol 2.30.0) as a
|
||||
* self-describing list of directory groups: a group count, then for each group
|
||||
* the relative owner directory followed by that directory's rule records (run
|
||||
* order = the sender's traversal/rule order). Rules of one directory are
|
||||
* appended to the sink contiguously, so runs reproduce the compilation order.
|
||||
* Bounded by MAX_FILTER_RULES / MAX_FILTER_BYTES and MAX_PROTECT_PATTERN_LEN so
|
||||
* the peer never sees a frame it would reject. The sender enforces exactly the
|
||||
* receiver's limits (including the cumulative owner+pattern byte budget) and
|
||||
* fails with a clear local error instead of emitting a frame that would abort
|
||||
* the transfer with STATUS_ERROR. */
|
||||
bool delete_filter_dir_rules_send(int fd, const FilterRuleList* rules) {
|
||||
int count = rules ? rules->count : 0;
|
||||
if (count < 0 || count > MAX_FILTER_RULES) {
|
||||
log_message(LOG_LEVEL_ERROR, "too many per-directory filter rules: %d (maximum %d)", count,
|
||||
MAX_FILTER_RULES);
|
||||
return false;
|
||||
}
|
||||
size_t bytes = 0;
|
||||
for (int i = 0; i < count; i++) {
|
||||
const FilterRule* rule = rules->items[i];
|
||||
const char* owner = filter_dir_rule_owner(rule);
|
||||
size_t owner_len = strlen(owner);
|
||||
size_t pattern_len = rule && rule->pattern ? strlen(rule->pattern) : 0;
|
||||
if (!rule || !rule->pattern || pattern_len == 0) {
|
||||
log_message(LOG_LEVEL_ERROR, "invalid per-directory filter pattern");
|
||||
return false;
|
||||
}
|
||||
if (pattern_len > MAX_PROTECT_PATTERN_LEN) {
|
||||
log_message(LOG_LEVEL_ERROR,
|
||||
"per-directory filter pattern exceeds %d bytes (use a shorter pattern)",
|
||||
MAX_PROTECT_PATTERN_LEN);
|
||||
return false;
|
||||
}
|
||||
if (!(owner_len == 0 || (owner[0] != '/' && !has_path_traversal(owner)))) {
|
||||
log_message(LOG_LEVEL_ERROR, "invalid per-directory filter owner directory");
|
||||
return false;
|
||||
}
|
||||
if (owner_len + pattern_len > MAX_FILTER_BYTES - bytes) {
|
||||
log_message(LOG_LEVEL_ERROR, "per-directory filter rules exceed %d bytes", MAX_FILTER_BYTES);
|
||||
return false;
|
||||
}
|
||||
bytes += owner_len + pattern_len;
|
||||
}
|
||||
int groups = 0;
|
||||
for (int i = 0; i < count;) {
|
||||
const char* owner = filter_dir_rule_owner(rules->items[i]);
|
||||
groups++;
|
||||
i++;
|
||||
while (i < count && strcmp(filter_dir_rule_owner(rules->items[i]), owner) == 0)
|
||||
i++;
|
||||
}
|
||||
if (!send_int(fd, groups))
|
||||
return false;
|
||||
for (int i = 0; i < count;) {
|
||||
const char* owner = filter_dir_rule_owner(rules->items[i]);
|
||||
int start = i;
|
||||
i++;
|
||||
while (i < count && strcmp(filter_dir_rule_owner(rules->items[i]), owner) == 0)
|
||||
i++;
|
||||
if (!send_wire_str(fd, owner) || !send_int(fd, i - start))
|
||||
return false;
|
||||
for (int j = start; j < i; j++) {
|
||||
const FilterRule* rule = rules->items[j];
|
||||
if (!send_int(fd, (int)rule->action) || !send_int(fd, (int)rule->sides) ||
|
||||
!send_int(fd, rule->anchored ? 1 : 0) || !send_int(fd, rule->dir_only ? 1 : 0) ||
|
||||
!send_int(fd, rule->negate ? 1 : 0) || !send_int(fd, rule->no_inherit ? 1 : 0) ||
|
||||
!send_wire_str(fd, rule->pattern))
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Read one wire flag (an int restricted to 0/1). */
|
||||
static bool receive_flag(int fd, bool* value) {
|
||||
int raw;
|
||||
if (!receive_int(fd, &raw) || (raw != 0 && raw != 1))
|
||||
return false;
|
||||
*value = raw != 0;
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Read the per-directory filter block emitted by delete_filter_dir_rules_send.
|
||||
* Reconstructs a flat FilterRuleList whose rules carry their owner directory;
|
||||
* `*out` is NULL when the sender transmitted no rules. Every bound is enforced
|
||||
* (group/rule counts, owner/pattern bytes, pattern length, action/sides domain)
|
||||
* so a malicious peer can neither overread nor allocate unboundedly. Returns
|
||||
* false on a malformed frame (the caller signals STATUS_ERROR). */
|
||||
bool delete_filter_dir_rules_receive(int fd, FilterRuleList** out) {
|
||||
if (!out)
|
||||
return false;
|
||||
*out = NULL;
|
||||
int groups;
|
||||
if (!receive_int(fd, &groups) || groups < 0 || groups > MAX_FILTER_RULES)
|
||||
return false;
|
||||
if (groups == 0)
|
||||
return true;
|
||||
FilterRuleList* list = filter_rule_list_create();
|
||||
if (!list)
|
||||
return false;
|
||||
int total_rules = 0;
|
||||
size_t bytes = 0;
|
||||
for (int g = 0; g < groups; g++) {
|
||||
char* dir = receive_wire_str(fd);
|
||||
if (!dir)
|
||||
goto fail;
|
||||
size_t dir_bytes = strlen(dir);
|
||||
if (!(dir[0] == '\0' || (dir[0] != '/' && !has_path_traversal(dir))) ||
|
||||
dir_bytes > MAX_FILTER_BYTES - bytes) {
|
||||
free(dir);
|
||||
goto fail;
|
||||
}
|
||||
bytes += dir_bytes;
|
||||
int rule_count;
|
||||
if (!receive_int(fd, &rule_count) || rule_count < 0 || rule_count > MAX_FILTER_RULES ||
|
||||
rule_count > MAX_FILTER_RULES - total_rules) {
|
||||
free(dir);
|
||||
goto fail;
|
||||
}
|
||||
for (int r = 0; r < rule_count; r++) {
|
||||
int action, sides;
|
||||
bool anchored, dir_only, negate, no_inherit;
|
||||
if (!receive_int(fd, &action) ||
|
||||
(action != FILTER_ACTION_EXCLUDE && action != FILTER_ACTION_INCLUDE) ||
|
||||
!receive_int(fd, &sides) || sides < (int)FILTER_SIDE_SENDER ||
|
||||
sides > (int)(FILTER_SIDE_SENDER | FILTER_SIDE_RECEIVER) ||
|
||||
!receive_flag(fd, &anchored) || !receive_flag(fd, &dir_only) ||
|
||||
!receive_flag(fd, &negate) || !receive_flag(fd, &no_inherit)) {
|
||||
free(dir);
|
||||
goto fail;
|
||||
}
|
||||
char* pattern = receive_wire_str(fd);
|
||||
size_t pattern_bytes = pattern ? strlen(pattern) : 0;
|
||||
if (!pattern || pattern_bytes == 0 || pattern_bytes > MAX_PROTECT_PATTERN_LEN ||
|
||||
pattern_bytes > MAX_FILTER_BYTES - bytes) {
|
||||
free(pattern);
|
||||
free(dir);
|
||||
goto fail;
|
||||
}
|
||||
bytes += pattern_bytes;
|
||||
FilterRule* rule = calloc(1, sizeof(FilterRule));
|
||||
if (!rule) {
|
||||
free(pattern);
|
||||
free(dir);
|
||||
goto fail;
|
||||
}
|
||||
rule->action = (FilterAction)action;
|
||||
rule->sides = (unsigned)sides;
|
||||
rule->anchored = anchored;
|
||||
rule->dir_only = dir_only;
|
||||
rule->negate = negate;
|
||||
rule->no_inherit = no_inherit;
|
||||
rule->owner = str_dup(dir);
|
||||
rule->pattern = pattern;
|
||||
if (!rule->owner || !filter_rule_list_add(list, rule)) {
|
||||
filter_rule_free(rule);
|
||||
free(dir);
|
||||
goto fail;
|
||||
}
|
||||
total_rules++;
|
||||
}
|
||||
free(dir);
|
||||
}
|
||||
*out = list;
|
||||
return true;
|
||||
fail:
|
||||
filter_rule_list_free(list);
|
||||
return false;
|
||||
}
|
||||
|
||||
static int send_plan_node(int fd, DeletePlanSender* sender, PlanNode* node) {
|
||||
if (!send_status(fd, STATUS_DELETE_PLAN))
|
||||
return -1;
|
||||
@@ -328,7 +506,8 @@ static int send_plan_node(int fd, DeletePlanSender* sender, PlanNode* node) {
|
||||
if (!sender->config_sent) {
|
||||
if (send_str_section(fd, sender->protected_prefixes) != 0 ||
|
||||
send_str_section(fd, sender->size_skipped) != 0 ||
|
||||
send_str_section(fd, sender->missing_args) != 0)
|
||||
send_str_section(fd, sender->missing_args) != 0 ||
|
||||
!delete_filter_dir_rules_send(fd, sender->per_dir_rules))
|
||||
return -1;
|
||||
sender->config_sent = true;
|
||||
}
|
||||
@@ -355,7 +534,8 @@ static int send_config_only(int fd, DeletePlanSender* sender) {
|
||||
return -1;
|
||||
if (send_str_section(fd, sender->protected_prefixes) != 0 ||
|
||||
send_str_section(fd, sender->size_skipped) != 0 ||
|
||||
send_str_section(fd, sender->missing_args) != 0)
|
||||
send_str_section(fd, sender->missing_args) != 0 ||
|
||||
!delete_filter_dir_rules_send(fd, sender->per_dir_rules))
|
||||
return -1;
|
||||
sender->config_sent = true;
|
||||
if (!send_int(fd, 0)) /* apply = false */
|
||||
@@ -472,15 +652,19 @@ struct DeletePlanSession {
|
||||
ArrayList* protected_prefixes;
|
||||
ArrayList* size_skipped;
|
||||
ArrayList* missing;
|
||||
/* Received per-directory filter rules (protocol 2.30.0), or NULL. Evaluated
|
||||
deepest-directory-first for every candidate extra so a destination-only
|
||||
entry matching only a per-directory rule is protected. */
|
||||
FilterRuleList* per_dir_rules;
|
||||
ArrayList* deferred;
|
||||
DeletePathObserver observer;
|
||||
void* observer_context;
|
||||
};
|
||||
|
||||
/* Report one path the session truly removed (no-op without an observer). */
|
||||
static void notify_deleted(DeletePlanSession* session, const char* rel) {
|
||||
static void notify_deleted(DeletePlanSession* session, const char* rel, DeleteEntryType type) {
|
||||
if (session && session->observer && rel)
|
||||
session->observer(session->observer_context, rel);
|
||||
session->observer(session->observer_context, rel, type);
|
||||
}
|
||||
|
||||
/* A removed directory is reported with rsync's trailing slash (`deleting dir/`)
|
||||
@@ -491,13 +675,13 @@ static void notify_deleted_dir(DeletePlanSession* session, const char* rel) {
|
||||
size_t len = strlen(rel);
|
||||
char* with_slash = malloc(len + 2);
|
||||
if (!with_slash) {
|
||||
session->observer(session->observer_context, rel);
|
||||
session->observer(session->observer_context, rel, DELETE_ENTRY_DIR);
|
||||
return;
|
||||
}
|
||||
memcpy(with_slash, rel, len);
|
||||
with_slash[len] = '/';
|
||||
with_slash[len + 1] = '\0';
|
||||
session->observer(session->observer_context, with_slash);
|
||||
session->observer(session->observer_context, with_slash, DELETE_ENTRY_DIR);
|
||||
free(with_slash);
|
||||
}
|
||||
|
||||
@@ -532,6 +716,7 @@ void delete_plan_session_destroy(DeletePlanSession* session) {
|
||||
array_list_delete(session->protected_prefixes);
|
||||
array_list_delete(session->size_skipped);
|
||||
array_list_delete(session->missing);
|
||||
filter_rule_list_free(session->per_dir_rules);
|
||||
array_list_delete(session->deferred);
|
||||
free(session);
|
||||
}
|
||||
@@ -604,16 +789,19 @@ static int open_plan_dir(const Config* config, const char* dir) {
|
||||
|
||||
typedef struct {
|
||||
DeleteSkipSet set;
|
||||
/* Receiver-side delete-protection rules received on the config frame (NULL
|
||||
when the sender sent none). Evaluated per extra so a protect/risk rule is
|
||||
/* Receiver-side delete-protection rules. `base` is the config-frame
|
||||
command-line set and `dir` the received per-directory set (both NULL when
|
||||
the sender sent none). Evaluated per extra so a protect/risk rule is
|
||||
honored under --delete-during/--delete-delay exactly like the whole-tree
|
||||
commit walker. */
|
||||
const FilterRuleList* protect_rules;
|
||||
DeleteProtectRules protect;
|
||||
} PlanSkips;
|
||||
|
||||
static bool build_plan_skips(const Config* config, const DeletePlanSession* session,
|
||||
PlanSkips* out) {
|
||||
out->protect_rules = config->protect_rules;
|
||||
out->protect.base_rules = config->protect_rules;
|
||||
out->protect.dir_rules = session->per_dir_rules;
|
||||
out->protect.backup_suffix = delete_backup_suffix(config);
|
||||
/* The per-directory plan walk keeps each basis path verbatim (it does not
|
||||
convert an absolute under-root path to its root-relative form, unlike the
|
||||
whole-tree commit walk). */
|
||||
@@ -711,8 +899,8 @@ static bool process_extra_dir(int dirfd, const char* name, const char* child_rel
|
||||
return errno == ENOTEMPTY || errno == EEXIST;
|
||||
}
|
||||
|
||||
static bool process_extra_file(int dirfd, const char* name, const char* child_rel, bool force_now,
|
||||
DeletePlanSession* session) {
|
||||
static bool process_extra_file(int dirfd, const char* name, const char* child_rel, mode_t mode,
|
||||
bool force_now, DeletePlanSession* session) {
|
||||
if (session->defer && !force_now) {
|
||||
return defer_add(session, child_rel);
|
||||
}
|
||||
@@ -724,7 +912,7 @@ static bool process_extra_file(int dirfd, const char* name, const char* child_re
|
||||
session->deleted++;
|
||||
session->planned++;
|
||||
log_deleted(child_rel);
|
||||
notify_deleted(session, child_rel);
|
||||
notify_deleted(session, child_rel, delete_entry_type_of_mode(mode));
|
||||
} else if (errno != ENOENT) {
|
||||
return false;
|
||||
}
|
||||
@@ -778,10 +966,8 @@ static bool process_children(int dirfd, const char* dir_rel, const ArrayList* ke
|
||||
bool is_dir = entries[i].is_dir;
|
||||
bool in_keep_dirs = is_dir && list_contains_str(keep_dirs, entries[i].name);
|
||||
bool in_keep_files = !is_dir && list_contains_str(keep_files, entries[i].name);
|
||||
bool rule_protected =
|
||||
skips->protect_rules &&
|
||||
filter_rules_apply_side(skips->protect_rules, child_rel, entries[i].name, is_dir,
|
||||
FILTER_SIDE_RECEIVER) == FILTER_ACTION_PROTECT;
|
||||
bool rule_protected = delete_protect_verdict(&skips->protect, child_rel, entries[i].name,
|
||||
is_dir) == FILTER_ACTION_PROTECT;
|
||||
if (in_keep_dirs || in_keep_files || rule_protected) {
|
||||
shielded[i] = true;
|
||||
local_survives = true;
|
||||
@@ -828,7 +1014,8 @@ static bool process_children(int dirfd, const char* dir_rel, const ArrayList* ke
|
||||
operation_ok = false;
|
||||
continue;
|
||||
}
|
||||
if (!process_extra_file(dirfd, entries[i].name, child_rel, force[i] || force_now, session))
|
||||
if (!process_extra_file(dirfd, entries[i].name, child_rel, entries[i].mode,
|
||||
force[i] || force_now, session))
|
||||
operation_ok = false;
|
||||
free(child_rel);
|
||||
}
|
||||
@@ -902,7 +1089,8 @@ int delete_plan_session_receive(DeletePlanSession* session, const Config* config
|
||||
if (has_config) {
|
||||
if (session->config_seen || !read_section(fd, session->protected_prefixes, true, &bytes) ||
|
||||
!read_section(fd, session->size_skipped, true, &bytes) ||
|
||||
!read_section(fd, session->missing, true, &bytes)) {
|
||||
!read_section(fd, session->missing, true, &bytes) ||
|
||||
!delete_filter_dir_rules_receive(fd, &session->per_dir_rules)) {
|
||||
send_status(fd, STATUS_ERROR);
|
||||
return -1;
|
||||
}
|
||||
@@ -1032,7 +1220,7 @@ static bool apply_deferred_path(DeletePlanSession* session, const Config* config
|
||||
session->deleted++;
|
||||
session->planned++;
|
||||
log_deleted(rel);
|
||||
notify_deleted(session, rel);
|
||||
notify_deleted(session, rel, delete_entry_type_of_mode(st.st_mode));
|
||||
} else if (errno != ENOENT) {
|
||||
close(parent_fd);
|
||||
free(leaf);
|
||||
|
||||
@@ -25,6 +25,19 @@
|
||||
* --delete-missing-args exact deletions, the shared --max-delete budget and,
|
||||
* for --delete-delay, the snapshotted extras. */
|
||||
|
||||
/* Per-directory filter-rule block (protocol 2.30.0). The sender compiles the
|
||||
* source's per-directory merge rules as it scans and streams them so the
|
||||
* receiver can re-derive the receiver-side protect/risk verdicts for
|
||||
* destination-only entries. The wire format is a group count, then for each
|
||||
* directory group its relative owner path followed by that directory's rule
|
||||
* records (action, sides, anchored, dir-only, negate, no-inherit, pattern).
|
||||
* All bounds (MAX_FILTER_RULES, MAX_FILTER_BYTES, MAX_PROTECT_PATTERN_LEN) are
|
||||
* enforced on both sides; a malformed receive frame signals STATUS_ERROR and
|
||||
* returns false. Receive yields a flat FilterRuleList whose rules carry their
|
||||
* owner, or NULL when no rules were sent. */
|
||||
bool delete_filter_dir_rules_send(int fd, const FilterRuleList* rules);
|
||||
bool delete_filter_dir_rules_receive(int fd, FilterRuleList** out);
|
||||
|
||||
/* ---- Sender: plan builder ---- */
|
||||
|
||||
typedef struct DeletePlanSender DeletePlanSender;
|
||||
@@ -53,7 +66,8 @@ bool delete_plan_sender_empty(const DeletePlanSender* sender);
|
||||
* block is always transmitted by delete_plan_send_root(), on a config-only
|
||||
* carrier frame when the scope allows no directory plan. */
|
||||
void delete_plan_sender_set_config(DeletePlanSender* sender, const ArrayList* protected_prefixes,
|
||||
const ArrayList* size_skipped, const ArrayList* missing_args);
|
||||
const ArrayList* size_skipped, const ArrayList* missing_args,
|
||||
const FilterRuleList* per_dir_rules);
|
||||
/* Send the root plan (even before any data, so root extras are handled like
|
||||
* rsync's first generator directory), after transmitting the per-run config
|
||||
* block on its own carrier frame. Returns -1 on I/O error. */
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
#include "delta.h"
|
||||
#include "log.h"
|
||||
#include "protocol.h"
|
||||
#include <errno.h>
|
||||
#include <stdint.h>
|
||||
#include <limits.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#define XXH_STATIC_LINKING_ONLY
|
||||
#define XXH_IMPLEMENTATION
|
||||
@@ -82,6 +84,64 @@ DeltaSignature* delta_signature_create_seeded(const void* old_file_data, uint64_
|
||||
return sig;
|
||||
}
|
||||
|
||||
/* Bounded read of exactly `len` bytes at `off`; retries on EINTR. */
|
||||
static bool pread_all(int fd, void* buf, size_t len, uint64_t off) {
|
||||
uint8_t* p = buf;
|
||||
size_t done = 0;
|
||||
while (done < len) {
|
||||
ssize_t n = pread(fd, p + done, len - done, (off_t)(off + done));
|
||||
if (n < 0 && errno == EINTR)
|
||||
continue;
|
||||
if (n <= 0)
|
||||
return false;
|
||||
done += (size_t)n;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
DeltaSignature* delta_signature_create_fd_seeded(int fd, uint64_t old_file_size,
|
||||
uint32_t block_size, uint32_t seed) {
|
||||
if (fd < 0 || old_file_size == 0 || block_size == 0 || block_size > DELTA_BLOCK_SIZE_MAX ||
|
||||
old_file_size > UINT32_MAX * (uint64_t)block_size)
|
||||
return NULL;
|
||||
uint32_t block_count = (uint32_t)((old_file_size + block_size - 1) / block_size);
|
||||
/* Bound the signature's own memory (block_count * sizeof(DeltaBlockSig)). */
|
||||
if (block_count == 0 || block_count > MAX_DELTA_BLOCKS)
|
||||
return NULL;
|
||||
DeltaSignature* sig = protocol_alloc(sizeof(DeltaSignature));
|
||||
if (!sig)
|
||||
return NULL;
|
||||
sig->file_size = old_file_size;
|
||||
sig->block_size = block_size;
|
||||
sig->block_count = block_count;
|
||||
sig->blocks = protocol_alloc((size_t)block_count * sizeof(DeltaBlockSig));
|
||||
if (!sig->blocks) {
|
||||
free(sig);
|
||||
return NULL;
|
||||
}
|
||||
uint8_t* block = malloc(block_size);
|
||||
if (!block) {
|
||||
free(sig->blocks);
|
||||
free(sig);
|
||||
return NULL;
|
||||
}
|
||||
for (uint32_t i = 0; i < block_count; i++) {
|
||||
uint64_t offset = (uint64_t)i * block_size;
|
||||
uint32_t len =
|
||||
(uint32_t)((old_file_size - offset < block_size) ? (old_file_size - offset) : block_size);
|
||||
if (!pread_all(fd, block, len, offset)) {
|
||||
free(block);
|
||||
free(sig->blocks);
|
||||
free(sig);
|
||||
return NULL;
|
||||
}
|
||||
sig->blocks[i].adler32 = delta_adler32(block, len);
|
||||
sig->blocks[i].xxhash = delta_xxhash32_seeded(block, len, seed);
|
||||
}
|
||||
free(block);
|
||||
return sig;
|
||||
}
|
||||
|
||||
Data* delta_signature_serialize(const DeltaSignature* sig) {
|
||||
if (!sig)
|
||||
return NULL;
|
||||
@@ -687,6 +747,130 @@ void* delta_apply(const void* old_data, uint64_t old_size, const Delta* delta,
|
||||
return output;
|
||||
}
|
||||
|
||||
void* delta_apply_fd(int src_fd, uint64_t old_size, const Delta* delta, uint32_t block_size) {
|
||||
if (!delta || block_size == 0 || block_size > DELTA_BLOCK_SIZE_MAX ||
|
||||
(delta->instruction_count > 0 && !delta->instructions) || delta->new_file_size == 0 ||
|
||||
delta->new_file_size > SIZE_MAX)
|
||||
return NULL;
|
||||
|
||||
void* output = protocol_alloc((size_t)delta->new_file_size);
|
||||
if (!output)
|
||||
return NULL;
|
||||
|
||||
uint8_t* out = (uint8_t*)output;
|
||||
uint64_t out_pos = 0;
|
||||
|
||||
for (uint32_t i = 0; i < delta->instruction_count; i++) {
|
||||
if (delta->instructions[i].type == DELTA_INSTR_BLOCK_MATCH) {
|
||||
uint64_t src_offset = (uint64_t)delta->instructions[i].match.block_index * block_size;
|
||||
if (src_offset > UINT64_MAX - delta->instructions[i].match.block_offset) {
|
||||
free(output);
|
||||
return NULL;
|
||||
}
|
||||
src_offset += delta->instructions[i].match.block_offset;
|
||||
uint32_t len = delta->instructions[i].match.length;
|
||||
|
||||
if (src_offset > old_size || (uint64_t)len > old_size - src_offset ||
|
||||
out_pos > delta->new_file_size || (uint64_t)len > delta->new_file_size - out_pos) {
|
||||
free(output);
|
||||
return NULL;
|
||||
}
|
||||
if (!pread_all(src_fd, out + out_pos, len, src_offset)) {
|
||||
free(output);
|
||||
return NULL;
|
||||
}
|
||||
out_pos += len;
|
||||
} else if (delta->instructions[i].type == DELTA_INSTR_LITERAL) {
|
||||
uint32_t len = delta->instructions[i].literal.length;
|
||||
if (out_pos > delta->new_file_size || (uint64_t)len > delta->new_file_size - out_pos) {
|
||||
free(output);
|
||||
return NULL;
|
||||
}
|
||||
memcpy(out + out_pos, delta->instructions[i].literal.data, len);
|
||||
out_pos += len;
|
||||
} else {
|
||||
free(output);
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
if (out_pos != delta->new_file_size) {
|
||||
free(output);
|
||||
return NULL;
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
bool delta_apply_to_fd(const void* old_data, int src_fd, uint64_t old_size, const Delta* delta,
|
||||
uint32_t block_size, int dst_fd) {
|
||||
if (!delta || (old_data == NULL && src_fd < 0) || block_size == 0 ||
|
||||
block_size > DELTA_BLOCK_SIZE_MAX || (delta->instruction_count > 0 && !delta->instructions))
|
||||
return false;
|
||||
const int chunk = 1 << 20;
|
||||
uint8_t* buf = malloc((size_t)chunk);
|
||||
if (!buf)
|
||||
return false;
|
||||
uint64_t out_pos = 0;
|
||||
bool ok = true;
|
||||
for (uint32_t i = 0; i < delta->instruction_count && ok; i++) {
|
||||
uint64_t src_offset = 0;
|
||||
uint64_t len = 0;
|
||||
const uint8_t* lit = NULL;
|
||||
if (delta->instructions[i].type == DELTA_INSTR_BLOCK_MATCH) {
|
||||
src_offset = (uint64_t)delta->instructions[i].match.block_index * block_size;
|
||||
if (src_offset > UINT64_MAX - delta->instructions[i].match.block_offset ||
|
||||
src_offset + delta->instructions[i].match.block_offset > old_size) {
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
src_offset += delta->instructions[i].match.block_offset;
|
||||
len = delta->instructions[i].match.length;
|
||||
if (len > old_size - src_offset) {
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
} else if (delta->instructions[i].type == DELTA_INSTR_LITERAL) {
|
||||
lit = delta->instructions[i].literal.data;
|
||||
len = delta->instructions[i].literal.length;
|
||||
} else {
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
if (out_pos > delta->new_file_size || len > delta->new_file_size - out_pos) {
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
uint64_t done = 0;
|
||||
while (ok && done < len) {
|
||||
size_t want = (len - done) < (uint64_t)chunk ? (size_t)(len - done) : (size_t)chunk;
|
||||
if (lit) {
|
||||
memcpy(buf, lit + done, want);
|
||||
} else if (old_data) {
|
||||
memcpy(buf, (const uint8_t*)old_data + src_offset + done, want);
|
||||
} else if (!pread_all(src_fd, buf, want, src_offset + done)) {
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
const uint8_t* p = buf;
|
||||
size_t written = 0;
|
||||
while (written < want) {
|
||||
ssize_t n = write(dst_fd, p + written, want - written);
|
||||
if (n < 0 && errno == EINTR)
|
||||
continue;
|
||||
if (n <= 0) {
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
written += (size_t)n;
|
||||
}
|
||||
done += want;
|
||||
}
|
||||
out_pos += len;
|
||||
}
|
||||
free(buf);
|
||||
return ok && out_pos == delta->new_file_size;
|
||||
}
|
||||
|
||||
void delta_destroy(Delta* delta) {
|
||||
if (!delta)
|
||||
return;
|
||||
|
||||
@@ -61,6 +61,13 @@ DeltaSignature* delta_signature_create(const void* old_file_data, uint64_t old_f
|
||||
* identical to the unseeded function. */
|
||||
DeltaSignature* delta_signature_create_seeded(const void* old_file_data, uint64_t old_file_size,
|
||||
uint32_t block_size, uint32_t seed);
|
||||
/* Streaming equivalent of delta_signature_create_seeded: reads the basis blocks
|
||||
* from `fd` in bounded chunks, so an arbitrarily large basis can be signed
|
||||
* without materializing it. The signature itself is bounded (MAX_DELTA_BLOCKS
|
||||
* entries); an over-large basis returns NULL and the caller falls back to a
|
||||
* whole-file transfer. */
|
||||
DeltaSignature* delta_signature_create_fd_seeded(int fd, uint64_t old_file_size,
|
||||
uint32_t block_size, uint32_t seed);
|
||||
Data* delta_signature_serialize(const DeltaSignature* sig);
|
||||
DeltaSignature* delta_signature_deserialize(const Data* data);
|
||||
void delta_signature_destroy(DeltaSignature* sig);
|
||||
@@ -75,6 +82,15 @@ Delta* delta_compute_seeded(const void* new_file_data, uint64_t new_file_size,
|
||||
Data* delta_serialize(const Delta* delta);
|
||||
Delta* delta_deserialize(const Data* data);
|
||||
void* delta_apply(const void* old_data, uint64_t old_size, const Delta* delta, uint32_t block_size);
|
||||
/* Streaming equivalent of delta_apply: matched blocks are read from `src_fd` as
|
||||
* they are emitted, so the basis never has to be resident. */
|
||||
void* delta_apply_fd(int src_fd, uint64_t old_size, const Delta* delta, uint32_t block_size);
|
||||
/* Fully streamed reconstruction: matched blocks come from `old_data` (when
|
||||
* non-NULL) or are read from `src_fd`, and the reconstructed bytes are written
|
||||
* straight to `dst_fd` in bounded chunks, so a reconstructed file larger than
|
||||
* memory is never materialized. */
|
||||
bool delta_apply_to_fd(const void* old_data, int src_fd, uint64_t old_size, const Delta* delta,
|
||||
uint32_t block_size, int dst_fd);
|
||||
void delta_destroy(Delta* delta);
|
||||
|
||||
bool delta_should_attempt(uint64_t old_size, uint64_t new_size, uint64_t max_file_size);
|
||||
|
||||
@@ -16,6 +16,8 @@
|
||||
|
||||
#include "data.h"
|
||||
#include "checksum.h"
|
||||
#include "chunk.h"
|
||||
#include "compression.h"
|
||||
#include "delta.h"
|
||||
#include "file.h"
|
||||
#include "file_store.h"
|
||||
@@ -209,6 +211,7 @@ File* file_create(const char* path) {
|
||||
file->dir_time_only = false;
|
||||
file->basis_link = NULL;
|
||||
file->basis_copy = NULL;
|
||||
file->data_spool = false;
|
||||
file->link_group = 0;
|
||||
file->link_first = false;
|
||||
file->hardlink_target = NULL;
|
||||
@@ -238,8 +241,11 @@ void file_destroy(void* item) {
|
||||
file->send_path = NULL;
|
||||
free(file->basis_link);
|
||||
file->basis_link = NULL;
|
||||
if (file->data_spool && file->basis_copy)
|
||||
unlink(file->basis_copy);
|
||||
free(file->basis_copy);
|
||||
file->basis_copy = NULL;
|
||||
file->data_spool = false;
|
||||
free(file->hardlink_target);
|
||||
file->hardlink_target = NULL;
|
||||
free(file->symlink_target);
|
||||
@@ -381,6 +387,261 @@ size_t file_content_to_buffer(File* file) {
|
||||
return bytes_read;
|
||||
}
|
||||
|
||||
/* ---- Streamed whole-file payload receive ----
|
||||
*
|
||||
* A whole-file data frame is a uint64 length followed by that many bytes. When
|
||||
* the logical payload is at or below the receiver's streaming bound the
|
||||
* historical charged whole-buffer path is kept (decompressing in one shot for
|
||||
* `-z`). Above the bound the frame is read in bounded chunks and written into
|
||||
* a spool temp file next to the destination, decompressing incrementally for
|
||||
* zstd/zlib (lz4's block format cannot be streamed and keeps the buffered path).
|
||||
* The spool is then installed by the existing bounded-buffer basis-copy path
|
||||
* (file_copy_basis_stream_attrs), so the atomic temp+rename store, --partial/
|
||||
* --partial-dir, --delay-updates, --preallocate and metadata/xattr application
|
||||
* are all reused unchanged. Only bounded buffers (64 KiB read chunk + the
|
||||
* decompressor's 256 KiB output window) are ever live. */
|
||||
|
||||
#define FILE_PAYLOAD_READ_CHUNK (64 * 1024)
|
||||
#define FILE_PAYLOAD_PEEK_MAX 32
|
||||
|
||||
/* Create a confined spool temp file in the destination's directory. Returns an
|
||||
* open write fd and an owned absolute path, or -1 (errno set). The parent walk
|
||||
* creates missing directories exactly as a normal store would. */
|
||||
static int file_spool_create(const char* dest_path, char** out_spool_path) {
|
||||
*out_spool_path = NULL;
|
||||
char* leaf = NULL;
|
||||
int dirfd = file_open_secure_parent(dest_path, &leaf, true);
|
||||
free(leaf);
|
||||
if (dirfd < 0)
|
||||
return -1;
|
||||
char name[64];
|
||||
for (unsigned int i = 0; i < 100; i++) {
|
||||
snprintf(name, sizeof(name), ".fastsync-spool.%ld.%llu", (long)getpid(), next_temp_sequence());
|
||||
int fd = openat(dirfd, name, O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC | O_NOFOLLOW, 0600);
|
||||
if (fd < 0) {
|
||||
if (errno != EEXIST)
|
||||
break;
|
||||
continue;
|
||||
}
|
||||
char* copy = str_dup(dest_path);
|
||||
const char* dir = copy ? dirname(copy) : NULL;
|
||||
size_t need = dir ? strlen(dir) + 1 + strlen(name) + 1 : 0;
|
||||
char* full = need ? malloc(need) : NULL;
|
||||
if (!full) {
|
||||
free(copy);
|
||||
close(fd);
|
||||
unlinkat(dirfd, name, 0);
|
||||
close(dirfd);
|
||||
return -1;
|
||||
}
|
||||
snprintf(full, need, "%s/%s", dir, name);
|
||||
free(copy);
|
||||
close(dirfd);
|
||||
*out_spool_path = full;
|
||||
return fd;
|
||||
}
|
||||
close(dirfd);
|
||||
return -1;
|
||||
}
|
||||
|
||||
int file_spool_for_payload(const char* dest_path, char** out_spool_path) {
|
||||
return file_spool_create(dest_path, out_spool_path);
|
||||
}
|
||||
|
||||
bool file_receive_payload(int fd, bool compress, unsigned long long expected_size,
|
||||
const char* dest_path, unsigned long long stream_limit, Data** out_buffer,
|
||||
char** out_spool, unsigned long long* out_size) {
|
||||
if (out_buffer)
|
||||
*out_buffer = NULL;
|
||||
if (out_spool)
|
||||
*out_spool = NULL;
|
||||
if (out_size)
|
||||
*out_size = 0;
|
||||
if (!out_buffer || !out_spool || !out_size || !dest_path)
|
||||
return false;
|
||||
|
||||
unsigned long long frame_size = 0;
|
||||
if (!receive_n_data(fd, &frame_size, sizeof(frame_size)))
|
||||
return false;
|
||||
/* A frame larger than MAX_DATA_PAYLOAD_SIZE is allowed only on the streamed
|
||||
path, which never materializes it; the buffered path's
|
||||
receive_data_alloc/body enforces the historical bound itself. Only a size
|
||||
unrepresentable on this platform is rejected here. */
|
||||
if (frame_size > SIZE_MAX) {
|
||||
log_message(LOG_LEVEL_ERROR, "Data size %llu is not representable", frame_size);
|
||||
return false;
|
||||
}
|
||||
|
||||
unsigned char prefix[FILE_PAYLOAD_PEEK_MAX];
|
||||
size_t prefix_len = 0;
|
||||
bool stream;
|
||||
if (expected_size != 0) {
|
||||
/* The check frame already told us the logical size: no need to peek. */
|
||||
stream = expected_size > stream_limit;
|
||||
} else if (!compress) {
|
||||
stream = frame_size > stream_limit;
|
||||
} else {
|
||||
/* Non-incremental compressed frame with unknown logical size: peek the
|
||||
header to decide, so a small compressed frame that expands past the bound
|
||||
still streams instead of allocating the whole logical image. */
|
||||
size_t want = frame_size < sizeof(prefix) ? (size_t)frame_size : sizeof(prefix);
|
||||
if (want > 0 && !receive_n_data(fd, prefix, want))
|
||||
return false;
|
||||
prefix_len = want;
|
||||
unsigned long long logical = compression_peek_frame_content_size(prefix, prefix_len);
|
||||
stream = logical != 0 ? logical > stream_limit : frame_size > stream_limit;
|
||||
}
|
||||
|
||||
if (!stream) {
|
||||
Data* frame;
|
||||
if (prefix_len > 0) {
|
||||
frame = receive_data_alloc(fd, frame_size);
|
||||
if (!frame)
|
||||
return false;
|
||||
memcpy(frame->data, prefix, prefix_len);
|
||||
if (frame_size > prefix_len &&
|
||||
!receive_n_data(fd, (char*)frame->data + prefix_len, (size_t)(frame_size - prefix_len))) {
|
||||
data_destroy(frame);
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
frame = receive_data_body(fd, frame_size);
|
||||
if (!frame)
|
||||
return false;
|
||||
}
|
||||
if (compress) {
|
||||
unsigned long long bound =
|
||||
expected_size != 0 ? expected_size : (unsigned long long)MAX_RECEIVE_WHOLE_FILE_SIZE;
|
||||
Data* uncompressed = data_decompress_limited(frame, (size_t)bound);
|
||||
ProtocolSession* owner = frame->owner;
|
||||
data_destroy(frame);
|
||||
if (!uncompressed)
|
||||
return false;
|
||||
if (expected_size != 0 && uncompressed->size != expected_size) {
|
||||
data_destroy(uncompressed);
|
||||
return false;
|
||||
}
|
||||
if (!data_charge_session(uncompressed, owner, uncompressed->size)) {
|
||||
data_destroy(uncompressed);
|
||||
return false;
|
||||
}
|
||||
*out_buffer = uncompressed;
|
||||
*out_size = uncompressed->size;
|
||||
return true;
|
||||
}
|
||||
*out_buffer = frame;
|
||||
*out_size = frame->size;
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Streamed path. */
|
||||
int spool_fd = file_spool_create(dest_path, out_spool);
|
||||
if (spool_fd < 0)
|
||||
return false;
|
||||
|
||||
CompressionStreamDecompressor* dec = NULL;
|
||||
size_t header_len = 0;
|
||||
unsigned long long learned_size = expected_size;
|
||||
if (compress) {
|
||||
unsigned char hdr[1 + 4];
|
||||
size_t hdr_have = 0;
|
||||
if (prefix_len >= 1) {
|
||||
hdr[0] = prefix[0];
|
||||
hdr_have = 1;
|
||||
} else {
|
||||
if (!receive_n_data(fd, hdr, 1))
|
||||
goto stream_fail;
|
||||
hdr_have = 1;
|
||||
}
|
||||
CompressionAlgo algo = (CompressionAlgo)hdr[0];
|
||||
if (!compression_algo_valid((int)algo) || algo == COMPRESSION_ALGO_LZ4)
|
||||
goto stream_fail;
|
||||
header_len = (algo == COMPRESSION_ALGO_ZLIB || algo == COMPRESSION_ALGO_ZLIBX) ? 5 : 1;
|
||||
while (hdr_have < header_len) {
|
||||
size_t need = header_len - hdr_have;
|
||||
if (prefix_len > hdr_have) {
|
||||
size_t avail = prefix_len - hdr_have;
|
||||
size_t take = avail < need ? avail : need;
|
||||
memcpy(hdr + hdr_have, prefix + hdr_have, take);
|
||||
hdr_have += take;
|
||||
} else {
|
||||
if (!receive_n_data(fd, hdr + hdr_have, need))
|
||||
goto stream_fail;
|
||||
hdr_have = header_len;
|
||||
}
|
||||
}
|
||||
if (header_len == 5) {
|
||||
uint32_t raw = 0;
|
||||
for (int i = 0; i < 4; i++)
|
||||
raw |= (uint32_t)hdr[1 + i] << (8 * i);
|
||||
if (expected_size != 0 && raw != expected_size)
|
||||
goto stream_fail;
|
||||
if (learned_size == 0)
|
||||
learned_size = raw;
|
||||
}
|
||||
dec = compression_stream_decompressor_create(algo, learned_size);
|
||||
if (!dec)
|
||||
goto stream_fail;
|
||||
}
|
||||
|
||||
if (frame_size < header_len)
|
||||
goto stream_fail;
|
||||
{
|
||||
unsigned long long body_remaining = frame_size - header_len;
|
||||
if (prefix_len > header_len) {
|
||||
size_t avail = prefix_len - header_len;
|
||||
if (compress) {
|
||||
bool done = false;
|
||||
if (!compression_stream_decompressor_feed(dec, prefix + header_len, avail, spool_fd, &done))
|
||||
goto stream_fail;
|
||||
} else if (!write_all(spool_fd, prefix + header_len, avail)) {
|
||||
goto stream_fail;
|
||||
}
|
||||
body_remaining -= avail;
|
||||
}
|
||||
unsigned char buf[FILE_PAYLOAD_READ_CHUNK];
|
||||
while (body_remaining > 0) {
|
||||
size_t want = body_remaining < sizeof(buf) ? (size_t)body_remaining : (size_t)sizeof(buf);
|
||||
if (!receive_n_data(fd, buf, want))
|
||||
goto stream_fail;
|
||||
if (compress) {
|
||||
bool done = false;
|
||||
if (!compression_stream_decompressor_feed(dec, buf, want, spool_fd, &done))
|
||||
goto stream_fail;
|
||||
} else if (!write_all(spool_fd, buf, want)) {
|
||||
goto stream_fail;
|
||||
}
|
||||
body_remaining -= want;
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
unsigned long long total = compress ? compression_stream_decompressor_total(dec) : frame_size;
|
||||
if (learned_size != 0 && total != learned_size)
|
||||
goto stream_fail;
|
||||
*out_size = total;
|
||||
}
|
||||
if (dec)
|
||||
compression_stream_decompressor_destroy(dec);
|
||||
if (close(spool_fd) != 0) {
|
||||
spool_fd = -1;
|
||||
goto stream_fail;
|
||||
}
|
||||
return true;
|
||||
|
||||
stream_fail:
|
||||
if (dec)
|
||||
compression_stream_decompressor_destroy(dec);
|
||||
if (spool_fd >= 0)
|
||||
close(spool_fd);
|
||||
if (*out_spool) {
|
||||
unlink(*out_spool);
|
||||
free(*out_spool);
|
||||
*out_spool = NULL;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/* ---- Secure filesystem primitives ---- */
|
||||
|
||||
bool file_path_exists_secure(const char* path) {
|
||||
|
||||
@@ -177,6 +177,24 @@ bool file_copy_basis_stream_attrs(const char* path, const char* basis_path,
|
||||
const FileMetadata* metadata, FileAttrPolicy policy, bool update,
|
||||
bool use_fsync, const FileXattrList* xattrs, bool fake_super,
|
||||
const char* temp_dir);
|
||||
/* Receive one length-prefixed whole-file data frame, streaming the payload
|
||||
* through a bounded buffer when it (or its known logical size) exceeds
|
||||
* `stream_limit`. On success exactly one of *out_buffer / *out_spool is set:
|
||||
* - *out_buffer: the historical charged whole-buffer Data (caller destroys);
|
||||
* - *out_spool: an owned temp path holding the payload, installed through the
|
||||
* File's basis_copy field with File.data_spool set so file_destroy unlinks
|
||||
* it. The destination policy/metadata/atomic-store handling is then the
|
||||
* existing bounded-buffer basis install (file_copy_basis_stream_attrs).
|
||||
* `expected_size` (0 = unknown) is the logical size from the check frame;
|
||||
* `dest_path` locates the spool next to the destination; `compress` selects
|
||||
* incremental decompression. Returns false on any framing/I/O/size error. */
|
||||
bool file_receive_payload(int fd, bool compress, unsigned long long expected_size,
|
||||
const char* dest_path, unsigned long long stream_limit, Data** out_buffer,
|
||||
char** out_spool, unsigned long long* out_size);
|
||||
/* Create a confined spool temp file next to `dest_path`; returns an open write
|
||||
* fd and an owned absolute path (to be installed via File.basis_copy with
|
||||
* File.data_spool set, and unlinked by file_destroy). */
|
||||
int file_spool_for_payload(const char* dest_path, char** out_spool_path);
|
||||
/* Protocol 2.28.0 receiver-stat variants: like the two above but additionally
|
||||
* report through `dirs_created` (when non-NULL) how many parent directories the
|
||||
* confined secure walk had to create that lie strictly below `count_floor` (a
|
||||
|
||||
+64
-29
@@ -58,36 +58,41 @@ File* file_receive(const Config* config, int file_descriptor) {
|
||||
file_destroy(file);
|
||||
return NULL;
|
||||
}
|
||||
Data* file_data = receive_data_limited(file_descriptor, MAX_RECEIVE_WHOLE_FILE_SIZE);
|
||||
if (file_data == NULL) {
|
||||
bool compress =
|
||||
config->use_compression && !compression_should_skip_with_suffixes(
|
||||
file->path, config->skip_compress_suffixes,
|
||||
config->skip_compress_set ? config->skip_compress_count : -1);
|
||||
char* dest_path = path_cat(config->receive_root_directory, file->path);
|
||||
if (!dest_path) {
|
||||
file_destroy(file);
|
||||
return NULL;
|
||||
}
|
||||
if (config->use_compression &&
|
||||
!compression_should_skip_with_suffixes(file->path, config->skip_compress_suffixes,
|
||||
config->skip_compress_set ? config->skip_compress_count
|
||||
: -1)) {
|
||||
Data* file_data_uncompressed = data_decompress_limited(file_data, MAX_RECEIVE_WHOLE_FILE_SIZE);
|
||||
ProtocolSession* owner = file_data->owner;
|
||||
data_destroy(file_data);
|
||||
if (file_data_uncompressed == NULL) {
|
||||
Data* buffer = NULL;
|
||||
char* spool = NULL;
|
||||
unsigned long long size = 0;
|
||||
bool ok = file_receive_payload(file_descriptor, compress, 0, dest_path,
|
||||
protocol_whole_file_receive_limit(), &buffer, &spool, &size);
|
||||
free(dest_path);
|
||||
if (!ok) {
|
||||
file_destroy(file);
|
||||
return NULL;
|
||||
}
|
||||
if (!data_charge_session(file_data_uncompressed, owner, file_data_uncompressed->size)) {
|
||||
data_destroy(file_data_uncompressed);
|
||||
if (spool) {
|
||||
Data* reserved = data_create_reserve((size_t)size);
|
||||
if (!reserved) {
|
||||
unlink(spool);
|
||||
free(spool);
|
||||
file_destroy(file);
|
||||
return NULL;
|
||||
}
|
||||
if (file_data_uncompressed->size > MAX_FILE_DATA_SIZE) {
|
||||
data_destroy(file_data_uncompressed);
|
||||
file_destroy(file);
|
||||
return NULL;
|
||||
}
|
||||
file_data = file_data_uncompressed;
|
||||
}
|
||||
data_destroy(file->data);
|
||||
file->data = file_data;
|
||||
file->data = reserved;
|
||||
file->basis_copy = spool;
|
||||
file->data_spool = true;
|
||||
} else {
|
||||
data_destroy(file->data);
|
||||
file->data = buffer;
|
||||
}
|
||||
return file;
|
||||
}
|
||||
|
||||
@@ -97,12 +102,14 @@ bool dir_metadata_should_capture(const Config* config) {
|
||||
/* Directory metadata is captured when a directory attribute is actually
|
||||
* requested: -p/--perms (directory modes), -t/--times (directory mtimes,
|
||||
* unless -O/--omit-dir-times suppresses them), -o/-g (directory ownership),
|
||||
* or -X/-A (directory xattrs/ACLs). --atimes/-U alone does not pull
|
||||
* directory metadata (matching the original dir-time bundle). */
|
||||
* -X/-A (directory xattrs/ACLs), or --fake-super (whose reserved %stat record
|
||||
* is written on the directory itself, so its metadata must travel).
|
||||
* --atimes/-U alone does not pull directory metadata (matching the original
|
||||
* dir-time bundle). */
|
||||
return config && config->use_metadata &&
|
||||
(config->preserve_perms || (config->preserve_times && !config->omit_dir_times) ||
|
||||
config->preserve_owner || config->preserve_group || config->preserve_xattrs ||
|
||||
config->preserve_acls);
|
||||
config->preserve_acls || config->fake_super);
|
||||
}
|
||||
|
||||
void dir_time_list_init(DirTimeList* list) {
|
||||
@@ -200,12 +207,19 @@ void dir_metadata_list_apply(const DirTimeList* list, const char* root_directory
|
||||
bool apply_times = config->preserve_times && !config->omit_dir_times;
|
||||
bool apply_mode = config->preserve_perms;
|
||||
bool apply_xattrs = config->use_xattrs;
|
||||
bool apply_fake_super = config->fake_super;
|
||||
/* Ownership is applied through the active identity snapshot (which no-ops
|
||||
* unless an ownership request is active), and xattrs only when -X/-A was
|
||||
* negotiated. Times/mode keep their own per-attribute gates. */
|
||||
bool have_any = apply_times || apply_mode || apply_xattrs || identity_active_enabled();
|
||||
* unless an ownership request is active), xattrs only when -X/-A was
|
||||
* negotiated, and the --fake-super record whenever the flag is active.
|
||||
* Times/mode keep their own per-attribute gates. */
|
||||
bool have_any =
|
||||
apply_times || apply_mode || apply_xattrs || apply_fake_super || identity_active_enabled();
|
||||
if (!have_any)
|
||||
return;
|
||||
/* Built once: the --fake-super replay uses it to apply only the recorded
|
||||
* permission bits (the special bits stay in the record, exactly like the
|
||||
* regular-file fake-super receiver). */
|
||||
FileAttrPolicy policy = file_attr_policy_from_config(config);
|
||||
for (size_t i = 0; i < list->count; i++) {
|
||||
char* dir_path = path_cat(root_directory, list->paths[i]);
|
||||
if (!dir_path)
|
||||
@@ -255,10 +269,12 @@ void dir_metadata_list_apply(const DirTimeList* list, const char* root_directory
|
||||
free(escaped_path);
|
||||
}
|
||||
}
|
||||
if (apply_mode) {
|
||||
/* The final directory mode (after any --chmod) is computed once so the
|
||||
--fake-super record can carry it even when the on-disk replay is
|
||||
restricted to the permission bits below. */
|
||||
mode_t dir_mode = list->entries[i].mode;
|
||||
bool mode_ready = true;
|
||||
if (config->chmod_spec && *config->chmod_spec &&
|
||||
if (apply_mode && config->chmod_spec && *config->chmod_spec &&
|
||||
!chmod_apply(dir_mode, config->chmod_spec, &dir_mode)) {
|
||||
char* escaped_path = output_escape(dir_path, log_get_8_bit_output());
|
||||
log_message(LOG_LEVEL_WARNING, "Failed to apply --chmod to directory %s",
|
||||
@@ -266,7 +282,13 @@ void dir_metadata_list_apply(const DirTimeList* list, const char* root_directory
|
||||
free(escaped_path);
|
||||
mode_ready = false;
|
||||
}
|
||||
if (mode_ready) {
|
||||
/* Under --fake-super the normal fchmod below still applies the mode, but
|
||||
the fake-super replay that follows narrows the on-disk result to the
|
||||
recorded permission bits (the full mode, including setuid/setgid/sticky,
|
||||
lives only in the record). Keeping the normal fchmod first means a
|
||||
filesystem without xattr support still gets the directory mode rather than
|
||||
silently losing it. */
|
||||
if (apply_mode && mode_ready) {
|
||||
/* rsync -p copies the source directory mode exactly, including
|
||||
* group/other write and the setgid/sticky bits. Setuid/setgid/sticky
|
||||
* are super-user activities: when the connection forbade them
|
||||
@@ -286,6 +308,19 @@ void dir_metadata_list_apply(const DirTimeList* list, const char* root_directory
|
||||
free(escaped_path);
|
||||
}
|
||||
}
|
||||
/* --fake-super: park the directory's full stat (rsync 3.4.1's exact
|
||||
grammar) on the directory ITSELF, then replay only the recorded
|
||||
permission bits fd-relative. The special bits live only in the record
|
||||
and the recorded ownership is never real-chowned: the resolved ids are
|
||||
stored for a later privileged restore, exactly like the file path. Runs
|
||||
before the xattr apply so a mode change cannot clobber the ACL mask. */
|
||||
if (apply_fake_super && dir_fd >= 0) {
|
||||
uint32_t store_uid = 0;
|
||||
uint32_t store_gid = 0;
|
||||
identity_resolve_storage_ids((int32_t)list->entries[i].uid, (int32_t)list->entries[i].gid,
|
||||
&store_uid, &store_gid);
|
||||
fake_super_store_fd(dir_fd, store_uid, store_gid, (uint32_t)dir_mode, 0, 0);
|
||||
fake_super_restore_fd(dir_fd, policy);
|
||||
}
|
||||
/* xattrs/ACLs last: a mode change can rewrite the ACL mask, so the ACL
|
||||
xattrs must be (re)applied after fchmod. */
|
||||
|
||||
@@ -817,6 +817,21 @@ static FileSaveResult file_save_directory_to_disk(const FileSavePlan* plan, bool
|
||||
} else if (ok && identity_copy_as_active()) {
|
||||
ok = false;
|
||||
}
|
||||
/* --fake-super: park the directory's full stat in rsync's reserved
|
||||
user.rsync.%stat xattr as soon as the directory exists. This makes even a
|
||||
direct file_save_to_disk_full() caller -- which never runs the deferred
|
||||
DirTimeList pass -- produce an rsync-readable fake-super record. The record
|
||||
carries the full mode/uid/gid; the permission bits are replayed by the
|
||||
deferred pass (never inline, so a restrictive mode cannot block child
|
||||
creation) and the recorded ownership is never real-chowned. Best-effort:
|
||||
fake_super_store_fd() logs and skips a failure, never failing the entry. */
|
||||
if (ok && plan->config && plan->config->fake_super && file->metadata && dir_fd >= 0) {
|
||||
uint32_t store_uid = 0;
|
||||
uint32_t store_gid = 0;
|
||||
identity_resolve_storage_ids((int32_t)file->metadata->uid, (int32_t)file->metadata->gid,
|
||||
&store_uid, &store_gid);
|
||||
fake_super_store_fd(dir_fd, store_uid, store_gid, (uint32_t)file->metadata->mode, 0, 0);
|
||||
}
|
||||
/* The final source MODE is deliberately NOT applied inline. A restrictive
|
||||
source mode (for example 0555) would make the directory unwritable before
|
||||
its children are created, so a non-root receiver fails each child with
|
||||
|
||||
+127
-1
@@ -44,12 +44,138 @@ bool file_send_single_calls(File* file, int file_descriptor, bool use_metadata,
|
||||
send_path, NULL, -1, 0, false);
|
||||
}
|
||||
|
||||
/* Stream-compress a whole source file into a temp file, then transmit it as the
|
||||
* normal length-prefixed data frame. Used when the source was too large to
|
||||
* load (File.data.data == NULL): the source is read in bounded chunks through a
|
||||
* streaming codec, so neither the raw nor the compressed image is held in
|
||||
* memory. The compressed length must be known before the frame is sent (the
|
||||
* wire is length-prefixed), so the stream lands in a private temp file first. */
|
||||
bool file_send_compressed_stream_with_skip(File* file, int file_descriptor, bool use_metadata,
|
||||
int compression_level, bool send_path,
|
||||
char* const* skip_suffixes, int skip_count,
|
||||
int compression_threads, bool send_xattrs) {
|
||||
/* The caller has already decided this file compresses; the skip list is part
|
||||
of the public signature for symmetry with the buffered path. */
|
||||
(void)skip_suffixes;
|
||||
(void)skip_count;
|
||||
if (!file || !file->path || !file->data || file->data->size == 0 || file->data->data != NULL)
|
||||
return false;
|
||||
CompressionAlgo algo = compression_get_algo();
|
||||
CompressionStreamCompressor* compressor =
|
||||
compression_stream_compressor_create(algo, compression_level, compression_threads);
|
||||
if (!compressor) {
|
||||
log_message(LOG_LEVEL_ERROR, "streaming compression is not available for this codec; "
|
||||
"the source was not loaded for the buffered path");
|
||||
return false;
|
||||
}
|
||||
|
||||
int src = file_open_for_read(file->path);
|
||||
if (src < 0) {
|
||||
compression_stream_compressor_destroy(compressor);
|
||||
return false;
|
||||
}
|
||||
struct stat src_st;
|
||||
if (fstat(src, &src_st) != 0 || !S_ISREG(src_st.st_mode) ||
|
||||
(unsigned long long)src_st.st_size < file->data->size) {
|
||||
close(src);
|
||||
compression_stream_compressor_destroy(compressor);
|
||||
return false;
|
||||
}
|
||||
|
||||
const char* tmpdir = getenv("TMPDIR");
|
||||
if (!tmpdir || tmpdir[0] == '\0')
|
||||
tmpdir = "/tmp";
|
||||
size_t tmplen = strlen(tmpdir) + strlen("/fastsync-z-XXXXXX") + 1;
|
||||
char* tmpl = malloc(tmplen);
|
||||
if (!tmpl) {
|
||||
close(src);
|
||||
compression_stream_compressor_destroy(compressor);
|
||||
return false;
|
||||
}
|
||||
snprintf(tmpl, tmplen, "%s/fastsync-z-XXXXXX", tmpdir);
|
||||
int tmp_fd = mkstemp(tmpl);
|
||||
if (tmp_fd < 0) {
|
||||
log_perror("Could not create compression temp file");
|
||||
free(tmpl);
|
||||
close(src);
|
||||
compression_stream_compressor_destroy(compressor);
|
||||
return false;
|
||||
}
|
||||
unlink(tmpl);
|
||||
free(tmpl);
|
||||
|
||||
bool ok = compression_stream_compressor_begin(compressor, file->data->size, tmp_fd);
|
||||
unsigned char buf[64 * 1024];
|
||||
unsigned long long remaining = file->data->size;
|
||||
while (ok && remaining > 0) {
|
||||
size_t want = remaining < sizeof(buf) ? (size_t)remaining : sizeof(buf);
|
||||
ssize_t got = read(src, buf, want);
|
||||
if (got <= 0) {
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
if (!compression_stream_compressor_feed(compressor, buf, (size_t)got, tmp_fd))
|
||||
ok = false;
|
||||
remaining -= (unsigned long long)got;
|
||||
}
|
||||
if (ok)
|
||||
ok = compression_stream_compressor_finish(compressor, tmp_fd);
|
||||
close(src);
|
||||
compression_stream_compressor_destroy(compressor);
|
||||
if (!ok) {
|
||||
close(tmp_fd);
|
||||
return false;
|
||||
}
|
||||
struct stat tmp_st;
|
||||
if (fstat(tmp_fd, &tmp_st) != 0 || tmp_st.st_size < 0) {
|
||||
close(tmp_fd);
|
||||
return false;
|
||||
}
|
||||
unsigned long long compressed_size = (unsigned long long)tmp_st.st_size;
|
||||
if (lseek(tmp_fd, 0, SEEK_SET) == (off_t)-1) {
|
||||
close(tmp_fd);
|
||||
return false;
|
||||
}
|
||||
|
||||
ok = true;
|
||||
if (send_path && !send_wire_str(file_descriptor, file_wire_path(file)))
|
||||
ok = false;
|
||||
if (ok && use_metadata && !metadata_send(file_descriptor, file->metadata))
|
||||
ok = false;
|
||||
if (ok && send_xattrs && !xattr_send(file_descriptor, file ? file->xattrs : NULL))
|
||||
ok = false;
|
||||
if (ok && !send_n_data(file_descriptor, &compressed_size, sizeof(compressed_size)))
|
||||
ok = false;
|
||||
unsigned long long left = compressed_size;
|
||||
while (ok && left > 0) {
|
||||
size_t want = left < sizeof(buf) ? (size_t)left : sizeof(buf);
|
||||
ssize_t got = read(tmp_fd, buf, want);
|
||||
if (got <= 0 || !send_n_data(file_descriptor, buf, (size_t)got)) {
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
left -= (unsigned long long)got;
|
||||
}
|
||||
close(tmp_fd);
|
||||
return ok;
|
||||
}
|
||||
|
||||
bool file_send_single_calls_with_skip(File* file, int file_descriptor, bool use_metadata,
|
||||
int compression_level, bool send_path,
|
||||
char* const* skip_suffixes, int skip_count,
|
||||
int compression_threads, bool send_xattrs) {
|
||||
if (!file || !file->path || !file->data || (file->data->size != 0 && !file->data->data))
|
||||
if (!file || !file->path || !file->data)
|
||||
return false;
|
||||
/* A source too large to load is streamed: compression streams through a
|
||||
* temp file, no compression must have taken the sendfile path instead. */
|
||||
if (file->data->size != 0 && file->data->data == NULL) {
|
||||
if (compression_level <= 0 ||
|
||||
compression_should_skip_with_suffixes(file->path, skip_suffixes, skip_count))
|
||||
return false;
|
||||
return file_send_compressed_stream_with_skip(file, file_descriptor, use_metadata,
|
||||
compression_level, send_path, skip_suffixes,
|
||||
skip_count, compression_threads, send_xattrs);
|
||||
}
|
||||
const Data* data_to_send = file->data;
|
||||
Data* compressed_data = NULL;
|
||||
if (compression_level > 0 &&
|
||||
|
||||
@@ -13,6 +13,12 @@ bool file_send_single_calls_with_skip(File* file, int file_descriptor, bool use_
|
||||
int compression_level, bool send_path,
|
||||
char* const* skip_suffixes, int skip_count,
|
||||
int compression_threads, bool send_xattrs);
|
||||
/* Stream-compress an unloaded whole source file into a temp file and send it as
|
||||
* the usual data frame (see file_send.c). */
|
||||
bool file_send_compressed_stream_with_skip(File* file, int file_descriptor, bool use_metadata,
|
||||
int compression_level, bool send_path,
|
||||
char* const* skip_suffixes, int skip_count,
|
||||
int compression_threads, bool send_xattrs);
|
||||
bool file_send_sendfile(File* file, int file_descriptor, bool use_metadata, int compression_level,
|
||||
bool send_path);
|
||||
bool file_send_sendfile_with_skip(File* file, int file_descriptor, bool use_metadata,
|
||||
|
||||
@@ -62,6 +62,11 @@ typedef struct {
|
||||
* This lets a basis larger than any whole-file bound materialize without
|
||||
* buffering it; the source metadata on `metadata` is applied afterwards. */
|
||||
char* basis_copy;
|
||||
/* Receiver-only. When set, `basis_copy` points at a receiver-created spool
|
||||
* temp file holding a STREAMED whole-file payload (rather than a --copy-dest
|
||||
* basis). file_destroy unlinks it after the install consumes it, so an
|
||||
* over-limit file leaves no scratch behind. */
|
||||
bool data_spool;
|
||||
/* --hard-links (-H), sender + receiver wire state. link_group is a run-local
|
||||
* id shared by every member of one source inode (0 = not part of a group).
|
||||
* The FIRST member (link_first == true) carries its data on the wire and is
|
||||
|
||||
+333
-90
@@ -21,6 +21,28 @@ void filter_rule_free(FilterRule* rule) {
|
||||
free(rule);
|
||||
}
|
||||
|
||||
FilterRule* filter_rule_clone(const FilterRule* rule) {
|
||||
if (!rule)
|
||||
return NULL;
|
||||
FilterRule* copy = calloc(1, sizeof(FilterRule));
|
||||
if (!copy)
|
||||
return NULL;
|
||||
copy->action = rule->action;
|
||||
copy->sides = rule->sides;
|
||||
copy->anchored = rule->anchored;
|
||||
copy->dir_only = rule->dir_only;
|
||||
copy->negate = rule->negate;
|
||||
copy->perishable = rule->perishable;
|
||||
copy->no_inherit = rule->no_inherit;
|
||||
copy->owner = str_dup(rule->owner ? rule->owner : "");
|
||||
copy->pattern = str_dup(rule->pattern ? rule->pattern : "");
|
||||
if (!copy->owner || !copy->pattern) {
|
||||
filter_rule_free(copy);
|
||||
return NULL;
|
||||
}
|
||||
return copy;
|
||||
}
|
||||
|
||||
FilterRuleList* filter_rule_list_create(void) {
|
||||
return calloc(1, sizeof(FilterRuleList));
|
||||
}
|
||||
@@ -48,37 +70,94 @@ void filter_rule_list_free(FilterRuleList* list) {
|
||||
for (int i = 0; i < list->count; i++)
|
||||
filter_rule_free(list->items[i]);
|
||||
for (int i = 0; i < list->dir_merge_count; i++)
|
||||
free(list->dir_merge_names[i]);
|
||||
free(list->dir_merge_names);
|
||||
free(list->dir_merges[i].name);
|
||||
free(list->dir_merges);
|
||||
free(list->items);
|
||||
free(list);
|
||||
}
|
||||
|
||||
/* Append an implicit exclude rule for the merge file itself (rsync's 'e'
|
||||
* modifier). The rule is owned by the transfer root and matches the basename
|
||||
* anywhere, exactly like rsync's EXCLUDE_SELF (a dual-sided exclude: it hides
|
||||
* the file and protects its destination mirror from --delete). */
|
||||
static bool filter_list_add_exclude_self(FilterRuleList* list, const char* name) {
|
||||
const char* base = strrchr(name, '/');
|
||||
base = base ? base + 1 : name;
|
||||
if (base[0] == '\0')
|
||||
return true;
|
||||
FilterRule* rule = calloc(1, sizeof(FilterRule));
|
||||
if (!rule)
|
||||
return false;
|
||||
rule->action = FILTER_ACTION_EXCLUDE;
|
||||
rule->sides = FILTER_SIDE_SENDER | FILTER_SIDE_RECEIVER;
|
||||
rule->pattern = str_dup(base);
|
||||
if (!rule->pattern || !filter_rule_set_owner(rule, "")) {
|
||||
filter_rule_free(rule);
|
||||
return false;
|
||||
}
|
||||
if (!filter_rule_list_add(list, rule)) {
|
||||
filter_rule_free(rule);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Register a per-directory merge-file basename (for "dir-merge NAME"/": NAME"
|
||||
* and -F's .rsync-filter). Duplicate names are ignored. */
|
||||
bool filter_rule_list_add_dir_merge(FilterRuleList* list, const char* name) {
|
||||
return filter_rule_list_add_dir_merge_ex(list, name, false, false, false, false, false);
|
||||
}
|
||||
|
||||
bool filter_rule_list_add_dir_merge_ex(FilterRuleList* list, const char* name, bool no_prefixes,
|
||||
bool include, bool word_split, bool no_inherit,
|
||||
bool exclude_self) {
|
||||
if (!list || !name || name[0] == '\0')
|
||||
return false;
|
||||
for (int i = 0; i < list->dir_merge_count; i++) {
|
||||
if (strcmp(list->dir_merge_names[i], name) == 0)
|
||||
if (strcmp(list->dir_merges[i].name, name) == 0) {
|
||||
/* rsync keeps the first registration (first-wins), but the 'e' modifier
|
||||
is a list side effect, not a registration field: honor it on the
|
||||
duplicate path too, adding the implicit exclude-self rule at most
|
||||
once. */
|
||||
if (exclude_self && !list->dir_merges[i].exclude_self) {
|
||||
if (!filter_list_add_exclude_self(list, name))
|
||||
return false;
|
||||
list->dir_merges[i].exclude_self = true;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
if (list->dir_merge_count == list->dir_merge_capacity) {
|
||||
if (list->dir_merge_capacity > INT_MAX / 2)
|
||||
return false;
|
||||
int new_cap = list->dir_merge_capacity > 0 ? list->dir_merge_capacity * 2 : 4;
|
||||
char** grown = realloc(list->dir_merge_names, (size_t)new_cap * sizeof(char*));
|
||||
FilterDirMerge* grown = realloc(list->dir_merges, (size_t)new_cap * sizeof(*grown));
|
||||
if (!grown)
|
||||
return false;
|
||||
list->dir_merge_names = grown;
|
||||
list->dir_merges = grown;
|
||||
list->dir_merge_capacity = new_cap;
|
||||
}
|
||||
char* dup = str_dup(name);
|
||||
if (!dup)
|
||||
return false;
|
||||
list->dir_merge_names[list->dir_merge_count++] = dup;
|
||||
if (exclude_self && !filter_list_add_exclude_self(list, name)) {
|
||||
free(dup);
|
||||
return false;
|
||||
}
|
||||
FilterDirMerge* entry = &list->dir_merges[list->dir_merge_count];
|
||||
entry->name = dup;
|
||||
entry->no_prefixes = no_prefixes;
|
||||
entry->include = include;
|
||||
entry->word_split = word_split;
|
||||
entry->no_inherit = no_inherit;
|
||||
entry->exclude_self = exclude_self;
|
||||
list->dir_merge_count++;
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool set_rule_owner(FilterRule* rule, const char* owner) {
|
||||
bool filter_rule_set_owner(FilterRule* rule, const char* owner) {
|
||||
if (!rule)
|
||||
return false;
|
||||
char* dup = str_dup(owner ? owner : "");
|
||||
if (!dup)
|
||||
return false;
|
||||
@@ -177,10 +256,11 @@ static bool is_unsupported_modifier_char(char c) {
|
||||
return c == 'e' || c == 'n' || c == 'w';
|
||||
}
|
||||
|
||||
/* Merge-file modifiers rsync accepts on merge/dir-merge rules: 'e', 'n', 'w'
|
||||
* and '-' (do not transfer the merge file). */
|
||||
/* Merge-file modifiers rsync accepts on merge/dir-merge rules: 'e' (exclude
|
||||
* self), 'n' (no inherit), 'w' (word split), '-' (bare excludes) and '+'
|
||||
* (bare includes). */
|
||||
static bool is_merge_modifier_char(char c) {
|
||||
return c == 'e' || c == 'n' || c == 'w' || c == '-';
|
||||
return c == 'e' || c == 'n' || c == 'w' || c == '-' || c == '+';
|
||||
}
|
||||
|
||||
/* Characters that count as part of a modifier run for `kind` when deciding
|
||||
@@ -228,8 +308,10 @@ static char unsupported_modifier_in_token(const char* tok, RuleKind kind) {
|
||||
* generic syntax error so callers can emit a precise diagnostic. */
|
||||
static bool parse_rule_syntax(const char* text, RuleKind* kind, unsigned* sides,
|
||||
bool* sides_explicit, bool* negate, bool* anchored_mod,
|
||||
bool* perishable, bool* xattr, bool* cvs_inject,
|
||||
const char** pat_start, size_t* pat_len, char* bad_mod) {
|
||||
bool* perishable, bool* xattr, bool* cvs_inject, bool* no_prefixes,
|
||||
bool* include_defaults, bool* word_split, bool* no_inherit,
|
||||
bool* exclude_self, const char** pat_start, size_t* pat_len,
|
||||
char* bad_mod) {
|
||||
const char* p = text;
|
||||
*sides = FILTER_SIDE_SENDER | FILTER_SIDE_RECEIVER;
|
||||
*sides_explicit = false;
|
||||
@@ -238,6 +320,11 @@ static bool parse_rule_syntax(const char* text, RuleKind* kind, unsigned* sides,
|
||||
*perishable = false;
|
||||
*xattr = false;
|
||||
*cvs_inject = false;
|
||||
*no_prefixes = false;
|
||||
*include_defaults = false;
|
||||
*word_split = false;
|
||||
*no_inherit = false;
|
||||
*exclude_self = false;
|
||||
*pat_start = NULL;
|
||||
*pat_len = 0;
|
||||
*bad_mod = '\0';
|
||||
@@ -312,6 +399,21 @@ static bool parse_rule_syntax(const char* text, RuleKind* kind, unsigned* sides,
|
||||
case 'C':
|
||||
*cvs_inject = true;
|
||||
break;
|
||||
case '-':
|
||||
*no_prefixes = true;
|
||||
break;
|
||||
case '+':
|
||||
*include_defaults = true;
|
||||
break;
|
||||
case 'e':
|
||||
*exclude_self = true;
|
||||
break;
|
||||
case 'n':
|
||||
*no_inherit = true;
|
||||
break;
|
||||
case 'w':
|
||||
*word_split = true;
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
@@ -347,11 +449,13 @@ FilterRule* filter_rule_parse(const char* line, const FilterParseOptions* opts,
|
||||
RuleKind kind = RULE_KIND_UNKNOWN;
|
||||
unsigned sides;
|
||||
bool sides_explicit, negate, anchored_mod, perishable, xattr, cvs_inject;
|
||||
bool no_prefixes, include_defaults, word_split, no_inherit, exclude_self;
|
||||
const char* pat;
|
||||
size_t pat_len;
|
||||
char bad_mod;
|
||||
if (!parse_rule_syntax(p, &kind, &sides, &sides_explicit, &negate, &anchored_mod, &perishable,
|
||||
&xattr, &cvs_inject, &pat, &pat_len, &bad_mod)) {
|
||||
&xattr, &cvs_inject, &no_prefixes, &include_defaults, &word_split,
|
||||
&no_inherit, &exclude_self, &pat, &pat_len, &bad_mod)) {
|
||||
if (bad_mod != '\0')
|
||||
filter_set_error(err, err_size, "unsupported filter modifier '%c'", bad_mod);
|
||||
else
|
||||
@@ -504,7 +608,7 @@ static bool filter_list_append_cvs(FilterRuleList* list, unsigned sides) {
|
||||
}
|
||||
memcpy(rule->pattern, CVS_DEFAULTS[i].pattern, plen);
|
||||
rule->pattern[plen] = '\0';
|
||||
if (!set_rule_owner(rule, "")) {
|
||||
if (!filter_rule_set_owner(rule, "")) {
|
||||
filter_rule_free(rule);
|
||||
return false;
|
||||
}
|
||||
@@ -522,16 +626,138 @@ static bool filter_list_parse_append_depth(FilterRuleList* list, const char* lin
|
||||
const FilterParseOptions* opts, const char* base_dir,
|
||||
int depth, char* err, size_t err_size);
|
||||
|
||||
/* Read a merge file and splice its rules into `list`. A relative path is
|
||||
* resolved below `base_dir` when given, else used as-is (rsync resolves a
|
||||
* command-line merge file relative to the current directory). */
|
||||
/* Append one merge-file token/line to `list`, honoring the merge rule's
|
||||
* no-prefix/include mode. In no-prefix mode the token is a bare pattern whose
|
||||
* include/exclude default comes from the merge rule (rsync's "-"/"+" merge
|
||||
* modifiers); otherwise the token is parsed as a full filter rule. */
|
||||
static bool filter_merge_append_token(FilterRuleList* list, const char* token,
|
||||
const FilterDirMerge* spec, const FilterParseOptions* opts,
|
||||
const char* base_dir, int depth, char* err, size_t err_size) {
|
||||
if (spec->no_prefixes || spec->include) {
|
||||
size_t tlen = strlen(token);
|
||||
char* text = malloc(tlen + 3);
|
||||
if (!text) {
|
||||
filter_set_error(err, err_size, "memory allocation failed");
|
||||
return false;
|
||||
}
|
||||
text[0] = spec->include ? '+' : '-';
|
||||
text[1] = ' ';
|
||||
memcpy(text + 2, token, tlen + 1);
|
||||
bool ok = filter_list_parse_append_depth(list, text, opts, base_dir, depth + 1, err, err_size);
|
||||
free(text);
|
||||
return ok;
|
||||
}
|
||||
return filter_list_parse_append_depth(list, token, opts, base_dir, depth + 1, err, err_size);
|
||||
}
|
||||
|
||||
/* Read a merge file's tokens/lines into `list` for `spec`. A `w` merge rule
|
||||
* word-splits on whitespace (turning comments off); otherwise lines are parsed
|
||||
* and whole-line `#` comments skipped. When `owner_rel` is non-NULL the newly
|
||||
* added rules are owned by that directory; a no-inherit spec marks them so they
|
||||
* apply only there. Returns false on parse/allocation failure. */
|
||||
static bool filter_merge_read(FilterRuleList* list, FILE* fp, const char* display_path,
|
||||
const FilterDirMerge* spec, const FilterParseOptions* opts,
|
||||
const char* base_dir, const char* owner_rel, int depth, char* err,
|
||||
size_t err_size) {
|
||||
/* Lowest list index this read is responsible for. A "clear"/"!" inside the
|
||||
* file resets list->count to 0 (freeing the caller's earlier rules too), so
|
||||
* the base must follow it down: otherwise post-clear rules sit below the
|
||||
* original count and never receive an owner (nor no-inherit) and are missed
|
||||
* by the rollback. */
|
||||
int floor = list->count;
|
||||
char* line = NULL;
|
||||
size_t cap = 0;
|
||||
bool ok = true;
|
||||
while (ok) {
|
||||
ssize_t n = utils_getdelim_bounded(fp, &line, &cap, '\n', UTILS_MAX_LINE_LEN);
|
||||
if (n < 0) {
|
||||
if (errno == EFBIG)
|
||||
filter_set_error(err, err_size, "line in %s exceeds %d bytes", display_path,
|
||||
(int)UTILS_MAX_LINE_LEN);
|
||||
else
|
||||
filter_set_error(err, err_size, "error reading %s: %s", display_path, strerror(errno));
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
if (n == 0)
|
||||
break;
|
||||
if (spec->word_split) {
|
||||
/* Whitespace-separated tokens; newlines are ordinary separators and
|
||||
* comments are disabled. */
|
||||
const char* s = line;
|
||||
while (*s) {
|
||||
while (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r')
|
||||
s++;
|
||||
if (*s == '\0')
|
||||
break;
|
||||
const char* start = s;
|
||||
while (*s != '\0' && *s != ' ' && *s != '\t' && *s != '\n' && *s != '\r')
|
||||
s++;
|
||||
size_t tlen = (size_t)(s - start);
|
||||
char* token = malloc(tlen + 1);
|
||||
if (!token) {
|
||||
filter_set_error(err, err_size, "memory allocation failed");
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
memcpy(token, start, tlen);
|
||||
token[tlen] = '\0';
|
||||
if (!filter_merge_append_token(list, token, spec, opts, base_dir, depth, err, err_size))
|
||||
ok = false;
|
||||
if (list->count < floor)
|
||||
floor = list->count; /* a "clear" reset the list below this read's base */
|
||||
free(token);
|
||||
}
|
||||
} else {
|
||||
const char* lp = line;
|
||||
while (*lp == ' ' || *lp == '\t')
|
||||
lp++;
|
||||
if (*lp == '\0' || *lp == '\n' || *lp == '\r' || *lp == '#')
|
||||
continue;
|
||||
if (!filter_merge_append_token(list, lp, spec, opts, base_dir, depth, err, err_size))
|
||||
ok = false;
|
||||
if (list->count < floor)
|
||||
floor = list->count; /* a "clear" reset the list below this read's base */
|
||||
}
|
||||
}
|
||||
free(line);
|
||||
if (!ok) {
|
||||
/* Drop every live rule this read is responsible for. After a "clear" that
|
||||
* base is 0, so the post-clear rules are freed too instead of leaking. */
|
||||
for (int i = floor; i < list->count; i++)
|
||||
filter_rule_free(list->items[i]);
|
||||
list->count = floor;
|
||||
return false;
|
||||
}
|
||||
for (int i = floor; i < list->count; i++) {
|
||||
FilterRule* rule = list->items[i];
|
||||
if (spec->no_inherit)
|
||||
rule->no_inherit = true;
|
||||
if (owner_rel && !filter_rule_set_owner(rule, owner_rel)) {
|
||||
filter_set_error(err, err_size, "memory allocation failed");
|
||||
for (int j = floor; j < list->count; j++)
|
||||
filter_rule_free(list->items[j]);
|
||||
list->count = floor;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Read a single-instance merge file and splice its rules into `list`. A
|
||||
* relative path is resolved below `base_dir` when given, else used as-is (rsync
|
||||
* resolves a command-line merge file relative to the current directory). */
|
||||
static bool filter_list_merge_file(FilterRuleList* list, const char* name,
|
||||
const FilterParseOptions* opts, const char* base_dir, int depth,
|
||||
char* err, size_t err_size) {
|
||||
const FilterDirMerge* spec, const FilterParseOptions* opts,
|
||||
const char* base_dir, int depth, char* err, size_t err_size) {
|
||||
if (name[0] == '\0') {
|
||||
filter_set_error(err, err_size, "merge requires a filename");
|
||||
return false;
|
||||
}
|
||||
if (spec->exclude_self && !filter_list_add_exclude_self(list, name)) {
|
||||
filter_set_error(err, err_size, "memory allocation failed");
|
||||
return false;
|
||||
}
|
||||
char* path =
|
||||
(base_dir && base_dir[0] && name[0] != '/') ? path_cat(base_dir, name) : str_dup(name);
|
||||
if (!path) {
|
||||
@@ -544,29 +770,7 @@ static bool filter_list_merge_file(FilterRuleList* list, const char* name,
|
||||
free(path);
|
||||
return false;
|
||||
}
|
||||
char* line = NULL;
|
||||
size_t cap = 0;
|
||||
bool ok = true;
|
||||
while (true) {
|
||||
ssize_t n = utils_getdelim_bounded(fp, &line, &cap, '\n', UTILS_MAX_LINE_LEN);
|
||||
if (n < 0) {
|
||||
filter_set_error(err, err_size, "error reading merge file '%s'", path);
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
if (n == 0)
|
||||
break;
|
||||
const char* lp = line;
|
||||
while (*lp == ' ' || *lp == '\t')
|
||||
lp++;
|
||||
if (*lp == '\0' || *lp == '\n' || *lp == '\r' || *lp == '#')
|
||||
continue;
|
||||
if (!filter_list_parse_append_depth(list, lp, opts, base_dir, depth + 1, err, err_size)) {
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
free(line);
|
||||
bool ok = filter_merge_read(list, fp, path, spec, opts, base_dir, NULL, depth, err, err_size);
|
||||
fclose(fp);
|
||||
free(path);
|
||||
return ok;
|
||||
@@ -590,11 +794,13 @@ static bool filter_list_parse_append_depth(FilterRuleList* list, const char* lin
|
||||
RuleKind kind = RULE_KIND_UNKNOWN;
|
||||
unsigned sides;
|
||||
bool sides_explicit, negate, anchored_mod, perishable, xattr, cvs_inject;
|
||||
bool no_prefixes, include_defaults, word_split, no_inherit, exclude_self;
|
||||
const char* pat;
|
||||
size_t pat_len;
|
||||
char bad_mod;
|
||||
if (!parse_rule_syntax(p, &kind, &sides, &sides_explicit, &negate, &anchored_mod, &perishable,
|
||||
&xattr, &cvs_inject, &pat, &pat_len, &bad_mod)) {
|
||||
&xattr, &cvs_inject, &no_prefixes, &include_defaults, &word_split,
|
||||
&no_inherit, &exclude_self, &pat, &pat_len, &bad_mod)) {
|
||||
if (bad_mod != '\0')
|
||||
filter_set_error(err, err_size, "unsupported filter modifier '%c': %s", bad_mod, p);
|
||||
else
|
||||
@@ -640,7 +846,15 @@ static bool filter_list_parse_append_depth(FilterRuleList* list, const char* lin
|
||||
}
|
||||
memcpy(name, pat, pat_len);
|
||||
name[pat_len] = '\0';
|
||||
bool ok = filter_list_merge_file(list, name, opts, base_dir, depth, err, err_size);
|
||||
/* A single-instance merge has no inheritance, so 'n' is meaningless; the
|
||||
* other merge modifiers still shape how the file is read. */
|
||||
FilterDirMerge spec = {.name = name,
|
||||
.no_prefixes = no_prefixes,
|
||||
.include = include_defaults,
|
||||
.word_split = word_split,
|
||||
.no_inherit = false,
|
||||
.exclude_self = exclude_self};
|
||||
bool ok = filter_list_merge_file(list, name, &spec, opts, base_dir, depth, err, err_size);
|
||||
free(name);
|
||||
return ok;
|
||||
}
|
||||
@@ -656,7 +870,8 @@ static bool filter_list_parse_append_depth(FilterRuleList* list, const char* lin
|
||||
}
|
||||
memcpy(name, pat, pat_len);
|
||||
name[pat_len] = '\0';
|
||||
bool ok = filter_rule_list_add_dir_merge(list, name);
|
||||
bool ok = filter_rule_list_add_dir_merge_ex(list, name, no_prefixes, include_defaults,
|
||||
word_split, no_inherit, exclude_self);
|
||||
free(name);
|
||||
if (!ok) {
|
||||
filter_set_error(err, err_size, "memory allocation failed");
|
||||
@@ -717,27 +932,30 @@ FilterRuleList* filter_base_build(const char* const* rule_texts, int rule_count,
|
||||
/* Undo the rules and dir-merge registrations that one merge file appended,
|
||||
* leaving the caller's earlier content intact. A "clear" rule inside the file
|
||||
* frees every rule, including the caller's; clamp to the surviving count so
|
||||
* those already-freed rules are never resurrected and freed a second time. */
|
||||
* those already-freed rules are never resurrected and freed a second time.
|
||||
* (filter_merge_read() has already rolled its own range back by the time this
|
||||
* runs, so on a post-clear failure `list->count` is below `rules_before` and
|
||||
* this is a no-op for the rules.) */
|
||||
static void filter_file_rollback(FilterRuleList* list, int rules_before, int dir_merges_before) {
|
||||
int first = rules_before < list->count ? rules_before : list->count;
|
||||
for (int i = first; i < list->count; i++)
|
||||
filter_rule_free(list->items[i]);
|
||||
list->count = first;
|
||||
for (int i = dir_merges_before; i < list->dir_merge_count; i++)
|
||||
free(list->dir_merge_names[i]);
|
||||
free(list->dir_merges[i].name);
|
||||
list->dir_merge_count = dir_merges_before;
|
||||
}
|
||||
|
||||
bool filter_file_append(FilterRuleList* list, const char* dir_path, const char* name,
|
||||
bool filter_dir_merge_append(FilterRuleList* list, const char* dir_path, const FilterDirMerge* spec,
|
||||
const char* owner_rel, const FilterParseOptions* opts, bool* exists,
|
||||
char* err, size_t err_size) {
|
||||
if (err && err_size > 0)
|
||||
err[0] = '\0';
|
||||
if (exists)
|
||||
*exists = false;
|
||||
if (!list)
|
||||
if (!list || !spec || !spec->name)
|
||||
return false;
|
||||
char* filter_path = path_cat(dir_path, name);
|
||||
char* filter_path = path_cat(dir_path, spec->name);
|
||||
if (!filter_path) {
|
||||
filter_set_error(err, err_size, "memory allocation failed");
|
||||
return false;
|
||||
@@ -748,7 +966,7 @@ bool filter_file_append(FilterRuleList* list, const char* dir_path, const char*
|
||||
if (errno == ENOENT || errno == ENOTDIR)
|
||||
return true;
|
||||
char* escaped_dir = output_escape(dir_path, log_get_8_bit_output());
|
||||
log_message(LOG_LEVEL_WARNING, "Could not read %s in %s: %s", name,
|
||||
log_message(LOG_LEVEL_WARNING, "Could not read %s in %s: %s", spec->name,
|
||||
escaped_dir ? escaped_dir : "<allocation failed>", strerror(errno));
|
||||
free(escaped_dir);
|
||||
return true;
|
||||
@@ -757,51 +975,25 @@ bool filter_file_append(FilterRuleList* list, const char* dir_path, const char*
|
||||
*exists = true;
|
||||
int rules_before = list->count;
|
||||
int dir_merges_before = list->dir_merge_count;
|
||||
char* line = NULL;
|
||||
size_t line_cap = 0;
|
||||
bool ok = true;
|
||||
while (true) {
|
||||
ssize_t n = utils_getdelim_bounded(fp, &line, &line_cap, '\n', UTILS_MAX_LINE_LEN);
|
||||
if (n < 0) {
|
||||
if (errno == EFBIG) {
|
||||
filter_set_error(err, err_size, "line in %s exceeds %d bytes", name,
|
||||
(int)UTILS_MAX_LINE_LEN);
|
||||
} else {
|
||||
filter_set_error(err, err_size, "error reading %s: %s", name, strerror(errno));
|
||||
}
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
if (n == 0)
|
||||
break;
|
||||
const char* p = line;
|
||||
while (*p == ' ' || *p == '\t')
|
||||
p++;
|
||||
if (*p == '\0' || *p == '\n' || *p == '\r' || *p == '#')
|
||||
continue;
|
||||
/* Merge files inside a per-directory file resolve relative to that
|
||||
directory. */
|
||||
if (!filter_list_parse_append_depth(list, p, opts, dir_path, 0, err, err_size)) {
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
free(line);
|
||||
bool ok =
|
||||
filter_merge_read(list, fp, spec->name, spec, opts, dir_path, owner_rel, 0, err, err_size);
|
||||
fclose(fp);
|
||||
if (!ok) {
|
||||
filter_file_rollback(list, rules_before, dir_merges_before);
|
||||
return false;
|
||||
}
|
||||
for (int i = rules_before; i < list->count; i++) {
|
||||
if (!set_rule_owner(list->items[i], owner_rel)) {
|
||||
filter_set_error(err, err_size, "memory allocation failed");
|
||||
filter_file_rollback(list, rules_before, dir_merges_before);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool filter_file_append(FilterRuleList* list, const char* dir_path, const char* name,
|
||||
const char* owner_rel, const FilterParseOptions* opts, bool* exists,
|
||||
char* err, size_t err_size) {
|
||||
FilterDirMerge spec = {.name = (char*)name};
|
||||
return filter_dir_merge_append(list, dir_path, &spec, owner_rel, opts, exists, err, err_size);
|
||||
}
|
||||
|
||||
FilterRuleList* filter_file_read_named(const char* dir_path, const char* name,
|
||||
const char* owner_rel, const FilterParseOptions* opts,
|
||||
bool* exists, char* err, size_t err_size) {
|
||||
@@ -843,11 +1035,16 @@ static FilterAction rule_matches(const FilterRule* rule, const char* rel_path, c
|
||||
return FILTER_ACTION_NONE;
|
||||
if (!(rule->sides & side))
|
||||
return FILTER_ACTION_NONE;
|
||||
/* A rule applies only to entries below its owner directory. */
|
||||
/* A rule applies only to entries below its owner directory. The receive
|
||||
* root's destination-relative coordinate may be written as "." (the
|
||||
* synced-directory sentinel), which is the same scope as the empty owner. */
|
||||
const char* owner = rule->owner;
|
||||
if (owner && strcmp(owner, ".") == 0)
|
||||
owner = "";
|
||||
const char* rel2 = rel_path;
|
||||
if (rule->owner && rule->owner[0] != '\0') {
|
||||
size_t owner_len = strlen(rule->owner);
|
||||
if (strncmp(rule->owner, rel_path, owner_len) != 0)
|
||||
if (owner && owner[0] != '\0') {
|
||||
size_t owner_len = strlen(owner);
|
||||
if (strncmp(owner, rel_path, owner_len) != 0)
|
||||
return FILTER_ACTION_NONE;
|
||||
if (rel_path[owner_len] != '/')
|
||||
return FILTER_ACTION_NONE;
|
||||
@@ -855,6 +1052,10 @@ static FilterAction rule_matches(const FilterRule* rule, const char* rel_path, c
|
||||
}
|
||||
if (rel2[0] == '\0')
|
||||
return FILTER_ACTION_NONE;
|
||||
/* A no-inherit rule ('n' on its dir-merge) applies only to direct children of
|
||||
* its owner directory, never to deeper entries. */
|
||||
if (rule->no_inherit && strchr(rel2, '/') != NULL)
|
||||
return FILTER_ACTION_NONE;
|
||||
bool matched;
|
||||
if (rule->dir_only && !is_dir)
|
||||
matched = false;
|
||||
@@ -884,3 +1085,45 @@ FilterAction filter_rules_apply_side(const FilterRuleList* list, const char* rel
|
||||
}
|
||||
return FILTER_ACTION_NONE;
|
||||
}
|
||||
|
||||
FilterAction filter_dir_rules_apply_side(const FilterRuleList* dir_rules, const char* rel_path,
|
||||
const char* leaf, bool is_dir) {
|
||||
if (!dir_rules || !rel_path)
|
||||
return FILTER_ACTION_NONE;
|
||||
size_t len = strlen(rel_path);
|
||||
if (len == 0)
|
||||
return FILTER_ACTION_NONE;
|
||||
/* The containing directory of rel_path is the prefix before its final '/'. */
|
||||
size_t owner_len = 0;
|
||||
for (size_t i = 0; i < len; i++) {
|
||||
if (rel_path[i] == '/')
|
||||
owner_len = i;
|
||||
}
|
||||
for (;;) {
|
||||
for (int i = 0; i < dir_rules->count; i++) {
|
||||
const FilterRule* rule = dir_rules->items[i];
|
||||
const char* rule_owner = rule && rule->owner ? rule->owner : "";
|
||||
/* "." is the receive root's coordinate (see rule_matches). */
|
||||
if (strcmp(rule_owner, ".") == 0)
|
||||
rule_owner = "";
|
||||
size_t rule_owner_len = strlen(rule_owner);
|
||||
if (rule_owner_len != owner_len)
|
||||
continue;
|
||||
if (owner_len != 0 && memcmp(rule_owner, rel_path, owner_len) != 0)
|
||||
continue;
|
||||
FilterAction action = rule_matches(rule, rel_path, leaf, is_dir, FILTER_SIDE_RECEIVER);
|
||||
if (action != FILTER_ACTION_NONE)
|
||||
return action;
|
||||
}
|
||||
if (owner_len == 0)
|
||||
break;
|
||||
/* Move to the parent directory: the last '/' before owner_len. */
|
||||
size_t parent = 0;
|
||||
for (size_t j = 0; j < owner_len; j++) {
|
||||
if (rel_path[j] == '/')
|
||||
parent = j;
|
||||
}
|
||||
owner_len = parent;
|
||||
}
|
||||
return FILTER_ACTION_NONE;
|
||||
}
|
||||
|
||||
+61
-11
@@ -25,11 +25,12 @@
|
||||
* Modifiers: '/' absolute anchor, '!' negate match, 'C' inject CVS defaults,
|
||||
* 's' sender side, 'r' receiver side, 'p' perishable. The rsync 'x'
|
||||
* (xattr-name) modifier is not implemented and is rejected explicitly
|
||||
* everywhere. The merge-file modifiers 'e' (exclude the merge file itself),
|
||||
* 'n' (do not inherit the merge file), 'w' (word-split the merge file) and '-'
|
||||
* (do not transfer the merge file) are accepted and consumed only on merge/
|
||||
* dir-merge rules (rejected on every other rule, matching rsync); their
|
||||
* semantics are not implemented and they are otherwise ignored.
|
||||
* everywhere. The merge-only modifiers are accepted only on merge/dir-merge
|
||||
* rules (rejected on every other rule, matching rsync): 'e' excludes the merge
|
||||
* file itself, 'n' makes the merged rules non-inheriting (they apply only to
|
||||
* the directory that holds the merge file), 'w' word-splits the merge file on
|
||||
* whitespace instead of lines, and '-' reads the merge file as a list of bare
|
||||
* exclude patterns with no rule prefixes.
|
||||
* A trailing '/' makes a pattern match directories only. A leading '/' anchors
|
||||
* the pattern to its owner directory.
|
||||
*/
|
||||
@@ -55,18 +56,32 @@ typedef struct {
|
||||
bool dir_only; /* pattern had a trailing '/': matches directories only */
|
||||
bool negate; /* '!' modifier: match succeeds when the pattern does not */
|
||||
bool perishable; /* 'p' modifier (ignored in deleted directories) */
|
||||
bool no_inherit; /* 'n' on the owning dir-merge: applies only in `owner` */
|
||||
char* owner; /* owning directory rel path ("" == transfer root) */
|
||||
char* pattern; /* cleaned glob pattern (no leading '/', no trailing '/') */
|
||||
} FilterRule;
|
||||
|
||||
/* One per-directory merge-file registration ("dir-merge NAME"/": NAME", "merge
|
||||
* NAME"/". NAME" and -F's .rsync-filter) together with the merge-only modifiers
|
||||
* parsed from the rule. The scanner reads `name` in every directory it
|
||||
* traverses and applies `no_prefixes`/`include`/`word_split`/`no_inherit`/
|
||||
* `exclude_self` while merging the file's rules. */
|
||||
typedef struct {
|
||||
char* name;
|
||||
bool no_prefixes; /* '-' : file holds only bare exclude patterns */
|
||||
bool include; /* '+' : file holds only bare include patterns */
|
||||
bool word_split; /* 'w' : split the file on whitespace, not lines */
|
||||
bool no_inherit; /* 'n' : the merged rules do not inherit below their dir */
|
||||
bool exclude_self; /* 'e' : exclude the merge file itself from the transfer */
|
||||
} FilterDirMerge;
|
||||
|
||||
typedef struct FilterRuleList {
|
||||
FilterRule** items; /* owned array of rule pointers */
|
||||
int count;
|
||||
int capacity;
|
||||
/* Per-directory merge-file basenames registered by "dir-merge NAME"/": NAME"
|
||||
* or by -F (.rsync-filter). Owned strings; the scanner reads each name in
|
||||
* every directory it traverses. */
|
||||
char** dir_merge_names;
|
||||
/* Per-directory merge-file registrations. Owned; the scanner reads each
|
||||
* name in every directory it traverses. */
|
||||
FilterDirMerge* dir_merges;
|
||||
int dir_merge_count;
|
||||
int dir_merge_capacity;
|
||||
} FilterRuleList;
|
||||
@@ -83,13 +98,28 @@ typedef struct {
|
||||
FilterRule* filter_rule_parse(const char* line, const FilterParseOptions* opts, char* err,
|
||||
size_t err_size);
|
||||
void filter_rule_free(FilterRule* rule);
|
||||
/* Deep-copy a rule (owned pattern/owner). Returns NULL on allocation failure. */
|
||||
FilterRule* filter_rule_clone(const FilterRule* rule);
|
||||
/* Replace a rule's owner directory (owned copy of `owner`, "" for the transfer
|
||||
* root). Returns false on allocation failure, leaving the rule unchanged.
|
||||
* Used to re-express a mirrored per-directory rule in the receiver's
|
||||
* destination-relative coordinate system. */
|
||||
bool filter_rule_set_owner(FilterRule* rule, const char* owner);
|
||||
|
||||
FilterRuleList* filter_rule_list_create(void);
|
||||
/* Append a fully-parsed rule (takes ownership). Returns false on OOM. */
|
||||
bool filter_rule_list_add(FilterRuleList* list, FilterRule* rule);
|
||||
/* Register a per-directory merge-file basename (idempotent). Returns false on
|
||||
* OOM. Used by the scanner to read custom "dir-merge" files. */
|
||||
/* Register a per-directory merge-file basename (idempotent, no modifiers).
|
||||
* Returns false on OOM. Used by the scanner to read custom "dir-merge" files. */
|
||||
bool filter_rule_list_add_dir_merge(FilterRuleList* list, const char* name);
|
||||
/* Register a per-directory merge file with its merge-only modifiers. On a
|
||||
* duplicate name the existing registration is kept (rsync's first wins) and true
|
||||
* is returned. When `exclude_self` is set an implicit exclude rule for the
|
||||
* merge file's basename is appended to the list at this position, matching
|
||||
* rsync's `e` modifier. Returns false on OOM. */
|
||||
bool filter_rule_list_add_dir_merge_ex(FilterRuleList* list, const char* name, bool no_prefixes,
|
||||
bool include, bool word_split, bool no_inherit,
|
||||
bool exclude_self);
|
||||
/* Parse `line` and append it. Handles "clear"/"!" (resets the list), "merge
|
||||
* FILE"/". FILE" (splices the file's rules) and "dir-merge NAME"/": NAME"
|
||||
* (registers a per-directory filename). Returns false and fills `err` on bad
|
||||
@@ -122,6 +152,15 @@ bool filter_file_append(FilterRuleList* list, const char* dir_path, const char*
|
||||
const char* owner_rel, const FilterParseOptions* opts, bool* exists,
|
||||
char* err, size_t err_size);
|
||||
|
||||
/* Append a per-directory merge file honoring its merge-only modifiers: `-`
|
||||
* reads every (word-split, when `w`) token as a bare exclude, `+` as a bare
|
||||
* include, and `n` marks each read rule non-inheriting. A plain name behaves
|
||||
* like filter_file_append. A missing file yields *exists=false with no error;
|
||||
* returns false only on a parse/allocation failure (message in `err`). */
|
||||
bool filter_dir_merge_append(FilterRuleList* list, const char* dir_path, const FilterDirMerge* spec,
|
||||
const char* owner_rel, const FilterParseOptions* opts, bool* exists,
|
||||
char* err, size_t err_size);
|
||||
|
||||
/* filter_file_read_named with the default ".rsync-filter" name. */
|
||||
FilterRuleList* filter_file_read(const char* dir_path, const char* owner_rel, bool* exists,
|
||||
char* err, size_t err_size);
|
||||
@@ -135,4 +174,15 @@ FilterRuleList* filter_file_read(const char* dir_path, const char* owner_rel, bo
|
||||
FilterAction filter_rules_apply_side(const FilterRuleList* list, const char* rel_path,
|
||||
const char* leaf, bool is_dir, unsigned side);
|
||||
|
||||
/* Evaluate a received per-directory rule set for the receiver side, using
|
||||
* rsync's per-directory-before-ancestors order: the entry's containing
|
||||
* directory's rules are tried first, then each ancestor's, then the receive
|
||||
* root's. `dir_rules` is a flat list whose rules carry `owner`; a rule applies
|
||||
* only when `owner` is exactly the directory being examined (a no-inherit rule
|
||||
* therefore applies only to that directory's direct children). Returns the
|
||||
* first matching rule's receiver verdict (PROTECT/RISK) or FILTER_ACTION_NONE.
|
||||
* The caller evaluates the command-line base rules after this chain. */
|
||||
FilterAction filter_dir_rules_apply_side(const FilterRuleList* dir_rules, const char* rel_path,
|
||||
const char* leaf, bool is_dir);
|
||||
|
||||
#endif
|
||||
|
||||
+16
-6
@@ -62,14 +62,16 @@ bool format_dest_state_send(int fd, const OutputDestState* state) {
|
||||
if (!state)
|
||||
return false;
|
||||
int32_t has_old = state->existed ? 1 : 0;
|
||||
int32_t target_matches = state->target_matches ? 1 : 0;
|
||||
uint64_t size = (uint64_t)state->size;
|
||||
int64_t mtime = (int64_t)state->mtime_sec;
|
||||
int64_t mtime_nsec = state->mtime_nsec;
|
||||
uint32_t mode = state->mode;
|
||||
int32_t uid = state->uid;
|
||||
int32_t gid = state->gid;
|
||||
return send_n_data(fd, &has_old, sizeof(has_old)) && send_n_data(fd, &size, sizeof(size)) &&
|
||||
send_n_data(fd, &mtime, sizeof(mtime)) &&
|
||||
return send_n_data(fd, &has_old, sizeof(has_old)) &&
|
||||
send_n_data(fd, &target_matches, sizeof(target_matches)) &&
|
||||
send_n_data(fd, &size, sizeof(size)) && send_n_data(fd, &mtime, sizeof(mtime)) &&
|
||||
send_n_data(fd, &mtime_nsec, sizeof(mtime_nsec)) && send_n_data(fd, &mode, sizeof(mode)) &&
|
||||
send_n_data(fd, &uid, sizeof(uid)) && send_n_data(fd, &gid, sizeof(gid));
|
||||
}
|
||||
@@ -78,14 +80,16 @@ bool format_dest_state_receive(int fd, OutputDestState* state) {
|
||||
if (!state)
|
||||
return false;
|
||||
int32_t has_old = 0;
|
||||
int32_t target_matches = 0;
|
||||
uint64_t size = 0;
|
||||
int64_t mtime = 0;
|
||||
int64_t mtime_nsec = 0;
|
||||
uint32_t mode = 0;
|
||||
int32_t uid = 0;
|
||||
int32_t gid = 0;
|
||||
if (!receive_n_data(fd, &has_old, sizeof(has_old)) || !receive_n_data(fd, &size, sizeof(size)) ||
|
||||
!receive_n_data(fd, &mtime, sizeof(mtime)) ||
|
||||
if (!receive_n_data(fd, &has_old, sizeof(has_old)) ||
|
||||
!receive_n_data(fd, &target_matches, sizeof(target_matches)) ||
|
||||
!receive_n_data(fd, &size, sizeof(size)) || !receive_n_data(fd, &mtime, sizeof(mtime)) ||
|
||||
!receive_n_data(fd, &mtime_nsec, sizeof(mtime_nsec)) ||
|
||||
!receive_n_data(fd, &mode, sizeof(mode)) || !receive_n_data(fd, &uid, sizeof(uid)) ||
|
||||
!receive_n_data(fd, &gid, sizeof(gid)))
|
||||
@@ -93,6 +97,7 @@ bool format_dest_state_receive(int fd, OutputDestState* state) {
|
||||
memset(state, 0, sizeof(*state));
|
||||
state->known = true;
|
||||
state->existed = has_old != 0;
|
||||
state->target_matches = target_matches != 0;
|
||||
state->size = size;
|
||||
state->mtime_sec = mtime;
|
||||
state->mtime_nsec = mtime_nsec;
|
||||
@@ -105,9 +110,10 @@ bool format_dest_state_receive(int fd, OutputDestState* state) {
|
||||
bool format_stats_send(int fd, const ReceiverStats* stats) {
|
||||
if (!stats)
|
||||
return false;
|
||||
unsigned long long fields[8] = {
|
||||
unsigned long long fields[12] = {
|
||||
stats->matched_data, stats->deleted_files, stats->would_delete_count, stats->literal_bytes,
|
||||
stats->created_reg, stats->created_dir, stats->created_link, stats->created_special,
|
||||
stats->deleted_reg, stats->deleted_dir, stats->deleted_link, stats->deleted_special,
|
||||
};
|
||||
return send_n_data(fd, fields, sizeof(fields));
|
||||
}
|
||||
@@ -115,7 +121,7 @@ bool format_stats_send(int fd, const ReceiverStats* stats) {
|
||||
bool format_stats_receive(int fd, ReceiverStats* stats) {
|
||||
if (!stats)
|
||||
return false;
|
||||
unsigned long long fields[8] = {0};
|
||||
unsigned long long fields[12] = {0};
|
||||
if (!receive_n_data(fd, fields, sizeof(fields)))
|
||||
return false;
|
||||
memset(stats, 0, sizeof(*stats));
|
||||
@@ -127,5 +133,9 @@ bool format_stats_receive(int fd, ReceiverStats* stats) {
|
||||
stats->created_dir = fields[5];
|
||||
stats->created_link = fields[6];
|
||||
stats->created_special = fields[7];
|
||||
stats->deleted_reg = fields[8];
|
||||
stats->deleted_dir = fields[9];
|
||||
stats->deleted_link = fields[10];
|
||||
stats->deleted_special = fields[11];
|
||||
return true;
|
||||
}
|
||||
|
||||
+25
-6
@@ -17,10 +17,18 @@
|
||||
/* Pre-transfer destination snapshot, reported by the receiver when the wire
|
||||
* config carries report_dest_info. `known` distinguishes "no report was
|
||||
* requested/received" from "the destination did not exist" (`existed == false`
|
||||
* with `known == true`). */
|
||||
* with `known == true`).
|
||||
*
|
||||
* `target_matches` is meaningful only for a symlink destination (protocol
|
||||
* 2.30.0): the receiver compares its on-disk link target with the incoming
|
||||
* target and reports whether they are equal, so the sender can render rsync's
|
||||
* `cLc........` (target changed) versus `.L..t......` (attributes only) and
|
||||
* suppress an unchanged symlink's line entirely. It is always false for every
|
||||
* other entry kind. */
|
||||
typedef struct {
|
||||
bool known;
|
||||
bool existed;
|
||||
bool target_matches;
|
||||
unsigned long long size;
|
||||
long long mtime_sec;
|
||||
long long mtime_nsec;
|
||||
@@ -57,17 +65,24 @@ bool format_dest_state_send(int fd, const OutputDestState* state);
|
||||
bool format_dest_state_receive(int fd, OutputDestState* state);
|
||||
|
||||
/* End-of-transfer receiver counters reported through STATUS_STATS (protocol
|
||||
* 2.25.0, extended in 2.28.0) when the wire config carries report_stats.
|
||||
* `would_delete_count` is the number of destination-relative paths the receiver
|
||||
* would have deleted in a -n/--dry-run --delete run; that many wire strings
|
||||
* immediately follow the fixed record (sent/read by the caller).
|
||||
* 2.25.0, extended in 2.28.0 and 2.30.0) when the wire config carries
|
||||
* report_stats. `would_delete_count` is the number of destination-relative
|
||||
* paths the receiver would have deleted in a -n/--dry-run --delete run; that
|
||||
* many wire strings immediately follow the fixed record (sent/read by the
|
||||
* caller).
|
||||
*
|
||||
* Protocol 2.28.0 adds the receiver-observed counters the sender cannot see:
|
||||
* `literal_bytes` is the file data the receiver actually stored literally
|
||||
* (whole files plus the literal fragments of a delta) and the four `created_*`
|
||||
* counters split the destination entries the receiver newly created by type,
|
||||
* reproducing rsync's `Number of created files` breakdown and an exact
|
||||
* `Literal data` for a delta run. */
|
||||
* `Literal data` for a delta run.
|
||||
*
|
||||
* Protocol 2.30.0 appends the four `deleted_*` counters: the same reg/dir/link/
|
||||
* special split for the entries the receiver ACTUALLY removed, so `--stats` can
|
||||
* render rsync's `Number of deleted files: X (reg: A, dir: B, link: C,
|
||||
* special: D)` parenthetical. The scalar `deleted_files` stays the authoritative
|
||||
* total (the breakdown is a strict partition of it). */
|
||||
typedef struct {
|
||||
unsigned long long matched_data;
|
||||
unsigned long long deleted_files;
|
||||
@@ -77,6 +92,10 @@ typedef struct {
|
||||
unsigned long long created_dir;
|
||||
unsigned long long created_link;
|
||||
unsigned long long created_special;
|
||||
unsigned long long deleted_reg;
|
||||
unsigned long long deleted_dir;
|
||||
unsigned long long deleted_link;
|
||||
unsigned long long deleted_special;
|
||||
} ReceiverStats;
|
||||
|
||||
/* Fixed-width STATUS_STATS counter record. The status frame and the optional
|
||||
|
||||
@@ -276,7 +276,7 @@ static bool identity_wire_map_valid(const IdentityMap* map) {
|
||||
}
|
||||
if (map->to < IDENTITY_CURRENT)
|
||||
return false;
|
||||
if (map->to_name && strlen(map->to_name) > 255)
|
||||
if (map->to_name && strlen(map->to_name) > IDENTITY_MAX_NAME_LEN)
|
||||
return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -6,6 +6,11 @@
|
||||
#include <stdint.h>
|
||||
#include <sys/types.h>
|
||||
|
||||
/* Maximum length of a receiver-resolved identity name in a FROM:TO map's TO
|
||||
* field. Bounded so a malicious/huge name can never cross the wire (see
|
||||
* identity_wire_map_valid). */
|
||||
#define IDENTITY_MAX_NAME_LEN 255
|
||||
|
||||
/*
|
||||
* Identity mapping: --numeric-ids / --usermap / --groupmap / --chown / --copy-as.
|
||||
*
|
||||
|
||||
+267
-249
@@ -44,99 +44,53 @@ bool receive_file_xattrs(File* file, int fd, const Config* config) {
|
||||
return true;
|
||||
}
|
||||
|
||||
static File* receive_delta_file(int fd, const Config* config, const char* check_path,
|
||||
void* old_data, unsigned long long old_size, bool* failed) {
|
||||
if (!old_data) {
|
||||
free(old_data); /* defensive: old_data is always non-NULL today */
|
||||
*failed = true;
|
||||
return NULL;
|
||||
}
|
||||
static bool receive_file_payload_into(File* file, int fd, const Config* config,
|
||||
const char* dest_path, unsigned long long expected_size);
|
||||
|
||||
DeltaSignature* sig = delta_signature_create_seeded(old_data, old_size, config->delta_block_size,
|
||||
(uint32_t)config->checksum_seed);
|
||||
if (!sig) {
|
||||
free(old_data);
|
||||
*failed = true;
|
||||
return NULL;
|
||||
}
|
||||
/* Receive a STATUS_DELTA_DATA response: the sender's delta against the basis we
|
||||
signed. Deserializes, decompresses and applies the delta (in memory or
|
||||
through a spool temp file), then receives the metadata/xattr block and
|
||||
installs the reconstructed payload. Takes ownership of `old_data` and `sig`,
|
||||
releasing both on every path. */
|
||||
static File* receive_delta_data_branch(int fd, const Config* config, const char* check_path,
|
||||
void* old_data, unsigned long long old_size, int basis_fd,
|
||||
DeltaSignature* sig, bool* failed) {
|
||||
Data* raw_delta = NULL;
|
||||
Delta* delta = NULL;
|
||||
void* new_data = NULL;
|
||||
char* spool = NULL;
|
||||
File* file = NULL;
|
||||
|
||||
Data* sig_data = delta_signature_serialize(sig);
|
||||
if (!sig_data) {
|
||||
delta_signature_destroy(sig);
|
||||
free(old_data);
|
||||
*failed = true;
|
||||
return NULL;
|
||||
}
|
||||
raw_delta = receive_data_limited(fd, MAX_RECEIVE_WHOLE_FILE_SIZE);
|
||||
if (!raw_delta)
|
||||
goto fail;
|
||||
|
||||
bool sig_sent = send_status(fd, STATUS_DELTA_SIGNATURE) && send_data(fd, sig_data);
|
||||
data_destroy(sig_data);
|
||||
|
||||
if (!sig_sent) {
|
||||
delta_signature_destroy(sig);
|
||||
free(old_data);
|
||||
*failed = true;
|
||||
return NULL;
|
||||
}
|
||||
|
||||
Status resp;
|
||||
if (!receive_status(fd, &resp)) {
|
||||
delta_signature_destroy(sig);
|
||||
free(old_data);
|
||||
*failed = true;
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (resp == STATUS_DELTA_DATA) {
|
||||
Data* delta_data = receive_data_limited(fd, MAX_RECEIVE_WHOLE_FILE_SIZE);
|
||||
if (!delta_data) {
|
||||
delta_signature_destroy(sig);
|
||||
free(old_data);
|
||||
*failed = true;
|
||||
return NULL;
|
||||
}
|
||||
|
||||
Data* raw_delta = delta_data;
|
||||
if (config->use_compression &&
|
||||
!compression_should_skip_with_suffixes(
|
||||
check_path, config->skip_compress_suffixes,
|
||||
config->skip_compress_set ? config->skip_compress_count : -1)) {
|
||||
ProtocolSession* owner = delta_data->owner;
|
||||
raw_delta = data_decompress_limited(delta_data, MAX_RECEIVE_WHOLE_FILE_SIZE);
|
||||
data_destroy(delta_data);
|
||||
if (!raw_delta) {
|
||||
free(old_data);
|
||||
delta_signature_destroy(sig);
|
||||
*failed = true;
|
||||
return NULL;
|
||||
}
|
||||
!compression_should_skip_with_suffixes(check_path, config->skip_compress_suffixes,
|
||||
config->skip_compress_set ? config->skip_compress_count
|
||||
: -1)) {
|
||||
ProtocolSession* owner = raw_delta->owner;
|
||||
Data* decompressed = data_decompress_limited(raw_delta, MAX_RECEIVE_WHOLE_FILE_SIZE);
|
||||
data_destroy(raw_delta);
|
||||
raw_delta = decompressed;
|
||||
if (!raw_delta)
|
||||
goto fail;
|
||||
/* Charge the decompressed delta to the connection budget (the paired
|
||||
wire buffer's charge was just released). */
|
||||
if (!data_charge_session(raw_delta, owner, raw_delta->size)) {
|
||||
data_destroy(raw_delta);
|
||||
free(old_data);
|
||||
delta_signature_destroy(sig);
|
||||
*failed = true;
|
||||
return NULL;
|
||||
}
|
||||
if (!data_charge_session(raw_delta, owner, raw_delta->size))
|
||||
goto fail;
|
||||
}
|
||||
|
||||
Delta* delta = delta_deserialize(raw_delta);
|
||||
delta = delta_deserialize(raw_delta);
|
||||
data_destroy(raw_delta);
|
||||
if (!delta) {
|
||||
free(old_data);
|
||||
delta_signature_destroy(sig);
|
||||
*failed = true;
|
||||
return NULL;
|
||||
}
|
||||
raw_delta = NULL;
|
||||
if (!delta)
|
||||
goto fail;
|
||||
|
||||
uint64_t new_size = delta->new_file_size;
|
||||
if (new_size > MAX_RECEIVE_WHOLE_FILE_SIZE || new_size > SIZE_MAX) {
|
||||
delta_destroy(delta);
|
||||
free(old_data);
|
||||
delta_signature_destroy(sig);
|
||||
if (new_size > SIZE_MAX) {
|
||||
send_status(fd, STATUS_ERROR);
|
||||
*failed = true;
|
||||
return NULL;
|
||||
goto fail;
|
||||
}
|
||||
/* Wire-stats tally: bytes taken straight from the basis file (matched
|
||||
delta blocks) and bytes shipped literally (protocol 2.28.0). Computed
|
||||
@@ -149,134 +103,180 @@ static File* receive_delta_file(int fd, const Config* config, const char* check_
|
||||
else if (delta->instructions[k].type == DELTA_INSTR_LITERAL)
|
||||
literal += delta->instructions[k].literal.length;
|
||||
}
|
||||
void* new_data = delta_apply(old_data, old_size, delta, config->delta_block_size);
|
||||
|
||||
/* A reconstructed file above the streaming bound is written into a spool
|
||||
temp file through delta_apply_to_fd; a smaller one keeps the historical
|
||||
in-memory reconstruction. */
|
||||
if (new_size > protocol_whole_file_receive_limit()) {
|
||||
char* dest_path = path_cat(config->receive_root_directory, check_path);
|
||||
int spool_fd = dest_path ? file_spool_for_payload(dest_path, &spool) : -1;
|
||||
free(dest_path);
|
||||
if (spool_fd < 0) {
|
||||
send_status(fd, STATUS_ERROR);
|
||||
goto fail;
|
||||
}
|
||||
bool applied =
|
||||
delta_apply_to_fd(old_data, basis_fd, old_size, delta, config->delta_block_size, spool_fd);
|
||||
if (close(spool_fd) != 0)
|
||||
applied = false;
|
||||
delta_destroy(delta);
|
||||
|
||||
if (!new_data) {
|
||||
free(old_data);
|
||||
delta_signature_destroy(sig);
|
||||
*failed = true;
|
||||
return NULL;
|
||||
delta = NULL;
|
||||
if (!applied) {
|
||||
send_status(fd, STATUS_ERROR);
|
||||
goto fail;
|
||||
}
|
||||
} else {
|
||||
new_data = old_data ? delta_apply(old_data, old_size, delta, config->delta_block_size)
|
||||
: delta_apply_fd(basis_fd, old_size, delta, config->delta_block_size);
|
||||
delta_destroy(delta);
|
||||
delta = NULL;
|
||||
if (!new_data)
|
||||
goto fail;
|
||||
}
|
||||
|
||||
File* file = file_create(check_path);
|
||||
if (!file) {
|
||||
free(new_data);
|
||||
free(old_data);
|
||||
delta_signature_destroy(sig);
|
||||
*failed = true;
|
||||
return NULL;
|
||||
}
|
||||
file = file_create(check_path);
|
||||
if (!file)
|
||||
goto fail;
|
||||
file->matched_bytes = matched;
|
||||
file->literal_bytes = literal;
|
||||
|
||||
if (config->use_metadata) {
|
||||
int meta_ok = 1;
|
||||
file->metadata = metadata_receive(fd, &meta_ok);
|
||||
if (!meta_ok) {
|
||||
file_destroy(file);
|
||||
free(new_data);
|
||||
free(old_data);
|
||||
delta_signature_destroy(sig);
|
||||
*failed = true;
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
if (!receive_file_xattrs(file, fd, config)) {
|
||||
file_destroy(file);
|
||||
free(new_data);
|
||||
free(old_data);
|
||||
delta_signature_destroy(sig);
|
||||
*failed = true;
|
||||
return NULL;
|
||||
if (!meta_ok)
|
||||
goto fail;
|
||||
}
|
||||
if (!receive_file_xattrs(file, fd, config))
|
||||
goto fail;
|
||||
|
||||
Data* replacement = data_create(new_data, (size_t)new_size);
|
||||
if (replacement == NULL) {
|
||||
file_destroy(file);
|
||||
free(old_data);
|
||||
delta_signature_destroy(sig);
|
||||
if (spool) {
|
||||
Data* reserved = data_create_reserve((size_t)new_size);
|
||||
if (reserved == NULL) {
|
||||
send_status(fd, STATUS_ERROR);
|
||||
*failed = true;
|
||||
return NULL;
|
||||
goto fail;
|
||||
}
|
||||
data_destroy(file->data);
|
||||
file->data = reserved;
|
||||
file->basis_copy = spool;
|
||||
spool = NULL; /* ownership moved into file->basis_copy */
|
||||
file->data_spool = true;
|
||||
} else {
|
||||
Data* replacement = data_create(new_data, (size_t)new_size);
|
||||
new_data = NULL; /* data_create owns, and frees, the buffer on failure */
|
||||
if (replacement == NULL) {
|
||||
send_status(fd, STATUS_ERROR);
|
||||
goto fail;
|
||||
}
|
||||
data_destroy(file->data);
|
||||
file->data = replacement;
|
||||
}
|
||||
|
||||
free(old_data);
|
||||
delta_signature_destroy(sig);
|
||||
return file;
|
||||
|
||||
fail:
|
||||
free(new_data);
|
||||
if (spool) {
|
||||
unlink(spool);
|
||||
free(spool);
|
||||
}
|
||||
file_destroy(file);
|
||||
delta_destroy(delta);
|
||||
data_destroy(raw_delta);
|
||||
free(old_data);
|
||||
delta_signature_destroy(sig);
|
||||
*failed = true;
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (resp == STATUS_NEXT) {
|
||||
/* Receive a STATUS_NEXT response: the sender declined the delta and will send
|
||||
the whole file. Releases the basis signature and snapshot, then receives the
|
||||
metadata/xattr block and the full payload. Takes ownership of `old_data` and
|
||||
`sig`, releasing both immediately. */
|
||||
static File* receive_next_branch(int fd, const Config* config, const char* check_path,
|
||||
unsigned long long expected_size, void* old_data,
|
||||
DeltaSignature* sig, bool* failed) {
|
||||
delta_signature_destroy(sig);
|
||||
free(old_data);
|
||||
|
||||
File* file = file_create(check_path);
|
||||
if (!file) {
|
||||
*failed = true;
|
||||
return NULL;
|
||||
}
|
||||
if (!file)
|
||||
goto fail;
|
||||
|
||||
if (config->use_metadata) {
|
||||
int meta_ok = 1;
|
||||
file->metadata = metadata_receive(fd, &meta_ok);
|
||||
if (!meta_ok) {
|
||||
file_destroy(file);
|
||||
*failed = true;
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
if (!receive_file_xattrs(file, fd, config)) {
|
||||
file_destroy(file);
|
||||
*failed = true;
|
||||
return NULL;
|
||||
if (!meta_ok)
|
||||
goto fail;
|
||||
}
|
||||
if (!receive_file_xattrs(file, fd, config))
|
||||
goto fail;
|
||||
|
||||
Data* file_data = receive_data_limited(fd, MAX_RECEIVE_WHOLE_FILE_SIZE);
|
||||
if (file_data == NULL) {
|
||||
file_destroy(file);
|
||||
*failed = true;
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (config->use_compression &&
|
||||
!compression_should_skip_with_suffixes(
|
||||
file->path, config->skip_compress_suffixes,
|
||||
config->skip_compress_set ? config->skip_compress_count : -1)) {
|
||||
Data* uncompressed = data_decompress_limited(file_data, MAX_RECEIVE_WHOLE_FILE_SIZE);
|
||||
ProtocolSession* owner = file_data->owner;
|
||||
data_destroy(file_data);
|
||||
if (uncompressed == NULL) {
|
||||
file_destroy(file);
|
||||
*failed = true;
|
||||
return NULL;
|
||||
}
|
||||
if (!data_charge_session(uncompressed, owner, uncompressed->size)) {
|
||||
data_destroy(uncompressed);
|
||||
file_destroy(file);
|
||||
send_status(fd, STATUS_ERROR);
|
||||
*failed = true;
|
||||
return NULL;
|
||||
}
|
||||
if (uncompressed->size > MAX_FILE_DATA_SIZE) {
|
||||
data_destroy(uncompressed);
|
||||
file_destroy(file);
|
||||
send_status(fd, STATUS_ERROR);
|
||||
*failed = true;
|
||||
return NULL;
|
||||
}
|
||||
file_data = uncompressed;
|
||||
}
|
||||
|
||||
data_destroy(file->data);
|
||||
file->data = file_data;
|
||||
char* dest_path = path_cat(config->receive_root_directory, check_path);
|
||||
if (!dest_path)
|
||||
goto fail;
|
||||
bool payload_ok = receive_file_payload_into(file, fd, config, dest_path, expected_size);
|
||||
free(dest_path);
|
||||
if (!payload_ok)
|
||||
goto fail;
|
||||
return file;
|
||||
|
||||
fail:
|
||||
file_destroy(file);
|
||||
*failed = true;
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* Delta handshake dispatcher: sign the basis, ship the signature, then hand the
|
||||
response off to the matching branch helper. Takes ownership of `old_data`
|
||||
(and, once created, `sig`); sets `*failed` on every error path. */
|
||||
static File* receive_delta_file(int fd, const Config* config, const char* check_path,
|
||||
void* old_data, unsigned long long old_size,
|
||||
unsigned long long expected_size, int basis_fd, bool* failed) {
|
||||
/* The basis is either an in-memory snapshot (the destination file, bounded) or
|
||||
* a confined descriptor (a --fuzzy sibling, possibly larger than memory) that
|
||||
* is signed/applied in bounded chunks. */
|
||||
Data* sig_data = NULL;
|
||||
Status resp = STATUS_ERROR;
|
||||
bool sig_sent = false;
|
||||
/* A delta check needs at least one basis source: the in-memory destination
|
||||
* snapshot or a confined basis descriptor. */
|
||||
if (!old_data && basis_fd < 0) {
|
||||
*failed = true;
|
||||
return NULL;
|
||||
}
|
||||
DeltaSignature* sig =
|
||||
old_data ? delta_signature_create_seeded(old_data, old_size, config->delta_block_size,
|
||||
(uint32_t)config->checksum_seed)
|
||||
: delta_signature_create_fd_seeded(basis_fd, old_size, config->delta_block_size,
|
||||
(uint32_t)config->checksum_seed);
|
||||
if (!sig)
|
||||
goto fail;
|
||||
|
||||
sig_data = delta_signature_serialize(sig);
|
||||
if (!sig_data)
|
||||
goto fail;
|
||||
|
||||
sig_sent = send_status(fd, STATUS_DELTA_SIGNATURE) && send_data(fd, sig_data);
|
||||
data_destroy(sig_data);
|
||||
sig_data = NULL;
|
||||
|
||||
if (!sig_sent || !receive_status(fd, &resp))
|
||||
goto fail;
|
||||
|
||||
if (resp == STATUS_DELTA_DATA)
|
||||
return receive_delta_data_branch(fd, config, check_path, old_data, old_size, basis_fd, sig,
|
||||
failed);
|
||||
|
||||
if (resp == STATUS_NEXT)
|
||||
return receive_next_branch(fd, config, check_path, expected_size, old_data, sig, failed);
|
||||
|
||||
send_status(fd, STATUS_ERROR);
|
||||
fail:
|
||||
data_destroy(sig_data);
|
||||
delta_signature_destroy(sig);
|
||||
free(old_data);
|
||||
send_status(fd, STATUS_ERROR);
|
||||
*failed = true;
|
||||
return NULL;
|
||||
}
|
||||
@@ -640,28 +640,29 @@ static bool fuzzy_candidate_better(const FuzzyCandidate* cand, const FuzzyCandid
|
||||
return strcmp(cand->name, best->name) < 0;
|
||||
}
|
||||
|
||||
/* Search the destination directory that will contain `check_path` for a
|
||||
* similar regular file usable as a --fuzzy delta basis and return its full
|
||||
* content in a malloc'd (protocol_alloc) buffer. Returns NULL (with *out_size
|
||||
* = 0) when no candidate qualifies, which means the caller performs the normal
|
||||
* whole-file transfer. */
|
||||
static void* fuzzy_basis_find_and_load(const Config* config, const char* check_path,
|
||||
/* Search the destination directory that will contain `check_path` for a similar
|
||||
* regular file usable as a --fuzzy delta basis and return an open, confined
|
||||
* read descriptor to it (with *out_size set). Returns -1 (with *out_size 0)
|
||||
* when no candidate qualifies, which means the caller performs the normal
|
||||
* whole-file transfer. The basis is signed/applied by streaming its descriptor,
|
||||
* so no whole-basis buffer is ever needed and its size is not capped. */
|
||||
static int fuzzy_basis_find_and_open(const Config* config, const char* check_path,
|
||||
unsigned long long check_size, time_t check_mtime,
|
||||
long check_mtime_nsec, unsigned long long* out_size) {
|
||||
*out_size = 0;
|
||||
if (!config || !config->receive_root_directory || !config->fuzzy || !config->use_delta ||
|
||||
!check_path || check_size > MAX_RECEIVE_WHOLE_FILE_SIZE)
|
||||
return NULL;
|
||||
!check_path)
|
||||
return -1;
|
||||
|
||||
char* full_path = path_cat(config->receive_root_directory, check_path);
|
||||
if (!full_path)
|
||||
return NULL;
|
||||
return -1;
|
||||
char* leaf = NULL;
|
||||
int dir_fd = file_open_secure_parent(full_path, &leaf, false);
|
||||
if (dir_fd < 0 || !leaf) {
|
||||
free(leaf);
|
||||
free(full_path);
|
||||
return NULL;
|
||||
return -1;
|
||||
}
|
||||
size_t target_len = strlen(leaf);
|
||||
/* A target basename longer than FUZZY_NAME_LIMIT can never pass the name gate
|
||||
@@ -670,7 +671,7 @@ static void* fuzzy_basis_find_and_load(const Config* config, const char* check_p
|
||||
close(dir_fd);
|
||||
free(leaf);
|
||||
free(full_path);
|
||||
return NULL;
|
||||
return -1;
|
||||
}
|
||||
|
||||
int scanfd = dup(dir_fd);
|
||||
@@ -678,7 +679,7 @@ static void* fuzzy_basis_find_and_load(const Config* config, const char* check_p
|
||||
close(dir_fd);
|
||||
free(leaf);
|
||||
free(full_path);
|
||||
return NULL;
|
||||
return -1;
|
||||
}
|
||||
DIR* dir = fdopendir(scanfd);
|
||||
if (!dir) {
|
||||
@@ -686,7 +687,7 @@ static void* fuzzy_basis_find_and_load(const Config* config, const char* check_p
|
||||
close(dir_fd);
|
||||
free(leaf);
|
||||
free(full_path);
|
||||
return NULL;
|
||||
return -1;
|
||||
}
|
||||
|
||||
/* The weighted-distance scratch row is allocated once per scan (not once per
|
||||
@@ -697,7 +698,7 @@ static void* fuzzy_basis_find_and_load(const Config* config, const char* check_p
|
||||
close(dir_fd);
|
||||
free(leaf);
|
||||
free(full_path);
|
||||
return NULL;
|
||||
return -1;
|
||||
}
|
||||
int fname_suf_len = 0;
|
||||
const char* fname_suf = fuzzy_find_suffix(leaf, (int)target_len, &fname_suf_len);
|
||||
@@ -727,7 +728,7 @@ static void* fuzzy_basis_find_and_load(const Config* config, const char* check_p
|
||||
if (fstatat(dir_fd, name, &st, AT_SYMLINK_NOFOLLOW) != 0 || !S_ISREG(st.st_mode))
|
||||
continue;
|
||||
unsigned long long cand_size = (unsigned long long)st.st_size;
|
||||
if (cand_size == 0 || cand_size > MAX_RECEIVE_WHOLE_FILE_SIZE)
|
||||
if (cand_size == 0)
|
||||
continue;
|
||||
long cand_nsec = 0;
|
||||
#ifdef __linux__
|
||||
@@ -771,7 +772,7 @@ static void* fuzzy_basis_find_and_load(const Config* config, const char* check_p
|
||||
if (exact.name[0])
|
||||
best = exact;
|
||||
|
||||
void* basis = NULL;
|
||||
int basis_fd = -1;
|
||||
if (best.name[0]) {
|
||||
/* O_NONBLOCK: a name raced to a FIFO between the fstatat gate and this open
|
||||
would otherwise block the receive thread forever on open(2); with it the
|
||||
@@ -781,29 +782,55 @@ static void* fuzzy_basis_find_and_load(const Config* config, const char* check_p
|
||||
if (fd >= 0) {
|
||||
struct stat st;
|
||||
if (fstat(fd, &st) == 0 && S_ISREG(st.st_mode) &&
|
||||
(unsigned long long)st.st_size == best.size && best.size <= SIZE_MAX) {
|
||||
basis = protocol_alloc((size_t)best.size);
|
||||
if (basis) {
|
||||
size_t got = 0;
|
||||
while (got < (size_t)best.size) {
|
||||
ssize_t n = read(fd, (char*)basis + got, (size_t)best.size - got);
|
||||
if (n <= 0) {
|
||||
free(basis);
|
||||
basis = NULL;
|
||||
break;
|
||||
}
|
||||
got += (size_t)n;
|
||||
}
|
||||
}
|
||||
}
|
||||
(unsigned long long)st.st_size == best.size) {
|
||||
basis_fd = fd;
|
||||
} else {
|
||||
close(fd);
|
||||
}
|
||||
}
|
||||
}
|
||||
close(dir_fd);
|
||||
free(full_path);
|
||||
if (basis)
|
||||
if (basis_fd >= 0)
|
||||
*out_size = best.size;
|
||||
return basis;
|
||||
return basis_fd;
|
||||
}
|
||||
|
||||
/* Receive one whole-file data frame into `file`. A payload at or below the
|
||||
* receiver's streaming bound keeps the historical charged whole-buffer path; a
|
||||
* larger one is streamed into a spool temp file (decompressing incrementally)
|
||||
* and installed through the File's basis_copy field. `expected_size` is the
|
||||
* logical size from the check frame (0 when unknown, e.g. the non-incremental
|
||||
* path). */
|
||||
static bool receive_file_payload_into(File* file, int fd, const Config* config,
|
||||
const char* dest_path, unsigned long long expected_size) {
|
||||
bool compress =
|
||||
config->use_compression && !compression_should_skip_with_suffixes(
|
||||
file->path, config->skip_compress_suffixes,
|
||||
config->skip_compress_set ? config->skip_compress_count : -1);
|
||||
Data* buffer = NULL;
|
||||
char* spool = NULL;
|
||||
unsigned long long size = 0;
|
||||
if (!file_receive_payload(fd, compress, expected_size, dest_path,
|
||||
protocol_whole_file_receive_limit(), &buffer, &spool, &size)) {
|
||||
return false;
|
||||
}
|
||||
if (spool) {
|
||||
Data* reserved = data_create_reserve((size_t)size);
|
||||
if (!reserved) {
|
||||
unlink(spool);
|
||||
free(spool);
|
||||
return false;
|
||||
}
|
||||
data_destroy(file->data);
|
||||
file->data = reserved;
|
||||
file->basis_copy = spool;
|
||||
file->data_spool = true;
|
||||
} else {
|
||||
data_destroy(file->data);
|
||||
file->data = buffer;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Read the remainder of a full-file transfer after the receiver has already
|
||||
@@ -811,7 +838,8 @@ static void* fuzzy_basis_find_and_load(const Config* config, const char* check_p
|
||||
* data frame, and return an owned File. Shared by the plain full-transfer path
|
||||
* and the --append-verify prefix-mismatch fallback (a clean full transfer
|
||||
* instead of a corrupt prefix+tail blend). */
|
||||
static File* receive_full_file(int fd, const Config* config, const char* path) {
|
||||
static File* receive_full_file(int fd, const Config* config, const char* path,
|
||||
unsigned long long expected_size) {
|
||||
File* file = file_create(path);
|
||||
if (!file)
|
||||
return NULL;
|
||||
@@ -827,36 +855,17 @@ static File* receive_full_file(int fd, const Config* config, const char* path) {
|
||||
file_destroy(file);
|
||||
return NULL;
|
||||
}
|
||||
Data* file_data = receive_data_limited(fd, MAX_RECEIVE_WHOLE_FILE_SIZE);
|
||||
if (file_data == NULL) {
|
||||
char* dest_path = path_cat(config->receive_root_directory, path);
|
||||
if (!dest_path) {
|
||||
file_destroy(file);
|
||||
return NULL;
|
||||
}
|
||||
if (config->use_compression &&
|
||||
!compression_should_skip_with_suffixes(file->path, config->skip_compress_suffixes,
|
||||
config->skip_compress_set ? config->skip_compress_count
|
||||
: -1)) {
|
||||
Data* uncompressed = data_decompress_limited(file_data, MAX_RECEIVE_WHOLE_FILE_SIZE);
|
||||
ProtocolSession* owner = file_data->owner;
|
||||
data_destroy(file_data);
|
||||
if (uncompressed == NULL) {
|
||||
bool ok = receive_file_payload_into(file, fd, config, dest_path, expected_size);
|
||||
free(dest_path);
|
||||
if (!ok) {
|
||||
file_destroy(file);
|
||||
return NULL;
|
||||
}
|
||||
if (!data_charge_session(uncompressed, owner, uncompressed->size)) {
|
||||
data_destroy(uncompressed);
|
||||
file_destroy(file);
|
||||
return NULL;
|
||||
}
|
||||
if (uncompressed->size > MAX_FILE_DATA_SIZE) {
|
||||
data_destroy(uncompressed);
|
||||
file_destroy(file);
|
||||
return NULL;
|
||||
}
|
||||
file_data = uncompressed;
|
||||
}
|
||||
data_destroy(file->data);
|
||||
file->data = file_data;
|
||||
return file;
|
||||
}
|
||||
|
||||
@@ -974,13 +983,12 @@ static IncrementalCheckOutcome incremental_check_receive_request(IncrementalChec
|
||||
return INCREMENTAL_ERROR;
|
||||
}
|
||||
|
||||
/* A basis-configured run may materialize a file larger than the whole-file
|
||||
payload bound: a basis hit is streamed from the basis path (bounded
|
||||
buffers), so the check size is not itself an allocation. Every other
|
||||
path (delta/append/full) still applies MAX_RECEIVE_WHOLE_FILE_SIZE, and a
|
||||
miss simply falls through to the normal transfer with its own bound. */
|
||||
if (!config_has_basis(config) && state->check_size > MAX_RECEIVE_WHOLE_FILE_SIZE) {
|
||||
send_error_detail(fd, "check size exceeds receiver limit");
|
||||
/* No file-size refusal: a whole-file payload larger than the historical
|
||||
whole-file bound is streamed through a bounded buffer (see
|
||||
file_receive_payload). Only a size that cannot be represented on this
|
||||
platform is rejected. */
|
||||
if (state->check_size > SIZE_MAX) {
|
||||
send_error_detail(fd, "check size is not representable");
|
||||
return INCREMENTAL_ERROR;
|
||||
}
|
||||
|
||||
@@ -1411,7 +1419,7 @@ static IncrementalCheckOutcome incremental_check_try_append_resume(IncrementalCh
|
||||
close(state->old_fd);
|
||||
state->old_fd = -1;
|
||||
}
|
||||
*out_file = receive_full_file(fd, config, check_path);
|
||||
*out_file = receive_full_file(fd, config, check_path, check_size);
|
||||
return INCREMENTAL_FILE;
|
||||
}
|
||||
|
||||
@@ -1428,20 +1436,24 @@ static IncrementalCheckOutcome incremental_check_try_append_resume(IncrementalCh
|
||||
if (config->use_metadata) {
|
||||
int meta_ok = 1;
|
||||
meta = metadata_receive(fd, &meta_ok);
|
||||
if (!meta_ok)
|
||||
if (!meta_ok) {
|
||||
file_metadata_destroy(meta);
|
||||
return INCREMENTAL_ERROR;
|
||||
}
|
||||
}
|
||||
if (config->use_xattrs) {
|
||||
int xok = 0;
|
||||
append_xattrs = xattr_receive(fd, &xok, config->preserve_acls);
|
||||
if (!xok) {
|
||||
xattr_list_free(append_xattrs);
|
||||
file_metadata_destroy(meta);
|
||||
return INCREMENTAL_ERROR;
|
||||
}
|
||||
}
|
||||
Data* tail = receive_data_limited(fd, MAX_RECEIVE_WHOLE_FILE_SIZE);
|
||||
if (tail == NULL) {
|
||||
xattr_list_free(append_xattrs);
|
||||
file_metadata_destroy(meta);
|
||||
return INCREMENTAL_ERROR;
|
||||
}
|
||||
if (config->use_compression &&
|
||||
@@ -1453,16 +1465,19 @@ static IncrementalCheckOutcome incremental_check_try_append_resume(IncrementalCh
|
||||
data_destroy(tail);
|
||||
if (uncompressed == NULL) {
|
||||
xattr_list_free(append_xattrs);
|
||||
file_metadata_destroy(meta);
|
||||
return INCREMENTAL_ERROR;
|
||||
}
|
||||
if (!data_charge_session(uncompressed, owner, uncompressed->size)) {
|
||||
data_destroy(uncompressed);
|
||||
xattr_list_free(append_xattrs);
|
||||
file_metadata_destroy(meta);
|
||||
return INCREMENTAL_ERROR;
|
||||
}
|
||||
if (uncompressed->size > MAX_FILE_DATA_SIZE) {
|
||||
data_destroy(uncompressed);
|
||||
xattr_list_free(append_xattrs);
|
||||
file_metadata_destroy(meta);
|
||||
return INCREMENTAL_ERROR;
|
||||
}
|
||||
tail = uncompressed;
|
||||
@@ -1475,6 +1490,7 @@ static IncrementalCheckOutcome incremental_check_try_append_resume(IncrementalCh
|
||||
send_status(fd, STATUS_ERROR);
|
||||
data_destroy(tail);
|
||||
xattr_list_free(append_xattrs);
|
||||
file_metadata_destroy(meta);
|
||||
return INCREMENTAL_ERROR;
|
||||
}
|
||||
size_t full_size = (size_t)check_size;
|
||||
@@ -1482,6 +1498,7 @@ static IncrementalCheckOutcome incremental_check_try_append_resume(IncrementalCh
|
||||
if (!full) {
|
||||
data_destroy(tail);
|
||||
xattr_list_free(append_xattrs);
|
||||
file_metadata_destroy(meta);
|
||||
return INCREMENTAL_ERROR;
|
||||
}
|
||||
if (old_size > 0 && state->old_data)
|
||||
@@ -1496,6 +1513,7 @@ static IncrementalCheckOutcome incremental_check_try_append_resume(IncrementalCh
|
||||
if (!file) {
|
||||
free(full);
|
||||
xattr_list_free(append_xattrs);
|
||||
file_metadata_destroy(meta);
|
||||
return INCREMENTAL_ERROR;
|
||||
}
|
||||
file->metadata = meta;
|
||||
@@ -1516,8 +1534,9 @@ static IncrementalCheckOutcome incremental_check_try_delta(IncrementalCheckState
|
||||
bool try_delta, File** out_file) {
|
||||
if (try_delta && state->old_data != NULL) {
|
||||
bool delta_failed = false;
|
||||
File* delta_file = receive_delta_file(state->fd, state->config, state->check_path,
|
||||
state->old_data, state->old_size, &delta_failed);
|
||||
File* delta_file =
|
||||
receive_delta_file(state->fd, state->config, state->check_path, state->old_data,
|
||||
state->old_size, state->check_size, -1, &delta_failed);
|
||||
state->old_data = NULL; /* receive_delta_file consumes the snapshot on every path */
|
||||
if (delta_file) {
|
||||
*out_file = delta_file;
|
||||
@@ -1540,14 +1559,14 @@ static IncrementalCheckOutcome incremental_check_try_fuzzy(IncrementalCheckState
|
||||
if (!config->fuzzy || !config->use_delta)
|
||||
return INCREMENTAL_CONTINUE;
|
||||
unsigned long long fuzzy_size = 0;
|
||||
void* fuzzy_basis = fuzzy_basis_find_and_load(config, state->check_path, state->check_size,
|
||||
int fuzzy_fd = fuzzy_basis_find_and_open(config, state->check_path, state->check_size,
|
||||
(time_t)state->check_mtime,
|
||||
(long)state->check_mtime_nsec, &fuzzy_size);
|
||||
if (fuzzy_basis != NULL) {
|
||||
if (fuzzy_fd >= 0) {
|
||||
bool fuzzy_failed = false;
|
||||
File* fuzzy_file = receive_delta_file(state->fd, config, state->check_path, fuzzy_basis,
|
||||
fuzzy_size, &fuzzy_failed);
|
||||
fuzzy_basis = NULL; /* receive_delta_file consumes the buffer on every path */
|
||||
File* fuzzy_file = receive_delta_file(state->fd, config, state->check_path, NULL, fuzzy_size,
|
||||
state->check_size, fuzzy_fd, &fuzzy_failed);
|
||||
close(fuzzy_fd);
|
||||
if (fuzzy_file) {
|
||||
*out_file = fuzzy_file;
|
||||
return INCREMENTAL_FILE;
|
||||
@@ -1555,7 +1574,6 @@ static IncrementalCheckOutcome incremental_check_try_fuzzy(IncrementalCheckState
|
||||
if (fuzzy_failed)
|
||||
return INCREMENTAL_ERROR;
|
||||
}
|
||||
free(fuzzy_basis);
|
||||
return INCREMENTAL_CONTINUE;
|
||||
}
|
||||
|
||||
@@ -1567,7 +1585,7 @@ static File* incremental_check_receive_full(IncrementalCheckState* state) {
|
||||
close(state->old_fd);
|
||||
state->old_fd = -1;
|
||||
}
|
||||
return receive_full_file(state->fd, state->config, state->check_path);
|
||||
return receive_full_file(state->fd, state->config, state->check_path, state->check_size);
|
||||
}
|
||||
|
||||
/* Core implementation. `would_transfer` (may be NULL) is set true only on the
|
||||
|
||||
+87
-20
@@ -1,4 +1,5 @@
|
||||
#include "log.h"
|
||||
#include "utils.h"
|
||||
#include <errno.h>
|
||||
#include <stdbool.h>
|
||||
#include <stdarg.h>
|
||||
@@ -16,6 +17,7 @@ static bool info_flags_explicit = false;
|
||||
static FILE* log_fp = NULL;
|
||||
static _Thread_local bool eight_bit_output;
|
||||
static LogStderrMode stderr_mode = LOG_STDERR_ERRORS;
|
||||
static LogClientMsgSink client_msg_sink = NULL;
|
||||
|
||||
/* Serializes access to log_fp and makes each emitted line atomic: the
|
||||
* timestamp prefix, formatted body, and trailing newline are written as one
|
||||
@@ -77,6 +79,51 @@ LogStderrMode log_get_stderr_mode(void) {
|
||||
return stderr_mode;
|
||||
}
|
||||
|
||||
void log_set_client_msg_sink(LogClientMsgSink sink) {
|
||||
client_msg_sink = sink;
|
||||
}
|
||||
|
||||
LogClientMsgSink log_get_client_msg_sink(void) {
|
||||
return client_msg_sink;
|
||||
}
|
||||
|
||||
/* Format just the message body (no prefix/newline) into a freshly allocated
|
||||
* buffer. Shared by log_message (which may hand the body to a client-message
|
||||
* sink) and log_client_message. Returns NULL on allocation/format failure. */
|
||||
static char* format_log_body(const char* format, va_list args) {
|
||||
va_list copy;
|
||||
va_copy(copy, args);
|
||||
int body_len = vsnprintf(NULL, 0, format, copy);
|
||||
va_end(copy);
|
||||
if (body_len < 0)
|
||||
return NULL;
|
||||
char* body = malloc((size_t)body_len + 1);
|
||||
if (!body)
|
||||
return NULL;
|
||||
vsnprintf(body, (size_t)body_len + 1, format, args);
|
||||
return body;
|
||||
}
|
||||
|
||||
/* Assemble a complete log line (prefix + body + newline) from an already
|
||||
* formatted body. Returns NULL on allocation failure. */
|
||||
static char* format_log_line_from_body(LogLevel log_level, const struct tm* t, const char* body) {
|
||||
char prefix[64];
|
||||
int prefix_len = snprintf(
|
||||
prefix, sizeof(prefix), "%04d-%02d-%02d %02d:%02d:%02d [%s]: ", t->tm_year + 1900,
|
||||
t->tm_mon + 1, t->tm_mday, t->tm_hour, t->tm_min, t->tm_sec, log_level_strings[log_level]);
|
||||
if (prefix_len < 0 || prefix_len >= (int)sizeof(prefix))
|
||||
return NULL;
|
||||
size_t body_len = strlen(body);
|
||||
char* line = malloc((size_t)prefix_len + body_len + 2); /* body + '\n' + NUL */
|
||||
if (!line)
|
||||
return NULL;
|
||||
memcpy(line, prefix, (size_t)prefix_len);
|
||||
memcpy(line + prefix_len, body, body_len);
|
||||
line[(size_t)prefix_len + body_len] = '\n';
|
||||
line[(size_t)prefix_len + body_len + 1] = '\0';
|
||||
return line;
|
||||
}
|
||||
|
||||
/* Format one complete log line (timestamp prefix + body + newline) into a
|
||||
* freshly allocated buffer. This is pure CPU/malloc work and must happen
|
||||
* OUTSIDE the log mutex: the mutex only guards the log_fp pointer, so a
|
||||
@@ -84,26 +131,11 @@ LogStderrMode log_get_stderr_mode(void) {
|
||||
* on allocation/formatting failure. */
|
||||
static char* format_log_line(LogLevel log_level, const struct tm* t, const char* format,
|
||||
va_list args) {
|
||||
char prefix[64];
|
||||
int prefix_len = snprintf(
|
||||
prefix, sizeof(prefix), "%04d-%02d-%02d %02d:%02d:%02d [%s]: ", t->tm_year + 1900,
|
||||
t->tm_mon + 1, t->tm_mday, t->tm_hour, t->tm_min, t->tm_sec, log_level_strings[log_level]);
|
||||
if (prefix_len < 0 || prefix_len >= (int)sizeof(prefix))
|
||||
char* body = format_log_body(format, args);
|
||||
if (!body)
|
||||
return NULL;
|
||||
va_list copy;
|
||||
va_copy(copy, args);
|
||||
int body_len = vsnprintf(NULL, 0, format, copy);
|
||||
va_end(copy);
|
||||
if (body_len < 0)
|
||||
return NULL;
|
||||
size_t total = (size_t)prefix_len + (size_t)body_len;
|
||||
char* line = malloc(total + 2); /* body bytes + '\n' + NUL */
|
||||
if (!line)
|
||||
return NULL;
|
||||
memcpy(line, prefix, (size_t)prefix_len);
|
||||
vsnprintf(line + prefix_len, (size_t)body_len + 1, format, args);
|
||||
line[total] = '\n';
|
||||
line[total + 1] = '\0';
|
||||
char* line = format_log_line_from_body(log_level, t, body);
|
||||
free(body);
|
||||
return line;
|
||||
}
|
||||
|
||||
@@ -121,6 +153,26 @@ static void emit_log_line(FILE* console, const char* line) {
|
||||
mtx_unlock(&log_mutex);
|
||||
}
|
||||
|
||||
void log_client_message(const char* message) {
|
||||
if (!message)
|
||||
return;
|
||||
/* The body is peer-controlled: escape every non-printable byte (newlines,
|
||||
CR, ANSI ESC, ...) so a hostile client cannot forge log lines or inject
|
||||
terminal control sequences. output_escape() is the codebase's canonical
|
||||
escaper and leaves printable text untouched. */
|
||||
char* escaped = output_escape(message, log_get_8_bit_output());
|
||||
if (!escaped)
|
||||
return;
|
||||
/* Route through the ordinary log level / destination gate (log_message):
|
||||
this respects --log-file, the configured stderr mode and the level
|
||||
threshold instead of always writing to stderr. The wire body carries no
|
||||
severity, so the forwarded diagnostic is emitted as a warning -- the
|
||||
lowest level the default gate admits, which keeps the peer's messages
|
||||
visible without bypassing --quiet. */
|
||||
log_message(LOG_LEVEL_WARNING, "%s", escaped);
|
||||
free(escaped);
|
||||
}
|
||||
|
||||
void log_message(LogLevel log_level, const char* format, ...) {
|
||||
if (log_level < current_log_level)
|
||||
return;
|
||||
@@ -138,8 +190,23 @@ void log_message(LogLevel log_level, const char* format, ...) {
|
||||
|
||||
va_list args;
|
||||
va_start(args, format);
|
||||
char* line = format_log_line(log_level, &t, format, args);
|
||||
char* body = format_log_body(format, args);
|
||||
va_end(args);
|
||||
if (!body)
|
||||
return;
|
||||
/* LOG_STDERR_CLIENT: hand the diagnostic to the client-message channel. A
|
||||
sink that takes ownership suppresses the local write; otherwise (no sink
|
||||
yet, or the peer connection is not up) fall through to local output so the
|
||||
diagnostic is never lost. */
|
||||
if (stderr_mode == LOG_STDERR_CLIENT) {
|
||||
LogClientMsgSink sink = client_msg_sink;
|
||||
if (sink && sink(body)) {
|
||||
free(body);
|
||||
return;
|
||||
}
|
||||
}
|
||||
char* line = format_log_line_from_body(log_level, &t, body);
|
||||
free(body);
|
||||
if (!line)
|
||||
return;
|
||||
emit_log_line(dest_io, line);
|
||||
|
||||
+20
-1
@@ -15,7 +15,11 @@
|
||||
#endif
|
||||
|
||||
typedef enum { LOG_LEVEL_DEBUG, LOG_LEVEL_INFO, LOG_LEVEL_WARNING, LOG_LEVEL_ERROR } LogLevel;
|
||||
typedef enum { LOG_STDERR_ERRORS, LOG_STDERR_ALL } LogStderrMode;
|
||||
/* --stderr=MODE destinations. ERRORS keeps errors on stderr and everything
|
||||
* else on stdout; ALL sends every message to stderr; CLIENT routes the client's
|
||||
* own diagnostics over the protocol stream to the peer's stderr (rsync's
|
||||
* --stderr=client / --no-msgs2stderr). */
|
||||
typedef enum { LOG_STDERR_ERRORS, LOG_STDERR_ALL, LOG_STDERR_CLIENT } LogStderrMode;
|
||||
|
||||
typedef enum {
|
||||
LOG_DEBUG_IO = 1u << 0,
|
||||
@@ -86,5 +90,20 @@ void log_set_8_bit_output(bool enabled);
|
||||
bool log_get_8_bit_output(void);
|
||||
void log_set_stderr_mode(LogStderrMode mode);
|
||||
LogStderrMode log_get_stderr_mode(void);
|
||||
/* Write a message a peer forwarded over the client-message channel to this
|
||||
* process's stderr (and log file), with the standard log prefix. Used by the
|
||||
* server side of rsync's --stderr=client. */
|
||||
void log_client_message(const char* message);
|
||||
|
||||
/* Sink for LOG_STDERR_CLIENT. log_message() passes the un-prefixed message
|
||||
* body to the installed sink; a `true` return means the sink took ownership
|
||||
* (e.g. queued it for protocol transmission) and the message must NOT also be
|
||||
* written locally. A `false` return (or a NULL sink) makes log_message fall
|
||||
* back to the normal local destination, so a diagnostic emitted before the peer
|
||||
* connection exists is never lost (rsync's documented fallback). The sink may
|
||||
* be called from any thread and must be tolerant of that. */
|
||||
typedef bool (*LogClientMsgSink)(const char* message);
|
||||
void log_set_client_msg_sink(LogClientMsgSink sink);
|
||||
LogClientMsgSink log_get_client_msg_sink(void);
|
||||
|
||||
#endif
|
||||
|
||||
@@ -34,6 +34,7 @@ PipelineContextSender* pipeline_context_sender_create(Config* config, Queue* que
|
||||
context->synced_dirs = NULL;
|
||||
context->plan_dirs = NULL;
|
||||
context->missing_args = NULL;
|
||||
context->per_dir_rules = NULL;
|
||||
context->scan_had_io_error = false;
|
||||
context->remove_source_files = NULL;
|
||||
context->early_delete = false;
|
||||
@@ -53,6 +54,7 @@ PipelineContextSender* pipeline_context_sender_create(Config* config, Queue* que
|
||||
context->dir_entries_mutex_init = false;
|
||||
atomic_init(&context->dir_count, 0);
|
||||
context->delete_limit = false;
|
||||
context->partial = false;
|
||||
int init = 0;
|
||||
if (config->use_metadata) {
|
||||
context->dir_entries = array_list_create(file_destroy);
|
||||
@@ -209,6 +211,8 @@ void pipeline_context_sender_destroy(PipelineContextSender* context) {
|
||||
array_list_delete(context->plan_dirs);
|
||||
if (context->missing_args)
|
||||
array_list_delete(context->missing_args);
|
||||
if (context->per_dir_rules)
|
||||
filter_rule_list_free(context->per_dir_rules);
|
||||
if (context->remove_source_files)
|
||||
array_list_delete(context->remove_source_files);
|
||||
if (context->dir_entries)
|
||||
|
||||
@@ -67,6 +67,12 @@ typedef struct {
|
||||
them in the manifest frame's third section and the receiver deletes each as
|
||||
an explicit request. */
|
||||
ArrayList* missing_args;
|
||||
/* Per-directory filter rules the source scan compiled (protocol 2.30.0),
|
||||
sent with the delete manifest/plan config so the receiver can re-derive the
|
||||
per-directory protect/risk set. NULL when --delete is off. Populated by
|
||||
the scanner (parallel workers append under mutex_scanner) or the early
|
||||
pre-scan. */
|
||||
FilterRuleList* per_dir_rules;
|
||||
/* A source I/O error (unreadable directory) was recorded during the scan.
|
||||
Set by the pre-scan (before the threads start) or by the scanner thread
|
||||
under mutex_scanner; the caller turns it into a non-zero exit when
|
||||
@@ -134,6 +140,11 @@ typedef struct {
|
||||
deletion (STATUS_DELETE_LIMIT): the transfer succeeded and the process must
|
||||
exit 25 like rsync. Read by the caller after the sender thread is joined. */
|
||||
bool delete_limit;
|
||||
/* Set by the sender thread when the receiver reported STATUS_PARTIAL (a
|
||||
per-entry receiver failure that did not abort the stream): the transfer
|
||||
otherwise succeeded, successfully stored --remove-source-files sources were
|
||||
removed, and the process must exit 23 like rsync. Read after join. */
|
||||
bool partial;
|
||||
} PipelineContextSender;
|
||||
|
||||
/* `config` is borrowed and must outlive the context: destroy does NOT free it,
|
||||
|
||||
+74
-16
@@ -38,6 +38,27 @@ static atomic_ullong io_bytes_read = 0;
|
||||
|
||||
static unsigned long long global_bwlimit(void);
|
||||
|
||||
/* Runtime whole-file receive bound (see protocol.h). Resolved once; an
|
||||
* override can only LOWER the ceiling, never raise it above the protocol
|
||||
* constant, so the wire/security bound is unchanged. A parse failure or a
|
||||
* non-positive value leaves the default in place. */
|
||||
unsigned long long protocol_whole_file_receive_limit(void) {
|
||||
static atomic_ullong cached = 0;
|
||||
unsigned long long value = atomic_load_explicit(&cached, memory_order_relaxed);
|
||||
if (value != 0)
|
||||
return value;
|
||||
value = MAX_RECEIVE_WHOLE_FILE_SIZE;
|
||||
const char* env = getenv("FASTSYNC_MAX_WHOLE_FILE_SIZE");
|
||||
if (env && env[0] != '\0') {
|
||||
char* end = NULL;
|
||||
unsigned long long parsed = strtoull(env, &end, 10);
|
||||
if (end && *end == '\0' && parsed > 0 && parsed < value)
|
||||
value = parsed;
|
||||
}
|
||||
atomic_store_explicit(&cached, value, memory_order_relaxed);
|
||||
return value;
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------------- *
|
||||
* Transport vtable implementations.
|
||||
*
|
||||
@@ -664,6 +685,10 @@ static const char* status_to_string(Status status) {
|
||||
return "DELETE_LIMIT";
|
||||
case STATUS_DEST_INFO:
|
||||
return "DEST_INFO";
|
||||
case STATUS_CLIENT_MSG:
|
||||
return "CLIENT_MSG";
|
||||
case STATUS_PARTIAL:
|
||||
return "PARTIAL";
|
||||
default:
|
||||
return "UNKNOWN";
|
||||
}
|
||||
@@ -672,10 +697,10 @@ static const char* status_to_string(Status status) {
|
||||
/* Reject a raw wire status outside the known enum range before it is handed to
|
||||
* callers, so an unknown/corrupt frame fails as a protocol error instead of
|
||||
* being silently interpreted as an unexpected-but-valid verdict. STATUS_OK is
|
||||
* the first enumerator and STATUS_STATS the last, so the range check accepts
|
||||
* the first enumerator and STATUS_PARTIAL the last, so the range check accepts
|
||||
* every status the protocol defines. */
|
||||
static bool status_is_valid(Status status) {
|
||||
return status >= STATUS_OK && status <= STATUS_STATS;
|
||||
return status >= STATUS_OK && status <= STATUS_PARTIAL;
|
||||
}
|
||||
|
||||
/* Shared string send/receive implementation. `redact` selects whether the
|
||||
@@ -765,17 +790,11 @@ bool protocol_send_data(ProtocolSession* session, const Data* data) {
|
||||
return true;
|
||||
}
|
||||
|
||||
Data* protocol_receive_data_limited(ProtocolSession* session, unsigned long long maximum_size) {
|
||||
Data* protocol_receive_data_alloc(ProtocolSession* session, unsigned long long size) {
|
||||
if (!session)
|
||||
return NULL;
|
||||
unsigned long long size = 0;
|
||||
if (!protocol_receive_n_data(session, &size, sizeof(unsigned long long)))
|
||||
if (size > MAX_DATA_PAYLOAD_SIZE)
|
||||
return NULL;
|
||||
if (size > MAX_DATA_PAYLOAD_SIZE || size > maximum_size) {
|
||||
log_message(LOG_LEVEL_ERROR, "Data size %llu exceeds maximum %llu", size,
|
||||
(unsigned long long)MAX_DATA_PAYLOAD_SIZE);
|
||||
return NULL;
|
||||
}
|
||||
if (size > SIZE_MAX)
|
||||
return NULL;
|
||||
size_t allocation_size = size == 0 ? 1 : (size_t)size;
|
||||
@@ -790,12 +809,6 @@ Data* protocol_receive_data_limited(ProtocolSession* session, unsigned long long
|
||||
protocol_release_memory_for_session(session, allocation_size);
|
||||
return NULL;
|
||||
}
|
||||
if (!protocol_receive_n_data(session, data, (size_t)size)) {
|
||||
free(data);
|
||||
protocol_release_memory_for_session(session, allocation_size);
|
||||
return NULL;
|
||||
}
|
||||
log_debug_message(LOG_DEBUG_PROTO, "Received %llu data", size);
|
||||
Data* result = data_create(data, (size_t)size);
|
||||
if (!result) {
|
||||
protocol_release_memory_for_session(session, allocation_size);
|
||||
@@ -806,6 +819,32 @@ Data* protocol_receive_data_limited(ProtocolSession* session, unsigned long long
|
||||
return result;
|
||||
}
|
||||
|
||||
Data* protocol_receive_data_body(ProtocolSession* session, unsigned long long size) {
|
||||
Data* result = protocol_receive_data_alloc(session, size);
|
||||
if (!result)
|
||||
return NULL;
|
||||
if (!protocol_receive_n_data(session, result->data, (size_t)size)) {
|
||||
data_destroy(result);
|
||||
return NULL;
|
||||
}
|
||||
log_debug_message(LOG_DEBUG_PROTO, "Received %llu data", size);
|
||||
return result;
|
||||
}
|
||||
|
||||
Data* protocol_receive_data_limited(ProtocolSession* session, unsigned long long maximum_size) {
|
||||
if (!session)
|
||||
return NULL;
|
||||
unsigned long long size = 0;
|
||||
if (!protocol_receive_n_data(session, &size, sizeof(unsigned long long)))
|
||||
return NULL;
|
||||
if (size > MAX_DATA_PAYLOAD_SIZE || size > maximum_size) {
|
||||
log_message(LOG_LEVEL_ERROR, "Data size %llu exceeds maximum %llu", size,
|
||||
(unsigned long long)MAX_DATA_PAYLOAD_SIZE);
|
||||
return NULL;
|
||||
}
|
||||
return protocol_receive_data_body(session, size);
|
||||
}
|
||||
|
||||
bool protocol_send_int(ProtocolSession* session, int data) {
|
||||
if (!protocol_send_n_data(session, &data, sizeof(int)))
|
||||
return false;
|
||||
@@ -1110,6 +1149,12 @@ Data* receive_data(int fd) {
|
||||
Data* receive_data_limited(int fd, unsigned long long maximum_size) {
|
||||
return protocol_receive_data_limited(legacy_session(fd, -1), maximum_size);
|
||||
}
|
||||
Data* receive_data_body(int fd, unsigned long long size) {
|
||||
return protocol_receive_data_body(legacy_session(fd, -1), size);
|
||||
}
|
||||
Data* receive_data_alloc(int fd, unsigned long long size) {
|
||||
return protocol_receive_data_alloc(legacy_session(fd, -1), size);
|
||||
}
|
||||
bool send_int(int fd, int data) {
|
||||
return protocol_send_int(legacy_session(-1, fd), data);
|
||||
}
|
||||
@@ -1144,6 +1189,19 @@ bool send_error_detail(int fd, const char* message) {
|
||||
return send_status(fd, STATUS_ERROR_DETAIL) && send_str(fd, message);
|
||||
}
|
||||
|
||||
bool send_client_message(int fd, const char* message) {
|
||||
if (!message)
|
||||
message = "";
|
||||
char bounded[MAX_CLIENT_MSG_BYTES + 1];
|
||||
size_t len = strlen(message);
|
||||
if (len > MAX_CLIENT_MSG_BYTES) {
|
||||
memcpy(bounded, message, MAX_CLIENT_MSG_BYTES);
|
||||
bounded[MAX_CLIENT_MSG_BYTES] = '\0';
|
||||
message = bounded;
|
||||
}
|
||||
return send_status(fd, STATUS_CLIENT_MSG) && send_str(fd, message);
|
||||
}
|
||||
|
||||
const char* protocol_last_error(void) {
|
||||
return io_error_detail;
|
||||
}
|
||||
|
||||
+76
-15
@@ -15,6 +15,12 @@
|
||||
* this for a rejection and the detail frame stays a small, fixed bound. */
|
||||
#define MAX_ERROR_DETAIL_BYTES 4096
|
||||
|
||||
/* Hard cap on a client diagnostic forwarded over the STATUS_CLIENT_MSG channel
|
||||
* (protocol 2.30.0, rsync's --stderr=client). The body is reused from the
|
||||
* bounded-string wire helper and sliced to this many bytes before it is sent,
|
||||
* so a peer can never be made to retain more than this per message. */
|
||||
#define MAX_CLIENT_MSG_BYTES 4096
|
||||
|
||||
/* Maximum uncompressed file payload accepted by the receiver's whole-file
|
||||
* paths. A single whole file is charged against the per-connection memory
|
||||
* reservation (MAX_CONNECTION_MEMORY) and against the server allocation
|
||||
@@ -26,6 +32,16 @@
|
||||
/* Maximum allowed data payload size for receive_data (whole-file bound) */
|
||||
#define MAX_DATA_PAYLOAD_SIZE MAX_RECEIVE_WHOLE_FILE_SIZE
|
||||
|
||||
/* Runtime whole-file receive bound. It defaults to MAX_RECEIVE_WHOLE_FILE_SIZE
|
||||
* and exists so the test suite can lower the ceiling (via the
|
||||
* FASTSYNC_MAX_WHOLE_FILE_SIZE environment variable, a byte count) and exercise
|
||||
* the streaming path with a small, fast transfer. A payload at or below the
|
||||
* bound keeps the historical whole-buffer path; a larger one is streamed
|
||||
* through a bounded buffer. The value is resolved once per process and never
|
||||
* exceeds the compile-time ceiling, so a malicious environment cannot raise it
|
||||
* beyond the protocol limit. */
|
||||
unsigned long long protocol_whole_file_receive_limit(void);
|
||||
|
||||
/* Maximum chunk size (64 MB) — prevents unbounded allocation from the wire */
|
||||
#define MAX_CHUNK_SIZE (64ULL * 1024 * 1024)
|
||||
/* Files larger than this are not kept fully in memory while loading: the
|
||||
@@ -123,8 +139,13 @@ enum NET_STATUS {
|
||||
STATUS_KEEPALIVE,
|
||||
STATUS_ABORT,
|
||||
STATUS_CHECK_BATCH,
|
||||
/* An explicit directory entry (--dirs): the sender transmits only the path;
|
||||
* the receiver creates the directory below the receive root. */
|
||||
/* An explicit directory entry (--dirs / an empty source directory): the sender
|
||||
* transmits the path and, when metadata/xattrs are negotiated, their blocks;
|
||||
* the receiver creates the directory below the receive root. Protocol 2.30.0
|
||||
* inserts an int32 probe flag right after the status when report_dest_info is
|
||||
* negotiated: probe=1 is a report-only frame (path only; the receiver answers
|
||||
* STATUS_DEST_INFO and creates nothing), probe=0 is a real create that is
|
||||
* answered with the directory's pre-transfer state before it is created. */
|
||||
STATUS_MKDIR,
|
||||
/* --append / --append-verify tail resume. STATUS_APPEND is sent by the
|
||||
* receiver after a per-file STATUS_CHECK when the existing destination file
|
||||
@@ -207,16 +228,22 @@ enum NET_STATUS {
|
||||
* limit stopped deletions"). Appended after STATUS_DRY_RUN_TRANSFER so no
|
||||
* existing status is renumbered. */
|
||||
STATUS_DELETE_LIMIT,
|
||||
/* Destination-state report for output parity (protocol 2.23.0). When the
|
||||
* wire config carries report_dest_info=true, the receiver answers every
|
||||
* per-file STATUS_CHECK request with STATUS_DEST_INFO FIRST, followed by a
|
||||
* fixed record describing the pre-transfer destination entry
|
||||
* (int32 has_old; uint64 size; int64 mtime; int64 mtime_nsec; uint32 mode;
|
||||
* int32 uid; int32 gid). The ordinary STATUS_OK/STATUS_NEXT/... verdict
|
||||
* follows, so the sender can render rsync-accurate -i/--out-format columns
|
||||
* (new vs modified, and which of size/time/perms/owner/group differ) without
|
||||
* changing the transfer decision itself. Appended after
|
||||
* STATUS_DELETE_LIMIT so no existing status is renumbered. */
|
||||
/* Destination-state report for output parity (protocol 2.23.0; extended to
|
||||
* directories/symlinks in 2.30.0). When the wire config carries
|
||||
* report_dest_info=true, the receiver answers every per-file STATUS_CHECK
|
||||
* request with STATUS_DEST_INFO FIRST, followed by a fixed record describing
|
||||
* the pre-transfer destination entry (int32 has_old; int32 target_matches;
|
||||
* uint64 size; int64 mtime; int64 mtime_nsec; uint32 mode; int32 uid;
|
||||
* int32 gid). The ordinary STATUS_OK/STATUS_NEXT/... verdict follows, so the
|
||||
* sender can render rsync-accurate -i/--out-format columns (new vs modified,
|
||||
* and which of size/time/perms/owner/group differ) without changing the
|
||||
* transfer decision itself. Protocol 2.30.0 also uses this record for
|
||||
* STATUS_MKDIR and STATUS_SYMLINK: the sender consumes it into the entry's
|
||||
* dest_state before emitting its change line, and target_matches reports
|
||||
* whether an existing symlink's on-disk target already equals the incoming
|
||||
* one (so the sender can render `cLc........` vs `.L..t......` and suppress
|
||||
* an unchanged symlink). Appended after STATUS_DELETE_LIMIT so no existing
|
||||
* status is renumbered. */
|
||||
STATUS_DEST_INFO,
|
||||
/* Per-directory delete plan (protocol 2.24.0). The sender of a
|
||||
* --delete-during/--delete-delay transfer streams one frame per source
|
||||
@@ -238,9 +265,29 @@ enum NET_STATUS {
|
||||
* config carries report_stats=true, the receiver sends this status once,
|
||||
* immediately before its terminal success status, followed by a fixed stats
|
||||
* record (see format_stats_send/receive in format.h) and, when the run is a
|
||||
* --dry-run with --delete, the would-delete path list. Appended after
|
||||
* STATUS_DELETE_PLAN so no existing status is renumbered. */
|
||||
STATUS_STATS
|
||||
* --dry-run with --delete, the would-delete path list. Protocol 2.30.0
|
||||
* appends the four deleted_reg/dir/link/special counters to that record, so
|
||||
* --stats can render rsync's `Number of deleted files` per-type breakdown.
|
||||
* Appended after STATUS_DELETE_PLAN so no existing status is renumbered. */
|
||||
STATUS_STATS,
|
||||
/* Client diagnostic channel (protocol 2.30.0, rsync's --stderr=client /
|
||||
* --no-msgs2stderr). When the client's --stderr mode is `client`, the
|
||||
* client forwards its own diagnostics over this client->server frame
|
||||
* (STATUS_CLIENT_MSG followed by a bounded length-prefixed string, capped at
|
||||
* MAX_CLIENT_MSG_BYTES) instead of writing them to its local stderr. The
|
||||
* receiver reads the string and writes it to the server's stderr (respecting
|
||||
* the server log destination). Appended after STATUS_STATS so no existing
|
||||
* status is renumbered. */
|
||||
STATUS_CLIENT_MSG,
|
||||
/* Receiver-side partial transfer (protocol 2.30.0). Sent by the receiver as
|
||||
* the terminal status INSTEAD of STATUS_OK when one or more entries failed
|
||||
* per-entry without aborting the stream (currently a --devices mknod
|
||||
* EPERM/EACCES). The transfer otherwise succeeded and every successfully
|
||||
* stored file was acknowledged, so the sender may still remove
|
||||
* --remove-source-files sources; the sender maps this to rsync's exit code
|
||||
* 23 ("partial transfer due to error"), distinct from a fatal STATUS_ERROR.
|
||||
* Appended after STATUS_CLIENT_MSG so no existing status is renumbered. */
|
||||
STATUS_PARTIAL
|
||||
};
|
||||
|
||||
void io_set_fds(int read_fd, int write_fd);
|
||||
@@ -333,6 +380,15 @@ char* receive_str_redacted(int file_descriptor);
|
||||
bool send_data(int file_descriptor, const Data* data);
|
||||
Data* receive_data(int file_descriptor);
|
||||
Data* receive_data_limited(int file_descriptor, unsigned long long maximum_size);
|
||||
/* Read exactly `size` bytes as a charged Data body. The length-prefixed
|
||||
* receive_data_limited() reads the header itself; this variant is for callers
|
||||
* that must inspect the declared size (and possibly stream the body instead)
|
||||
* before allocating. `size` must already be within MAX_DATA_PAYLOAD_SIZE. */
|
||||
Data* receive_data_body(int file_descriptor, unsigned long long size);
|
||||
/* Allocate (and charge) a `size`-byte Data body without reading it; the caller
|
||||
* fills `result->data` itself. Used when a frame's leading bytes must be
|
||||
* inspected before the rest of the body is read. */
|
||||
Data* receive_data_alloc(int file_descriptor, unsigned long long size);
|
||||
bool send_int(int file_descriptor, int data);
|
||||
bool receive_int(int file_descriptor, int* data);
|
||||
bool send_status(int file_descriptor, Status status);
|
||||
@@ -341,6 +397,11 @@ bool receive_status(int file_descriptor, Status* status);
|
||||
* length-prefixed string. Over-long messages are sliced and NULL is treated
|
||||
* as "". Returns false if the status or the string could not be sent. */
|
||||
bool send_error_detail(int file_descriptor, const char* message);
|
||||
/* Send STATUS_CLIENT_MSG followed by a bounded (<= MAX_CLIENT_MSG_BYTES)
|
||||
* length-prefixed string carrying a client diagnostic. Over-long messages are
|
||||
* sliced and NULL is treated as "". Returns false if the status or the string
|
||||
* could not be sent. */
|
||||
bool send_client_message(int file_descriptor, const char* message);
|
||||
/* Human-readable reason captured from the most recent STATUS_ERROR_DETAIL
|
||||
* received on this thread, or "" when the last status was a bare STATUS_ERROR
|
||||
* (or no detail was seen). Thread-local, and valid until the next non-keepalive
|
||||
|
||||
@@ -134,7 +134,7 @@ Server* server_create_ex(int port, const ServerBindOptions* bind_opts) {
|
||||
|
||||
server->file_descriptor = file_descriptor;
|
||||
server->ssl_ctx = NULL;
|
||||
server->max_connections = 100;
|
||||
server->max_connections = SERVER_DEFAULT_MAX_CONNECTIONS;
|
||||
server->active_connections = 0;
|
||||
server->limit_registry = NULL;
|
||||
|
||||
|
||||
@@ -11,6 +11,10 @@
|
||||
* stored here so the transport layer does not depend on daemon config. */
|
||||
struct DaemonLimitRegistry;
|
||||
|
||||
/* Connection cap applied by server_create_ex() until the daemon's configured
|
||||
* `max connections` overrides it via server_set_max_connections(). */
|
||||
#define SERVER_DEFAULT_MAX_CONNECTIONS 100
|
||||
|
||||
typedef struct Server {
|
||||
struct sockaddr_storage address;
|
||||
unsigned int address_length;
|
||||
|
||||
+2
-2
@@ -450,7 +450,7 @@ void fake_super_store_fd(int fd, uint32_t uid, uint32_t gid, uint32_t mode, uint
|
||||
if (len <= 0 || (size_t)len >= sizeof(record))
|
||||
return;
|
||||
if (fsetxattr(fd, FAKESUPER_XATTR, record, (size_t)len, 0) != 0) {
|
||||
log_message(LOG_LEVEL_WARNING, "--fake-super: could not store %s on destination file: %s",
|
||||
log_message(LOG_LEVEL_WARNING, "--fake-super: could not store %s on destination entry: %s",
|
||||
FAKESUPER_XATTR, strerror(errno));
|
||||
}
|
||||
}
|
||||
@@ -550,7 +550,7 @@ bool fake_super_restore_fd(int fd, FileAttrPolicy policy) {
|
||||
} else if (metadata_mode_for_policy((mode_t)(ul_mode & 0777U), cur.st_mode, policy, &want)) {
|
||||
if (fchmod(fd, want) != 0)
|
||||
log_message(LOG_LEVEL_WARNING,
|
||||
"--fake-super: could not restore mode on destination file: %s",
|
||||
"--fake-super: could not restore mode on destination entry: %s",
|
||||
strerror(errno));
|
||||
}
|
||||
}
|
||||
|
||||
+10
-6
@@ -140,21 +140,25 @@ bool xattr_apply_path_nofollow(int parent_fd, const char* leaf, const FileXattrL
|
||||
/* --fake-super: write the source uid/gid/mode/rdev record into the reserved
|
||||
* FAKESUPER_XATTR on `fd`, using rsync 3.4.1's exact grammar (see the key
|
||||
* comment above). `mode` is the full st_mode including its S_IFMT bits.
|
||||
* Best-effort (logged, never fatal). Only meaningful when metadata was
|
||||
* transmitted so the values exist. */
|
||||
* `fd` may be a regular file, a faked char/block device (written as a regular
|
||||
* file), or a DIRECTORY: rsync stores a directory's faked mode/uid/gid in the
|
||||
* reserved xattr on the directory itself. Best-effort (logged, never fatal).
|
||||
* Only meaningful when metadata was transmitted so the values exist. */
|
||||
void fake_super_store_fd(int fd, uint32_t uid, uint32_t gid, uint32_t mode, uint32_t rdev_major,
|
||||
uint32_t rdev_minor);
|
||||
|
||||
/* --fake-super replay: parse the FAKESUPER_XATTR record previously written on
|
||||
* `fd` by fake_super_store_fd and re-apply the recorded permission bits
|
||||
* fd-relative. The recorded uid/gid are deliberately NOT chowned for real:
|
||||
* --fake-super only RECORDS ownership (the caller stores the resolved mapping
|
||||
* via identity_resolve_storage_ids), it never performs a real chown. The
|
||||
* fd-relative. `fd` may be a regular file, a faked device, or a DIRECTORY;
|
||||
* fgetxattr/fchmod work identically on a directory descriptor. The recorded
|
||||
* uid/gid are deliberately NOT chowned for real: --fake-super only RECORDS
|
||||
* ownership (the caller stores the resolved mapping via
|
||||
* identity_resolve_storage_ids), it never performs a real chown. The
|
||||
* recorded rdev is retained for a later privileged restore but is not acted on
|
||||
* here. Best-effort: absence of the xattr or a malformed record is a silent
|
||||
* no-op that never fails the transfer. The MODE leg is applied only when
|
||||
* policy.perms||policy.executability, and the recorded special bits
|
||||
* (setuid/setgid/sticky) are NOT applied to the real file -- exactly like
|
||||
* (setuid/setgid/sticky) are NOT applied to the real entry -- exactly like
|
||||
* rsync's fake-super receiver, which stores the full mode in the xattr but
|
||||
* strips the special bits on disk. mtime is not part of the record; the normal
|
||||
* metadata path carries it (policy.times) exactly as rsync sets the file's own
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
hello
|
||||
@@ -0,0 +1 @@
|
||||
x
|
||||
@@ -0,0 +1 @@
|
||||
y
|
||||
@@ -0,0 +1 @@
|
||||
a.txt
|
||||
@@ -0,0 +1 @@
|
||||
b.txt
|
||||
@@ -0,0 +1 @@
|
||||
world
|
||||
@@ -0,0 +1 @@
|
||||
deep
|
||||
@@ -0,0 +1 @@
|
||||
../a.txt
|
||||
@@ -0,0 +1 @@
|
||||
hello
|
||||
Symlink
+1
@@ -0,0 +1 @@
|
||||
b.txt
|
||||
@@ -0,0 +1 @@
|
||||
world
|
||||
@@ -0,0 +1 @@
|
||||
deep
|
||||
Symlink
+1
@@ -0,0 +1 @@
|
||||
../a.txt
|
||||
@@ -0,0 +1 @@
|
||||
hello
|
||||
@@ -0,0 +1 @@
|
||||
world
|
||||
@@ -0,0 +1 @@
|
||||
hello
|
||||
@@ -0,0 +1 @@
|
||||
world
|
||||
@@ -0,0 +1 @@
|
||||
hello
|
||||
@@ -0,0 +1 @@
|
||||
world
|
||||
@@ -0,0 +1 @@
|
||||
hello
|
||||
@@ -0,0 +1 @@
|
||||
world
|
||||
@@ -34,7 +34,7 @@ class ServerManager:
|
||||
self._proc = None
|
||||
self._port = None
|
||||
|
||||
def start(self, extra_args=None):
|
||||
def start(self, extra_args=None, env=None):
|
||||
self.stop()
|
||||
self._port = _find_free_port()
|
||||
# Plain TCP is intentionally explicit in the server; integration tests
|
||||
@@ -42,7 +42,11 @@ class ServerManager:
|
||||
cmd = SERVER_CMD + ["-p", str(self._port), "--allow-unauthenticated"]
|
||||
if extra_args:
|
||||
cmd += extra_args
|
||||
self._proc = subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
proc_env = dict(os.environ)
|
||||
if env:
|
||||
proc_env.update(env)
|
||||
self._proc = subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||||
env=proc_env)
|
||||
_wait_for_port(self._port, timeout=5)
|
||||
|
||||
def stop(self):
|
||||
|
||||
@@ -17,6 +17,7 @@ Run locally::
|
||||
python3 -m pytest tests/integration/test_differential_parity.py -n 4 --dist=load -m parity
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
import warnings
|
||||
@@ -105,6 +106,15 @@ def seed_backup(_src, rroot, froot):
|
||||
_mk(os.path.join(root, "a.txt"), b"OLD-CONTENT\n", _OLD_MTIME)
|
||||
|
||||
|
||||
def seed_delay_updates(_src, rroot, froot):
|
||||
"""A changed file plus an extra, so --delay-updates (and its implied
|
||||
--delete-after) has both a publication and a deletion to order."""
|
||||
for root in (rroot, froot):
|
||||
_mk(os.path.join(root, "a.txt"), b"OLD-CONTENT\n", _OLD_MTIME)
|
||||
_mk(os.path.join(root, "extra.txt"), b"extra\n", _OLD_MTIME)
|
||||
_mk(os.path.join(root, "extradir", "z.txt"), b"z\n", _OLD_MTIME)
|
||||
|
||||
|
||||
def seed_size_only(_src, rroot, froot):
|
||||
for root in (rroot, froot):
|
||||
_mk(os.path.join(root, "a.txt"), b"XXXXXXXXXXX\n", _OLD_MTIME)
|
||||
@@ -127,6 +137,66 @@ def seed_filter_protect(_src, rroot, froot):
|
||||
_mk(os.path.join(root, "sub", "other2.txt"), b"nested dest-only other\n", _OLD_MTIME)
|
||||
|
||||
|
||||
def seed_perdir_protect(_src, rroot, froot):
|
||||
"""Per-directory `.rsync-filter` carrying `P` rules on the source and both
|
||||
destinations, plus destination-only extras. The `.log` extras must survive
|
||||
--delete under every timing while the other extras go; the source's
|
||||
`.rsync-filter` (which FastSync carries to the receiver) and the seeded
|
||||
destination one (which rsync's receiver reads) are byte-identical."""
|
||||
for root in (_src, rroot, froot):
|
||||
_mk(os.path.join(root, ".rsync-filter"), b"P extra.log\nP nested.log\n")
|
||||
for root in (rroot, froot):
|
||||
_mk(os.path.join(root, "extra.log"), b"dest-only protected\n", _OLD_MTIME)
|
||||
_mk(os.path.join(root, "other.txt"), b"dest-only deleted\n", _OLD_MTIME)
|
||||
_mk(os.path.join(root, "sub", "nested.log"), b"nested protected\n", _OLD_MTIME)
|
||||
_mk(os.path.join(root, "sub", "other2.txt"), b"nested deleted\n", _OLD_MTIME)
|
||||
|
||||
|
||||
def seed_perdir_exclude(_src, rroot, froot):
|
||||
"""Per-directory unqualified exclude (`-`): dual-sided, so it protects the
|
||||
matching destination-only extra (and is opted back in by
|
||||
--delete-excluded)."""
|
||||
for root in (_src, rroot, froot):
|
||||
_mk(os.path.join(root, ".rsync-filter"), b"- extra.log\n")
|
||||
for root in (rroot, froot):
|
||||
_mk(os.path.join(root, "extra.log"), b"dest-only excluded\n", _OLD_MTIME)
|
||||
_mk(os.path.join(root, "other.txt"), b"dest-only deleted\n", _OLD_MTIME)
|
||||
|
||||
|
||||
def seed_perdir_subdir_protect(_src, rroot, froot):
|
||||
"""A SUBDIRECTORY-owned `.rsync-filter` (its owner is not the transfer root):
|
||||
the receiver must re-derive the `P` rules in the destination-relative
|
||||
coordinate system, otherwise the destination-only nested extras are wrongly
|
||||
deleted (silent data loss). A root-level extra is included so a too-broad
|
||||
rule would over-protect. The file is seeded on both destinations because
|
||||
rsync's receiver reads the per-directory file locally for delete-during."""
|
||||
for root in (_src, rroot, froot):
|
||||
_mk(os.path.join(root, "sub", ".rsync-filter"), b"P nested.log\nP extra.log\n")
|
||||
for root in (rroot, froot):
|
||||
_mk(os.path.join(root, "sub", "nested.log"), b"dest-only protected\n", _OLD_MTIME)
|
||||
_mk(os.path.join(root, "sub", "extra.log"), b"dest-only protected 2\n", _OLD_MTIME)
|
||||
_mk(os.path.join(root, "sub", "other.txt"), b"dest-only deleted\n", _OLD_MTIME)
|
||||
_mk(os.path.join(root, "root_extra.txt"), b"root dest-only deleted\n", _OLD_MTIME)
|
||||
|
||||
|
||||
def seed_perdir_subdir_exclude(_src, rroot, froot):
|
||||
"""A SUBDIRECTORY-owned unqualified exclude (`-`): dual-sided, so it protects
|
||||
the matching destination-only nested extra under plain --delete and is opted
|
||||
back in by --delete-excluded."""
|
||||
for root in (_src, rroot, froot):
|
||||
_mk(os.path.join(root, "sub", ".rsync-filter"), b"- nested.log\n")
|
||||
for root in (rroot, froot):
|
||||
_mk(os.path.join(root, "sub", "nested.log"), b"dest-only excluded\n", _OLD_MTIME)
|
||||
_mk(os.path.join(root, "sub", "other.txt"), b"dest-only deleted\n", _OLD_MTIME)
|
||||
|
||||
|
||||
def _seed_rules(content):
|
||||
def seed(_src, _rroot, _froot):
|
||||
_mk(os.path.join(_src, ".rules"), content)
|
||||
|
||||
return seed
|
||||
|
||||
|
||||
def seed_max_delete(_src, rroot, froot):
|
||||
for root in (rroot, froot):
|
||||
_mk(os.path.join(root, "extra1.txt"), b"e1\n", _OLD_MTIME)
|
||||
@@ -244,6 +314,14 @@ _CASES = [
|
||||
H.Case("delete_commit", "basic", ["-a", "--delete-after"], seed=seed_extras,
|
||||
fastsync_flags=["-a", "--delete-commit"], server_args=DELETE,
|
||||
ref="FastSync-only --delete-commit == rsync --delete-after"),
|
||||
# #317: --delay-updates stages under a per-run unique name and publishes
|
||||
# every update before the implied --delete-after removes extras.
|
||||
H.Case("delay_updates", "basic", ["-a", "--delay-updates"],
|
||||
seed=seed_delay_updates, ref="--delay-updates stages then publishes"),
|
||||
H.Case("delay_updates_delete", "basic",
|
||||
["-a", "--delay-updates", "--delete"], seed=seed_delay_updates,
|
||||
server_args=DELETE, ci=True,
|
||||
ref="--delay-updates implies --delete-after (publish before delete)"),
|
||||
H.Case("delete_excluded", "filters",
|
||||
["-a", "--delete", "--delete-excluded", "--exclude=*.log"],
|
||||
seed=seed_delete_excluded, server_args=DELETE, ref="--delete-excluded"),
|
||||
@@ -271,6 +349,81 @@ _CASES = [
|
||||
["-a", "--delete-after", "--filter=P *.log"],
|
||||
seed=seed_filter_protect, server_args=DELETE, ci=True,
|
||||
ref="--filter P/--protect under the whole-tree --delete-after commit"),
|
||||
# Per-directory merge rules (#315): the receiver must re-derive the
|
||||
# protect/risk verdict from the carried per-directory rules, so a
|
||||
# destination-only entry matching ONLY a per-directory rule is shielded.
|
||||
H.Case("filter_perdir_protect", "filters",
|
||||
["-a", "-F", "--delete"],
|
||||
seed=seed_perdir_protect, server_args=DELETE, ci=True,
|
||||
ref="-F per-directory P rule under the default --delete timing"),
|
||||
H.Case("filter_perdir_protect_during", "filters",
|
||||
["-a", "-F", "--delete-during"],
|
||||
seed=seed_perdir_protect, server_args=DELETE, ci=True,
|
||||
ref="-F per-directory P rule under --delete-during"),
|
||||
H.Case("filter_perdir_protect_delay", "filters",
|
||||
["-a", "-F", "--delete-delay"],
|
||||
seed=seed_perdir_protect, server_args=DELETE, ci=True,
|
||||
ref="-F per-directory P rule under --delete-delay"),
|
||||
H.Case("filter_perdir_protect_before", "filters",
|
||||
["-a", "-F", "--delete-before"],
|
||||
seed=seed_perdir_protect, server_args=DELETE, ci=True,
|
||||
ref="-F per-directory P rule under the whole-tree --delete-before commit"),
|
||||
H.Case("filter_perdir_protect_after", "filters",
|
||||
["-a", "-F", "--delete-after"],
|
||||
seed=seed_perdir_protect, server_args=DELETE, ci=True,
|
||||
ref="-F per-directory P rule under the whole-tree --delete-after commit"),
|
||||
H.Case("filter_perdir_exclude_protect", "filters",
|
||||
["-a", "-F", "--delete"],
|
||||
seed=seed_perdir_exclude, server_args=DELETE, ci=True,
|
||||
ref="-F per-directory exclude protects its destination mirror"),
|
||||
H.Case("filter_perdir_exclude_deleted", "filters",
|
||||
["-a", "-F", "--delete", "--delete-excluded"],
|
||||
seed=seed_perdir_exclude, server_args=DELETE, ci=True,
|
||||
ref="-F per-directory exclude under --delete-excluded is at risk"),
|
||||
# #316: a rule owned by a SUBDIRECTORY (not the transfer root) must be
|
||||
# re-expressed in the receiver's destination-relative coordinate system, or
|
||||
# the dest-only extras it protects are silently deleted.
|
||||
H.Case("filter_perdir_subdir_protect", "filters",
|
||||
["-a", "-F", "--delete"],
|
||||
seed=seed_perdir_subdir_protect, server_args=DELETE, ci=True,
|
||||
ref="-F subdirectory-owned P rule under the default --delete timing"),
|
||||
H.Case("filter_perdir_subdir_protect_during", "filters",
|
||||
["-a", "-F", "--delete-during"],
|
||||
seed=seed_perdir_subdir_protect, server_args=DELETE, ci=True,
|
||||
ref="-F subdirectory-owned P rule under --delete-during"),
|
||||
H.Case("filter_perdir_subdir_protect_delay", "filters",
|
||||
["-a", "-F", "--delete-delay"],
|
||||
seed=seed_perdir_subdir_protect, server_args=DELETE, ci=True,
|
||||
ref="-F subdirectory-owned P rule under --delete-delay"),
|
||||
H.Case("filter_perdir_subdir_protect_before", "filters",
|
||||
["-a", "-F", "--delete-before"],
|
||||
seed=seed_perdir_subdir_protect, server_args=DELETE, ci=True,
|
||||
ref="-F subdirectory-owned P rule under the whole-tree --delete-before commit"),
|
||||
H.Case("filter_perdir_subdir_protect_after", "filters",
|
||||
["-a", "-F", "--delete-after"],
|
||||
seed=seed_perdir_subdir_protect, server_args=DELETE, ci=True,
|
||||
ref="-F subdirectory-owned P rule under the whole-tree --delete-after commit"),
|
||||
H.Case("filter_perdir_subdir_exclude_protect", "filters",
|
||||
["-a", "-F", "--delete"],
|
||||
seed=seed_perdir_subdir_exclude, server_args=DELETE, ci=True,
|
||||
ref="-F subdirectory-owned exclude protects its destination mirror"),
|
||||
H.Case("filter_perdir_subdir_exclude_deleted", "filters",
|
||||
["-a", "-F", "--delete", "--delete-excluded"],
|
||||
seed=seed_perdir_subdir_exclude, server_args=DELETE, ci=True,
|
||||
ref="-F subdirectory-owned exclude under --delete-excluded is at risk"),
|
||||
# Merge-file modifiers (#315): e/n/w/- semantics match rsync 3.4.1.
|
||||
H.Case("dir_merge_e", "filters", ["-a", "--filter=:e .rules"],
|
||||
seed=_seed_rules(b"- *.log\n"), ci=True,
|
||||
ref="dir-merge,e excludes the merge file itself"),
|
||||
H.Case("dir_merge_n", "filters", ["-a", "--filter=:n .rules"],
|
||||
seed=_seed_rules(b"- *.log\n"), ci=True,
|
||||
ref="dir-merge,n does not inherit into subdirectories"),
|
||||
H.Case("dir_merge_dash", "filters", ["-a", "--filter=:- .rules"],
|
||||
seed=_seed_rules(b"*.log\n*.bin\n"), ci=True,
|
||||
ref="dir-merge,- reads the file as bare exclude patterns"),
|
||||
H.Case("dir_merge_w", "filters", ["-a", "--filter=:-w .rules"],
|
||||
seed=_seed_rules(b"*.log *.bin\n"), ci=True,
|
||||
ref="dir-merge,w word-splits bare patterns on whitespace"),
|
||||
|
||||
# --- relative / dirs --------------------------------------------------
|
||||
H.Case("relative_general", "basic", ["-a", "-R"], layout=H.MIRROR_ABS,
|
||||
@@ -666,6 +819,157 @@ def test_added_and_deleted_between_runs(parity_server_factory):
|
||||
_run_and_check(case_id, result)
|
||||
|
||||
|
||||
def _seed_dest_tree(src, root):
|
||||
"""Copy `src`'s tree into `root` (the transfer mirror), preserving symlinks
|
||||
and directory mtimes, so a second differential run starts from an existing
|
||||
destination exactly like a seeded rsync run."""
|
||||
os.makedirs(root, exist_ok=True)
|
||||
for dirpath, dirnames, filenames in os.walk(src):
|
||||
rel = os.path.relpath(dirpath, src)
|
||||
for name in dirnames:
|
||||
s = os.path.join(dirpath, name)
|
||||
d = os.path.join(root, rel, name) if rel != "." else os.path.join(root, name)
|
||||
if os.path.islink(s):
|
||||
continue
|
||||
os.makedirs(d, exist_ok=True)
|
||||
for name in filenames:
|
||||
s = os.path.join(dirpath, name)
|
||||
d = os.path.join(root, rel, name) if rel != "." else os.path.join(root, name)
|
||||
os.makedirs(os.path.dirname(d), exist_ok=True)
|
||||
if os.path.islink(s):
|
||||
if os.path.lexists(d):
|
||||
os.remove(d)
|
||||
os.symlink(os.readlink(s), d)
|
||||
else:
|
||||
shutil.copy2(s, d)
|
||||
if rel != ".":
|
||||
os.utime(os.path.join(root, rel), None)
|
||||
os.utime(root, None)
|
||||
|
||||
|
||||
# A full rsync itemize code (11 columns) followed by the name. H._ITEMIZE_RE
|
||||
# only matches created (`+`) entries, so the changed-attribute codes this test
|
||||
# asserts need their own matcher.
|
||||
_ITEMIZE_LINE_RE = re.compile(r"^[<>ch.*][fdLDS].{9} ")
|
||||
|
||||
|
||||
def _itemize_dir_link_lines(text):
|
||||
"""The itemize lines for directory and symlink entries, excluding the
|
||||
transfer-root `./` line (FastSync emits it unconditionally; a documented
|
||||
residual)."""
|
||||
out = []
|
||||
for line in (text or "").splitlines():
|
||||
line = line.rstrip()
|
||||
if not line or not _ITEMIZE_LINE_RE.match(line):
|
||||
continue
|
||||
name = line.rsplit(" ", 1)[-1]
|
||||
if name == "./":
|
||||
continue
|
||||
if name.endswith("/") or " -> " in line:
|
||||
out.append(line)
|
||||
return sorted(out)
|
||||
|
||||
|
||||
@requires_rsync
|
||||
@parity
|
||||
def test_itemize_rerun_dirs_symlinks_matches_rsync(parity_server_factory):
|
||||
"""#314: a re-run reports directory/symlink destination state like rsync.
|
||||
|
||||
On an unchanged tree FastSync emits no per-directory `cd+++++++++` (or
|
||||
symlink) lines, and after a changed directory mtime / symlink target it
|
||||
renders rsync's `.d..t......` / `cLc........` instead of `cd`/`cL`."""
|
||||
case_id = "itemize_rerun_dirs_symlinks"
|
||||
src = os.path.join(TEST_DATA_DIR, "parity_itemds_src")
|
||||
rdst = os.path.join(TEST_DATA_DIR, "parity_itemds_rdst")
|
||||
fdst = os.path.join(TEST_DATA_DIR, "parity_itemds_fdst")
|
||||
clean_dir(src)
|
||||
_mk(os.path.join(src, "sub", "b.txt"), b"nested\n")
|
||||
os.makedirs(os.path.join(src, "emptydir"), exist_ok=True)
|
||||
os.symlink("a.txt", os.path.join(src, "link"))
|
||||
_mk(os.path.join(src, "a.txt"), b"top\n")
|
||||
clean_dir(rdst)
|
||||
clean_dir(fdst)
|
||||
server = parity_server_factory(SUPER)
|
||||
rroot = rdst
|
||||
froot = get_dest_received_dir(fdst, src)
|
||||
_seed_dest_tree(src, rroot)
|
||||
_seed_dest_tree(src, froot)
|
||||
|
||||
# Unchanged re-run: no directory or symlink itemize lines from either tool.
|
||||
rs = H.run_rsync(src, rdst, ["-a", "-i"])
|
||||
fs, _ = H.run_fastsync(src, fdst, ["-a", "-i", "--incremental"], server.port)
|
||||
assert rs.returncode == 0, rs.stderr
|
||||
assert fs.returncode == 0, fs.stderr
|
||||
assert _itemize_dir_link_lines(rs.stdout) == []
|
||||
fast_unchanged = _itemize_dir_link_lines(fs.stdout)
|
||||
assert fast_unchanged == [], f"unchanged re-run itemized dirs/links: {fast_unchanged}"
|
||||
|
||||
# Change the directory mtime and the symlink target, then re-run.
|
||||
_pin(os.path.join(src, "sub"), _OLD_MTIME)
|
||||
os.remove(os.path.join(src, "link"))
|
||||
os.symlink("b.txt", os.path.join(src, "link"))
|
||||
rs = H.run_rsync(src, rdst, ["-a", "-i"])
|
||||
fs, _ = H.run_fastsync(src, fdst, ["-a", "-i", "--incremental"], server.port)
|
||||
assert rs.returncode == 0, rs.stderr
|
||||
assert fs.returncode == 0, fs.stderr
|
||||
expected = _itemize_dir_link_lines(rs.stdout)
|
||||
actual = _itemize_dir_link_lines(fs.stdout)
|
||||
assert actual == expected, f"rsync={rs.stdout!r} fastsync={fs.stdout!r}"
|
||||
assert any(line.endswith(" sub/") and line.startswith(".d..t") for line in actual), actual
|
||||
assert any(line.startswith("cLc") and " -> b.txt" in line for line in actual), actual
|
||||
|
||||
|
||||
def _deleted_breakdown_line(text):
|
||||
for line in (text or "").splitlines():
|
||||
if line.startswith("Number of deleted files:"):
|
||||
return " ".join(line.split())
|
||||
return ""
|
||||
|
||||
|
||||
@requires_rsync
|
||||
@parity
|
||||
def test_stats_deleted_breakdown_matches_rsync(parity_server_factory):
|
||||
"""#316: `--stats` renders rsync's per-type `Number of deleted files`
|
||||
breakdown for removed regular files, directories, symlinks and a special."""
|
||||
case_id = "stats_deleted_breakdown"
|
||||
src = os.path.join(TEST_DATA_DIR, "parity_delbd_src")
|
||||
rdst = os.path.join(TEST_DATA_DIR, "parity_delbd_rdst")
|
||||
fdst = os.path.join(TEST_DATA_DIR, "parity_delbd_fdst")
|
||||
clean_dir(src)
|
||||
_mk(os.path.join(src, "keep.txt"), b"keep\n")
|
||||
server = parity_server_factory(DELETE)
|
||||
|
||||
def seed(_src, rroot, froot):
|
||||
for root in (rroot, froot):
|
||||
_mk(os.path.join(root, "extra1.txt"), b"e1\n", _OLD_MTIME)
|
||||
_mk(os.path.join(root, "extradir", "inside.txt"), b"e2\n", _OLD_MTIME)
|
||||
os.makedirs(os.path.join(root, "extradir"), exist_ok=True)
|
||||
link = os.path.join(root, "extralink")
|
||||
if not os.path.lexists(link):
|
||||
os.symlink("keep.txt", link)
|
||||
fifo = os.path.join(root, "extrafifo")
|
||||
if not os.path.exists(fifo):
|
||||
os.mkfifo(fifo)
|
||||
|
||||
def extra(_src, _rroot, _froot, rs, fs):
|
||||
rs_line = _deleted_breakdown_line(rs.stdout)
|
||||
fs_line = _deleted_breakdown_line(fs.stdout)
|
||||
if not rs_line:
|
||||
return ["rsync printed no deleted-files line"]
|
||||
if rs_line != fs_line:
|
||||
return [f"deleted breakdown rsync={rs_line!r} fastsync={fs_line!r}"]
|
||||
if "reg:" not in rs_line or "dir:" not in rs_line or \
|
||||
"link:" not in rs_line or "special:" not in rs_line:
|
||||
return [f"breakdown missing a category: {rs_line!r}"]
|
||||
return []
|
||||
|
||||
result = H.run_differential(
|
||||
src, rdst, fdst, ["-a", "--delete", "--stats"],
|
||||
["-a", "--delete", "--stats", "--incremental"], server,
|
||||
seed=seed, extra_check=extra)
|
||||
_run_and_check(case_id, result, ref="--stats deleted per-type breakdown")
|
||||
|
||||
|
||||
@requires_rsync
|
||||
@parity
|
||||
def test_one_file_system(parity_server_factory):
|
||||
|
||||
@@ -36,7 +36,7 @@ from common import ( # noqa: E402
|
||||
verify_transfer,
|
||||
)
|
||||
|
||||
PROTOCOL_VERSION = b"2.29.0"
|
||||
PROTOCOL_VERSION = b"2.30.0"
|
||||
STATUS_MANIFEST = 5
|
||||
STATUS_OK = 0
|
||||
|
||||
|
||||
@@ -203,9 +203,9 @@ class TestDeviceSpecial:
|
||||
flags=["--devices"], port=port)
|
||||
finally:
|
||||
out, err = _stop_captured_server(server)
|
||||
assert result.returncode != 0, (
|
||||
f"a failed device mknod must be a transfer error like rsync (got exit 0): "
|
||||
f"{(out + err)[:300]}"
|
||||
assert result.returncode == 23, (
|
||||
f"a failed device mknod must exit 23 (rsync partial transfer), got "
|
||||
f"{result.returncode}: {(out + err)[:300]}"
|
||||
)
|
||||
received = get_dest_received_dir(DEVICE_DEST, DEVICE_SOURCE)
|
||||
assert not os.path.lexists(os.path.join(received, "chardev")), (
|
||||
@@ -215,6 +215,38 @@ class TestDeviceSpecial:
|
||||
f"receiver did not log the device creation error: out={out!r} err={err!r}"
|
||||
)
|
||||
|
||||
@pytest.mark.skipif(os.geteuid() != 0, reason="requires root to create device nodes")
|
||||
def test_devices_nonroot_partial_removes_transferred_sources(self):
|
||||
"""rsync parity for a partial receiver run under --remove-source-files:
|
||||
the successfully transferred regular source is still removed, the
|
||||
un-creatable device source is kept, and the client exits 23 (verified
|
||||
against rsync 3.4.1: it removes ok.txt/ok2.txt, keeps the device, and
|
||||
exits 23)."""
|
||||
if os.geteuid() != 0 or shutil.which("setpriv") is None:
|
||||
pytest.skip("requires root + setpriv to run the receiver unprivileged")
|
||||
self._setup()
|
||||
os.mknod(os.path.join(DEVICE_SOURCE, "chardev"), stat.S_IFCHR | 0o666,
|
||||
os.makedev(1, 3))
|
||||
os.makedirs(DEVICE_DEST, exist_ok=True)
|
||||
os.chmod(DEVICE_DEST, 0o777)
|
||||
server, port = _start_captured_server(
|
||||
prefix=["setpriv", "--reuid=65534", "--regid=65534", "--clear-groups"])
|
||||
try:
|
||||
result, _ = run_client(DEVICE_SOURCE, DEVICE_DEST,
|
||||
flags=["--devices", "--remove-source-files"], port=port)
|
||||
finally:
|
||||
out, err = _stop_captured_server(server)
|
||||
assert result.returncode == 23, (
|
||||
f"a partial receiver run must exit 23, got {result.returncode}: "
|
||||
f"{(out + err)[:300]}"
|
||||
)
|
||||
assert not os.path.exists(os.path.join(DEVICE_SOURCE, "plain.txt")), (
|
||||
"a successfully transferred source must be removed even on a partial run"
|
||||
)
|
||||
assert os.path.exists(os.path.join(DEVICE_SOURCE, "chardev")), (
|
||||
"the source device that failed to materialize must be kept"
|
||||
)
|
||||
|
||||
@pytest.mark.skipif(os.geteuid() != 0, reason="requires root to create device nodes")
|
||||
def test_devices_recreates_real_char_device(self, shared_server):
|
||||
"""Root-only: a source char device node is recreated on the destination
|
||||
@@ -334,6 +366,57 @@ def setup_test_data():
|
||||
shutil.rmtree(DEST_DIR, ignore_errors=True)
|
||||
|
||||
|
||||
class TestClientStderrChannel:
|
||||
"""--stderr=client: the client's own diagnostics go to the peer's stderr."""
|
||||
|
||||
@pytest.mark.ci
|
||||
def test_client_diagnostic_reaches_server_stderr(self):
|
||||
"""A client-side warning emitted during the transfer is forwarded over
|
||||
the STATUS_CLIENT_MSG channel and printed on the server's stderr, not the
|
||||
client's. A dangling symlink under -L is the deterministic trigger."""
|
||||
source = os.path.join(TEST_DATA_DIR, "client_msg_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "client_msg_dst")
|
||||
clean_dir(source)
|
||||
clean_dir(dest)
|
||||
with open(os.path.join(source, "plain.txt"), "wb") as f:
|
||||
f.write(b"payload\n")
|
||||
os.symlink("no-such-referent", os.path.join(source, "dangling"))
|
||||
server, port = _start_captured_server()
|
||||
try:
|
||||
result, _ = run_client(source, dest, flags=["-L", "--stderr=client"],
|
||||
port=port)
|
||||
finally:
|
||||
out, err = _stop_captured_server(server)
|
||||
assert "symlink has no referent" in (out + err), (
|
||||
f"client diagnostic did not reach the server stderr: out={out!r} err={err!r}"
|
||||
)
|
||||
assert "symlink has no referent" not in (result.stderr or ""), (
|
||||
f"client diagnostic must not also be written locally: {result.stderr!r}"
|
||||
)
|
||||
|
||||
@pytest.mark.ci
|
||||
def test_no_msgs2stderr_alias_uses_client_channel(self):
|
||||
"""--no-msgs2stderr is rsync's spelling of --stderr=client and now
|
||||
forwards the client's diagnostics to the server too."""
|
||||
source = os.path.join(TEST_DATA_DIR, "client_msg_alias_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "client_msg_alias_dst")
|
||||
clean_dir(source)
|
||||
clean_dir(dest)
|
||||
with open(os.path.join(source, "plain.txt"), "wb") as f:
|
||||
f.write(b"payload\n")
|
||||
os.symlink("no-such-referent", os.path.join(source, "dangling"))
|
||||
server, port = _start_captured_server()
|
||||
try:
|
||||
result, _ = run_client(source, dest, flags=["-L", "--no-msgs2stderr"],
|
||||
port=port)
|
||||
finally:
|
||||
out, err = _stop_captured_server(server)
|
||||
assert "symlink has no referent" in (out + err), (
|
||||
f"--no-msgs2stderr did not route to the server: out={out!r} err={err!r}"
|
||||
)
|
||||
assert "symlink has no referent" not in (result.stderr or "")
|
||||
|
||||
|
||||
class TestDryRun:
|
||||
def test_trust_sender_transfer_completes(self, shared_server):
|
||||
"""--trust-sender is a receiver-local policy (never sent to the peer).
|
||||
@@ -2810,6 +2893,37 @@ class TestItemizeChanges:
|
||||
result, _ = run_client(SOURCE_DIR, DEST_DIR, flags=["-i", "--dry-run"])
|
||||
assert result.returncode == 0, f"dry-run -i failed: {result.stderr[:200]}"
|
||||
|
||||
def test_chunk_serialization_probes_ancestor_dir_state(self, shared_server):
|
||||
"""#314: --chunk-serialization + -i must probe ancestor directory state
|
||||
so a pre-existing directory with a changed mtime itemizes as an
|
||||
attribute change (`.d..t......`) instead of being rendered as created
|
||||
(`cd+++++++++`)."""
|
||||
source = os.path.join(TEST_DATA_DIR, "itemize_chunk_serial_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "itemize_chunk_serial_dst")
|
||||
clean_dir(source)
|
||||
clean_dir(dest)
|
||||
subdir = os.path.join(source, "sub")
|
||||
os.makedirs(subdir)
|
||||
with open(os.path.join(subdir, "file.txt"), "wb") as fh:
|
||||
fh.write(b"payload\n")
|
||||
|
||||
result, _ = run_client(source, dest, flags=["--preserve"], port=shared_server.port)
|
||||
assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}"
|
||||
|
||||
# Change only the source directory's mtime; its contents stay identical
|
||||
# so only the directory's time attribute differs on the rerun.
|
||||
os.utime(subdir, (1_000_000_000, 1_000_000_000))
|
||||
|
||||
result, _ = run_client(source, dest,
|
||||
flags=["--preserve", "-i", "--chunk-serialization"],
|
||||
port=shared_server.port)
|
||||
assert result.returncode == 0, f"chunk-serialization -i failed: {result.stderr[:200]}"
|
||||
dir_lines = [line for line in result.stdout.splitlines() if line.endswith(" sub/")]
|
||||
assert dir_lines == [".d..t...... sub/"], (
|
||||
f"expected an attribute-change dir line, got {dir_lines!r}; "
|
||||
f"full stdout={result.stdout!r}"
|
||||
)
|
||||
|
||||
def test_changed_file_on_second_incremental_run_prints_exactly_one_line(self, shared_server):
|
||||
"""A changed file itemizes exactly once on an incremental rerun while
|
||||
unchanged files print nothing (no double emission)."""
|
||||
@@ -2975,12 +3089,12 @@ class TestDelayUpdates:
|
||||
"staging directory left behind after a successful delayed transfer"
|
||||
|
||||
@pytest.mark.skipif(shutil.which("rsync") is None, reason="rsync not installed")
|
||||
def test_delay_updates_staging_name_collision_residual(self):
|
||||
"""Documented residual (RSYNC_COMPAT.md `--delay-updates` row): FastSync
|
||||
uses a fixed `.fastsync-stage` staging name and wipes a pre-existing tree
|
||||
of that name at the start of a delayed run (crash-leftover cleanup),
|
||||
even without `--delete`; rsync leaves a genuine destination entry of that
|
||||
name untouched. Pins the divergence that keeps the row Divergent."""
|
||||
def test_delay_updates_staging_name_collision_preserved(self):
|
||||
"""rsync parity (RSYNC_COMPAT.md `--delay-updates` row): the receiver
|
||||
stages under a per-run unique name, so a genuine pre-existing
|
||||
destination entry named like the reserved staging prefix (`.fastsync-
|
||||
stage`) is never wiped -- even without `--delete`. rsync likewise
|
||||
leaves a real destination entry of its own temp name untouched."""
|
||||
source = self._make_source("delay_collide_src")
|
||||
rdst = os.path.join(TEST_DATA_DIR, "delay_collide_rdst")
|
||||
fdst = os.path.join(TEST_DATA_DIR, "delay_collide_fdst")
|
||||
@@ -3006,8 +3120,11 @@ class TestDelayUpdates:
|
||||
result, _ = run_client(source, fdst, flags=["--delay-updates"],
|
||||
port=server.port)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||
assert not os.path.exists(os.path.join(fdst, self.STAGING)), \
|
||||
"FastSync did not wipe the reserved staging name (residual changed)"
|
||||
assert _read_file(os.path.join(fdst, self.STAGING, "keepme.txt")) == b"genuine user data\n", \
|
||||
"FastSync destroyed a genuine destination entry named like the staging prefix"
|
||||
# The per-run staging directory itself is removed after a clean run.
|
||||
leftovers = [n for n in os.listdir(fdst) if n.startswith(self.STAGING + ".")]
|
||||
assert leftovers == [], f"per-run staging directories left behind: {leftovers}"
|
||||
|
||||
@pytest.mark.parametrize("mt", [False, True])
|
||||
def test_delay_updates_incremental_rerun_no_leftovers(self, shared_server, mt):
|
||||
@@ -3047,10 +3164,11 @@ class TestDelayUpdates:
|
||||
|
||||
@pytest.mark.parametrize("mt", [False, True])
|
||||
def test_delete_with_delay_updates(self, mt):
|
||||
"""--delete runs before publication, so the delete walker must not treat
|
||||
the staging directory as a set of extras: a changed file must still be
|
||||
published after genuine extras are removed. Uses its own server started
|
||||
with --allow-delete (the shared session server refuses deletion)."""
|
||||
"""rsync parity: --delay-updates implies --delete-after, so every staged
|
||||
update is published first and the genuine extras are removed only after
|
||||
that (the delete walker must never treat the staging directory as a set
|
||||
of extras). Uses its own server started with --allow-delete (the shared
|
||||
session server refuses deletion)."""
|
||||
source = os.path.join(TEST_DATA_DIR, "delay_delete_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "delay_delete_dst")
|
||||
clean_dir(source)
|
||||
@@ -3080,6 +3198,65 @@ class TestDelayUpdates:
|
||||
assert not os.path.exists(os.path.join(received, "extra.txt")), \
|
||||
"genuine extra file was not deleted"
|
||||
assert not os.path.isdir(os.path.join(dest, self.STAGING))
|
||||
assert [n for n in os.listdir(dest) if n.startswith(self.STAGING + ".")] == []
|
||||
|
||||
@pytest.mark.parametrize("mt", [False, True])
|
||||
def test_delay_updates_delete_keeps_backup(self, mt):
|
||||
"""The --backup/--delay-updates interplay: the old destination file is
|
||||
moved aside at publication, and that backup survives the implied
|
||||
--delete-after pass (rsync never treats a backup file as an extra)."""
|
||||
source = os.path.join(TEST_DATA_DIR, "delay_bak_del_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "delay_bak_del_dst")
|
||||
clean_dir(source)
|
||||
clean_dir(dest)
|
||||
with open(os.path.join(source, "f.txt"), "wb") as fh:
|
||||
fh.write(b"NEW")
|
||||
received = get_dest_received_dir(dest, source)
|
||||
os.makedirs(received, exist_ok=True)
|
||||
with open(os.path.join(received, "f.txt"), "wb") as fh:
|
||||
fh.write(b"OLD")
|
||||
os.utime(os.path.join(received, "f.txt"), (1_500_000_000, 1_500_000_000))
|
||||
# A pre-existing backup-looking extra must also be shielded.
|
||||
with open(os.path.join(received, "stale.txt~"), "wb") as fh:
|
||||
fh.write(b"stale backup")
|
||||
with ServerManager() as server:
|
||||
server.start(extra_args=["--allow-delete"])
|
||||
flags = ["--delete", "--backup", "--delay-updates"] + (["--threads"] if mt else [])
|
||||
result, _ = run_client(source, dest, flags=flags, port=server.port)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||
assert _read_file(os.path.join(received, "f.txt")) == b"NEW"
|
||||
assert _read_file(os.path.join(received, "f.txt~")) == b"OLD", \
|
||||
"the publication backup was removed by the delete-after pass"
|
||||
assert os.path.exists(os.path.join(received, "stale.txt~")), \
|
||||
"a pre-existing backup-suffixed entry was deleted"
|
||||
|
||||
@pytest.mark.parametrize("mt", [False, True])
|
||||
def test_delay_updates_failed_run_leaves_no_staged_files(self, shared_server, mt):
|
||||
"""A run that fails before publication installs nothing and removes the
|
||||
per-run staging directory (no staged leftovers)."""
|
||||
source = os.path.join(TEST_DATA_DIR, "delay_fail_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "delay_fail_dst")
|
||||
clean_dir(source)
|
||||
clean_dir(dest)
|
||||
with open(os.path.join(source, "top.txt"), "wb") as fh:
|
||||
fh.write(b"top\n")
|
||||
os.makedirs(os.path.join(source, "sub"))
|
||||
with open(os.path.join(source, "sub", "deep.txt"), "wb") as fh:
|
||||
fh.write(b"deep\n")
|
||||
# Plant a regular file where the "sub" directory must be created so the
|
||||
# nested publish fails (the top-level file still publishes first).
|
||||
received = get_dest_received_dir(dest, source)
|
||||
os.makedirs(received)
|
||||
with open(os.path.join(received, "sub"), "wb") as fh:
|
||||
fh.write(b"blocker")
|
||||
|
||||
flags = ["--delay-updates"] + (["--threads"] if mt else [])
|
||||
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
|
||||
assert result.returncode != 0, "a blocked nested publish must fail the run"
|
||||
assert not os.path.lexists(os.path.join(received, "sub", "deep.txt")), \
|
||||
"a staged file appeared despite the failed run"
|
||||
assert [n for n in os.listdir(dest) if n.startswith(self.STAGING + ".")] == [], \
|
||||
"the per-run staging directory survived a failed run"
|
||||
|
||||
def test_delay_updates_rejects_reserved_backup_dir(self):
|
||||
"""--backup-dir equal to the internal staging name must be rejected so
|
||||
@@ -6865,6 +7042,72 @@ class TestExtendedAttributes:
|
||||
f"is not interoperable: ours={rec!r} rsync={out_rec!r}"
|
||||
)
|
||||
|
||||
@pytest.mark.ci
|
||||
def test_fake_super_directory_rsync_interop(self, shared_server):
|
||||
"""#319: --fake-super fakes DIRECTORIES too. A recursive -a
|
||||
--fake-super run must write rsync 3.4.1's `user.rsync.%stat` record on
|
||||
the directory itself (full mode with S_IFDIR + special bits, rdev 0,0,
|
||||
uid:gid), replay only the permission bits on disk, and real rsync must
|
||||
read the tree and re-emit the identical record."""
|
||||
rsync = shutil.which("rsync")
|
||||
if rsync is None:
|
||||
pytest.skip("rsync not installed")
|
||||
source, dest = self._source_and_dest("fakesuper_dir_interop")
|
||||
sub = os.path.join(source, "subdir")
|
||||
os.makedirs(sub)
|
||||
with open(os.path.join(sub, "f.txt"), "wb") as fh:
|
||||
fh.write(b"dir interop\n")
|
||||
if not _xattr_supported(sub):
|
||||
pytest.skip("filesystem does not support user xattrs")
|
||||
# A special bit (setgid) is exactly what a fake-super record exists to
|
||||
# carry: rsync only re-emits a directory record when there is something
|
||||
# it cannot represent on disk (a special bit, or a mode it would widen
|
||||
# to keep the owner's rwx). Skip cleanly when the filesystem drops it.
|
||||
os.chmod(sub, 0o2751)
|
||||
if stat.S_IMODE(os.stat(sub).st_mode) & 0o7000 == 0:
|
||||
pytest.skip("filesystem drops directory special bits")
|
||||
uid = os.stat(sub).st_uid
|
||||
|
||||
result, _ = run_client(source, dest, flags=["-a", "--fake-super"],
|
||||
port=shared_server.port)
|
||||
assert result.returncode == 0, \
|
||||
f"-a --fake-super dir sync failed: {(result.stderr or result.stdout)[:300]}"
|
||||
received = get_dest_received_dir(dest, source)
|
||||
dst_sub = os.path.join(received, "subdir")
|
||||
assert os.path.isdir(dst_sub), "the directory entry was not transferred"
|
||||
|
||||
rec = os.getxattr(dst_sub, "user.rsync.%stat").decode()
|
||||
fields = rec.split()
|
||||
assert len(fields) == 3, f"unexpected rsync fake-super record {rec!r}"
|
||||
mode_field, rdev_field, owner_field = fields
|
||||
assert rdev_field == "0,0", f"directory rdev must be 0,0, got {rdev_field!r}"
|
||||
assert int(mode_field, 8) & 0o170000 == stat.S_IFDIR, (
|
||||
f"recorded mode {mode_field!r} must carry S_IFDIR"
|
||||
)
|
||||
assert int(mode_field, 8) & 0o7777 == 0o2751, (
|
||||
f"recorded mode {mode_field!r} must carry the full source mode 02751"
|
||||
)
|
||||
assert owner_field.split(":")[0] == str(uid), \
|
||||
f"recorded uid {owner_field!r} != source uid {uid}"
|
||||
# Permission bits only on disk: the setgid bit stays in the record.
|
||||
assert stat.S_IMODE(os.stat(dst_sub).st_mode) == 0o751, (
|
||||
"the directory's special bits must not be installed on disk"
|
||||
)
|
||||
|
||||
# Real rsync reads FastSync's directory record and re-emits it verbatim.
|
||||
out = os.path.join(TEST_DATA_DIR, "fakesuper_dir_interop_rsync")
|
||||
clean_dir(out)
|
||||
rs = subprocess.run([rsync, "-aX", "--fake-super", received + "/", out + "/"],
|
||||
capture_output=True, text=True, timeout=120)
|
||||
assert rs.returncode == 0, (
|
||||
f"rsync could not read FastSync's fake-super directory tree: {rs.stderr[:300]}"
|
||||
)
|
||||
out_rec = os.getxattr(os.path.join(out, "subdir"), "user.rsync.%stat").decode()
|
||||
assert out_rec == rec, (
|
||||
"rsync re-emitted a different directory fake-super record; FastSync's "
|
||||
f"grammar is not interoperable: ours={rec!r} rsync={out_rec!r}"
|
||||
)
|
||||
|
||||
@pytest.mark.ci
|
||||
def test_directory_xattrs_preserved(self, shared_server):
|
||||
"""#286.3: -aX must preserve user.* xattrs on DIRECTORIES, not just files."""
|
||||
|
||||
@@ -0,0 +1,222 @@
|
||||
"""#318: whole-file streaming above the receiver's 256 MiB ceiling.
|
||||
|
||||
The receiver's historical whole-file bound (``MAX_RECEIVE_WHOLE_FILE_SIZE``,
|
||||
256 MiB) refused any single-file payload above it. The transfer engine now
|
||||
streams such a payload (and the basis read/verify/hash) through a bounded buffer
|
||||
and spools it to a temp file, so arbitrarily large single files transfer without
|
||||
being materialized in memory.
|
||||
|
||||
To exercise the streaming path deterministically and quickly, these tests lower
|
||||
the receiver bound with the test-only ``FASTSYNC_MAX_WHOLE_FILE_SIZE`` hook (it
|
||||
can only lower, never raise, the protocol ceiling) and transfer a file a few
|
||||
times larger than the lowered bound. A real >256 MiB transfer is covered once,
|
||||
unmarked, so it runs in the full suite but not the fast PR gate.
|
||||
"""
|
||||
import hashlib
|
||||
import os
|
||||
import random
|
||||
import shutil
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from common import ( # noqa: E402
|
||||
ServerManager,
|
||||
TEST_DATA_DIR,
|
||||
clean_dir,
|
||||
get_dest_received_dir,
|
||||
run_client,
|
||||
)
|
||||
|
||||
LOW_BOUND = 1024 * 1024
|
||||
FILE_SIZE = 3 * 1024 * 1024
|
||||
OLD_MTIME = 1_500_000_000
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def small_bound_server():
|
||||
"""A server whose whole-file streaming bound is 1 MiB."""
|
||||
server = ServerManager()
|
||||
server.start(extra_args=["--allow-super"],
|
||||
env={"FASTSYNC_MAX_WHOLE_FILE_SIZE": str(LOW_BOUND)})
|
||||
yield server
|
||||
server.stop()
|
||||
|
||||
|
||||
def _payload(n):
|
||||
rng = random.Random(0xC0FFEE)
|
||||
return rng.randbytes(n)
|
||||
|
||||
|
||||
def _write(path, data, mtime=None):
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
with open(path, "wb") as fh:
|
||||
fh.write(data)
|
||||
if mtime is not None:
|
||||
os.utime(path, (mtime, mtime))
|
||||
|
||||
|
||||
def _resolved(dest, source, rel):
|
||||
return os.path.join(get_dest_received_dir(dest, source), rel)
|
||||
|
||||
|
||||
def _no_spool_leftovers(dest):
|
||||
leftovers = []
|
||||
for root, _dirs, files in os.walk(dest):
|
||||
leftovers += [os.path.join(root, f) for f in files if ".fastsync-spool." in f]
|
||||
return leftovers
|
||||
|
||||
|
||||
class TestStreamedWholeFile:
|
||||
"""A file above the (lowered) bound transfers correctly in every mode."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"flags",
|
||||
[
|
||||
["-a"],
|
||||
["-a", "--incremental"],
|
||||
["-a", "-z"],
|
||||
["-a", "--incremental", "-z"],
|
||||
["-a", "--threads", "--incremental"],
|
||||
["-a", "--inplace"],
|
||||
["-a", "--partial"],
|
||||
],
|
||||
)
|
||||
def test_above_bound_transfers(self, small_bound_server, flags):
|
||||
tag = "_".join(f.strip("-") for f in flags) or "default"
|
||||
source = os.path.join(TEST_DATA_DIR, f"stream_src_{tag}")
|
||||
dest = os.path.join(TEST_DATA_DIR, f"stream_dst_{tag}")
|
||||
clean_dir(source)
|
||||
clean_dir(dest)
|
||||
data = _payload(FILE_SIZE)
|
||||
_write(os.path.join(source, "big.bin"), data, OLD_MTIME)
|
||||
if "--inplace" in flags:
|
||||
# --inplace only matters when the destination already exists.
|
||||
_write(_resolved(dest, source, "big.bin"), b"stale", OLD_MTIME)
|
||||
|
||||
result, _ = run_client(source, dest, flags=flags, port=small_bound_server.port)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:400]
|
||||
|
||||
got = os.path.join(get_dest_received_dir(dest, source), "big.bin")
|
||||
assert os.path.exists(got), "streamed file was not written"
|
||||
with open(got, "rb") as fh:
|
||||
assert fh.read() == data, "streamed file content mismatch"
|
||||
assert _no_spool_leftovers(dest) == [], "a spool temp file leaked"
|
||||
|
||||
|
||||
class TestStreamedBasis:
|
||||
"""A basis above the bound is streamed, not refused (compare/copy/link)."""
|
||||
|
||||
def _seed(self, dest, source, data):
|
||||
clean_dir(source)
|
||||
clean_dir(dest)
|
||||
_write(os.path.join(source, "big.bin"), data, OLD_MTIME)
|
||||
# FastSync resolves a relative basis DIR against the destination and
|
||||
# appends the transfer-relative name.
|
||||
_write(os.path.join(dest, "basis", "big.bin"), data, OLD_MTIME)
|
||||
|
||||
def test_compare_dest_above_bound(self, small_bound_server):
|
||||
source = os.path.join(TEST_DATA_DIR, "sbasis_cmp_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "sbasis_cmp_dst")
|
||||
data = _payload(FILE_SIZE)
|
||||
self._seed(dest, source, data)
|
||||
result, _ = run_client(source, dest,
|
||||
flags=["-a", "--compare-dest=basis", "--incremental"],
|
||||
port=small_bound_server.port)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:400]
|
||||
# compare-dest never copies: an already-present destination stays sparse.
|
||||
assert not os.path.exists(_resolved(dest, source, "big.bin"))
|
||||
|
||||
def test_copy_dest_above_bound(self, small_bound_server):
|
||||
source = os.path.join(TEST_DATA_DIR, "sbasis_cpy_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "sbasis_cpy_dst")
|
||||
data = _payload(FILE_SIZE)
|
||||
self._seed(dest, source, data)
|
||||
result, _ = run_client(source, dest,
|
||||
flags=["-a", "--copy-dest=basis", "--incremental"],
|
||||
port=small_bound_server.port)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:400]
|
||||
got = _resolved(dest, source, "big.bin")
|
||||
assert os.path.exists(got)
|
||||
with open(got, "rb") as fh:
|
||||
assert fh.read() == data
|
||||
assert os.stat(got).st_ino != os.stat(os.path.join(dest, "basis", "big.bin")).st_ino
|
||||
assert _no_spool_leftovers(dest) == []
|
||||
|
||||
def test_link_dest_above_bound(self, small_bound_server):
|
||||
source = os.path.join(TEST_DATA_DIR, "sbasis_lnk_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "sbasis_lnk_dst")
|
||||
data = _payload(FILE_SIZE)
|
||||
self._seed(dest, source, data)
|
||||
result, _ = run_client(source, dest,
|
||||
flags=["-a", "--link-dest=basis", "--incremental"],
|
||||
port=small_bound_server.port)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:400]
|
||||
got = _resolved(dest, source, "big.bin")
|
||||
assert os.path.exists(got)
|
||||
with open(got, "rb") as fh:
|
||||
assert fh.read() == data
|
||||
assert os.stat(got).st_ino == os.stat(os.path.join(dest, "basis", "big.bin")).st_ino
|
||||
|
||||
|
||||
class TestFuzzyAboveBound:
|
||||
"""-y/--fuzzy reuses a basis above the bound by streaming its signature."""
|
||||
|
||||
def test_fuzzy_oversized_sibling(self, small_bound_server):
|
||||
source = os.path.join(TEST_DATA_DIR, "sfuzzy_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "sfuzzy_dst")
|
||||
clean_dir(source)
|
||||
clean_dir(dest)
|
||||
base = _payload(FILE_SIZE)
|
||||
sibling = bytearray(base)
|
||||
sibling[FILE_SIZE // 2:FILE_SIZE // 2 + 4096] = bytes(
|
||||
(b + 1) % 256 for b in sibling[FILE_SIZE // 2:FILE_SIZE // 2 + 4096])
|
||||
_write(os.path.join(source, "report_v2.txt"), base)
|
||||
_write(os.path.join(get_dest_received_dir(dest, source), "report_v1.txt"), bytes(sibling))
|
||||
result, _ = run_client(
|
||||
source, dest,
|
||||
flags=["-a", "--incremental", "--delta", "--fuzzy", "--stats"],
|
||||
port=small_bound_server.port)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:400]
|
||||
got = _resolved(dest, source, "report_v2.txt")
|
||||
with open(got, "rb") as fh:
|
||||
assert fh.read() == base, "fuzzy reconstruction mismatch"
|
||||
assert _no_spool_leftovers(dest) == []
|
||||
|
||||
|
||||
class TestRealLargeFile:
|
||||
"""A real >256 MiB transfer, run only in the full (non-PR-gate) suite."""
|
||||
|
||||
def test_real_300mib_transfer(self, shared_server):
|
||||
source = os.path.join(TEST_DATA_DIR, "real_large_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "real_large_dst")
|
||||
clean_dir(source)
|
||||
clean_dir(dest)
|
||||
n = 300 * 1024 * 1024
|
||||
# Deterministic, compressible pattern written in bounded chunks.
|
||||
chunk = bytes(range(256)) * 4096
|
||||
digest = hashlib.sha256()
|
||||
with open(os.path.join(source, "big.bin"), "wb") as fh:
|
||||
written = 0
|
||||
while written < n:
|
||||
piece = chunk[: min(len(chunk), n - written)]
|
||||
fh.write(piece)
|
||||
digest.update(piece)
|
||||
written += len(piece)
|
||||
|
||||
result, _ = run_client(source, dest, flags=["-a", "--incremental"],
|
||||
port=shared_server.port)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:400]
|
||||
got = os.path.join(get_dest_received_dir(dest, source), "big.bin")
|
||||
assert os.path.getsize(got) == n
|
||||
got_digest = hashlib.sha256()
|
||||
with open(got, "rb") as fh:
|
||||
while True:
|
||||
block = fh.read(1 << 20)
|
||||
if not block:
|
||||
break
|
||||
got_digest.update(block)
|
||||
assert got_digest.hexdigest() == digest.hexdigest()
|
||||
shutil.rmtree(source, ignore_errors=True)
|
||||
shutil.rmtree(dest, ignore_errors=True)
|
||||
@@ -536,3 +536,78 @@ class TestFilterProtect:
|
||||
assert "extra.log" not in result.stdout, result.stdout
|
||||
assert os.path.exists(os.path.join(received, "extra.log"))
|
||||
assert os.path.exists(os.path.join(received, "other.txt"))
|
||||
|
||||
@pytest.mark.ci
|
||||
def test_perdir_protect_dest_only_matches_rsync(self):
|
||||
"""#315: a `P` rule inside a per-directory `.rsync-filter` is carried to
|
||||
the receiver, so a destination-only extra matching ONLY that rule is
|
||||
shielded under --delete. Both roots carry the same filter file (rsync's
|
||||
receiver reads the destination one; FastSync carries the source's)."""
|
||||
source = os.path.join(TEST_DATA_DIR, "fpdp_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "fpdp_dst")
|
||||
rdst = os.path.join(TEST_DATA_DIR, "fpdp_rdst")
|
||||
clean_dir(source)
|
||||
_write(os.path.join(source, "keep.txt"), b"keep\n")
|
||||
_write(os.path.join(source, ".rsync-filter"), b"P extra.log\n")
|
||||
clean_dir(rdst)
|
||||
_write(os.path.join(rdst, ".rsync-filter"), b"P extra.log\n")
|
||||
_write(os.path.join(rdst, "extra.log"), b"extra\n")
|
||||
_write(os.path.join(rdst, "other.txt"), b"other\n")
|
||||
|
||||
rsync_result = _rsync(["-aF", "--delete", source + "/", rdst + "/"])
|
||||
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||
assert os.path.exists(os.path.join(rdst, "extra.log")), "rsync did not protect extra.log"
|
||||
assert not os.path.exists(os.path.join(rdst, "other.txt"))
|
||||
|
||||
clean_dir(dest)
|
||||
received = get_dest_received_dir(dest, source)
|
||||
os.makedirs(received, exist_ok=True)
|
||||
_write(os.path.join(received, ".rsync-filter"), b"P extra.log\n")
|
||||
_write(os.path.join(received, "extra.log"), b"extra\n")
|
||||
_write(os.path.join(received, "other.txt"), b"other\n")
|
||||
with ServerManager() as server:
|
||||
server.start(extra_args=["--allow-delete"])
|
||||
result, _ = run_client(source, dest, flags=["-aF", "--delete"], port=server.port)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||
assert os.path.exists(os.path.join(received, "extra.log")), (
|
||||
"FastSync must protect a destination-only per-directory P match like rsync")
|
||||
assert not os.path.exists(os.path.join(received, "other.txt"))
|
||||
|
||||
@requires_rsync
|
||||
@pytest.mark.ci
|
||||
def test_perdir_protect_dry_run_enumeration(self, shared_server):
|
||||
"""#315: the -n/--dry-run would-delete enumeration also honors the
|
||||
carried per-directory rules, matching rsync's `*deleting` set: a
|
||||
destination-only entry matching only a `.rsync-filter` P rule is not
|
||||
reported (nor removed)."""
|
||||
source = os.path.join(TEST_DATA_DIR, "fpdp_nd_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "fpdp_nd_dst")
|
||||
rdst = os.path.join(TEST_DATA_DIR, "fpdp_nd_rdst")
|
||||
clean_dir(source)
|
||||
_write(os.path.join(source, "keep.txt"), b"keep\n")
|
||||
_write(os.path.join(source, ".rsync-filter"), b"P extra.log\n")
|
||||
received = get_dest_received_dir(dest, source)
|
||||
clean_dir(rdst)
|
||||
clean_dir(received)
|
||||
for root in (rdst, received):
|
||||
_write(os.path.join(root, "keep.txt"), b"keep\n")
|
||||
_write(os.path.join(root, ".rsync-filter"), b"P extra.log\n")
|
||||
_write(os.path.join(root, "extra.log"), b"extra\n")
|
||||
_write(os.path.join(root, "other.txt"), b"other\n")
|
||||
|
||||
rsync_result = _rsync(["-an", "-i", "-F", "--delete", source + "/", rdst + "/"])
|
||||
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||
rsync_del = sorted(l for l in rsync_result.stdout.splitlines()
|
||||
if l.startswith("*deleting"))
|
||||
assert rsync_del == ["*deleting other.txt"], f"unexpected rsync set: {rsync_del}"
|
||||
|
||||
with ServerManager() as server:
|
||||
server.start(extra_args=["--allow-delete"])
|
||||
result, _ = run_client(source, dest, flags=["-aF", "-n", "-i", "--delete"],
|
||||
port=server.port)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||
fs_del = sorted(l for l in (result.stdout or "").splitlines()
|
||||
if l.startswith("*deleting"))
|
||||
assert fs_del == rsync_del, f"rsync={rsync_del}\nfastsync={fs_del}"
|
||||
assert os.path.exists(os.path.join(received, "extra.log"))
|
||||
assert os.path.exists(os.path.join(received, "other.txt"))
|
||||
|
||||
@@ -133,14 +133,14 @@ def _seed_protocol_source(source):
|
||||
class TestProtocol:
|
||||
@pytest.mark.ci
|
||||
def test_protocol_current_version_accepted(self, shared_server):
|
||||
"""--protocol=2.29.0 (the current PROTOCOL_VERSION) is accepted and the
|
||||
"""--protocol=2.30.0 (the current PROTOCOL_VERSION) is accepted and the
|
||||
transfer completes normally."""
|
||||
source = os.path.join(TEST_DATA_DIR, "proto_ok_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "proto_ok_dst")
|
||||
shutil.rmtree(dest, ignore_errors=True)
|
||||
os.makedirs(dest)
|
||||
_seed_protocol_source(source)
|
||||
result, _ = run_client(source, dest, flags=["--protocol=2.29.0"],
|
||||
result, _ = run_client(source, dest, flags=["--protocol=2.30.0"],
|
||||
port=shared_server.port)
|
||||
assert result.returncode == 0, \
|
||||
f"--protocol current run failed: {(result.stderr or result.stdout)[:400]}"
|
||||
@@ -157,7 +157,7 @@ class TestProtocol:
|
||||
shutil.rmtree(dest, ignore_errors=True)
|
||||
os.makedirs(dest)
|
||||
_seed_protocol_source(source)
|
||||
for bad in ("2.28.0", "2.27.0", "2.26.0", "2.25.0", "2.24.0", "2.23.0", "2.22.0", "2.21.0", "2.20.0",
|
||||
for bad in ("2.29.0", "2.28.0", "2.27.0", "2.26.0", "2.25.0", "2.24.0", "2.23.0", "2.22.0", "2.21.0", "2.20.0",
|
||||
"2.19.0", "2.18.0", "2.17.0", "2.15.0", "2.16.0", "216", "31"):
|
||||
result, _ = run_client(source, dest, flags=[f"--protocol={bad}"],
|
||||
port=shared_server.port)
|
||||
|
||||
+75
-10
@@ -352,7 +352,7 @@ static void test_parse_args_protocol_accept_current() {
|
||||
Config* cfg = valid_client_config();
|
||||
EXPECT_NOT_NULL(cfg);
|
||||
char* argv_equals[] = {"fastsync", "--source-dir", "/src",
|
||||
"--dest-dir", "/dst", "--protocol=2.29.0"};
|
||||
"--dest-dir", "/dst", "--protocol=2.30.0"};
|
||||
int positional_args[2];
|
||||
int positional_count = 0;
|
||||
EXPECT_EQ_INT(parse_args(cfg, 6, argv_equals, positional_args, &positional_count), 0);
|
||||
@@ -362,7 +362,7 @@ static void test_parse_args_protocol_accept_current() {
|
||||
cfg = valid_client_config();
|
||||
EXPECT_NOT_NULL(cfg);
|
||||
char* argv_space[] = {"fastsync", "--source-dir", "/src", "--dest-dir",
|
||||
"/dst", "--protocol", "2.29.0"};
|
||||
"/dst", "--protocol", "2.30.0"};
|
||||
positional_count = 0;
|
||||
EXPECT_EQ_INT(parse_args(cfg, 7, argv_space, positional_args, &positional_count), 0);
|
||||
EXPECT_EQ_STR(cfg->version, PROTOCOL_VERSION);
|
||||
@@ -372,10 +372,10 @@ static void test_parse_args_protocol_accept_current() {
|
||||
/* Any --protocol value other than the current PROTOCOL_VERSION must end in
|
||||
* failure (parse_args simply stores it; validate_config rejects it up front). */
|
||||
static void test_parse_args_protocol_rejects_other_versions() {
|
||||
static const char* const bad_versions[] = {"2.17", "2.16", "2.15.0", "2.16.0", "2.17.0",
|
||||
"2.18.0", "2.19.0", "2.20.0", "2.21.0", "2.22.0",
|
||||
"2.23.0", "2.24.0", "2.25.0", "2.26.0", "2.27.0",
|
||||
"2.28.0", "216", "31", "abc", ""};
|
||||
static const char* const bad_versions[] = {
|
||||
"2.17", "2.16", "2.15.0", "2.16.0", "2.17.0", "2.18.0", "2.19.0",
|
||||
"2.20.0", "2.21.0", "2.22.0", "2.23.0", "2.24.0", "2.25.0", "2.26.0",
|
||||
"2.27.0", "2.28.0", "2.29.0", "216", "31", "abc", ""};
|
||||
for (size_t i = 0; i < sizeof(bad_versions) / sizeof(bad_versions[0]); i++) {
|
||||
Config* cfg = valid_client_config();
|
||||
EXPECT_NOT_NULL(cfg);
|
||||
@@ -2323,9 +2323,9 @@ static void test_parse_args_8_bit_output() {
|
||||
}
|
||||
|
||||
static void test_parse_args_stderr_modes() {
|
||||
static const char* const modes[] = {"errors", "all", "e", "a"};
|
||||
static const LogStderrMode expected[] = {LOG_STDERR_ERRORS, LOG_STDERR_ALL, LOG_STDERR_ERRORS,
|
||||
LOG_STDERR_ALL};
|
||||
static const char* const modes[] = {"errors", "all", "client", "e", "a", "c"};
|
||||
static const LogStderrMode expected[] = {LOG_STDERR_ERRORS, LOG_STDERR_ALL, LOG_STDERR_CLIENT,
|
||||
LOG_STDERR_ERRORS, LOG_STDERR_ALL, LOG_STDERR_CLIENT};
|
||||
for (size_t i = 0; i < sizeof(modes) / sizeof(modes[0]); i++) {
|
||||
Config* cfg = config_create();
|
||||
char option[32];
|
||||
@@ -2340,8 +2340,29 @@ static void test_parse_args_stderr_modes() {
|
||||
log_set_stderr_mode(LOG_STDERR_ERRORS);
|
||||
}
|
||||
|
||||
/* rsync's deprecated --msgs2stderr / --no-msgs2stderr spellings map to
|
||||
* --stderr=all and --stderr=client respectively; the client-message channel
|
||||
* that `client` needs now exists (protocol 2.30.0). */
|
||||
static void test_parse_args_msgs2stderr_aliases() {
|
||||
Config* cfg = config_create();
|
||||
char* argv_all[] = {"fastsync", "--msgs2stderr", "/src", "/dst"};
|
||||
int positional_args[2];
|
||||
int positional_count = 0;
|
||||
EXPECT_EQ_INT(parse_args(cfg, 4, argv_all, positional_args, &positional_count), 0);
|
||||
EXPECT_EQ_INT(log_get_stderr_mode(), LOG_STDERR_ALL);
|
||||
config_delete(cfg);
|
||||
|
||||
cfg = config_create();
|
||||
char* argv_client[] = {"fastsync", "--no-msgs2stderr", "/src", "/dst"};
|
||||
positional_count = 0;
|
||||
EXPECT_EQ_INT(parse_args(cfg, 4, argv_client, positional_args, &positional_count), 0);
|
||||
EXPECT_EQ_INT(log_get_stderr_mode(), LOG_STDERR_CLIENT);
|
||||
config_delete(cfg);
|
||||
log_set_stderr_mode(LOG_STDERR_ERRORS);
|
||||
}
|
||||
|
||||
static void test_parse_args_rejects_unsupported_stderr_modes() {
|
||||
static const char* const modes[] = {"client", "c", "invalid"};
|
||||
static const char* const modes[] = {"invalid", "x", ""};
|
||||
for (size_t i = 0; i < sizeof(modes) / sizeof(modes[0]); i++) {
|
||||
Config* cfg = config_create();
|
||||
char option[32];
|
||||
@@ -2956,6 +2977,48 @@ static void test_parse_args_delay_updates() {
|
||||
config_delete(cfg);
|
||||
}
|
||||
|
||||
/* rsync parity: --delay-updates implies --delete-after when --delete is
|
||||
active (all updates publish first, then extras are removed). An explicit
|
||||
other timing is overridden; without --delete no timing is set. */
|
||||
static void test_parse_args_delay_updates_implies_delete_after() {
|
||||
Config* cfg = config_create();
|
||||
char* argv[] = {"fastsync", "--delay-updates", "--delete", "/src", "/dst"};
|
||||
int positional_args[2];
|
||||
int positional_count = 0;
|
||||
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0);
|
||||
EXPECT_TRUE(cfg->use_delete);
|
||||
EXPECT_TRUE(cfg->delete_after);
|
||||
EXPECT_FALSE(cfg->delete_before);
|
||||
EXPECT_FALSE(cfg->delete_during);
|
||||
EXPECT_FALSE(cfg->delete_delay);
|
||||
cfg->send_directory = str_dup("/src");
|
||||
cfg->receive_root_directory = str_dup("/dst");
|
||||
EXPECT_TRUE(validate_config(cfg));
|
||||
config_delete(cfg);
|
||||
|
||||
/* An explicit conflicting timing is normalized to delete-after. */
|
||||
cfg = config_create();
|
||||
char* argv_before[] = {"fastsync", "--delay-updates", "--delete-before", "/src", "/dst"};
|
||||
positional_count = 0;
|
||||
EXPECT_EQ_INT(parse_args(cfg, 5, argv_before, positional_args, &positional_count), 0);
|
||||
EXPECT_TRUE(cfg->use_delete);
|
||||
EXPECT_TRUE(cfg->delete_after);
|
||||
EXPECT_FALSE(cfg->delete_before);
|
||||
config_delete(cfg);
|
||||
|
||||
/* Without --delete there is no deletion, so no timing is selected. */
|
||||
cfg = config_create();
|
||||
char* argv_alone[] = {"fastsync", "--delay-updates", "/src", "/dst"};
|
||||
positional_count = 0;
|
||||
EXPECT_EQ_INT(parse_args(cfg, 4, argv_alone, positional_args, &positional_count), 0);
|
||||
EXPECT_FALSE(cfg->use_delete);
|
||||
EXPECT_FALSE(cfg->delete_after);
|
||||
EXPECT_FALSE(cfg->delete_before);
|
||||
EXPECT_FALSE(cfg->delete_during);
|
||||
EXPECT_FALSE(cfg->delete_delay);
|
||||
config_delete(cfg);
|
||||
}
|
||||
|
||||
/* rsync rejects --delay-updates with --inplace; FastSync must too. */
|
||||
static void test_validate_config_delay_updates_rejects_inplace() {
|
||||
Config* cfg = valid_client_config();
|
||||
@@ -5242,6 +5305,7 @@ void test_client_cli() {
|
||||
test_parse_args_ignore_times();
|
||||
test_parse_args_8_bit_output();
|
||||
test_parse_args_stderr_modes();
|
||||
test_parse_args_msgs2stderr_aliases();
|
||||
test_parse_args_rejects_unsupported_stderr_modes();
|
||||
test_parse_args_secluded_args();
|
||||
test_parse_args_chunk_serialization_long_form();
|
||||
@@ -5290,6 +5354,7 @@ void test_client_cli() {
|
||||
test_parse_args_checksum_seed();
|
||||
test_parse_args_temp_dir();
|
||||
test_parse_args_delay_updates();
|
||||
test_parse_args_delay_updates_implies_delete_after();
|
||||
test_validate_config_delay_updates_rejects_inplace();
|
||||
test_validate_config_delay_updates_rejects_reserved_backup_dir();
|
||||
test_parse_args_files_from();
|
||||
|
||||
+8
-5
@@ -2924,7 +2924,7 @@ static void golden_config_populate(Config* c) {
|
||||
array_list_add(c->filters, str_dup("- /sub/dir/"));
|
||||
}
|
||||
|
||||
/* The pinned golden frame (protocol 2.29.0). The values below are the only
|
||||
/* The pinned golden frame (protocol 2.30.0). The values below are the only
|
||||
* thing that ties the generated table to the historical wire format; update
|
||||
* them ONLY with a PROTOCOL_VERSION bump and a documented reason. The 2.24.0
|
||||
* delete-plan wave changed only the version string; 2.25.0 appended the
|
||||
@@ -2936,10 +2936,13 @@ static void golden_config_populate(Config* c) {
|
||||
* (project decision), so the frame grew by one int to 886 bytes. The 2.29.0
|
||||
* symlink-xattr wave changes only the version string: the config-frame layout
|
||||
* is unchanged (use_xattrs already crosses the wire); the STATUS_SYMLINK frame
|
||||
* body grows instead. The byte-exact values are recomputed for the merged
|
||||
* layout. */
|
||||
* body grows instead. The 2.30.0 client-message/partial wave changes only the
|
||||
* version string: the config-frame layout is unchanged (the new
|
||||
* STATUS_CLIENT_MSG and STATUS_PARTIAL statuses are not part of this frame), so
|
||||
* the length stays 886 and only the hash moves. The byte-exact values are
|
||||
* recomputed for the merged layout. */
|
||||
#define GOLDEN_WIRE_LEN 886
|
||||
#define GOLDEN_WIRE_HASH 17827864270611927842ULL
|
||||
#define GOLDEN_WIRE_HASH 4169866417069573876ULL
|
||||
|
||||
static unsigned long long fnv1a_64(const unsigned char* buf, size_t len) {
|
||||
unsigned long long h = 1469598103934665603ULL;
|
||||
@@ -3021,7 +3024,7 @@ static unsigned long long capture_wire_hash(const Config* cfg, size_t* out_len)
|
||||
return h;
|
||||
}
|
||||
|
||||
/* Byte-for-byte wire compatibility guard (protocol 2.29.0). The expected hash
|
||||
/* Byte-for-byte wire compatibility guard (protocol 2.30.0). The expected hash
|
||||
* pins the pre-X-macro byte stream; the refactor MUST NOT change it. */
|
||||
static void test_config_wire_golden() {
|
||||
if (is_running_under_valgrind())
|
||||
|
||||
@@ -87,8 +87,10 @@ static void test_delay_updates_no_final_before_publish() {
|
||||
const char* final_path = "test_delay_tmp/sub/file.txt";
|
||||
/* Before publication the final destination must not contain the file. */
|
||||
EXPECT_FALSE(file_path_exists_secure(final_path));
|
||||
/* The complete staged copy must live inside the staging tree. */
|
||||
char* staged = path_cat("test_delay_tmp/.fastsync-stage", "/sub/file.txt");
|
||||
/* The complete staged copy must live inside the per-run staging tree. */
|
||||
EXPECT_NOT_NULL(cfg->delay_context->staging_name);
|
||||
EXPECT_EQ_INT(strncmp(cfg->delay_context->staging_name, ".fastsync-stage.", 16), 0);
|
||||
char* staged = path_cat(cfg->delay_context->staging_root, "/sub/file.txt");
|
||||
EXPECT_NOT_NULL(staged);
|
||||
// cppcheck-suppress knownConditionTrueFalse
|
||||
if (staged) {
|
||||
@@ -125,6 +127,8 @@ static void test_delay_updates_publish_installs_files() {
|
||||
const char* final_path = "test_delay_pub_tmp/sub/file.txt";
|
||||
EXPECT_FALSE(file_path_exists_secure(final_path));
|
||||
|
||||
char* staging_root = str_dup(cfg->delay_context->staging_root);
|
||||
EXPECT_NOT_NULL(staging_root);
|
||||
EXPECT_TRUE(delay_updates_publish(cfg->delay_context, cfg));
|
||||
/* After a successful publish the file is installed and staging is gone. */
|
||||
char* content = read_all(final_path);
|
||||
@@ -134,7 +138,8 @@ static void test_delay_updates_publish_installs_files() {
|
||||
EXPECT_EQ_STR(content, "published payload");
|
||||
free(content);
|
||||
}
|
||||
EXPECT_FALSE(file_path_exists_secure("test_delay_pub_tmp/.fastsync-stage"));
|
||||
EXPECT_FALSE(file_path_exists_secure(staging_root));
|
||||
free(staging_root);
|
||||
|
||||
out:
|
||||
file_destroy(f);
|
||||
@@ -158,11 +163,17 @@ static void test_delay_updates_cleanup_removes_staged() {
|
||||
goto out;
|
||||
|
||||
EXPECT_EQ_INT(file_save_to_disk_full(root, f, cfg), FILE_SAVE_WRITTEN);
|
||||
EXPECT_TRUE(file_path_exists_secure("test_delay_clean_tmp/.fastsync-stage/sub/file.txt"));
|
||||
char* staged_file = path_cat(cfg->delay_context->staging_root, "/sub/file.txt");
|
||||
char* staging_root = str_dup(cfg->delay_context->staging_root);
|
||||
EXPECT_NOT_NULL(staged_file);
|
||||
EXPECT_NOT_NULL(staging_root);
|
||||
EXPECT_TRUE(file_path_exists_secure(staged_file));
|
||||
|
||||
delay_updates_cleanup(cfg->delay_context);
|
||||
EXPECT_FALSE(file_path_exists_secure("test_delay_clean_tmp/.fastsync-stage"));
|
||||
EXPECT_FALSE(file_path_exists_secure(staging_root));
|
||||
EXPECT_FALSE(file_path_exists_secure("test_delay_clean_tmp/sub/file.txt"));
|
||||
free(staged_file);
|
||||
free(staging_root);
|
||||
|
||||
out:
|
||||
file_destroy(f);
|
||||
@@ -274,6 +285,55 @@ out:
|
||||
remove_tree(root);
|
||||
}
|
||||
|
||||
/* Every context picks its own staging directory name, so two delayed
|
||||
transfers to the same root can never share (and corrupt) a staging tree. */
|
||||
static void test_delay_updates_unique_staging_name() {
|
||||
DelayUpdatesContext* first = delay_updates_context_create("test_delay_uniq_tmp");
|
||||
DelayUpdatesContext* second = delay_updates_context_create("test_delay_uniq_tmp");
|
||||
EXPECT_NOT_NULL(first);
|
||||
EXPECT_NOT_NULL(second);
|
||||
/* cppcheck-suppress knownConditionTrueFalse -- the EXPECT_NOT_NULL checks above return on NULL */
|
||||
if (first && second) {
|
||||
EXPECT_EQ_INT(strncmp(first->staging_name, ".fastsync-stage.", 16), 0);
|
||||
EXPECT_EQ_INT(strncmp(second->staging_name, ".fastsync-stage.", 16), 0);
|
||||
EXPECT_TRUE(strcmp(first->staging_name, second->staging_name) != 0);
|
||||
EXPECT_TRUE(strcmp(first->staging_root, second->staging_root) != 0);
|
||||
}
|
||||
delay_updates_context_destroy(first);
|
||||
delay_updates_context_destroy(second);
|
||||
}
|
||||
|
||||
/* A pre-existing destination entry at the exact (random) staging path is not
|
||||
ours: prepare() must refuse rather than wipe it. */
|
||||
static void test_delay_updates_prepare_refuses_non_owned_collision() {
|
||||
const char* root = "test_delay_collide_tmp";
|
||||
remove_tree(root);
|
||||
DelayUpdatesContext* context = delay_updates_context_create(root);
|
||||
EXPECT_NOT_NULL(context);
|
||||
// cppcheck-suppress knownConditionTrueFalse
|
||||
if (!context)
|
||||
return;
|
||||
/* Plant a genuine directory with user data at the exact staging path. */
|
||||
EXPECT_TRUE(file_ensure_directory_secure(context->staging_root));
|
||||
char* inner = path_cat(context->staging_root, "keepme.txt");
|
||||
EXPECT_NOT_NULL(inner);
|
||||
// cppcheck-suppress knownConditionTrueFalse
|
||||
if (inner) {
|
||||
EXPECT_TRUE(file_write_to_disk(inner, "genuine", 7, false, false));
|
||||
EXPECT_FALSE(delay_updates_prepare(context));
|
||||
char* content = read_all(inner);
|
||||
EXPECT_NOT_NULL(content);
|
||||
// cppcheck-suppress knownConditionTrueFalse
|
||||
if (content) {
|
||||
EXPECT_EQ_STR(content, "genuine");
|
||||
free(content);
|
||||
}
|
||||
free(inner);
|
||||
}
|
||||
delay_updates_context_destroy(context);
|
||||
remove_tree(root);
|
||||
}
|
||||
|
||||
/* The reserved staging name must be recognizable for validation, including
|
||||
with a trailing slash. */
|
||||
static void test_delay_updates_reserved_name_helper() {
|
||||
@@ -288,6 +348,8 @@ static void test_delay_updates_reserved_name_helper() {
|
||||
|
||||
void test_delay_updates() {
|
||||
test_delay_updates_reserved_name_helper();
|
||||
test_delay_updates_unique_staging_name();
|
||||
test_delay_updates_prepare_refuses_non_owned_collision();
|
||||
test_delay_updates_no_final_before_publish();
|
||||
test_delay_updates_publish_installs_files();
|
||||
test_delay_updates_cleanup_removes_staged();
|
||||
|
||||
@@ -11,8 +11,23 @@
|
||||
#include <string.h>
|
||||
#include <sys/socket.h>
|
||||
#include <sys/stat.h>
|
||||
#include <threads.h>
|
||||
#include <unistd.h>
|
||||
|
||||
/* Discards everything written to `fd` until EOF, so a sender that regresses to
|
||||
* emitting an over-budget frame does not block forever on a full socket. */
|
||||
typedef struct {
|
||||
int fd;
|
||||
} DrainArg;
|
||||
|
||||
static int drain_fd_thread(void* arg) {
|
||||
DrainArg* drain = arg;
|
||||
char buffer[8192];
|
||||
while (read(drain->fd, buffer, sizeof(buffer)) > 0)
|
||||
;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Send one STATUS_DELETE_PLAN body (the leading status is consumed by the
|
||||
* caller/receiver entry point) describing `dir` with no kept children. */
|
||||
static void send_plan_frame(int fd, const char* dir) {
|
||||
@@ -225,6 +240,7 @@ static void send_config_only_frame(int fd, const char* missing_path) {
|
||||
EXPECT_TRUE(send_int(fd, 0)); /* size-skipped */
|
||||
EXPECT_TRUE(send_int(fd, 1)); /* missing args */
|
||||
EXPECT_TRUE(send_wire_str(fd, missing_path));
|
||||
EXPECT_TRUE(send_int(fd, 0)); /* per-directory filter-rule block is empty */
|
||||
EXPECT_TRUE(send_int(fd, 0)); /* apply = false */
|
||||
EXPECT_TRUE(send_wire_str(fd, "."));
|
||||
EXPECT_TRUE(send_int(fd, 0));
|
||||
@@ -265,10 +281,201 @@ static void test_config_only_frame_applies_missing_args(void) {
|
||||
config_delete(config);
|
||||
}
|
||||
|
||||
/* The per-directory filter-rule block (protocol 2.30.0) must be bounded on
|
||||
* receive: every count, the action/sides domain, the owner-directory syntax and
|
||||
* the pattern length are validated so a hostile peer can neither overread nor
|
||||
* allocate unboundedly. It also round-trips a valid group faithfully. */
|
||||
static void test_filter_dir_rules_receive_bounds(void) {
|
||||
int p[2];
|
||||
FilterRuleList* out = NULL;
|
||||
|
||||
/* Group count beyond the cap is rejected. */
|
||||
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
|
||||
EXPECT_TRUE(send_int(p[1], MAX_FILTER_RULES + 1));
|
||||
EXPECT_FALSE(delete_filter_dir_rules_receive(p[0], &out));
|
||||
EXPECT_NULL(out);
|
||||
close(p[0]);
|
||||
close(p[1]);
|
||||
|
||||
/* A negative group count is rejected. */
|
||||
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
|
||||
EXPECT_TRUE(send_int(p[1], -1));
|
||||
EXPECT_FALSE(delete_filter_dir_rules_receive(p[0], &out));
|
||||
EXPECT_NULL(out);
|
||||
close(p[0]);
|
||||
close(p[1]);
|
||||
|
||||
/* An empty block is valid and yields NULL. */
|
||||
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
|
||||
EXPECT_TRUE(send_int(p[1], 0));
|
||||
EXPECT_TRUE(delete_filter_dir_rules_receive(p[0], &out));
|
||||
EXPECT_NULL(out);
|
||||
close(p[0]);
|
||||
close(p[1]);
|
||||
|
||||
/* An unknown action is a protocol error. */
|
||||
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
|
||||
EXPECT_TRUE(send_int(p[1], 1));
|
||||
EXPECT_TRUE(send_wire_str(p[1], ""));
|
||||
EXPECT_TRUE(send_int(p[1], 1));
|
||||
EXPECT_TRUE(send_int(p[1], 999));
|
||||
EXPECT_FALSE(delete_filter_dir_rules_receive(p[0], &out));
|
||||
EXPECT_NULL(out);
|
||||
close(p[0]);
|
||||
close(p[1]);
|
||||
|
||||
/* An absolute owner directory is rejected (confinement). */
|
||||
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
|
||||
EXPECT_TRUE(send_int(p[1], 1));
|
||||
EXPECT_TRUE(send_wire_str(p[1], "/etc"));
|
||||
EXPECT_TRUE(send_int(p[1], 0));
|
||||
EXPECT_FALSE(delete_filter_dir_rules_receive(p[0], &out));
|
||||
EXPECT_NULL(out);
|
||||
close(p[0]);
|
||||
close(p[1]);
|
||||
|
||||
/* An over-long pattern is rejected before allocation. */
|
||||
{
|
||||
char* big = malloc(MAX_PROTECT_PATTERN_LEN + 2);
|
||||
EXPECT_NOT_NULL(big);
|
||||
memset(big, 'a', MAX_PROTECT_PATTERN_LEN + 1);
|
||||
big[MAX_PROTECT_PATTERN_LEN + 1] = '\0';
|
||||
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
|
||||
EXPECT_TRUE(send_int(p[1], 1));
|
||||
EXPECT_TRUE(send_wire_str(p[1], ""));
|
||||
EXPECT_TRUE(send_int(p[1], 1));
|
||||
EXPECT_TRUE(send_int(p[1], (int)FILTER_ACTION_EXCLUDE));
|
||||
EXPECT_TRUE(send_int(p[1], (int)(FILTER_SIDE_SENDER | FILTER_SIDE_RECEIVER)));
|
||||
EXPECT_TRUE(send_int(p[1], 0));
|
||||
EXPECT_TRUE(send_int(p[1], 0));
|
||||
EXPECT_TRUE(send_int(p[1], 0));
|
||||
EXPECT_TRUE(send_int(p[1], 0));
|
||||
EXPECT_TRUE(send_wire_str(p[1], big));
|
||||
EXPECT_FALSE(delete_filter_dir_rules_receive(p[0], &out));
|
||||
EXPECT_NULL(out);
|
||||
close(p[0]);
|
||||
close(p[1]);
|
||||
free(big);
|
||||
}
|
||||
|
||||
/* A valid group round-trips its owner, no-inherit flag and pattern. */
|
||||
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
|
||||
EXPECT_TRUE(send_int(p[1], 1));
|
||||
EXPECT_TRUE(send_wire_str(p[1], "sub"));
|
||||
EXPECT_TRUE(send_int(p[1], 1));
|
||||
EXPECT_TRUE(send_int(p[1], (int)FILTER_ACTION_EXCLUDE));
|
||||
EXPECT_TRUE(send_int(p[1], (int)(FILTER_SIDE_SENDER | FILTER_SIDE_RECEIVER)));
|
||||
EXPECT_TRUE(send_int(p[1], 0));
|
||||
EXPECT_TRUE(send_int(p[1], 0));
|
||||
EXPECT_TRUE(send_int(p[1], 0));
|
||||
EXPECT_TRUE(send_int(p[1], 1)); /* no_inherit */
|
||||
EXPECT_TRUE(send_wire_str(p[1], "*.log"));
|
||||
EXPECT_TRUE(delete_filter_dir_rules_receive(p[0], &out));
|
||||
EXPECT_NOT_NULL(out);
|
||||
EXPECT_EQ_INT(out->count, 1);
|
||||
EXPECT_EQ_STR(out->items[0]->owner, "sub");
|
||||
EXPECT_EQ_STR(out->items[0]->pattern, "*.log");
|
||||
EXPECT_TRUE(out->items[0]->no_inherit);
|
||||
filter_rule_list_free(out);
|
||||
close(p[0]);
|
||||
close(p[1]);
|
||||
}
|
||||
|
||||
/* The per-directory rule sender enforces exactly the receiver's limits: an
|
||||
* over-long pattern and an over-budget owner+pattern total are rejected locally
|
||||
* with a clear error instead of emitting a frame the peer would abort the
|
||||
* transfer on. A valid block still round-trips. */
|
||||
static void test_filter_dir_rules_send_bounds(void) {
|
||||
int p[2];
|
||||
|
||||
/* An over-long pattern is rejected before anything is written. */
|
||||
{
|
||||
FilterRuleList* list = filter_rule_list_create();
|
||||
EXPECT_NOT_NULL(list);
|
||||
FilterRule* rule = calloc(1, sizeof(FilterRule));
|
||||
EXPECT_NOT_NULL(rule);
|
||||
rule->action = FILTER_ACTION_EXCLUDE;
|
||||
rule->sides = FILTER_SIDE_SENDER | FILTER_SIDE_RECEIVER;
|
||||
rule->owner = str_dup("sub");
|
||||
rule->pattern = malloc(MAX_PROTECT_PATTERN_LEN + 2);
|
||||
EXPECT_NOT_NULL(rule->owner);
|
||||
EXPECT_NOT_NULL(rule->pattern);
|
||||
memset(rule->pattern, 'a', MAX_PROTECT_PATTERN_LEN + 1);
|
||||
rule->pattern[MAX_PROTECT_PATTERN_LEN + 1] = '\0';
|
||||
EXPECT_TRUE(filter_rule_list_add(list, rule));
|
||||
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
|
||||
EXPECT_FALSE(delete_filter_dir_rules_send(p[1], list));
|
||||
close(p[0]);
|
||||
close(p[1]);
|
||||
filter_rule_list_free(list);
|
||||
}
|
||||
|
||||
/* A cumulative owner+pattern total over MAX_FILTER_BYTES is rejected. */
|
||||
{
|
||||
FilterRuleList* list = filter_rule_list_create();
|
||||
EXPECT_NOT_NULL(list);
|
||||
int per = MAX_PROTECT_PATTERN_LEN;
|
||||
int need = MAX_FILTER_BYTES / per + 1;
|
||||
EXPECT_TRUE(need < MAX_FILTER_RULES);
|
||||
for (int i = 0; i < need; i++) {
|
||||
FilterRule* rule = calloc(1, sizeof(FilterRule));
|
||||
EXPECT_NOT_NULL(rule);
|
||||
rule->action = FILTER_ACTION_EXCLUDE;
|
||||
rule->sides = FILTER_SIDE_SENDER | FILTER_SIDE_RECEIVER;
|
||||
rule->owner = str_dup("");
|
||||
rule->pattern = malloc((size_t)per + 1);
|
||||
EXPECT_NOT_NULL(rule->owner);
|
||||
EXPECT_NOT_NULL(rule->pattern);
|
||||
memset(rule->pattern, 'b', (size_t)per);
|
||||
rule->pattern[per] = '\0';
|
||||
EXPECT_TRUE(filter_rule_list_add(list, rule));
|
||||
}
|
||||
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
|
||||
DrainArg drain = {p[0]};
|
||||
thrd_t drainer;
|
||||
EXPECT_EQ_INT(thrd_create(&drainer, drain_fd_thread, &drain), thrd_success);
|
||||
bool sent = delete_filter_dir_rules_send(p[1], list);
|
||||
close(p[1]);
|
||||
thrd_join(drainer, NULL);
|
||||
EXPECT_FALSE(sent);
|
||||
close(p[0]);
|
||||
filter_rule_list_free(list);
|
||||
}
|
||||
|
||||
/* A valid block still round-trips through send -> receive. */
|
||||
{
|
||||
FilterRuleList* list = filter_rule_list_create();
|
||||
EXPECT_NOT_NULL(list);
|
||||
FilterRule* rule = calloc(1, sizeof(FilterRule));
|
||||
EXPECT_NOT_NULL(rule);
|
||||
rule->action = FILTER_ACTION_EXCLUDE;
|
||||
rule->sides = FILTER_SIDE_SENDER | FILTER_SIDE_RECEIVER;
|
||||
rule->owner = str_dup("sub");
|
||||
rule->pattern = str_dup("*.log");
|
||||
EXPECT_NOT_NULL(rule->owner);
|
||||
EXPECT_NOT_NULL(rule->pattern);
|
||||
EXPECT_TRUE(filter_rule_list_add(list, rule));
|
||||
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
|
||||
EXPECT_TRUE(delete_filter_dir_rules_send(p[1], list));
|
||||
FilterRuleList* out = NULL;
|
||||
EXPECT_TRUE(delete_filter_dir_rules_receive(p[0], &out));
|
||||
EXPECT_NOT_NULL(out);
|
||||
EXPECT_EQ_INT(out->count, 1);
|
||||
EXPECT_EQ_STR(out->items[0]->owner, "sub");
|
||||
EXPECT_EQ_STR(out->items[0]->pattern, "*.log");
|
||||
filter_rule_list_free(out);
|
||||
close(p[0]);
|
||||
close(p[1]);
|
||||
filter_rule_list_free(list);
|
||||
}
|
||||
}
|
||||
|
||||
void test_delete_plan(void) {
|
||||
test_delete_delay_refilled_dir_removed_recursively();
|
||||
test_delete_delay_removed_file_counted();
|
||||
test_delete_delay_max_delete_bounds_actual();
|
||||
test_delete_delay_actual_removal_charges_budget();
|
||||
test_config_only_frame_applies_missing_args();
|
||||
test_filter_dir_rules_receive_bounds();
|
||||
test_filter_dir_rules_send_bounds();
|
||||
}
|
||||
|
||||
@@ -5,6 +5,8 @@
|
||||
#include "file.h"
|
||||
#include "file_receive.h"
|
||||
#include "data.h"
|
||||
#include "compression.h"
|
||||
#include "delta.h"
|
||||
#include "config.h"
|
||||
#include "charset.h"
|
||||
#include "utils.h"
|
||||
@@ -2465,7 +2467,172 @@ static void test_manifest_would_delete_protects_absolute_basis() {
|
||||
free(extra);
|
||||
}
|
||||
|
||||
/* #318: a whole-file payload above the streaming bound must be written to a
|
||||
* spool temp file in bounded chunks, not materialized in memory. Exercises the
|
||||
* raw and zstd-compressed paths and asserts the exact bytes land in the spool. */
|
||||
static void test_file_receive_payload_streams(void) {
|
||||
const char* dir = "test_file_stream_tmp";
|
||||
char dest_path[512];
|
||||
snprintf(dest_path, sizeof(dest_path), "%s/out.bin", dir);
|
||||
mkdir(dir, 0777);
|
||||
|
||||
const unsigned long long stream_limit = 4096;
|
||||
size_t size = 20000;
|
||||
unsigned char* payload = malloc(size);
|
||||
EXPECT_NOT_NULL(payload);
|
||||
for (size_t i = 0; i < size; i++)
|
||||
payload[i] = (unsigned char)((i * 7 + 3) & 0xff);
|
||||
|
||||
/* Raw (uncompressed) streamed payload. */
|
||||
{
|
||||
int p[2];
|
||||
EXPECT_EQ_INT(pipe(p), 0);
|
||||
unsigned long long hdr = size;
|
||||
EXPECT_TRUE(send_n_data(p[1], &hdr, sizeof(hdr)));
|
||||
EXPECT_TRUE(send_n_data(p[1], payload, size));
|
||||
Data* buffer = NULL;
|
||||
char* spool = NULL;
|
||||
unsigned long long out_size = 0;
|
||||
EXPECT_TRUE(file_receive_payload(p[0], false, size, dest_path, stream_limit, &buffer, &spool,
|
||||
&out_size));
|
||||
EXPECT_NULL(buffer);
|
||||
EXPECT_NOT_NULL(spool);
|
||||
EXPECT_TRUE(out_size == size);
|
||||
/* cppcheck-suppress knownConditionTrueFalse -- EXPECT_NOT_NULL/EXPECT_TRUE above returns on
|
||||
* failure */
|
||||
if (spool) {
|
||||
FILE* fh = fopen(spool, "rb");
|
||||
EXPECT_NOT_NULL(fh);
|
||||
/* cppcheck-suppress knownConditionTrueFalse -- EXPECT_NOT_NULL/EXPECT_TRUE above returns on
|
||||
* failure */
|
||||
if (fh) {
|
||||
unsigned char* got = malloc(size);
|
||||
EXPECT_TRUE(fread(got, 1, size, fh) == size);
|
||||
EXPECT_EQ_INT(memcmp(got, payload, size), 0);
|
||||
free(got);
|
||||
fclose(fh);
|
||||
}
|
||||
unlink(spool);
|
||||
free(spool);
|
||||
}
|
||||
close(p[0]);
|
||||
close(p[1]);
|
||||
}
|
||||
|
||||
/* zstd-compressed payload whose logical size exceeds the bound: the frame is
|
||||
* decompressed incrementally straight into the spool. */
|
||||
{
|
||||
unsigned char* copy = malloc(size);
|
||||
EXPECT_NOT_NULL(copy);
|
||||
memcpy(copy, payload, size);
|
||||
Data* raw = data_create(copy, size); /* data_create takes ownership of copy */
|
||||
Data* compressed = data_compress_codec(raw, COMPRESSION_ALGO_ZSTD, 3, 0);
|
||||
data_destroy(raw);
|
||||
EXPECT_NOT_NULL(compressed);
|
||||
/* cppcheck-suppress knownConditionTrueFalse -- EXPECT_NOT_NULL/EXPECT_TRUE above returns on
|
||||
* failure */
|
||||
if (compressed) {
|
||||
int p[2];
|
||||
EXPECT_EQ_INT(pipe(p), 0);
|
||||
EXPECT_TRUE(send_data(p[1], compressed));
|
||||
Data* buffer = NULL;
|
||||
char* spool = NULL;
|
||||
unsigned long long out_size = 0;
|
||||
EXPECT_TRUE(file_receive_payload(p[0], true, size, dest_path, stream_limit, &buffer, &spool,
|
||||
&out_size));
|
||||
EXPECT_NULL(buffer);
|
||||
EXPECT_NOT_NULL(spool);
|
||||
EXPECT_TRUE(out_size == size);
|
||||
/* cppcheck-suppress knownConditionTrueFalse -- EXPECT_NOT_NULL/EXPECT_TRUE above returns on
|
||||
* failure */
|
||||
if (spool) {
|
||||
FILE* fh = fopen(spool, "rb");
|
||||
EXPECT_NOT_NULL(fh);
|
||||
/* cppcheck-suppress knownConditionTrueFalse -- EXPECT_NOT_NULL/EXPECT_TRUE above returns on
|
||||
* failure */
|
||||
if (fh) {
|
||||
unsigned char* got = malloc(size);
|
||||
EXPECT_TRUE(fread(got, 1, size, fh) == size);
|
||||
EXPECT_EQ_INT(memcmp(got, payload, size), 0);
|
||||
free(got);
|
||||
fclose(fh);
|
||||
}
|
||||
unlink(spool);
|
||||
free(spool);
|
||||
}
|
||||
close(p[0]);
|
||||
close(p[1]);
|
||||
data_destroy(compressed);
|
||||
}
|
||||
}
|
||||
|
||||
free(payload);
|
||||
unlink(dest_path);
|
||||
rmdir(dir);
|
||||
}
|
||||
|
||||
/* #318: the fd-based delta helpers must match the in-memory ones and stream the
|
||||
* reconstruction to a descriptor without allocating the whole output. */
|
||||
static void test_delta_stream_helpers(void) {
|
||||
const unsigned char basis[] = {0x11, 0x22, 0x33, 0x44};
|
||||
const char* basis_path = "test_file_delta_basis.bin";
|
||||
int bfd = open(basis_path, O_RDWR | O_CREAT | O_TRUNC, 0600);
|
||||
EXPECT_TRUE(bfd >= 0);
|
||||
EXPECT_TRUE(write(bfd, basis, sizeof(basis)) == (ssize_t)sizeof(basis));
|
||||
EXPECT_TRUE(lseek(bfd, 0, SEEK_SET) == 0);
|
||||
|
||||
DeltaSignature* fd_sig = delta_signature_create_fd_seeded(bfd, sizeof(basis), 4, 0);
|
||||
DeltaSignature* mem_sig = delta_signature_create_seeded(basis, sizeof(basis), 4, 0);
|
||||
EXPECT_NOT_NULL(fd_sig);
|
||||
EXPECT_NOT_NULL(mem_sig);
|
||||
/* cppcheck-suppress knownConditionTrueFalse -- EXPECT_NOT_NULL/EXPECT_TRUE above returns on
|
||||
* failure */
|
||||
if (fd_sig && mem_sig) {
|
||||
EXPECT_TRUE(fd_sig->block_count == mem_sig->block_count);
|
||||
for (uint32_t i = 0; i < fd_sig->block_count; i++) {
|
||||
EXPECT_TRUE(fd_sig->blocks[i].adler32 == mem_sig->blocks[i].adler32);
|
||||
EXPECT_TRUE(fd_sig->blocks[i].xxhash == mem_sig->blocks[i].xxhash);
|
||||
}
|
||||
}
|
||||
delta_signature_destroy(fd_sig);
|
||||
delta_signature_destroy(mem_sig);
|
||||
|
||||
DeltaInstruction instrs[2];
|
||||
instrs[0].type = DELTA_INSTR_BLOCK_MATCH;
|
||||
instrs[0].match.block_index = 0;
|
||||
instrs[0].match.block_offset = 0;
|
||||
instrs[0].match.length = 4;
|
||||
instrs[1].type = DELTA_INSTR_LITERAL;
|
||||
instrs[1].literal.data = (uint8_t*)"XY";
|
||||
instrs[1].literal.length = 2;
|
||||
Delta delta;
|
||||
delta.new_file_size = 6;
|
||||
delta.instruction_count = 2;
|
||||
delta.instructions = instrs;
|
||||
delta.delta_size = 0;
|
||||
|
||||
int out[2];
|
||||
EXPECT_EQ_INT(pipe(out), 0);
|
||||
EXPECT_TRUE(delta_apply_to_fd(NULL, bfd, sizeof(basis), &delta, 4, out[1]));
|
||||
close(out[1]);
|
||||
unsigned char got[6] = {0};
|
||||
size_t total = 0;
|
||||
while (total < sizeof(got)) {
|
||||
ssize_t n = read(out[0], got + total, sizeof(got) - total);
|
||||
if (n <= 0)
|
||||
break;
|
||||
total += (size_t)n;
|
||||
}
|
||||
EXPECT_TRUE(total == sizeof(got));
|
||||
EXPECT_TRUE(memcmp(got, "\x11\x22\x33\x44XY", 6) == 0);
|
||||
close(out[0]);
|
||||
close(bfd);
|
||||
unlink(basis_path);
|
||||
}
|
||||
|
||||
void test_file() {
|
||||
test_file_receive_payload_streams();
|
||||
test_delta_stream_helpers();
|
||||
test_file_create();
|
||||
test_file_special_rdev_valid();
|
||||
test_file_destroy_null();
|
||||
|
||||
+226
-29
@@ -1,6 +1,7 @@
|
||||
#include "test_filter.h"
|
||||
#include "filter.h"
|
||||
#include "test_utils.h"
|
||||
#include "utils.h"
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
@@ -64,44 +65,47 @@ static void test_filter_list_rejects_unsupported_modifiers() {
|
||||
}
|
||||
|
||||
/* rsync accepts the merge-file modifiers e/n/w/- on merge and dir-merge rules.
|
||||
* They must be consumed so they never leak into the merge filename. */
|
||||
* They must be consumed so they never leak into the merge filename, and their
|
||||
* semantics (exclude-self, no-inherit, word-split, no-prefixes) must be
|
||||
* applied while the file is read. */
|
||||
static void test_filter_list_accepts_merge_modifiers() {
|
||||
char tmpl[] = "/tmp/fastsync_filter_mmod_XXXXXX";
|
||||
EXPECT_TRUE(mkdtemp(tmpl) != NULL);
|
||||
char path[512];
|
||||
snprintf(path, sizeof(path), "%s/rules", tmpl);
|
||||
FILE* fp = fopen(path, "w");
|
||||
char prefixed[512];
|
||||
char bare[512];
|
||||
char words[512];
|
||||
snprintf(prefixed, sizeof(prefixed), "%s/prefixed", tmpl);
|
||||
snprintf(bare, sizeof(bare), "%s/bare", tmpl);
|
||||
snprintf(words, sizeof(words), "%s/words", tmpl);
|
||||
FILE* fp = fopen(prefixed, "w");
|
||||
EXPECT_NOT_NULL(fp);
|
||||
fputs("- *.tmp\n", fp);
|
||||
fclose(fp);
|
||||
fp = fopen(bare, "w");
|
||||
EXPECT_NOT_NULL(fp);
|
||||
fputs("*.log\n*.tmp\n", fp);
|
||||
fclose(fp);
|
||||
fp = fopen(words, "w");
|
||||
EXPECT_NOT_NULL(fp);
|
||||
fputs("*.log *.tmp\n", fp);
|
||||
fclose(fp);
|
||||
|
||||
/* merge with e/n/w/- consumes the modifiers and reads the right file. */
|
||||
static const char* const fmts[] = {
|
||||
"merge,e %s", "merge,n %s", "merge,w %s", "merge,- %s", ".e %s", ".- %s",
|
||||
};
|
||||
for (size_t i = 0; i < sizeof(fmts) / sizeof(fmts[0]); i++) {
|
||||
FilterRuleList* list = filter_rule_list_create();
|
||||
EXPECT_NOT_NULL(list);
|
||||
char rule[600];
|
||||
char err[256] = "";
|
||||
snprintf(rule, sizeof(rule), fmts[i], path);
|
||||
bool ok = filter_rule_list_parse_append(list, rule, NULL, NULL, err, sizeof(err));
|
||||
if (!ok)
|
||||
printf(" merge rule '%s' errored: %s\n", rule, err);
|
||||
EXPECT_TRUE(ok);
|
||||
EXPECT_EQ_INT(list->count, 1);
|
||||
EXPECT_EQ_STR(list->items[0]->pattern, "*.tmp");
|
||||
filter_rule_list_free(list);
|
||||
}
|
||||
|
||||
/* dir-merge with e/n/w/- registers the basename without the modifiers. */
|
||||
/* dir-merge with e/n/w/- registers the basename without the modifiers and
|
||||
* records the modifier flags; 'e' appends an exclude-self rule. */
|
||||
static const struct {
|
||||
const char* rule;
|
||||
const char* want;
|
||||
bool no_prefixes;
|
||||
bool word_split;
|
||||
bool no_inherit;
|
||||
bool exclude_self;
|
||||
} drules[] = {
|
||||
{"dir-merge,e .rules", ".rules"}, {"dir-merge,n .rules", ".rules"},
|
||||
{"dir-merge,w .rules", ".rules"}, {"dir-merge,- .rules", ".rules"},
|
||||
{":e .rules", ".rules"}, {":- .rules", ".rules"},
|
||||
{"dir-merge,e .rules", ".rules", false, false, false, true},
|
||||
{"dir-merge,n .rules", ".rules", false, false, true, false},
|
||||
{"dir-merge,w .rules", ".rules", false, true, false, false},
|
||||
{"dir-merge,- .rules", ".rules", true, false, false, false},
|
||||
{":e .rules", ".rules", false, false, false, true},
|
||||
{":- .rules", ".rules", true, false, false, false},
|
||||
};
|
||||
for (size_t i = 0; i < sizeof(drules) / sizeof(drules[0]); i++) {
|
||||
FilterRuleList* list = filter_rule_list_create();
|
||||
@@ -112,11 +116,78 @@ static void test_filter_list_accepts_merge_modifiers() {
|
||||
printf(" dir-merge rule '%s' errored: %s\n", drules[i].rule, err);
|
||||
EXPECT_TRUE(ok);
|
||||
EXPECT_EQ_INT(list->dir_merge_count, 1);
|
||||
EXPECT_EQ_STR(list->dir_merge_names[0], drules[i].want);
|
||||
EXPECT_EQ_STR(list->dir_merges[0].name, drules[i].want);
|
||||
EXPECT_EQ_INT(list->dir_merges[0].no_prefixes, drules[i].no_prefixes);
|
||||
EXPECT_EQ_INT(list->dir_merges[0].word_split, drules[i].word_split);
|
||||
EXPECT_EQ_INT(list->dir_merges[0].no_inherit, drules[i].no_inherit);
|
||||
EXPECT_EQ_INT(list->dir_merges[0].exclude_self, drules[i].exclude_self);
|
||||
if (drules[i].exclude_self) {
|
||||
EXPECT_EQ_INT(list->count, 1);
|
||||
EXPECT_EQ_STR(list->items[0]->pattern, ".rules");
|
||||
EXPECT_EQ_INT(list->items[0]->action, FILTER_ACTION_EXCLUDE);
|
||||
} else {
|
||||
EXPECT_EQ_INT(list->count, 0);
|
||||
}
|
||||
filter_rule_list_free(list);
|
||||
}
|
||||
|
||||
unlink(path);
|
||||
/* merge,n reads the file normally; no-inherit is meaningless for a single
|
||||
* merge so the rule is not marked. */
|
||||
{
|
||||
FilterRuleList* list = filter_rule_list_create();
|
||||
char err[256] = "";
|
||||
char rule[600];
|
||||
snprintf(rule, sizeof(rule), "merge,n %s", prefixed);
|
||||
EXPECT_TRUE(filter_rule_list_parse_append(list, rule, NULL, NULL, err, sizeof(err)));
|
||||
EXPECT_EQ_INT(list->count, 1);
|
||||
EXPECT_EQ_STR(list->items[0]->pattern, "*.tmp");
|
||||
EXPECT_FALSE(list->items[0]->no_inherit);
|
||||
filter_rule_list_free(list);
|
||||
}
|
||||
|
||||
/* merge,e adds an implicit exclude for the merge file's basename. */
|
||||
{
|
||||
FilterRuleList* list = filter_rule_list_create();
|
||||
char err[256] = "";
|
||||
char rule[600];
|
||||
snprintf(rule, sizeof(rule), "merge,e %s", prefixed);
|
||||
EXPECT_TRUE(filter_rule_list_parse_append(list, rule, NULL, NULL, err, sizeof(err)));
|
||||
EXPECT_EQ_INT(list->count, 2);
|
||||
EXPECT_EQ_STR(list->items[0]->pattern, "prefixed");
|
||||
EXPECT_EQ_INT(list->items[0]->action, FILTER_ACTION_EXCLUDE);
|
||||
EXPECT_EQ_STR(list->items[1]->pattern, "*.tmp");
|
||||
filter_rule_list_free(list);
|
||||
}
|
||||
|
||||
/* merge,- reads the file as bare exclude patterns with no prefix parsing. */
|
||||
{
|
||||
FilterRuleList* list = filter_rule_list_create();
|
||||
char err[256] = "";
|
||||
char rule[600];
|
||||
snprintf(rule, sizeof(rule), "merge,- %s", bare);
|
||||
EXPECT_TRUE(filter_rule_list_parse_append(list, rule, NULL, NULL, err, sizeof(err)));
|
||||
EXPECT_EQ_INT(list->count, 2);
|
||||
EXPECT_EQ_STR(list->items[0]->pattern, "*.log");
|
||||
EXPECT_EQ_STR(list->items[1]->pattern, "*.tmp");
|
||||
filter_rule_list_free(list);
|
||||
}
|
||||
|
||||
/* merge,-w word-splits bare patterns on whitespace. */
|
||||
{
|
||||
FilterRuleList* list = filter_rule_list_create();
|
||||
char err[256] = "";
|
||||
char rule[600];
|
||||
snprintf(rule, sizeof(rule), "merge,w- %s", words);
|
||||
EXPECT_TRUE(filter_rule_list_parse_append(list, rule, NULL, NULL, err, sizeof(err)));
|
||||
EXPECT_EQ_INT(list->count, 2);
|
||||
EXPECT_EQ_STR(list->items[0]->pattern, "*.log");
|
||||
EXPECT_EQ_STR(list->items[1]->pattern, "*.tmp");
|
||||
filter_rule_list_free(list);
|
||||
}
|
||||
|
||||
unlink(prefixed);
|
||||
unlink(bare);
|
||||
unlink(words);
|
||||
rmdir(tmpl);
|
||||
}
|
||||
|
||||
@@ -187,6 +258,59 @@ static void test_filter_list_accepts_supported_rules_and_modifiers() {
|
||||
}
|
||||
}
|
||||
|
||||
/* A "clear"/"!" inside a merge file resets the list to empty. Rules read after
|
||||
* it must still be owned by the merge file's directory (and marked no-inherit
|
||||
* when the dir-merge says so). The base index must follow the clear down: when
|
||||
* it was captured before the clear, post-clear rules sat below it and were left
|
||||
* globally owned by "" (and unmarked). */
|
||||
static void test_filter_merge_clear_then_owner() {
|
||||
char tmpl[] = "/tmp/fastsync_filter_clear_XXXXXX";
|
||||
EXPECT_TRUE(mkdtemp(tmpl) != NULL);
|
||||
char path[512];
|
||||
snprintf(path, sizeof(path), "%s/.rsync-filter", tmpl);
|
||||
FILE* fp = fopen(path, "w");
|
||||
EXPECT_NOT_NULL(fp);
|
||||
fputs("- *.tmp\n!\nP *.log\n", fp);
|
||||
fclose(fp);
|
||||
|
||||
FilterRuleList* list = filter_rule_list_create();
|
||||
EXPECT_NOT_NULL(list);
|
||||
char err[256] = "";
|
||||
/* A pre-existing rule that the in-file clear must discard. */
|
||||
EXPECT_TRUE(filter_rule_list_parse_append(list, "- keep.txt", NULL, NULL, err, sizeof(err)));
|
||||
EXPECT_EQ_INT(list->count, 1);
|
||||
|
||||
FilterDirMerge spec = {.name = ".rsync-filter", .no_inherit = true};
|
||||
bool exists = false;
|
||||
EXPECT_TRUE(filter_dir_merge_append(list, tmpl, &spec, "sub", NULL, &exists, err, sizeof(err)));
|
||||
EXPECT_TRUE(exists);
|
||||
/* Only the post-clear rule survives, owned by "sub" and no-inherit. */
|
||||
EXPECT_EQ_INT(list->count, 1);
|
||||
EXPECT_EQ_STR(list->items[0]->pattern, "*.log");
|
||||
EXPECT_EQ_STR(list->items[0]->owner, "sub");
|
||||
EXPECT_TRUE(list->items[0]->no_inherit);
|
||||
filter_rule_list_free(list);
|
||||
|
||||
/* A parse failure after the clear must roll the list back to the post-clear
|
||||
* base (empty here), freeing the post-clear rule rather than retaining it. */
|
||||
fp = fopen(path, "w");
|
||||
EXPECT_NOT_NULL(fp);
|
||||
fputs("- *.tmp\n!\nP *.log\n-e bogus\n", fp);
|
||||
fclose(fp);
|
||||
list = filter_rule_list_create();
|
||||
EXPECT_NOT_NULL(list);
|
||||
EXPECT_TRUE(filter_rule_list_parse_append(list, "- keep.txt", NULL, NULL, err, sizeof(err)));
|
||||
EXPECT_EQ_INT(list->count, 1);
|
||||
exists = false;
|
||||
EXPECT_FALSE(filter_dir_merge_append(list, tmpl, &spec, "sub", NULL, &exists, err, sizeof(err)));
|
||||
EXPECT_TRUE(exists);
|
||||
EXPECT_EQ_INT(list->count, 0);
|
||||
filter_rule_list_free(list);
|
||||
|
||||
unlink(path);
|
||||
rmdir(tmpl);
|
||||
}
|
||||
|
||||
static void test_filter_list_merge_file_still_supported() {
|
||||
char tmpl[] = "/tmp/fastsync_filter_XXXXXX";
|
||||
EXPECT_TRUE(mkdtemp(tmpl) != NULL);
|
||||
@@ -283,12 +407,85 @@ static void test_filter_rules_apply_supported_modifiers() {
|
||||
}
|
||||
}
|
||||
|
||||
static FilterRule* chain_rule(const char* owner, const char* pattern, FilterAction action,
|
||||
bool no_inherit) {
|
||||
FilterRule* rule = calloc(1, sizeof(FilterRule));
|
||||
if (!rule)
|
||||
return NULL;
|
||||
rule->action = action;
|
||||
rule->sides = FILTER_SIDE_SENDER | FILTER_SIDE_RECEIVER;
|
||||
rule->owner = str_dup(owner);
|
||||
rule->pattern = str_dup(pattern);
|
||||
rule->no_inherit = no_inherit;
|
||||
if (!rule->owner || !rule->pattern) {
|
||||
filter_rule_free(rule);
|
||||
return NULL;
|
||||
}
|
||||
return rule;
|
||||
}
|
||||
|
||||
/* The receiver's per-directory chain: a containing directory's rules win over
|
||||
* an ancestor's (deepest-first), root rules are inherited, and a no-inherit
|
||||
* rule applies only to its own directory's direct children. */
|
||||
static void test_filter_dir_rules_chain(void) {
|
||||
FilterRuleList* list = filter_rule_list_create();
|
||||
EXPECT_NOT_NULL(list);
|
||||
FilterRule* root_log = chain_rule("", "*.log", FILTER_ACTION_EXCLUDE, false);
|
||||
FilterRule* sub_keep = chain_rule("sub", "keep.log", FILTER_ACTION_INCLUDE, false);
|
||||
FilterRule* sub_tmp = chain_rule("sub", "*.tmp", FILTER_ACTION_EXCLUDE, true);
|
||||
EXPECT_NOT_NULL(root_log);
|
||||
EXPECT_NOT_NULL(sub_keep);
|
||||
EXPECT_NOT_NULL(sub_tmp);
|
||||
EXPECT_TRUE(filter_rule_list_add(list, root_log));
|
||||
EXPECT_TRUE(filter_rule_list_add(list, sub_keep));
|
||||
EXPECT_TRUE(filter_rule_list_add(list, sub_tmp));
|
||||
|
||||
/* Root rule inherited by every directory (leaf match). */
|
||||
EXPECT_EQ_INT(filter_dir_rules_apply_side(list, "a.log", "a.log", false), FILTER_ACTION_PROTECT);
|
||||
EXPECT_EQ_INT(filter_dir_rules_apply_side(list, "sub/a.log", "a.log", false),
|
||||
FILTER_ACTION_PROTECT);
|
||||
/* The deeper include overrides the inherited root exclude. */
|
||||
EXPECT_EQ_INT(filter_dir_rules_apply_side(list, "sub/keep.log", "keep.log", false),
|
||||
FILTER_ACTION_RISK);
|
||||
/* No-inherit applies directly in its owner... */
|
||||
EXPECT_EQ_INT(filter_dir_rules_apply_side(list, "sub/x.tmp", "x.tmp", false),
|
||||
FILTER_ACTION_PROTECT);
|
||||
/* ...but not below it. */
|
||||
EXPECT_EQ_INT(filter_dir_rules_apply_side(list, "sub/deep/x.tmp", "x.tmp", false),
|
||||
FILTER_ACTION_NONE);
|
||||
/* No matching rule. */
|
||||
EXPECT_EQ_INT(filter_dir_rules_apply_side(list, "sub/deep/plain.txt", "plain.txt", false),
|
||||
FILTER_ACTION_NONE);
|
||||
filter_rule_list_free(list);
|
||||
}
|
||||
|
||||
static void test_filter_rule_clone_copies_every_field(void) {
|
||||
char err[128] = "";
|
||||
FilterRule* original = filter_rule_parse("-!p /a/*.o", NULL, err, sizeof(err));
|
||||
EXPECT_NOT_NULL(original);
|
||||
FilterRule* copy = filter_rule_clone(original);
|
||||
EXPECT_NOT_NULL(copy);
|
||||
EXPECT_TRUE(copy != original);
|
||||
EXPECT_EQ_INT(copy->action, original->action);
|
||||
EXPECT_EQ_INT(copy->sides, original->sides);
|
||||
EXPECT_EQ_INT(copy->anchored, original->anchored);
|
||||
EXPECT_EQ_INT(copy->dir_only, original->dir_only);
|
||||
EXPECT_EQ_INT(copy->negate, original->negate);
|
||||
EXPECT_EQ_INT(copy->perishable, original->perishable);
|
||||
EXPECT_EQ_STR(copy->pattern, original->pattern);
|
||||
filter_rule_free(original);
|
||||
filter_rule_free(copy);
|
||||
}
|
||||
|
||||
void test_filter() {
|
||||
test_filter_list_rejects_xattr_modifier();
|
||||
test_filter_list_rejects_unsupported_modifiers();
|
||||
test_filter_list_accepts_merge_modifiers();
|
||||
test_filter_list_accepts_supported_rules_and_modifiers();
|
||||
test_filter_list_merge_file_still_supported();
|
||||
test_filter_merge_clear_then_owner();
|
||||
test_filter_rule_parse_rejects_unsupported_and_keeps_supported();
|
||||
test_filter_rules_apply_supported_modifiers();
|
||||
test_filter_dir_rules_chain();
|
||||
test_filter_rule_clone_copies_every_field();
|
||||
}
|
||||
|
||||
@@ -64,6 +64,7 @@ static void test_dest_state_roundtrip() {
|
||||
memset(&out, 0, sizeof(out));
|
||||
out.known = true;
|
||||
out.existed = true;
|
||||
out.target_matches = true;
|
||||
out.size = 123456789ULL;
|
||||
out.mtime_sec = 1700000000;
|
||||
out.mtime_nsec = 123456789;
|
||||
@@ -76,6 +77,7 @@ static void test_dest_state_roundtrip() {
|
||||
EXPECT_TRUE(format_dest_state_receive(fds[1], &in));
|
||||
EXPECT_TRUE(in.known);
|
||||
EXPECT_TRUE(in.existed);
|
||||
EXPECT_TRUE(in.target_matches);
|
||||
EXPECT_TRUE(in.size == out.size);
|
||||
EXPECT_TRUE(in.mtime_sec == out.mtime_sec);
|
||||
EXPECT_TRUE(in.mtime_nsec == out.mtime_nsec);
|
||||
@@ -103,6 +105,10 @@ static void test_stats_roundtrip() {
|
||||
out.created_dir = 4;
|
||||
out.created_link = 2;
|
||||
out.created_special = 1;
|
||||
out.deleted_reg = 9;
|
||||
out.deleted_dir = 6;
|
||||
out.deleted_link = 3;
|
||||
out.deleted_special = 2;
|
||||
ReceiverStats in;
|
||||
memset(&in, 0, sizeof(in));
|
||||
EXPECT_TRUE(format_stats_send(fds[0], &out));
|
||||
@@ -115,6 +121,10 @@ static void test_stats_roundtrip() {
|
||||
EXPECT_TRUE(in.created_dir == out.created_dir);
|
||||
EXPECT_TRUE(in.created_link == out.created_link);
|
||||
EXPECT_TRUE(in.created_special == out.created_special);
|
||||
EXPECT_TRUE(in.deleted_reg == out.deleted_reg);
|
||||
EXPECT_TRUE(in.deleted_dir == out.deleted_dir);
|
||||
EXPECT_TRUE(in.deleted_link == out.deleted_link);
|
||||
EXPECT_TRUE(in.deleted_special == out.deleted_special);
|
||||
close(fds[0]);
|
||||
close(fds[1]);
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user