diff --git a/RSYNC_COMPAT.md b/RSYNC_COMPAT.md index 8baf6b6..5e5c322 100644 --- a/RSYNC_COMPAT.md +++ b/RSYNC_COMPAT.md @@ -54,7 +54,7 @@ This document maps rsync's full feature set to FastSync's current implementation | `--exclude-from=FILE` | Read exclude patterns from file | ✅ Implemented | Reads patterns from file | | `--include-from=FILE` | Read include patterns from file | ✅ Implemented | Reads patterns from file | | `--filter=RULE` | Add file-filtering rule | ✅ Implemented | Long option only: rsync's short `-f` conflicts with FastSync sendfile (see FastSync-specific list), so `-f` is not reassigned. Supported subset: `+`/`-` include/exclude, implicit-exclude patterns, `include`/`exclude` word forms, a leading `/` anchor (to the transfer root, or to a `.rsync-filter` file's directory), and a trailing `/` for dir-only rules; first match wins with a default of include inside the filter layer. Filters are an independent layer from `--exclude`/`--include` (an entry must pass both). Rejected with a clear error (no silent no-ops): `merge`/`dir-merge`/`hide`/`show`/`protect`/`risk`/`clear` words, rules that begin with `:`/`.`/`!` (merge/dir-merge/list-clear shorthands), and include/exclude modifiers other than `/` (`! C s r p x`) | -| `--files-from=FILE` | Read source file list from file | ✅ Implemented | Entries are paths relative to the source root (leading `./` stripped, `..`/absolute entries rejected at parse time, blank lines ignored; NUL-delimited with `-0`). A listed regular file is transferred; a listed directory transfers its whole subtree (FastSync recursion is always on, unlike rsync's non-recursive default). Non-listed paths and their subtrees are pruned by the scanner. A listed entry that does not exist under the source (and an empty list) is a hard error reported before any transfer; listing `.` (whole tree) and empty listed directories are fine. Scalability note: `file_list_affects` is O(list size) per scanned entry, so a very large `--files-from` list against a huge tree is quadratic; lists are typically small enough that this is acceptable, but it is the documented bound. The delete manifest still derives from what was actually sent, so `--delete` stays consistent with the subset | +| `--files-from=FILE` | Read source file list from file | ✅ Implemented | Entries are paths relative to the source root (leading `./` stripped, `..`/absolute entries rejected at parse time, blank lines ignored; NUL-delimited with `-0`). A listed regular file is transferred; a listed directory transfers its whole subtree (FastSync recursion is always on, unlike rsync's non-recursive default). Non-listed paths and their subtrees are pruned by the scanner. A listed entry that does not exist under the source (and an empty list) is a hard error reported before any transfer, unless `--ignore-missing-args` / `--delete-missing-args` is given (see the Safety & Security rows): those flags downgrade the listed-but-missing case to a skip and, for `--delete-missing-args`, a destination deletion; an empty list stays a hard error in every mode. Listing `.` (whole tree) and empty listed directories are fine. Scalability note: `file_list_affects` is O(list size) per scanned entry, so a very large `--files-from` list against a huge tree is quadratic; lists are typically small enough that this is acceptable, but it is the documented bound. The delete manifest still derives from what was actually sent, so `--delete` stays consistent with the subset | | `-0`, `--from0` | Delimit *-from files with NULs | ✅ Implemented | `--files-from` entries become NUL-delimited; the flag may appear before or after `--files-from` on the command line. NUL mode preserves entry bytes exactly (trailing CR/LF are part of the name; only newline mode trims them) | | `--max-size=SIZE` | Skip files larger than SIZE | ✅ Implemented | `max_size` in scanner | | `--min-size=SIZE` | Skip files smaller than SIZE | ✅ Implemented | `min_size` in scanner | @@ -144,6 +144,24 @@ limit"). These wire/layout changes bumped `PROTOCOL_VERSION` **2.8.0 → 2.9.0** `delete_excluded`, `prune_empty_dirs`, `max_delete` — round-trip unchanged and are validated on receive. +**Missing-args note (Phase 3, missing-args wave):** `--ignore-missing-args` and +`--delete-missing-args` are implemented as described in the Safety & Security +rows. Wire impact: the `STATUS_MANIFEST` frame now carries a **third section** — +a list of destination-relative **exact-delete paths** (the missing entries' +mirrors) — and the config frame gained a `delete_missing_args` boolean +(`ignore_missing_args` stays client-only, exactly like `ignore_errors`). These +wire/layout changes bumped `PROTOCOL_VERSION` **2.9.0 → 2.10.0** (peers must +match). The receiver validates the third section identically to the keep-set +(non-empty, relative, traversal-free; `MAX_MANIFEST_ENTRIES` per section, a +single `MAX_MANIFEST_BYTES` budget shared across all three). On commit the +receiver runs the exact-path deletions FIRST (`manifest_delete_missing_args`: +confined per-path unlink/rmdir, deep removal only under `--force`/`--delete`, +staging/basis protected, never blocked by the protected-prefix list) and then +the ordinary extras walk when `--delete` is active (`manifest_delete_all`). A +client may request the exact-path deletions without `--delete`; the server's +`--allow-delete` policy gates them exactly like `--delete`, so an unauthorized +server ignores the request while the missing entries are still skipped. + The deletion walker is now **all-or-nothing**: before any unlink it rehearses the deletion (an fd-relative walk identical to the delete pass, counting every regular file it would unlink and every directory it would remove) and refuses to @@ -299,8 +317,8 @@ do NOT imply `--delete`; without `--delete` they are inert (matching rsync). | `--max-alloc=SIZE` | Limit a single memory allocation | ✅ Implemented | Caps the largest single allocation; binary units, default 1G | | `--trust-sender` | Trust remote sender's file list | ❌ Not Implemented | | | `--old-args` | Disable modern arg protection | ✅ Implemented | SSH-only legacy mode; restores raw remote command construction and permits shell interpretation of the configured server path | -| `--ignore-missing-args` | Ignore missing source args | ❌ Not Implemented | | -| `--delete-missing-args` | Delete missing source args | ❌ Not Implemented | | +| `--ignore-missing-args` | Ignore missing source args | ✅ Implemented | FastSync has a single source-root argument (which always exists), so the "explicitly requested source arguments" are the `--files-from` entries and the flags only ever apply there (inert without `--files-from`, like `-R`). Without the flag a listed-but-missing entry stays a hard pre-transfer error (nothing is transferred). With it each missing entry is skipped: nothing is sent for it, it never enters the keep-set, and the run succeeds for the rest — an all-missing non-empty list succeeds transferring nothing, matching rsync. `--dirs` + `--files-from` missing entries are skipped the same way. Every skipped entry is logged and a per-run warning names the count, so the handling is never a silent no-op. Divergences: an EMPTY `--files-from` file stays a hard error in every mode (no argument was requested at all; rsync likewise reports "no source files specified"); missing-arg skipping only applies to the pre-transfer list validation, so an entry that is present at preflight and vanishes mid-transfer still fails (matching rsync, whose flag "does not affect subsequent vanished-file errors"); `--no-ignore-missing-args` is not a supported negation | +| `--delete-missing-args` | Delete missing source args | ✅ Implemented | Implies `--ignore-missing-args` (order-independent) and additionally removes each missing entry's destination mirror receiver-side. The mirror is computed exactly like a present sibling's wire path: the bare relative entry under `-R`, otherwise the full source-mirror path below the destination root. rsync parity, verified against the man page: it does **not** imply `--delete` generally and is "independent of any other type of delete processing" — unrelated destination extras are untouched unless `--delete` is also present. Composition with `--delete` + timing: the exact-path deletions commit with the manifest, early for `--delete-before`/`--delete-during`, else only after a fully-successful transfer (delete-after/commit). A non-empty directory mirror is removed only when `--force` or `--delete` is in effect (otherwise it is left with a warning and the run continues, like rsync); an absent mirror is a no-op. An explicitly listed missing arg is a user request, not an excluded file: its deletion is never blocked by the filter-exclusion protection of excluded destination mirrors (a mirror sitting inside a filter-excluded directory is still removed). Safety/policy: gated by the server `--allow-delete` policy like `--delete`; the request paths cross the wire only in the delete-manifest frame and are confined by the same receiver validation as the keep-set (non-empty, relative, traversal-free, bounded by the per-section/per-frame manifest caps); the `--delay-updates` staging directory and basis snapshots are protected exactly as in the extras walker. Divergence: the missing-args deletions are not counted toward `--max-delete` (they are explicit per-path requests, not discovered extras). See the Phase-3 wire note below for the `PROTOCOL_VERSION` bump | ## 16. Batch Operations diff --git a/src/client/client_cli.c b/src/client/client_cli.c index 995a5cc..4d208da 100644 --- a/src/client/client_cli.c +++ b/src/client/client_cli.c @@ -460,6 +460,8 @@ static const OptionEntry OPTION_TABLE[] = { {"--ignore-errors", NULL, OPT_FLAG, offsetof(Config, ignore_errors)}, {"--force", NULL, OPT_FLAG, offsetof(Config, force_delete)}, {"--prune-empty-dirs", NULL, OPT_FLAG, offsetof(Config, prune_empty_dirs)}, + {"--ignore-missing-args", NULL, OPT_FLAG, offsetof(Config, ignore_missing_args)}, + {"--delete-missing-args", NULL, OPT_FLAG, offsetof(Config, delete_missing_args)}, {"--source-dir", NULL, OPT_STRING, offsetof(Config, send_directory)}, {"--dest-dir", NULL, OPT_STRING, offsetof(Config, receive_root_directory)}, @@ -739,6 +741,12 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args, entry->offset == offsetof(Config, delete_delay) || entry->offset == offsetof(Config, delete_after)) config->use_delete = true; + /* --delete-missing-args implies --ignore-missing-args (missing entries + are skipped for deletion instead of failing the run). The implication + is order-independent because it is applied over the final parsed + config. */ + if (entry->offset == offsetof(Config, delete_missing_args)) + config->ignore_missing_args = true; continue; } diff --git a/src/client/client_send.c b/src/client/client_send.c index 03cb828..aff2b13 100644 --- a/src/client/client_send.c +++ b/src/client/client_send.c @@ -104,6 +104,7 @@ static bool prepare_scanner(const Config* config, int num_threads, PreparedScann options->relative = config->relative; options->prune_empty_dirs = config->prune_empty_dirs; options->ignore_io_errors = config->ignore_errors; + options->ignore_missing_args = config->ignore_missing_args || config->delete_missing_args; options->excluded_paths = NULL; options->excluded_mutex = NULL; return true; @@ -182,13 +183,40 @@ static bool no_implied_dirs_files_from_valid(const Config* config) { return true; } +/* The destination-relative mirror path for a missing --files-from entry: where + a PRESENT entry with the same name would have been written. With -R that is + the entry's bare relative path (the bare wire path the receiver uses); + otherwise it is the full source mirror below the destination root + (`send_directory` joined to the entry, leading '/' stripped), exactly the + path the manifest records for a present sibling. Returns an owned string, or + NULL on allocation failure. */ +static char* files_from_missing_dest_path(const Config* config, const char* entry) { + if (config->relative) + return str_dup(entry); + char* joined = path_cat(config->send_directory, entry); + if (!joined) + return NULL; + const char* rel = *joined == '/' ? joined + 1 : joined; + char* dup = str_dup(rel); + free(joined); + return dup; +} + /* --files-from semantics: every listed entry must resolve under the source * root, otherwise rsync reports a hard error instead of silently transferring * nothing. An empty list is also an error. An entry of "." (the whole tree) - * and listed-but-empty directories are valid. Runs before any transfer so the - * failure is surfaced uniformly in the single-threaded, -m, dry-run and - * --list-only paths. */ -static bool files_from_list_valid(const Config* config) { + * and listed-but-empty directories are valid. With --ignore-missing-args + * (implied by --delete-missing-args) a listed-but-missing entry is instead + * skipped: nothing is transferred for it, it never enters the keep-set and the + * run succeeds for the rest (an all-missing non-empty list succeeds + * transferring nothing, matching rsync). With --delete-missing-args + * `missing_dest` (when non-NULL) collects the entry's destination-relative + * mirror for the receiver's exact-deletion request. An empty list stays a + * hard error in every mode (nothing was requested at all). Runs before any + * transfer so the failure/skip is surfaced uniformly in the single-threaded, + * -m, dry-run and --list-only paths. */ +static bool files_from_list_check(const Config* config, ArrayList* missing_dest, int* skipped_out) { + *skipped_out = 0; const FileListSet* set = (const FileListSet*)config->files_from_set; if (!set) return true; @@ -201,6 +229,7 @@ static bool files_from_list_valid(const Config* config) { config->files_from ? config->files_from : ""); return false; } + bool ignore = config->ignore_missing_args || config->delete_missing_args; for (int i = 0; i < set->count; i++) { const char* entry = set->entries[i]; if (entry[0] == '\0') @@ -212,13 +241,51 @@ static bool files_from_list_valid(const Config* config) { } struct stat st; if (lstat(full, &st) != 0) { + free(full); + if (ignore) { + (*skipped_out)++; + log_info_message(LOG_INFO_MISC, "skipping missing --files-from entry '%s'", entry); + if (config->delete_missing_args && missing_dest) { + char* mirror = files_from_missing_dest_path(config, entry); + if (!mirror || !array_list_add(missing_dest, mirror)) { + free(mirror); + log_message(LOG_LEVEL_ERROR, "memory allocation failed while validating --files-from"); + return false; + } + } + continue; + } log_message(LOG_LEVEL_ERROR, "--files-from entry '%s' not found in source '%s'", entry, config->send_directory); - free(full); return false; } free(full); } + if (*skipped_out > 0) { + if (config->delete_missing_args) { + /* --list-only never deletes and a --dry-run only shows intent, so the + summary must not claim a real deletion happened in those modes. */ + if (config->list_only) + log_message(LOG_LEVEL_WARNING, + "--delete-missing-args: %d missing --files-from entr%s skipped (--list-only " + "never deletes)", + *skipped_out, *skipped_out == 1 ? "y" : "ies"); + else if (config->dry_run) + log_message(LOG_LEVEL_WARNING, + "--delete-missing-args: %d missing --files-from entr%s would be deleted from " + "the destination (dry run)", + *skipped_out, *skipped_out == 1 ? "y" : "ies"); + else + log_message( + LOG_LEVEL_WARNING, + "--delete-missing-args: %d missing --files-from entr%s will be deleted from the " + "destination", + *skipped_out, *skipped_out == 1 ? "y" : "ies"); + } else if (config->ignore_missing_args) + log_message(LOG_LEVEL_WARNING, + "--ignore-missing-args: ignored %d missing --files-from entr%s", *skipped_out, + *skipped_out == 1 ? "y" : "ies"); + } return no_implied_dirs_files_from_valid(config); } @@ -458,15 +525,30 @@ static void pipeline_cancel(PipelineContextSender* context) { /* Print dry-run manifest showing files that would be transferred. Returns 0 on success. */ static int send_dry_run_manifest(const Config* config) { - if (!files_from_list_valid(config)) + int skipped = 0; + ArrayList* missing_dest = NULL; + if (config->delete_missing_args) { + missing_dest = array_list_create(free); + if (!missing_dest) + return -1; + } + if (!files_from_list_check(config, missing_dest, &skipped)) { + if (missing_dest) + array_list_delete(missing_dest); return -1; + } PreparedScanner prepared; - if (!prepare_scanner(config, 0, &prepared)) + if (!prepare_scanner(config, 0, &prepared)) { + if (missing_dest) + array_list_delete(missing_dest); return -1; + } DirectoryScanner* scanner = directory_scanner_create_with_options(config->send_directory, &prepared.options); if (!scanner) { prepared_scanner_destroy(&prepared); + if (missing_dest) + array_list_delete(missing_dest); return -1; } Chunk* chunk; @@ -484,6 +566,8 @@ static int send_dry_run_manifest(const Config* config) { chunk_destroy(chunk); directory_scanner_destroy(scanner); prepared_scanner_destroy(&prepared); + if (missing_dest) + array_list_delete(missing_dest); return -1; } if (config->human_readable) @@ -501,6 +585,17 @@ static int send_dry_run_manifest(const Config* config) { } directory_scanner_destroy(scanner); prepared_scanner_destroy(&prepared); + /* --delete-missing-args: the missing entries' destination mirrors render as + would-be deletions (rsync's dry-run also lists its *deleting lines). */ + if (missing_dest && !config->quiet) { + for (int i = 0; i < missing_dest->size; i++) { + char* escaped = output_escape((char*)missing_dest->items[i], config->eight_bit_output); + printf(" %s (missing; would be deleted)\n", escaped ? escaped : ""); + free(escaped); + } + } + if (missing_dest) + array_list_delete(missing_dest); if (!config->quiet) { if (config->human_readable) printf("Total: %d files, %s\n", file_count, @@ -537,7 +632,8 @@ static int compare_list_entries(const void* left, const void* right) { * Directory lines are not printed because the scanner only yields regular * transfer candidates. Returns 0 on success, 1 on error. */ static int send_list_only(const Config* config) { - if (!files_from_list_valid(config)) + int skipped = 0; + if (!files_from_list_check(config, NULL, &skipped)) return 1; PreparedScanner prepared; if (!prepare_scanner(config, 0, &prepared)) @@ -626,22 +722,26 @@ static int send_list_only(const Config* config) { } /* Send the delete manifest (keep-set paths plus the protected excluded - prefixes) to the server. Returns 0 on success, -1 on failure. When - --delete-excluded is given `protected` is empty: excluded destination - mirrors are then ordinary extras and are removed. Both sections are - unbounded on the sender; the receiver enforces MAX_MANIFEST_ENTRIES per - section and a single MAX_MANIFEST_BYTES budget shared across the two - sections, rejecting (with STATUS_ERROR) an over-budget frame. A heavily - filtered source whose exclusion list is large therefore fails the run - cleanly on the receiver rather than being truncated. */ -static int send_delete_manifest(int fd, ArrayList* manifest, ArrayList* protected_prefixes) { - if (!manifest) - return -1; + prefixes and the --delete-missing-args exact-delete paths) to the server. + Returns 0 on success, -1 on failure. When --delete-excluded is given + `protected` is empty: excluded destination mirrors are then ordinary extras + and are removed. When --delete-missing-args is active `missing_args` holds + the destination mirrors of missing --files-from entries: each is an explicit + receiver-side deletion request, independent of the extras walk. A NULL + keep-set / protected / missing list transmits an empty section. All three + sections are unbounded on the sender; the receiver enforces + MAX_MANIFEST_ENTRIES per section and a single MAX_MANIFEST_BYTES budget + shared across the sections, rejecting (with STATUS_ERROR) an over-budget + frame. A heavily filtered source whose exclusion list is large therefore + fails the run cleanly on the receiver rather than being truncated. */ +static int send_delete_manifest(int fd, ArrayList* manifest, ArrayList* protected_prefixes, + ArrayList* missing_args) { if (!send_status(fd, STATUS_MANIFEST)) return -1; - if (!send_int(fd, manifest->size)) + int keep_count = manifest ? manifest->size : 0; + if (!send_int(fd, keep_count)) return -1; - for (int i = 0; i < manifest->size; i++) { + for (int i = 0; i < keep_count; i++) { if (!send_str(fd, (char*)manifest->items[i])) return -1; } @@ -652,6 +752,13 @@ static int send_delete_manifest(int fd, ArrayList* manifest, ArrayList* protecte if (!send_str(fd, (char*)protected_prefixes->items[i])) return -1; } + int missing_count = missing_args ? missing_args->size : 0; + if (!send_int(fd, missing_count)) + return -1; + for (int i = 0; i < missing_count; i++) { + if (!send_str(fd, (char*)missing_args->items[i])) + return -1; + } return 0; } @@ -667,10 +774,11 @@ static int send_delete_manifest(int fd, ArrayList* manifest, ArrayList* protecte #define DELETE_ACK_TIMEOUT_SEC 3600 static bool send_delete_manifest_early(Client* client, ArrayList* manifest, - ArrayList* protected_prefixes) { + ArrayList* protected_prefixes, ArrayList* missing_args) { if (!client || !manifest) return false; - if (send_delete_manifest(client->file_descriptor, manifest, protected_prefixes) != 0) + if (send_delete_manifest(client->file_descriptor, manifest, protected_prefixes, missing_args) != + 0) return false; Status ack; if (!receive_status_timed(client->file_descriptor, &ack, DELETE_ACK_TIMEOUT_SEC)) @@ -1031,7 +1139,8 @@ static int send_chunks_multithreaded(void* pipeline_context) { if (context->early_delete) { /* The keep-set manifest was prebuilt by a path-only pre-scan. Transmit it and wait for the receiver to delete extras before streaming any data. */ - if (!send_delete_manifest_early(client, context->manifest, context->excluded_paths)) { + if (!send_delete_manifest_early(client, context->manifest, context->excluded_paths, + context->missing_args)) { pipeline_cancel(context); disconnect_transfer_client(client); mark_sender_done(context); @@ -1066,7 +1175,13 @@ static int send_chunks_multithreaded(void* pipeline_context) { goto send_fail; } if (send_delete_manifest(client->file_descriptor, context->manifest, - context->excluded_paths) != 0) + context->excluded_paths, context->missing_args) != 0) + goto send_fail; + } else if (context->config->delete_missing_args && !context->early_delete) { + /* --delete-missing-args without --delete: no keep-set is built, but the + exact-delete paths still ride the same manifest frame (commit once the + transfer succeeded). */ + if (send_delete_manifest(client->file_descriptor, NULL, NULL, context->missing_args) != 0) goto send_fail; } bool ok = finalize_transfer(client, context->config, context->remove_source_files); @@ -1322,10 +1437,23 @@ int send_files(Config* config) { return send_list_only(config); if (config->dry_run) return send_dry_run_manifest(config); - if (!files_from_list_valid(config)) + ArrayList* missing_args = NULL; + int skipped = 0; + if (config->delete_missing_args) { + missing_args = array_list_create(free); + if (!missing_args) + return 1; + } + if (!files_from_list_check(config, missing_args, &skipped)) { + if (missing_args) + array_list_delete(missing_args); return 1; - if (config_has_basis(config) && !basis_oversize_preflight(config)) + } + if (config_has_basis(config) && !basis_oversize_preflight(config)) { + if (missing_args) + array_list_delete(missing_args); return 1; + } Client* client = connect_transfer_client(config); if (!client) { @@ -1392,7 +1520,7 @@ int send_files(Config* config) { "with an empty keep-set (--delete)"); prescan_ok = false; } else { - early_ok = send_delete_manifest_early(client, early_manifest, excluded); + early_ok = send_delete_manifest_early(client, early_manifest, excluded, missing_args); } } array_list_delete(early_manifest); @@ -1478,16 +1606,23 @@ int send_files(Config* config) { "an empty keep-set (--delete)"); goto send_fail; } - if (manifest) { - /* Late (commit) ordering: all file data is out; transmit the keep-set - manifest so the receiver deletes only after the transfer succeeds. */ - if (send_delete_manifest(client->file_descriptor, manifest, excluded) != 0) { - array_list_delete(manifest); - manifest = NULL; + if ((manifest || config->delete_missing_args) && !delete_early) { + /* Late (commit) ordering: all file data is out; transmit the manifest so + the receiver commits the extras walk (--delete) and/or the + --delete-missing-args exact-path deletions only after the transfer + succeeds. In the early modes (--delete-before/--delete-during) the + manifest already went out up front, so nothing is re-sent here. */ + if (send_delete_manifest(client->file_descriptor, manifest, excluded, missing_args) != 0) { + if (manifest) { + array_list_delete(manifest); + manifest = NULL; + } goto send_fail; } - array_list_delete(manifest); - manifest = NULL; + if (manifest) { + array_list_delete(manifest); + manifest = NULL; + } } bool ok = finalize_transfer(client, config, remove_sources); if (!ok && config->use_delete) @@ -1526,6 +1661,8 @@ send_fail: array_list_delete(manifest); if (excluded) array_list_delete(excluded); + if (missing_args) + array_list_delete(missing_args); if (remove_sources) array_list_delete(remove_sources); if (scanner) @@ -1544,10 +1681,23 @@ int send_files_multithreaded(Config** config_ptr) { return send_list_only(config); if (config->dry_run) return send_dry_run_manifest(config); - if (!files_from_list_valid(config)) + ArrayList* missing_args = NULL; + int skipped = 0; + if (config->delete_missing_args) { + missing_args = array_list_create(free); + if (!missing_args) + return 1; + } + if (!files_from_list_check(config, missing_args, &skipped)) { + if (missing_args) + array_list_delete(missing_args); return 1; - if (config_has_basis(config) && !basis_oversize_preflight(config)) + } + if (config_has_basis(config) && !basis_oversize_preflight(config)) { + if (missing_args) + array_list_delete(missing_args); return 1; + } long pages = sysconf(_SC_AVPHYS_PAGES); long page_size = sysconf(_SC_PAGE_SIZE); @@ -1574,9 +1724,13 @@ int send_files_multithreaded(Config** config_ptr) { if (!context) { queue_destroy(q1); queue_destroy(q2); + if (missing_args) + array_list_delete(missing_args); return 1; } - *config_ptr = NULL; /* context now owns config through all remaining paths */ + context->missing_args = missing_args; + missing_args = NULL; /* owned by the context from here on */ + *config_ptr = NULL; /* context now owns config through all remaining paths */ bool collect_excluded = config->use_delete && !config->delete_excluded; if (config->use_delete) { context->manifest = array_list_create(free); diff --git a/src/client/scanner.c b/src/client/scanner.c index f72f5b0..87fa020 100644 --- a/src/client/scanner.c +++ b/src/client/scanner.c @@ -352,6 +352,7 @@ DirectoryScanner* directory_scanner_create_with_options(const char* root_directo scanner->excluded_paths = options->excluded_paths; scanner->excluded_mutex = options->excluded_mutex; scanner->ignore_io_errors = options->ignore_io_errors; + scanner->ignore_missing_args = options->ignore_missing_args; scanner->io_error = false; scanner->dirs_mode = options->dirs; scanner->relative_mode = options->relative && options->file_list != NULL; @@ -591,6 +592,16 @@ static File* dirs_file_for_entry(DirectoryScanner* scanner, const char* entry) { } struct stat link_stats; if (lstat(abs_path, &link_stats) != 0) { + /* --ignore-missing-args (implied by --delete-missing-args): an explicitly + listed entry that does not exist under the source is a preflight-detected + missing argument and is skipped here, exactly as the recursive scan skips + nothing (missing entries never appear there). Without the flags it stays + a hard pre-transfer error. */ + if (scanner->ignore_missing_args) { + log_info_message(LOG_INFO_MISC, "skipping missing --files-from entry '%s'", entry); + free(abs_path); + return NULL; + } log_message(LOG_LEVEL_ERROR, "--dirs listed entry is not present under the source: %s", entry); free(abs_path); scanner->failed = true; diff --git a/src/client/scanner.h b/src/client/scanner.h index fe92309..6c4e5fe 100644 --- a/src/client/scanner.h +++ b/src/client/scanner.h @@ -59,6 +59,11 @@ typedef struct { /* --ignore-errors: an unreadable directory during the scan is recorded as an * I/O error and skipped instead of aborting the scan. Client-only. */ bool ignore_io_errors; + /* --ignore-missing-args (implied by --delete-missing-args): an explicitly + * --files-from-listed entry that does not exist under the source is skipped + * instead of failing (the --dirs generator is the only scanner path that + * observes a listed-but-missing entry). */ + bool ignore_missing_args; } ScannerOptions; /* Internal per-scanner filter state. FilterNode chains represent the ordered @@ -112,6 +117,9 @@ typedef struct { mtx_t* excluded_mutex; /* --ignore-errors: continue past unreadable directories (records io_error). */ bool ignore_io_errors; + /* --ignore-missing-args: --dirs listed-but-missing entries are skipped, not + fatal (see ScannerOptions.ignore_missing_args). */ + bool ignore_missing_args; /* A directory could not be opened (I/O error, e.g. EACCES). With --ignore-errors the scan continues past it and the caller decides what to do; `failed` is reserved for fatal errors that always abort the scan. */ diff --git a/src/client/usage.c b/src/client/usage.c index f4ee960..11cbb07 100644 --- a/src/client/usage.c +++ b/src/client/usage.c @@ -46,6 +46,12 @@ void print_usage(void) { printf(" deletion\n"); printf(" --force A file may replace a destination directory by removing\n"); printf(" that (non-empty) directory first\n"); + printf(" --ignore-missing-args A --files-from entry that does not exist under the\n"); + printf(" source is silently skipped instead of failing the run\n"); + printf(" --delete-missing-args Implies --ignore-missing-args; also deletes each missing\n"); + printf(" entry's destination mirror receiver-side. Independent of\n"); + printf(" --delete (it does not imply --delete; a non-empty directory\n"); + printf(" mirror is removed only with --force or --delete)\n"); printf(" --prune-empty-dirs Do not transfer empty directory entries (--dirs mode);\n"); printf(" recursive transfers never send empty dirs. rsync's -m\n"); printf(" short form stays FastSync multithreading\n"); diff --git a/src/server/receiver.c b/src/server/receiver.c index 0556292..ea6d29f 100644 --- a/src/server/receiver.c +++ b/src/server/receiver.c @@ -191,7 +191,9 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver so the sender only starts streaming once the deletion committed (or failed). This is the rsync delete-before/delete-during window: a later transfer failure does not restore these deletions. */ - bool deletion_ok = config->use_delete ? manifest_delete_extras(config, manifest) : true; + bool deletion_ok = (config->use_delete || config->delete_missing_args) + ? manifest_delete_all(config, manifest) + : true; delete_manifest_free(manifest); if (!deletion_ok) { send_status(file_descriptor, STATUS_ERROR); @@ -199,9 +201,10 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver } if (!send_status(file_descriptor, STATUS_OK)) goto fail; - } else if (config->use_delete) { - /* Plain --delete / --delete-after / --delete-delay: hold the keep-set - and commit the deletion only after STATUS_FINISHED. */ + } else if (config->use_delete || config->delete_missing_args) { + /* Plain --delete / --delete-after / --delete-delay and the + --delete-missing-args exact-path deletions: hold the manifest and + commit it only after STATUS_FINISHED. */ if (deferred_manifest) { log_message(LOG_LEVEL_ERROR, "Received a second delete manifest"); delete_manifest_free(deferred_manifest); @@ -246,7 +249,7 @@ int receiver_process_pending(Config* config, int file_descriptor, const Receiver *pending_manifest = deferred_manifest; deferred_manifest = NULL; } else { - bool deletion_ok = manifest_delete_extras(config, deferred_manifest); + bool deletion_ok = manifest_delete_all(config, deferred_manifest); delete_manifest_free(deferred_manifest); deferred_manifest = NULL; if (!deletion_ok) { diff --git a/src/server/server.c b/src/server/server.c index a8aa7e8..b042ffb 100644 --- a/src/server/server.c +++ b/src/server/server.c @@ -180,6 +180,11 @@ void handler(int file_descriptor) { return; } config->use_delete = config->use_delete && allow_delete; + /* --delete-missing-args deletes destination mirrors receiver-side, so it is + deletion and stays gated by the same --allow-delete server policy. When + the server policy is off the flag is inert (the missing entries are still + skipped via its implied --ignore-missing-args, but nothing is deleted). */ + config->delete_missing_args = config->delete_missing_args && allow_delete; /* --mkpath: create the destination root (and its missing leading components) before anything else; without it the root must pre-exist. A failure here aborts the connection cleanly before any file data is exchanged. */ @@ -262,7 +267,7 @@ void handler(int file_descriptor) { known to have succeeded. Remove the extras before publishing a --delay-updates run; the walker skips the staging directory. */ if (context->deferred_manifest) { - if (!manifest_delete_extras(config, context->deferred_manifest)) { + if (!manifest_delete_all(config, context->deferred_manifest)) { transfer_ok = false; } delete_manifest_free(context->deferred_manifest); diff --git a/src/shared/config.c b/src/shared/config.c index 49c25f0..09e733d 100644 --- a/src/shared/config.c +++ b/src/shared/config.c @@ -96,6 +96,8 @@ static void config_set_defaults(Config* config) { config->max_delete = -1; config->ignore_errors = false; config->force_delete = false; + config->ignore_missing_args = false; + config->delete_missing_args = false; config->filters = NULL; config->files_from = NULL; config->files_from_set = NULL; @@ -165,10 +167,11 @@ static bool validate_received_config(const Config* config) { valid_wire_bool(config->inplace) && valid_wire_bool(config->append) && valid_wire_bool(config->use_fsync) && valid_wire_bool(config->append_verify) && valid_wire_bool(config->delete_excluded) && valid_wire_bool(config->force_delete) && - valid_wire_bool(config->delete_after) && valid_wire_bool(config->delete_delay) && - valid_wire_bool(config->delete_during) && valid_wire_bool(config->relative) && - valid_wire_bool(config->prune_empty_dirs) && valid_wire_bool(config->delay_updates) && - valid_wire_bool(config->mkpath) && !(config->delay_updates && config->inplace) && + valid_wire_bool(config->delete_missing_args) && valid_wire_bool(config->delete_after) && + valid_wire_bool(config->delete_delay) && valid_wire_bool(config->delete_during) && + valid_wire_bool(config->relative) && valid_wire_bool(config->prune_empty_dirs) && + valid_wire_bool(config->delay_updates) && valid_wire_bool(config->mkpath) && + !(config->delay_updates && config->inplace) && !(config->delay_updates && delay_updates_staging_name_conflict(config->backup_dir)) && valid_wire_bool(config->partial) && valid_wire_bool(config->delete_before) && valid_wire_bool(config->checksum) && valid_wire_bool(config->eight_bit_output) && @@ -421,9 +424,10 @@ static bool send_selection_options(int fd, const Config* c) { send_int(fd, c->inplace) && send_int(fd, c->delay_updates) && send_int(fd, c->append) && send_int(fd, c->use_fsync) && send_int(fd, c->append_verify) && send_int(fd, c->delete_excluded) && send_int(fd, c->force_delete) && - send_int(fd, c->delete_after) && send_n_data(fd, &c->max_delete, sizeof(c->max_delete)) && - send_int(fd, c->relative) && send_int(fd, c->prune_empty_dirs) && - send_int(fd, c->mkpath) && send_int(fd, c->delete_during) && send_int(fd, c->delete_delay); + send_int(fd, c->delete_missing_args) && send_int(fd, c->delete_after) && + send_n_data(fd, &c->max_delete, sizeof(c->max_delete)) && send_int(fd, c->relative) && + send_int(fd, c->prune_empty_dirs) && send_int(fd, c->mkpath) && + send_int(fd, c->delete_during) && send_int(fd, c->delete_delay); } static bool send_skip_compress_options(int fd, const Config* c) { @@ -532,9 +536,18 @@ static bool receive_file_options(int fd, Config* c) { } static bool receive_selection_options(int fd, Config* c) { - bool* flags[] = {&c->ignore_existing, &c->existing, &c->update, &c->inplace, - &c->delay_updates, &c->append, &c->use_fsync, &c->append_verify, - &c->delete_excluded, &c->force_delete, &c->delete_after}; + bool* flags[] = {&c->ignore_existing, + &c->existing, + &c->update, + &c->inplace, + &c->delay_updates, + &c->append, + &c->use_fsync, + &c->append_verify, + &c->delete_excluded, + &c->force_delete, + &c->delete_missing_args, + &c->delete_after}; for (size_t i = 0; i < sizeof(flags) / sizeof(flags[0]); i++) { if (!receive_wire_bool(fd, flags[i])) return false; diff --git a/src/shared/config.h b/src/shared/config.h index 34b1552..41a7634 100644 --- a/src/shared/config.h +++ b/src/shared/config.h @@ -145,6 +145,19 @@ typedef struct Config { * directory by removing that (possibly non-empty, symlink-safe) directory * tree first, instead of failing the write. Crosses the wire. */ bool force_delete; + /* --ignore-missing-args (client-only, never serialized): a --files-from + * entry that does not exist under the source is silently skipped instead of + * failing the run. Sender-side only: nothing is sent for it and it never + * enters the keep-set. Implied by --delete-missing-args. */ + bool ignore_missing_args; + /* --delete-missing-args: implies --ignore-missing-args; additionally each + * missing entry's destination mirror (computed like a present entry's wire + * path) is deleted receiver-side. Crosses the wire and is gated by the + * server's --allow-delete policy like --delete. rsync-parity: independent + * of ordinary --delete processing (it does not imply --delete); a non-empty + * directory mirror is only removed with --force or --delete in effect, and + * the missing-args deletions are not counted toward --max-delete. */ + bool delete_missing_args; // Issue #129: Advanced file selection. These fields are CLIENT-ONLY: they are // never serialized to the wire (the receiver must not learn them). @@ -231,7 +244,7 @@ typedef struct Config { DelayUpdatesContext* delay_context; } Config; -#define PROTOCOL_VERSION "2.9.0" +#define PROTOCOL_VERSION "2.10.0" #define DEFAULT_CHUNK_SIZE (10 * 1024 * 1024) /* Upper bound on total basis-dir entries (rsync caps --link-dest at 20). */ #define MAX_BASIS_DIRS 64 diff --git a/src/shared/file_receive.c b/src/shared/file_receive.c index a937787..d373a60 100644 --- a/src/shared/file_receive.c +++ b/src/shared/file_receive.c @@ -1380,12 +1380,16 @@ File* file_receive_directory(int file_descriptor) { /* Read a delete-manifest frame (the STATUS_MANIFEST leading code has already been consumed): a keep-set entry count followed by that many destination-relative paths, then a protected-prefix count followed by that - many destination-relative prefixes. The frame is self-delimiting (the counts - are authoritative), so the caller decides what to do next and continues - reading the following STATUS_* frame. Returns an owned DeleteManifest, or - NULL after sending STATUS_ERROR when the frame is malformed (bad count, - empty/absolute path, path traversal, or an aggregate size beyond - MAX_MANIFEST_BYTES). */ + many destination-relative prefixes, then (protocol 2.10.0+) a missing-args + count followed by that many destination-relative delete paths. The frame is + self-delimiting (the counts are authoritative), so the caller decides what to + do next and continues reading the following STATUS_* frame. Every section is + validated identically: an entry must be non-empty, relative and traversal-free + and the aggregate length across ALL sections is capped by MAX_MANIFEST_BYTES + (so the missing-args deletion requests are confined like the rest of the + manifest). Returns an owned DeleteManifest, or NULL after sending STATUS_ERROR + when the frame is malformed (bad count, empty/absolute path, path traversal, + or an aggregate size beyond MAX_MANIFEST_BYTES). */ static bool receive_manifest_section(int fd, ArrayList* list, size_t* manifest_bytes) { int count; if (!receive_int(fd, &count)) { @@ -1418,14 +1422,16 @@ DeleteManifest* receive_manifest_entries(int fd) { } manifest->keeps = array_list_create(free); manifest->protected = array_list_create(free); - if (!manifest->keeps || !manifest->protected) { + manifest->missing = array_list_create(free); + if (!manifest->keeps || !manifest->protected || !manifest->missing) { delete_manifest_free(manifest); send_status(fd, STATUS_ERROR); return NULL; } size_t manifest_bytes = 0; if (!receive_manifest_section(fd, manifest->keeps, &manifest_bytes) || - !receive_manifest_section(fd, manifest->protected, &manifest_bytes)) { + !receive_manifest_section(fd, manifest->protected, &manifest_bytes) || + !receive_manifest_section(fd, manifest->missing, &manifest_bytes)) { delete_manifest_free(manifest); return NULL; } @@ -1437,6 +1443,7 @@ void delete_manifest_free(DeleteManifest* manifest) { return; array_list_delete(manifest->keeps); array_list_delete(manifest->protected); + array_list_delete(manifest->missing); free(manifest); } @@ -1518,3 +1525,154 @@ bool manifest_delete_extras(const Config* config, DeleteManifest* manifest) { } return true; } + +/* --delete-missing-args exact-path deletions: each destination mirror in + manifest->missing is an explicit user request, so it is removed even when the + ordinary extras walk (with its protected prefixes) would leave it alone. The + --delay-updates staging directory and basis snapshots are receiver artifacts + and stay protected exactly as in the extras walker. A regular file or + symlink is unlinked, an empty directory removed, and a NON-empty directory is + removed recursively only when --delete or --force is in effect (rsync parity: + the man page says a non-empty directory mirror is only deleted with --force + or --delete); otherwise it is left with a warning and the run continues. A + mirror that does not exist is a no-op. Returns false only on a genuine error + (a confinement failure on a validated path or an I/O error), which fails the + run. */ +bool manifest_delete_missing_args(const Config* config, DeleteManifest* manifest) { + if (!config || !manifest) + return false; + if (!manifest->missing || manifest->missing->size == 0) + return true; + fprintf(stderr, "Deleting destination mirrors of missing source arguments...\n"); + int skip_count = (config->delay_updates ? 1 : 0) + config->basis_count; + DeleteSkipEntry* skips = NULL; + if (skip_count > 0) { + skips = calloc((size_t)skip_count, sizeof(DeleteSkipEntry)); + if (!skips) + return false; + int idx = 0; + if (config->delay_updates) { + skips[idx].prefix = DELAY_UPDATES_STAGING_DIR; + skips[idx].top_level_only = true; + idx++; + } + for (int i = 0; i < config->basis_count; i++) { + skips[idx].prefix = config->basis_dirs[i].path; + skips[idx].top_level_only = false; + idx++; + } + } + bool ok = true; + for (int i = 0; i < manifest->missing->size; i++) { + const char* rel = (const char*)manifest->missing->items[i]; + if (!rel || *rel == '\0' || *rel == '/' || has_path_traversal(rel)) { + /* Defensive only: receive_manifest_entries already validated every + section identically, so a controlled peer never reaches this branch. */ + log_message(LOG_LEVEL_ERROR, "invalid missing-args delete path"); + ok = false; + continue; + } + bool at_root = strchr(rel, '/') == NULL; + if (path_under_skip_prefix(rel, at_root, skips, skip_count)) { + char* escaped = output_escape(rel, log_get_8_bit_output()); + log_message(LOG_LEVEL_WARNING, + "missing-args path '%s' is protected (staging directory or basis snapshot); " + "not deleting", + escaped ? escaped : ""); + free(escaped); + continue; + } + char* full = path_cat(config->receive_root_directory, rel); + if (!full) { + ok = false; + continue; + } + char* leaf = NULL; + int parent_fd = file_open_secure_parent(full, &leaf, false); + if (parent_fd < 0) { + /* The mirror's parent directory may itself not exist on the destination + (a deeper missing entry whose leading directories were never created). + That is a no-op -- there is nothing to delete -- matching + file_remove_tree_secure's absent-path handling; only a genuine I/O + error (EACCES, a symlink loop, ...) fails the run. */ + bool absent = errno == ENOENT || errno == ENOTDIR; + free(full); + free(leaf); + if (!absent) + ok = false; + continue; + } + struct stat st; + if (fstatat(parent_fd, leaf, &st, AT_SYMLINK_NOFOLLOW) != 0) { + /* Already absent: nothing to delete (a no-op, not a deletion). */ + if (errno != ENOENT) + ok = false; + close(parent_fd); + free(leaf); + free(full); + continue; + } + bool removed = false; + if (S_ISDIR(st.st_mode)) { + if (unlinkat(parent_fd, leaf, AT_REMOVEDIR) == 0) { + removed = true; + } else if (errno == ENOTEMPTY || errno == EEXIST) { + close(parent_fd); + parent_fd = -1; + free(leaf); + leaf = NULL; + if (config->use_delete || config->force_delete) { + if (!file_remove_tree_secure(full)) + ok = false; + else + removed = true; + } else { + char* escaped = output_escape(rel, log_get_8_bit_output()); + log_message(LOG_LEVEL_WARNING, + "missing-args destination '%s' is a non-empty directory; use --force or " + "--delete to remove it", + escaped ? escaped : ""); + free(escaped); + } + } else if (errno != ENOENT) { + ok = false; + } + } else { + if (unlinkat(parent_fd, leaf, 0) == 0) { + removed = true; + } else if (errno != ENOENT) { + ok = false; + } + } + if (removed) { + char* escaped = output_escape(rel, log_get_8_bit_output()); + fprintf(stderr, " Deleted: %s\n", escaped ? escaped : ""); + free(escaped); + } + if (parent_fd >= 0) + close(parent_fd); + free(leaf); + free(full); + if (!ok) + break; + } + free(skips); + return ok; +} + +/* Commit every deletion family the manifest carries. The --delete-missing-args + exact-path deletions run FIRST: they are explicit user requests and must not + be blocked by the extras walker's filter-exclusion protection (a protected + leftover inside a missing-argument directory must not make that user-requested + removal fail). The ordinary extras walk then runs when --delete is active. + Returns true when there was nothing to do or every requested deletion + committed. */ +bool manifest_delete_all(const Config* config, DeleteManifest* manifest) { + if (!config || !manifest) + return false; + if (config->delete_missing_args && !manifest_delete_missing_args(config, manifest)) + return false; + if (config->use_delete && !manifest_delete_extras(config, manifest)) + return false; + return true; +} diff --git a/src/shared/file_receive.h b/src/shared/file_receive.h index 3bc1cd0..54dae5c 100644 --- a/src/shared/file_receive.h +++ b/src/shared/file_receive.h @@ -16,17 +16,22 @@ File* receive_incremental_check(int fd, const Config* config, bool* skipped); prefixes the sender asks the receiver never to delete (paths excluded on the source, protected at any depth). When --delete-excluded is given the sender transmits an empty protected list so excluded destination mirrors are treated - as ordinary extras. */ + as ordinary extras. With --delete-missing-args a third section (`missing`) + carries the destination mirrors of explicitly-listed source entries that do + not exist: each is an exact deletion request, independent of the ordinary + extras walk (never blocked by the protected prefixes) and processed when the + manifest is committed. */ typedef struct DeleteManifest { ArrayList* keeps; ArrayList* protected; + ArrayList* missing; } DeleteManifest; void delete_manifest_free(DeleteManifest* manifest); /* Read a delete-manifest frame: keep count + keeps, then protected count + - protected prefixes (self-delimiting; the leading STATUS_MANIFEST code has been - consumed). Returns an owned DeleteManifest, or NULL after signalling - STATUS_ERROR on a malformed frame. */ + protected prefixes, then missing count + missing paths (self-delimiting; the + leading STATUS_MANIFEST code has been consumed). Returns an owned + DeleteManifest, or NULL after signalling STATUS_ERROR on a malformed frame. */ DeleteManifest* receive_manifest_entries(int fd); /* Remove destination entries under config->receive_root_directory that are not in `manifest` (bounded, all-or-nothing walk; staging-dir, basis-dir and @@ -34,6 +39,20 @@ DeleteManifest* receive_manifest_entries(int fd); caller decides WHEN to run it based on the negotiated delete timing. Returns false (and the transfer fails) when the deletion cannot be committed. */ bool manifest_delete_extras(const Config* config, DeleteManifest* manifest); +/* --delete-missing-args exact-path deletions: remove each destination mirror + in `manifest->missing` (never blocked by the protected prefixes, staging dir + and basis dirs excluded). A regular file/symlink is unlinked; an empty + directory is removed; a NON-empty directory is removed recursively only when + --delete or --force is in effect, otherwise it is left with a warning (rsync + parity). A missing path is a no-op. Returns false only on a genuine + confinement or I/O error (the run then fails); tolerated per-path cases are + reported and skipped. */ +bool manifest_delete_missing_args(const Config* config, DeleteManifest* manifest); +/* Run every deletion family the manifest carries: the --delete-missing-args + exact-path deletions first (user requests are not blocked by exclusion + protection), then the ordinary extras walk when --delete is active. Returns + true when nothing to do or everything committed. */ +bool manifest_delete_all(const Config* config, DeleteManifest* manifest); /* Outcome of a single file_save_to_disk operation. The receiver needs to distinguish "written" from "skipped" so --remove-source-files can be told diff --git a/src/shared/multiprocessing.c b/src/shared/multiprocessing.c index 32270d3..2179c09 100644 --- a/src/shared/multiprocessing.c +++ b/src/shared/multiprocessing.c @@ -27,6 +27,7 @@ PipelineContextSender* pipeline_context_sender_create(Config* config, Queue* que context->loader_done = false; context->manifest = NULL; context->excluded_paths = NULL; + context->missing_args = NULL; context->scan_had_io_error = false; context->remove_source_files = NULL; context->early_delete = false; @@ -86,6 +87,8 @@ void pipeline_context_sender_destroy(PipelineContextSender* context) { } if (context->excluded_paths) array_list_delete(context->excluded_paths); + if (context->missing_args) + array_list_delete(context->missing_args); if (context->remove_source_files) array_list_delete(context->remove_source_files); config_delete(context->config); diff --git a/src/shared/multiprocessing.h b/src/shared/multiprocessing.h index d515282..3d33a10 100644 --- a/src/shared/multiprocessing.h +++ b/src/shared/multiprocessing.h @@ -32,6 +32,12 @@ typedef struct { scanner's exclusion sink) or, in the early modes, by the path-only pre-scan on the calling thread before the pipeline starts. */ ArrayList* excluded_paths; + /* --delete-missing-args: the destination-relative mirrors of the --files-from + entries that are missing under the source. Computed by the preflight on + the calling thread before the pipeline starts; the sender thread transmits + them in the manifest frame's third section and the receiver deletes each as + an explicit request. */ + ArrayList* missing_args; /* A source I/O error (unreadable directory) was recorded during the scan. Set by the pre-scan (before the threads start) or by the scanner thread under mutex_scanner; the caller turns it into a non-zero exit when diff --git a/src/shared/utils.c b/src/shared/utils.c index d2dbf89..308d88f 100644 --- a/src/shared/utils.c +++ b/src/shared/utils.c @@ -208,8 +208,8 @@ static bool is_dir_in_manifest(const char* rel_path, ArrayList* manifest) { therefore protects "a" and "a/b/c" but not "ab". Entries with top_level_only set only protect DIRECT children of the receive root (at_root); nested directories that share such a name stay ordinary destination content. */ -static bool path_under_skip_prefix(const char* child_rel, bool at_root, - const DeleteSkipEntry* skips, int skip_count) { +bool path_under_skip_prefix(const char* child_rel, bool at_root, const DeleteSkipEntry* skips, + int skip_count) { for (int i = 0; i < skip_count; i++) { if (skips[i].top_level_only && !at_root) continue; diff --git a/src/shared/utils.h b/src/shared/utils.h index 4206095..5603038 100644 --- a/src/shared/utils.h +++ b/src/shared/utils.h @@ -32,6 +32,11 @@ typedef struct { const char* prefix; bool top_level_only; } DeleteSkipEntry; +/* True when child_rel is, or lies below, one of the protected entries (a prefix + "a" protects "a" and "a/b/c" but not "ab"; top_level_only entries protect + only DIRECT children of the destination root, i.e. child_rel has no '/'). */ +bool path_under_skip_prefix(const char* child_rel, bool at_root, const DeleteSkipEntry* skips, + int skip_count); /* Remove files/dirs under dest_root that are not listed in manifest without ever descending into a protected prefix (see DeleteSkipEntry). When max_delete is not SIZE_MAX the run is all-or-nothing: extras are counted diff --git a/tests/integration/test_features.py b/tests/integration/test_features.py index 422c21d..167edd4 100644 --- a/tests/integration/test_features.py +++ b/tests/integration/test_features.py @@ -1742,6 +1742,278 @@ class TestRelativeFilesFrom: "unlisted relative file was not deleted" +class TestMissingArgs: + """--ignore-missing-args / --delete-missing-args: a --files-from entry that + does not exist under the source is skipped instead of failing the run, and + (delete-missing) its destination mirror is removed receiver-side. Following + rsync, --delete-missing-args implies --ignore-missing-args but is + independent of --delete: unrelated extras stay unless --delete is also + given, and the missing-args deletion (an explicit user request) is never + blocked by filter-exclusion protection.""" + + def _make_source(self, name): + source = os.path.join(TEST_DATA_DIR, name) + clean_dir(source) + for rel, content in { + "a.txt": b"a\n", + "sub/b.txt": b"b\n", + "keep.txt": b"keep\n", + "prot/kept.txt": b"kept\n", + }.items(): + full = os.path.join(source, rel) + os.makedirs(os.path.dirname(full), exist_ok=True) + with open(full, "wb") as fh: + fh.write(content) + return source + + @pytest.mark.parametrize("mt", [False, True]) + def test_missing_entry_is_hard_error_before_transfer(self, shared_server, mt): + source = self._make_source("mg_default_src") + dest = os.path.join(TEST_DATA_DIR, "mg_default_dst") + clean_dir(dest) + lst = _write_rel_list(b"a.txt\ngone.txt\nsub/b.txt\n") + flags = ["--files-from", lst] + (["-m"] if mt else []) + result, _ = run_client(source, dest, flags=flags, port=shared_server.port) + assert result.returncode != 0, "a listed-but-missing entry did not fail the run" + assert "gone.txt" in (result.stderr or result.stdout) + received = get_dest_received_dir(dest, source) + assert not os.path.isfile(os.path.join(received, "a.txt")), \ + "the transfer started despite the missing-entry hard error" + + @pytest.mark.parametrize("mt", [False, True]) + def test_ignore_missing_args_transfers_the_rest(self, shared_server, mt): + source = self._make_source("mg_ignore_src") + dest = os.path.join(TEST_DATA_DIR, "mg_ignore_dst") + clean_dir(dest) + lst = _write_rel_list(b"a.txt\ngone.txt\nsub/b.txt\n") + flags = ["--files-from", lst, "--ignore-missing-args"] + (["-m"] if mt else []) + result, _ = run_client(source, dest, flags=flags, port=shared_server.port) + assert result.returncode == 0, f"ignore-missing-args sync failed: {result.stderr[:300]}" + received = get_dest_received_dir(dest, source) + assert _read_file(os.path.join(received, "a.txt")) == b"a\n" + assert _read_file(os.path.join(received, "sub", "b.txt")) == b"b\n" + assert not os.path.exists(os.path.join(received, "gone.txt")), \ + "nothing was transferred for the missing entry" + assert "--ignore-missing-args" in (result.stderr or result.stdout), \ + "the skipped entry must be observable (not a silent no-op)" + + @pytest.mark.parametrize("mt", [False, True]) + def test_all_missing_entries_succeed_transferring_nothing(self, shared_server, mt): + source = self._make_source("mg_all_missing_src") + dest = os.path.join(TEST_DATA_DIR, "mg_all_missing_dst") + clean_dir(dest) + lst = _write_rel_list(b"gone1.txt\ngone2.txt\n") + flags = ["--files-from", lst, "--ignore-missing-args"] + (["-m"] if mt else []) + result, _ = run_client(source, dest, flags=flags, port=shared_server.port) + assert result.returncode == 0, \ + f"all-missing run should succeed (rsync parity): {result.stderr[:300]}" + received = get_dest_received_dir(dest, source) + assert not os.path.exists(os.path.join(received, "gone1.txt")) + + @pytest.mark.parametrize("mt", [False, True]) + def test_empty_list_stays_a_hard_error(self, shared_server, mt): + source = self._make_source("mg_empty_src") + dest = os.path.join(TEST_DATA_DIR, "mg_empty_dst") + clean_dir(dest) + lst = _write_rel_list(b"") + flags = ["--files-from", lst, "--ignore-missing-args"] + (["-m"] if mt else []) + result, _ = run_client(source, dest, flags=flags, port=shared_server.port) + assert result.returncode != 0, "an empty --files-from list must stay a hard error" + assert "contains no entries" in (result.stderr or result.stdout) + + @pytest.mark.parametrize("mt", [False, True]) + def test_delete_missing_removes_mirror_not_unrelated(self, mt): + """-R layout: --delete-missing-args deletes exactly the missing entry's + destination mirror (bare relative path) and leaves unrelated extras + untouched; with --delete also present the unrelated extras go too.""" + source = self._make_source("mg_del_src") + dest = os.path.join(TEST_DATA_DIR, "mg_del_dst") + clean_dir(dest) + with ServerManager() as server: + server.start(extra_args=["--allow-delete"]) + seed = _write_rel_list(b"a.txt\nsub/b.txt\n") + result, _ = run_client(source, dest, + flags=["--files-from", seed, "-R"] + (["-m"] if mt else []), + port=server.port) + assert result.returncode == 0, f"seed -R sync failed: {result.stderr[:200]}" + assert os.path.isfile(os.path.join(dest, "a.txt")) + assert os.path.isfile(os.path.join(dest, "sub", "b.txt")) + + # Plant the missing entry's destination mirror and an unrelated extra. + with open(os.path.join(dest, "gone.txt"), "w") as fh: + fh.write("stale mirror") + with open(os.path.join(dest, "unrelated.txt"), "w") as fh: + fh.write("unrelated") + + lst = _write_rel_list(b"a.txt\ngone.txt\nsub/b.txt\n") + flags = ["--files-from", lst, "-R", "--delete-missing-args"] + (["-m"] if mt else []) + result, _ = run_client(source, dest, flags=flags, port=server.port) + assert result.returncode == 0, f"delete-missing sync failed: {result.stderr[:300]}" + assert not os.path.exists(os.path.join(dest, "gone.txt")), \ + "the missing entry's destination mirror was not deleted" + assert os.path.isfile(os.path.join(dest, "unrelated.txt")), \ + "--delete-missing-args removed an unrelated extra (only --delete may)" + assert os.path.isfile(os.path.join(dest, "a.txt")) + assert os.path.isfile(os.path.join(dest, "sub", "b.txt")) + + # Now with --delete the unrelated extra is an ordinary extra and must go. + lst2 = _write_rel_list(b"a.txt\ngone.txt\nsub/b.txt\n") + flags2 = ["--files-from", lst2, "-R", "--delete-missing-args", "--delete"] + \ + (["-m"] if mt else []) + result, _ = run_client(source, dest, flags=flags2, port=server.port) + assert result.returncode == 0, f"delete-missing + delete sync failed: {result.stderr[:300]}" + assert not os.path.exists(os.path.join(dest, "unrelated.txt")), \ + "--delete did not remove the unrelated extra" + assert not os.path.exists(os.path.join(dest, "gone.txt")) + assert os.path.isfile(os.path.join(dest, "a.txt")) + + @pytest.mark.parametrize("mt", [False, True]) + def test_delete_missing_mirror_outside_relative_layout(self, mt): + """Without -R the missing entry's mirror mirrors the full source path + below the destination root, exactly like a present sibling's.""" + source = self._make_source("mg_del_nor_src") + dest = os.path.join(TEST_DATA_DIR, "mg_del_nor_dst") + clean_dir(dest) + with ServerManager() as server: + server.start(extra_args=["--allow-delete"]) + # Full-tree seed places every current source file in the mirrored layout. + result, _ = run_client(source, dest, flags=["--delete"], port=server.port) + assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}" + received = get_dest_received_dir(dest, source) + assert os.path.isfile(os.path.join(received, "a.txt")) + + # Plant a stale mirror for an entry not (yet) on the source. + with open(os.path.join(received, "gone.txt"), "w") as fh: + fh.write("stale") + lst = _write_rel_list(b"a.txt\ngone.txt\n") + result, _ = run_client(source, dest, + flags=["--files-from", lst, "--delete-missing-args"], + port=server.port) + assert result.returncode == 0, f"delete-missing no-R sync failed: {result.stderr[:300]}" + assert not os.path.exists(os.path.join(received, "gone.txt")), \ + "the full-source-mirror path of the missing entry was not deleted" + assert os.path.isfile(os.path.join(received, "a.txt")) + + @pytest.mark.parametrize("mt", [False, True]) + def test_delete_missing_args_not_blocked_by_exclude_protection(self, mt): + """A missing-arg mirror that sits under a filter-excluded directory is an + explicit user request, so --delete-missing-args removes it even though an + ordinary --delete honours the exclusion protection (rsync parity). Uses + the non-relative layout: exclusion protection is only recorded there.""" + source = self._make_source("mg_excl_src") + dest = os.path.join(TEST_DATA_DIR, "mg_excl_dst") + clean_dir(dest) + with ServerManager() as server: + server.start(extra_args=["--allow-delete"]) + # Full-tree seed mirrors the whole source below the destination root. + result, _ = run_client(source, dest, port=server.port) + assert result.returncode == 0, f"seed failed: {result.stderr[:200]}" + received = get_dest_received_dir(dest, source) + assert os.path.isfile(os.path.join(received, "prot", "kept.txt")) + + # A stale mirror under the (now excluded) prot/ directory, plus an extra. + with open(os.path.join(received, "prot", "gone.txt"), "w") as fh: + fh.write("stale") + with open(os.path.join(received, "extra.txt"), "w") as fh: + fh.write("extra") + + lst = _write_rel_list(b"a.txt\nprot/gone.txt\n") + flags = ["--files-from", lst, "--filter=- prot/", "--delete-missing-args", + "--delete"] + (["-m"] if mt else []) + result, _ = run_client(source, dest, flags=flags, port=server.port) + assert result.returncode == 0, f"delete-missing exclude sync failed: {result.stderr[:300]}" + assert not os.path.exists(os.path.join(received, "prot", "gone.txt")), \ + "the explicit missing-arg deletion was blocked by exclusion protection" + assert os.path.isfile(os.path.join(received, "prot", "kept.txt")), \ + "the excluded-but-present destination file must stay (default protection)" + assert not os.path.exists(os.path.join(received, "extra.txt")), \ + "--delete did not remove the unrelated extra" + assert os.path.isfile(os.path.join(received, "a.txt")) + + @pytest.mark.parametrize("mt", [False, True]) + def test_delete_missing_args_with_delete_before(self, mt): + """--delete-before (early delete timing) composes with --delete-missing-args: + the exact-path deletions commit with the early manifest, before data, and + --delete-before implies --delete (so unrelated extras go too).""" + source = self._make_source("mg_early_src") + dest = os.path.join(TEST_DATA_DIR, "mg_early_dst") + clean_dir(dest) + with open(os.path.join(dest, "gone.txt"), "w") as fh: + fh.write("stale") + with open(os.path.join(dest, "extra.txt"), "w") as fh: + fh.write("extra") + with ServerManager() as server: + server.start(extra_args=["--allow-delete"]) + lst = _write_rel_list(b"a.txt\ngone.txt\ngone2.txt\n") + flags = ["--files-from", lst, "-R", "--delete-missing-args", "--delete-before"] + \ + (["-m"] if mt else []) + result, _ = run_client(source, dest, flags=flags, port=server.port) + assert result.returncode == 0, f"early delete-missing sync failed: {result.stderr[:300]}" + assert not os.path.exists(os.path.join(dest, "gone.txt")), \ + "early timing did not remove the missing-arg mirror" + assert os.path.isfile(os.path.join(dest, "a.txt")), "a.txt was not transferred" + assert not os.path.exists(os.path.join(dest, "extra.txt")), \ + "--delete-before implies --delete: unrelated extras must go" + + @pytest.mark.parametrize("mt", [False, True]) + @pytest.mark.parametrize("relative", [False, True]) + def test_delete_missing_deep_entry_with_absent_parent(self, mt, relative): + """A missing entry whose destination mirror's parent directory does not + exist is a no-op (nothing to delete), never a run failure: the + exact-path deletions must not abort the --delete extras walk. Covers + the -R bare-relative layout and the full source-mirror layout.""" + source = self._make_source("mg_deep_src") + dest = os.path.join(TEST_DATA_DIR, "mg_deep_dst") + clean_dir(dest) + rel_flags = ["-R"] if relative else [] + with ServerManager() as server: + server.start(extra_args=["--allow-delete"]) + if relative: + target_root = dest + else: + # Non-relative layout: seed a.txt so the receive-root mirror + # tree exists (its sub/ sibling deliberately does not). + seed = _write_rel_list(b"a.txt\n") + result, _ = run_client(source, dest, + flags=["--files-from", seed] + rel_flags, + port=server.port) + assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}" + target_root = get_dest_received_dir(dest, source) + assert os.path.isfile(os.path.join(target_root, "a.txt")) + with open(os.path.join(target_root, "extra.txt"), "w") as fh: + fh.write("extra") + + lst = _write_rel_list(b"a.txt\nsub/gone.txt\n") + flags = ["--files-from", lst, "--delete-missing-args", "--delete"] + rel_flags + \ + (["-m"] if mt else []) + result, _ = run_client(source, dest, flags=flags, port=server.port) + assert result.returncode == 0, \ + f"deep missing-entry sync failed: {result.stderr[:300]}" + assert _read_file(os.path.join(target_root, "a.txt")) == b"a\n" + assert not os.path.exists(os.path.join(target_root, "extra.txt")), \ + "--delete extras walk was aborted by the absent-parent missing entry" + assert not os.path.exists(os.path.join(target_root, "sub")), \ + "the absent parent directory of the missing entry was created" + + @pytest.mark.parametrize("mt", [False, True]) + def test_dirs_missing_entry_skipped_in_scanner(self, shared_server, mt): + """--dirs + --files-from: a listed-but-missing entry is skipped in the + --dirs generator (which would otherwise hard-fail), transferring the + rest of the list.""" + source = self._make_source("mg_dirs_src") + dest = os.path.join(TEST_DATA_DIR, "mg_dirs_dst") + clean_dir(dest) + lst = _write_rel_list(b"a.txt\ngone.txt\n") + flags = ["--files-from", lst, "--dirs", "-R", "--ignore-missing-args"] + \ + (["-m"] if mt else []) + result, _ = run_client(source, dest, flags=flags, port=shared_server.port) + assert result.returncode == 0, f"--dirs ignore-missing sync failed: {result.stderr[:300]}" + assert _read_file(os.path.join(dest, "a.txt")) == b"a\n", \ + "the listed present file was not transferred" + assert not os.path.exists(os.path.join(dest, "gone.txt")), \ + "a directory/file was created for the missing --dirs entry" + + class TestNoImpliedDirs: """--no-implied-dirs (only meaningful with -R + --files-from) refuses to place a listed file whose parent directory is not itself listed.""" diff --git a/tests/test_client_cli.c b/tests/test_client_cli.c index 5882bae..d5d041d 100644 --- a/tests/test_client_cli.c +++ b/tests/test_client_cli.c @@ -1855,6 +1855,72 @@ static void test_parse_args_max_delete_inert_without_delete() { config_delete(cfg); } +/* --ignore-missing-args / --delete-missing-args parse onto their config fields. + * --delete-missing-args implies --ignore-missing-args (order-independent), + * does NOT imply --delete (rsync: independent of other delete processing), and + * the config stays valid in every combination. */ +static void test_parse_args_missing_args_flags() { + Config* cfg = config_create(); + int positional_args[2]; + int positional_count = 0; + char* argv[] = {"fastsync", "--ignore-missing-args", "/src", "/dst"}; + EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0); + EXPECT_TRUE(cfg->ignore_missing_args); + EXPECT_FALSE(cfg->delete_missing_args); + EXPECT_FALSE(cfg->use_delete); + cfg->send_directory = str_dup("/src"); + cfg->receive_root_directory = str_dup("/dst"); + EXPECT_TRUE(validate_config(cfg)); + config_delete(cfg); + + cfg = config_create(); + positional_count = 0; + char* argv2[] = {"fastsync", "--delete-missing-args", "/src", "/dst"}; + EXPECT_EQ_INT(parse_args(cfg, 4, argv2, positional_args, &positional_count), 0); + EXPECT_TRUE(cfg->delete_missing_args); + EXPECT_TRUE(cfg->ignore_missing_args); + EXPECT_FALSE(cfg->use_delete); + cfg->send_directory = str_dup("/src"); + cfg->receive_root_directory = str_dup("/dst"); + EXPECT_TRUE(validate_config(cfg)); + config_delete(cfg); + + /* The implication is order-independent: even with the explicit flag first. */ + cfg = config_create(); + positional_count = 0; + char* argv3[] = {"fastsync", "--ignore-missing-args", "--delete-missing-args", "/src", "/dst"}; + EXPECT_EQ_INT(parse_args(cfg, 5, argv3, positional_args, &positional_count), 0); + EXPECT_TRUE(cfg->ignore_missing_args); + EXPECT_TRUE(cfg->delete_missing_args); + config_delete(cfg); + + /* --delete-missing-args composes with --delete (both active) and with a + delete-timing flag (which implies --delete); timing stays valid. */ + cfg = config_create(); + positional_count = 0; + char* argv4[] = {"fastsync", "--delete", "--delete-missing-args", "/src", "/dst"}; + EXPECT_EQ_INT(parse_args(cfg, 5, argv4, positional_args, &positional_count), 0); + EXPECT_TRUE(cfg->use_delete); + EXPECT_TRUE(cfg->delete_missing_args); + EXPECT_TRUE(cfg->ignore_missing_args); + cfg->send_directory = str_dup("/src"); + cfg->receive_root_directory = str_dup("/dst"); + EXPECT_TRUE(validate_config(cfg)); + config_delete(cfg); + + cfg = config_create(); + positional_count = 0; + char* argv5[] = {"fastsync", "--delete-before", "--delete-missing-args", "/src", "/dst"}; + EXPECT_EQ_INT(parse_args(cfg, 5, argv5, positional_args, &positional_count), 0); + EXPECT_TRUE(cfg->use_delete); + EXPECT_TRUE(cfg->delete_before); + EXPECT_TRUE(cfg->delete_missing_args); + cfg->send_directory = str_dup("/src"); + cfg->receive_root_directory = str_dup("/dst"); + EXPECT_TRUE(validate_config(cfg)); + config_delete(cfg); +} + void test_client_cli() { test_validate_config_required_paths(); test_validate_config_incompatible_options(); @@ -1956,4 +2022,5 @@ void test_client_cli() { test_parse_args_delete_policy_flags(); test_parse_args_delete_policy_invalid_values(); test_parse_args_max_delete_inert_without_delete(); + test_parse_args_missing_args_flags(); } diff --git a/tests/test_config.c b/tests/test_config.c index ced2318..542fe61 100644 --- a/tests/test_config.c +++ b/tests/test_config.c @@ -619,6 +619,57 @@ static void test_config_delete_policy_wire_roundtrip() { } } +/* --delete-missing-args crosses the wire (the receiver executes the exact-path + deletions) while --ignore-missing-args is client-only: the receiver must + observe delete_missing_args unchanged and ignore_missing_args always false. */ +static void test_config_delete_missing_args_wire_roundtrip() { + if (is_running_under_valgrind()) + return; + + struct { + bool delete_missing_args, ignore_missing_args; + } cases[] = { + {false, false}, + {true, false}, + {true, true}, + }; + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + int p[2]; + EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0); + pid_t pid = fork(); + if (pid == 0) { + close(p[1]); + io_set_fds(p[0], p[0]); + Config* recv = config_receive(p[0]); + bool ok = recv != NULL; + if (ok) { + ok = recv->delete_missing_args == cases[i].delete_missing_args && + /* ignore_missing_args never crosses the wire. */ + recv->ignore_missing_args == false; + } + config_delete(recv); + close(p[0]); + _exit(ok ? 0 : 1); + } else { + close(p[0]); + io_set_fds(p[1], p[1]); + Config* send_cfg = config_create(); + EXPECT_NOT_NULL(send_cfg); + send_cfg->send_directory = str_dup("/src"); + send_cfg->receive_root_directory = str_dup("/dst"); + send_cfg->delete_missing_args = cases[i].delete_missing_args; + send_cfg->ignore_missing_args = cases[i].ignore_missing_args; + bool sent = config_send(p[1], send_cfg); + int status; + waitpid(pid, &status, 0); + close(p[1]); + config_delete(send_cfg); + EXPECT_TRUE(sent); + EXPECT_TRUE(WIFEXITED(status) && WEXITSTATUS(status) == 0); + } + } +} + /* Basis-dir lists survive the config wire: each entry's type and path must round-trip unchanged. */ static void test_config_basis_roundtrip() { @@ -771,6 +822,7 @@ void test_config() { test_config_delete_timing_wire_roundtrip(); test_config_delete_timing_conflict_rejected(); test_config_delete_policy_wire_roundtrip(); + test_config_delete_missing_args_wire_roundtrip(); test_config_basis_roundtrip(); test_config_basis_wire_rejects_escaping(); test_config_basis_normalization(); diff --git a/tests/test_server.c b/tests/test_server.c index df0ab31..6fc70ba 100644 --- a/tests/test_server.c +++ b/tests/test_server.c @@ -231,7 +231,10 @@ static char* make_check_root(const char* tag) { } static void write_check_file(const char* dir, const char* name, const char* content) { - char path[1024]; + /* Sized so a caller that passes a PATH_MAX-bounded `dir` (e.g. one of the + test's own char[1024] stack buffers) still provably fits with the joined + name, keeping -Werror=format-truncation quiet. */ + char path[4096]; snprintf(path, sizeof(path), "%s/%s", dir, name); int fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0644); if (fd >= 0) { @@ -493,6 +496,7 @@ static void test_late_manifest_abort_frees_keepset() { EXPECT_TRUE(send_int(p[1], 1)); EXPECT_TRUE(send_str(p[1], "keep.txt")); EXPECT_TRUE(send_int(p[1], 0)); /* protected-prefix section is empty */ + EXPECT_TRUE(send_int(p[1], 0)); /* missing-args section is empty */ EXPECT_TRUE(send_status(p[1], STATUS_ABORT)); DeleteManifest* pending = NULL; @@ -516,6 +520,7 @@ static void test_late_manifest_eof_frees_keepset() { EXPECT_TRUE(send_int(p[1], 1)); EXPECT_TRUE(send_str(p[1], "keep.txt")); EXPECT_TRUE(send_int(p[1], 0)); /* protected-prefix section is empty */ + EXPECT_TRUE(send_int(p[1], 0)); /* missing-args section is empty */ shutdown(p[1], SHUT_WR); DeleteManifest* pending = NULL; @@ -539,10 +544,12 @@ static void test_late_second_manifest_frees_both() { EXPECT_TRUE(send_int(p[1], 1)); EXPECT_TRUE(send_str(p[1], "first.txt")); EXPECT_TRUE(send_int(p[1], 0)); /* protected-prefix section is empty */ + EXPECT_TRUE(send_int(p[1], 0)); /* missing-args section is empty */ EXPECT_TRUE(send_status(p[1], STATUS_MANIFEST)); EXPECT_TRUE(send_int(p[1], 1)); EXPECT_TRUE(send_str(p[1], "second.txt")); EXPECT_TRUE(send_int(p[1], 0)); /* protected-prefix section is empty */ + EXPECT_TRUE(send_int(p[1], 0)); /* missing-args section is empty */ DeleteManifest* pending = NULL; EXPECT_EQ_INT(run_pending_receiver(cfg, p[0], &pending), -1); @@ -553,6 +560,170 @@ static void test_late_second_manifest_frees_both() { config_delete(cfg); } +/* A delete-manifest frame with a third (missing-args) section round-trips: the + receiver keeps all three sections and the missing paths are confined exactly + like the keep-set (a traversal entry in the missing section is rejected). + receive_manifest_entries() reads the counts directly (the leading + STATUS_MANIFEST code is consumed by the caller, so these frames do not send + it). */ +static void test_receive_manifest_three_sections() { + Config* cfg = config_create(); + EXPECT_NOT_NULL(cfg); + cfg->receive_root_directory = str_dup("/tmp/dst"); + int p[2]; + EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0); + io_set_fds(p[0], p[1]); + + EXPECT_TRUE(send_int(p[1], 1)); + EXPECT_TRUE(send_str(p[1], "keep.txt")); + EXPECT_TRUE(send_int(p[1], 1)); + EXPECT_TRUE(send_str(p[1], "protected.txt")); + EXPECT_TRUE(send_int(p[1], 2)); + EXPECT_TRUE(send_str(p[1], "gone.txt")); + EXPECT_TRUE(send_str(p[1], "dir/gone.bin")); + + DeleteManifest* manifest = receive_manifest_entries(p[0]); + EXPECT_NOT_NULL(manifest); + EXPECT_EQ_INT(manifest->keeps->size, 1); + EXPECT_EQ_STR((char*)manifest->keeps->items[0], "keep.txt"); + EXPECT_EQ_INT(manifest->protected->size, 1); + EXPECT_EQ_STR((char*)manifest->protected->items[0], "protected.txt"); + EXPECT_EQ_INT(manifest->missing->size, 2); + EXPECT_EQ_STR((char*)manifest->missing->items[0], "gone.txt"); + EXPECT_EQ_STR((char*)manifest->missing->items[1], "dir/gone.bin"); + delete_manifest_free(manifest); + + /* A traversal entry in the third section is rejected like every other. */ + EXPECT_TRUE(send_int(p[1], 0)); + EXPECT_TRUE(send_int(p[1], 0)); + EXPECT_TRUE(send_int(p[1], 1)); + EXPECT_TRUE(send_str(p[1], "../escape")); + EXPECT_NULL(receive_manifest_entries(p[0])); + Status status; + EXPECT_TRUE(receive_status(p[1], &status)); + EXPECT_EQ_INT(status, STATUS_ERROR); + + close(p[0]); + close(p[1]); + config_delete(cfg); +} + +/* --delete-missing-args exact-path deletions: regular files and empty + directories are removed, a non-empty directory survives without + --force/--delete and is recursively removed with --force or --delete, and a + missing mirror is a no-op. */ +static void test_manifest_delete_missing_args() { + char* root = make_check_root("qmissing"); + EXPECT_NOT_NULL(root); + write_check_file(root, "gone.txt", "stale"); + char empty_dir[1024], full_dir[1024], inner[1024]; + snprintf(empty_dir, sizeof(empty_dir), "%s/empty_dir", root); + snprintf(full_dir, sizeof(full_dir), "%s/full_dir", root); + snprintf(inner, sizeof(inner), "%s/full_dir/inner.txt", root); + EXPECT_EQ_INT(mkdir(empty_dir, 0755), 0); + EXPECT_EQ_INT(mkdir(full_dir, 0755), 0); + write_check_file(full_dir, "inner.txt", "content"); + + Config* cfg = config_create(); + EXPECT_NOT_NULL(cfg); + cfg->receive_root_directory = str_dup(root); + cfg->delete_missing_args = true; + + DeleteManifest* manifest = calloc(1, sizeof(DeleteManifest)); + EXPECT_NOT_NULL(manifest); + manifest->keeps = array_list_create(free); + manifest->protected = array_list_create(free); + manifest->missing = array_list_create(free); + EXPECT_TRUE(array_list_add(manifest->missing, str_dup("gone.txt"))); + EXPECT_TRUE(array_list_add(manifest->missing, str_dup("empty_dir"))); + EXPECT_TRUE(array_list_add(manifest->missing, str_dup("full_dir"))); + EXPECT_TRUE(array_list_add(manifest->missing, str_dup("never_here.txt"))); + /* A deeper entry whose destination parent directory does not exist is a + no-op (nothing to delete), never a failure. */ + EXPECT_TRUE(array_list_add(manifest->missing, str_dup("no_parent_here/gone.txt"))); + + /* Without --delete/--force the non-empty directory survives (rsync parity). */ + EXPECT_TRUE(manifest_delete_missing_args(cfg, manifest)); + char path[1024]; + snprintf(path, sizeof(path), "%s/gone.txt", root); + EXPECT_EQ_INT(access(path, F_OK), -1); + snprintf(path, sizeof(path), "%s/empty_dir", root); + EXPECT_EQ_INT(access(path, F_OK), -1); + snprintf(path, sizeof(path), "%s/full_dir", root); + EXPECT_EQ_INT(access(path, F_OK), 0); + EXPECT_EQ_INT(access(inner, F_OK), 0); + + /* With --force the non-empty directory mirror is removed recursively. */ + cfg->force_delete = true; + EXPECT_TRUE(array_list_add(manifest->missing, str_dup("full_dir"))); + EXPECT_TRUE(manifest_delete_missing_args(cfg, manifest)); + EXPECT_EQ_INT(access(full_dir, F_OK), -1); + snprintf(path, sizeof(path), "%s/no_parent_here", root); + EXPECT_EQ_INT(access(path, F_OK), -1); + + delete_manifest_free(manifest); + config_delete(cfg); + remove(full_dir); + rmdir(empty_dir); + rmdir(root); + free(root); +} + +/* A delete-missing-args manifest parked by the commit path is committed after + STATUS_FINISHED: the mirror that exists is removed, a missing mirror is a + no-op, and unrelated destination content is untouched (no --delete). */ +static void test_receiver_pending_commits_missing_args() { + char* root = make_check_root("qmisscomm"); + EXPECT_NOT_NULL(root); + write_check_file(root, "gone.txt", "stale"); + write_check_file(root, "extra.txt", "unrelated"); + + Config* cfg = config_create(); + EXPECT_NOT_NULL(cfg); + cfg->send_directory = str_dup("/src"); + cfg->receive_root_directory = str_dup(root); + cfg->delete_missing_args = true; + + int p[2]; + EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0); + io_set_fds(p[0], p[1]); + io_set_bwlimit(0); + + EXPECT_TRUE(send_status(p[1], STATUS_MANIFEST)); + EXPECT_TRUE(send_int(p[1], 0)); /* keep-set empty */ + EXPECT_TRUE(send_int(p[1], 0)); /* protected empty */ + EXPECT_TRUE(send_int(p[1], 2)); + EXPECT_TRUE(send_str(p[1], "gone.txt")); + EXPECT_TRUE(send_str(p[1], "never_here.txt")); + EXPECT_TRUE(send_status(p[1], STATUS_FINISHED)); + + /* NULL pending: the single-threaded commit path deletes at FINISHED. The + sink sends the terminal STATUS_OK success frame. */ + ReceiverSink sink = {.send_success = true}; + EXPECT_EQ_INT(receiver_process_pending(cfg, p[0], &sink, NULL), 0); + Status ack; + EXPECT_TRUE(receive_status(p[1], &ack)); + EXPECT_EQ_INT(ack, STATUS_OK); + + char path[1024]; + snprintf(path, sizeof(path), "%s/gone.txt", root); + EXPECT_EQ_INT(access(path, F_OK), -1); + snprintf(path, sizeof(path), "%s/extra.txt", root); + EXPECT_EQ_INT(access(path, F_OK), 0); + + close(p[0]); + close(p[1]); + config_delete(cfg); + { + /* remove fixtures */ + char pth[1024]; + snprintf(pth, sizeof(pth), "%s/extra.txt", root); + remove(pth); + rmdir(root); + } + free(root); +} + void test_server() { if (!is_running_under_valgrind()) { test_receive_files_finished(); @@ -566,5 +737,8 @@ void test_server() { test_late_manifest_abort_frees_keepset(); test_late_manifest_eof_frees_keepset(); test_late_second_manifest_frees_both(); + test_receive_manifest_three_sections(); + test_manifest_delete_missing_args(); + test_receiver_pending_commits_missing_args(); } }