From 00829fd265506c040d195b5a8fe0bf39a9cbcab9 Mon Sep 17 00:00:00 2001 From: TapTap Date: Sun, 20 Sep 2026 14:55:47 +0200 Subject: [PATCH] parity: --info=mount/stats, --stats dir breakdown, --debug categories --info=mount now prints rsync's mount-point skip line (matching rsync 3.4.1, which emits it for repeated -xx and drops the mount-point dir); --info=stats enables the same block as --stats; -x is repeatable. --stats counts traversed directories for the Number of files breakdown even when no directory metadata is captured (-r without -t/-p). --debug enables real output for flist/del/hash/deltasum/recv/filter/send at their natural FastSync events (synthetic categories stay inert). --stats and --debug rows keep their documented residual status. --- RSYNC_COMPAT.md | 6 +- src/client/client_cli.c | 51 +++-- src/client/client_send.c | 50 ++++- src/client/scanner.c | 65 ++++++- src/client/scanner.h | 15 +- src/shared/config.c | 2 +- src/shared/config.h | 4 +- src/shared/log.h | 20 +- src/shared/multiprocessing.c | 1 + src/shared/multiprocessing.h | 4 + tests/integration/test_output_parity.py | 40 ++++ tests/integration/test_parity_debug.py | 105 +++++++++++ .../test_parity_info_mount_stats.py | 174 ++++++++++++++++++ tests/test_client_cli.c | 17 +- 14 files changed, 519 insertions(+), 35 deletions(-) create mode 100644 tests/integration/test_parity_debug.py create mode 100644 tests/integration/test_parity_info_mount_stats.py diff --git a/RSYNC_COMPAT.md b/RSYNC_COMPAT.md index 8701c30..22df531 100644 --- a/RSYNC_COMPAT.md +++ b/RSYNC_COMPAT.md @@ -85,8 +85,8 @@ Every one of those has an entry below with its remaining caveats. | `-q`, `--quiet` | Suppress non-error messages | ✅ Parity | Suppresses client output while preserving errors | | `--help` | Show help | ✅ Parity | Prints usage and exits. A lone `-h` with no other transfer arguments also prints help (protocol 2.26.0), matching the rsync idiom; `-h` alongside a transfer keeps its rsync meaning of `--human-readable` (see that row) | | `-V`, `--version` | Print version | ✅ Parity | | -| `--info=FLAGS` | Fine-grained info verbosity | ⚠️ Caveat | Accepts rsync 3.4.1's full `--info` vocabulary — `backup`, `copy`, `del`, `flist`, `misc`, `mount`, `name`, `nonreg`, `progress`, `remove`, `skip`, `stats`, `symsafe`, `all`, `none` — with optional level suffixes (`--info=stats2`), so a valid rsync invocation is never rejected up front. Protocol 2.27.0 wires the categories that map to a real FastSync event, matching rsync's line format: `name` prints the updated entry names (with the ` -> target` link suffix), `flist` prints `sending incremental file list`, `del` prints `deleting PATH` (or `*deleting PATH` under `-i`/`--out-format`) for both dry-run would-delete and real deletions (real runs carry the removed paths over the new `report_deletes` wire bool), `remove` prints `sender removed PATH`, `nonreg` prints `skipping non-regular file "NAME"`, `progress` drives the per-file progress output, and `copy`/`misc`/`skip`/`stats` keep their existing channels. `none` suppresses info output, explicit flags override `--verbose`, and a genuinely unknown name is still rejected by name (matching rsync). **Fixed (no-wire):** `--info=name2` (and higher) also prints rsync's `NAME is uptodate` lines for entries the receiver already has, and `--info=name` emits the leading transfer-root `./` name line before the first transferred entry (the marker rides in the existing `info_level` bitset; differential tests vs rsync 3.4.1). **Caveat:** the root `./` line is emitted before the first transferred name rather than keyed off rsync's root-attribute-change decision, so a pre-existing root that rsync leaves untouched can differ; the categories with no client-observable event stay accepted-but-silent — `symsafe`, `mount`, and `backup` (the backup happens on the receiver, which FastSync's protocol does not echo back); and `skip` maps to FastSync's sender-side skip logging rather than rsync's receiver-side "not creating new file" lines | -| `--debug=FLAGS` | Fine-grained debug verbosity | ⚠️ Caveat | Protocol 2.26.0 accepts rsync 3.4.1's full `--debug` vocabulary with optional level suffixes. FastSync emits for its own channels (`io`, `proto`, `pack`, `util`, plus the aliases `hl`/`owner`); the rsync-only categories (`acl`, `filter`, `send`, ...) are accepted silently. `--debug=help` lists the flags; a genuinely unknown name is rejected by name. **Caveat:** most accepted rsync categories produce no output (e.g. `acl`, `filter`, `send`, `flist`, `del`, `deltasum`, `hash`, `recv`, `time`), so they are accepted for CLI compatibility only | +| `--info=FLAGS` | Fine-grained info verbosity | ⚠️ Caveat | Accepts rsync 3.4.1's full `--info` vocabulary — `backup`, `copy`, `del`, `flist`, `misc`, `mount`, `name`, `nonreg`, `progress`, `remove`, `skip`, `stats`, `symsafe`, `all`, `none` — with optional level suffixes (`--info=stats2`), so a valid rsync invocation is never rejected up front. Protocol 2.27.0 wires the categories that map to a real FastSync event, matching rsync's line format: `name` prints the updated entry names (with the ` -> target` link suffix), `flist` prints `sending incremental file list`, `del` prints `deleting PATH` (or `*deleting PATH` under `-i`/`--out-format`) for both dry-run would-delete and real deletions (real runs carry the removed paths over the new `report_deletes` wire bool), `remove` prints `sender removed PATH`, `nonreg` prints `skipping non-regular file "NAME"`, `progress` drives the per-file progress output, `copy`/`misc`/`skip` keep their existing channels, `stats` enables the same transfer-statistics block as `--stats`, and `mount` prints rsync's `[sender] skipping mount-point dir NAME` when `-xx` drops a mount-point directory (plain `-x` keeps the empty directory entry and stays silent, matching rsync; both differential-tested). `none` suppresses info output, explicit flags override `--verbose`, and a genuinely unknown name is still rejected by name (matching rsync). **Fixed (no-wire):** `--info=name2` (and higher) also prints rsync's `NAME is uptodate` lines for entries the receiver already has, and `--info=name` emits the leading transfer-root `./` name line before the first transferred entry (the marker rides in the existing `info_level` bitset; differential tests vs rsync 3.4.1). **Caveat:** the root `./` line is emitted before the first transferred name rather than keyed off rsync's root-attribute-change decision, so a pre-existing root that rsync leaves untouched can differ; the categories with no client-observable event stay accepted-but-silent — `symsafe` and `backup` (the backup happens on the receiver, which FastSync's protocol does not echo back); and `skip` maps to FastSync's sender-side skip logging rather than rsync's receiver-side "not creating new file" lines | +| `--debug=FLAGS` | Fine-grained debug verbosity | ⚠️ Caveat | Protocol 2.26.0 accepts rsync 3.4.1's full `--debug` vocabulary with optional level suffixes. FastSync emits for its own channels (`io`, `proto`, `pack`, `util`, plus the aliases `hl`/`owner`) and maps the remaining categories that have a natural FastSync event onto real debug output: `flist` (per-directory scan progress), `del` (receiver-removed paths, riding the existing `report_deletes` wire bool), `hash`/`deltasum` (whole-file hashing and delta-sum generation), `recv` (receiver verdicts/signatures), `filter` (selection/exclusion decisions) and `send` (files handed to the sender). A normal run prints none of it; `--debug=help` lists the flags and a genuinely unknown name is rejected by name. **Caveat:** the output is FastSync's own timestamped debug format (it does not reproduce rsync's exact per-category lines), and the synthetic/rsync-internal categories (`acl`, `backup`, `bind`, `time`, ...) stay accepted-but-silent, so the row remains ⚠️ | | `--stderr=MODE` | Change stderr output mode | ❌ Divergent | `errors` (default) and `all` are supported; `client` is rejected with a clear error (`--stderr=client is not supported`) because FastSync has no rsync client-message channel — the rejection itself is the documented behavior (Phase 7 Wave B decision). The modes that exist work; the missing rsync channel cannot be emulated without a wire change | | `--msgs2stderr`, `--no-msgs2stderr` | Deprecated `--stderr` aliases | ⚠️ Caveat | `--msgs2stderr` maps to `--stderr=all` (supported, matching rsync). `--no-msgs2stderr` is rsync's spelling of `--stderr=client`, which FastSync has no client-message channel for, so it maps to the errors-only default instead of reproducing rsync's client mode. See `--stderr=MODE` | | `--no-motd` | Suppress daemon MOTD | ✅ Parity | Client-only display switch (Wave C): the daemon still sends the configured `motd file` on a `host::module/path` connection; the client reads and discards the frame without showing it. Without the flag the MOTD is printed to stdout after the config/auth handshake and escaped so control bytes cannot inject terminal sequences | @@ -98,7 +98,7 @@ Every one of those has an entry below with its remaining caveats. | Flag | Rsync Description | FastSync Status | Notes | |------|-------------------|-----------------|-------| -| `--stats` | Give transfer stats | ⚠️ Caveat | Prints transfer statistics. Protocol 2.25.0 populates the receiver-only counters the sender cannot observe (`Matched data`, `Number of deleted files`) from the receiver's `STATUS_STATS` report; the sender tracks the scanned file list per type so `Number of files` carries rsync's `(reg: X, dir: Y, link: Z, special: W)` breakdown (directories come from the scanner's captured directory list, present for `-a`/`-t`/`-p`), `Number of regular files transferred` excludes symlinks/specials and up-to-date files, `Total file size` includes symlink target lengths, and `Total transferred file size` counts only transferred files. **Protocol 2.28.0 extends `STATUS_STATS`** with receiver-observed `literal_bytes` and the four `created_*` counters: `Number of created files` now carries rsync's `(reg/dir/link/special)` breakdown (the receiver reports which destination entries it newly created, including implicitly-created parent directories below the transfer root) and `Literal data` is exact for a delta transfer (the receiver counts the literal fragments it stored, not the whole source size) — all differential-tested in the sequential and `--threads` paths against rsync 3.4.1 for fresh-create, update and delta shapes. **Remaining divergences:** a recursive scan that preserves no directory attribute (`-r` without `-t`/`-p`) captures no directory entries, so the `dir:` category is then omitted from `Number of files`; rsync's per-type breakdown on `Number of deleted files` is not reproduced; and `Total bytes sent`/`received` are FastSync wire bytes framed differently from rsync's, so they are not numerically comparable | +| `--stats` | Give transfer stats | ⚠️ Caveat | Prints transfer statistics. Protocol 2.25.0 populates the receiver-only counters the sender cannot observe (`Matched data`, `Number of deleted files`) from the receiver's `STATUS_STATS` report; the sender tracks the scanned file list per type so `Number of files` carries rsync's `(reg: X, dir: Y, link: Z, special: W)` breakdown (directories come from the scanner's captured directory list for `-a`/`-t`/`-p`, or from a lightweight traversed-directory counter on a plain `-r` run so the `dir:` category is present there too), `Number of regular files transferred` excludes symlinks/specials and up-to-date files, `Total file size` includes symlink target lengths, and `Total transferred file size` counts only transferred files. **Protocol 2.28.0 extends `STATUS_STATS`** with receiver-observed `literal_bytes` and the four `created_*` counters: `Number of created files` now carries rsync's `(reg/dir/link/special)` breakdown (the receiver reports which destination entries it newly created, including implicitly-created parent directories below the transfer root) and `Literal data` is exact for a delta transfer (the receiver counts the literal fragments it stored, not the whole source size) — all differential-tested in the sequential and `--threads` paths against rsync 3.4.1 for fresh-create, update and delta shapes. **Remaining divergences:** rsync's per-type breakdown on `Number of deleted files` is not reproduced; and `Total bytes sent`/`received` are FastSync wire bytes framed differently from rsync's, so they are not numerically comparable | | `-h`, `--human-readable` | Human-readable numbers | ✅ Parity | Formats transfer byte and rate counts using rsync's **decimal** (base-1000) units, matching rsync `-h` (e.g. `1.23M`), not binary units. **A lone `-h` with no transfer arguments prints help instead** (protocol 2.26.0), matching the rsync idiom; `-h` alongside a transfer remains human-readable | | `-i`, `--itemize-changes` | Per-file change summary | ✅ Parity | Prints rsync-style `>f+++++++++` lines to stdout only for files actually sent (also under `-j`/`--threads`); unchanged files print nothing, matching single-`-i` behavior | | `--progress` | Show progress | ⚠️ Caveat | Protocol 2.25.0 prints rsync-style per-file progress blocks (percent, transferred/total bytes, rate, elapsed, `(xfr#N, to-chk=M/T)`) fed by the receiver's `STATUS_STATS`, in both the sequential and `--threads` send paths. FastSync also prints rsync's leading `./` transfer-root line and, when progress is requested (`--progress`/`-P`/`--info=progress`) and not `--quiet`, runs a **paths-only metadata pre-scan** (no file reads, no hashing) that supplies rsync's file-list total `T` for the `to-chk` denominator and the directory names; `--delete-during`/`--delete-delay` reuse their existing keep-set pre-scan instead of walking twice, and non-progress runs are untouched. Per-directory name lines are emitted (trailing `/`), and symlink (` -> target`) and special entries are named too, so a **fresh multi-directory tree's name set and `to-chk` denominator match rsync 3.4.1** (differential test, sequential and `--threads`) and a **single-file transfer's name lines and deterministic frames remain byte-identical** to rsync. **Remaining divergences:** rsync emits entries in sorted depth-first order while FastSync streams them in the scanner's readdir/BFS order, so the interleaving and the `to-chk` numerator differ (the denominator matches); the leading `./` root line is emitted unconditionally rather than keyed off rsync's root-attribute-change decision, and an ancestor directory line is emitted whenever a child transfers (rsync suppresses it when the directory itself is unchanged); on a re-run, entries without a quick-check (symlinks, empty directories) are still named where rsync stays silent; and the rate/ETA are wall-clock dependent | diff --git a/src/client/client_cli.c b/src/client/client_cli.c index c830ca9..8004d2d 100644 --- a/src/client/client_cli.c +++ b/src/client/client_cli.c @@ -504,9 +504,8 @@ static bool split_flag_level(const char* token, char* name, size_t name_size, in * of rsync's `symsafe`, `hlink`, and `own`. */ static bool is_accepted_debug_category(const char* name) { static const char* const categories[] = { - "acl", "backup", "bind", "chdir", "cmd", "connect", "del", "deltasum", - "dup", "exit", "filter", "flist", "fuzzy", "genr", "hash", "hl", - "hlink", "iconv", "nstr", "own", "owner", "recv", "send", "time", + "acl", "backup", "bind", "chdir", "cmd", "connect", "dup", "exit", "fuzzy", + "genr", "hl", "hlink", "iconv", "nstr", "own", "owner", "time", }; for (size_t i = 0; i < sizeof(categories) / sizeof(categories[0]); i++) { if (strcmp(name, categories[i]) == 0) @@ -518,7 +517,6 @@ static bool is_accepted_debug_category(const char* name) { static bool is_accepted_info_category(const char* name) { static const char* const categories[] = { "backup", - "mount", "syms", "symsafe", }; @@ -571,6 +569,18 @@ static int parse_debug_flags(const char* value, Config* config) { flag = LOG_DEBUG_PACK; } else if (strcmp(name, "util") == 0) { flag = LOG_DEBUG_UTIL; + } else if (strcmp(name, "flist") == 0) { + flag = LOG_DEBUG_FLIST; + } else if (strcmp(name, "del") == 0) { + flag = LOG_DEBUG_DEL; + } else if (strcmp(name, "hash") == 0 || strcmp(name, "deltasum") == 0) { + flag = LOG_DEBUG_HASH; + } else if (strcmp(name, "recv") == 0) { + flag = LOG_DEBUG_RECV; + } else if (strcmp(name, "filter") == 0) { + flag = LOG_DEBUG_FILTER; + } else if (strcmp(name, "send") == 0) { + flag = LOG_DEBUG_SEND; } else if (is_accepted_debug_category(name)) { continue; } else { @@ -645,9 +655,12 @@ static int parse_info_flags(const char* value, Config* config) { flag = LOG_INFO_MISC; else if (strcmp(name, "skip") == 0) flag = LOG_INFO_SKIP; - else if (strcmp(name, "stats") == 0) + else if (strcmp(name, "stats") == 0) { flag = LOG_INFO_STATS; - else if (strcmp(name, "del") == 0) + /* `--info=stats` requests the same transfer-statistics block as + `--stats`; `--info=stats0` turns it back off. */ + config->stats = level > 0; + } else if (strcmp(name, "del") == 0) flag = LOG_INFO_DEL; else if (strcmp(name, "remove") == 0) flag = LOG_INFO_REMOVE; @@ -655,6 +668,8 @@ static int parse_info_flags(const char* value, Config* config) { flag = LOG_INFO_FLIST; else if (strcmp(name, "nonreg") == 0) flag = LOG_INFO_NONREG; + else if (strcmp(name, "mount") == 0) + flag = LOG_INFO_MOUNT; else if (strcmp(name, "progress") == 0) flag = LOG_INFO_PROGRESS; else if (is_accepted_info_category(name)) @@ -1213,7 +1228,13 @@ static int apply_table_option(Config* config, const OptionEntry* entry, const ch void* field = (char*)config + entry->offset; switch (entry->kind) { case OPT_FLAG: - *(bool*)field = true; + /* -x/--one-file-system is repeatable in rsync: `-xx` increments the level so + the scanner drops mount-point directories instead of recreating them + empty. Everything else is a plain boolean. */ + if (entry->offset == offsetof(Config, one_file_system)) + (*(int*)field)++; + else + *(bool*)field = true; return 0; case OPT_NOOP: return 0; @@ -2574,7 +2595,11 @@ static bool cli_handle_outbuf_option(CliParseCtx* ctx) { * load --files-from once every argument has been seen. Returns 0 on success, * -1 on error. */ static int cli_finalize_config(Config* config, bool verbose, bool no_delta, bool no_incremental) { - set_log_level(config->quiet ? LOG_LEVEL_ERROR : (verbose ? LOG_LEVEL_DEBUG : LOG_LEVEL_WARNING)); + /* An explicit --debug=FLAGS enables the debug log level by itself (rsync + behaviour); -v enables every other INFO-level message. */ + bool debug_enabled = verbose || config->debug_level != 0; + set_log_level(config->quiet ? LOG_LEVEL_ERROR + : (debug_enabled ? LOG_LEVEL_DEBUG : LOG_LEVEL_WARNING)); /* rsync's plain --delete defaults to delete-during (--del): each directory's extras are removed as that directory is processed, so space is freed progressively and a tight destination never has to hold the whole old+new @@ -2764,10 +2789,12 @@ static int cli_finalize_config(Config* config, bool verbose, bool no_delta, bool } /* --info=del on a real --delete run asks the receiver to report the paths it actually removed; the report rides the STATUS_STATS path list, so the wire - stats frame must be negotiated too. */ - config->report_deletes = config->use_delete && !config->dry_run && - ((config->info_level & LOG_INFO_DEL) != 0 || config->itemize_changes || - config->out_format != NULL); + stats frame must be negotiated too. --debug=del needs the same paths, so + it opts into the existing report (no new wire field). */ + config->report_deletes = + config->use_delete && !config->dry_run && + ((config->info_level & LOG_INFO_DEL) != 0 || config->itemize_changes || + config->out_format != NULL || (config->debug_level & LOG_DEBUG_DEL) != 0); config->report_stats = config->stats || config->show_progress || (config->info_level & LOG_INFO_PROGRESS) || format_needs_wire || config->report_deletes || (config->dry_run && config->use_delete); diff --git a/src/client/client_send.c b/src/client/client_send.c index c775bc2..535cb5d 100644 --- a/src/client/client_send.c +++ b/src/client/client_send.c @@ -129,6 +129,21 @@ static void stats_type_breakdown(const TransferStats* stats, char* out, size_t o out_size); } +/* rsync's `Number of files` counts every directory. A recursive scan that + preserves a directory attribute captures them in `dir_entries`; a `-r` scan + (no -t/-p) captures nothing, so fall back to the scanner's shared counter of + traversed directories that are not already represented by an inline + directory entry. The -d generator counts its explicit directory entries + inline and does not traverse, so it is excluded here. */ +static unsigned long long dir_count_for_stats(const Config* config, const ArrayList* dir_entries, + atomic_ullong* counter) { + if (config == NULL || config->dirs || config->list_only) + return 0; + if (dir_metadata_should_capture(config)) + return dir_entries != NULL ? (unsigned long long)dir_entries->size : 0; + return counter != NULL ? (unsigned long long)atomic_load(counter) : 0; +} + /* Print the rsync `--stats` block on stdout. The source-side flist and transferred counters come from `stats` (filled while scanning/sending), the receiver-only counters from the STATUS_STATS frame, and the wire byte totals @@ -387,6 +402,15 @@ static bool info_flag_enabled(const Config* config, LogInfoFlag flag) { static void print_delete_reports(const Config* config, const ArrayList* paths) { if (!config || !paths || config->quiet) return; + /* --debug=del is independent of the --info=del/itemize/out-format display: + emit the debug trace even when no deletion line would be printed. */ + if (log_debug_enabled(LOG_DEBUG_DEL)) { + for (int i = 0; i < paths->size; i++) { + const char* raw = (const char*)paths->items[i]; + const char* path = delete_display_path(config, raw); + log_debug_message(LOG_DEBUG_DEL, "del: %s", path ? path : raw); + } + } if (!(config->itemize_changes || config->out_format != NULL || info_flag_enabled(config, LOG_INFO_DEL))) return; @@ -665,6 +689,7 @@ static bool prepare_scanner(const Config* config, int num_threads, PreparedScann options->ignore_io_errors = config->ignore_errors; options->ignore_missing_args = config->ignore_missing_args || config->delete_missing_args; options->note_nonreg = (config->info_level & LOG_INFO_NONREG) != 0 && !config->quiet; + options->note_mount = (config->info_level & LOG_INFO_MOUNT) != 0 && !config->quiet; options->send_directory = config->send_directory; options->eight_bit_output = config->eight_bit_output; options->excluded_paths = NULL; @@ -672,6 +697,9 @@ static bool prepare_scanner(const Config* config, int num_threads, PreparedScann options->size_skipped_paths = NULL; options->synced_dirs = NULL; options->hardlinks = NULL; + /* Set by the real send paths; NULL for the metadata-only scans (progress + pre-count, batch) that must not perturb the sender's --stats counter. */ + options->dir_count = NULL; /* P7 Wave D: capture source directory metadata when a directory attribute is requested (-p for modes, -t for times unless -O omits them). Whether they are APPLIED is decided receiver-side. */ @@ -730,6 +758,8 @@ static bool progress_precount_scan(const Config* config, ProgressPrecount* out) ScannerOptions local = prepared.options; local.list_dirs = true; local.note_nonreg = false; + local.note_mount = false; + local.dir_count = NULL; local.use_metadata = false; local.preserve_xattrs = false; local.preserve_acls = false; @@ -1336,6 +1366,7 @@ static bool finalize_transfer(Client* client, const Config* config, ArrayList* r return false; if (status == STATUS_STATS) { ReceiverStats scratch; + log_debug_message(LOG_DEBUG_RECV, "recv: receiver stats"); /* A real --info=del run carries the actually-removed paths in the stats frame's path list; collect and print them in rsync's format. */ ArrayList* deleted = config->report_deletes ? array_list_create(free) : NULL; @@ -1885,6 +1916,8 @@ static int incremental_check(Client* client, File* file, const Config* config, if (!file_checksum(file, (ChecksumAlgo)config->checksum_algo, config->checksum_seed, digest, sizeof(digest), &digest_len)) return -1; + log_debug_message(LOG_DEBUG_HASH, "hash: %s (algo %d)", file_wire_path(file), + config->checksum_algo); uint8_t wire_len = (uint8_t)digest_len; if (!send_n_data(client->file_descriptor, &wire_len, sizeof(wire_len)) || !send_n_data(client->file_descriptor, digest, wire_len)) @@ -1919,6 +1952,7 @@ static int incremental_check(Client* client, File* file, const Config* config, log_server_rejection("Server reported error for file"); return -1; } + log_debug_message(LOG_DEBUG_RECV, "recv: check reply for %s", file_wire_path(file)); if (s == STATUS_OK) return 1; if (s == STATUS_DELTA_SIGNATURE) { @@ -1933,6 +1967,8 @@ static int incremental_check(Client* client, File* file, const Config* config, send_status(client->file_descriptor, STATUS_ERROR); return -1; } + log_debug_message(LOG_DEBUG_RECV, "recv: delta signature for %s (%d blocks)", + file_wire_path(file), sig->block_count); *out_sig = sig; return 2; } @@ -1973,6 +2009,7 @@ static int incremental_check(Client* client, File* file, const Config* config, static int send_delta(Client* client, File* file, DeltaSignature* sig, Config* config) { Delta* delta = delta_compute_seeded(file->data->data, file->data->size, sig, config->delta_block_size, (uint32_t)config->checksum_seed); + log_debug_message(LOG_DEBUG_HASH, "deltasum: %s", file_wire_path(file)); /* The receiver is blocked after sending the signature. Every local fallback therefore needs the explicit NEXT response before full data. */ if (!delta) @@ -2446,6 +2483,7 @@ static int send_single_file(Client* client, File* file, Config* config, bool use bool use_sendfile) { int compression_level = config->use_compression ? config->compression_level : 0; log_info_message(LOG_INFO_COPY, "Transferring %s", file->path); + log_debug_message(LOG_DEBUG_SEND, "send: %s", file_wire_path(file)); if (!use_incremental) { if (use_sendfile) { @@ -2886,8 +2924,8 @@ static int send_chunks_multithreaded(void* pipeline_context) { "server reported a deletion failure (--delete); see the server log for the reason"); if (ok) remove_transferred_sources(context->config, context->remove_source_files); - if (context->dir_entries) - context->stats.flist_dir += (unsigned long long)context->dir_entries->size; + context->stats.flist_dir += + dir_count_for_stats(context->config, context->dir_entries, &context->dir_count); report_transfer_stats(context->config, &context->stats, start, &recv_stats); log_info_message(LOG_INFO_STATS, "Transfer summary: %llu files, %.1f MB", context->stats.transferred_regular, @@ -2924,6 +2962,7 @@ static int scan_directory_multithreaded(void* pipeline_context) { parallel workers append under the context's dedicated mutex. */ prepared.options.dir_entries = context->dir_entries; prepared.options.dir_entries_mutex = &context->dir_entries_mutex; + prepared.options.dir_count = context->config->stats ? &context->dir_count : NULL; if (!append_implied_dir_times(context->config, context->dir_entries)) { pipeline_cancel(context); protocol_session_unbind(); @@ -3192,6 +3231,9 @@ int send_files(Config* config) { /* P7 Wave D: captured source directory times, transmitted in trailing STATUS_DIR_TIMES frame(s) (only when metadata rides the wire). */ ArrayList* dir_entries = NULL; + /* --stats directory accounting for the no-metadata (-r) case. */ + atomic_ullong dir_count; + atomic_init(&dir_count, 0); /* Protected excluded prefixes (delete-excluded default protection). */ ArrayList* excluded = NULL; /* Size-pruned prefixes (always protected) and synchronized directories. */ @@ -3373,6 +3415,7 @@ int send_files(Config* config) { the directory-time list (otherwise every directory would be captured twice). */ prepared.options.dir_entries = dir_entries; + prepared.options.dir_count = config->stats ? &dir_count : NULL; scanner = directory_scanner_create_with_options(config->send_directory, &prepared.options); if (!scanner) goto send_fail; @@ -3526,8 +3569,7 @@ int send_files(Config* config) { from the scanner's captured directory list (present whenever a directory attribute is preserved, e.g. -a/-t/-p). The -d generator counts its explicit directory entries inline instead. */ - if (dir_entries) - transfer_stats.flist_dir += (unsigned long long)dir_entries->size; + transfer_stats.flist_dir += dir_count_for_stats(config, dir_entries, &dir_count); report_transfer_stats(config, &transfer_stats, start, &recv_stats); log_info_message(LOG_INFO_STATS, "Transfer summary: %llu files, %.1f MB", transfer_stats.transferred_regular, diff --git a/src/client/scanner.c b/src/client/scanner.c index 99fec2c..1adbab8 100644 --- a/src/client/scanner.c +++ b/src/client/scanner.c @@ -219,8 +219,8 @@ typedef struct { /* --one-file-system (-x) decision. Only directories can carry a different * device than their parent (mount points), so this is checked when a child * directory is about to be descended into. */ -bool scanner_same_filesystem(bool one_file_system, dev_t root_device, dev_t entry_device) { - return !one_file_system || entry_device == root_device; +bool scanner_same_filesystem(int one_file_system, dev_t root_device, dev_t entry_device) { + return one_file_system <= 0 || entry_device == root_device; } /* Build a payload-less directory File carrying the captured metadata (when @@ -456,6 +456,40 @@ static void scanner_note_nonreg(const ScannerOptions* options, const char* fs_pa fflush(stdout); } +/* rsync 3.4.1's `--info=mount` line, emitted when `-xx` drops a mount-point + * directory: `[sender] skipping mount-point dir NAME` (the client is the + * sender). Plain `-x` keeps the empty directory and prints nothing, matching + * rsync. */ +static void scanner_note_mount(const ScannerOptions* options, const char* fs_path) { + if (!options || !options->note_mount || !fs_path) + return; + const char* rel = utils_strip_transfer_root(fs_path, options->send_directory); + char* escaped = output_escape(rel, options->eight_bit_output); + printf("[sender] skipping mount-point dir %s\n", escaped ? escaped : rel); + free(escaped); + fflush(stdout); +} + +/* --debug=filter: a selection/filter decision dropped an entry. */ +static void scanner_note_filter(const ScannerOptions* options, const char* name) { + if (!options || !log_debug_enabled(LOG_DEBUG_FILTER) || !name) + return; + log_debug_message(LOG_DEBUG_FILTER, "filter: excluded %s", name); +} + +/* Account for a directory that will not be represented by an inline directory + * entry. Paired with scanner_dir_count_uncount for empty directories that are + * emitted inline, so every traversed directory is counted exactly once. */ +static void scanner_dir_count_count(const ScannerOptions* options) { + if (options && options->dir_count) + atomic_fetch_add(options->dir_count, 1); +} + +static void scanner_dir_count_uncount(const ScannerOptions* options) { + if (options && options->dir_count) + atomic_fetch_sub(options->dir_count, 1); +} + /* A user-selection exclusion (--filter/-C/per-dir or --exclude/--include). */ static void scanner_record_excluded(DirectoryScanner* scanner, const char* fs_path) { scanner_record_protected(scanner, fs_path, scanner->options.excluded_paths); @@ -1096,6 +1130,9 @@ static bool scanner_emit_empty_dir(DirectoryScanner* scanner, ArrayList* chunk_d file_destroy(dir); return false; } + /* The directory was counted when it was opened; this inline entry represents + it, so drop the counter to avoid counting it twice in --stats. */ + scanner_dir_count_uncount(&scanner->options); return true; } @@ -1185,6 +1222,8 @@ static int open_next_directory(DirectoryScanner* scanner) { scanner->failed = true; return -1; } + scanner_dir_count_count(&scanner->options); + log_debug_message(LOG_DEBUG_FLIST, "flist: scanning %s", scanner->current_path); if (scanner->options.capture_dir_times && !scanner_capture_dir_time( scanner->options.dir_entries, scanner->options.dir_entries_mutex, scanner->root_path, @@ -1692,6 +1731,7 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) { break; } if (!passes_selection) { + scanner_note_filter(&scanner->options, name); free(rel_copy); continue; } @@ -1700,6 +1740,13 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) { free(rel_copy); if (!scanner_same_filesystem(scanner->options.one_file_system, scanner->root_dev, stats.st_dev)) { + if (scanner->options.one_file_system > 1) { + /* rsync's -xx drops the mount-point directory entirely (the plain -x + path below keeps it as an empty directory) and prints the + --info=mount line when that category is enabled. */ + scanner_note_mount(&scanner->options, cur_path); + continue; + } /* rsync's -x/--one-file-system emits the mount-point directory entry itself (so the destination gets an empty directory) but does NOT descend into it. Build a payload-less directory File and hand it to @@ -2113,6 +2160,7 @@ static void scan_root_entry(const ScannerOptions* options, const FilterNode* roo free(prefixed); } if (!passes) { + scanner_note_filter(options, entry->d_name); free(rel); free(cur_path); return; @@ -2120,6 +2168,14 @@ static void scan_root_entry(const ScannerOptions* options, const FilterNode* roo } if (is_dir) { if (!scanner_same_filesystem(options->one_file_system, root_dev, st.st_dev)) { + if (options->one_file_system > 1) { + /* -xx: drop the mount-point directory entirely (rsync) and print the + --info=mount line when enabled. */ + scanner_note_mount(options, cur_path); + free(rel); + free(cur_path); + return; + } /* -x/--one-file-system: emit the mount-point directory entry (empty) but do not descend into it (see the sequential scanner for the same rule). */ File* mount = file_create(cur_path); @@ -2257,6 +2313,7 @@ static bool scan_root_directory(ParallelScanner* ps, const char* root_directory, ps->failed = true; return false; } + log_debug_message(LOG_DEBUG_FLIST, "flist: scanning %s", root_directory); const struct dirent* entry; while ((entry = readdir(dir)) != NULL) { if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) @@ -2421,6 +2478,10 @@ ParallelScanner* parallel_scanner_create_with_options(const char* root_directory parallel_scanner_destroy(ps); return NULL; } + /* The root itself is a traversed directory (rsync counts it in + `Number of files`); the worker DirectoryScanners account for every + subdirectory below it. */ + scanner_dir_count_count(options); /* P7 Wave D: the parallel scanner never runs a DirectoryScanner over the transfer root itself (it hands the root's immediate subdirectories to workers), so capture the root's directory time here. */ diff --git a/src/client/scanner.h b/src/client/scanner.h index 55fc3bb..a9fa3d0 100644 --- a/src/client/scanner.h +++ b/src/client/scanner.h @@ -51,7 +51,7 @@ typedef struct { bool copy_dirlinks; bool munge_links; bool checksum; - bool one_file_system; + int one_file_system; /* Phase 4 special/devices: whether device nodes (--devices) and special files * (--specials) are preserved via recreation, and whether --copy-devices * copies a device's content as an ordinary regular file. */ @@ -130,6 +130,17 @@ typedef struct { /* --info=nonreg: print rsync's `skipping non-regular file "NAME"` line for a * non-regular entry that is not being preserved. Client-only. */ bool note_nonreg; + /* --info=mount: print rsync's `[sender] skipping mount-point dir NAME` when + * -xx drops a mount-point directory. Client-only. */ + bool note_mount; + /* --stats directory accounting for a `-r` run (no -t/-p): a shared counter of + * traversed directories that are NOT otherwise represented by an inline + * directory entry (rsync still counts every directory in `Number of files`). + * Incremented when a directory is opened and decremented when an empty + * directory is emitted inline (so it is counted exactly once). Atomic + * because the parallel scanner's workers share it; NULL disables the + * accounting. Client-only. */ + atomic_ullong* dir_count; /* Source root and 8-bit-output policy used to render a `--info=nonreg` name * relative to the transfer root. Borrowed read-only. */ const char* send_directory; @@ -266,7 +277,7 @@ void directory_scanner_destroy(DirectoryScanner* scanner); /* --one-file-system (-x) decision: a directory entry may be descended into * only when the option is disabled or the entry lives on the same device as * the transfer root. Exposed so tests can exercise the rule directly. */ -bool scanner_same_filesystem(bool one_file_system, dev_t root_device, dev_t entry_device); +bool scanner_same_filesystem(int one_file_system, dev_t root_device, dev_t entry_device); /* Relative path of an on-disk path below `root` ("" == the root itself, NULL * when `fs_path` is not under `root`). Handles trailing slashes and a root of diff --git a/src/shared/config.c b/src/shared/config.c index d0e25a8..2a2941b 100644 --- a/src/shared/config.c +++ b/src/shared/config.c @@ -79,7 +79,7 @@ static void config_set_defaults(Config* config) { config->cvs_exclude = false; config->per_dir_filter = false; config->per_dir_filter_count = 0; - config->one_file_system = false; + config->one_file_system = 0; config->no_implied_dirs = false; config->dirs = false; config->rsh_command = NULL; diff --git a/src/shared/config.h b/src/shared/config.h index 2eb6d3b..c4d7130 100644 --- a/src/shared/config.h +++ b/src/shared/config.h @@ -463,7 +463,9 @@ typedef struct Config { * /.rsync-filter' (the .rsync-filter files themselves are transferred); a * repeated -F adds --filter='- .rsync-filter' so they are excluded too. */ int per_dir_filter_count; - bool one_file_system; /* -x/--one-file-system: do not cross filesystem boundaries */ + int one_file_system; /* -x/--one-file-system: do not cross filesystem boundaries. + Repeated -x (rsync's -xx) drops the mount-point + directory entirely instead of recreating it empty. */ /* --no-implied-dirs: client-only. With -R, do not transfer the source * metadata of the parent directories implied by a listed path; an unlisted * implied parent is still created (with default attributes) so the listed diff --git a/src/shared/log.h b/src/shared/log.h index 52cf38f..b3a258c 100644 --- a/src/shared/log.h +++ b/src/shared/log.h @@ -13,7 +13,19 @@ typedef enum { LOG_DEBUG_PROTO = 1u << 1, LOG_DEBUG_PACK = 1u << 2, LOG_DEBUG_UTIL = 1u << 3, - LOG_DEBUG_ALL = (1u << 4) - 1, + /* rsync --debug categories that now map to a natural FastSync event: + * flist (file-list scan progress), del (deletions), hash/deltasum + * (whole-file hashing and delta-sum generation), recv (receiver + * responses/signatures), filter (selection/exclusion decisions) and send + * (files handed to the sender). Only emitted when the category is + * explicitly enabled; a normal run stays silent. */ + LOG_DEBUG_FLIST = 1u << 4, + LOG_DEBUG_DEL = 1u << 5, + LOG_DEBUG_HASH = 1u << 6, + LOG_DEBUG_RECV = 1u << 7, + LOG_DEBUG_FILTER = 1u << 8, + LOG_DEBUG_SEND = 1u << 9, + LOG_DEBUG_ALL = (1u << 10) - 1, } LogDebugFlag; typedef enum { @@ -38,9 +50,13 @@ typedef enum { the info_level bitset (there is no separate Config field) and is never set by --info=all (which selects level 1). */ LOG_INFO_NAME_UPTODATE = 1u << 10, + /* --info=mount: print rsync's `[sender] skipping mount-point dir NAME` when + * -xx/--one-file-system drops a mount-point directory (FastSync's client is + * the sender). */ + LOG_INFO_MOUNT = 1u << 11, LOG_INFO_ALL = LOG_INFO_COPY | LOG_INFO_MISC | LOG_INFO_SKIP | LOG_INFO_STATS | LOG_INFO_DEL | LOG_INFO_REMOVE | LOG_INFO_NAME | LOG_INFO_FLIST | LOG_INFO_NONREG | - LOG_INFO_PROGRESS, + LOG_INFO_PROGRESS | LOG_INFO_MOUNT, } LogInfoFlag; void log_message(LogLevel log_level, const char* message, ...); diff --git a/src/shared/multiprocessing.c b/src/shared/multiprocessing.c index c45109a..307c736 100644 --- a/src/shared/multiprocessing.c +++ b/src/shared/multiprocessing.c @@ -50,6 +50,7 @@ PipelineContextSender* pipeline_context_sender_create(Config* config, Queue* que protocol_session_set_max_alloc(&context->allocation_session, config->max_alloc); context->dir_entries = NULL; context->dir_entries_mutex_init = false; + atomic_init(&context->dir_count, 0); context->delete_limit = false; int init = 0; if (config->use_metadata) { diff --git a/src/shared/multiprocessing.h b/src/shared/multiprocessing.h index a6fda8c..e53c542 100644 --- a/src/shared/multiprocessing.h +++ b/src/shared/multiprocessing.h @@ -119,6 +119,10 @@ typedef struct { ArrayList* dir_entries; mtx_t dir_entries_mutex; bool dir_entries_mutex_init; + /* --stats directory accounting for a `-r` scan (no directory metadata): + shared by the parallel scanner workers, read by the sender thread once the + scanner is done. See ScannerOptions.dir_count. */ + atomic_ullong dir_count; /* Set by the sender thread when the receiver reported a --max-delete-capped deletion (STATUS_DELETE_LIMIT): the transfer succeeded and the process must exit 25 like rsync. Read by the caller after the sender thread is joined. */ diff --git a/tests/integration/test_output_parity.py b/tests/integration/test_output_parity.py index 8cae816..0710d78 100644 --- a/tests/integration/test_output_parity.py +++ b/tests/integration/test_output_parity.py @@ -662,6 +662,46 @@ class TestWireStatsParity: assert re.match(r"Number of created files: 1 \(reg: 1\)$", r_created), r_created assert f_created == r_created, (r_created, f_created) + @requires_rsync + @pytest.mark.ci + @pytest.mark.parametrize("mt", [False, True]) + def test_stats_r_directory_breakdown_matches_rsync(self, shared_server, mt): + """A recursive `-r` scan (no -t/-p) exposes no directory metadata, but + rsync still counts every directory in `Number of files`; the sender's + lightweight directory counter must reproduce the `dir: N` category.""" + source = os.path.join(TEST_DATA_DIR, "wire_stdir_src") + dest = os.path.join(TEST_DATA_DIR, "wire_stdir_dst") + rdst = os.path.join(TEST_DATA_DIR, "wire_stdir_rdst") + clean_dir(source) + clean_dir(dest) + clean_dir(rdst) + os.makedirs(os.path.join(source, "sub", "deep")) + os.makedirs(os.path.join(source, "empty")) + for rel in ("a.txt", os.path.join("sub", "b.txt"), os.path.join("sub", "deep", "c.txt")): + with open(os.path.join(source, rel), "wb") as fh: + fh.write(b"x\n") + os.makedirs(get_dest_received_dir(dest, source), exist_ok=True) + + rsync_result = _rsync(["-r", "--stats", source + "/", rdst + "/"]) + assert rsync_result.returncode == 0, rsync_result.stderr + flags = ["-r", "--stats"] + (["--threads"] if mt else []) + result, _ = run_client(source, dest, flags=flags, port=shared_server.port) + assert result.returncode == 0, result.stderr[:300] + + def stats_line(text, key): + for line in text.splitlines(): + if line.startswith(key + ":"): + return line + return None + + r_files = stats_line(rsync_result.stdout, "Number of files") + f_files = stats_line(result.stdout, "Number of files") + # 3 regular files, 4 directories (root, sub, sub/deep, empty). + assert re.match(r"Number of files: 7 \(reg: 3, dir: 4\)$", r_files), r_files + assert f_files == r_files, (r_files, f_files) + assert (stats_line(result.stdout, "Number of regular files transferred") == + stats_line(rsync_result.stdout, "Number of regular files transferred")) + @requires_rsync @pytest.mark.ci @pytest.mark.parametrize("mt", [False, True]) diff --git a/tests/integration/test_parity_debug.py b/tests/integration/test_parity_debug.py new file mode 100644 index 0000000..b094887 --- /dev/null +++ b/tests/integration/test_parity_debug.py @@ -0,0 +1,105 @@ +"""`--debug=FLAGS` natural-event categories (no-wire). + +FastSync maps the rsync `--debug` categories that correspond to a real event it +already performs (``flist``, ``del``, ``hash``/``deltasum``, ``recv``, +``filter`` and ``send``) onto debug output. A normal run prints none of it. +""" +import os +import sys + +import pytest + +sys.path.insert(0, os.path.dirname(__file__)) +from common import TEST_DATA_DIR, run_client, clean_dir, get_dest_received_dir, ServerManager + + +def _make_tree(root): + clean_dir(root) + os.makedirs(os.path.join(root, "sub")) + with open(os.path.join(root, "a.txt"), "wb") as fh: + fh.write(b"alpha\n") + with open(os.path.join(root, "keep.log"), "wb") as fh: + fh.write(b"log\n") + with open(os.path.join(root, "sub", "b.txt"), "wb") as fh: + fh.write(b"beta\n") + + +@pytest.mark.ci +def test_debug_flist_and_send_emit_output(shared_server): + """`--debug=flist,send` produces category-tagged debug output.""" + source = os.path.join(TEST_DATA_DIR, "dbg_src") + dest = os.path.join(TEST_DATA_DIR, "dbg_dst") + _make_tree(source) + clean_dir(dest) + result, _ = run_client(source, dest, flags=["-a", "--debug=flist,send"], + port=shared_server.port) + assert result.returncode == 0, (result.stderr or result.stdout)[:300] + assert "flist: scanning" in result.stdout, result.stdout + assert "send: " in result.stdout, result.stdout + + +@pytest.mark.ci +def test_debug_filter_emits_excluded_entry(shared_server): + source = os.path.join(TEST_DATA_DIR, "dbg_filter_src") + dest = os.path.join(TEST_DATA_DIR, "dbg_filter_dst") + _make_tree(source) + clean_dir(dest) + result, _ = run_client(source, dest, + flags=["-a", "--debug=filter", "--exclude=*.log"], + port=shared_server.port) + assert result.returncode == 0, (result.stderr or result.stdout)[:300] + assert "filter: excluded keep.log" in result.stdout, result.stdout + + +@pytest.mark.ci +def test_debug_hash_and_recv_emit_on_incremental(shared_server): + source = os.path.join(TEST_DATA_DIR, "dbg_hash_src") + dest = os.path.join(TEST_DATA_DIR, "dbg_hash_dst") + _make_tree(source) + clean_dir(dest) + result, _ = run_client(source, dest, + flags=["-a", "--incremental", "--checksum", + "--debug=hash,recv"], + port=shared_server.port) + assert result.returncode == 0, (result.stderr or result.stdout)[:300] + assert "hash: " in result.stdout, result.stdout + assert "recv: " in result.stdout, result.stdout + + +@pytest.mark.ci +def test_debug_del_emits_deleted_path(): + """`--debug=del` reports the paths the receiver actually removed. + + A deletion-capable server is required (the shared fixture refuses + client-requested deletion).""" + source = os.path.join(TEST_DATA_DIR, "dbg_del_src") + dest = os.path.join(TEST_DATA_DIR, "dbg_del_dst") + _make_tree(source) + clean_dir(dest) + seeded = get_dest_received_dir(dest, source) + os.makedirs(seeded) + with open(os.path.join(seeded, "extra.tmp"), "wb") as fh: + fh.write(b"stale\n") + server = ServerManager() + server.start(extra_args=["--allow-super", "--allow-delete"]) + try: + result, _ = run_client(source, dest, flags=["-a", "--delete", "--debug=del"], + port=server.port) + finally: + server.stop() + assert result.returncode == 0, (result.stderr or result.stdout)[:300] + assert "del: " in result.stdout and "extra.tmp" in result.stdout, result.stdout + assert not os.path.exists(os.path.join(seeded, "extra.tmp")) + + +@pytest.mark.ci +def test_normal_run_has_no_debug_output(shared_server): + source = os.path.join(TEST_DATA_DIR, "dbg_quiet_src") + dest = os.path.join(TEST_DATA_DIR, "dbg_quiet_dst") + _make_tree(source) + clean_dir(dest) + result, _ = run_client(source, dest, flags=["-a"], port=shared_server.port) + assert result.returncode == 0, (result.stderr or result.stdout)[:300] + assert "[DEBUG]" not in result.stdout + assert "flist: scanning" not in result.stdout + assert "send: " not in result.stdout diff --git a/tests/integration/test_parity_info_mount_stats.py b/tests/integration/test_parity_info_mount_stats.py new file mode 100644 index 0000000..fa9e50e --- /dev/null +++ b/tests/integration/test_parity_info_mount_stats.py @@ -0,0 +1,174 @@ +"""Differential parity for `--info=mount` and `--info=stats` (no-wire). + +Both behaviours are compared against real rsync 3.4.1: + +* `--info=mount` prints rsync's ``[sender] skipping mount-point dir NAME`` line + when ``-xx`` drops a mount-point directory. Plain ``-x`` keeps the empty + directory and stays silent, exactly like rsync. +* `--info=stats` requests the same transfer-statistics block as `--stats` + (rsync spells the full block ``--info=stats2``/``--stats``). + +The tests are skipped when rsync is unavailable. +""" +import os +import re +import shutil +import subprocess +import sys + +import pytest + +sys.path.insert(0, os.path.dirname(__file__)) +from common import TEST_DATA_DIR, run_client, clean_dir, get_dest_received_dir + +RSYNC = shutil.which("rsync") +requires_rsync = pytest.mark.skipif(RSYNC is None, reason="rsync 3.4.1 not installed") + + +def _rsync(args): + env = dict(os.environ, LC_ALL="C") + return subprocess.run([RSYNC] + args, capture_output=True, text=True, env=env, timeout=120) + + +def _cross_device_mount_tree(source): + """Build a source whose ``nested_link`` is a symlink onto a tmpfs directory. + + ``--copy-links`` dereferences it so ``-x`` sees a mount-point directory on a + different device. Returns the probe path to remove, or skips the test when + no cross-device filesystem is available. + """ + local = os.stat(".") + shm = "/dev/shm" + try: + shm_stat = os.stat(shm) + except OSError: + pytest.skip("/dev/shm not available") + if shm_stat.st_dev == local.st_dev: + pytest.skip("no cross-device filesystem available") + + clean_dir(source) + with open(os.path.join(source, "keep.txt"), "wb") as fh: + fh.write(b"keep\n") + probe = os.path.join(shm, f"fastsync_info_mount_{os.getpid()}") + shutil.rmtree(probe, ignore_errors=True) + os.makedirs(probe) + with open(os.path.join(probe, "inside.txt"), "wb") as fh: + fh.write(b"cross\n") + try: + os.symlink(probe, os.path.join(source, "nested_link")) + except OSError: + shutil.rmtree(probe, ignore_errors=True) + pytest.skip("cannot create symlink") + return probe + + +@requires_rsync +@pytest.mark.ci +def test_info_mount_xx_matches_rsync(shared_server): + """`-xx --info=mount` drops the mount-point dir and prints rsync's line.""" + source = os.path.join(TEST_DATA_DIR, "info_mount_src") + dest = os.path.join(TEST_DATA_DIR, "info_mount_dst") + rdst = os.path.join(TEST_DATA_DIR, "info_mount_rdst") + probe = _cross_device_mount_tree(source) + clean_dir(dest) + clean_dir(rdst) + flags = ["-a", "--copy-links", "-xx", "--info=mount"] + try: + rsync_result = _rsync(flags + [source + "/", rdst + "/"]) + assert rsync_result.returncode == 0, rsync_result.stderr + result, _ = run_client(source, dest, flags=flags, port=shared_server.port) + assert result.returncode == 0, (result.stderr or result.stdout)[:300] + + expected = "[sender] skipping mount-point dir nested_link" + assert expected in rsync_result.stdout, rsync_result.stdout + assert expected in result.stdout, (result.stdout, result.stderr) + + received = get_dest_received_dir(dest, source) + assert os.path.exists(os.path.join(received, "keep.txt")) + # -xx omits the mount-point directory entirely. + assert not os.path.exists(os.path.join(received, "nested_link")) + assert not os.path.exists(os.path.join(rdst, "nested_link")) + finally: + shutil.rmtree(probe, ignore_errors=True) + + +@requires_rsync +@pytest.mark.ci +def test_info_mount_single_x_is_silent(shared_server): + """Plain `-x` keeps the empty mount-point directory and prints no line.""" + source = os.path.join(TEST_DATA_DIR, "info_mount1_src") + dest = os.path.join(TEST_DATA_DIR, "info_mount1_dst") + rdst = os.path.join(TEST_DATA_DIR, "info_mount1_rdst") + probe = _cross_device_mount_tree(source) + clean_dir(dest) + clean_dir(rdst) + flags = ["-a", "--copy-links", "-x", "--info=mount"] + try: + rsync_result = _rsync(flags + [source + "/", rdst + "/"]) + assert rsync_result.returncode == 0, rsync_result.stderr + result, _ = run_client(source, dest, flags=flags, port=shared_server.port) + assert result.returncode == 0, (result.stderr or result.stdout)[:300] + + assert "skipping mount-point dir" not in rsync_result.stdout + assert "skipping mount-point dir" not in result.stdout + + received = get_dest_received_dir(dest, source) + assert os.path.isdir(os.path.join(received, "nested_link")) + assert not os.path.exists(os.path.join(received, "nested_link", "inside.txt")) + assert os.path.isdir(os.path.join(rdst, "nested_link")) + assert not os.path.exists(os.path.join(rdst, "nested_link", "inside.txt")) + finally: + shutil.rmtree(probe, ignore_errors=True) + + +def _make_stats_tree(root): + clean_dir(root) + os.makedirs(os.path.join(root, "sub")) + with open(os.path.join(root, "a.txt"), "wb") as fh: + fh.write(b"alpha\n") + with open(os.path.join(root, "sub", "b.txt"), "wb") as fh: + fh.write(b"beta\n") + + +def _pick_stats(text): + keys = ("Number of files", "Number of regular files transferred", "Total file size", + "Total transferred file size", "Literal data", "Matched data") + out = {} + for line in text.splitlines(): + for key in keys: + if line.startswith(key + ":"): + out[key] = line + return out + + +@requires_rsync +@pytest.mark.ci +def test_info_stats_emits_full_stats_block(shared_server): + """`--info=stats` is the same full block as `--stats` and matches rsync.""" + source = os.path.join(TEST_DATA_DIR, "info_stats_src") + dest = os.path.join(TEST_DATA_DIR, "info_stats_dst") + rdst = os.path.join(TEST_DATA_DIR, "info_stats_rdst") + dest2 = os.path.join(TEST_DATA_DIR, "info_stats_dst2") + _make_stats_tree(source) + for path in (dest, rdst, dest2): + clean_dir(path) + os.makedirs(get_dest_received_dir(path, source), exist_ok=True) + + rsync_result = _rsync(["-a", "--stats", source + "/", rdst + "/"]) + assert rsync_result.returncode == 0, rsync_result.stderr + + info_result, _ = run_client(source, dest, flags=["-a", "--info=stats"], + port=shared_server.port) + assert info_result.returncode == 0, (info_result.stderr or info_result.stdout)[:300] + stats_result, _ = run_client(source, dest2, flags=["-a", "--stats"], + port=shared_server.port) + assert stats_result.returncode == 0, (stats_result.stderr or stats_result.stdout)[:300] + + # --info=stats must print the same block as --stats... + assert _pick_stats(info_result.stdout) == _pick_stats(stats_result.stdout), ( + f"info={info_result.stdout} stats={stats_result.stdout}") + # ...and the protocol-independent counters must match real rsync. + assert _pick_stats(info_result.stdout) == _pick_stats(rsync_result.stdout), ( + f"rsync={_pick_stats(rsync_result.stdout)} fastsync={_pick_stats(info_result.stdout)}") + assert re.search(r"^Number of files: \d+ \(reg: 2, dir: 2\)$", info_result.stdout, + re.MULTILINE), info_result.stdout diff --git a/tests/test_client_cli.c b/tests/test_client_cli.c index 468987b..b511dd5 100644 --- a/tests/test_client_cli.c +++ b/tests/test_client_cli.c @@ -762,8 +762,9 @@ static void test_parse_args_debug_flags() { int positional_count = 0; EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0); - EXPECT_EQ_INT(cfg->debug_level, LOG_DEBUG_ALL); - EXPECT_EQ_INT(get_log_debug_flags(), LOG_DEBUG_ALL); + EXPECT_EQ_INT(cfg->debug_level, LOG_DEBUG_IO | LOG_DEBUG_PROTO | LOG_DEBUG_PACK | LOG_DEBUG_UTIL); + EXPECT_EQ_INT(get_log_debug_flags(), + LOG_DEBUG_IO | LOG_DEBUG_PROTO | LOG_DEBUG_PACK | LOG_DEBUG_UTIL); config_delete(cfg); } @@ -1421,10 +1422,9 @@ static void test_parse_args_info_name_and_help() { config_delete(cfg); } -/* rsync 3.4.1's full --info/--debug vocabulary parses. The info categories - * with a FastSync event set their flag; the remaining rsync-only categories - * (backup/mount/symsafe/syms) parse but stay silent. Every --debug category - * listed here is FastSync-silent, so debug_level stays 0. */ +/* rsync 3.4.1's full --info/--debug vocabulary parses. The categories with a + * FastSync event set their flag; the remaining rsync-only categories + * (backup/symsafe/syms, acl/bind/chdir/...) parse but stay silent. */ static void test_parse_args_rsync_flag_vocabulary_accepted() { Config* cfg = config_create(); char* argv[] = {"fastsync", "--info=backup,del,flist,mount,nonreg,progress,remove,symsafe,syms", @@ -1436,9 +1436,10 @@ static void test_parse_args_rsync_flag_vocabulary_accepted() { int positional_count = 0; EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0); - EXPECT_EQ_INT(cfg->info_level, LOG_INFO_DEL | LOG_INFO_FLIST | LOG_INFO_NONREG | + EXPECT_EQ_INT(cfg->info_level, LOG_INFO_DEL | LOG_INFO_FLIST | LOG_INFO_MOUNT | LOG_INFO_NONREG | LOG_INFO_PROGRESS | LOG_INFO_REMOVE); - EXPECT_EQ_INT(cfg->debug_level, 0); + EXPECT_EQ_INT(cfg->debug_level, LOG_DEBUG_DEL | LOG_DEBUG_FLIST | LOG_DEBUG_HASH | + LOG_DEBUG_RECV | LOG_DEBUG_FILTER | LOG_DEBUG_SEND); config_delete(cfg); }