From f3ac4df4d04dacfbc317e8c5a882919365301392 Mon Sep 17 00:00:00 2001 From: TapTap Date: Fri, 18 Sep 2026 20:44:24 +0200 Subject: [PATCH] fix(parity): rsync bwlimit units, --info categories, --ignore-errors deletion semantics - --bwlimit: faithful port of rsync 3.4.1 parse_size_arg (default KiB/s, binary K/M/G/T/P, decimal KB/MB, KiB/MiB, decimals, 0 = unlimited, 512-byte floor, (size+512)/1024 quantization). Unit tests + docs. - --info: wire del/remove/name/flist/nonreg/progress to real FastSync events in rsync's line format (deleting PATH, sender removed NAME, name lines, 'sending incremental file list', skipping non-regular file "NAME"); name no longer aliases copy; --info=progress drives the progress path and report_stats. - --ignore-errors: match rsync's default -- a source I/O error skips deletion unless --ignore-errors, while the readable tree still transfers and the run exits 23. Covers all delete timings and both send paths. --- src/client/change_list.c | 35 +++++++- src/client/client_cli.c | 134 ++++++++++++++++++++++++---- src/client/client_send.c | 136 ++++++++++++++++++++++++----- src/client/scanner.c | 40 ++++++++- src/client/scanner.h | 18 +++- src/client/usage.c | 4 +- src/shared/log.h | 15 +++- src/shared/multiprocessing.h | 5 ++ src/shared/protocol.c | 4 + src/shared/protocol.h | 1 + tests/integration/test_features.py | 20 +++-- tests/test_client_cli.c | 73 ++++++++++++++-- 12 files changed, 427 insertions(+), 58 deletions(-) diff --git a/src/client/change_list.c b/src/client/change_list.c index e6889ef..99d0b35 100644 --- a/src/client/change_list.c +++ b/src/client/change_list.c @@ -1,5 +1,6 @@ #include "change_list.h" #include "checksum.h" +#include "log.h" #include "utils.h" #include #include @@ -69,8 +70,10 @@ static bool strbuf_append(StrBuf* buf, const char* text) { } bool change_list_enabled(const Config* config) { - return config != NULL && (config->itemize_changes || config->out_format != NULL || - (config->log_file != NULL && config->log_file_format != NULL)); + return config != NULL && + (config->itemize_changes || config->out_format != NULL || + (config->log_file != NULL && config->log_file_format != NULL) || + (config->info_level & LOG_INFO_NAME) != 0); } /* ---- Itemize code ---- */ @@ -192,6 +195,24 @@ char* change_render_itemize(const Config* config, const ChangeEvent* event) { return line.data; } +/* rsync's `--info=name` line for an updated entry: the transfer-relative name + * (trailing slash for directories) plus the ` -> target` / ` => target` link + * suffix. `--info=name` does not alter an itemize/out-format run. */ +static char* change_render_name(const ChangeEvent* event) { + StrBuf line = {0}; + bool ok = append_name(&line, event) && append_link_suffix(&line, event); + if (!ok) { + strbuf_free(&line); + return NULL; + } + if (line.data == NULL) { + line.data = str_dup(""); + if (!line.data) + return NULL; + } + return line.data; +} + /* ---- --out-format / --log-file-format ---- */ /* rsync 3.4.1's `%C` uses the negotiated TRANSFER checksum (the first name of a @@ -453,6 +474,16 @@ void change_emit(const Config* config, const ChangeEvent* event) { print_escaped_line(stdout, line, config->eight_bit_output); free(line); } + } else if ((config->info_level & LOG_INFO_NAME) != 0 && + !(config->show_progress || (config->info_level & LOG_INFO_PROGRESS))) { + /* --info=name without -i/--out-format: print the updated entry's name. The + --progress path owns the name line when progress output is active (it + emits the same names before the progress frames), so do not duplicate. */ + char* line = change_render_name(event); + if (line != NULL) { + print_escaped_line(stdout, line, config->eight_bit_output); + free(line); + } } if (to_log) { char* line = change_render_format(config->log_file_format, config, event); diff --git a/src/client/client_cli.c b/src/client/client_cli.c index 6680bcb..a0b93ad 100644 --- a/src/client/client_cli.c +++ b/src/client/client_cli.c @@ -501,7 +501,7 @@ static bool is_accepted_debug_category(const char* name) { static bool is_accepted_info_category(const char* name) { static const char* const categories[] = { - "backup", "del", "flist", "mount", "nonreg", "progress", "remove", "syms", "symsafe", + "mount", "syms", "symsafe", }; for (size_t i = 0; i < sizeof(categories) / sizeof(categories[0]); i++) { if (strcmp(name, categories[i]) == 0) @@ -608,14 +608,28 @@ static int parse_info_flags(const char* value, Config* config) { free(flags); return 1; } - if (strcmp(name, "copy") == 0 || strcmp(name, "name") == 0) + if (strcmp(name, "copy") == 0) flag = LOG_INFO_COPY; + else if (strcmp(name, "name") == 0) + flag = LOG_INFO_NAME; else if (strcmp(name, "misc") == 0) flag = LOG_INFO_MISC; else if (strcmp(name, "skip") == 0) flag = LOG_INFO_SKIP; else if (strcmp(name, "stats") == 0) flag = LOG_INFO_STATS; + else if (strcmp(name, "del") == 0) + flag = LOG_INFO_DEL; + else if (strcmp(name, "remove") == 0) + flag = LOG_INFO_REMOVE; + else if (strcmp(name, "flist") == 0) + flag = LOG_INFO_FLIST; + else if (strcmp(name, "nonreg") == 0) + flag = LOG_INFO_NONREG; + else if (strcmp(name, "backup") == 0) + flag = LOG_INFO_BACKUP; + else if (strcmp(name, "progress") == 0) + flag = LOG_INFO_PROGRESS; else if (is_accepted_info_category(name)) continue; else { @@ -1819,22 +1833,107 @@ static int set_log_file_option(Config* config, const char* log_path) { return 0; } -/* Apply a --bwlimit value (kilobytes per second). Returns 0 on success, -1 on - * error. */ +/* Faithful port of rsync 3.4.1's `parse_size_arg(bwlimit_arg, 'K', "bwlimit", + * 512, -1, True)`: a default KiB suffix, binary (1024) multipliers unless a + * `b`/`B` decimal suffix or explicit `iB` is given, an optional decimal + * fraction, the P/T/G/M/K suffixes, and the special rules that a value of 0 + * means "no limit" while any other value below 512 bytes is rejected. The + * parsed byte count is then quantized to whole KiB exactly like rsync's + * `bwlimit = (size + 512) / 1024`. Returns 0 on success, -1 on a parse error. */ +static int parse_bwlimit_value(const char* value, unsigned long long* bytes_per_sec_out) { + const char* arg = value; + int reps; + long long mult; + while (*arg >= '0' && *arg <= '9') + arg++; + if (*arg != '\0' && + (*arg == '.' || *arg == localeconv()->decimal_point[0])) + for (arg++; *arg >= '0' && *arg <= '9'; arg++) {} + + char suffix = *arg && *arg != '+' && *arg != '-' ? *arg++ : 'K'; + switch (suffix) { + case 'b': + case 'B': + reps = 0; + break; + case 'k': + case 'K': + reps = 1; + break; + case 'm': + case 'M': + reps = 2; + break; + case 'g': + case 'G': + reps = 3; + break; + case 't': + case 'T': + reps = 4; + break; + case 'p': + case 'P': + reps = 5; + break; + default: + log_message(LOG_LEVEL_ERROR, "--bwlimit=%s is invalid", value ? value : ""); + return -1; + } + if (*arg == 'b' || *arg == 'B') { + mult = 1000; + arg++; + } else if (*arg == '\0' || *arg == '+' || *arg == '-') { + mult = 1024; + } else if ((arg[0] == 'i' || arg[0] == 'I') && (arg[1] == 'b' || arg[1] == 'B')) { + mult = 1024; + arg += 2; + } else { + log_message(LOG_LEVEL_ERROR, "--bwlimit=%s is invalid", value ? value : ""); + return -1; + } + + long long size = 1; + for (int i = 0; i < reps; i++) { + if (size > LLONG_MAX / mult) { + log_message(LOG_LEVEL_ERROR, "--bwlimit=%s is too large", value ? value : ""); + return -1; + } + size *= mult; + } + size = (long long)((double)size * atof(value)); + if ((*arg == '+' || *arg == '-') && arg[1] == '1' && arg != value) { + size += atoi(arg); + arg += 2; + } + if (*arg != '\0' || size < 0) { + log_message(LOG_LEVEL_ERROR, "--bwlimit=%s is %s", value ? value : "", + size < 0 ? "too large" : "invalid"); + return -1; + } + if (size != 0 && size < 512) { + log_message(LOG_LEVEL_ERROR, "--bwlimit=%s is too small (min: 512 or 0 for unlimited)", + value ? value : ""); + return -1; + } + long long kib = size == 0 ? 0 : (size + 512) / 1024; + if (kib > (long long)(ULLONG_MAX / 1024)) { + log_message(LOG_LEVEL_ERROR, "--bwlimit=%s is too large", value ? value : ""); + return -1; + } + *bytes_per_sec_out = (unsigned long long)kib * 1024; + return 0; +} + +/* Apply a --bwlimit value using rsync 3.4.1's units/semantics. Returns 0 on + * success, -1 on error. */ static int set_bwlimit_option(const char* value) { - unsigned long long kbps; - if (parse_ull_arg(value, &kbps, "--bwlimit") != 0) + unsigned long long bytes_per_sec; + if (parse_bwlimit_value(value, &bytes_per_sec) != 0) return -1; - if (kbps == 0) { - log_message(LOG_LEVEL_ERROR, "--bwlimit must be a positive integer"); - return -1; - } - if (kbps > ULLONG_MAX / 1024) { - log_message(LOG_LEVEL_ERROR, "--bwlimit value too large"); - return -1; - } - io_set_bwlimit(kbps * 1024); - log_info_message(LOG_INFO_MISC, "Set bandwidth limit to %llu KB/s", kbps); + io_set_bwlimit(bytes_per_sec); + log_info_message(LOG_INFO_MISC, "Set bandwidth limit to %llu KB/s", + bytes_per_sec / 1024); return 0; } @@ -2549,7 +2648,8 @@ static int cli_finalize_config(Config* config, bool verbose, bool no_delta, bool } } } - config->report_stats = config->stats || config->show_progress || format_needs_wire || + config->report_stats = config->stats || config->show_progress || + (config->info_level & LOG_INFO_PROGRESS) || format_needs_wire || (config->dry_run && config->use_delete); return 0; } diff --git a/src/client/client_send.c b/src/client/client_send.c index 676cdd7..253a2c2 100644 --- a/src/client/client_send.c +++ b/src/client/client_send.c @@ -66,6 +66,16 @@ static void log_server_rejection(const char* context) { } } +/* rsync's --ignore-errors semantics: an I/O error during the transfer normally + * suppresses deletion entirely ("IO error encountered -- skipping file + * deletion"); --ignore-errors lets the deletion run anyway. FastSync always + * continues past an unreadable subdirectory so the readable tree transfers, and + * always reports the partial transfer (exit 23); this only decides whether the + * deletion phase is skipped. Returns true when deletion may proceed. */ +static bool ignore_errors_allows_delete(const Config* config, bool had_io_error) { + return !had_io_error || (config && config->ignore_errors); +} + static const char* display_bytes(unsigned long long bytes, bool human_readable, char* buffer, size_t buffer_size) { if (human_readable && format_human_size_decimal(bytes, buffer, buffer_size)) @@ -293,12 +303,23 @@ static void progress_final_frame(unsigned long long size, char* out, size_t out_ rate, units, rembuf, g_progress_xferred, to_chk, total); } +static bool info_flag_enabled(const Config* config, LogInfoFlag flag) { + return config != NULL && (config->info_level & flag) != 0; +} + static void client_progress_begin(const Config* config) { - g_progress_active = config->show_progress && !config->quiet; + g_progress_active = (config->show_progress || info_flag_enabled(config, LOG_INFO_PROGRESS)) && + !config->quiet; g_progress_xferred = 0; g_progress_seen = 0; - if (!g_progress_active) + if (!g_progress_active) { + /* `--info=flist` prints rsync's file-list header even without progress. */ + if (!config->quiet && info_flag_enabled(config, LOG_INFO_FLIST)) { + printf("sending incremental file list\n"); + fflush(stdout); + } return; + } printf("sending incremental file list\n"); /* rsync prints the transfer-root directory's name before the first file when that directory is created; FastSync mirrors the source root below the @@ -428,6 +449,9 @@ static bool prepare_scanner(const Config* config, int num_threads, PreparedScann options->prune_empty_dirs = config->prune_empty_dirs; options->ignore_io_errors = config->ignore_errors; options->ignore_missing_args = config->ignore_missing_args || config->delete_missing_args; + options->note_nonreg = (config->info_level & LOG_INFO_NONREG) != 0 && !config->quiet; + options->send_directory = config->send_directory; + options->eight_bit_output = config->eight_bit_output; options->excluded_paths = NULL; options->excluded_mutex = NULL; options->size_skipped_paths = NULL; @@ -936,6 +960,8 @@ static void source_file_destroy(void* item) { } } +static const char* delete_display_path(const Config* config, const char* path); + /* Remove only the same regular source file that was sent. */ static void remove_transferred_sources(const Config* config, ArrayList* paths) { if (!config->remove_source_files || !paths) @@ -973,6 +999,13 @@ static void remove_transferred_sources(const Config* config, ArrayList* paths) { log_message(LOG_LEVEL_WARNING, "Could not remove source file %s", escaped_path ? escaped_path : ""); free(escaped_path); + } else if (info_flag_enabled(config, LOG_INFO_REMOVE) && !config->quiet) { + /* rsync's --info=remove line: the transfer-relative name. */ + const char* rel = delete_display_path(config, source->path); + char* escaped = output_escape(rel, config->eight_bit_output); + printf("sender removed %s\n", escaped ? escaped : rel); + free(escaped); + fflush(stdout); } close(dirfd); } @@ -1540,8 +1573,11 @@ static bool scan_paths_only(const Config* config, const ScannerOptions* options, ArrayList* manifest, DeletePlanSender* plans, bool* io_error_out) { if (io_error_out) *io_error_out = false; - DirectoryScanner* scanner = - directory_scanner_create_with_options(config->send_directory, options); + ScannerOptions local = *options; + /* The pre-scan is a paths-only pass with no client output; it must not emit + --info=nonreg lines (the data pass does that once). */ + local.note_nonreg = false; + DirectoryScanner* scanner = directory_scanner_create_with_options(config->send_directory, &local); if (!scanner) return false; bool ok = true; @@ -2045,10 +2081,10 @@ static int send_dry_run_remote(Config* config) { array_list_delete(would_delete); goto dry_fail; } - /* rsync prints `*deleting PATH` when itemizing (or `deleting PATH` with - --out-format / -v); the plain-total output used here has no delete - counterpart, so only the itemize/out-format cases are rendered. */ - if (!config->quiet && (config->itemize_changes || config->out_format != NULL)) { + /* rsync prints `*deleting PATH` when itemizing, `deleting PATH` under + --info=del/--info=remove, and the --out-format expansion when set. */ + if (!config->quiet && (config->itemize_changes || config->out_format != NULL || + info_flag_enabled(config, LOG_INFO_DEL))) { for (int i = 0; i < would_delete->size; i++) { const char* raw = (const char*)would_delete->items[i]; const char* path = delete_display_path(config, raw); @@ -2071,7 +2107,10 @@ static int send_dry_run_remote(Config* config) { } } else { char* escaped = output_escape(path, config->eight_bit_output); - printf("*deleting %s\n", escaped ? escaped : path); + if (config->itemize_changes) + printf("*deleting %s\n", escaped ? escaped : path); + else + printf("deleting %s\n", escaped ? escaped : path); free(escaped); } } @@ -2599,7 +2638,8 @@ static int send_chunks_multithreaded(void* pipeline_context) { "unscanned source mirrors are not deleted"); else log_message(LOG_LEVEL_WARNING, "transfer stopped early (stop deadline)"); - } else if (context->config->use_delete && !context->early_delete && !context->delete_plans) { + } else if (context->config->use_delete && !context->early_delete && !context->delete_plans && + !context->delete_suppressed) { /* Empty keep-set + scan I/O error must not delete the whole destination (the source may not be genuinely empty -- see send_files). */ bool empty_io; @@ -2612,12 +2652,20 @@ static int send_chunks_multithreaded(void* pipeline_context) { "with an empty keep-set (--delete)"); goto send_fail; } - if (send_delete_manifest(client->file_descriptor, context->manifest, context->excluded_paths, - context->size_skipped_paths, context->missing_args, - context->synced_dirs) != 0) + /* rsync default: an I/O error suppresses deletion unless --ignore-errors. + The keep-set manifest is not sent, so the receiver removes nothing. */ + mtx_lock(&context->mutex_scanner); + bool scan_io_now = context->scan_had_io_error; + mtx_unlock(&context->mutex_scanner); + if (!ignore_errors_allows_delete(context->config, scan_io_now)) { + log_message(LOG_LEVEL_WARNING, "IO error encountered -- skipping file deletion"); + } else if (send_delete_manifest(client->file_descriptor, context->manifest, + context->excluded_paths, context->size_skipped_paths, + context->missing_args, context->synced_dirs) != 0) { goto send_fail; + } } else if (context->config->delete_missing_args && !context->early_delete && - !context->delete_plans) { + !context->delete_suppressed && !context->delete_plans) { /* --delete-missing-args without --delete: no keep-set is built, but the exact-delete paths still ride the same manifest frame (commit once the transfer succeeded). */ @@ -2744,7 +2792,8 @@ static int scan_directory_multithreaded(void* pipeline_context) { failed = use_dscanner ? directory_scanner_failed(dscanner) : parallel_scanner_failed(scanner); break; } - if (context->config->use_delete && !context->early_delete && !context->delete_plans) { + if (context->config->use_delete && !context->early_delete && !context->delete_plans && + !context->delete_suppressed) { mtx_lock(&context->mutex_scanner); bool manifest_ok = add_chunk_to_manifest(context->manifest, current_chunk); mtx_unlock(&context->mutex_scanner); @@ -3037,6 +3086,7 @@ int send_files(Config* config) { bool prescan_ok = scan_paths_only(config, &prepared.options, early_manifest, NULL, &had_scan_io); bool early_ok = false; + bool skip_delete = false; if (prescan_ok) { /* A scan that hit an I/O error and produced NO keep entries is ambiguous (the source may not be genuinely empty -- part of it was unreadable), @@ -3048,6 +3098,11 @@ int send_files(Config* config) { "source scan hit an I/O error before finding any file; refusing to delete " "with an empty keep-set (--delete)"); prescan_ok = false; + } else if (!ignore_errors_allows_delete(config, had_scan_io)) { + /* rsync default: an I/O error suppresses deletion unless + --ignore-errors. Skip the manifest; the transfer still proceeds. */ + log_message(LOG_LEVEL_WARNING, "IO error encountered -- skipping file deletion"); + skip_delete = true; } else { early_ok = send_delete_manifest_early(client, early_manifest, excluded, size_skipped, missing_args, synced_dirs); @@ -3059,7 +3114,7 @@ int send_files(Config* config) { prepared.options.excluded_paths = NULL; prepared.options.size_skipped_paths = NULL; prepared.options.synced_dirs = NULL; - if (!prescan_ok || !early_ok) + if (!prescan_ok || (!early_ok && !skip_delete)) goto send_fail; } else if (delete_per_dir) { /* --delete-during/--delete-delay: build one plan per source directory from a @@ -3073,6 +3128,7 @@ int send_files(Config* config) { prepared.options.plan_dirs = plan_dirs; bool prescan_ok = scan_paths_only(config, &prepared.options, NULL, plan_sender, &had_scan_io); bool plans_ok = false; + bool skip_delete = false; if (prescan_ok) { const char* walk_root = delete_plan_walk_root(config, synced_dirs); const ArrayList* scope = @@ -3084,6 +3140,15 @@ int send_files(Config* config) { "source scan hit an I/O error before finding any file; refusing to delete " "with an empty keep-set (--delete)"); prescan_ok = false; + } else if (!ignore_errors_allows_delete(config, had_scan_io)) { + /* rsync default: an I/O error suppresses deletion unless + --ignore-errors. Drop the plans; the transfer still proceeds. */ + log_message(LOG_LEVEL_WARNING, "IO error encountered -- skipping file deletion"); + delete_plan_sender_destroy(plan_sender); + plan_sender = NULL; + array_list_delete(plan_dirs); + plan_dirs = NULL; + skip_delete = true; } else { plans_ok = delete_plan_send_root(client->file_descriptor, plan_sender) == 0; } @@ -3092,7 +3157,7 @@ int send_files(Config* config) { prepared.options.size_skipped_paths = NULL; prepared.options.synced_dirs = NULL; prepared.options.plan_dirs = NULL; - if (!prescan_ok || !plans_ok) + if (!prescan_ok || (!plans_ok && !skip_delete)) goto send_fail; } else if (config->use_delete) { manifest = array_list_create(free); @@ -3222,7 +3287,17 @@ int send_files(Config* config) { "an empty keep-set (--delete)"); goto send_fail; } - if ((manifest || config->delete_missing_args) && !delete_early && !delete_per_dir) { + /* rsync default: a scan I/O error suppresses deletion unless + --ignore-errors, even in the late (commit) modes. Drop the keep-set so + the receiver removes nothing; the readable tree still transferred. */ + bool late_delete = (manifest || config->delete_missing_args) && !delete_early && !delete_per_dir; + if (late_delete && !ignore_errors_allows_delete(config, had_scan_io)) { + log_message(LOG_LEVEL_WARNING, "IO error encountered -- skipping file deletion"); + if (manifest) { + array_list_delete(manifest); + manifest = NULL; + } + } else if (late_delete) { /* Late (commit) ordering: all file data is out; transmit the manifest so the receiver commits the extras walk (--delete) and/or the --delete-missing-args exact-path deletions only after the transfer @@ -3476,8 +3551,28 @@ int send_files_multithreaded(Config** config_ptr) { pipeline_context_sender_destroy(context); return 1; } - if (!per_dir) + if (context->scan_had_io_error && !ignore_errors_allows_delete(config, true)) { + /* rsync default: an I/O error suppresses deletion unless + --ignore-errors. Drop the prebuilt keep-set so nothing is sent; the + data pass still transfers the readable tree and exits 23. */ + log_message(LOG_LEVEL_WARNING, "IO error encountered -- skipping file deletion"); + if (context->manifest) { + array_list_delete(context->manifest); + context->manifest = NULL; + } + if (context->delete_plans) { + delete_plan_sender_destroy(context->delete_plans); + context->delete_plans = NULL; + } + if (context->plan_dirs) { + array_list_delete(context->plan_dirs); + context->plan_dirs = NULL; + } + /* A later --delete pass must not try to rebuild/send a keep-set. */ + context->delete_suppressed = true; + } else if (!per_dir) { context->early_delete = true; + } } else { context->manifest = array_list_create(free); if (!context->manifest) { @@ -3488,7 +3583,8 @@ int send_files_multithreaded(Config** config_ptr) { } if (config->remove_source_files) context->remove_source_files = array_list_create(source_file_destroy); - if ((config->use_delete && !context->manifest && !context->delete_plans) || + if ((config->use_delete && !context->manifest && !context->delete_plans && + !context->delete_suppressed) || (config->remove_source_files && !context->remove_source_files)) { pipeline_context_sender_destroy(context); return 1; diff --git a/src/client/scanner.c b/src/client/scanner.c index 61e7dd9..021334a 100644 --- a/src/client/scanner.c +++ b/src/client/scanner.c @@ -432,6 +432,30 @@ static void scanner_record_protected(DirectoryScanner* scanner, const char* fs_p scanner->failed = true; } +/* rsync's `--info=nonreg` line for a non-regular entry that is not being + * preserved: `skipping non-regular file "NAME"`. The name is the path relative + * to the transfer root, so it matches rsync's displayed name. */ +static void scanner_note_nonreg(const ScannerOptions* options, const char* fs_path) { + if (!options || !options->note_nonreg || !fs_path) + return; + const char* rel = *fs_path == '/' ? fs_path + 1 : fs_path; + const char* root = options->send_directory; + if (root != NULL) { + while (*root == '/') + root++; + size_t root_len = strlen(root); + while (root_len > 0 && root[root_len - 1] == '/') + root_len--; + if (root_len > 0 && strncmp(root, rel, root_len) == 0 && + (rel[root_len] == '/' || rel[root_len] == '\0')) + rel += root_len + (rel[root_len] == '/' ? 1 : 0); + } + char* escaped = output_escape(rel, options->eight_bit_output); + printf("skipping non-regular file \"%s\"\n", escaped ? escaped : rel); + free(escaped); + fflush(stdout); +} + /* A user-selection exclusion (--filter/-C/per-dir or --exclude/--include). */ static void scanner_record_excluded(DirectoryScanner* scanner, const char* fs_path) { scanner_record_protected(scanner, fs_path, scanner->options.excluded_paths); @@ -954,11 +978,18 @@ static int open_next_directory(DirectoryScanner* scanner) { scanner->current_rel = NULL; free(scanner->current_path); scanner->current_path = NULL; - if (!scanner->options.ignore_io_errors || is_root_seed) { + if (is_root_seed) { + /* The transfer ROOT being unreadable is always fatal: an empty keep-set + would delete the whole destination. Mark the scan as errored so the + client can report the partial-transfer exit code (rsync's 23). */ + scanner->root_io_error = true; scanner->failed = true; return -1; } - /* --ignore-errors: record the I/O error and keep scanning the rest. */ + /* A subdirectory that cannot be opened is always skipped (rsync continues + with a partial transfer), whether or not --ignore-errors is set. The + error is recorded so the client exits 23; --ignore-errors only changes + what the deletion phase does with the recorded error. */ continue; } if (open_directory_filter_context(scanner, inherited) != 0) { @@ -1554,6 +1585,7 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) { scanner->options.preserve_specials, scanner->options.copy_devices, file, &stats); if (special == SCANNER_SPECIAL_SKIP) { + scanner_note_nonreg(&scanner->options, file->path); free(rel_copy); file_destroy(file); continue; @@ -1604,7 +1636,7 @@ bool directory_scanner_failed(const DirectoryScanner* scanner) { } bool directory_scanner_had_io_error(const DirectoryScanner* scanner) { - return scanner != NULL && scanner->io_error; + return scanner != NULL && (scanner->io_error || scanner->root_io_error); } typedef struct { @@ -1974,6 +2006,7 @@ static void scan_root_entry(const ScannerOptions* options, const FilterNode* roo ScannerSpecial special = scanner_prepare_special( options->preserve_devices, options->preserve_specials, options->copy_devices, file, &st); if (special == SCANNER_SPECIAL_SKIP) { + scanner_note_nonreg(ps->options, file->path); free(rel); file_destroy(file); return; @@ -2134,6 +2167,7 @@ ParallelScanner* parallel_scanner_create_with_options(const char* root_directory return NULL; } ps->allocation_session = allocation_session; + ps->options = options; ArrayList* root_files = array_list_create(file_destroy); ArrayList* subdirs = array_list_create(free); diff --git a/src/client/scanner.h b/src/client/scanner.h index 3aeab8c..37c8625 100644 --- a/src/client/scanner.h +++ b/src/client/scanner.h @@ -121,9 +121,18 @@ typedef struct { * it) and to emit its plan after the data stream, when no file frame would * otherwise trigger it. Guarded by `excluded_mutex`. */ ArrayList* plan_dirs; - /* --ignore-errors: an unreadable directory during the scan is recorded as an - * I/O error and skipped instead of aborting the scan. Client-only. */ + /* --ignore-errors: an unreadable subdirectory no longer aborts the scan (it + * is always skipped so the rest of the tree transfers); this flag is kept so + * the client can distinguish the option state when deciding deletion policy. + * Client-only. */ bool ignore_io_errors; + /* --info=nonreg: print rsync's `skipping non-regular file "NAME"` line for a + * non-regular entry that is not being preserved. Client-only. */ + bool note_nonreg; + /* Source root and 8-bit-output policy used to render a `--info=nonreg` name + * relative to the transfer root. Borrowed read-only. */ + const char* send_directory; + bool eight_bit_output; /* --ignore-missing-args (implied by --delete-missing-args): an explicitly * --files-from-listed entry that does not exist under the source is skipped * instead of failing (the --dirs generator is the only scanner path that @@ -186,6 +195,10 @@ typedef struct { --ignore-errors the scan continues past it and the caller decides what to do; `failed` is reserved for fatal errors that always abort the scan. */ bool io_error; + /* The transfer ROOT could not be opened. It is always fatal, even under + --ignore-errors, but the client still maps it to rsync's partial-transfer + exit (23) rather than a generic failure. */ + bool root_io_error; } DirectoryScanner; typedef struct { @@ -206,6 +219,7 @@ typedef struct { Chunk* initial_chunk; ProtocolSession* allocation_session; FilterNode* root_filter_node; /* root .rsync-filter context (owned by ps) */ + const ScannerOptions* options; /* borrowed scan options (--info=nonreg output) */ } ParallelScanner; DirectoryScanner* directory_scanner_create(const char* root_directory, bool use_metadata, diff --git a/src/client/usage.c b/src/client/usage.c index c393766..58d24cc 100644 --- a/src/client/usage.c +++ b/src/client/usage.c @@ -243,7 +243,9 @@ void print_usage(void) { printf(" reusable digest is sent (keep the file mode 0600)\n"); printf(" --no-motd Suppress display of the daemon's MOTD (the server\n"); printf(" still sends it; the client just does not show it)\n"); - printf(" --bwlimit Bandwidth limit in kilobytes per second\n"); + printf(" --bwlimit=RATE Limit socket I/O bandwidth (default unit KiB/s,\n"); + printf(" rsync-style: 0 = no limit; K/M/G/T/P suffixes are\n"); + printf(" binary, KB/MB decimal, KiB/MiB binary; decimals allowed)\n"); printf(" --tls Enable TLS encryption\n"); printf(" --cert TLS certificate file (PEM)\n"); printf(" --key TLS private key file (PEM)\n"); diff --git a/src/shared/log.h b/src/shared/log.h index 0e5a2ff..bce412a 100644 --- a/src/shared/log.h +++ b/src/shared/log.h @@ -21,7 +21,20 @@ typedef enum { LOG_INFO_MISC = 1u << 1, LOG_INFO_SKIP = 1u << 2, LOG_INFO_STATS = 1u << 3, - LOG_INFO_ALL = LOG_INFO_COPY | LOG_INFO_MISC | LOG_INFO_SKIP | LOG_INFO_STATS, + /* rsync categories that map to a FastSync event (emitted in rsync's line + * format): del (deletions), remove (sender-side source removal), name + * (transferred entry names), flist (file-list header), nonreg (skipped + * non-regular files), backup (backed-up files), progress (per-file progress). */ + LOG_INFO_DEL = 1u << 4, + LOG_INFO_REMOVE = 1u << 5, + LOG_INFO_NAME = 1u << 6, + LOG_INFO_FLIST = 1u << 7, + LOG_INFO_NONREG = 1u << 8, + LOG_INFO_BACKUP = 1u << 9, + LOG_INFO_PROGRESS = 1u << 10, + LOG_INFO_ALL = LOG_INFO_COPY | LOG_INFO_MISC | LOG_INFO_SKIP | LOG_INFO_STATS | LOG_INFO_DEL | + LOG_INFO_REMOVE | LOG_INFO_NAME | LOG_INFO_FLIST | LOG_INFO_NONREG | + LOG_INFO_BACKUP | LOG_INFO_PROGRESS, } LogInfoFlag; void log_message(LogLevel log_level, const char* message, ...); diff --git a/src/shared/multiprocessing.h b/src/shared/multiprocessing.h index 58f0af9..3886d84 100644 --- a/src/shared/multiprocessing.h +++ b/src/shared/multiprocessing.h @@ -83,6 +83,11 @@ typedef struct { thread transmits the root plan before any data and the remaining plans alongside the chunks. Set once before the worker threads start. */ DeletePlanSender* delete_plans; + /* A scan I/O error without --ignore-errors suppressed deletion: the prebuilt + keep-set/plans were dropped, and the streaming scanner must not build a + fresh manifest or re-send the per-directory plans. Set once before the + worker threads start. */ + bool delete_suppressed; mtx_t mutex_progress; int total_files; unsigned long long progress_bytes; diff --git a/src/shared/protocol.c b/src/shared/protocol.c index 70c0d8a..b5ceb0b 100644 --- a/src/shared/protocol.c +++ b/src/shared/protocol.c @@ -183,6 +183,10 @@ void io_set_bwlimit(unsigned long long bytes_per_sec) { mtx_unlock(&bw_mutex); } +unsigned long long io_get_bwlimit(void) { + return global_bwlimit(); +} + void protocol_session_set_bwlimit(ProtocolSession* session, unsigned long long bytes_per_sec) { if (!session) return; diff --git a/src/shared/protocol.h b/src/shared/protocol.h index 30bc5af..a9f55ee 100644 --- a/src/shared/protocol.h +++ b/src/shared/protocol.h @@ -207,6 +207,7 @@ enum NET_STATUS { void io_set_fds(int read_fd, int write_fd); void io_set_bwlimit(unsigned long long bytes_per_sec); +unsigned long long io_get_bwlimit(void); void io_set_ssl(SSL* ssl); SSL* io_get_ssl(void); diff --git a/tests/integration/test_features.py b/tests/integration/test_features.py index 2f9e686..77876b1 100644 --- a/tests/integration/test_features.py +++ b/tests/integration/test_features.py @@ -4463,11 +4463,11 @@ class TestDeletePolicy: @pytest.mark.parametrize("mt", [False, True]) @pytest.mark.setpriv def test_ignore_errors_keeps_deletion_active_on_scan_error(self, mt): - """A source I/O error (unreadable subdirectory) aborts the run so no - deletion happens by default; --ignore-errors continues, still transfers - the readable tree and still deletes, single-threaded and under -m. Run - as an unprivileged user so the mode-000 directory is genuinely - unreadable.""" + """rsync's --ignore-errors semantics: a source I/O error (unreadable + subdirectory) makes the run continue and transfer the readable tree, but + the default suppresses deletion ("IO error encountered -- skipping file + deletion"); --ignore-errors lets deletion proceed. Both exit 23. Run as + an unprivileged user so the mode-000 directory is genuinely unreadable.""" if os.geteuid() != 0 or shutil.which("setpriv") is None: pytest.skip("requires root + setpriv to drop privileges for the client") tag = f"ioerr_{os.getpid()}_{mt}" @@ -4487,11 +4487,15 @@ class TestDeletePolicy: try: os.chmod(os.path.join(source, "locked"), 0) - # Default: scan error aborts the run; nothing is deleted. + # Default: the scan continues past the unreadable dir and the + # readable tree transfers, but deletion is skipped (exit 23). self._write(os.path.join(received, "extra.txt"), b"extra\n") flags = ["--delete"] + (["--threads"] if mt else []) result = self._run_client_as_nobody(source, dest, server.port, flags) - assert result.returncode != 0, "unreadable source dir did not fail the run" + assert result.returncode == 23, \ + f"unreadable source dir should exit 23 (got {result.returncode})" + assert os.path.exists(os.path.join(received, "top.txt")), \ + "readable tree did not transfer past the I/O error" assert os.path.exists(os.path.join(received, "extra.txt")), \ "default run deleted although the scan hit an I/O error" @@ -4499,6 +4503,8 @@ class TestDeletePolicy: self._write(os.path.join(received, "extra.txt"), b"extra\n") flags = ["--delete", "--ignore-errors"] + (["--threads"] if mt else []) result = self._run_client_as_nobody(source, dest, server.port, flags) + assert result.returncode == 23, \ + f"--ignore-errors run should still exit 23 (got {result.returncode})" assert not os.path.exists(os.path.join(received, "extra.txt")), \ f"--ignore-errors did not keep deletion active: {result.stderr[:300]}" assert not os.path.exists(os.path.join(received, "locked")), \ diff --git a/tests/test_client_cli.c b/tests/test_client_cli.c index 2972a8a..a9ae488 100644 --- a/tests/test_client_cli.c +++ b/tests/test_client_cli.c @@ -7,6 +7,7 @@ #include "delta.h" #include "file_list.h" #include "log.h" +#include "protocol.h" #include "test_utils.h" #include "utils.h" #include @@ -1327,7 +1328,7 @@ static void test_parse_args_rejects_invalid_info_flag() { config_delete(cfg); } -/* rsync's info "name" category maps to fastsync's per-file name logging, and +/* rsync's info "name" category maps to fastsync's per-file name output, and * --info=help prints the flag list and exits without error. */ static void test_parse_args_info_name_and_help() { Config* cfg = config_create(); @@ -1335,7 +1336,7 @@ static void test_parse_args_info_name_and_help() { int positional_args[2]; int positional_count = 0; EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0); - EXPECT_EQ_INT(cfg->info_level, LOG_INFO_COPY); + EXPECT_EQ_INT(cfg->info_level, LOG_INFO_NAME); config_delete(cfg); cfg = config_create(); @@ -1345,8 +1346,10 @@ static void test_parse_args_info_name_and_help() { config_delete(cfg); } -/* rsync 3.4.1's remaining --info/--debug categories parse successfully but - * have no FastSync output wired to them, so they must not set any log flag. */ +/* rsync 3.4.1's full --info/--debug vocabulary parses. The info categories + * with a FastSync event set their flag; the remaining rsync-only categories + * (mount/symsafe/syms) parse but stay silent. Every --debug category listed + * here is FastSync-silent, so debug_level stays 0. */ static void test_parse_args_rsync_flag_vocabulary_accepted() { Config* cfg = config_create(); char* argv[] = {"fastsync", "--info=backup,del,flist,mount,nonreg,progress,remove,symsafe,syms", @@ -1358,7 +1361,8 @@ static void test_parse_args_rsync_flag_vocabulary_accepted() { int positional_count = 0; EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0); - EXPECT_EQ_INT(cfg->info_level, 0); + EXPECT_EQ_INT(cfg->info_level, LOG_INFO_BACKUP | LOG_INFO_DEL | LOG_INFO_FLIST | + LOG_INFO_NONREG | LOG_INFO_PROGRESS | LOG_INFO_REMOVE); EXPECT_EQ_INT(cfg->debug_level, 0); config_delete(cfg); } @@ -3870,6 +3874,64 @@ static void test_parse_args_unsigned_options_reject_sign() { config_delete(cfg); } +/* --bwlimit must parse with rsync 3.4.1's units and quantization: a bare value + * is KiB/s, K/M/G/T/P are binary multipliers, KB/MB are decimal, KiB/MiB are + * binary, decimals are rounded to whole KiB like rsync's (size + 512) / 1024, + * and 0 (or an empty value) means "no limit". */ +static void test_parse_args_bwlimit_rsync_units() { + struct { + const char* value; + unsigned long long expected; /* bytes/sec */ + int ok; + } cases[] = { + {"100", 100ULL * 1024, 1}, + {"0", 0, 1}, + {"", 0, 1}, + {"1.5", 2ULL * 1024, 1}, + {"100K", 100ULL * 1024, 1}, + {"100KiB", 100ULL * 1024, 1}, + {"100KB", (100000ULL + 512) / 1024 * 1024, 1}, + {"1M", 1024ULL * 1024, 1}, + {"1MB", (1000000ULL + 512) / 1024 * 1024, 1}, + {"1.5m", 1536ULL * 1024, 1}, + {"1G", 1024ULL * 1024 * 1024, 1}, + {"1000B", (1000ULL + 512) / 1024 * 1024, 1}, + {"100B", 0, 0}, /* below the 512-byte floor (not 0) */ + {"0.4", 0, 0}, /* 409 bytes, below the floor */ + {"511", 511ULL * 1024, 1}, + {"-1", 0, 0}, + {"abc", 0, 0}, + {"1x", 0, 0}, + }; + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + Config* cfg = config_create(); + EXPECT_NOT_NULL(cfg); + int positional_args[2]; + int positional_count = 0; + char option[32]; + snprintf(option, sizeof(option), "--bwlimit=%s", cases[i].value); + char* argv[] = {"fastsync", option, "/src", "/dst"}; + int rc = parse_args(cfg, 4, argv, positional_args, &positional_count); + if (cases[i].ok) { + EXPECT_EQ_INT(rc, 0); + EXPECT_TRUE(io_get_bwlimit() == cases[i].expected); + } else { + EXPECT_EQ_INT(rc, -1); + } + config_delete(cfg); + } + + Config* cfg = config_create(); + EXPECT_NOT_NULL(cfg); + int positional_args[2]; + int positional_count = 0; + char* argv[] = {"fastsync", "--bwlimit", "512", "/src", "/dst"}; + EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0); + EXPECT_TRUE(io_get_bwlimit() == 512ULL * 1024); + config_delete(cfg); + io_set_bwlimit(0); +} + /* --dry-run must not emit a batch file, so it is rejected alongside * --read-batch/--only-write-batch. */ static void test_validate_config_dry_run_rejects_write_batch() { @@ -4579,6 +4641,7 @@ void test_client_cli() { test_parse_args_password_file(); test_parse_args_pattern_file_oversized_rejected(); test_parse_args_unsigned_options_reject_sign(); + test_parse_args_bwlimit_rsync_units(); test_validate_config_dry_run_rejects_write_batch(); test_parse_args_short_clustering(); test_parse_args_attached_short_values();