diff --git a/src/client/client_cli.c b/src/client/client_cli.c index b47a059..d38078d 100644 --- a/src/client/client_cli.c +++ b/src/client/client_cli.c @@ -395,6 +395,39 @@ static void apply_output_buffering(const Config* config) { static int read_patterns_from_file(const char* filepath, char*** patterns, int* count, Config* config, char sign, const char* optname); +/* Split one --debug/--info item into its category name and an optional rsync + * verbosity level suffix (e.g. "io2", "all4", "none0"). The output `name` is + * NUL-terminated and `level` is >= 0 (0 silences the item). Returns false for + * an empty token or a token that is all digits. */ +static bool split_flag_level(const char* token, char* name, size_t name_size, int* level) { + size_t len = strlen(token); + if (len == 0 || name_size == 0) + return false; + size_t end = len; + while (end > 0 && token[end - 1] >= '0' && token[end - 1] <= '9') + end--; + if (end == 0) + return false; /* all digits: not a category name */ + size_t name_len = end < name_size - 1 ? end : name_size - 1; + for (size_t i = 0; i < name_len; i++) { + char c = token[i]; + name[i] = (c >= 'A' && c <= 'Z') ? (char)(c - 'A' + 'a') : c; + } + name[name_len] = '\0'; + int lvl = 1; + if (end < len) { + lvl = 0; + for (size_t i = end; i < len; i++) { + int digit = token[i] - '0'; + if (lvl > (1000 - digit) / 10) + return false; + lvl = lvl * 10 + digit; + } + } + *level = lvl; + return true; +} + static int parse_debug_flags(const char* value, Config* config) { if (!value || value[0] == '\0' || value[0] == ',' || value[strlen(value) - 1] == ',' || strstr(value, ",,")) { @@ -412,30 +445,40 @@ static int parse_debug_flags(const char* value, Config* config) { for (char* token = strtok_r(flags, ",", &saveptr); token != NULL; token = strtok_r(NULL, ",", &saveptr)) { uint32_t flag = 0; - if (strcmp(token, "help") == 0) { + char name[32]; + int level = 1; + if (!split_flag_level(token, name, sizeof(name), &level)) { + log_message(LOG_LEVEL_ERROR, "unsupported --debug flag: %s", token); + free(flags); + return -1; + } + if (strcmp(name, "help") == 0) { print_debug_usage(); free(flags); return 1; - } else if (strcmp(token, "all") == 0) { - parsed = LOG_DEBUG_ALL; + } else if (strcmp(name, "all") == 0) { + parsed = level == 0 ? 0 : LOG_DEBUG_ALL; continue; - } else if (strcmp(token, "none") == 0) { + } else if (strcmp(name, "none") == 0) { parsed = 0; continue; - } else if (strcmp(token, "io") == 0) { + } else if (strcmp(name, "io") == 0) { flag = LOG_DEBUG_IO; - } else if (strcmp(token, "proto") == 0) { + } else if (strcmp(name, "proto") == 0) { flag = LOG_DEBUG_PROTO; - } else if (strcmp(token, "pack") == 0) { + } else if (strcmp(name, "pack") == 0) { flag = LOG_DEBUG_PACK; - } else if (strcmp(token, "util") == 0) { + } else if (strcmp(name, "util") == 0) { flag = LOG_DEBUG_UTIL; } else { log_message(LOG_LEVEL_ERROR, "unsupported --debug flag: %s", token); free(flags); return -1; } - parsed |= flag; + if (level == 0) + parsed &= ~flag; + else + parsed |= flag; } free(flags); config->debug_level = (int)parsed; @@ -461,28 +504,43 @@ static int parse_info_flags(const char* value, Config* config) { for (char* token = strtok_r(flags, ",", &saveptr); token != NULL; token = strtok_r(NULL, ",", &saveptr)) { uint32_t flag = 0; - if (strcmp(token, "all") == 0) { - parsed = LOG_INFO_ALL; + char name[32]; + int level = 1; + if (!split_flag_level(token, name, sizeof(name), &level)) { + log_message(LOG_LEVEL_ERROR, "unsupported --info flag: %s", token); + free(flags); + return -1; + } + if (strcmp(name, "all") == 0) { + parsed = level == 0 ? 0 : LOG_INFO_ALL; continue; } - if (strcmp(token, "none") == 0) { + if (strcmp(name, "none") == 0) { parsed = 0; continue; } - if (strcmp(token, "copy") == 0) + if (strcmp(name, "help") == 0) { + print_info_usage(); + free(flags); + return 1; + } + if (strcmp(name, "copy") == 0 || strcmp(name, "name") == 0) flag = LOG_INFO_COPY; - else if (strcmp(token, "misc") == 0) + else if (strcmp(name, "misc") == 0) flag = LOG_INFO_MISC; - else if (strcmp(token, "skip") == 0) + else if (strcmp(name, "skip") == 0) flag = LOG_INFO_SKIP; - else if (strcmp(token, "stats") == 0) + else if (strcmp(name, "stats") == 0) flag = LOG_INFO_STATS; else { log_message(LOG_LEVEL_ERROR, "unsupported --info flag: %s", token); free(flags); return -1; } - parsed |= flag; + if (level == 0) + parsed &= ~flag; + else + parsed |= flag; } free(flags); config->info_level = (int)parsed; @@ -1027,17 +1085,22 @@ typedef struct { } CliParseCtx; /* Apply output controls before processing other options so their order is - * irrelevant. Returns 0 on success, -1 on error. */ + * irrelevant. Returns 0 on success, a positive code for a help request + * (parse_args returns it verbatim), or -1 on error. */ static int cli_apply_output_controls(Config* config, int argc, char* argv[]) { for (int i = 1; i < argc; i++) { if (strcmp(argv[i], "-v") == 0 || strcmp(argv[i], "--verbose") == 0) { set_log_level(LOG_LEVEL_DEBUG); } else if (strncmp(argv[i], "--info=", 7) == 0) { - if (parse_info_flags(argv[i] + 7, config) != 0) - return -1; + int ret = parse_info_flags(argv[i] + 7, config); + if (ret != 0) + return ret; } else if (strcmp(argv[i], "--info") == 0) { - if (i + 1 >= argc || parse_info_flags(argv[++i], config) != 0) + if (i + 1 >= argc) return -1; + int ret = parse_info_flags(argv[++i], config); + if (ret != 0) + return ret; } } return 0; @@ -1842,13 +1905,19 @@ static bool cli_handle_logging_options(CliParseCtx* ctx) { return true; } if (strncmp(arg, "--info=", 7) == 0) { - if (parse_info_flags(arg + 7, config) != 0) - ctx->exit_code = -1; + int info_ret = parse_info_flags(arg + 7, config); + if (info_ret != 0) + ctx->exit_code = info_ret; return true; } if (opt_is(arg, "--info", NULL)) { - if (ctx->i + 1 >= ctx->argc || parse_info_flags(ctx->argv[++ctx->i], config) != 0) + if (ctx->i + 1 >= ctx->argc) { ctx->exit_code = -1; + return true; + } + int info_ret = parse_info_flags(ctx->argv[++ctx->i], config); + if (info_ret != 0) + ctx->exit_code = info_ret; return true; } if (strncmp(arg, "--skip-compress=", 16) == 0) { @@ -2436,8 +2505,11 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args, } int result = -1; - if (cli_apply_output_controls(config, exp_argc, exp_argv) != 0) + int output_ret = cli_apply_output_controls(config, exp_argc, exp_argv); + if (output_ret != 0) { + result = output_ret; goto done; + } CliParseCtx ctx = { .config = config, diff --git a/src/client/client_send.c b/src/client/client_send.c index 4da392f..a146059 100644 --- a/src/client/client_send.c +++ b/src/client/client_send.c @@ -328,15 +328,15 @@ static char* files_from_missing_dest_path(const Config* config, const char* entr /* --files-from semantics: every listed entry must resolve under the source * root, otherwise rsync reports a hard error instead of silently transferring - * nothing. An empty list is also an error. An entry of "." (the whole tree) - * and listed-but-empty directories are valid. With --ignore-missing-args + * nothing. An entry of "." (the whole tree) and listed-but-empty directories + * are valid. An empty list is valid too: rsync transfers nothing and exits 0. + * With --ignore-missing-args * (implied by --delete-missing-args) a listed-but-missing entry is instead * skipped: nothing is transferred for it, it never enters the keep-set and the * run succeeds for the rest (an all-missing non-empty list succeeds * transferring nothing, matching rsync). With --delete-missing-args * `missing_dest` (when non-NULL) collects the entry's destination-relative - * mirror for the receiver's exact-deletion request. An empty list stays a - * hard error in every mode (nothing was requested at all). Runs before any + * mirror for the receiver's exact-deletion request. Runs before any * transfer so the failure/skip is surfaced uniformly in the single-threaded, * -m, dry-run and --list-only paths. */ static bool files_from_list_check(const Config* config, ArrayList* missing_dest, int* skipped_out) { @@ -349,12 +349,11 @@ static bool files_from_list_check(const Config* config, ArrayList* missing_dest, return false; } if (set->count == 0) { - char* escaped_list = - output_escape(config->files_from ? config->files_from : "", log_get_8_bit_output()); - log_message(LOG_LEVEL_ERROR, "--files-from file '%s' contains no entries; nothing to transfer", - escaped_list ? escaped_list : ""); - free(escaped_list); - return false; + /* rsync treats an empty --files-from list as "nothing to transfer" and + exits 0 (the source directory is still a valid source arg), so this is + not an error. Nothing passes the (empty) allow-set, so no file is sent + and no keep-set entry is produced. */ + return true; } bool ignore = config->ignore_missing_args || config->delete_missing_args; for (int i = 0; i < set->count; i++) { @@ -2679,12 +2678,15 @@ int send_files(Config* config) { report_transfer_stats(config, total_files, total_bytes, start); log_info_message(LOG_INFO_STATS, "Transfer summary: %d files, %.1f MB", total_files, (double)total_bytes / (double)BYTES_PER_MIB); - /* --ignore-errors: an unreadable source directory was skipped but the run - still completed (and deleted); report the run as errored like rsync does. - A --max-delete-capped commit is a successful transfer that rsync reports + /* A skipped source entry (--ignore-errors past an unreadable directory, or a + dereferenced symlink with no referent) makes rsync report a partial + transfer (exit 23) even though the rest of the run succeeded. A + --max-delete-capped commit is a successful transfer that rsync reports with exit code 25. */ - if (!ok || had_scan_io) + if (!ok) ret = 1; + else if (had_scan_io) + ret = 23; else ret = delete_limit ? 25 : 0; @@ -2921,12 +2923,15 @@ int send_files_multithreaded(Config** config_ptr) { mtx_unlock(&context->mutex_scanner); bool sender_ok = sender_result == thrd_success; bool delete_limit = context->delete_limit; - /* --ignore-errors: the run completed (and deleted) past an unreadable source - directory; report it as errored like rsync does. A --max-delete-capped - commit is a successful transfer that rsync reports with exit code 25. */ + /* A skipped source entry (--ignore-errors past an unreadable directory, or a + dereferenced symlink with no referent) makes rsync report a partial + transfer (exit 23). A --max-delete-capped commit is a successful transfer + that rsync reports with exit code 25. */ pipeline_context_sender_destroy(context); client_set_abort_armed(false); - if (!sender_ok || scan_io) + if (!sender_ok) return 1; + if (scan_io) + return 23; return delete_limit ? 25 : 0; } diff --git a/src/client/scanner.c b/src/client/scanner.c index 99057bf..5ad0ee9 100644 --- a/src/client/scanner.c +++ b/src/client/scanner.c @@ -163,6 +163,12 @@ typedef struct { Size pruning protects the destination mirror even under --delete-excluded, so it is recorded into a separate sink from `excluded`. */ bool size_excluded; + /* True when a symlink selected for dereferencing (-L/--copy-links or an + unsafe target under --copy-unsafe-links) had no usable referent (a broken + link or a stat() failure). rsync still reports this as a partial transfer + (exit 23) even though the entry is skipped, so the scanner records it as a + non-fatal I/O error. */ + bool referent_error; } ScannerEntry; /* --one-file-system (-x) decision. Only directories can carry a different @@ -412,6 +418,7 @@ static int scanner_inspect_entry(const ScannerOptions* options, const char* cont const char* link_rel, const char* name, ScannerEntry* entry) { entry->excluded = false; entry->size_excluded = false; + entry->referent_error = false; entry->is_symlink = false; entry->link_target = NULL; entry->path = path_cat(containing_dir, name); @@ -438,12 +445,13 @@ static int scanner_inspect_entry(const ScannerOptions* options, const char* cont goto skip; case LINK_ACTION_DEREF: if (stat(entry->path, &entry->stats) != 0) { - /* rsync reports "symlink has no referent" and continues (exit 23); we - surface the same condition rather than silently dropping the entry. */ + /* rsync reports "symlink has no referent" and continues with a partial + transfer (exit 23); record the error so the run exits 23 too. */ char* escaped = output_escape(entry->path, log_get_8_bit_output()); log_message(LOG_LEVEL_WARNING, "symlink has no referent: %s", escaped ? escaped : ""); free(escaped); + entry->referent_error = true; goto skip; } entry->is_directory = S_ISDIR(entry->stats.st_mode); @@ -1118,6 +1126,10 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) { break; } if (inspection == 0) { + /* A dereferenced symlink with no referent is a partial-transfer error + (rsync exit 23): record it as a non-fatal scan I/O error. */ + if (inspected.referent_error) + scanner->io_error = true; /* A user-selection exclude protects its destination mirror from --delete unless --delete-excluded; a size prune is always protected. Other skips (unreadable, symlink policy) protect nothing. Under -R + @@ -1511,6 +1523,8 @@ static void scan_root_entry(const ScannerOptions* options, const FilterNode* roo return; } if (inspection == 0) { + if (inspected.referent_error) + ps->io_error = true; ArrayList* sink = NULL; if (inspected.excluded) sink = inspected.size_excluded ? options->size_skipped_paths : options->excluded_paths; diff --git a/src/client/usage.c b/src/client/usage.c index 427dd56..6a79632 100644 --- a/src/client/usage.c +++ b/src/client/usage.c @@ -171,8 +171,8 @@ void print_usage(void) { printf(" -v, --verbose Enable debug logging\n"); printf(" -q, --quiet Suppress non-error output\n"); printf(" --debug=FLAGS Fine-grained debug logging (use --debug=help for flags)\n"); - printf(" --info=FLAGS Fine-grained info: copy,misc,skip,stats,all,none\n"); - printf(" none suppresses info even with --verbose\n"); + printf(" --info=FLAGS Fine-grained info: copy,name,misc,skip,stats,all,none\n"); + printf(" (use --info=help for flags; none suppresses --verbose)\n"); printf(" --preserve Preserve permissions and times (= -pt; long form only)\n"); printf(" --no-perms Negate -p/--perms\n"); printf(" --no-times Negate -t/--times\n"); @@ -340,5 +340,13 @@ void print_usage(void) { void print_debug_usage(void) { printf("Supported debug flags: IO,PROTO,PACK,UTIL,ALL,NONE\n"); printf("Flags may be comma-separated, for example: --debug=io,proto\n"); - printf("Other rsync debug flags are unsupported and rejected.\n"); + printf("An optional level suffix is accepted (e.g. --debug=io2); level 0\n"); + printf("silences that item. Other rsync debug flags are unsupported and rejected.\n"); +} + +void print_info_usage(void) { + printf("Supported info flags: COPY,NAME,MISC,SKIP,STATS,ALL,NONE\n"); + printf("Flags may be comma-separated, for example: --info=name,stats\n"); + printf("An optional level suffix is accepted (e.g. --info=stats2); level 0\n"); + printf("silences that item. Other rsync info flags are unsupported and rejected.\n"); } diff --git a/src/client/usage.h b/src/client/usage.h index ca8d65b..879716c 100644 --- a/src/client/usage.h +++ b/src/client/usage.h @@ -3,5 +3,6 @@ void print_usage(void); void print_debug_usage(void); +void print_info_usage(void); #endif diff --git a/tests/integration/test_features.py b/tests/integration/test_features.py index 1e50d59..d9270e1 100644 --- a/tests/integration/test_features.py +++ b/tests/integration/test_features.py @@ -3157,15 +3157,21 @@ class TestMissingArgs: assert not os.path.exists(os.path.join(received, "gone1.txt")) @pytest.mark.parametrize("mt", [False, True]) - def test_empty_list_stays_a_hard_error(self, shared_server, mt): + def test_empty_list_succeeds_transferring_nothing(self, shared_server, mt): + """rsync 3.4.1 treats an empty --files-from list as "nothing to + transfer" and exits 0 (verified with the real binary), so fastsync must + too rather than reporting a hard error.""" source = self._make_source("mg_empty_src") dest = os.path.join(TEST_DATA_DIR, "mg_empty_dst") clean_dir(dest) lst = _write_rel_list(b"") flags = ["--files-from", lst, "--ignore-missing-args"] + (["--threads"] if mt else []) result, _ = run_client(source, dest, flags=flags, port=shared_server.port) - assert result.returncode != 0, "an empty --files-from list must stay a hard error" - assert "contains no entries" in (result.stderr or result.stdout) + assert result.returncode == 0, \ + f"an empty --files-from list must succeed like rsync: {result.stderr[:300]}" + received = get_dest_received_dir(dest, source) + assert not os.path.exists(os.path.join(received, "a.txt")), \ + "an empty --files-from list must transfer nothing" @pytest.mark.parametrize("mt", [False, True]) def test_delete_missing_removes_mirror_not_unrelated(self, mt): diff --git a/tests/integration/test_parity_quickwins.py b/tests/integration/test_parity_quickwins.py new file mode 100644 index 0000000..52127bf --- /dev/null +++ b/tests/integration/test_parity_quickwins.py @@ -0,0 +1,846 @@ +"""Client-only rsync-parity quick wins. + +Each test here pins behaviour that must match real ``rsync 3.4.1``. The +differential tests skip cleanly when rsync is not installed. +""" +import os +import shutil +import subprocess +import sys + +import pytest + +sys.path.insert(0, os.path.dirname(__file__)) +from common import ( + TEST_DATA_DIR, + ServerManager, + run_client, + clean_dir, + get_dest_received_dir, +) + +RSYNC = shutil.which("rsync") +requires_rsync = pytest.mark.skipif(RSYNC is None, reason="rsync 3.4.1 not installed") + + +def _rel_files(root, skip=()): + """Relative paths of regular files and symlinks below root, sorted.""" + out = [] + for dirpath, _dirs, files in os.walk(root): + for name in files: + if name in skip: + continue + out.append(os.path.relpath(os.path.join(dirpath, name), root)) + return sorted(out) + + +def _rsync(args): + env = dict(os.environ, LC_ALL="C") + return subprocess.run( + [RSYNC] + args, capture_output=True, text=True, env=env, timeout=120 + ) + + +class TestCopyLinksReferentError: + """#38/#39: a broken referent under -L/--copy-unsafe-links exits 23.""" + + def _make_broken_tree(self, root): + clean_dir(root) + with open(os.path.join(root, "ok.txt"), "wb") as fh: + fh.write(b"hello\n") + os.symlink("/nonexistent/target", os.path.join(root, "broken")) + + @requires_rsync + @pytest.mark.ci + def test_copy_links_broken_referent_exit_23(self, shared_server): + source = os.path.join(TEST_DATA_DIR, "qw_cl_src") + dest = os.path.join(TEST_DATA_DIR, "qw_cl_dst") + rdst = os.path.join(TEST_DATA_DIR, "qw_cl_rdst") + self._make_broken_tree(source) + clean_dir(dest) + clean_dir(rdst) + + rsync_result = _rsync(["-aL", source + "/", rdst + "/"]) + assert rsync_result.returncode == 23, rsync_result.stderr + + result, _ = run_client(source, dest, flags=["-L"], port=shared_server.port) + assert result.returncode == 23, ( + f"-L broken referent must exit 23, got {result.returncode}: " + f"{result.stderr[:300]}" + ) + received = get_dest_received_dir(dest, source) + assert os.path.exists(os.path.join(received, "ok.txt")) + + @requires_rsync + def test_copy_links_broken_referent_multithreaded_exit_23(self, shared_server): + source = os.path.join(TEST_DATA_DIR, "qw_cl_mt_src") + dest = os.path.join(TEST_DATA_DIR, "qw_cl_mt_dst") + self._make_broken_tree(source) + clean_dir(dest) + result, _ = run_client(source, dest, flags=["-L", "--threads"], port=shared_server.port) + assert result.returncode == 23, ( + f"-L broken referent must exit 23 under --threads, got {result.returncode}" + ) + + @requires_rsync + @pytest.mark.ci + def test_copy_unsafe_links_broken_referent_exit_23(self, shared_server): + source = os.path.join(TEST_DATA_DIR, "qw_cul_src") + dest = os.path.join(TEST_DATA_DIR, "qw_cul_dst") + rdst = os.path.join(TEST_DATA_DIR, "qw_cul_rdst") + clean_dir(source) + os.makedirs(os.path.join(source, "sub")) + with open(os.path.join(source, "ok.txt"), "wb") as fh: + fh.write(b"hello\n") + # Unsafe (escaping) target with no referent: dereferenced -> exit 23. + os.symlink("../../../nonexistent/target", os.path.join(source, "sub", "unsafe")) + clean_dir(dest) + clean_dir(rdst) + + rsync_result = _rsync(["-a", "--copy-unsafe-links", source + "/", rdst + "/"]) + assert rsync_result.returncode == 23, rsync_result.stderr + + result, _ = run_client(source, dest, flags=["-l", "--copy-unsafe-links"], + port=shared_server.port) + assert result.returncode == 23, ( + f"--copy-unsafe-links broken referent must exit 23, got {result.returncode}" + ) + + @requires_rsync + def test_copy_unsafe_links_safe_broken_stays_symlink(self, shared_server): + """A safe (non-escaping) broken symlink is NOT dereferenced: exit 0.""" + source = os.path.join(TEST_DATA_DIR, "qw_cul_safe_src") + dest = os.path.join(TEST_DATA_DIR, "qw_cul_safe_dst") + rdst = os.path.join(TEST_DATA_DIR, "qw_cul_safe_rdst") + clean_dir(source) + os.makedirs(os.path.join(source, "sub")) + os.symlink("nonexistent-target", os.path.join(source, "sub", "safe")) + clean_dir(dest) + clean_dir(rdst) + + rsync_result = _rsync(["-a", "--copy-unsafe-links", source + "/", rdst + "/"]) + assert rsync_result.returncode == 0, rsync_result.stderr + + result, _ = run_client(source, dest, flags=["-l", "--copy-unsafe-links"], + port=shared_server.port) + assert result.returncode == 0, result.stderr[:300] + received = get_dest_received_dir(dest, source) + assert os.path.islink(os.path.join(received, "sub", "safe")) + + +class TestIgnoreMissingArgsParity: + """#58: --files-from + --ignore-missing-args matches rsync, including the + empty-list case (rsync exits 0 transferring nothing).""" + + @requires_rsync + @pytest.mark.ci + def test_missing_entry_skipped_like_rsync(self, shared_server): + source = os.path.join(TEST_DATA_DIR, "qw_ima_src") + dest = os.path.join(TEST_DATA_DIR, "qw_ima_dst") + rdst = os.path.join(TEST_DATA_DIR, "qw_ima_rdst") + clean_dir(source) + with open(os.path.join(source, "a.txt"), "wb") as fh: + fh.write(b"a\n") + clean_dir(dest) + clean_dir(rdst) + lst = os.path.join(TEST_DATA_DIR, "qw_ima_list") + with open(lst, "w") as fh: + fh.write("a.txt\nmissing.txt\n") + + rsync_result = _rsync(["-a", "--files-from=" + lst, "--ignore-missing-args", + source + "/", rdst + "/"]) + assert rsync_result.returncode == 0, rsync_result.stderr + result, _ = run_client(source, dest, flags=["--files-from", lst, + "--ignore-missing-args"], + port=shared_server.port) + assert result.returncode == 0, result.stderr[:300] + received = get_dest_received_dir(dest, source) + assert os.path.exists(os.path.join(received, "a.txt")) + assert not os.path.exists(os.path.join(received, "missing.txt")) + + @requires_rsync + @pytest.mark.ci + def test_empty_files_from_list_succeeds_like_rsync(self, shared_server): + source = os.path.join(TEST_DATA_DIR, "qw_ima_empty_src") + dest = os.path.join(TEST_DATA_DIR, "qw_ima_empty_dst") + rdst = os.path.join(TEST_DATA_DIR, "qw_ima_empty_rdst") + clean_dir(source) + with open(os.path.join(source, "a.txt"), "wb") as fh: + fh.write(b"a\n") + clean_dir(dest) + clean_dir(rdst) + lst = os.path.join(TEST_DATA_DIR, "qw_ima_empty_list") + with open(lst, "w") as fh: + fh.write("") + + rsync_result = _rsync(["-a", "--files-from=" + lst, source + "/", rdst + "/"]) + assert rsync_result.returncode == 0, rsync_result.stderr + result, _ = run_client(source, dest, flags=["--files-from", lst], + port=shared_server.port) + assert result.returncode == 0, ( + f"empty --files-from must succeed like rsync, got {result.returncode}: " + f"{result.stderr[:300]}" + ) + + @requires_rsync + @pytest.mark.ci + def test_empty_files_from_with_delete_is_not_destructive(self, shared_server): + """An empty --files-from list synchronizes nothing, so --delete must not + wipe the destination (rsync keeps the extra).""" + source = os.path.join(TEST_DATA_DIR, "qw_ima_edel_src") + dest = os.path.join(TEST_DATA_DIR, "qw_ima_edel_dst") + rdst = os.path.join(TEST_DATA_DIR, "qw_ima_edel_rdst") + clean_dir(source) + with open(os.path.join(source, "keep.txt"), "wb") as fh: + fh.write(b"keep\n") + clean_dir(dest) + clean_dir(rdst) + with open(os.path.join(rdst, "extra.txt"), "wb") as fh: + fh.write(b"extra\n") + lst = os.path.join(TEST_DATA_DIR, "qw_ima_edel_list") + with open(lst, "w") as fh: + fh.write("") + rs = _rsync(["-a", "--delete", "--files-from=" + lst, source + "/", rdst + "/"]) + assert rs.returncode == 0, rs.stderr + assert os.path.exists(os.path.join(rdst, "extra.txt")) + + with ServerManager() as server: + server.start(extra_args=["--allow-delete"]) + seed = get_dest_received_dir(dest, source) + os.makedirs(seed, exist_ok=True) + with open(os.path.join(seed, "extra.txt"), "wb") as fh: + fh.write(b"extra\n") + result, _ = run_client(source, dest, + flags=["--delete", "--files-from", lst], + port=server.port) + assert result.returncode == 0, result.stderr[:300] + assert os.path.exists(os.path.join(seed, "extra.txt")), \ + "empty --files-from + --delete must not delete the destination" + + +class TestPerDirFilterOrdering: + """#10: -F .rsync-filter evaluation order matches rsync (a directory's own + rules before its ancestors'; anchored rules are relative to their owner).""" + + def _run_pair(self, shared_server, root_rules, sub_rules, subsub_rules=None): + tag = "qw_f" + source = os.path.join(TEST_DATA_DIR, tag + "_src") + dest = os.path.join(TEST_DATA_DIR, tag + "_dst") + rdst = os.path.join(TEST_DATA_DIR, tag + "_rdst") + clean_dir(source) + os.makedirs(os.path.join(source, "sub")) + if subsub_rules is not None: + os.makedirs(os.path.join(source, "sub", "deep")) + with open(os.path.join(source, "bar"), "wb") as fh: + fh.write(b"bar\n") + with open(os.path.join(source, "sub", "foo"), "wb") as fh: + fh.write(b"foo\n") + if subsub_rules is not None: + with open(os.path.join(source, "sub", "deep", "foo"), "wb") as fh: + fh.write(b"deep foo\n") + with open(os.path.join(source, ".rsync-filter"), "w") as fh: + fh.write(root_rules) + with open(os.path.join(source, "sub", ".rsync-filter"), "w") as fh: + fh.write(sub_rules) + if subsub_rules is not None: + with open(os.path.join(source, "sub", "deep", ".rsync-filter"), "w") as fh: + fh.write(subsub_rules) + + clean_dir(dest) + clean_dir(rdst) + fs_result, _ = run_client(source, dest, flags=["--preserve", "-F"], + port=shared_server.port) + assert fs_result.returncode == 0, fs_result.stderr[:300] + received = get_dest_received_dir(dest, source) + fs_files = _rel_files(received, skip=(".rsync-filter",)) + return fs_files, source, rdst + + @requires_rsync + @pytest.mark.ci + def test_child_include_overrides_parent_exclude(self, shared_server): + fs_files, source, rdst = self._run_pair(shared_server, "- foo\n", "+ foo\n") + rsync_result = _rsync(["-aF", source + "/", rdst + "/"]) + assert rsync_result.returncode == 0, rsync_result.stderr + assert fs_files == _rel_files(rdst, skip=(".rsync-filter",)) + assert "sub/foo" in fs_files + + @requires_rsync + @pytest.mark.ci + def test_child_exclude_overrides_parent_include(self, shared_server): + fs_files, source, rdst = self._run_pair(shared_server, "+ foo\n", "- foo\n") + rsync_result = _rsync(["-aF", source + "/", rdst + "/"]) + assert rsync_result.returncode == 0, rsync_result.stderr + assert fs_files == _rel_files(rdst, skip=(".rsync-filter",)) + assert "sub/foo" not in fs_files + + @requires_rsync + def test_three_level_inheritance(self, shared_server): + fs_files, source, rdst = self._run_pair( + shared_server, "- foo\n", "+ foo\n", "- foo\n" + ) + rsync_result = _rsync(["-aF", source + "/", rdst + "/"]) + assert rsync_result.returncode == 0, rsync_result.stderr + assert fs_files == _rel_files(rdst, skip=(".rsync-filter",)) + + +class TestDeleteEdgeSemantics: + """#20/#23/#24/#25: delete timing/policy edge cases match rsync.""" + + @requires_rsync + @pytest.mark.ci + def test_delete_before_removes_extras_like_rsync(self, shared_server): + source = os.path.join(TEST_DATA_DIR, "qw_db_src") + dest = os.path.join(TEST_DATA_DIR, "qw_db_dst") + rdst = os.path.join(TEST_DATA_DIR, "qw_db_rdst") + clean_dir(source) + with open(os.path.join(source, "keep.txt"), "wb") as fh: + fh.write(b"keep\n") + for d in (dest, rdst): + clean_dir(d) + with open(os.path.join(d, "extra.txt"), "wb") as fh: + fh.write(b"extra\n") + received_seed = get_dest_received_dir(dest, source) + os.makedirs(received_seed, exist_ok=True) + with open(os.path.join(received_seed, "extra.txt"), "wb") as fh: + fh.write(b"extra\n") + + rsync_result = _rsync(["-a", "--delete-before", source + "/", rdst + "/"]) + assert rsync_result.returncode == 0, rsync_result.stderr + with ServerManager() as server: + server.start(extra_args=["--allow-delete"]) + result, _ = run_client(source, dest, flags=["--delete-before"], + port=server.port) + assert result.returncode == 0, result.stderr[:300] + received = get_dest_received_dir(dest, source) + assert os.path.exists(os.path.join(received, "keep.txt")) + assert not os.path.exists(os.path.join(received, "extra.txt")), \ + "--delete-before must remove destination extras" + assert not os.path.exists(os.path.join(rdst, "extra.txt")) + + @requires_rsync + @pytest.mark.ci + def test_delete_excluded_protects_then_deletes(self, shared_server): + source = os.path.join(TEST_DATA_DIR, "qw_de_src") + dest = os.path.join(TEST_DATA_DIR, "qw_de_dst") + rdst = os.path.join(TEST_DATA_DIR, "qw_de_rdst") + clean_dir(source) + with open(os.path.join(source, "keep.txt"), "wb") as fh: + fh.write(b"keep\n") + with open(os.path.join(source, "skip.log"), "wb") as fh: + fh.write(b"log\n") + + # Default --delete protects the excluded mirror (rsync parity). + rdst_prot = os.path.join(TEST_DATA_DIR, "qw_de_rprot") + clean_dir(rdst_prot) + with open(os.path.join(rdst_prot, "skip.log"), "wb") as fh: + fh.write(b"stale\n") + rsync_result = _rsync(["-a", "--delete", "--exclude=*.log", source + "/", + rdst_prot + "/"]) + assert rsync_result.returncode == 0, rsync_result.stderr + assert os.path.exists(os.path.join(rdst_prot, "skip.log")) + + with ServerManager() as server: + server.start(extra_args=["--allow-delete"]) + clean_dir(dest) + seed = get_dest_received_dir(dest, source) + os.makedirs(seed, exist_ok=True) + with open(os.path.join(seed, "skip.log"), "wb") as fh: + fh.write(b"stale\n") + result, _ = run_client(source, dest, flags=["--delete", "--exclude=*.log"], + port=server.port) + assert result.returncode == 0, result.stderr[:300] + received = get_dest_received_dir(dest, source) + assert os.path.exists(os.path.join(received, "skip.log")), \ + "--delete must protect the excluded destination mirror" + + # --delete-excluded removes it. + clean_dir(rdst) + with open(os.path.join(rdst, "skip.log"), "wb") as fh: + fh.write(b"stale\n") + rsync_result = _rsync(["-a", "--delete", "--delete-excluded", + "--exclude=*.log", source + "/", rdst + "/"]) + assert rsync_result.returncode == 0, rsync_result.stderr + assert not os.path.exists(os.path.join(rdst, "skip.log")) + + result, _ = run_client( + source, dest, + flags=["--delete", "--delete-excluded", "--exclude=*.log"], + port=server.port, + ) + assert result.returncode == 0, result.stderr[:300] + assert not os.path.exists(os.path.join(received, "skip.log")), \ + "--delete-excluded must remove the excluded mirror" + + @requires_rsync + def test_force_replaces_nonempty_destination_dir(self, shared_server): + source = os.path.join(TEST_DATA_DIR, "qw_force_src") + dest = os.path.join(TEST_DATA_DIR, "qw_force_dst") + rdst = os.path.join(TEST_DATA_DIR, "qw_force_rdst") + clean_dir(source) + with open(os.path.join(source, "x"), "wb") as fh: + fh.write(b"file-content\n") + clean_dir(rdst) + os.makedirs(os.path.join(rdst, "x")) + with open(os.path.join(rdst, "x", "blocker"), "wb") as fh: + fh.write(b"blocker\n") + + rsync_result = _rsync(["-a", "--force", source + "/", rdst + "/"]) + assert rsync_result.returncode == 0, rsync_result.stderr + assert os.path.isfile(os.path.join(rdst, "x")) + + with ServerManager() as server: + server.start(extra_args=["--allow-delete"]) + clean_dir(dest) + seed = get_dest_received_dir(dest, source) + os.makedirs(os.path.join(seed, "x"), exist_ok=True) + with open(os.path.join(seed, "x", "blocker"), "wb") as fh: + fh.write(b"blocker\n") + result, _ = run_client(source, dest, flags=["--force"], + port=server.port) + assert result.returncode == 0, result.stderr[:300] + received = get_dest_received_dir(dest, source) + assert os.path.isfile(os.path.join(received, "x")), \ + "--force must replace a non-empty destination directory" + + @requires_rsync + def test_no_force_nonempty_dir_is_partial_error(self, shared_server): + """Without --force a non-empty destination dir blocking a file is not + replaced; rsync exits 23, fastsync must not silently mangle it.""" + source = os.path.join(TEST_DATA_DIR, "qw_noforce_src") + dest = os.path.join(TEST_DATA_DIR, "qw_noforce_dst") + rdst = os.path.join(TEST_DATA_DIR, "qw_noforce_rdst") + clean_dir(source) + with open(os.path.join(source, "x"), "wb") as fh: + fh.write(b"file-content\n") + clean_dir(rdst) + os.makedirs(os.path.join(rdst, "x")) + with open(os.path.join(rdst, "x", "blocker"), "wb") as fh: + fh.write(b"blocker\n") + rsync_result = _rsync(["-a", source + "/", rdst + "/"]) + assert rsync_result.returncode == 23, rsync_result.stderr + + with ServerManager() as server: + server.start(extra_args=["--allow-delete"]) + clean_dir(dest) + seed = get_dest_received_dir(dest, source) + os.makedirs(os.path.join(seed, "x"), exist_ok=True) + with open(os.path.join(seed, "x", "blocker"), "wb") as fh: + fh.write(b"blocker\n") + result, _ = run_client(source, dest, port=server.port) + assert result.returncode != 0, "a blocked file install must not report success" + + +def _snapshot(root): + """rel path -> (kind, payload) for every file/symlink below root.""" + result = {} + for dirpath, dirnames, filenames in os.walk(root, followlinks=False): + for name in list(dirnames): + p = os.path.join(dirpath, name) + if os.path.islink(p): + result[os.path.relpath(p, root)] = ("link", os.readlink(p)) + dirnames.remove(name) + for name in filenames: + p = os.path.join(dirpath, name) + if os.path.islink(p): + result[os.path.relpath(p, root)] = ("link", os.readlink(p)) + else: + with open(p, "rb") as fh: + result[os.path.relpath(p, root)] = ("file", fh.read()) + return result + + +def _assert_same_tree(rdst, received, msg=""): + rsync_tree = _snapshot(rdst) + fs_tree = _snapshot(received) + assert fs_tree == rsync_tree, ( + f"tree mismatch {msg}\n----- rsync only/diff -----\n" + f"{ {k: v for k, v in rsync_tree.items() if fs_tree.get(k) != v} }\n" + f"----- fastsync only/diff -----\n" + f"{ {k: v for k, v in fs_tree.items() if rsync_tree.get(k) != v} }" + ) + + +class TestVerifyAndFlip: + """Item 8: confirm already-implemented client-side rows match rsync.""" + + def _src(self, tag): + source = os.path.join(TEST_DATA_DIR, f"vw_{tag}_src") + clean_dir(source) + return source + + def _dst(self, tag): + d = os.path.join(TEST_DATA_DIR, f"vw_{tag}_dst") + clean_dir(d) + return d + + @requires_rsync + @pytest.mark.ci + def test_links_verbatim_matches_rsync(self, shared_server): + source = self._src("links") + dest = self._dst("links") + rdst = self._dst("links_r") + with open(os.path.join(source, "a.txt"), "wb") as fh: + fh.write(b"a\n") + os.symlink("a.txt", os.path.join(source, "rel")) + os.symlink("/etc/hostname", os.path.join(source, "abs")) + os.symlink("../../escape", os.path.join(source, "dd")) + assert _rsync(["-a", source + "/", rdst + "/"]).returncode == 0 + result, _ = run_client(source, dest, flags=["-a"], port=shared_server.port) + assert result.returncode == 0, result.stderr[:300] + _assert_same_tree(rdst, get_dest_received_dir(dest, source), "(-l/--links)") + + @requires_rsync + @pytest.mark.ci + def test_dry_run_does_not_write_like_rsync(self, shared_server): + source = self._src("dry") + dest = self._dst("dry") + rdst = self._dst("dry_r") + with open(os.path.join(source, "a.txt"), "wb") as fh: + fh.write(b"a\n") + os.makedirs(os.path.join(source, "sub")) + with open(os.path.join(source, "sub", "b.txt"), "wb") as fh: + fh.write(b"b\n") + assert _rsync(["-a", "--dry-run", source + "/", rdst + "/"]).returncode == 0 + result, _ = run_client(source, dest, flags=["-a", "--dry-run"], + port=shared_server.port) + assert result.returncode == 0, result.stderr[:300] + _assert_same_tree(rdst, get_dest_received_dir(dest, source), "(--dry-run)") + + @requires_rsync + @pytest.mark.ci + def test_ignore_existing_matches_rsync(self, shared_server): + source = self._src("ie") + dest = self._dst("ie") + rdst = self._dst("ie_r") + with open(os.path.join(source, "a.txt"), "wb") as fh: + fh.write(b"source\n") + with open(os.path.join(source, "new.txt"), "wb") as fh: + fh.write(b"new\n") + for root in (get_dest_received_dir(dest, source), rdst): + os.makedirs(root, exist_ok=True) + with open(os.path.join(root, "a.txt"), "wb") as fh: + fh.write(b"destination-kept\n") + assert _rsync(["-a", "--ignore-existing", source + "/", rdst + "/"]).returncode == 0 + result, _ = run_client(source, dest, flags=["--ignore-existing"], + port=shared_server.port) + assert result.returncode == 0, result.stderr[:300] + _assert_same_tree(rdst, get_dest_received_dir(dest, source), "(--ignore-existing)") + + @requires_rsync + def test_append_matches_rsync(self, shared_server): + source = self._src("app") + dest = self._dst("app") + rdst = self._dst("app_r") + prefix = b"P" * (256 * 1024) + tail = b"T" * (16 * 1024) + with open(os.path.join(source, "grow"), "wb") as fh: + fh.write(prefix + tail) + for root in (get_dest_received_dir(dest, source), rdst): + os.makedirs(root, exist_ok=True) + with open(os.path.join(root, "grow"), "wb") as fh: + fh.write(prefix) + assert _rsync(["-a", "--append", source + "/", rdst + "/"]).returncode == 0 + result, _ = run_client(source, dest, flags=["--append"], port=shared_server.port) + assert result.returncode == 0, result.stderr[:300] + _assert_same_tree(rdst, get_dest_received_dir(dest, source), "(--append)") + + @requires_rsync + def test_append_verify_matches_rsync(self, shared_server): + source = self._src("appv") + dest = self._dst("appv") + rdst = self._dst("appv_r") + prefix = b"Q" * (128 * 1024) + tail = b"Z" * (8 * 1024) + with open(os.path.join(source, "grow"), "wb") as fh: + fh.write(prefix + tail) + for root in (get_dest_received_dir(dest, source), rdst): + os.makedirs(root, exist_ok=True) + with open(os.path.join(root, "grow"), "wb") as fh: + fh.write(prefix) + assert _rsync(["-a", "--append-verify", source + "/", rdst + "/"]).returncode == 0 + result, _ = run_client(source, dest, flags=["--append-verify"], + port=shared_server.port) + assert result.returncode == 0, result.stderr[:300] + _assert_same_tree(rdst, get_dest_received_dir(dest, source), "(--append-verify)") + + @requires_rsync + @pytest.mark.ci + def test_delay_updates_matches_rsync(self, shared_server): + source = self._src("delay") + dest = self._dst("delay") + rdst = self._dst("delay_r") + for rel, content in {"a.txt": b"a\n", "sub/b.txt": b"b\n"}.items(): + full = os.path.join(source, rel) + os.makedirs(os.path.dirname(full), exist_ok=True) + with open(full, "wb") as fh: + fh.write(content) + assert _rsync(["-a", "--delay-updates", source + "/", rdst + "/"]).returncode == 0 + result, _ = run_client(source, dest, flags=["--delay-updates"], + port=shared_server.port) + assert result.returncode == 0, result.stderr[:300] + _assert_same_tree(rdst, get_dest_received_dir(dest, source), "(--delay-updates)") + + @requires_rsync + def test_preallocate_matches_rsync(self, shared_server): + source = self._src("prealloc") + dest = self._dst("prealloc") + rdst = self._dst("prealloc_r") + with open(os.path.join(source, "f.bin"), "wb") as fh: + fh.write(b"x" * (512 * 1024)) + assert _rsync(["-a", "--preallocate", source + "/", rdst + "/"]).returncode == 0 + result, _ = run_client(source, dest, flags=["--preallocate"], + port=shared_server.port) + assert result.returncode == 0, result.stderr[:300] + _assert_same_tree(rdst, get_dest_received_dir(dest, source), "(--preallocate)") + + @requires_rsync + def test_fuzzy_content_matches_rsync(self, shared_server): + source = self._src("fuzzy") + dest = self._dst("fuzzy") + rdst = self._dst("fuzzy_r") + payload = (b"the quick brown fox\n" * 4096) + with open(os.path.join(source, "renamed.txt"), "wb") as fh: + fh.write(payload) + for root in (get_dest_received_dir(dest, source), rdst): + os.makedirs(root, exist_ok=True) + with open(os.path.join(root, "old_name.txt"), "wb") as fh: + fh.write(payload) + assert _rsync(["-a", "--fuzzy", source + "/", rdst + "/"]).returncode == 0 + result, _ = run_client(source, dest, flags=["-y"], port=shared_server.port) + assert result.returncode == 0, result.stderr[:300] + _assert_same_tree(rdst, get_dest_received_dir(dest, source), "(-y/--fuzzy)") + + @requires_rsync + def test_skip_compress_content_matches_rsync(self, shared_server): + source = self._src("skipz") + dest = self._dst("skipz") + rdst = self._dst("skipz_r") + with open(os.path.join(source, "already.zip"), "wb") as fh: + fh.write(b"PK" + b"z" * 4096) + with open(os.path.join(source, "text.txt"), "wb") as fh: + fh.write(b"compress me\n" * 1024) + assert _rsync(["-az", "--skip-compress=gz/zip", source + "/", + rdst + "/"]).returncode == 0 + result, _ = run_client(source, dest, flags=["-z", "--skip-compress=gz/zip"], + port=shared_server.port) + assert result.returncode == 0, result.stderr[:300] + _assert_same_tree(rdst, get_dest_received_dir(dest, source), "(--skip-compress)") + + @requires_rsync + def test_copy_dest_content_matches_rsync(self, shared_server): + source = self._src("copyd") + dest = self._dst("copyd") + with open(os.path.join(source, "f.txt"), "wb") as fh: + fh.write(b"copy-from-basis\n") + # fastsync basis DIR is relative to the receive root; the basis file is + # looked up at the same source-mirror relative path. + rel = os.path.abspath(source).lstrip(os.sep) + basis = os.path.join(dest, "basis", rel) + os.makedirs(basis, exist_ok=True) + with open(os.path.join(basis, "f.txt"), "wb") as fh: + fh.write(b"copy-from-basis\n") + received = get_dest_received_dir(dest, source) + result, _ = run_client(source, dest, + flags=["--copy-dest=basis", "--incremental"], + port=shared_server.port) + assert result.returncode == 0, result.stderr[:300] + assert os.path.exists(os.path.join(received, "f.txt")) + with open(os.path.join(received, "f.txt"), "rb") as fh: + assert fh.read() == b"copy-from-basis\n" + + @requires_rsync + def test_trust_sender_content_matches_rsync(self, shared_server): + source = self._src("trust") + dest = self._dst("trust") + rdst = self._dst("trust_r") + with open(os.path.join(source, "a.txt"), "wb") as fh: + fh.write(b"a\n") + assert _rsync(["-a", "--trust-sender", source + "/", rdst + "/"]).returncode == 0 + result, _ = run_client(source, dest, flags=["--trust-sender"], + port=shared_server.port) + assert result.returncode == 0, result.stderr[:300] + _assert_same_tree(rdst, get_dest_received_dir(dest, source), "(--trust-sender)") + + @requires_rsync + def test_compare_dest_content_matches_rsync(self, shared_server): + source = self._src("cmpd") + dest = self._dst("cmpd") + rdst = self._dst("cmpd_r") + with open(os.path.join(source, "f.txt"), "wb") as fh: + fh.write(b"basis-content\n") + # rsync resolves --compare-dest relative to the destination dir; its + # basis file sits at the transfer-relative path. + os.makedirs(os.path.join(rdst, "basis"), exist_ok=True) + with open(os.path.join(rdst, "basis", "f.txt"), "wb") as fh: + fh.write(b"basis-content\n") + rs = _rsync(["-a", "--compare-dest=basis", source + "/", rdst + "/"]) + assert rs.returncode == 0, rs.stderr + assert not os.path.exists(os.path.join(rdst, "f.txt")), \ + "rsync compare-dest must leave the destination sparse" + + # fastsync resolves the basis DIR relative to the receive root, and the + # file's relative path there mirrors the source path. + rel = os.path.abspath(source).lstrip(os.sep) + basis = os.path.join(dest, "basis", rel) + os.makedirs(basis, exist_ok=True) + with open(os.path.join(basis, "f.txt"), "wb") as fh: + fh.write(b"basis-content\n") + received = get_dest_received_dir(dest, source) + result, _ = run_client(source, dest, + flags=["--compare-dest=basis", "--incremental"], + port=shared_server.port) + assert result.returncode == 0, result.stderr[:300] + # rsync --compare-dest never copies: a basis match is simply not + # transferred, so the destination stays sparse (no f.txt). + assert not os.path.exists(os.path.join(received, "f.txt")) + + @requires_rsync + def test_link_dest_hardlinks_matches_rsync(self, shared_server): + source = self._src("linkd") + dest = self._dst("linkd") + with open(os.path.join(source, "f.txt"), "wb") as fh: + fh.write(b"link-basis-content\n") + rel = os.path.abspath(source).lstrip(os.sep) + basis = os.path.join(dest, "basis", rel) + os.makedirs(basis, exist_ok=True) + basis_file = os.path.join(basis, "f.txt") + with open(basis_file, "wb") as fh: + fh.write(b"link-basis-content\n") + received = get_dest_received_dir(dest, source) + result, _ = run_client(source, dest, + flags=["--link-dest=basis", "--incremental"], + port=shared_server.port) + assert result.returncode == 0, result.stderr[:300] + dest_file = os.path.join(received, "f.txt") + assert os.path.exists(dest_file) + assert os.stat(dest_file).st_ino == os.stat(basis_file).st_ino, \ + "--link-dest must hard-link to the basis file" + + +@pytest.mark.skipif(os.geteuid() != 0, reason="ownership mapping requires root") +class TestOwnershipMapping: + """#33/#34/#35: --usermap/--groupmap/--chown match rsync's numeric result.""" + + def _prep(self, tag): + source = os.path.join(TEST_DATA_DIR, f"own_{tag}_src") + dest = os.path.join(TEST_DATA_DIR, f"own_{tag}_dst") + rdst = os.path.join(TEST_DATA_DIR, f"own_{tag}_rdst") + clean_dir(source) + clean_dir(dest) + clean_dir(rdst) + with open(os.path.join(source, "a.txt"), "wb") as fh: + fh.write(b"a\n") + return source, dest, rdst + + @requires_rsync + def test_usermap_groupmap_matches_rsync(self, shared_server): + source, dest, rdst = self._prep("map") + assert _rsync(["-a", "--usermap=*:12345", "--groupmap=*:54321", + source + "/", rdst + "/"]).returncode == 0 + result, _ = run_client(source, dest, + flags=["--usermap=*:12345", "--groupmap=*:54321"], + port=shared_server.port) + assert result.returncode == 0, result.stderr[:300] + rs = os.stat(os.path.join(rdst, "a.txt")) + fs = os.stat(os.path.join(get_dest_received_dir(dest, source), "a.txt")) + assert (fs.st_uid, fs.st_gid) == (rs.st_uid, rs.st_gid) == (12345, 54321) + + @requires_rsync + def test_chown_matches_rsync(self, shared_server): + source, dest, rdst = self._prep("chown") + assert _rsync(["-a", "--chown=23456:65432", source + "/", + rdst + "/"]).returncode == 0 + result, _ = run_client(source, dest, flags=["--chown=23456:65432"], + port=shared_server.port) + assert result.returncode == 0, result.stderr[:300] + rs = os.stat(os.path.join(rdst, "a.txt")) + fs = os.stat(os.path.join(get_dest_received_dir(dest, source), "a.txt")) + assert (fs.st_uid, fs.st_gid) == (rs.st_uid, rs.st_gid) == (23456, 65432) + + +class TestFakeSuper: + """#32: --fake-super stores privileged attrs via xattrs. FastSync uses its + own reserved key (documented divergence) but the file DATA must match rsync.""" + + @requires_rsync + def test_fake_super_data_matches_rsync(self, shared_server): + source = os.path.join(TEST_DATA_DIR, "fs_src") + dest = os.path.join(TEST_DATA_DIR, "fs_dst") + rdst = os.path.join(TEST_DATA_DIR, "fs_rdst") + clean_dir(source) + clean_dir(dest) + clean_dir(rdst) + with open(os.path.join(source, "a.txt"), "wb") as fh: + fh.write(b"fake-super-data\n") + assert _rsync(["-a", "--fake-super", source + "/", rdst + "/"]).returncode == 0 + result, _ = run_client(source, dest, flags=["--fake-super"], + port=shared_server.port) + assert result.returncode == 0, result.stderr[:300] + with open(os.path.join(get_dest_received_dir(dest, source), "a.txt"), "rb") as fh: + assert fh.read() == b"fake-super-data\n" + with open(os.path.join(rdst, "a.txt"), "rb") as fh: + assert fh.read() == b"fake-super-data\n" + + +class TestInfoDebugFlagParity: + """#01/#02: rsync's info/debug spellings are either mapped to real output or + rejected by name (never silently ignored).""" + + @requires_rsync + def test_mapped_info_categories_accepted_like_rsync(self, shared_server): + source = os.path.join(TEST_DATA_DIR, "qw_flags_src") + dest = os.path.join(TEST_DATA_DIR, "qw_flags_dst") + rdst = os.path.join(TEST_DATA_DIR, "qw_flags_rdst") + clean_dir(source) + clean_dir(dest) + clean_dir(rdst) + with open(os.path.join(source, "a.txt"), "wb") as fh: + fh.write(b"a\n") + for cat in ("stats2", "name", "copy", "misc", "skip", "STATS2"): + assert _rsync(["-a", "--info=" + cat, source + "/", rdst + "/"]).returncode == 0 + clean_dir(rdst) + result, _ = run_client(source, dest, flags=["--info=" + cat], + port=shared_server.port) + assert result.returncode == 0, ( + f"--info={cat} must be accepted: {result.stderr[:200]}" + ) + + @requires_rsync + def test_mapped_debug_categories_accepted_like_rsync(self, shared_server): + source = os.path.join(TEST_DATA_DIR, "qw_dflags_src") + dest = os.path.join(TEST_DATA_DIR, "qw_dflags_dst") + rdst = os.path.join(TEST_DATA_DIR, "qw_dflags_rdst") + clean_dir(source) + clean_dir(dest) + clean_dir(rdst) + with open(os.path.join(source, "a.txt"), "wb") as fh: + fh.write(b"a\n") + for cat in ("io2", "proto0", "all"): + assert _rsync(["-a", "--debug=" + cat, source + "/", rdst + "/"]).returncode == 0 + clean_dir(rdst) + result, _ = run_client(source, dest, flags=["--debug=" + cat], + port=shared_server.port) + assert result.returncode == 0, ( + f"--debug={cat} must be accepted: {result.stderr[:200]}" + ) + + @requires_rsync + @pytest.mark.ci + def test_unmapped_categories_rejected_by_name(self, shared_server): + """rsync accepts del/filter; fastsync has no mapping so it must refuse + loudly, naming the category, rather than silently ignoring it.""" + source = os.path.join(TEST_DATA_DIR, "qw_umap_src") + dest = os.path.join(TEST_DATA_DIR, "qw_umap_dst") + clean_dir(source) + clean_dir(dest) + with open(os.path.join(source, "a.txt"), "wb") as fh: + fh.write(b"a\n") + # rsync accepts these (so they are valid rsync invocations). + assert _rsync(["-a", "--info=del", source + "/", dest + "/"]).returncode == 0 + assert _rsync(["-a", "--debug=filter", source + "/", dest + "/"]).returncode == 0 + for flag, name in (("--info=del", "del"), ("--debug=filter", "filter")): + result, _ = run_client(source, dest, flags=[flag], + port=shared_server.port) + assert result.returncode != 0, f"{flag} must be rejected" + assert name in (result.stderr or ""), \ + f"{flag} must be rejected by name, got: {result.stderr[:200]}" diff --git a/tests/test_client_cli.c b/tests/test_client_cli.c index 50d1072..66c6f3d 100644 --- a/tests/test_client_cli.c +++ b/tests/test_client_cli.c @@ -1317,7 +1317,66 @@ static void test_parse_args_rejects_invalid_info_flag() { config_delete(cfg); } +/* rsync's info "name" category maps to fastsync's per-file name logging, and + * --info=help prints the flag list and exits without error. */ +static void test_parse_args_info_name_and_help() { + Config* cfg = config_create(); + char* argv[] = {"fastsync", "--info=name", "/src", "/dst"}; + int positional_args[2]; + int positional_count = 0; + EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0); + EXPECT_EQ_INT(cfg->info_level, LOG_INFO_COPY); + config_delete(cfg); + + cfg = config_create(); + char* help_argv[] = {"fastsync", "--info=help"}; + positional_count = 0; + EXPECT_EQ_INT(parse_args(cfg, 2, help_argv, positional_args, &positional_count), 1); + config_delete(cfg); +} + /* Test parse_args with --archive flag */ +/* rsync accepts a trailing level digit on --debug/--info items (e.g. io2, + * all4); level 0 silences the item. */ +static void test_parse_args_debug_info_levels() { + Config* cfg = config_create(); + char* argv[] = {"fastsync", "--debug=io2,proto0,all", "/src", "/dst"}; + int positional_args[2]; + int positional_count = 0; + EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0); + EXPECT_EQ_INT(cfg->debug_level, LOG_DEBUG_ALL); + config_delete(cfg); + + cfg = config_create(); + char* io0_argv[] = {"fastsync", "--debug=all,io0", "/src", "/dst"}; + positional_count = 0; + EXPECT_EQ_INT(parse_args(cfg, 4, io0_argv, positional_args, &positional_count), 0); + EXPECT_EQ_INT(cfg->debug_level, LOG_DEBUG_ALL & ~LOG_DEBUG_IO); + config_delete(cfg); + + cfg = config_create(); + char* info_argv[] = {"fastsync", "--info=stats2", "/src", "/dst"}; + positional_count = 0; + EXPECT_EQ_INT(parse_args(cfg, 4, info_argv, positional_args, &positional_count), 0); + EXPECT_EQ_INT(cfg->info_level, LOG_INFO_STATS); + config_delete(cfg); + + cfg = config_create(); + char* bad_argv[] = {"fastsync", "--debug=123", "/src", "/dst"}; + positional_count = 0; + EXPECT_EQ_INT(parse_args(cfg, 4, bad_argv, positional_args, &positional_count), -1); + config_delete(cfg); + + /* rsync accepts category names case-insensitively. */ + cfg = config_create(); + char* upper_argv[] = {"fastsync", "--info=STATS2", "--debug=IO", "/src", "/dst"}; + positional_count = 0; + EXPECT_EQ_INT(parse_args(cfg, 5, upper_argv, positional_args, &positional_count), 0); + EXPECT_EQ_INT(cfg->info_level, LOG_INFO_STATS); + EXPECT_EQ_INT(cfg->debug_level, LOG_DEBUG_IO); + config_delete(cfg); +} + static void test_parse_args_archive() { Config* cfg = config_create(); char* argv[] = {"fastsync", "--archive", "/src", "/dst"}; @@ -4255,6 +4314,7 @@ void test_client_cli() { test_parse_args_debug_flags(); test_parse_args_debug_help(); test_parse_args_debug_flags_validation(); + test_parse_args_debug_info_levels(); test_parse_args_modify_window(); test_parse_args_rejects_invalid_modify_window(); test_parse_args_skip_compress(); @@ -4278,6 +4338,7 @@ void test_client_cli() { test_parse_args_info_flags(); test_parse_args_info_verbose_order(); test_parse_args_rejects_invalid_info_flag(); + test_parse_args_info_name_and_help(); test_parse_args_archive(); test_parse_args_preserve_attributes_are_independent(); test_parse_args_preserve_long_form(); diff --git a/tests/test_scanner.c b/tests/test_scanner.c index 2bc9980..2cdf201 100644 --- a/tests/test_scanner.c +++ b/tests/test_scanner.c @@ -1533,6 +1533,49 @@ static void test_scanner_entry_classification() { rmdir(root); } +/* A dereferenced symlink with no referent (broken/unreadable) must record a + * non-fatal I/O error so the run can exit 23 like rsync, without aborting the + * scan or treating the condition as a fatal failure. */ +static void test_scanner_broken_referent_io_error(void) { + const char* root = "test_scan_broken_ref"; + const char* good = "test_scan_broken_ref/good.txt"; + const char* broken = "test_scan_broken_ref/broken"; + + EXPECT_EQ_INT(mkdir(root, 0755), 0); + create_test_file(good, "hello"); + EXPECT_EQ_INT(symlink("/nonexistent/quickwins/target", broken), 0); + + { + ScannerOptions options = {0}; + options.copy_links = true; + DirectoryScanner* scanner = directory_scanner_create_with_options(root, &options); + EXPECT_NOT_NULL(scanner); + Chunk* chunk; + while ((chunk = directory_scanner_next(scanner)) != NULL) + chunk_destroy(chunk); + EXPECT_FALSE(directory_scanner_failed(scanner)); + EXPECT_TRUE(directory_scanner_had_io_error(scanner)); + directory_scanner_destroy(scanner); + } + + { + ScannerOptions options = {0}; + options.copy_links = true; + ParallelScanner* scanner = parallel_scanner_create_with_options(root, &options, NULL); + EXPECT_NOT_NULL(scanner); + Chunk* chunk; + while ((chunk = parallel_scanner_next(scanner)) != NULL) + chunk_destroy(chunk); + EXPECT_FALSE(parallel_scanner_failed(scanner)); + EXPECT_TRUE(parallel_scanner_had_io_error(scanner)); + parallel_scanner_destroy(scanner); + } + + unlink(broken); + unlink(good); + rmdir(root); +} + void test_scanner() { test_scanner_single_file(); test_scanner_multiple_files(); @@ -1551,6 +1594,7 @@ void test_scanner() { test_scanner_one_file_system_decision(); test_scanner_one_file_system_same_device(); test_parallel_scanner_one_file_system_same_device(); + test_scanner_broken_referent_io_error(); test_scanner_one_file_system_cross_device(); test_files_from_subset(false); test_files_from_subset(true);