fix(parity): client quick wins for rsync 3.4.1 (copy-links exit 23, info/debug flags, empty files-from, -F ordering, delete edges)

This commit is contained in:
2026-09-16 22:21:45 +02:00
parent 1116da9f64
commit e674b25213
9 changed files with 1108 additions and 51 deletions
+95 -23
View File
@@ -395,6 +395,39 @@ static void apply_output_buffering(const Config* config) {
static int read_patterns_from_file(const char* filepath, char*** patterns, int* count, static int read_patterns_from_file(const char* filepath, char*** patterns, int* count,
Config* config, char sign, const char* optname); Config* config, char sign, const char* optname);
/* Split one --debug/--info item into its category name and an optional rsync
* verbosity level suffix (e.g. "io2", "all4", "none0"). The output `name` is
* NUL-terminated and `level` is >= 0 (0 silences the item). Returns false for
* an empty token or a token that is all digits. */
static bool split_flag_level(const char* token, char* name, size_t name_size, int* level) {
size_t len = strlen(token);
if (len == 0 || name_size == 0)
return false;
size_t end = len;
while (end > 0 && token[end - 1] >= '0' && token[end - 1] <= '9')
end--;
if (end == 0)
return false; /* all digits: not a category name */
size_t name_len = end < name_size - 1 ? end : name_size - 1;
for (size_t i = 0; i < name_len; i++) {
char c = token[i];
name[i] = (c >= 'A' && c <= 'Z') ? (char)(c - 'A' + 'a') : c;
}
name[name_len] = '\0';
int lvl = 1;
if (end < len) {
lvl = 0;
for (size_t i = end; i < len; i++) {
int digit = token[i] - '0';
if (lvl > (1000 - digit) / 10)
return false;
lvl = lvl * 10 + digit;
}
}
*level = lvl;
return true;
}
static int parse_debug_flags(const char* value, Config* config) { static int parse_debug_flags(const char* value, Config* config) {
if (!value || value[0] == '\0' || value[0] == ',' || value[strlen(value) - 1] == ',' || if (!value || value[0] == '\0' || value[0] == ',' || value[strlen(value) - 1] == ',' ||
strstr(value, ",,")) { strstr(value, ",,")) {
@@ -412,29 +445,39 @@ static int parse_debug_flags(const char* value, Config* config) {
for (char* token = strtok_r(flags, ",", &saveptr); token != NULL; for (char* token = strtok_r(flags, ",", &saveptr); token != NULL;
token = strtok_r(NULL, ",", &saveptr)) { token = strtok_r(NULL, ",", &saveptr)) {
uint32_t flag = 0; uint32_t flag = 0;
if (strcmp(token, "help") == 0) { char name[32];
int level = 1;
if (!split_flag_level(token, name, sizeof(name), &level)) {
log_message(LOG_LEVEL_ERROR, "unsupported --debug flag: %s", token);
free(flags);
return -1;
}
if (strcmp(name, "help") == 0) {
print_debug_usage(); print_debug_usage();
free(flags); free(flags);
return 1; return 1;
} else if (strcmp(token, "all") == 0) { } else if (strcmp(name, "all") == 0) {
parsed = LOG_DEBUG_ALL; parsed = level == 0 ? 0 : LOG_DEBUG_ALL;
continue; continue;
} else if (strcmp(token, "none") == 0) { } else if (strcmp(name, "none") == 0) {
parsed = 0; parsed = 0;
continue; continue;
} else if (strcmp(token, "io") == 0) { } else if (strcmp(name, "io") == 0) {
flag = LOG_DEBUG_IO; flag = LOG_DEBUG_IO;
} else if (strcmp(token, "proto") == 0) { } else if (strcmp(name, "proto") == 0) {
flag = LOG_DEBUG_PROTO; flag = LOG_DEBUG_PROTO;
} else if (strcmp(token, "pack") == 0) { } else if (strcmp(name, "pack") == 0) {
flag = LOG_DEBUG_PACK; flag = LOG_DEBUG_PACK;
} else if (strcmp(token, "util") == 0) { } else if (strcmp(name, "util") == 0) {
flag = LOG_DEBUG_UTIL; flag = LOG_DEBUG_UTIL;
} else { } else {
log_message(LOG_LEVEL_ERROR, "unsupported --debug flag: %s", token); log_message(LOG_LEVEL_ERROR, "unsupported --debug flag: %s", token);
free(flags); free(flags);
return -1; return -1;
} }
if (level == 0)
parsed &= ~flag;
else
parsed |= flag; parsed |= flag;
} }
free(flags); free(flags);
@@ -461,27 +504,42 @@ static int parse_info_flags(const char* value, Config* config) {
for (char* token = strtok_r(flags, ",", &saveptr); token != NULL; for (char* token = strtok_r(flags, ",", &saveptr); token != NULL;
token = strtok_r(NULL, ",", &saveptr)) { token = strtok_r(NULL, ",", &saveptr)) {
uint32_t flag = 0; uint32_t flag = 0;
if (strcmp(token, "all") == 0) { char name[32];
parsed = LOG_INFO_ALL; int level = 1;
if (!split_flag_level(token, name, sizeof(name), &level)) {
log_message(LOG_LEVEL_ERROR, "unsupported --info flag: %s", token);
free(flags);
return -1;
}
if (strcmp(name, "all") == 0) {
parsed = level == 0 ? 0 : LOG_INFO_ALL;
continue; continue;
} }
if (strcmp(token, "none") == 0) { if (strcmp(name, "none") == 0) {
parsed = 0; parsed = 0;
continue; continue;
} }
if (strcmp(token, "copy") == 0) if (strcmp(name, "help") == 0) {
print_info_usage();
free(flags);
return 1;
}
if (strcmp(name, "copy") == 0 || strcmp(name, "name") == 0)
flag = LOG_INFO_COPY; flag = LOG_INFO_COPY;
else if (strcmp(token, "misc") == 0) else if (strcmp(name, "misc") == 0)
flag = LOG_INFO_MISC; flag = LOG_INFO_MISC;
else if (strcmp(token, "skip") == 0) else if (strcmp(name, "skip") == 0)
flag = LOG_INFO_SKIP; flag = LOG_INFO_SKIP;
else if (strcmp(token, "stats") == 0) else if (strcmp(name, "stats") == 0)
flag = LOG_INFO_STATS; flag = LOG_INFO_STATS;
else { else {
log_message(LOG_LEVEL_ERROR, "unsupported --info flag: %s", token); log_message(LOG_LEVEL_ERROR, "unsupported --info flag: %s", token);
free(flags); free(flags);
return -1; return -1;
} }
if (level == 0)
parsed &= ~flag;
else
parsed |= flag; parsed |= flag;
} }
free(flags); free(flags);
@@ -1027,17 +1085,22 @@ typedef struct {
} CliParseCtx; } CliParseCtx;
/* Apply output controls before processing other options so their order is /* Apply output controls before processing other options so their order is
* irrelevant. Returns 0 on success, -1 on error. */ * irrelevant. Returns 0 on success, a positive code for a help request
* (parse_args returns it verbatim), or -1 on error. */
static int cli_apply_output_controls(Config* config, int argc, char* argv[]) { static int cli_apply_output_controls(Config* config, int argc, char* argv[]) {
for (int i = 1; i < argc; i++) { for (int i = 1; i < argc; i++) {
if (strcmp(argv[i], "-v") == 0 || strcmp(argv[i], "--verbose") == 0) { if (strcmp(argv[i], "-v") == 0 || strcmp(argv[i], "--verbose") == 0) {
set_log_level(LOG_LEVEL_DEBUG); set_log_level(LOG_LEVEL_DEBUG);
} else if (strncmp(argv[i], "--info=", 7) == 0) { } else if (strncmp(argv[i], "--info=", 7) == 0) {
if (parse_info_flags(argv[i] + 7, config) != 0) int ret = parse_info_flags(argv[i] + 7, config);
return -1; if (ret != 0)
return ret;
} else if (strcmp(argv[i], "--info") == 0) { } else if (strcmp(argv[i], "--info") == 0) {
if (i + 1 >= argc || parse_info_flags(argv[++i], config) != 0) if (i + 1 >= argc)
return -1; return -1;
int ret = parse_info_flags(argv[++i], config);
if (ret != 0)
return ret;
} }
} }
return 0; return 0;
@@ -1842,15 +1905,21 @@ static bool cli_handle_logging_options(CliParseCtx* ctx) {
return true; return true;
} }
if (strncmp(arg, "--info=", 7) == 0) { if (strncmp(arg, "--info=", 7) == 0) {
if (parse_info_flags(arg + 7, config) != 0) int info_ret = parse_info_flags(arg + 7, config);
ctx->exit_code = -1; if (info_ret != 0)
ctx->exit_code = info_ret;
return true; return true;
} }
if (opt_is(arg, "--info", NULL)) { if (opt_is(arg, "--info", NULL)) {
if (ctx->i + 1 >= ctx->argc || parse_info_flags(ctx->argv[++ctx->i], config) != 0) if (ctx->i + 1 >= ctx->argc) {
ctx->exit_code = -1; ctx->exit_code = -1;
return true; return true;
} }
int info_ret = parse_info_flags(ctx->argv[++ctx->i], config);
if (info_ret != 0)
ctx->exit_code = info_ret;
return true;
}
if (strncmp(arg, "--skip-compress=", 16) == 0) { if (strncmp(arg, "--skip-compress=", 16) == 0) {
if (parse_skip_compress(config, arg + 16) != 0) if (parse_skip_compress(config, arg + 16) != 0)
ctx->exit_code = -1; ctx->exit_code = -1;
@@ -2436,8 +2505,11 @@ int parse_args(Config* config, int argc, char* argv[], int* positional_args,
} }
int result = -1; int result = -1;
if (cli_apply_output_controls(config, exp_argc, exp_argv) != 0) int output_ret = cli_apply_output_controls(config, exp_argc, exp_argv);
if (output_ret != 0) {
result = output_ret;
goto done; goto done;
}
CliParseCtx ctx = { CliParseCtx ctx = {
.config = config, .config = config,
+23 -18
View File
@@ -328,15 +328,15 @@ static char* files_from_missing_dest_path(const Config* config, const char* entr
/* --files-from semantics: every listed entry must resolve under the source /* --files-from semantics: every listed entry must resolve under the source
* root, otherwise rsync reports a hard error instead of silently transferring * root, otherwise rsync reports a hard error instead of silently transferring
* nothing. An empty list is also an error. An entry of "." (the whole tree) * nothing. An entry of "." (the whole tree) and listed-but-empty directories
* and listed-but-empty directories are valid. With --ignore-missing-args * are valid. An empty list is valid too: rsync transfers nothing and exits 0.
* With --ignore-missing-args
* (implied by --delete-missing-args) a listed-but-missing entry is instead * (implied by --delete-missing-args) a listed-but-missing entry is instead
* skipped: nothing is transferred for it, it never enters the keep-set and the * skipped: nothing is transferred for it, it never enters the keep-set and the
* run succeeds for the rest (an all-missing non-empty list succeeds * run succeeds for the rest (an all-missing non-empty list succeeds
* transferring nothing, matching rsync). With --delete-missing-args * transferring nothing, matching rsync). With --delete-missing-args
* `missing_dest` (when non-NULL) collects the entry's destination-relative * `missing_dest` (when non-NULL) collects the entry's destination-relative
* mirror for the receiver's exact-deletion request. An empty list stays a * mirror for the receiver's exact-deletion request. Runs before any
* hard error in every mode (nothing was requested at all). Runs before any
* transfer so the failure/skip is surfaced uniformly in the single-threaded, * transfer so the failure/skip is surfaced uniformly in the single-threaded,
* -m, dry-run and --list-only paths. */ * -m, dry-run and --list-only paths. */
static bool files_from_list_check(const Config* config, ArrayList* missing_dest, int* skipped_out) { static bool files_from_list_check(const Config* config, ArrayList* missing_dest, int* skipped_out) {
@@ -349,12 +349,11 @@ static bool files_from_list_check(const Config* config, ArrayList* missing_dest,
return false; return false;
} }
if (set->count == 0) { if (set->count == 0) {
char* escaped_list = /* rsync treats an empty --files-from list as "nothing to transfer" and
output_escape(config->files_from ? config->files_from : "", log_get_8_bit_output()); exits 0 (the source directory is still a valid source arg), so this is
log_message(LOG_LEVEL_ERROR, "--files-from file '%s' contains no entries; nothing to transfer", not an error. Nothing passes the (empty) allow-set, so no file is sent
escaped_list ? escaped_list : "<allocation failed>"); and no keep-set entry is produced. */
free(escaped_list); return true;
return false;
} }
bool ignore = config->ignore_missing_args || config->delete_missing_args; bool ignore = config->ignore_missing_args || config->delete_missing_args;
for (int i = 0; i < set->count; i++) { for (int i = 0; i < set->count; i++) {
@@ -2679,12 +2678,15 @@ int send_files(Config* config) {
report_transfer_stats(config, total_files, total_bytes, start); report_transfer_stats(config, total_files, total_bytes, start);
log_info_message(LOG_INFO_STATS, "Transfer summary: %d files, %.1f MB", total_files, log_info_message(LOG_INFO_STATS, "Transfer summary: %d files, %.1f MB", total_files,
(double)total_bytes / (double)BYTES_PER_MIB); (double)total_bytes / (double)BYTES_PER_MIB);
/* --ignore-errors: an unreadable source directory was skipped but the run /* A skipped source entry (--ignore-errors past an unreadable directory, or a
still completed (and deleted); report the run as errored like rsync does. dereferenced symlink with no referent) makes rsync report a partial
A --max-delete-capped commit is a successful transfer that rsync reports transfer (exit 23) even though the rest of the run succeeded. A
--max-delete-capped commit is a successful transfer that rsync reports
with exit code 25. */ with exit code 25. */
if (!ok || had_scan_io) if (!ok)
ret = 1; ret = 1;
else if (had_scan_io)
ret = 23;
else else
ret = delete_limit ? 25 : 0; ret = delete_limit ? 25 : 0;
@@ -2921,12 +2923,15 @@ int send_files_multithreaded(Config** config_ptr) {
mtx_unlock(&context->mutex_scanner); mtx_unlock(&context->mutex_scanner);
bool sender_ok = sender_result == thrd_success; bool sender_ok = sender_result == thrd_success;
bool delete_limit = context->delete_limit; bool delete_limit = context->delete_limit;
/* --ignore-errors: the run completed (and deleted) past an unreadable source /* A skipped source entry (--ignore-errors past an unreadable directory, or a
directory; report it as errored like rsync does. A --max-delete-capped dereferenced symlink with no referent) makes rsync report a partial
commit is a successful transfer that rsync reports with exit code 25. */ transfer (exit 23). A --max-delete-capped commit is a successful transfer
that rsync reports with exit code 25. */
pipeline_context_sender_destroy(context); pipeline_context_sender_destroy(context);
client_set_abort_armed(false); client_set_abort_armed(false);
if (!sender_ok || scan_io) if (!sender_ok)
return 1; return 1;
if (scan_io)
return 23;
return delete_limit ? 25 : 0; return delete_limit ? 25 : 0;
} }
+16 -2
View File
@@ -163,6 +163,12 @@ typedef struct {
Size pruning protects the destination mirror even under --delete-excluded, Size pruning protects the destination mirror even under --delete-excluded,
so it is recorded into a separate sink from `excluded`. */ so it is recorded into a separate sink from `excluded`. */
bool size_excluded; bool size_excluded;
/* True when a symlink selected for dereferencing (-L/--copy-links or an
unsafe target under --copy-unsafe-links) had no usable referent (a broken
link or a stat() failure). rsync still reports this as a partial transfer
(exit 23) even though the entry is skipped, so the scanner records it as a
non-fatal I/O error. */
bool referent_error;
} ScannerEntry; } ScannerEntry;
/* --one-file-system (-x) decision. Only directories can carry a different /* --one-file-system (-x) decision. Only directories can carry a different
@@ -412,6 +418,7 @@ static int scanner_inspect_entry(const ScannerOptions* options, const char* cont
const char* link_rel, const char* name, ScannerEntry* entry) { const char* link_rel, const char* name, ScannerEntry* entry) {
entry->excluded = false; entry->excluded = false;
entry->size_excluded = false; entry->size_excluded = false;
entry->referent_error = false;
entry->is_symlink = false; entry->is_symlink = false;
entry->link_target = NULL; entry->link_target = NULL;
entry->path = path_cat(containing_dir, name); entry->path = path_cat(containing_dir, name);
@@ -438,12 +445,13 @@ static int scanner_inspect_entry(const ScannerOptions* options, const char* cont
goto skip; goto skip;
case LINK_ACTION_DEREF: case LINK_ACTION_DEREF:
if (stat(entry->path, &entry->stats) != 0) { if (stat(entry->path, &entry->stats) != 0) {
/* rsync reports "symlink has no referent" and continues (exit 23); we /* rsync reports "symlink has no referent" and continues with a partial
surface the same condition rather than silently dropping the entry. */ transfer (exit 23); record the error so the run exits 23 too. */
char* escaped = output_escape(entry->path, log_get_8_bit_output()); char* escaped = output_escape(entry->path, log_get_8_bit_output());
log_message(LOG_LEVEL_WARNING, "symlink has no referent: %s", log_message(LOG_LEVEL_WARNING, "symlink has no referent: %s",
escaped ? escaped : "<allocation failed>"); escaped ? escaped : "<allocation failed>");
free(escaped); free(escaped);
entry->referent_error = true;
goto skip; goto skip;
} }
entry->is_directory = S_ISDIR(entry->stats.st_mode); entry->is_directory = S_ISDIR(entry->stats.st_mode);
@@ -1118,6 +1126,10 @@ Chunk* directory_scanner_next(DirectoryScanner* scanner) {
break; break;
} }
if (inspection == 0) { if (inspection == 0) {
/* A dereferenced symlink with no referent is a partial-transfer error
(rsync exit 23): record it as a non-fatal scan I/O error. */
if (inspected.referent_error)
scanner->io_error = true;
/* A user-selection exclude protects its destination mirror from --delete /* A user-selection exclude protects its destination mirror from --delete
unless --delete-excluded; a size prune is always protected. Other unless --delete-excluded; a size prune is always protected. Other
skips (unreadable, symlink policy) protect nothing. Under -R + skips (unreadable, symlink policy) protect nothing. Under -R +
@@ -1511,6 +1523,8 @@ static void scan_root_entry(const ScannerOptions* options, const FilterNode* roo
return; return;
} }
if (inspection == 0) { if (inspection == 0) {
if (inspected.referent_error)
ps->io_error = true;
ArrayList* sink = NULL; ArrayList* sink = NULL;
if (inspected.excluded) if (inspected.excluded)
sink = inspected.size_excluded ? options->size_skipped_paths : options->excluded_paths; sink = inspected.size_excluded ? options->size_skipped_paths : options->excluded_paths;
+11 -3
View File
@@ -171,8 +171,8 @@ void print_usage(void) {
printf(" -v, --verbose Enable debug logging\n"); printf(" -v, --verbose Enable debug logging\n");
printf(" -q, --quiet Suppress non-error output\n"); printf(" -q, --quiet Suppress non-error output\n");
printf(" --debug=FLAGS Fine-grained debug logging (use --debug=help for flags)\n"); printf(" --debug=FLAGS Fine-grained debug logging (use --debug=help for flags)\n");
printf(" --info=FLAGS Fine-grained info: copy,misc,skip,stats,all,none\n"); printf(" --info=FLAGS Fine-grained info: copy,name,misc,skip,stats,all,none\n");
printf(" none suppresses info even with --verbose\n"); printf(" (use --info=help for flags; none suppresses --verbose)\n");
printf(" --preserve Preserve permissions and times (= -pt; long form only)\n"); printf(" --preserve Preserve permissions and times (= -pt; long form only)\n");
printf(" --no-perms Negate -p/--perms\n"); printf(" --no-perms Negate -p/--perms\n");
printf(" --no-times Negate -t/--times\n"); printf(" --no-times Negate -t/--times\n");
@@ -340,5 +340,13 @@ void print_usage(void) {
void print_debug_usage(void) { void print_debug_usage(void) {
printf("Supported debug flags: IO,PROTO,PACK,UTIL,ALL,NONE\n"); printf("Supported debug flags: IO,PROTO,PACK,UTIL,ALL,NONE\n");
printf("Flags may be comma-separated, for example: --debug=io,proto\n"); printf("Flags may be comma-separated, for example: --debug=io,proto\n");
printf("Other rsync debug flags are unsupported and rejected.\n"); printf("An optional level suffix is accepted (e.g. --debug=io2); level 0\n");
printf("silences that item. Other rsync debug flags are unsupported and rejected.\n");
}
void print_info_usage(void) {
printf("Supported info flags: COPY,NAME,MISC,SKIP,STATS,ALL,NONE\n");
printf("Flags may be comma-separated, for example: --info=name,stats\n");
printf("An optional level suffix is accepted (e.g. --info=stats2); level 0\n");
printf("silences that item. Other rsync info flags are unsupported and rejected.\n");
} }
+1
View File
@@ -3,5 +3,6 @@
void print_usage(void); void print_usage(void);
void print_debug_usage(void); void print_debug_usage(void);
void print_info_usage(void);
#endif #endif
+9 -3
View File
@@ -3157,15 +3157,21 @@ class TestMissingArgs:
assert not os.path.exists(os.path.join(received, "gone1.txt")) assert not os.path.exists(os.path.join(received, "gone1.txt"))
@pytest.mark.parametrize("mt", [False, True]) @pytest.mark.parametrize("mt", [False, True])
def test_empty_list_stays_a_hard_error(self, shared_server, mt): def test_empty_list_succeeds_transferring_nothing(self, shared_server, mt):
"""rsync 3.4.1 treats an empty --files-from list as "nothing to
transfer" and exits 0 (verified with the real binary), so fastsync must
too rather than reporting a hard error."""
source = self._make_source("mg_empty_src") source = self._make_source("mg_empty_src")
dest = os.path.join(TEST_DATA_DIR, "mg_empty_dst") dest = os.path.join(TEST_DATA_DIR, "mg_empty_dst")
clean_dir(dest) clean_dir(dest)
lst = _write_rel_list(b"") lst = _write_rel_list(b"")
flags = ["--files-from", lst, "--ignore-missing-args"] + (["--threads"] if mt else []) flags = ["--files-from", lst, "--ignore-missing-args"] + (["--threads"] if mt else [])
result, _ = run_client(source, dest, flags=flags, port=shared_server.port) result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
assert result.returncode != 0, "an empty --files-from list must stay a hard error" assert result.returncode == 0, \
assert "contains no entries" in (result.stderr or result.stdout) f"an empty --files-from list must succeed like rsync: {result.stderr[:300]}"
received = get_dest_received_dir(dest, source)
assert not os.path.exists(os.path.join(received, "a.txt")), \
"an empty --files-from list must transfer nothing"
@pytest.mark.parametrize("mt", [False, True]) @pytest.mark.parametrize("mt", [False, True])
def test_delete_missing_removes_mirror_not_unrelated(self, mt): def test_delete_missing_removes_mirror_not_unrelated(self, mt):
+846
View File
@@ -0,0 +1,846 @@
"""Client-only rsync-parity quick wins.
Each test here pins behaviour that must match real ``rsync 3.4.1``. The
differential tests skip cleanly when rsync is not installed.
"""
import os
import shutil
import subprocess
import sys
import pytest
sys.path.insert(0, os.path.dirname(__file__))
from common import (
TEST_DATA_DIR,
ServerManager,
run_client,
clean_dir,
get_dest_received_dir,
)
RSYNC = shutil.which("rsync")
requires_rsync = pytest.mark.skipif(RSYNC is None, reason="rsync 3.4.1 not installed")
def _rel_files(root, skip=()):
"""Relative paths of regular files and symlinks below root, sorted."""
out = []
for dirpath, _dirs, files in os.walk(root):
for name in files:
if name in skip:
continue
out.append(os.path.relpath(os.path.join(dirpath, name), root))
return sorted(out)
def _rsync(args):
env = dict(os.environ, LC_ALL="C")
return subprocess.run(
[RSYNC] + args, capture_output=True, text=True, env=env, timeout=120
)
class TestCopyLinksReferentError:
"""#38/#39: a broken referent under -L/--copy-unsafe-links exits 23."""
def _make_broken_tree(self, root):
clean_dir(root)
with open(os.path.join(root, "ok.txt"), "wb") as fh:
fh.write(b"hello\n")
os.symlink("/nonexistent/target", os.path.join(root, "broken"))
@requires_rsync
@pytest.mark.ci
def test_copy_links_broken_referent_exit_23(self, shared_server):
source = os.path.join(TEST_DATA_DIR, "qw_cl_src")
dest = os.path.join(TEST_DATA_DIR, "qw_cl_dst")
rdst = os.path.join(TEST_DATA_DIR, "qw_cl_rdst")
self._make_broken_tree(source)
clean_dir(dest)
clean_dir(rdst)
rsync_result = _rsync(["-aL", source + "/", rdst + "/"])
assert rsync_result.returncode == 23, rsync_result.stderr
result, _ = run_client(source, dest, flags=["-L"], port=shared_server.port)
assert result.returncode == 23, (
f"-L broken referent must exit 23, got {result.returncode}: "
f"{result.stderr[:300]}"
)
received = get_dest_received_dir(dest, source)
assert os.path.exists(os.path.join(received, "ok.txt"))
@requires_rsync
def test_copy_links_broken_referent_multithreaded_exit_23(self, shared_server):
source = os.path.join(TEST_DATA_DIR, "qw_cl_mt_src")
dest = os.path.join(TEST_DATA_DIR, "qw_cl_mt_dst")
self._make_broken_tree(source)
clean_dir(dest)
result, _ = run_client(source, dest, flags=["-L", "--threads"], port=shared_server.port)
assert result.returncode == 23, (
f"-L broken referent must exit 23 under --threads, got {result.returncode}"
)
@requires_rsync
@pytest.mark.ci
def test_copy_unsafe_links_broken_referent_exit_23(self, shared_server):
source = os.path.join(TEST_DATA_DIR, "qw_cul_src")
dest = os.path.join(TEST_DATA_DIR, "qw_cul_dst")
rdst = os.path.join(TEST_DATA_DIR, "qw_cul_rdst")
clean_dir(source)
os.makedirs(os.path.join(source, "sub"))
with open(os.path.join(source, "ok.txt"), "wb") as fh:
fh.write(b"hello\n")
# Unsafe (escaping) target with no referent: dereferenced -> exit 23.
os.symlink("../../../nonexistent/target", os.path.join(source, "sub", "unsafe"))
clean_dir(dest)
clean_dir(rdst)
rsync_result = _rsync(["-a", "--copy-unsafe-links", source + "/", rdst + "/"])
assert rsync_result.returncode == 23, rsync_result.stderr
result, _ = run_client(source, dest, flags=["-l", "--copy-unsafe-links"],
port=shared_server.port)
assert result.returncode == 23, (
f"--copy-unsafe-links broken referent must exit 23, got {result.returncode}"
)
@requires_rsync
def test_copy_unsafe_links_safe_broken_stays_symlink(self, shared_server):
"""A safe (non-escaping) broken symlink is NOT dereferenced: exit 0."""
source = os.path.join(TEST_DATA_DIR, "qw_cul_safe_src")
dest = os.path.join(TEST_DATA_DIR, "qw_cul_safe_dst")
rdst = os.path.join(TEST_DATA_DIR, "qw_cul_safe_rdst")
clean_dir(source)
os.makedirs(os.path.join(source, "sub"))
os.symlink("nonexistent-target", os.path.join(source, "sub", "safe"))
clean_dir(dest)
clean_dir(rdst)
rsync_result = _rsync(["-a", "--copy-unsafe-links", source + "/", rdst + "/"])
assert rsync_result.returncode == 0, rsync_result.stderr
result, _ = run_client(source, dest, flags=["-l", "--copy-unsafe-links"],
port=shared_server.port)
assert result.returncode == 0, result.stderr[:300]
received = get_dest_received_dir(dest, source)
assert os.path.islink(os.path.join(received, "sub", "safe"))
class TestIgnoreMissingArgsParity:
"""#58: --files-from + --ignore-missing-args matches rsync, including the
empty-list case (rsync exits 0 transferring nothing)."""
@requires_rsync
@pytest.mark.ci
def test_missing_entry_skipped_like_rsync(self, shared_server):
source = os.path.join(TEST_DATA_DIR, "qw_ima_src")
dest = os.path.join(TEST_DATA_DIR, "qw_ima_dst")
rdst = os.path.join(TEST_DATA_DIR, "qw_ima_rdst")
clean_dir(source)
with open(os.path.join(source, "a.txt"), "wb") as fh:
fh.write(b"a\n")
clean_dir(dest)
clean_dir(rdst)
lst = os.path.join(TEST_DATA_DIR, "qw_ima_list")
with open(lst, "w") as fh:
fh.write("a.txt\nmissing.txt\n")
rsync_result = _rsync(["-a", "--files-from=" + lst, "--ignore-missing-args",
source + "/", rdst + "/"])
assert rsync_result.returncode == 0, rsync_result.stderr
result, _ = run_client(source, dest, flags=["--files-from", lst,
"--ignore-missing-args"],
port=shared_server.port)
assert result.returncode == 0, result.stderr[:300]
received = get_dest_received_dir(dest, source)
assert os.path.exists(os.path.join(received, "a.txt"))
assert not os.path.exists(os.path.join(received, "missing.txt"))
@requires_rsync
@pytest.mark.ci
def test_empty_files_from_list_succeeds_like_rsync(self, shared_server):
source = os.path.join(TEST_DATA_DIR, "qw_ima_empty_src")
dest = os.path.join(TEST_DATA_DIR, "qw_ima_empty_dst")
rdst = os.path.join(TEST_DATA_DIR, "qw_ima_empty_rdst")
clean_dir(source)
with open(os.path.join(source, "a.txt"), "wb") as fh:
fh.write(b"a\n")
clean_dir(dest)
clean_dir(rdst)
lst = os.path.join(TEST_DATA_DIR, "qw_ima_empty_list")
with open(lst, "w") as fh:
fh.write("")
rsync_result = _rsync(["-a", "--files-from=" + lst, source + "/", rdst + "/"])
assert rsync_result.returncode == 0, rsync_result.stderr
result, _ = run_client(source, dest, flags=["--files-from", lst],
port=shared_server.port)
assert result.returncode == 0, (
f"empty --files-from must succeed like rsync, got {result.returncode}: "
f"{result.stderr[:300]}"
)
@requires_rsync
@pytest.mark.ci
def test_empty_files_from_with_delete_is_not_destructive(self, shared_server):
"""An empty --files-from list synchronizes nothing, so --delete must not
wipe the destination (rsync keeps the extra)."""
source = os.path.join(TEST_DATA_DIR, "qw_ima_edel_src")
dest = os.path.join(TEST_DATA_DIR, "qw_ima_edel_dst")
rdst = os.path.join(TEST_DATA_DIR, "qw_ima_edel_rdst")
clean_dir(source)
with open(os.path.join(source, "keep.txt"), "wb") as fh:
fh.write(b"keep\n")
clean_dir(dest)
clean_dir(rdst)
with open(os.path.join(rdst, "extra.txt"), "wb") as fh:
fh.write(b"extra\n")
lst = os.path.join(TEST_DATA_DIR, "qw_ima_edel_list")
with open(lst, "w") as fh:
fh.write("")
rs = _rsync(["-a", "--delete", "--files-from=" + lst, source + "/", rdst + "/"])
assert rs.returncode == 0, rs.stderr
assert os.path.exists(os.path.join(rdst, "extra.txt"))
with ServerManager() as server:
server.start(extra_args=["--allow-delete"])
seed = get_dest_received_dir(dest, source)
os.makedirs(seed, exist_ok=True)
with open(os.path.join(seed, "extra.txt"), "wb") as fh:
fh.write(b"extra\n")
result, _ = run_client(source, dest,
flags=["--delete", "--files-from", lst],
port=server.port)
assert result.returncode == 0, result.stderr[:300]
assert os.path.exists(os.path.join(seed, "extra.txt")), \
"empty --files-from + --delete must not delete the destination"
class TestPerDirFilterOrdering:
"""#10: -F .rsync-filter evaluation order matches rsync (a directory's own
rules before its ancestors'; anchored rules are relative to their owner)."""
def _run_pair(self, shared_server, root_rules, sub_rules, subsub_rules=None):
tag = "qw_f"
source = os.path.join(TEST_DATA_DIR, tag + "_src")
dest = os.path.join(TEST_DATA_DIR, tag + "_dst")
rdst = os.path.join(TEST_DATA_DIR, tag + "_rdst")
clean_dir(source)
os.makedirs(os.path.join(source, "sub"))
if subsub_rules is not None:
os.makedirs(os.path.join(source, "sub", "deep"))
with open(os.path.join(source, "bar"), "wb") as fh:
fh.write(b"bar\n")
with open(os.path.join(source, "sub", "foo"), "wb") as fh:
fh.write(b"foo\n")
if subsub_rules is not None:
with open(os.path.join(source, "sub", "deep", "foo"), "wb") as fh:
fh.write(b"deep foo\n")
with open(os.path.join(source, ".rsync-filter"), "w") as fh:
fh.write(root_rules)
with open(os.path.join(source, "sub", ".rsync-filter"), "w") as fh:
fh.write(sub_rules)
if subsub_rules is not None:
with open(os.path.join(source, "sub", "deep", ".rsync-filter"), "w") as fh:
fh.write(subsub_rules)
clean_dir(dest)
clean_dir(rdst)
fs_result, _ = run_client(source, dest, flags=["--preserve", "-F"],
port=shared_server.port)
assert fs_result.returncode == 0, fs_result.stderr[:300]
received = get_dest_received_dir(dest, source)
fs_files = _rel_files(received, skip=(".rsync-filter",))
return fs_files, source, rdst
@requires_rsync
@pytest.mark.ci
def test_child_include_overrides_parent_exclude(self, shared_server):
fs_files, source, rdst = self._run_pair(shared_server, "- foo\n", "+ foo\n")
rsync_result = _rsync(["-aF", source + "/", rdst + "/"])
assert rsync_result.returncode == 0, rsync_result.stderr
assert fs_files == _rel_files(rdst, skip=(".rsync-filter",))
assert "sub/foo" in fs_files
@requires_rsync
@pytest.mark.ci
def test_child_exclude_overrides_parent_include(self, shared_server):
fs_files, source, rdst = self._run_pair(shared_server, "+ foo\n", "- foo\n")
rsync_result = _rsync(["-aF", source + "/", rdst + "/"])
assert rsync_result.returncode == 0, rsync_result.stderr
assert fs_files == _rel_files(rdst, skip=(".rsync-filter",))
assert "sub/foo" not in fs_files
@requires_rsync
def test_three_level_inheritance(self, shared_server):
fs_files, source, rdst = self._run_pair(
shared_server, "- foo\n", "+ foo\n", "- foo\n"
)
rsync_result = _rsync(["-aF", source + "/", rdst + "/"])
assert rsync_result.returncode == 0, rsync_result.stderr
assert fs_files == _rel_files(rdst, skip=(".rsync-filter",))
class TestDeleteEdgeSemantics:
"""#20/#23/#24/#25: delete timing/policy edge cases match rsync."""
@requires_rsync
@pytest.mark.ci
def test_delete_before_removes_extras_like_rsync(self, shared_server):
source = os.path.join(TEST_DATA_DIR, "qw_db_src")
dest = os.path.join(TEST_DATA_DIR, "qw_db_dst")
rdst = os.path.join(TEST_DATA_DIR, "qw_db_rdst")
clean_dir(source)
with open(os.path.join(source, "keep.txt"), "wb") as fh:
fh.write(b"keep\n")
for d in (dest, rdst):
clean_dir(d)
with open(os.path.join(d, "extra.txt"), "wb") as fh:
fh.write(b"extra\n")
received_seed = get_dest_received_dir(dest, source)
os.makedirs(received_seed, exist_ok=True)
with open(os.path.join(received_seed, "extra.txt"), "wb") as fh:
fh.write(b"extra\n")
rsync_result = _rsync(["-a", "--delete-before", source + "/", rdst + "/"])
assert rsync_result.returncode == 0, rsync_result.stderr
with ServerManager() as server:
server.start(extra_args=["--allow-delete"])
result, _ = run_client(source, dest, flags=["--delete-before"],
port=server.port)
assert result.returncode == 0, result.stderr[:300]
received = get_dest_received_dir(dest, source)
assert os.path.exists(os.path.join(received, "keep.txt"))
assert not os.path.exists(os.path.join(received, "extra.txt")), \
"--delete-before must remove destination extras"
assert not os.path.exists(os.path.join(rdst, "extra.txt"))
@requires_rsync
@pytest.mark.ci
def test_delete_excluded_protects_then_deletes(self, shared_server):
source = os.path.join(TEST_DATA_DIR, "qw_de_src")
dest = os.path.join(TEST_DATA_DIR, "qw_de_dst")
rdst = os.path.join(TEST_DATA_DIR, "qw_de_rdst")
clean_dir(source)
with open(os.path.join(source, "keep.txt"), "wb") as fh:
fh.write(b"keep\n")
with open(os.path.join(source, "skip.log"), "wb") as fh:
fh.write(b"log\n")
# Default --delete protects the excluded mirror (rsync parity).
rdst_prot = os.path.join(TEST_DATA_DIR, "qw_de_rprot")
clean_dir(rdst_prot)
with open(os.path.join(rdst_prot, "skip.log"), "wb") as fh:
fh.write(b"stale\n")
rsync_result = _rsync(["-a", "--delete", "--exclude=*.log", source + "/",
rdst_prot + "/"])
assert rsync_result.returncode == 0, rsync_result.stderr
assert os.path.exists(os.path.join(rdst_prot, "skip.log"))
with ServerManager() as server:
server.start(extra_args=["--allow-delete"])
clean_dir(dest)
seed = get_dest_received_dir(dest, source)
os.makedirs(seed, exist_ok=True)
with open(os.path.join(seed, "skip.log"), "wb") as fh:
fh.write(b"stale\n")
result, _ = run_client(source, dest, flags=["--delete", "--exclude=*.log"],
port=server.port)
assert result.returncode == 0, result.stderr[:300]
received = get_dest_received_dir(dest, source)
assert os.path.exists(os.path.join(received, "skip.log")), \
"--delete must protect the excluded destination mirror"
# --delete-excluded removes it.
clean_dir(rdst)
with open(os.path.join(rdst, "skip.log"), "wb") as fh:
fh.write(b"stale\n")
rsync_result = _rsync(["-a", "--delete", "--delete-excluded",
"--exclude=*.log", source + "/", rdst + "/"])
assert rsync_result.returncode == 0, rsync_result.stderr
assert not os.path.exists(os.path.join(rdst, "skip.log"))
result, _ = run_client(
source, dest,
flags=["--delete", "--delete-excluded", "--exclude=*.log"],
port=server.port,
)
assert result.returncode == 0, result.stderr[:300]
assert not os.path.exists(os.path.join(received, "skip.log")), \
"--delete-excluded must remove the excluded mirror"
@requires_rsync
def test_force_replaces_nonempty_destination_dir(self, shared_server):
source = os.path.join(TEST_DATA_DIR, "qw_force_src")
dest = os.path.join(TEST_DATA_DIR, "qw_force_dst")
rdst = os.path.join(TEST_DATA_DIR, "qw_force_rdst")
clean_dir(source)
with open(os.path.join(source, "x"), "wb") as fh:
fh.write(b"file-content\n")
clean_dir(rdst)
os.makedirs(os.path.join(rdst, "x"))
with open(os.path.join(rdst, "x", "blocker"), "wb") as fh:
fh.write(b"blocker\n")
rsync_result = _rsync(["-a", "--force", source + "/", rdst + "/"])
assert rsync_result.returncode == 0, rsync_result.stderr
assert os.path.isfile(os.path.join(rdst, "x"))
with ServerManager() as server:
server.start(extra_args=["--allow-delete"])
clean_dir(dest)
seed = get_dest_received_dir(dest, source)
os.makedirs(os.path.join(seed, "x"), exist_ok=True)
with open(os.path.join(seed, "x", "blocker"), "wb") as fh:
fh.write(b"blocker\n")
result, _ = run_client(source, dest, flags=["--force"],
port=server.port)
assert result.returncode == 0, result.stderr[:300]
received = get_dest_received_dir(dest, source)
assert os.path.isfile(os.path.join(received, "x")), \
"--force must replace a non-empty destination directory"
@requires_rsync
def test_no_force_nonempty_dir_is_partial_error(self, shared_server):
"""Without --force a non-empty destination dir blocking a file is not
replaced; rsync exits 23, fastsync must not silently mangle it."""
source = os.path.join(TEST_DATA_DIR, "qw_noforce_src")
dest = os.path.join(TEST_DATA_DIR, "qw_noforce_dst")
rdst = os.path.join(TEST_DATA_DIR, "qw_noforce_rdst")
clean_dir(source)
with open(os.path.join(source, "x"), "wb") as fh:
fh.write(b"file-content\n")
clean_dir(rdst)
os.makedirs(os.path.join(rdst, "x"))
with open(os.path.join(rdst, "x", "blocker"), "wb") as fh:
fh.write(b"blocker\n")
rsync_result = _rsync(["-a", source + "/", rdst + "/"])
assert rsync_result.returncode == 23, rsync_result.stderr
with ServerManager() as server:
server.start(extra_args=["--allow-delete"])
clean_dir(dest)
seed = get_dest_received_dir(dest, source)
os.makedirs(os.path.join(seed, "x"), exist_ok=True)
with open(os.path.join(seed, "x", "blocker"), "wb") as fh:
fh.write(b"blocker\n")
result, _ = run_client(source, dest, port=server.port)
assert result.returncode != 0, "a blocked file install must not report success"
def _snapshot(root):
"""rel path -> (kind, payload) for every file/symlink below root."""
result = {}
for dirpath, dirnames, filenames in os.walk(root, followlinks=False):
for name in list(dirnames):
p = os.path.join(dirpath, name)
if os.path.islink(p):
result[os.path.relpath(p, root)] = ("link", os.readlink(p))
dirnames.remove(name)
for name in filenames:
p = os.path.join(dirpath, name)
if os.path.islink(p):
result[os.path.relpath(p, root)] = ("link", os.readlink(p))
else:
with open(p, "rb") as fh:
result[os.path.relpath(p, root)] = ("file", fh.read())
return result
def _assert_same_tree(rdst, received, msg=""):
rsync_tree = _snapshot(rdst)
fs_tree = _snapshot(received)
assert fs_tree == rsync_tree, (
f"tree mismatch {msg}\n----- rsync only/diff -----\n"
f"{ {k: v for k, v in rsync_tree.items() if fs_tree.get(k) != v} }\n"
f"----- fastsync only/diff -----\n"
f"{ {k: v for k, v in fs_tree.items() if rsync_tree.get(k) != v} }"
)
class TestVerifyAndFlip:
"""Item 8: confirm already-implemented client-side rows match rsync."""
def _src(self, tag):
source = os.path.join(TEST_DATA_DIR, f"vw_{tag}_src")
clean_dir(source)
return source
def _dst(self, tag):
d = os.path.join(TEST_DATA_DIR, f"vw_{tag}_dst")
clean_dir(d)
return d
@requires_rsync
@pytest.mark.ci
def test_links_verbatim_matches_rsync(self, shared_server):
source = self._src("links")
dest = self._dst("links")
rdst = self._dst("links_r")
with open(os.path.join(source, "a.txt"), "wb") as fh:
fh.write(b"a\n")
os.symlink("a.txt", os.path.join(source, "rel"))
os.symlink("/etc/hostname", os.path.join(source, "abs"))
os.symlink("../../escape", os.path.join(source, "dd"))
assert _rsync(["-a", source + "/", rdst + "/"]).returncode == 0
result, _ = run_client(source, dest, flags=["-a"], port=shared_server.port)
assert result.returncode == 0, result.stderr[:300]
_assert_same_tree(rdst, get_dest_received_dir(dest, source), "(-l/--links)")
@requires_rsync
@pytest.mark.ci
def test_dry_run_does_not_write_like_rsync(self, shared_server):
source = self._src("dry")
dest = self._dst("dry")
rdst = self._dst("dry_r")
with open(os.path.join(source, "a.txt"), "wb") as fh:
fh.write(b"a\n")
os.makedirs(os.path.join(source, "sub"))
with open(os.path.join(source, "sub", "b.txt"), "wb") as fh:
fh.write(b"b\n")
assert _rsync(["-a", "--dry-run", source + "/", rdst + "/"]).returncode == 0
result, _ = run_client(source, dest, flags=["-a", "--dry-run"],
port=shared_server.port)
assert result.returncode == 0, result.stderr[:300]
_assert_same_tree(rdst, get_dest_received_dir(dest, source), "(--dry-run)")
@requires_rsync
@pytest.mark.ci
def test_ignore_existing_matches_rsync(self, shared_server):
source = self._src("ie")
dest = self._dst("ie")
rdst = self._dst("ie_r")
with open(os.path.join(source, "a.txt"), "wb") as fh:
fh.write(b"source\n")
with open(os.path.join(source, "new.txt"), "wb") as fh:
fh.write(b"new\n")
for root in (get_dest_received_dir(dest, source), rdst):
os.makedirs(root, exist_ok=True)
with open(os.path.join(root, "a.txt"), "wb") as fh:
fh.write(b"destination-kept\n")
assert _rsync(["-a", "--ignore-existing", source + "/", rdst + "/"]).returncode == 0
result, _ = run_client(source, dest, flags=["--ignore-existing"],
port=shared_server.port)
assert result.returncode == 0, result.stderr[:300]
_assert_same_tree(rdst, get_dest_received_dir(dest, source), "(--ignore-existing)")
@requires_rsync
def test_append_matches_rsync(self, shared_server):
source = self._src("app")
dest = self._dst("app")
rdst = self._dst("app_r")
prefix = b"P" * (256 * 1024)
tail = b"T" * (16 * 1024)
with open(os.path.join(source, "grow"), "wb") as fh:
fh.write(prefix + tail)
for root in (get_dest_received_dir(dest, source), rdst):
os.makedirs(root, exist_ok=True)
with open(os.path.join(root, "grow"), "wb") as fh:
fh.write(prefix)
assert _rsync(["-a", "--append", source + "/", rdst + "/"]).returncode == 0
result, _ = run_client(source, dest, flags=["--append"], port=shared_server.port)
assert result.returncode == 0, result.stderr[:300]
_assert_same_tree(rdst, get_dest_received_dir(dest, source), "(--append)")
@requires_rsync
def test_append_verify_matches_rsync(self, shared_server):
source = self._src("appv")
dest = self._dst("appv")
rdst = self._dst("appv_r")
prefix = b"Q" * (128 * 1024)
tail = b"Z" * (8 * 1024)
with open(os.path.join(source, "grow"), "wb") as fh:
fh.write(prefix + tail)
for root in (get_dest_received_dir(dest, source), rdst):
os.makedirs(root, exist_ok=True)
with open(os.path.join(root, "grow"), "wb") as fh:
fh.write(prefix)
assert _rsync(["-a", "--append-verify", source + "/", rdst + "/"]).returncode == 0
result, _ = run_client(source, dest, flags=["--append-verify"],
port=shared_server.port)
assert result.returncode == 0, result.stderr[:300]
_assert_same_tree(rdst, get_dest_received_dir(dest, source), "(--append-verify)")
@requires_rsync
@pytest.mark.ci
def test_delay_updates_matches_rsync(self, shared_server):
source = self._src("delay")
dest = self._dst("delay")
rdst = self._dst("delay_r")
for rel, content in {"a.txt": b"a\n", "sub/b.txt": b"b\n"}.items():
full = os.path.join(source, rel)
os.makedirs(os.path.dirname(full), exist_ok=True)
with open(full, "wb") as fh:
fh.write(content)
assert _rsync(["-a", "--delay-updates", source + "/", rdst + "/"]).returncode == 0
result, _ = run_client(source, dest, flags=["--delay-updates"],
port=shared_server.port)
assert result.returncode == 0, result.stderr[:300]
_assert_same_tree(rdst, get_dest_received_dir(dest, source), "(--delay-updates)")
@requires_rsync
def test_preallocate_matches_rsync(self, shared_server):
source = self._src("prealloc")
dest = self._dst("prealloc")
rdst = self._dst("prealloc_r")
with open(os.path.join(source, "f.bin"), "wb") as fh:
fh.write(b"x" * (512 * 1024))
assert _rsync(["-a", "--preallocate", source + "/", rdst + "/"]).returncode == 0
result, _ = run_client(source, dest, flags=["--preallocate"],
port=shared_server.port)
assert result.returncode == 0, result.stderr[:300]
_assert_same_tree(rdst, get_dest_received_dir(dest, source), "(--preallocate)")
@requires_rsync
def test_fuzzy_content_matches_rsync(self, shared_server):
source = self._src("fuzzy")
dest = self._dst("fuzzy")
rdst = self._dst("fuzzy_r")
payload = (b"the quick brown fox\n" * 4096)
with open(os.path.join(source, "renamed.txt"), "wb") as fh:
fh.write(payload)
for root in (get_dest_received_dir(dest, source), rdst):
os.makedirs(root, exist_ok=True)
with open(os.path.join(root, "old_name.txt"), "wb") as fh:
fh.write(payload)
assert _rsync(["-a", "--fuzzy", source + "/", rdst + "/"]).returncode == 0
result, _ = run_client(source, dest, flags=["-y"], port=shared_server.port)
assert result.returncode == 0, result.stderr[:300]
_assert_same_tree(rdst, get_dest_received_dir(dest, source), "(-y/--fuzzy)")
@requires_rsync
def test_skip_compress_content_matches_rsync(self, shared_server):
source = self._src("skipz")
dest = self._dst("skipz")
rdst = self._dst("skipz_r")
with open(os.path.join(source, "already.zip"), "wb") as fh:
fh.write(b"PK" + b"z" * 4096)
with open(os.path.join(source, "text.txt"), "wb") as fh:
fh.write(b"compress me\n" * 1024)
assert _rsync(["-az", "--skip-compress=gz/zip", source + "/",
rdst + "/"]).returncode == 0
result, _ = run_client(source, dest, flags=["-z", "--skip-compress=gz/zip"],
port=shared_server.port)
assert result.returncode == 0, result.stderr[:300]
_assert_same_tree(rdst, get_dest_received_dir(dest, source), "(--skip-compress)")
@requires_rsync
def test_copy_dest_content_matches_rsync(self, shared_server):
source = self._src("copyd")
dest = self._dst("copyd")
with open(os.path.join(source, "f.txt"), "wb") as fh:
fh.write(b"copy-from-basis\n")
# fastsync basis DIR is relative to the receive root; the basis file is
# looked up at the same source-mirror relative path.
rel = os.path.abspath(source).lstrip(os.sep)
basis = os.path.join(dest, "basis", rel)
os.makedirs(basis, exist_ok=True)
with open(os.path.join(basis, "f.txt"), "wb") as fh:
fh.write(b"copy-from-basis\n")
received = get_dest_received_dir(dest, source)
result, _ = run_client(source, dest,
flags=["--copy-dest=basis", "--incremental"],
port=shared_server.port)
assert result.returncode == 0, result.stderr[:300]
assert os.path.exists(os.path.join(received, "f.txt"))
with open(os.path.join(received, "f.txt"), "rb") as fh:
assert fh.read() == b"copy-from-basis\n"
@requires_rsync
def test_trust_sender_content_matches_rsync(self, shared_server):
source = self._src("trust")
dest = self._dst("trust")
rdst = self._dst("trust_r")
with open(os.path.join(source, "a.txt"), "wb") as fh:
fh.write(b"a\n")
assert _rsync(["-a", "--trust-sender", source + "/", rdst + "/"]).returncode == 0
result, _ = run_client(source, dest, flags=["--trust-sender"],
port=shared_server.port)
assert result.returncode == 0, result.stderr[:300]
_assert_same_tree(rdst, get_dest_received_dir(dest, source), "(--trust-sender)")
@requires_rsync
def test_compare_dest_content_matches_rsync(self, shared_server):
source = self._src("cmpd")
dest = self._dst("cmpd")
rdst = self._dst("cmpd_r")
with open(os.path.join(source, "f.txt"), "wb") as fh:
fh.write(b"basis-content\n")
# rsync resolves --compare-dest relative to the destination dir; its
# basis file sits at the transfer-relative path.
os.makedirs(os.path.join(rdst, "basis"), exist_ok=True)
with open(os.path.join(rdst, "basis", "f.txt"), "wb") as fh:
fh.write(b"basis-content\n")
rs = _rsync(["-a", "--compare-dest=basis", source + "/", rdst + "/"])
assert rs.returncode == 0, rs.stderr
assert not os.path.exists(os.path.join(rdst, "f.txt")), \
"rsync compare-dest must leave the destination sparse"
# fastsync resolves the basis DIR relative to the receive root, and the
# file's relative path there mirrors the source path.
rel = os.path.abspath(source).lstrip(os.sep)
basis = os.path.join(dest, "basis", rel)
os.makedirs(basis, exist_ok=True)
with open(os.path.join(basis, "f.txt"), "wb") as fh:
fh.write(b"basis-content\n")
received = get_dest_received_dir(dest, source)
result, _ = run_client(source, dest,
flags=["--compare-dest=basis", "--incremental"],
port=shared_server.port)
assert result.returncode == 0, result.stderr[:300]
# rsync --compare-dest never copies: a basis match is simply not
# transferred, so the destination stays sparse (no f.txt).
assert not os.path.exists(os.path.join(received, "f.txt"))
@requires_rsync
def test_link_dest_hardlinks_matches_rsync(self, shared_server):
source = self._src("linkd")
dest = self._dst("linkd")
with open(os.path.join(source, "f.txt"), "wb") as fh:
fh.write(b"link-basis-content\n")
rel = os.path.abspath(source).lstrip(os.sep)
basis = os.path.join(dest, "basis", rel)
os.makedirs(basis, exist_ok=True)
basis_file = os.path.join(basis, "f.txt")
with open(basis_file, "wb") as fh:
fh.write(b"link-basis-content\n")
received = get_dest_received_dir(dest, source)
result, _ = run_client(source, dest,
flags=["--link-dest=basis", "--incremental"],
port=shared_server.port)
assert result.returncode == 0, result.stderr[:300]
dest_file = os.path.join(received, "f.txt")
assert os.path.exists(dest_file)
assert os.stat(dest_file).st_ino == os.stat(basis_file).st_ino, \
"--link-dest must hard-link to the basis file"
@pytest.mark.skipif(os.geteuid() != 0, reason="ownership mapping requires root")
class TestOwnershipMapping:
"""#33/#34/#35: --usermap/--groupmap/--chown match rsync's numeric result."""
def _prep(self, tag):
source = os.path.join(TEST_DATA_DIR, f"own_{tag}_src")
dest = os.path.join(TEST_DATA_DIR, f"own_{tag}_dst")
rdst = os.path.join(TEST_DATA_DIR, f"own_{tag}_rdst")
clean_dir(source)
clean_dir(dest)
clean_dir(rdst)
with open(os.path.join(source, "a.txt"), "wb") as fh:
fh.write(b"a\n")
return source, dest, rdst
@requires_rsync
def test_usermap_groupmap_matches_rsync(self, shared_server):
source, dest, rdst = self._prep("map")
assert _rsync(["-a", "--usermap=*:12345", "--groupmap=*:54321",
source + "/", rdst + "/"]).returncode == 0
result, _ = run_client(source, dest,
flags=["--usermap=*:12345", "--groupmap=*:54321"],
port=shared_server.port)
assert result.returncode == 0, result.stderr[:300]
rs = os.stat(os.path.join(rdst, "a.txt"))
fs = os.stat(os.path.join(get_dest_received_dir(dest, source), "a.txt"))
assert (fs.st_uid, fs.st_gid) == (rs.st_uid, rs.st_gid) == (12345, 54321)
@requires_rsync
def test_chown_matches_rsync(self, shared_server):
source, dest, rdst = self._prep("chown")
assert _rsync(["-a", "--chown=23456:65432", source + "/",
rdst + "/"]).returncode == 0
result, _ = run_client(source, dest, flags=["--chown=23456:65432"],
port=shared_server.port)
assert result.returncode == 0, result.stderr[:300]
rs = os.stat(os.path.join(rdst, "a.txt"))
fs = os.stat(os.path.join(get_dest_received_dir(dest, source), "a.txt"))
assert (fs.st_uid, fs.st_gid) == (rs.st_uid, rs.st_gid) == (23456, 65432)
class TestFakeSuper:
"""#32: --fake-super stores privileged attrs via xattrs. FastSync uses its
own reserved key (documented divergence) but the file DATA must match rsync."""
@requires_rsync
def test_fake_super_data_matches_rsync(self, shared_server):
source = os.path.join(TEST_DATA_DIR, "fs_src")
dest = os.path.join(TEST_DATA_DIR, "fs_dst")
rdst = os.path.join(TEST_DATA_DIR, "fs_rdst")
clean_dir(source)
clean_dir(dest)
clean_dir(rdst)
with open(os.path.join(source, "a.txt"), "wb") as fh:
fh.write(b"fake-super-data\n")
assert _rsync(["-a", "--fake-super", source + "/", rdst + "/"]).returncode == 0
result, _ = run_client(source, dest, flags=["--fake-super"],
port=shared_server.port)
assert result.returncode == 0, result.stderr[:300]
with open(os.path.join(get_dest_received_dir(dest, source), "a.txt"), "rb") as fh:
assert fh.read() == b"fake-super-data\n"
with open(os.path.join(rdst, "a.txt"), "rb") as fh:
assert fh.read() == b"fake-super-data\n"
class TestInfoDebugFlagParity:
"""#01/#02: rsync's info/debug spellings are either mapped to real output or
rejected by name (never silently ignored)."""
@requires_rsync
def test_mapped_info_categories_accepted_like_rsync(self, shared_server):
source = os.path.join(TEST_DATA_DIR, "qw_flags_src")
dest = os.path.join(TEST_DATA_DIR, "qw_flags_dst")
rdst = os.path.join(TEST_DATA_DIR, "qw_flags_rdst")
clean_dir(source)
clean_dir(dest)
clean_dir(rdst)
with open(os.path.join(source, "a.txt"), "wb") as fh:
fh.write(b"a\n")
for cat in ("stats2", "name", "copy", "misc", "skip", "STATS2"):
assert _rsync(["-a", "--info=" + cat, source + "/", rdst + "/"]).returncode == 0
clean_dir(rdst)
result, _ = run_client(source, dest, flags=["--info=" + cat],
port=shared_server.port)
assert result.returncode == 0, (
f"--info={cat} must be accepted: {result.stderr[:200]}"
)
@requires_rsync
def test_mapped_debug_categories_accepted_like_rsync(self, shared_server):
source = os.path.join(TEST_DATA_DIR, "qw_dflags_src")
dest = os.path.join(TEST_DATA_DIR, "qw_dflags_dst")
rdst = os.path.join(TEST_DATA_DIR, "qw_dflags_rdst")
clean_dir(source)
clean_dir(dest)
clean_dir(rdst)
with open(os.path.join(source, "a.txt"), "wb") as fh:
fh.write(b"a\n")
for cat in ("io2", "proto0", "all"):
assert _rsync(["-a", "--debug=" + cat, source + "/", rdst + "/"]).returncode == 0
clean_dir(rdst)
result, _ = run_client(source, dest, flags=["--debug=" + cat],
port=shared_server.port)
assert result.returncode == 0, (
f"--debug={cat} must be accepted: {result.stderr[:200]}"
)
@requires_rsync
@pytest.mark.ci
def test_unmapped_categories_rejected_by_name(self, shared_server):
"""rsync accepts del/filter; fastsync has no mapping so it must refuse
loudly, naming the category, rather than silently ignoring it."""
source = os.path.join(TEST_DATA_DIR, "qw_umap_src")
dest = os.path.join(TEST_DATA_DIR, "qw_umap_dst")
clean_dir(source)
clean_dir(dest)
with open(os.path.join(source, "a.txt"), "wb") as fh:
fh.write(b"a\n")
# rsync accepts these (so they are valid rsync invocations).
assert _rsync(["-a", "--info=del", source + "/", dest + "/"]).returncode == 0
assert _rsync(["-a", "--debug=filter", source + "/", dest + "/"]).returncode == 0
for flag, name in (("--info=del", "del"), ("--debug=filter", "filter")):
result, _ = run_client(source, dest, flags=[flag],
port=shared_server.port)
assert result.returncode != 0, f"{flag} must be rejected"
assert name in (result.stderr or ""), \
f"{flag} must be rejected by name, got: {result.stderr[:200]}"
+61
View File
@@ -1317,7 +1317,66 @@ static void test_parse_args_rejects_invalid_info_flag() {
config_delete(cfg); config_delete(cfg);
} }
/* rsync's info "name" category maps to fastsync's per-file name logging, and
* --info=help prints the flag list and exits without error. */
static void test_parse_args_info_name_and_help() {
Config* cfg = config_create();
char* argv[] = {"fastsync", "--info=name", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
EXPECT_EQ_INT(cfg->info_level, LOG_INFO_COPY);
config_delete(cfg);
cfg = config_create();
char* help_argv[] = {"fastsync", "--info=help"};
positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 2, help_argv, positional_args, &positional_count), 1);
config_delete(cfg);
}
/* Test parse_args with --archive flag */ /* Test parse_args with --archive flag */
/* rsync accepts a trailing level digit on --debug/--info items (e.g. io2,
* all4); level 0 silences the item. */
static void test_parse_args_debug_info_levels() {
Config* cfg = config_create();
char* argv[] = {"fastsync", "--debug=io2,proto0,all", "/src", "/dst"};
int positional_args[2];
int positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
EXPECT_EQ_INT(cfg->debug_level, LOG_DEBUG_ALL);
config_delete(cfg);
cfg = config_create();
char* io0_argv[] = {"fastsync", "--debug=all,io0", "/src", "/dst"};
positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, io0_argv, positional_args, &positional_count), 0);
EXPECT_EQ_INT(cfg->debug_level, LOG_DEBUG_ALL & ~LOG_DEBUG_IO);
config_delete(cfg);
cfg = config_create();
char* info_argv[] = {"fastsync", "--info=stats2", "/src", "/dst"};
positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, info_argv, positional_args, &positional_count), 0);
EXPECT_EQ_INT(cfg->info_level, LOG_INFO_STATS);
config_delete(cfg);
cfg = config_create();
char* bad_argv[] = {"fastsync", "--debug=123", "/src", "/dst"};
positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 4, bad_argv, positional_args, &positional_count), -1);
config_delete(cfg);
/* rsync accepts category names case-insensitively. */
cfg = config_create();
char* upper_argv[] = {"fastsync", "--info=STATS2", "--debug=IO", "/src", "/dst"};
positional_count = 0;
EXPECT_EQ_INT(parse_args(cfg, 5, upper_argv, positional_args, &positional_count), 0);
EXPECT_EQ_INT(cfg->info_level, LOG_INFO_STATS);
EXPECT_EQ_INT(cfg->debug_level, LOG_DEBUG_IO);
config_delete(cfg);
}
static void test_parse_args_archive() { static void test_parse_args_archive() {
Config* cfg = config_create(); Config* cfg = config_create();
char* argv[] = {"fastsync", "--archive", "/src", "/dst"}; char* argv[] = {"fastsync", "--archive", "/src", "/dst"};
@@ -4255,6 +4314,7 @@ void test_client_cli() {
test_parse_args_debug_flags(); test_parse_args_debug_flags();
test_parse_args_debug_help(); test_parse_args_debug_help();
test_parse_args_debug_flags_validation(); test_parse_args_debug_flags_validation();
test_parse_args_debug_info_levels();
test_parse_args_modify_window(); test_parse_args_modify_window();
test_parse_args_rejects_invalid_modify_window(); test_parse_args_rejects_invalid_modify_window();
test_parse_args_skip_compress(); test_parse_args_skip_compress();
@@ -4278,6 +4338,7 @@ void test_client_cli() {
test_parse_args_info_flags(); test_parse_args_info_flags();
test_parse_args_info_verbose_order(); test_parse_args_info_verbose_order();
test_parse_args_rejects_invalid_info_flag(); test_parse_args_rejects_invalid_info_flag();
test_parse_args_info_name_and_help();
test_parse_args_archive(); test_parse_args_archive();
test_parse_args_preserve_attributes_are_independent(); test_parse_args_preserve_attributes_are_independent();
test_parse_args_preserve_long_form(); test_parse_args_preserve_long_form();
+44
View File
@@ -1533,6 +1533,49 @@ static void test_scanner_entry_classification() {
rmdir(root); rmdir(root);
} }
/* A dereferenced symlink with no referent (broken/unreadable) must record a
* non-fatal I/O error so the run can exit 23 like rsync, without aborting the
* scan or treating the condition as a fatal failure. */
static void test_scanner_broken_referent_io_error(void) {
const char* root = "test_scan_broken_ref";
const char* good = "test_scan_broken_ref/good.txt";
const char* broken = "test_scan_broken_ref/broken";
EXPECT_EQ_INT(mkdir(root, 0755), 0);
create_test_file(good, "hello");
EXPECT_EQ_INT(symlink("/nonexistent/quickwins/target", broken), 0);
{
ScannerOptions options = {0};
options.copy_links = true;
DirectoryScanner* scanner = directory_scanner_create_with_options(root, &options);
EXPECT_NOT_NULL(scanner);
Chunk* chunk;
while ((chunk = directory_scanner_next(scanner)) != NULL)
chunk_destroy(chunk);
EXPECT_FALSE(directory_scanner_failed(scanner));
EXPECT_TRUE(directory_scanner_had_io_error(scanner));
directory_scanner_destroy(scanner);
}
{
ScannerOptions options = {0};
options.copy_links = true;
ParallelScanner* scanner = parallel_scanner_create_with_options(root, &options, NULL);
EXPECT_NOT_NULL(scanner);
Chunk* chunk;
while ((chunk = parallel_scanner_next(scanner)) != NULL)
chunk_destroy(chunk);
EXPECT_FALSE(parallel_scanner_failed(scanner));
EXPECT_TRUE(parallel_scanner_had_io_error(scanner));
parallel_scanner_destroy(scanner);
}
unlink(broken);
unlink(good);
rmdir(root);
}
void test_scanner() { void test_scanner() {
test_scanner_single_file(); test_scanner_single_file();
test_scanner_multiple_files(); test_scanner_multiple_files();
@@ -1551,6 +1594,7 @@ void test_scanner() {
test_scanner_one_file_system_decision(); test_scanner_one_file_system_decision();
test_scanner_one_file_system_same_device(); test_scanner_one_file_system_same_device();
test_parallel_scanner_one_file_system_same_device(); test_parallel_scanner_one_file_system_same_device();
test_scanner_broken_referent_io_error();
test_scanner_one_file_system_cross_device(); test_scanner_one_file_system_cross_device();
test_files_from_subset(false); test_files_from_subset(false);
test_files_from_subset(true); test_files_from_subset(true);