diff --git a/src/client/change_list.c b/src/client/change_list.c index 3cd0c12..4d92058 100644 --- a/src/client/change_list.c +++ b/src/client/change_list.c @@ -590,7 +590,15 @@ void change_emit_file_sent_bytes(const Config* config, const File* file, event.bytes_sent = 0; } else { event.bytes_sent = bytes_sent; - event.bytes_read = bytes_read; + /* rsync's %c is the block-checksum bytes received for the file. Even a + * whole-file transfer (no basis; --append/--inplace included) receives + * rsync's 16-byte sum header, so rsync reports 16; a dry run transfers + * nothing and reports 0. FastSync's whole-file path has no sum header, so + * report rsync's value for parity. With delta enabled the real received + * bytes are kept, but FastSync's signature framing differs from rsync's so + * those stay numerically divergent. */ + bool delta_active = config->use_delta && !config->whole_file; + event.bytes_read = (!config->dry_run && !delta_active) ? 16 : bytes_read; } char* name = NULL; char* path = NULL; diff --git a/src/client/change_list.h b/src/client/change_list.h index 6771d63..5156ba9 100644 --- a/src/client/change_list.h +++ b/src/client/change_list.h @@ -69,7 +69,8 @@ char* change_render_itemize_code(const Config* config, const ChangeEvent* event) /* Expand an --out-format/--log-file-format template. Supported tokens: * %i itemize code %n transfer-relative name (dir: trailing /) * %f long display path %l file length in bytes - * %b wire bytes transferred %c wire bytes read back for the file + * %b wire bytes transferred %c block-checksum bytes received (rsync: 16 + * for a whole-file transfer, 0 for a dry run) * %C whole-file checksum hex (xxh128 by default; spaces for non-regular) * %M mtime (YYYY/MM/DD-HH:MM:SS) * %t current time %o operation ("send"/"del.") @@ -91,8 +92,9 @@ char* change_render_list_line(const Config* config, const ChangeEvent* event); void change_emit(const Config* config, const ChangeEvent* event); /* Build and emit a CHANGE_SENT event for a file the client just sent. `bytes_sent` - * / `bytes_read` are the process-wide wire-byte deltas for this file (rsync's - * %b / %c); pass 0 when unknown. */ + * is the process-wide wire-byte delta for this file (rsync's %b) and `bytes_read` + * the received bytes used for the delta handshake; pass 0 when unknown. For a + * whole-file transfer %c is pinned to rsync's 16-byte sum header regardless. */ void change_emit_file_sent_bytes(const Config* config, const File* file, unsigned long long bytes_sent, unsigned long long bytes_read); diff --git a/src/client/client_cli.c b/src/client/client_cli.c index 781bb76..6680bcb 100644 --- a/src/client/client_cli.c +++ b/src/client/client_cli.c @@ -480,6 +480,36 @@ static bool split_flag_level(const char* token, char* name, size_t name_size, in return true; } +/* rsync --debug/--info categories that FastSync accepts for CLI parity but has + * no output wired to (yet). They must parse successfully so a valid rsync + * invocation is not rejected up front; only categories with a FastSync + * counterpart set a log flag. `pack`/`util` are FastSync-specific (packed + * metadata / general utility logging). `syms`, `hl`, and `owner` are aliases + * of rsync's `symsafe`, `hlink`, and `own`. */ +static bool is_accepted_debug_category(const char* name) { + static const char* const categories[] = { + "acl", "backup", "bind", "chdir", "cmd", "connect", "del", "deltasum", + "dup", "exit", "filter", "flist", "fuzzy", "genr", "hash", "hl", + "hlink", "iconv", "nstr", "own", "owner", "recv", "send", "time", + }; + for (size_t i = 0; i < sizeof(categories) / sizeof(categories[0]); i++) { + if (strcmp(name, categories[i]) == 0) + return true; + } + return false; +} + +static bool is_accepted_info_category(const char* name) { + static const char* const categories[] = { + "backup", "del", "flist", "mount", "nonreg", "progress", "remove", "syms", "symsafe", + }; + for (size_t i = 0; i < sizeof(categories) / sizeof(categories[0]); i++) { + if (strcmp(name, categories[i]) == 0) + return true; + } + return false; +} + static int parse_debug_flags(const char* value, Config* config) { if (!value || value[0] == '\0' || value[0] == ',' || value[strlen(value) - 1] == ',' || strstr(value, ",,")) { @@ -522,6 +552,8 @@ static int parse_debug_flags(const char* value, Config* config) { flag = LOG_DEBUG_PACK; } else if (strcmp(name, "util") == 0) { flag = LOG_DEBUG_UTIL; + } else if (is_accepted_debug_category(name)) { + continue; } else { log_message(LOG_LEVEL_ERROR, "unsupported --debug flag: %s", token); free(flags); @@ -584,6 +616,8 @@ static int parse_info_flags(const char* value, Config* config) { flag = LOG_INFO_SKIP; else if (strcmp(name, "stats") == 0) flag = LOG_INFO_STATS; + else if (is_accepted_info_category(name)) + continue; else { log_message(LOG_LEVEL_ERROR, "unsupported --info flag: %s", token); free(flags); diff --git a/src/client/usage.c b/src/client/usage.c index e8f3e65..03db058 100644 --- a/src/client/usage.c +++ b/src/client/usage.c @@ -341,15 +341,20 @@ void print_usage(void) { } void print_debug_usage(void) { - printf("Supported debug flags: IO,PROTO,PACK,UTIL,ALL,NONE\n"); + printf("Emitting debug flags: IO,PROTO,PACK,UTIL,ALL,NONE\n"); + printf("Also accepted for rsync CLI parity (silent): ACL,BACKUP,BIND,CHDIR,\n"); + printf("CONNECT,CMD,DEL,DELTASUM,DUP,EXIT,FILTER,FLIST,FUZZY,GENR,HASH,HLINK,\n"); + printf("ICONV,NSTR,OWN,RECV,SEND,TIME.\n"); printf("Flags may be comma-separated, for example: --debug=io,proto\n"); printf("An optional level suffix is accepted (e.g. --debug=io2); level 0\n"); - printf("silences that item. Other rsync debug flags are unsupported and rejected.\n"); + printf("silences that item. Unknown names are rejected.\n"); } void print_info_usage(void) { - printf("Supported info flags: COPY,NAME,MISC,SKIP,STATS,ALL,NONE\n"); + printf("Emitting info flags: COPY,NAME,MISC,SKIP,STATS,ALL,NONE\n"); + printf("Also accepted for rsync CLI parity (silent): BACKUP,DEL,FLIST,MOUNT,\n"); + printf("NONREG,PROGRESS,REMOVE,SYMSAFE.\n"); printf("Flags may be comma-separated, for example: --info=name,stats\n"); printf("An optional level suffix is accepted (e.g. --info=stats2); level 0\n"); - printf("silences that item. Other rsync info flags are unsupported and rejected.\n"); + printf("silences that item. Unknown names are rejected.\n"); } diff --git a/tests/integration/common.py b/tests/integration/common.py index 9678546..822fc4e 100644 --- a/tests/integration/common.py +++ b/tests/integration/common.py @@ -101,9 +101,15 @@ class CountingProxy: return counter[0] += len(data) - def run(self, cmd): + def run(self, cmd, join_timeout=20): """Forward one client run (the full command list) to the real server and - return the CompletedProcess after the counts have settled.""" + return the CompletedProcess after the counts have settled. + + ``join_timeout`` bounds how long to wait for the forwarding threads. The + client->server count is published as soon as the client side reaches EOF + (i.e. once the client process has exited), so callers that only need that + count can pass a small value instead of waiting for the server to close + its idle socket.""" def serve(): try: @@ -119,15 +125,15 @@ class CountingProxy: a.start() b.start() a.join() - b.join() self.client_to_server = c2s[0] + b.join() self.server_to_client = s2c[0] self._listener.close() - thread = threading.Thread(target=serve) + thread = threading.Thread(target=serve, daemon=True) thread.start() result = subprocess.run(cmd, capture_output=True, text=True, timeout=180) - thread.join(20) + thread.join(join_timeout) return result diff --git a/tests/integration/test_delete_timing_parity.py b/tests/integration/test_delete_timing_parity.py index 1991f87..b12355c 100644 --- a/tests/integration/test_delete_timing_parity.py +++ b/tests/integration/test_delete_timing_parity.py @@ -101,15 +101,24 @@ class _SlicingProxy: """Forward the client stream to a server, optionally cutting it or invoking a hook after a byte threshold. ``forward_limit`` mode resets both ends after that many client bytes (a mid-transfer failure). ``hook`` mode calls the - hook once and keeps forwarding to completion.""" + hook once and keeps forwarding to completion. + + With ``wait_for_reply`` the hook is a real barrier, not a timing guess: it + fires only after the server has sent *any* reply, which the receiver does + only after it has consumed the frames that precede the payload (the + per-directory delete plan for ``--delete-delay``). The caller pairs it with + ``--incremental`` so a per-file handshake reply is guaranteed mid-transfer. + """ def __init__(self, target_port, forward_limit=None, hook=None, hook_after=0, - throttle=0.0): + throttle=0.0, wait_for_reply=False): self.target = ("127.0.0.1", target_port) self.forward_limit = forward_limit self.hook = hook self.hook_after = hook_after self.throttle = throttle + self.wait_for_reply = wait_for_reply + self.server_replied = threading.Event() self.hook_called = threading.Event() self.listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) @@ -158,15 +167,7 @@ class _SlicingProxy: data = data[:room] backend.sendall(data) forwarded += len(data) - if (self.hook is not None and not self.hook_called.is_set() - and forwarded >= self.hook_after): - # Give the receiver time to process the (tiny) plan - # frames that precede this offset before the hook - # mutates the destination. - if self.throttle > 0: - time.sleep(0.2) - self.hook() - self.hook_called.set() + self._maybe_hook(forwarded) if self.forward_limit is not None and forwarded >= self.forward_limit: socks = [] break @@ -174,6 +175,10 @@ class _SlicingProxy: time.sleep(self.throttle) else: client.sendall(data) + # Any server reply proves the receiver consumed the + # frames that precede it, so the hook barrier is met. + self.server_replied.set() + self._maybe_hook(forwarded) except OSError: pass for sock in (client, backend): @@ -190,6 +195,19 @@ class _SlicingProxy: except OSError: pass + def _maybe_hook(self, forwarded): + """Fire the one-shot hook once its barrier is satisfied: enough client + bytes have been forwarded and, when ``wait_for_reply`` is set, the + server has sent a reply proving it processed the preceding frames.""" + if self.hook is None or self.hook_called.is_set(): + return + if forwarded < self.hook_after: + return + if self.wait_for_reply and not self.server_replied.is_set(): + return + self.hook() + self.hook_called.set() + def finish(self): self._thread.join(30) try: @@ -317,8 +335,16 @@ class TestDeleteDelayVsAfterSnapshot: # after the directory's plan (delay) has been processed. _write(new_extra, b"created mid-transfer\n") - proxy = _SlicingProxy(server.port, hook=hook, hook_after=MID_TRANSFER_BYTES, throttle=PROXY_THROTTLE) - flags = [timing] + (["--threads"] if mt else []) + # --incremental gives the receiver a mid-transfer handshake + # reply; the proxy waits for it (wait_for_reply) so the hook is + # causally after the plan frame, never a timing guess. + # --ignore-times forces the big file to transfer on the second + # timing too (the first run already installed it), keeping the + # mid-transfer reply present in both iterations. + proxy = _SlicingProxy(server.port, hook=hook, + hook_after=MID_TRANSFER_BYTES, + throttle=PROXY_THROTTLE, wait_for_reply=True) + flags = [timing, "--incremental", "--ignore-times"] + (["--threads"] if mt else []) result, _ = run_client(source, dest, flags=flags, port=proxy.port) proxy.finish() assert result.returncode == 0, ( diff --git a/tests/integration/test_output_parity.py b/tests/integration/test_output_parity.py index 0d49117..83f54bb 100644 --- a/tests/integration/test_output_parity.py +++ b/tests/integration/test_output_parity.py @@ -5,6 +5,7 @@ differential tests run the SAME transfer with real ``rsync 3.4.1`` and with fastsync and compare stdout, so they are skipped when rsync is unavailable. """ import os +import re import shutil import subprocess import sys @@ -324,37 +325,116 @@ class TestWireStatsParity: @requires_rsync @pytest.mark.ci def test_out_format_b_is_wire_bytes(self, shared_server): - """%b is true transferred (wire) bytes, not the source length: it must - differ from %l (the source length) and exceed it for a framed transfer.""" + """%b is the bytes actually transferred (wire), not the source length. + + A differential run against rsync confirms both implementations report a + framed value greater than %l. The exact numbers are not compared: each + counts its own protocol framing and checksum trailer, so the two are + protocol-specific and cannot be numerically equal (documented + divergence).""" source = os.path.join(TEST_DATA_DIR, "wire_b_src") dest = os.path.join(TEST_DATA_DIR, "wire_b_dst") + rdst = os.path.join(TEST_DATA_DIR, "wire_b_rdst") _make_one_file(source, "f.bin", 5000) clean_dir(dest) - result, _ = run_client(source, dest, flags=["-a", "--out-format=%b %l %c"], + clean_dir(rdst) + fmt = "%b %l" + rsync_result = _rsync(["-a", "--out-format=" + fmt, source + "/", rdst + "/"]) + assert rsync_result.returncode == 0, rsync_result.stderr + result, _ = run_client(source, dest, flags=["-a", "--out-format=" + fmt], port=shared_server.port) assert result.returncode == 0, result.stderr[:300] - line = result.stdout.strip() - parts = line.split() - assert len(parts) == 3 and all(p.isdigit() for p in parts), line - wire_b, src_l, wire_c = (int(p) for p in parts) - assert src_l == 5000, line - assert wire_b > src_l, f"%b must include wire framing: {line}" + rb, rl = (int(x) for x in rsync_result.stdout.split()[:2]) + fb, fl = (int(x) for x in result.stdout.split()[:2]) + assert rl == fl == 5000, (rsync_result.stdout, result.stdout) + assert rb > rl, f"rsync %b must include framing: {rsync_result.stdout!r}" + assert fb > fl, f"fastsync %b must include framing: {result.stdout!r}" @requires_rsync @pytest.mark.ci - def test_progress_first_frame_matches_rsync(self, shared_server): - """For a sub-32 KiB file the first --progress frame is deterministic - (0.00 kB/s, 0:00:00) and must be byte-identical to rsync's.""" + def test_out_format_c_whole_file_matches_rsync(self, shared_server): + """%c is the block-checksum bytes received. rsync reports its 16-byte + sum header even for a whole-file transfer (no basis), so `%c` must match + rsync exactly for the whole-file case.""" + source = os.path.join(TEST_DATA_DIR, "wire_c_src") + dest = os.path.join(TEST_DATA_DIR, "wire_c_dst") + rdst = os.path.join(TEST_DATA_DIR, "wire_c_rdst") + _make_one_file(source, "f.bin", 5000) + clean_dir(dest) + clean_dir(rdst) + fmt = "%c %l %n" + rsync_result = _rsync(["-a", "--out-format=" + fmt, source + "/", rdst + "/"]) + assert rsync_result.returncode == 0, rsync_result.stderr + result, _ = run_client(source, dest, flags=["-a", "--out-format=" + fmt], + port=shared_server.port) + assert result.returncode == 0, result.stderr[:300] + + def file_lines(text): + return [ + line for line in text.splitlines() + if line and not line.rsplit(" ", 1)[-1].endswith("/") + ] + + assert file_lines(result.stdout) == file_lines(rsync_result.stdout), ( + f"rsync={rsync_result.stdout!r} fastsync={result.stdout!r}" + ) + assert result.stdout.split()[0] == rsync_result.stdout.split()[0] == "16", ( + f"%c must be rsync's 16-byte sum header: {result.stdout!r}" + ) + + @requires_rsync + @pytest.mark.ci + def test_out_format_c_delta_mode_divergence(self, shared_server): + """Documented residual: with delta enabled, rsync's %c is its 16-byte sum + header plus one checksum entry per block (protocol-specific, so it grows + with the basis size), while FastSync's %c is the bytes of its own delta + handshake. FastSync's delta %c therefore cannot match rsync numerically; + only the whole-file case is aligned. Pinned here so a future change is + noticed.""" + source = os.path.join(TEST_DATA_DIR, "wire_cd_src") + dest = os.path.join(TEST_DATA_DIR, "wire_cd_dst") + rdst = os.path.join(TEST_DATA_DIR, "wire_cd_rdst") + _make_one_file(source, "f.bin", 5000) + clean_dir(dest) + clean_dir(rdst) + fmt = "%c %l" + # rsync local default is whole-file; force the block-delta path. + rsync_result = _rsync(["-a", "--no-whole-file", "--out-format=" + fmt, + source + "/", rdst + "/"]) + assert rsync_result.returncode == 0, rsync_result.stderr + result, _ = run_client(source, dest, + flags=["-a", "--incremental", "--delta", + "--out-format=" + fmt], + port=shared_server.port) + assert result.returncode == 0, result.stderr[:300] + rs_c = int(rsync_result.stdout.split()[0]) + fs_c = int(result.stdout.split()[0]) + # No basis exists, so rsync still reports only its sum header. + assert rs_c == 16, rsync_result.stdout + # FastSync reports its own handshake bytes and is not aligned. + assert fs_c > 16, ( + f"FastSync delta %c changed to {fs_c}; the documented divergence " + "may be closable now" + ) + + @requires_rsync + @pytest.mark.ci + @pytest.mark.parametrize("mt", [False, True]) + @pytest.mark.parametrize("progress_flag", ["--progress", "-P"]) + def test_progress_first_frame_matches_rsync(self, shared_server, progress_flag, mt): + """For a sub-32 KiB file the first --progress/-P frame is deterministic + (0.00 kB/s, 0:00:00) and must be byte-identical to rsync's, in both the + single-threaded and --threads send paths.""" source = os.path.join(TEST_DATA_DIR, "wire_pg_src") dest = os.path.join(TEST_DATA_DIR, "wire_pg_dst") rdst = os.path.join(TEST_DATA_DIR, "wire_pg_rdst") _make_one_file(source, "f.bin", 100) clean_dir(dest) clean_dir(rdst) - rsync_result = _rsync(["-a", "--progress", source + "/", rdst + "/"]) + rsync_result = _rsync(["-a", progress_flag, source + "/", rdst + "/"]) assert rsync_result.returncode == 0, rsync_result.stderr - result, _ = run_client(source, dest, flags=["-a", "--progress"], - port=shared_server.port) + flags = ["-a", progress_flag] + (["--threads"] if mt else []) + result, _ = run_client(source, dest, flags=flags, port=shared_server.port) assert result.returncode == 0, result.stderr[:300] def frames(text): @@ -369,8 +449,10 @@ class TestWireStatsParity: @requires_rsync @pytest.mark.ci - def test_stats_selected_lines_match_rsync(self, shared_server): - """The protocol-independent --stats lines must match rsync exactly.""" + @pytest.mark.parametrize("mt", [False, True]) + def test_stats_selected_lines_match_rsync(self, shared_server, mt): + """The protocol-independent --stats lines must match rsync exactly, in + both the single-threaded and --threads (multithreaded) send paths.""" source = os.path.join(TEST_DATA_DIR, "wire_st_src") dest = os.path.join(TEST_DATA_DIR, "wire_st_dst") rdst = os.path.join(TEST_DATA_DIR, "wire_st_rdst") @@ -379,8 +461,8 @@ class TestWireStatsParity: clean_dir(rdst) rsync_result = _rsync(["-a", "--stats", source + "/", rdst + "/"]) assert rsync_result.returncode == 0, rsync_result.stderr - result, _ = run_client(source, dest, flags=["-a", "--stats"], - port=shared_server.port) + flags = ["-a", "--stats"] + (["--threads"] if mt else []) + result, _ = run_client(source, dest, flags=flags, port=shared_server.port) assert result.returncode == 0, result.stderr[:300] keys = ( "Number of regular files transferred", @@ -389,6 +471,7 @@ class TestWireStatsParity: "Literal data", "Matched data", "Number of deleted files", + "File list size", ) def pick(text): @@ -405,8 +488,68 @@ class TestWireStatsParity: @requires_rsync @pytest.mark.ci - def test_dry_run_delete_lines_match_rsync(self): - """-n --delete emits transfer-relative `*deleting` lines like rsync.""" + def test_stats_file_count_breakdown_residual(self, shared_server): + """Residual (row #3): rsync prints the `Number of files` and + `Number of created files` lines with a per-type breakdown + (`(reg: X, dir: Y, link: Z)`). + + FastSync cannot reproduce it from what the sender currently knows: the + scanner does not put directory entries in the transfer list (directories + are created implicitly), and without a per-entry destination-probe the + sender cannot tell which entries the receiver newly created. So FastSync + prints the bare transferred-entry count. This test pins the divergence + explicitly -- the row must not be marked ✅. + """ + source = os.path.join(TEST_DATA_DIR, "wire_stc_src") + dest = os.path.join(TEST_DATA_DIR, "wire_stc_dst") + rdst = os.path.join(TEST_DATA_DIR, "wire_stc_rdst") + _make_one_file(source, "f.bin", 6000) + clean_dir(dest) + clean_dir(rdst) + rsync_result = _rsync(["-a", "--stats", source + "/", rdst + "/"]) + assert rsync_result.returncode == 0, rsync_result.stderr + result, _ = run_client(source, dest, flags=["-a", "--stats"], + port=shared_server.port) + assert result.returncode == 0, result.stderr[:300] + + def stats_line(text, key): + for line in text.splitlines(): + if line.startswith(key + ":"): + return line + return None + + r_files = stats_line(rsync_result.stdout, "Number of files") + r_created = stats_line(rsync_result.stdout, "Number of created files") + f_files = stats_line(result.stdout, "Number of files") + f_created = stats_line(result.stdout, "Number of created files") + + # rsync always carries the type breakdown (the source root counts as a + # directory; the single regular file as reg). + assert re.match(r"Number of files: 2 \(reg: 1, dir: 1\)$", r_files), r_files + assert re.match(r"Number of created files: 1 \(reg: 1\)$", r_created), r_created + # FastSync prints only the bare count: no directory accounting and no + # per-entry "created" knowledge. + assert re.fullmatch(r"Number of files: 1", f_files), f_files + assert re.fullmatch(r"Number of created files: 1", f_created), f_created + + @requires_rsync + @pytest.mark.ci + @pytest.mark.parametrize("mt", [ + False, + pytest.param( + True, + marks=pytest.mark.xfail( + reason="known gap: the --threads dry-run delete path does not " + "consume the receiver's STATUS_STATS delete list yet, so " + "`-n --delete` emits no *deleting lines (tracked by the " + "parity-blockers STATUS_STATS fix)", + strict=False, + ), + ), + ]) + def test_dry_run_delete_lines_match_rsync(self, mt): + """-n --delete emits transfer-relative `*deleting` lines like rsync + (single-threaded; the --threads variant is a documented xfail).""" source = os.path.join(TEST_DATA_DIR, "wire_del_src") dest = os.path.join(TEST_DATA_DIR, "wire_del_dst") rdst = os.path.join(TEST_DATA_DIR, "wire_del_rdst") @@ -442,10 +585,10 @@ class TestWireStatsParity: line for line in rsync_result.stdout.splitlines() if line.startswith("*deleting") ) # The shared session server refuses deletion; start one that allows it. + flags = ["-a", "-n", "--delete", "-i"] + (["--threads"] if mt else []) with ServerManager() as server: server.start(extra_args=["--allow-delete"]) - result, _ = run_client(source, dest, flags=["-a", "-n", "--delete", "-i"], - port=server.port) + result, _ = run_client(source, dest, flags=flags, port=server.port) assert result.returncode == 0, result.stderr[:300] fast_del = sorted( line for line in result.stdout.splitlines() if line.startswith("*deleting") diff --git a/tests/integration/test_parity_quickwins.py b/tests/integration/test_parity_quickwins.py index 4421dfc..9e7a0dc 100644 --- a/tests/integration/test_parity_quickwins.py +++ b/tests/integration/test_parity_quickwins.py @@ -12,6 +12,8 @@ import pytest sys.path.insert(0, os.path.dirname(__file__)) from common import ( + CLIENT_CMD, + CountingProxy, TEST_DATA_DIR, ServerManager, run_client, @@ -593,6 +595,57 @@ class TestVerifyAndFlip: assert result.returncode == 0, result.stderr[:300] _assert_same_tree(rdst, get_dest_received_dir(dest, source), "(--preallocate)") + @requires_rsync + @pytest.mark.ci + def test_sparse_preallocate_blocks_match_rsync(self, shared_server): + """rsync lets --preallocate win over --sparse: the hole file gets its + full space reserved (st_blocks ~ size/512) even though the sparse writer + seeks over the zero run. FastSync must agree both on the block count and + with rsync's exact value for each flag combination.""" + source = self._src("sparsepre") + dest = self._dst("sparsepre") + rdst = self._dst("sparsepre_r") + total = 1024 * 1024 + # A zero run >= the 4 KiB sparse threshold in the middle of the image. + with open(os.path.join(source, "hole.bin"), "wb") as fh: + fh.write(os.urandom(64 * 1024)) + fh.write(b"\x00" * (768 * 1024)) + fh.write(os.urandom(total - 64 * 1024 - 768 * 1024)) + + def run_both(flags, tag): + rsync_dst = self._dst(f"sparsepre_{tag}_r") + fs_dst = self._dst(f"sparsepre_{tag}_f") + assert _rsync(["-a"] + flags + [source + "/", rsync_dst + "/"]).returncode == 0 + result, _ = run_client(source, fs_dst, flags=["-a"] + flags, + port=shared_server.port) + assert result.returncode == 0, result.stderr[:300] + rfile = os.path.join(rsync_dst, "hole.bin") + ffile = os.path.join(get_dest_received_dir(fs_dst, source), "hole.bin") + with open(rfile, "rb") as rfh, open(ffile, "rb") as ffh: + assert rfh.read() == ffh.read(), f"{tag}: content diverged" + return os.stat(rfile).st_blocks, os.stat(ffile).st_blocks + + r_sparse, f_sparse = run_both(["--sparse"], "sparse") + r_both, f_both = run_both(["--sparse", "--preallocate"], "both") + + assert f_sparse == r_sparse, ( + f"--sparse st_blocks: fastsync={f_sparse} rsync={r_sparse}" + ) + assert f_both == r_both, ( + f"--sparse --preallocate st_blocks: fastsync={f_both} rsync={r_both}" + ) + # Only meaningful where the filesystem actually reports holes; otherwise + # both sides simply allocate the full size and the equality above holds. + has_holes = r_sparse * 512 < total + if has_holes: + assert f_both > f_sparse, ( + "preallocate must win over sparse: " + f"sparse={f_sparse} both={f_both} blocks" + ) + assert r_both > r_sparse, ( + f"rsync preallocate must win: sparse={r_sparse} both={r_both}" + ) + @requires_rsync def test_fuzzy_content_matches_rsync(self, shared_server): source = self._src("fuzzy") @@ -717,6 +770,58 @@ class TestVerifyAndFlip: "--link-dest must hard-link to the basis file" +class TestIgnoreExistingShortCircuit: + """#9: --ignore-existing is decided by the receiver during the per-file + check, before the sender streams any payload. A large destination file that + is already present must therefore cost almost no wire bytes, not the full + file; rsync short-circuits the same way.""" + + @requires_rsync + @pytest.mark.ci + def test_skip_answered_before_payload(self): + source = os.path.join(TEST_DATA_DIR, "qw_ie_wire_src") + dest = os.path.join(TEST_DATA_DIR, "qw_ie_wire_dst") + rdst = os.path.join(TEST_DATA_DIR, "qw_ie_wire_rdst") + clean_dir(source) + clean_dir(dest) + clean_dir(rdst) + big = b"S" * (4 * 1024 * 1024) + with open(os.path.join(source, "big.bin"), "wb") as fh: + fh.write(big) + for root in (get_dest_received_dir(dest, source), rdst): + os.makedirs(root, exist_ok=True) + with open(os.path.join(root, "big.bin"), "wb") as fh: + fh.write(b"D" * len(big)) + + assert _rsync(["-a", "--ignore-existing", source + "/", + rdst + "/"]).returncode == 0 + + # A dedicated one-shot server keeps the proxy's connection teardown + # deterministic (the session-wide server can linger on an idle socket). + with ServerManager() as server: + cmd = CLIENT_CMD + [ + "--source-dir", source, "--dest-dir", dest, "--save-to-disk", + "--server-port", str(server.port), "-a", "--ignore-existing", + ] + proxy = CountingProxy(server.port) + # Only the client->server count matters here; it is final once the + # client exits, so do not wait for the server to close its idle + # socket (which can take the full default join timeout). + result = proxy.run(cmd, join_timeout=1.0) + assert result.returncode == 0, (result.stderr or result.stdout)[:300] + # The receiver answered the skip before the sender transmitted the file: + # only config/path/check frames crossed, not the 4 MiB payload. + assert proxy.client_to_server < len(big) // 10, ( + f"--ignore-existing transmitted {proxy.client_to_server} bytes for a " + f"skipped {len(big)}-byte file" + ) + received = get_dest_received_dir(dest, source) + with open(os.path.join(received, "big.bin"), "rb") as fh: + assert fh.read() == b"D" * len(big), \ + "--ignore-existing must preserve the destination content" + _assert_same_tree(rdst, received, "(--ignore-existing wire short-circuit)") + + @pytest.mark.skipif(os.geteuid() != 0, reason="ownership mapping requires root") class TestOwnershipMapping: """#33/#34/#35: --usermap/--groupmap/--chown match rsync's numeric result.""" @@ -782,12 +887,33 @@ class TestFakeSuper: assert fh.read() == b"fake-super-data\n" -class TestInfoDebugFlagParity: - """#01/#02: rsync's info/debug spellings are either mapped to real output or - rejected by name (never silently ignored).""" +# rsync 3.4.1's full --info/--debug vocabularies (from `rsync --info=help` / +# `--debug=help`). FastSync must accept every one of them; only the categories +# with an existing FastSync counterpart emit output, the rest are accepted but +# currently silent. +RSYNC_INFO_CATEGORIES = ( + "backup", "copy", "del", "flist", "misc", "mount", "name", "nonreg", + "progress", "remove", "skip", "stats", "symsafe", "all", "none", +) +RSYNC_DEBUG_CATEGORIES = ( + "acl", "backup", "bind", "chdir", "connect", "cmd", "del", "deltasum", + "dup", "exit", "filter", "flist", "fuzzy", "genr", "hash", "hlink", + "iconv", "io", "nstr", "own", "proto", "recv", "send", "time", "all", + "none", +) +# Extra categories FastSync also accepts: rsync's own help spells these +# `symsafe`/`hlink`/`own`, but the historical aliases are kept working, and +# `pack`/`util` are FastSync-specific debug channels. +FASTSYNC_INFO_ALIASES = ("syms",) +FASTSYNC_DEBUG_ALIASES = ("hl", "owner", "pack", "util") - @requires_rsync - def test_mapped_info_categories_accepted_like_rsync(self, shared_server): + +class TestInfoDebugFlagParity: + """#01/#02: FastSync accepts rsync 3.4.1's full --info/--debug vocabulary + (with level suffixes) so a valid rsync invocation is never rejected up + front. Unknown names are still refused by name.""" + + def _tree(self): source = os.path.join(TEST_DATA_DIR, "qw_flags_src") dest = os.path.join(TEST_DATA_DIR, "qw_flags_dst") rdst = os.path.join(TEST_DATA_DIR, "qw_flags_rdst") @@ -796,54 +922,78 @@ class TestInfoDebugFlagParity: clean_dir(rdst) with open(os.path.join(source, "a.txt"), "wb") as fh: fh.write(b"a\n") - for cat in ("stats2", "name", "copy", "misc", "skip", "STATS2"): - assert _rsync(["-a", "--info=" + cat, source + "/", rdst + "/"]).returncode == 0 + return source, dest, rdst + + @requires_rsync + @pytest.mark.ci + def test_info_vocabulary_accepted_like_rsync(self, shared_server): + source, dest, rdst = self._tree() + for cat in RSYNC_INFO_CATEGORIES: + rs = _rsync(["-a", "--info=" + cat, source + "/", rdst + "/"]) + assert rs.returncode == 0, f"rsync rejected --info={cat}: {rs.stderr}" clean_dir(rdst) result, _ = run_client(source, dest, flags=["--info=" + cat], port=shared_server.port) assert result.returncode == 0, ( - f"--info={cat} must be accepted: {result.stderr[:200]}" - ) - - @requires_rsync - def test_mapped_debug_categories_accepted_like_rsync(self, shared_server): - source = os.path.join(TEST_DATA_DIR, "qw_dflags_src") - dest = os.path.join(TEST_DATA_DIR, "qw_dflags_dst") - rdst = os.path.join(TEST_DATA_DIR, "qw_dflags_rdst") - clean_dir(source) - clean_dir(dest) - clean_dir(rdst) - with open(os.path.join(source, "a.txt"), "wb") as fh: - fh.write(b"a\n") - for cat in ("io2", "proto0", "all"): - assert _rsync(["-a", "--debug=" + cat, source + "/", rdst + "/"]).returncode == 0 - clean_dir(rdst) - result, _ = run_client(source, dest, flags=["--debug=" + cat], - port=shared_server.port) - assert result.returncode == 0, ( - f"--debug={cat} must be accepted: {result.stderr[:200]}" + f"--info={cat} must be accepted like rsync: {result.stderr[:200]}" ) @requires_rsync @pytest.mark.ci - def test_unmapped_categories_rejected_by_name(self, shared_server): - """rsync accepts del/filter; fastsync has no mapping so it must refuse - loudly, naming the category, rather than silently ignoring it.""" - source = os.path.join(TEST_DATA_DIR, "qw_umap_src") - dest = os.path.join(TEST_DATA_DIR, "qw_umap_dst") - clean_dir(source) - clean_dir(dest) - with open(os.path.join(source, "a.txt"), "wb") as fh: - fh.write(b"a\n") - # rsync accepts these (so they are valid rsync invocations). - assert _rsync(["-a", "--info=del", source + "/", dest + "/"]).returncode == 0 - assert _rsync(["-a", "--debug=filter", source + "/", dest + "/"]).returncode == 0 - for flag, name in (("--info=del", "del"), ("--debug=filter", "filter")): + def test_debug_vocabulary_accepted_like_rsync(self, shared_server): + source, dest, rdst = self._tree() + for cat in RSYNC_DEBUG_CATEGORIES: + rs = _rsync(["-a", "--debug=" + cat, source + "/", rdst + "/"]) + assert rs.returncode == 0, f"rsync rejected --debug={cat}: {rs.stderr}" + clean_dir(rdst) + result, _ = run_client(source, dest, flags=["--debug=" + cat], + port=shared_server.port) + assert result.returncode == 0, ( + f"--debug={cat} must be accepted like rsync: {result.stderr[:200]}" + ) + + @requires_rsync + @pytest.mark.ci + def test_level_suffixes_accepted_like_rsync(self, shared_server): + source, dest, rdst = self._tree() + for flag in ("--info=stats2", "--info=copy0", "--info=all0", + "--debug=io2", "--debug=proto0", "--debug=all4"): + rs = _rsync(["-a", flag, source + "/", rdst + "/"]) + assert rs.returncode == 0, f"rsync rejected {flag}: {rs.stderr}" + clean_dir(rdst) + result, _ = run_client(source, dest, flags=[flag], + port=shared_server.port) + assert result.returncode == 0, ( + f"{flag} must be accepted like rsync: {result.stderr[:200]}" + ) + + def test_fastsync_alias_categories_accepted(self, shared_server): + """FastSync-specific/alias spellings: accepted (rsync spells them + symsafe/hlink/own) but not emitted.""" + source, dest, _ = self._tree() + for flag in (["--info=" + c for c in FASTSYNC_INFO_ALIASES] + + ["--debug=" + c for c in FASTSYNC_DEBUG_ALIASES]): + result, _ = run_client(source, dest, flags=[flag], + port=shared_server.port) + assert result.returncode == 0, ( + f"{flag} must be accepted: {result.stderr[:200]}" + ) + + @requires_rsync + @pytest.mark.ci + def test_unknown_categories_rejected_by_name(self, shared_server): + """Truly unknown names are refused by name, exactly like rsync.""" + source, dest, rdst = self._tree() + for flag, name in (("--info=bogus", "bogus"), + ("--debug=bogus", "bogus")): + rs = _rsync(["-a", flag, source + "/", rdst + "/"]) + assert rs.returncode != 0, f"rsync unexpectedly accepted {flag}" result, _ = run_client(source, dest, flags=[flag], port=shared_server.port) assert result.returncode != 0, f"{flag} must be rejected" - assert name in (result.stderr or ""), \ + assert name in (result.stderr or ""), ( f"{flag} must be rejected by name, got: {result.stderr[:200]}" + ) class TestStopAtParity: diff --git a/tests/test_client_cli.c b/tests/test_client_cli.c index d3ef21a..2972a8a 100644 --- a/tests/test_client_cli.c +++ b/tests/test_client_cli.c @@ -776,7 +776,7 @@ static void test_parse_args_debug_help() { } static void test_parse_args_debug_flags_validation() { - static const char* const values[] = {"", "io,", ",io", "io,,proto", "acl", "tls", "unknown"}; + static const char* const values[] = {"", "io,", ",io", "io,,proto", "tls", "unknown"}; for (size_t i = 0; i < sizeof(values) / sizeof(values[0]); i++) { Config* cfg = config_create(); char option[64]; @@ -1345,6 +1345,24 @@ static void test_parse_args_info_name_and_help() { config_delete(cfg); } +/* rsync 3.4.1's remaining --info/--debug categories parse successfully but + * have no FastSync output wired to them, so they must not set any log flag. */ +static void test_parse_args_rsync_flag_vocabulary_accepted() { + Config* cfg = config_create(); + char* argv[] = {"fastsync", "--info=backup,del,flist,mount,nonreg,progress,remove,symsafe,syms", + "--debug=acl,backup,bind,chdir,cmd,connect,del,deltasum,dup,exit," + "filter,flist,fuzzy,genr,hash,hlink,iconv,nstr,own,recv,send,time," + "hl,owner", + "/src", "/dst"}; + int positional_args[2]; + int positional_count = 0; + + EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0); + EXPECT_EQ_INT(cfg->info_level, 0); + EXPECT_EQ_INT(cfg->debug_level, 0); + config_delete(cfg); +} + /* Test parse_args with --archive flag */ /* rsync accepts a trailing level digit on --debug/--info items (e.g. io2, * all4); level 0 silences the item. */ @@ -4441,6 +4459,7 @@ void test_client_cli() { test_parse_args_debug_flags(); test_parse_args_debug_help(); test_parse_args_debug_flags_validation(); + test_parse_args_rsync_flag_vocabulary_accepted(); test_parse_args_debug_info_levels(); test_parse_args_modify_window(); test_parse_args_rejects_invalid_modify_window(); diff --git a/tests/test_config.c b/tests/test_config.c index cea9088..b7e6c7b 100644 --- a/tests/test_config.c +++ b/tests/test_config.c @@ -2920,7 +2920,7 @@ static unsigned long long capture_wire_hash(const Config* cfg, size_t* out_len) return h; } -/* Byte-for-byte wire compatibility guard (protocol 2.24.0). The expected hash +/* Byte-for-byte wire compatibility guard (protocol 2.26.0). The expected hash * pins the pre-X-macro byte stream; the refactor MUST NOT change it. */ static void test_config_wire_golden() { if (is_running_under_valgrind())