Merge branch 'feat/parity-wirestats' into feat/parity-completion
# Conflicts: # src/server/receiver_pipeline.c # src/shared/config.h # src/shared/protocol.h # tests/integration/test_fault_injection.py # tests/integration/test_preflight.py # tests/test_client_cli.c # tests/test_config.c
This commit is contained in:
@@ -36,7 +36,7 @@ from common import ( # noqa: E402
|
||||
verify_transfer,
|
||||
)
|
||||
|
||||
PROTOCOL_VERSION = b"2.24.0"
|
||||
PROTOCOL_VERSION = b"2.25.0"
|
||||
STATUS_MANIFEST = 5
|
||||
STATUS_OK = 0
|
||||
|
||||
|
||||
@@ -1855,8 +1855,8 @@ class TestDelete:
|
||||
)
|
||||
assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}"
|
||||
output = result.stdout + result.stderr
|
||||
assert "Sent " in output and "MB" in output, "--progress produced no stable byte marker"
|
||||
assert "Done." in output, "--progress did not report completion"
|
||||
assert "sending incremental file list" in output, "--progress produced no rsync header"
|
||||
assert "(xfr#" in output, "--progress produced no per-file xfr block"
|
||||
|
||||
def test_human_readable_stats(self, shared_server):
|
||||
clean_dir(DEST_DIR)
|
||||
@@ -1891,8 +1891,8 @@ class TestDelete:
|
||||
)
|
||||
assert result.returncode == 0, f"Exit {result.returncode}: {result.stderr[:100]}"
|
||||
output = result.stdout + result.stderr
|
||||
assert "Sent " in output
|
||||
assert "Done." in output
|
||||
assert "sending incremental file list" in output
|
||||
assert "(xfr#" in output
|
||||
|
||||
|
||||
class TestInfo:
|
||||
|
||||
@@ -12,7 +12,7 @@ import sys
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from common import TEST_DATA_DIR, run_client, clean_dir, get_dest_received_dir
|
||||
from common import TEST_DATA_DIR, run_client, clean_dir, get_dest_received_dir, ServerManager
|
||||
|
||||
RSYNC = shutil.which("rsync")
|
||||
requires_rsync = pytest.mark.skipif(RSYNC is None, reason="rsync 3.4.1 not installed")
|
||||
@@ -280,3 +280,174 @@ class TestListOnlyParity:
|
||||
assert fast_lines == rsync_lines, (
|
||||
f"rsync={rsync_lines}\nfastsync={fast_lines}"
|
||||
)
|
||||
|
||||
|
||||
def _make_one_file(root, name="f.bin", size=100):
|
||||
clean_dir(root)
|
||||
with open(os.path.join(root, name), "wb") as fh:
|
||||
fh.write(bytes((i * 7 + 3) & 0xFF for i in range(size)))
|
||||
|
||||
|
||||
class TestWireStatsParity:
|
||||
"""Wire-counter output parity: --out-format %b/%c/%C, --progress and
|
||||
--stats versus real rsync 3.4.1."""
|
||||
|
||||
@requires_rsync
|
||||
@pytest.mark.ci
|
||||
def test_out_format_checksum_matches_rsync(self, shared_server):
|
||||
"""%C (whole-file xxh128, seed 0) is protocol-independent, so the full
|
||||
`%C %l %n` line must be byte-identical to rsync."""
|
||||
source = os.path.join(TEST_DATA_DIR, "wire_ck_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "wire_ck_dst")
|
||||
rdst = os.path.join(TEST_DATA_DIR, "wire_ck_rdst")
|
||||
_make_one_file(source, "f.bin", 200000)
|
||||
clean_dir(dest)
|
||||
clean_dir(rdst)
|
||||
fmt = "%C %l %n"
|
||||
rsync_result = _rsync(["-a", "--out-format=" + fmt, source + "/", rdst + "/"])
|
||||
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||
result, _ = run_client(source, dest, flags=["-a", "--out-format=" + fmt],
|
||||
port=shared_server.port)
|
||||
assert result.returncode == 0, result.stderr[:300]
|
||||
|
||||
def file_lines(text):
|
||||
# Ignore the root directory entry: fastsync does not transfer the
|
||||
# source-root dir itself (a separate pre-existing divergence).
|
||||
return [
|
||||
line for line in text.splitlines() if not line.rsplit(" ", 1)[-1].endswith("/")
|
||||
]
|
||||
|
||||
assert file_lines(result.stdout) == file_lines(rsync_result.stdout), (
|
||||
f"rsync={rsync_result.stdout!r} fastsync={result.stdout!r}"
|
||||
)
|
||||
|
||||
@requires_rsync
|
||||
@pytest.mark.ci
|
||||
def test_out_format_b_is_wire_bytes(self, shared_server):
|
||||
"""%b is true transferred (wire) bytes, not the source length: it must
|
||||
differ from %l (the source length) and exceed it for a framed transfer."""
|
||||
source = os.path.join(TEST_DATA_DIR, "wire_b_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "wire_b_dst")
|
||||
_make_one_file(source, "f.bin", 5000)
|
||||
clean_dir(dest)
|
||||
result, _ = run_client(source, dest, flags=["-a", "--out-format=%b %l %c"],
|
||||
port=shared_server.port)
|
||||
assert result.returncode == 0, result.stderr[:300]
|
||||
line = result.stdout.strip()
|
||||
parts = line.split()
|
||||
assert len(parts) == 3 and all(p.isdigit() for p in parts), line
|
||||
wire_b, src_l, wire_c = (int(p) for p in parts)
|
||||
assert src_l == 5000, line
|
||||
assert wire_b > src_l, f"%b must include wire framing: {line}"
|
||||
|
||||
@requires_rsync
|
||||
@pytest.mark.ci
|
||||
def test_progress_first_frame_matches_rsync(self, shared_server):
|
||||
"""For a sub-32 KiB file the first --progress frame is deterministic
|
||||
(0.00 kB/s, 0:00:00) and must be byte-identical to rsync's."""
|
||||
source = os.path.join(TEST_DATA_DIR, "wire_pg_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "wire_pg_dst")
|
||||
rdst = os.path.join(TEST_DATA_DIR, "wire_pg_rdst")
|
||||
_make_one_file(source, "f.bin", 100)
|
||||
clean_dir(dest)
|
||||
clean_dir(rdst)
|
||||
rsync_result = _rsync(["-a", "--progress", source + "/", rdst + "/"])
|
||||
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||
result, _ = run_client(source, dest, flags=["-a", "--progress"],
|
||||
port=shared_server.port)
|
||||
assert result.returncode == 0, result.stderr[:300]
|
||||
|
||||
def frames(text):
|
||||
# subprocess text mode normalizes \r to \n (universal newlines).
|
||||
return [p for p in text.split("\n") if "%" in p]
|
||||
|
||||
rsync_frames = frames(rsync_result.stdout)
|
||||
fast_frames = frames(result.stdout)
|
||||
assert rsync_frames and fast_frames, (rsync_result.stdout, result.stdout)
|
||||
assert fast_frames[0] == rsync_frames[0], (rsync_frames[0], fast_frames[0])
|
||||
assert "(xfr#1," in fast_frames[-1], fast_frames[-1]
|
||||
|
||||
@requires_rsync
|
||||
@pytest.mark.ci
|
||||
def test_stats_selected_lines_match_rsync(self, shared_server):
|
||||
"""The protocol-independent --stats lines must match rsync exactly."""
|
||||
source = os.path.join(TEST_DATA_DIR, "wire_st_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "wire_st_dst")
|
||||
rdst = os.path.join(TEST_DATA_DIR, "wire_st_rdst")
|
||||
_make_one_file(source, "f.bin", 6000)
|
||||
clean_dir(dest)
|
||||
clean_dir(rdst)
|
||||
rsync_result = _rsync(["-a", "--stats", source + "/", rdst + "/"])
|
||||
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||
result, _ = run_client(source, dest, flags=["-a", "--stats"],
|
||||
port=shared_server.port)
|
||||
assert result.returncode == 0, result.stderr[:300]
|
||||
keys = (
|
||||
"Number of regular files transferred",
|
||||
"Total file size",
|
||||
"Total transferred file size",
|
||||
"Literal data",
|
||||
"Matched data",
|
||||
"Number of deleted files",
|
||||
)
|
||||
|
||||
def pick(text):
|
||||
out = {}
|
||||
for line in text.splitlines():
|
||||
for key in keys:
|
||||
if line.startswith(key + ":"):
|
||||
out[key] = line
|
||||
return out
|
||||
|
||||
assert pick(result.stdout) == pick(rsync_result.stdout), (
|
||||
f"rsync={pick(rsync_result.stdout)} fastsync={pick(result.stdout)}"
|
||||
)
|
||||
|
||||
@requires_rsync
|
||||
@pytest.mark.ci
|
||||
def test_dry_run_delete_lines_match_rsync(self):
|
||||
"""-n --delete emits transfer-relative `*deleting` lines like rsync."""
|
||||
source = os.path.join(TEST_DATA_DIR, "wire_del_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "wire_del_dst")
|
||||
rdst = os.path.join(TEST_DATA_DIR, "wire_del_rdst")
|
||||
clean_dir(source)
|
||||
clean_dir(dest)
|
||||
clean_dir(rdst)
|
||||
with open(os.path.join(source, "a.txt"), "wb") as fh:
|
||||
fh.write(b"a\n")
|
||||
for root, entries in (
|
||||
(rdst, {"extra.txt": b"x\n"}),
|
||||
(rdst, {"sub/y.txt": b"y\n", "extradir/z.txt": b"z\n"}),
|
||||
):
|
||||
for rel, data in entries.items():
|
||||
full = os.path.join(root, rel)
|
||||
os.makedirs(os.path.dirname(full), exist_ok=True)
|
||||
with open(full, "wb") as fh:
|
||||
fh.write(data)
|
||||
# FastSync mirrors the source's absolute path under dest.
|
||||
received = get_dest_received_dir(dest, source)
|
||||
for rel, data in (
|
||||
("extra.txt", b"x\n"),
|
||||
("sub/y.txt", b"y\n"),
|
||||
("extradir/z.txt", b"z\n"),
|
||||
):
|
||||
full = os.path.join(received, rel)
|
||||
os.makedirs(os.path.dirname(full), exist_ok=True)
|
||||
with open(full, "wb") as fh:
|
||||
fh.write(data)
|
||||
|
||||
rsync_result = _rsync(["-a", "-n", "--delete", "-i", source + "/", rdst + "/"])
|
||||
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||
rsync_del = sorted(
|
||||
line for line in rsync_result.stdout.splitlines() if line.startswith("*deleting")
|
||||
)
|
||||
# The shared session server refuses deletion; start one that allows it.
|
||||
with ServerManager() as server:
|
||||
server.start(extra_args=["--allow-delete"])
|
||||
result, _ = run_client(source, dest, flags=["-a", "-n", "--delete", "-i"],
|
||||
port=server.port)
|
||||
assert result.returncode == 0, result.stderr[:300]
|
||||
fast_del = sorted(
|
||||
line for line in result.stdout.splitlines() if line.startswith("*deleting")
|
||||
)
|
||||
assert fast_del == rsync_del, f"rsync={rsync_del}\nfastsync={fast_del}"
|
||||
|
||||
@@ -94,14 +94,14 @@ def _seed_protocol_source(source):
|
||||
class TestProtocol:
|
||||
@pytest.mark.ci
|
||||
def test_protocol_current_version_accepted(self, shared_server):
|
||||
"""--protocol=2.24.0 (the current PROTOCOL_VERSION) is accepted and the
|
||||
"""--protocol=2.25.0 (the current PROTOCOL_VERSION) is accepted and the
|
||||
transfer completes normally."""
|
||||
source = os.path.join(TEST_DATA_DIR, "proto_ok_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, "proto_ok_dst")
|
||||
shutil.rmtree(dest, ignore_errors=True)
|
||||
os.makedirs(dest)
|
||||
_seed_protocol_source(source)
|
||||
result, _ = run_client(source, dest, flags=["--protocol=2.24.0"],
|
||||
result, _ = run_client(source, dest, flags=["--protocol=2.25.0"],
|
||||
port=shared_server.port)
|
||||
assert result.returncode == 0, \
|
||||
f"--protocol current run failed: {(result.stderr or result.stdout)[:400]}"
|
||||
|
||||
@@ -317,7 +317,7 @@ static void test_parse_args_protocol_accept_current() {
|
||||
Config* cfg = valid_client_config();
|
||||
EXPECT_NOT_NULL(cfg);
|
||||
char* argv_equals[] = {"fastsync", "--source-dir", "/src",
|
||||
"--dest-dir", "/dst", "--protocol=2.24.0"};
|
||||
"--dest-dir", "/dst", "--protocol=2.25.0"};
|
||||
int positional_args[2];
|
||||
int positional_count = 0;
|
||||
EXPECT_EQ_INT(parse_args(cfg, 6, argv_equals, positional_args, &positional_count), 0);
|
||||
@@ -327,7 +327,7 @@ static void test_parse_args_protocol_accept_current() {
|
||||
cfg = valid_client_config();
|
||||
EXPECT_NOT_NULL(cfg);
|
||||
char* argv_space[] = {"fastsync", "--source-dir", "/src", "--dest-dir",
|
||||
"/dst", "--protocol", "2.24.0"};
|
||||
"/dst", "--protocol", "2.25.0"};
|
||||
positional_count = 0;
|
||||
EXPECT_EQ_INT(parse_args(cfg, 7, argv_space, positional_args, &positional_count), 0);
|
||||
EXPECT_EQ_STR(cfg->version, PROTOCOL_VERSION);
|
||||
|
||||
+6
-6
@@ -2775,14 +2775,14 @@ static void golden_config_populate(Config* c) {
|
||||
c->copy_as_gid = 222;
|
||||
}
|
||||
|
||||
/* The pinned golden frame (protocol 2.24.0). The values below are the only
|
||||
/* The pinned golden frame (protocol 2.25.0). The values below are the only
|
||||
* thing that ties the generated table to the historical wire format; update
|
||||
* them ONLY with a PROTOCOL_VERSION bump and a documented reason. The 2.24.0
|
||||
* per-directory delete-plan wave changes only the version string in the config
|
||||
* frame (the frame layout itself is unchanged from 2.23.0); the byte-exact hash
|
||||
* is recomputed for the new version bytes. */
|
||||
#define GOLDEN_WIRE_LEN 697
|
||||
#define GOLDEN_WIRE_HASH 13736055061412501670ULL
|
||||
* per-directory delete-plan wave changed only the version string in the config
|
||||
* frame; the 2.25.0 wire-stats wave appends one report_stats bool. The
|
||||
* byte-exact values are recomputed for the merged layout. */
|
||||
#define GOLDEN_WIRE_LEN 701
|
||||
#define GOLDEN_WIRE_HASH 16170466870400670271ULL
|
||||
|
||||
static unsigned long long fnv1a_64(const unsigned char* buf, size_t len) {
|
||||
unsigned long long h = 1469598103934665603ULL;
|
||||
|
||||
@@ -18,9 +18,10 @@
|
||||
|
||||
/* P8 config-frame tail: super_mode (4) + copy-as presence (4) + uid (4) + gid (4). */
|
||||
#define P8_TAIL_BYTES 16
|
||||
/* Protocol 2.23.0 appends one trailing bool (report_dest_info) AFTER the P8
|
||||
* tail, so the P8 fields sit this many bytes before the end of the frame. */
|
||||
#define OUTPUT_TAIL_BYTES 4
|
||||
/* Protocol 2.25.0 appends two trailing bools (report_dest_info, report_stats)
|
||||
* AFTER the P8 tail, so the P8 fields sit this many bytes before the end of the
|
||||
* frame. */
|
||||
#define OUTPUT_TAIL_BYTES 8
|
||||
|
||||
/* Smoke test for chunk_deserialize fuzz target */
|
||||
static void test_fuzz_chunk_deserialize() {
|
||||
|
||||
Reference in New Issue
Block a user