rsync 3.4.1 drop-in parity (#285-#297) + parity completion (protocol 2.26.0) #298
@@ -0,0 +1,15 @@
|
|||||||
|
B=/workspace/build-ci
|
||||||
|
D=/workspace/.probe
|
||||||
|
rm -rf $D && mkdir -p $D/src $D/dst
|
||||||
|
head -c 200000 /dev/urandom > $D/src/f.bin
|
||||||
|
$B/server -p 45995 --allow-unauthenticated >$D/srv.log 2>&1 &
|
||||||
|
SRV=$!; sleep 0.7
|
||||||
|
for flags in "" "--incremental" "--incremental --delta"; do
|
||||||
|
rm -rf $D/dst; mkdir -p $D/dst
|
||||||
|
echo "=== fastsync flags='$flags' fresh: %b %c %l %n ==="
|
||||||
|
$B/client --source-dir $D/src --dest-dir $D/dst --server-port 45995 --save-to-disk -a $flags --out-format="%b %c %l %n" 2>&1 | grep -v ERROR
|
||||||
|
done
|
||||||
|
echo "=== rsync whole-file: ==="
|
||||||
|
rm -rf $D/rsrc $D/rdst; mkdir -p $D/rsrc $D/rdst; head -c 200000 /dev/urandom > $D/rsrc/f.bin
|
||||||
|
rsync -a --out-format="%b %c %l %n" $D/rsrc/ $D/rdst/
|
||||||
|
kill $SRV 2>/dev/null || true
|
||||||
@@ -590,7 +590,15 @@ void change_emit_file_sent_bytes(const Config* config, const File* file,
|
|||||||
event.bytes_sent = 0;
|
event.bytes_sent = 0;
|
||||||
} else {
|
} else {
|
||||||
event.bytes_sent = bytes_sent;
|
event.bytes_sent = bytes_sent;
|
||||||
event.bytes_read = bytes_read;
|
/* rsync's %c is the block-checksum bytes received for the file. Even a
|
||||||
|
* whole-file transfer (no basis; --append/--inplace included) receives
|
||||||
|
* rsync's 16-byte sum header, so rsync reports 16; a dry run transfers
|
||||||
|
* nothing and reports 0. FastSync's whole-file path has no sum header, so
|
||||||
|
* report rsync's value for parity. With delta enabled the real received
|
||||||
|
* bytes are kept, but FastSync's signature framing differs from rsync's so
|
||||||
|
* those stay numerically divergent. */
|
||||||
|
bool delta_active = config->use_delta && !config->whole_file;
|
||||||
|
event.bytes_read = (!config->dry_run && !delta_active) ? 16 : bytes_read;
|
||||||
}
|
}
|
||||||
char* name = NULL;
|
char* name = NULL;
|
||||||
char* path = NULL;
|
char* path = NULL;
|
||||||
|
|||||||
@@ -69,7 +69,8 @@ char* change_render_itemize_code(const Config* config, const ChangeEvent* event)
|
|||||||
/* Expand an --out-format/--log-file-format template. Supported tokens:
|
/* Expand an --out-format/--log-file-format template. Supported tokens:
|
||||||
* %i itemize code %n transfer-relative name (dir: trailing /)
|
* %i itemize code %n transfer-relative name (dir: trailing /)
|
||||||
* %f long display path %l file length in bytes
|
* %f long display path %l file length in bytes
|
||||||
* %b wire bytes transferred %c wire bytes read back for the file
|
* %b wire bytes transferred %c block-checksum bytes received (rsync: 16
|
||||||
|
* for a whole-file transfer, 0 for a dry run)
|
||||||
* %C whole-file checksum hex (xxh128 by default; spaces for non-regular)
|
* %C whole-file checksum hex (xxh128 by default; spaces for non-regular)
|
||||||
* %M mtime (YYYY/MM/DD-HH:MM:SS)
|
* %M mtime (YYYY/MM/DD-HH:MM:SS)
|
||||||
* %t current time %o operation ("send"/"del.")
|
* %t current time %o operation ("send"/"del.")
|
||||||
@@ -91,8 +92,9 @@ char* change_render_list_line(const Config* config, const ChangeEvent* event);
|
|||||||
void change_emit(const Config* config, const ChangeEvent* event);
|
void change_emit(const Config* config, const ChangeEvent* event);
|
||||||
|
|
||||||
/* Build and emit a CHANGE_SENT event for a file the client just sent. `bytes_sent`
|
/* Build and emit a CHANGE_SENT event for a file the client just sent. `bytes_sent`
|
||||||
* / `bytes_read` are the process-wide wire-byte deltas for this file (rsync's
|
* is the process-wide wire-byte delta for this file (rsync's %b) and `bytes_read`
|
||||||
* %b / %c); pass 0 when unknown. */
|
* the received bytes used for the delta handshake; pass 0 when unknown. For a
|
||||||
|
* whole-file transfer %c is pinned to rsync's 16-byte sum header regardless. */
|
||||||
void change_emit_file_sent_bytes(const Config* config, const File* file,
|
void change_emit_file_sent_bytes(const Config* config, const File* file,
|
||||||
unsigned long long bytes_sent, unsigned long long bytes_read);
|
unsigned long long bytes_sent, unsigned long long bytes_read);
|
||||||
|
|
||||||
|
|||||||
@@ -324,21 +324,62 @@ class TestWireStatsParity:
|
|||||||
@requires_rsync
|
@requires_rsync
|
||||||
@pytest.mark.ci
|
@pytest.mark.ci
|
||||||
def test_out_format_b_is_wire_bytes(self, shared_server):
|
def test_out_format_b_is_wire_bytes(self, shared_server):
|
||||||
"""%b is true transferred (wire) bytes, not the source length: it must
|
"""%b is the bytes actually transferred (wire), not the source length.
|
||||||
differ from %l (the source length) and exceed it for a framed transfer."""
|
|
||||||
|
A differential run against rsync confirms both implementations report a
|
||||||
|
framed value greater than %l. The exact numbers are not compared: each
|
||||||
|
counts its own protocol framing and checksum trailer, so the two are
|
||||||
|
protocol-specific and cannot be numerically equal (documented
|
||||||
|
divergence)."""
|
||||||
source = os.path.join(TEST_DATA_DIR, "wire_b_src")
|
source = os.path.join(TEST_DATA_DIR, "wire_b_src")
|
||||||
dest = os.path.join(TEST_DATA_DIR, "wire_b_dst")
|
dest = os.path.join(TEST_DATA_DIR, "wire_b_dst")
|
||||||
|
rdst = os.path.join(TEST_DATA_DIR, "wire_b_rdst")
|
||||||
_make_one_file(source, "f.bin", 5000)
|
_make_one_file(source, "f.bin", 5000)
|
||||||
clean_dir(dest)
|
clean_dir(dest)
|
||||||
result, _ = run_client(source, dest, flags=["-a", "--out-format=%b %l %c"],
|
clean_dir(rdst)
|
||||||
|
fmt = "%b %l"
|
||||||
|
rsync_result = _rsync(["-a", "--out-format=" + fmt, source + "/", rdst + "/"])
|
||||||
|
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||||
|
result, _ = run_client(source, dest, flags=["-a", "--out-format=" + fmt],
|
||||||
port=shared_server.port)
|
port=shared_server.port)
|
||||||
assert result.returncode == 0, result.stderr[:300]
|
assert result.returncode == 0, result.stderr[:300]
|
||||||
line = result.stdout.strip()
|
rb, rl = (int(x) for x in rsync_result.stdout.split()[:2])
|
||||||
parts = line.split()
|
fb, fl = (int(x) for x in result.stdout.split()[:2])
|
||||||
assert len(parts) == 3 and all(p.isdigit() for p in parts), line
|
assert rl == fl == 5000, (rsync_result.stdout, result.stdout)
|
||||||
wire_b, src_l, wire_c = (int(p) for p in parts)
|
assert rb > rl, f"rsync %b must include framing: {rsync_result.stdout!r}"
|
||||||
assert src_l == 5000, line
|
assert fb > fl, f"fastsync %b must include framing: {result.stdout!r}"
|
||||||
assert wire_b > src_l, f"%b must include wire framing: {line}"
|
|
||||||
|
@requires_rsync
|
||||||
|
@pytest.mark.ci
|
||||||
|
def test_out_format_c_whole_file_matches_rsync(self, shared_server):
|
||||||
|
"""%c is the block-checksum bytes received. rsync reports its 16-byte
|
||||||
|
sum header even for a whole-file transfer (no basis), so `%c` must match
|
||||||
|
rsync exactly for the whole-file case."""
|
||||||
|
source = os.path.join(TEST_DATA_DIR, "wire_c_src")
|
||||||
|
dest = os.path.join(TEST_DATA_DIR, "wire_c_dst")
|
||||||
|
rdst = os.path.join(TEST_DATA_DIR, "wire_c_rdst")
|
||||||
|
_make_one_file(source, "f.bin", 5000)
|
||||||
|
clean_dir(dest)
|
||||||
|
clean_dir(rdst)
|
||||||
|
fmt = "%c %l %n"
|
||||||
|
rsync_result = _rsync(["-a", "--out-format=" + fmt, source + "/", rdst + "/"])
|
||||||
|
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||||
|
result, _ = run_client(source, dest, flags=["-a", "--out-format=" + fmt],
|
||||||
|
port=shared_server.port)
|
||||||
|
assert result.returncode == 0, result.stderr[:300]
|
||||||
|
|
||||||
|
def file_lines(text):
|
||||||
|
return [
|
||||||
|
line for line in text.splitlines()
|
||||||
|
if line and not line.rsplit(" ", 1)[-1].endswith("/")
|
||||||
|
]
|
||||||
|
|
||||||
|
assert file_lines(result.stdout) == file_lines(rsync_result.stdout), (
|
||||||
|
f"rsync={rsync_result.stdout!r} fastsync={result.stdout!r}"
|
||||||
|
)
|
||||||
|
assert result.stdout.split()[0] == rsync_result.stdout.split()[0] == "16", (
|
||||||
|
f"%c must be rsync's 16-byte sum header: {result.stdout!r}"
|
||||||
|
)
|
||||||
|
|
||||||
@requires_rsync
|
@requires_rsync
|
||||||
@pytest.mark.ci
|
@pytest.mark.ci
|
||||||
|
|||||||
Reference in New Issue
Block a user