TapTap/FastSync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

feat(basis): rsync quick-check default + FastSync-only --verify-basis

Browse Source 
- default basis match is rsync's metadata quick-check (size + mtime; size-only
  drops mtime; -I disables), no mandatory content digest
- new long-only --verify-basis (wire bool, protocol stays 2.28.0) restores the
  strict whole-file content equality
- --copy-dest re-applies source attributes; basis-hit 256 MiB cap removed by
  streaming the copy/hash; basis miss keeps the normal payload bound
- compare/copy/link-dest rows -> caveat; tally 116/13/28
This commit is contained in:
TapTap
2026-09-19 15:55:51 +02:00
parent 8ec8cb7203
commit 67076bf218
23 changed files with 940 additions and 255 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/integration/test_differential_parity.py
+111 -4
View File
@@ -28,6 +28,7 @@ from common import ( # noqa: E402
ServerManager,
TEST_DATA_DIR,
clean_dir,
get_dest_received_dir,
)
from parity_caveats import ASPECTS, caveat_for # noqa: E402
import parity_harness as H # noqa: E402
@@ -350,7 +351,9 @@ def _result_aspects(result):
_STANDALONE_REFS = {
"incremental_modified": "-i/--itemize-changes + incremental second run",
"compare_dest": "--compare-dest",
"copy_dest": "--copy-dest",
"link_dest": "--link-dest",
"verify_basis": "--verify-basis (FastSync-only)",
"added_and_deleted": "--delete across two runs",
"added_and_deleted_seed": "--delete across two runs",
"one_file_system": "-x/--one-file-system",
@@ -411,14 +414,17 @@ def test_compare_dest_skips_basis(parity_server_factory):
fdst = os.path.join(TEST_DATA_DIR, "parity_cmpd_fdst")
clean_dir(src)
_mk(os.path.join(src, "f.txt"), b"basis-content\n")
_pin(os.path.join(src, "f.txt"), _OLD_MTIME)
server = parity_server_factory(SUPER)
rel = os.path.abspath(src).lstrip(os.sep)
# rsync resolves --compare-dest relative to the destination dir; FastSync
# resolves it under the receive root and appends the mirrored source path.
# Both rely on rsync's size+mtime quick-check, so the basis mtime is pinned
# to the source's to keep the match deterministic across a second boundary.
def seed(_src, rroot, froot):
_mk(os.path.join(rroot, "basis", "f.txt"), b"basis-content\n")
_mk(os.path.join(fdst, "basis", rel, "f.txt"), b"basis-content\n")
_mk(os.path.join(rroot, "basis", "f.txt"), b"basis-content\n", _OLD_MTIME)
_mk(os.path.join(fdst, "basis", rel, "f.txt"), b"basis-content\n", _OLD_MTIME)
def extra(_src, rroot, froot, _rs, _fs):
out = []
@@ -446,12 +452,13 @@ def test_link_dest_hardlinks_basis(parity_server_factory):
fdst = os.path.join(TEST_DATA_DIR, "parity_linkd_fdst")
clean_dir(src)
_mk(os.path.join(src, "f.txt"), b"link-basis-content\n")
_pin(os.path.join(src, "f.txt"), _OLD_MTIME)
server = parity_server_factory(SUPER)
rel = os.path.abspath(src).lstrip(os.sep)
def seed(_src, rroot, froot):
_mk(os.path.join(rroot, "basis", "f.txt"), b"link-basis-content\n")
_mk(os.path.join(fdst, "basis", rel, "f.txt"), b"link-basis-content\n")
_mk(os.path.join(rroot, "basis", "f.txt"), b"link-basis-content\n", _OLD_MTIME)
_mk(os.path.join(fdst, "basis", rel, "f.txt"), b"link-basis-content\n", _OLD_MTIME)
def extra(_src, rroot, froot, _rs, _fs):
r_basis = os.stat(os.path.join(rroot, "basis", "f.txt")).st_ino
@@ -474,6 +481,106 @@ def test_link_dest_hardlinks_basis(parity_server_factory):
_run_and_check(case_id, result)
@requires_rsync
@parity
def test_copy_dest_copies_basis(parity_server_factory):
"""--copy-dest: a basis match is materialized as an independent copy with the
source's attributes, matching rsync (copy then fix attributes)."""
case_id = "copy_dest"
src = os.path.join(TEST_DATA_DIR, "parity_copyd_src")
rdst = os.path.join(TEST_DATA_DIR, "parity_copyd_rdst")
fdst = os.path.join(TEST_DATA_DIR, "parity_copyd_fdst")
clean_dir(src)
_mk(os.path.join(src, "f.txt"), b"copy-basis-content\n")
_pin(os.path.join(src, "f.txt"), 1_600_000_000)
os.chmod(os.path.join(src, "f.txt"), 0o755)
server = parity_server_factory(SUPER)
rel = os.path.abspath(src).lstrip(os.sep)
def seed(_src, rroot, froot):
# Basis content matches the source; give the basis a different mode so a
# wrong "keep basis attributes" implementation is visible.
_mk(os.path.join(rroot, "basis", "f.txt"), b"copy-basis-content\n",
1_600_000_000)
os.chmod(os.path.join(rroot, "basis", "f.txt"), 0o644)
_mk(os.path.join(fdst, "basis", rel, "f.txt"), b"copy-basis-content\n",
1_600_000_000)
os.chmod(os.path.join(fdst, "basis", rel, "f.txt"), 0o644)
def extra(_src, rroot, froot, _rs, _fs):
out = []
bases = {"rsync": os.path.join(rroot, "basis", "f.txt"),
"fastsync": os.path.join(fdst, "basis", rel, "f.txt")}
for label, root in (("rsync", rroot), ("fastsync", froot)):
target = os.path.join(root, "f.txt")
if not os.path.exists(target):
out.append(f"{label}: f.txt missing")
continue
if os.stat(target).st_ino == os.stat(bases[label]).st_ino:
out.append(f"{label}: f.txt is hard-linked, not copied")
if (os.stat(target).st_mode & 0o777) != 0o755:
out.append(f"{label}: f.txt mode "
f"{oct(os.stat(target).st_mode & 0o777)} != 0o755")
return out
result = H.run_differential(
src, rdst, fdst,
["-a", "--copy-dest=basis"],
["-a", f"--copy-dest={os.path.join(fdst, 'basis')}", "--incremental"],
server, seed=seed, ignore_paths=("basis",), extra_check=extra,
compare_modes=True)
_run_and_check(case_id, result)
@requires_rsync
@parity
def test_verify_basis_restores_strict_content(parity_server_factory):
"""Default matches rsync's metadata quick-check; FastSync-only
`--verify-basis` restores strict content equality and transfers the source
when a same-size/different-content basis would otherwise be trusted."""
case_id = "verify_basis"
src = os.path.join(TEST_DATA_DIR, "parity_vbasis_src")
rdst = os.path.join(TEST_DATA_DIR, "parity_vbasis_rdst")
fdst = os.path.join(TEST_DATA_DIR, "parity_vbasis_fdst")
clean_dir(src)
_mk(os.path.join(src, "f.txt"), b"AAAA\n")
_pin(os.path.join(src, "f.txt"), _OLD_MTIME)
server = parity_server_factory(SUPER)
rel = os.path.abspath(src).lstrip(os.sep)
def seed(_src, rroot, froot):
# Same size and mtime as the source, different bytes: a metadata
# quick-check trusts it; --verify-basis must not.
for root, basis_rel in ((rroot, os.path.join("basis", "f.txt")),
(fdst, os.path.join("basis", rel, "f.txt"))):
_mk(os.path.join(root, basis_rel), b"BBBB\n", _OLD_MTIME)
# Default: both tools trust the basis (rsync's quick check), so the
# destination carries the basis bytes and the trees match.
result = H.run_differential(
src, rdst, fdst,
["-a", "--link-dest=basis"],
["-a", f"--link-dest={os.path.join(fdst, 'basis')}", "--incremental"],
server, seed=seed, ignore_paths=("basis",))
_run_and_check(case_id + "_default", result)
# --verify-basis (FastSync only): the digest mismatch rejects the basis and
# the source is transferred, so the destination is the source bytes. rsync
# has no such flag; assert the FastSync outcome directly against the source.
fdst2 = os.path.join(TEST_DATA_DIR, "parity_vbasis_fdst2")
clean_dir(fdst2)
for root, basis_rel in ((fdst2, os.path.join("basis", rel, "f.txt")),):
_mk(os.path.join(root, basis_rel), b"BBBB\n", _OLD_MTIME)
result, _ = H.run_fastsync(src, fdst2,
["-a", f"--link-dest={os.path.join(fdst2, 'basis')}",
"--incremental", "--verify-basis"], server.port)
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
target = os.path.join(get_dest_received_dir(fdst2, src), "f.txt")
with open(target, "rb") as fh:
assert fh.read() == b"AAAA\n", \
"--verify-basis must reject the same-size/different-content basis"
@requires_rsync
@parity
def test_added_and_deleted_between_runs(parity_server_factory):
tests/integration/test_features.py
+96 -35
View File
@@ -4740,11 +4740,12 @@ class TestBasisDestDirs:
STAGING = ".fastsync-stage"
TS = 1577836800 # 2020-01-01 00:00:00 UTC, used to pin matching mtimes
# fixture files: source and basis share the mtime pin, so a basis "match"
# is decided purely by content (xxHash). unchanged.txt is byte-identical;
# changed.txt is byte-DIFFERENT but has the SAME SIZE as the source (and
# the same pinned mtime), which is what forces the content-hash gate;
# added.txt does not exist in the basis at all.
# fixture files: source and basis share the mtime pin, so the DEFAULT
# (rsync-parity) quick-check is a size+mtime match and trusts the basis even
# when the body differs. unchanged.txt is byte-identical; changed.txt is
# byte-DIFFERENT but has the SAME SIZE as the source (and the same pinned
# mtime), which is what the FastSync-only --verify-basis content gate
# rejects; added.txt does not exist in the basis at all.
UNCHANGED = "unchanged.txt"
CHANGED = "changed.txt"
ADDED = "added.txt"
@@ -4784,18 +4785,19 @@ class TestBasisDestDirs:
}
def _basis_tree(self, prefix):
# unchanged.txt is identical to the source; changed.txt has the SAME
# byte size and pinned mtime but a different body (equal size forces
# the xxHash gate); added.txt is missing from the basis.
# unchanged.txt is identical to the source; changed.txt has a DIFFERENT
# size (and body) so the size leg of the quick-check fails and it is
# transferred normally; added.txt is missing from the basis.
return {
self.UNCHANGED: b"stable content v1\n",
self.CHANGED: b"CHANGED CONTENT NOW\n",
self.CHANGED: b"CHANGED CONTENT NOW AND LONGER\n",
}
def test_same_size_different_content_is_not_a_basis_match(self, shared_server):
# Core safety property: equal size + pinned mtime but different content
# must NEVER be hard-linked or copied from the basis -- the xxHash gate
# rejects it and the sender's data is transferred instead.
def test_same_size_different_content_default_trusts_quick_check(self, shared_server):
# Default rsync-parity behavior: equal size + pinned mtime is a basis
# match, so the basis body is materialized/linked without reading it.
# This mirrors rsync 3.4.1's quick check (differential-tested in
# test_differential_parity.py::test_verify_basis_restores_strict_content).
for flag, basis_dir in (("--link-dest", "szlb"), ("--copy-dest", "szcp"),
("--compare-dest", "szcmp")):
source = self._make_source("basis_same_size_src",
@@ -4807,7 +4809,36 @@ class TestBasisDestDirs:
result, _ = run_client(source, dest, flags=[f"{flag}={basis_dir}"],
port=shared_server.port)
assert result.returncode == 0, \
f"{flag} same-size mismatch failed: {result.stderr[:300]}"
f"{flag} same-size quick-check failed: {result.stderr[:300]}"
received = get_dest_received_dir(dest, source)
dest_file = os.path.join(received, self.UNCHANGED)
if flag == "--compare-dest":
assert not os.path.exists(dest_file), \
f"{flag}: compare-dest must leave a matching file sparse"
else:
assert _read_file(dest_file) == b"SAME LENGTH BODY!", \
f"{flag}: default quick-check did not trust the basis body"
if flag == "--link-dest":
assert os.stat(dest_file).st_ino == os.stat(basis_file).st_ino, \
f"{flag}: basis was not hard-linked"
def test_verify_basis_rejects_same_size_different_content(self, shared_server):
# FastSync-only --verify-basis: the whole-file digest gate rejects the
# same-size/different-content basis, so the source data is transferred
# instead of the wrong basis bytes.
for flag, basis_dir in (("--link-dest", "vszlb"), ("--copy-dest", "vszcp"),
("--compare-dest", "vszcmp")):
source = self._make_source("basis_verify_src",
{self.UNCHANGED: b"same length body\n"})
dest = os.path.join(TEST_DATA_DIR, f"basis_verify_dst_{basis_dir}")
clean_dir(dest)
basis_file = self._seed_basis_file(dest, source, basis_dir, self.UNCHANGED,
b"SAME LENGTH BODY!")
result, _ = run_client(source, dest,
flags=[f"{flag}={basis_dir}", "--verify-basis"],
port=shared_server.port)
assert result.returncode == 0, \
f"{flag} --verify-basis failed: {result.stderr[:300]}"
received = get_dest_received_dir(dest, source)
dest_file = os.path.join(received, self.UNCHANGED)
assert _read_file(dest_file) == b"same length body\n", \
@@ -4839,11 +4870,13 @@ class TestBasisDestDirs:
self._source_tree("c")[self.ADDED], "added file not transferred"
@pytest.mark.ci
def test_dry_run_compare_dest_does_not_read_basis(self, shared_server):
# A dry-run --compare-dest must never read/hash the basis file: doing so
# is a 1-bit content oracle against the client-supplied digest. Even a
# byte-identical basis with a matching size+mtime is therefore reported
# as would-transfer, and nothing is created.
def test_dry_run_compare_dest_quick_check_does_not_read_basis(self, shared_server):
# A dry-run --compare-dest must never read/hash the basis file. Under
# the default metadata quick-check a matching basis is reported as a
# skip (matching rsync) without reading it; nothing is created. Under
# --verify-basis, which would require hashing, the dry-run cannot
# confirm the hit (that would be a 1-bit content oracle) and reports
# would-transfer instead.
source = self._make_source("basis_dry_src", {self.UNCHANGED: b"stable content v1\n"})
dest = os.path.join(TEST_DATA_DIR, "basis_dry_dst")
clean_dir(dest)
@@ -4854,11 +4887,30 @@ class TestBasisDestDirs:
port=shared_server.port)
assert result.returncode == 0, \
f"dry-run compare-dest failed: {result.stderr[:300]}"
assert self.UNCHANGED in result.stdout, (
"dry-run compare-dest silently skipped: receiver read the basis content"
assert self.UNCHANGED not in result.stdout, (
"dry-run compare-dest did not honor the metadata quick-check "
"(reported would-transfer for a matching basis)"
)
assert _snapshot_tree(dest) == before, "dry-run compare-dest mutated the destination"
# --verify-basis: the hit needs the basis content, which a dry-run must
# not read, so the file is reported as would-transfer.
dest2 = os.path.join(TEST_DATA_DIR, "basis_dry_verify_dst")
clean_dir(dest2)
self._seed_basis(dest2, source, "drybasis", {self.UNCHANGED: b"stable content v1\n"})
before2 = _snapshot_tree(dest2)
result, _ = run_client(source, dest2,
flags=["--compare-dest=drybasis", "--dry-run",
"--verify-basis"],
port=shared_server.port)
assert result.returncode == 0, \
f"dry-run --verify-basis compare-dest failed: {result.stderr[:300]}"
assert self.UNCHANGED in result.stdout, (
"dry-run --verify-basis must not read the basis to confirm a hit"
)
assert _snapshot_tree(dest2) == before2, \
"dry-run --verify-basis compare-dest mutated the destination"
def test_compare_dest_content_mismatch_forces_transfer(self, shared_server):
# The basis holds a file with a DIFFERENT body: even though it shares
# the mtime pin, the xxHash check fails and the data must be sent.
@@ -5097,27 +5149,36 @@ class TestBasisDestDirs:
assert os.stat(dest_file).st_ino != os.stat(basis_file).st_ino, \
"--ignore-times must not hard-link to a basis file"
def test_basis_refuses_file_above_whole_file_limit(self, shared_server):
# Every whole-file payload path in FastSync (basis dirs included) is
# bounded by MAX_RECEIVE_WHOLE_FILE_SIZE. rsync supports basis dirs for
# arbitrary sizes; FastSync refuses such a run up front with a clear
# diagnostic instead of letting the receiver abort the whole transfer
# mid-stream with no client-side explanation.
def test_basis_handles_file_above_whole_file_limit(self, shared_server):
# Track 5a: a basis hit streams the copy (and the --verify-basis digest
# streams the basis), so a source larger than the whole-file payload
# bound is supported for basis dirs exactly like rsync. A basis MISS
# still falls back to the normal transfer, which keeps its own bound.
source = self._make_source("basis_oversize_src", {"small.txt": b"ok\n"})
big = os.path.join(source, "huge.bin")
with open(big, "wb") as fh:
os.ftruncate(fh.fileno(), 256 * 1024 * 1024 + 4096)
dest = os.path.join(TEST_DATA_DIR, "basis_oversize_dst")
clean_dir(dest)
result, _ = run_client(source, dest, flags=["--link-dest=nope"],
port=shared_server.port)
assert result.returncode != 0, \
"basis run with an over-limit file unexpectedly succeeded"
assert "larger than" in result.stderr, \
f"no clear over-limit diagnostic: {result.stderr[:300]}"
received = get_dest_received_dir(dest, source)
assert not os.path.exists(received), \
"over-limit basis run transferred files before failing"
rel = os.path.relpath(received, dest)
basis_big = os.path.join(dest, "ob", rel, "huge.bin")
os.makedirs(os.path.dirname(basis_big), exist_ok=True)
shutil.copyfile(big, basis_big)
os.utime(basis_big, (self.TS, self.TS))
os.utime(big, (self.TS, self.TS))
result, _ = run_client(source, dest,
flags=["--link-dest=ob", "--incremental"],
port=shared_server.port)
assert result.returncode == 0, \
f"over-limit basis run failed: {result.stderr[:300]}"
dest_big = os.path.join(received, "huge.bin")
assert os.path.exists(dest_big), "over-limit basis hit was not materialized"
assert os.path.getsize(dest_big) == 256 * 1024 * 1024 + 4096
assert os.stat(dest_big).st_ino == os.stat(basis_big).st_ino, \
"over-limit --link-dest did not hard-link to the basis"
assert _read_file(os.path.join(received, "small.txt")) == b"ok\n"
def _random_payloads(size=2 * 1024 * 1024, changed=64 * 1024, seed=1234):
tests/integration/test_parity_quickwins.py
+43 -7
View File
@@ -719,13 +719,18 @@ class TestVerifyAndFlip:
source = self._src("cmpd")
dest = self._dst("cmpd")
rdst = self._dst("cmpd_r")
# Pin the mtime so rsync's size+mtime quick-check (and FastSync's
# default) matches deterministically across a second boundary.
OLD = 1_500_000_000
with open(os.path.join(source, "f.txt"), "wb") as fh:
fh.write(b"basis-content\n")
os.utime(os.path.join(source, "f.txt"), (OLD, OLD))
# rsync resolves --compare-dest relative to the destination dir; its
# basis file sits at the transfer-relative path.
os.makedirs(os.path.join(rdst, "basis"), exist_ok=True)
with open(os.path.join(rdst, "basis", "f.txt"), "wb") as fh:
fh.write(b"basis-content\n")
os.utime(os.path.join(rdst, "basis", "f.txt"), (OLD, OLD))
rs = _rsync(["-a", "--compare-dest=basis", source + "/", rdst + "/"])
assert rs.returncode == 0, rs.stderr
assert not os.path.exists(os.path.join(rdst, "f.txt")), \
@@ -738,6 +743,7 @@ class TestVerifyAndFlip:
os.makedirs(basis, exist_ok=True)
with open(os.path.join(basis, "f.txt"), "wb") as fh:
fh.write(b"basis-content\n")
os.utime(os.path.join(basis, "f.txt"), (OLD, OLD))
received = get_dest_received_dir(dest, source)
result, _ = run_client(source, dest,
flags=["--compare-dest=basis", "--incremental"],
@@ -751,14 +757,17 @@ class TestVerifyAndFlip:
def test_link_dest_hardlinks_matches_rsync(self, shared_server):
source = self._src("linkd")
dest = self._dst("linkd")
OLD = 1_500_000_000
with open(os.path.join(source, "f.txt"), "wb") as fh:
fh.write(b"link-basis-content\n")
os.utime(os.path.join(source, "f.txt"), (OLD, OLD))
rel = os.path.abspath(source).lstrip(os.sep)
basis = os.path.join(dest, "basis", rel)
os.makedirs(basis, exist_ok=True)
basis_file = os.path.join(basis, "f.txt")
with open(basis_file, "wb") as fh:
fh.write(b"link-basis-content\n")
os.utime(basis_file, (OLD, OLD))
received = get_dest_received_dir(dest, source)
result, _ = run_client(source, dest,
flags=["--link-dest=basis", "--incremental"],
@@ -771,12 +780,11 @@ class TestVerifyAndFlip:
@requires_rsync
def test_basis_dir_size_only_content_residual(self, shared_server):
"""Documented residual (RSYNC_COMPAT.md basis-dir rows): FastSync
xxHash-verifies a basis hit, while rsync's `--size-only` quick check
trusts the size alone. With a same-size, different-content basis,
rsync links/copies the wrong basis content while FastSync transfers the
source. This test pins both observed behaviors (FastSync is stricter,
so the rows are reclassified Divergent)."""
"""rsync parity (default): a basis hit is decided by the metadata
quick-check alone. With `--size-only`, a same-size, different-content
basis is trusted, so rsync links the basis content and FastSync must now
do the same instead of xxHash-verifying it. `--verify-basis` restores
the stricter content equality (covered by the differential test)."""
source = self._src("basissz")
rdest = self._dst("basissz_r")
fdest = self._dst("basissz_f")
@@ -806,9 +814,37 @@ class TestVerifyAndFlip:
flags=["-a", "--size-only", "--link-dest=basis", "--incremental"],
port=shared_server.port)
assert result.returncode == 0, result.stderr[:300]
with open(os.path.join(received, "f.txt"), "rb") as fh:
assert fh.read() == b"BBBB\n", \
"FastSync must trust the metadata quick-check exactly like rsync"
@requires_rsync
def test_verify_basis_restores_content_check(self, shared_server):
"""FastSync-only `--verify-basis`: a same-size, same-mtime basis with
different content is rejected by the whole-file digest, so the source is
transferred instead of installing the wrong basis bytes. The default
(no flag) installs the basis content, matching rsync."""
source = self._src("vbasis")
fdest = self._dst("vbasis_f")
with open(os.path.join(source, "f.txt"), "wb") as fh:
fh.write(b"AAAA\n")
OLD = 1_400_000_000
os.utime(os.path.join(source, "f.txt"), (OLD, OLD))
rel = os.path.abspath(source).lstrip(os.sep)
basis = os.path.join(fdest, "basis", rel)
os.makedirs(basis, exist_ok=True)
with open(os.path.join(basis, "f.txt"), "wb") as fh:
fh.write(b"BBBB\n")
os.utime(os.path.join(basis, "f.txt"), (OLD, OLD))
received = get_dest_received_dir(fdest, source)
result, _ = run_client(source, fdest,
flags=["-a", "--link-dest=basis", "--incremental",
"--verify-basis"],
port=shared_server.port)
assert result.returncode == 0, result.stderr[:300]
with open(os.path.join(received, "f.txt"), "rb") as fh:
assert fh.read() == b"AAAA\n", \
"FastSync must verify the basis content and transfer the source"
"--verify-basis must reject the same-size/different-content basis"
class TestIgnoreExistingShortCircuit:
tests/integration/test_preflight.py
+2 -2
View File
@@ -118,8 +118,8 @@ class TestProtocol:
shutil.rmtree(dest, ignore_errors=True)
os.makedirs(dest)
_seed_protocol_source(source)
for bad in ("2.22.0", "2.21.0", "2.20.0", "2.19.0", "2.18.0", "2.17.0", "2.15.0", "2.16.0",
"216", "31"):
for bad in ("2.27.0", "2.26.0", "2.25.0", "2.24.0", "2.23.0", "2.22.0", "2.21.0", "2.20.0",
"2.19.0", "2.18.0", "2.17.0", "2.15.0", "2.16.0", "216", "31"):
result, _ = run_client(source, dest, flags=[f"--protocol={bad}"],
port=shared_server.port)
assert result.returncode != 0, f"--protocol={bad} should be rejected"
tests/test_client_cli.c
+19
View File
@@ -340,6 +340,7 @@ static void test_parse_args_protocol_accept_current() {
static void test_parse_args_protocol_rejects_other_versions() {
static const char* const bad_versions[] = {"2.17", "2.16", "2.15.0", "2.16.0", "2.17.0",
"2.18.0", "2.19.0", "2.20.0", "2.21.0", "2.22.0",
"2.23.0", "2.24.0", "2.25.0", "2.26.0", "2.27.0",
"216", "31", "abc", ""};
for (size_t i = 0; i < sizeof(bad_versions) / sizeof(bad_versions[0]); i++) {
Config* cfg = valid_client_config();
@@ -1045,6 +1046,23 @@ static void test_parse_args_basis_dirs() {
config_delete(cfg);
}
/* --verify-basis (FastSync-only, long-only): default off; parses on as a plain
boolean and leaves the basis implications intact. */
static void test_parse_args_verify_basis() {
Config* cfg = config_create();
EXPECT_FALSE(cfg->verify_basis);
config_delete(cfg);
cfg = config_create();
int positional_args[2];
int positional_count = 0;
char* argv[] = {"fastsync", "--link-dest=prior", "--verify-basis", "/src", "/dst"};
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0);
EXPECT_TRUE(cfg->verify_basis);
EXPECT_TRUE(config_has_basis(cfg));
config_delete(cfg);
}
/* Escaping or degenerate basis-dir values must be rejected up front (they would
resolve outside the destination root on the receiver); an absolute path is
accepted (rsync parity) and canonicalized with its leading '/' preserved. */
@@ -4831,6 +4849,7 @@ void test_client_cli() {
test_parse_args_filter_rules();
test_parse_args_from0_cvs_filter_file_flags();
test_parse_args_basis_dirs();
test_parse_args_verify_basis();
test_parse_args_basis_invalid_paths();
test_validate_config_basis_rejects_chunk_serialization();
test_parse_args_delete_policy_flags();
tests/test_config.c
+8 -4
View File
@@ -2826,6 +2826,7 @@ static void golden_config_populate(Config* c) {
c->skip_compress_suffixes[1] = str_dup(".xz");
EXPECT_EQ_INT(config_basis_append(c, BASIS_DEST_COMPARE, "compare"), 0);
EXPECT_EQ_INT(config_basis_append(c, BASIS_DEST_LINK, "link"), 0);
c->verify_basis = true;
c->fuzzy = true;
c->checksum_algo = CHECKSUM_ALGO_MD5;
c->checksum_seed = 0x1122334455667788ULL;
@@ -2888,10 +2889,12 @@ static void golden_config_populate(Config* c) {
* report_stats bool, 2.26.0 appended the compression_algo int, 2.27.0 appended
* the report_deletes bool, and 2.28.0 changed only the version string and
* appended the receiver-side delete-protection rule block (the STATUS_STATS
* body also grew, but that is not part of this frame). The byte-exact values
* are recomputed for the merged layout. */
#define GOLDEN_WIRE_LEN 882
#define GOLDEN_WIRE_HASH 10588362715396735070ULL
* body also grew, but that is not part of this frame). Track 5a appends the
* FastSync-only verify_basis bool to the basis block WITHOUT a version bump
* (project decision), so the frame grew by one int to 886 bytes. The
* byte-exact values are recomputed for the merged layout. */
#define GOLDEN_WIRE_LEN 886
#define GOLDEN_WIRE_HASH 5809509022716816757ULL
static unsigned long long fnv1a_64(const unsigned char* buf, size_t len) {
unsigned long long h = 1469598103934665603ULL;
@@ -3040,6 +3043,7 @@ static void test_config_wire_golden_receive() {
recv->groupmap[0].to_name != NULL && strcmp(recv->groupmap[0].to_name, "root") == 0;
ok = ok && recv->basis_count == 2 && recv->basis_dirs[0].type == BASIS_DEST_COMPARE &&
recv->basis_dirs[1].type == BASIS_DEST_LINK;
ok = ok && recv->verify_basis;
ok = ok && recv->module != NULL && strcmp(recv->module, "goldenmod") == 0;
ok = ok && recv->copy_as_set && recv->copy_as_uid == 111 && recv->copy_as_gid == 222;
}
tests/test_file.c
+156
View File
@@ -6,6 +6,7 @@
#include "file_receive.h"
#include "data.h"
#include "config.h"
#include "charset.h"
#include "utils.h"
#include "protocol.h"
#include "test_utils.h"
@@ -1807,6 +1808,159 @@ static void test_receive_incremental_check_empty_path() {
config_delete(cfg);
}
/* Pure policy helpers behind the basis quick-check / --verify-basis decision. */
static void test_file_basis_quick_match_decision() {
Config* cfg = config_create();
EXPECT_NOT_NULL(cfg);
EXPECT_FALSE(file_basis_content_required(cfg));
cfg->verify_basis = true;
EXPECT_TRUE(file_basis_content_required(cfg));
cfg->verify_basis = false;
struct stat st;
memset(&st, 0, sizeof(st));
st.st_mtime = 1500000000;
#ifdef __linux__
st.st_mtim.tv_nsec = 500;
#endif
/* Equal size is required by the caller; this leg is the mtime / --size-only
rule. Equal mtime matches, a different mtime misses by default. */
EXPECT_TRUE(file_basis_quick_match(cfg, &st, 1500000000, 500));
EXPECT_FALSE(file_basis_quick_match(cfg, &st, 1500000001, 500));
cfg->size_only = true;
EXPECT_TRUE(file_basis_quick_match(cfg, &st, 1500000001, 500));
cfg->size_only = false;
cfg->modify_window = 2;
EXPECT_TRUE(file_basis_quick_match(cfg, &st, 1500000002, 500));
config_delete(cfg);
}
/* End-to-end handshake decision for a same-size, same-mtime, DIFFERENT-content
basis. Default (rsync parity): the metadata quick-check is trusted, the
receiver answers STATUS_OK and materializes the basis bytes. --verify-basis:
the whole-file digest is required, the basis is rejected and the receiver
asks for the source (STATUS_NEXT + full transfer). */
static void test_receive_incremental_check_basis_quick_check_and_verify() {
const char* root = "test_basis_quick_root";
const char* basis_dir = "test_basis_quick_root/basis";
const char* basis_file = "test_basis_quick_root/basis/f.txt";
unlink(basis_file);
unlink("test_basis_quick_root/f.txt");
rmdir(basis_dir);
rmdir(root);
EXPECT_EQ_INT(mkdir(root, 0755), 0);
EXPECT_EQ_INT(mkdir(basis_dir, 0755), 0);
const char* src_bytes = "AAAA";
const unsigned long long size = 4;
const time_t mtime = 1500000000;
{
FILE* fh = fopen(basis_file, "wb");
EXPECT_NOT_NULL(fh);
// cppcheck-suppress knownConditionTrueFalse
if (fh) {
EXPECT_EQ_INT((int)fwrite("BBBB", 1, (size_t)size, fh), (int)size);
fclose(fh);
}
}
struct timespec ts[2] = {{mtime, 0}, {mtime, 0}};
EXPECT_EQ_INT(utimensat(AT_FDCWD, basis_file, ts, 0), 0);
char root_abs[PATH_MAX];
EXPECT_NOT_NULL(realpath(root, root_abs));
int root_fd = open(root_abs, O_RDONLY | O_DIRECTORY | O_CLOEXEC);
EXPECT_TRUE(root_fd >= 0);
// cppcheck-suppress knownConditionTrueFalse
if (root_fd < 0) {
unlink(basis_file);
rmdir(basis_dir);
rmdir(root);
return;
}
EXPECT_TRUE(utils_set_authorized_root(root_fd, root_abs));
uint8_t digest[CHECKSUM_MAX_DIGEST_LEN];
size_t digest_len = 0;
EXPECT_TRUE(checksum_digest(CHECKSUM_ALGO_XXH64, 0, src_bytes, size, digest, sizeof(digest),
&digest_len));
/* Route protocol I/O through the explicit descriptors (a previous test group
may have left io_set_fds() bound to its own pipe). */
io_set_fds(-1, -1);
io_set_bwlimit(0);
for (int verify = 0; verify <= 1; verify++) {
Config* cfg = config_create();
EXPECT_NOT_NULL(cfg);
cfg->receive_root_directory = str_dup(root_abs);
cfg->checksum = false;
cfg->checksum_algo = CHECKSUM_ALGO_XXH64;
cfg->checksum_seed = 0;
cfg->use_incremental = true;
cfg->use_delta = false;
cfg->use_metadata = false;
cfg->verify_basis = (verify != 0);
EXPECT_EQ_INT(config_basis_append(cfg, BASIS_DEST_LINK, "basis"), 0);
int p[2];
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
EXPECT_TRUE(send_wire_str(p[1], "f.txt"));
unsigned long long check_size = size;
long long check_mtime = (long long)mtime;
long long check_mtime_nsec = 0;
EXPECT_TRUE(send_n_data(p[1], &check_size, sizeof(check_size)));
EXPECT_TRUE(send_n_data(p[1], &check_mtime, sizeof(check_mtime)));
EXPECT_TRUE(send_n_data(p[1], &check_mtime_nsec, sizeof(check_mtime_nsec)));
/* Only --verify-basis needs the digest (cfg->checksum is false) and the
pre-staged fallback full transfer the receiver will request. */
if (verify) {
uint8_t wire_len = (uint8_t)digest_len;
EXPECT_TRUE(send_n_data(p[1], &wire_len, sizeof(wire_len)));
EXPECT_TRUE(send_n_data(p[1], digest, digest_len));
char* payload_bytes = str_dup(src_bytes);
EXPECT_NOT_NULL(payload_bytes);
Data* payload = data_create(payload_bytes, size);
EXPECT_NOT_NULL(payload);
// cppcheck-suppress knownConditionTrueFalse
if (payload)
EXPECT_TRUE(send_data(p[1], payload));
data_destroy(payload);
}
bool skipped = false;
File* file = receive_incremental_check(p[0], cfg, &skipped);
EXPECT_NOT_NULL(file);
// cppcheck-suppress knownConditionTrueFalse
if (file) {
EXPECT_FALSE(skipped);
Status reply = STATUS_ERROR;
EXPECT_TRUE(receive_status(p[1], &reply));
if (verify) {
EXPECT_EQ_INT((int)reply, (int)STATUS_NEXT);
EXPECT_NOT_NULL(file->data->data);
EXPECT_TRUE(file->data->data != NULL && memcmp(file->data->data, src_bytes, size) == 0);
} else {
EXPECT_EQ_INT((int)reply, (int)STATUS_OK);
EXPECT_TRUE(file->skip);
/* The default quick-check hit materializes from the basis PATH at
install time (streaming), so no content is buffered on the File. */
EXPECT_NOT_NULL(file->basis_link);
EXPECT_NULL(file->data->data);
}
file_destroy(file);
}
close(p[0]);
close(p[1]);
config_delete(cfg);
}
utils_set_authorized_root(-1, NULL);
close(root_fd);
unlink(basis_file);
rmdir(basis_dir);
rmdir(root);
}
/* -K/--keep-dirlinks secure open: with an authorized root, a destination path
* component that is a symlink to an IN-ROOT directory is used as that directory
* (its referent is opened through a relative O_NOFOLLOW walk from the root fd,
@@ -2140,6 +2294,8 @@ void test_file() {
test_dir_time_list();
test_dir_time_list_cap();
test_receive_incremental_check_empty_path();
test_file_basis_quick_match_decision();
test_receive_incremental_check_basis_quick_check_and_verify();
test_keep_dirlinks_secure_open();
test_inplace_overwrite_clears_special_mode_bits();
test_inplace_overwrite_metadata_strips_special_bits();
tests/test_server.c
+26 -13
View File
@@ -1063,14 +1063,18 @@ static void test_incremental_check_fifo_destination_does_not_hang() {
}
/* A server-contacting --dry-run with an alternate basis dir must never read or
hash the basis file. An exact (size+mtime+content) basis match would
otherwise let a client probe the basis bytes against its own supplied digest
(a 1-bit content oracle). The dry-run decision is metadata-only, so even a
byte-identical basis is reported as would-transfer, not a compare-dest skip. */
static void test_incremental_check_dry_run_basis_does_not_read_content() {
hash the basis file. Under the default metadata quick-check a hit needs no
basis bytes, so a compare-dest match is reported as a skip (STATUS_OK) just
like a real run -- and still no content is read. Under --verify-basis a hit
would require hashing the basis against the client-supplied digest (a 1-bit
content oracle), which a dry-run must never do, so even a byte-identical
basis is reported as would-transfer. The destination is never materialized
in either arm. */
static void run_dry_run_basis_check(bool verify, Status expected) {
Config* cfg = config_create();
EXPECT_NOT_NULL(cfg);
cfg->dry_run = true;
cfg->verify_basis = verify;
char* root = make_check_root("dryb");
EXPECT_NOT_NULL(root);
cfg->receive_root_directory = str_dup(root);
@@ -1086,8 +1090,8 @@ static void test_incremental_check_dry_run_basis_does_not_read_content() {
EXPECT_EQ_INT(stat(basis_path, &bst), 0);
EXPECT_EQ_INT(config_basis_append(cfg, BASIS_DEST_COMPARE, "basis"), 0);
/* The (correct) source digest for the basis bytes: an unfixed dry-run would
read+hash the basis and treat this as an exact compare-dest hit. */
/* The (correct) source digest for the basis bytes: a buggy dry-run that read
and hashed the basis would treat this as an exact compare-dest hit. */
uint8_t digest[CHECKSUM_MAX_DIGEST_LEN];
size_t digest_len = 0;
EXPECT_TRUE(checksum_digest((ChecksumAlgo)cfg->checksum_algo, cfg->checksum_seed, content,
@@ -1106,7 +1110,8 @@ static void test_incremental_check_dry_run_basis_does_not_read_content() {
bool skipped = false;
bool would_transfer = false;
File* file = receive_incremental_check_ex(p[0], cfg, &skipped, &would_transfer);
bool ok = file == NULL && !skipped && would_transfer;
bool ok = file == NULL && skipped == (expected == STATUS_OK) &&
would_transfer == (expected != STATUS_OK);
file_destroy(file);
config_delete(cfg);
close(p[0]);
@@ -1124,13 +1129,16 @@ static void test_incremental_check_dry_run_basis_does_not_read_content() {
EXPECT_TRUE(send_n_data(p[1], &size, sizeof(size)));
EXPECT_TRUE(send_n_data(p[1], &mtime, sizeof(mtime)));
EXPECT_TRUE(send_n_data(p[1], &mtime_nsec, sizeof(mtime_nsec)));
uint8_t wire_len = (uint8_t)digest_len;
EXPECT_TRUE(send_n_data(p[1], &wire_len, sizeof(wire_len)));
EXPECT_TRUE(send_n_data(p[1], digest, digest_len));
/* The digest is only on the wire when --checksum or --verify-basis needs it
(cfg->checksum is false here); the default quick-check arm sends none. */
if (verify) {
uint8_t wire_len = (uint8_t)digest_len;
EXPECT_TRUE(send_n_data(p[1], &wire_len, sizeof(wire_len)));
EXPECT_TRUE(send_n_data(p[1], digest, digest_len));
}
Status s;
EXPECT_TRUE(receive_status(p[1], &s));
/* A skip here would mean the receiver read+hashed the basis file. */
EXPECT_EQ_INT(s, STATUS_DRY_RUN_TRANSFER);
EXPECT_EQ_INT(s, expected);
int status;
waitpid(pid, &status, 0);
@@ -1148,6 +1156,11 @@ static void test_incremental_check_dry_run_basis_does_not_read_content() {
}
}
static void test_incremental_check_dry_run_basis_does_not_read_content() {
run_dry_run_basis_check(false, STATUS_OK);
run_dry_run_basis_check(true, STATUS_DRY_RUN_TRANSFER);
}
/* B1: a FIFO planted in a --link-dest basis directory must not block
* basis_open_regular() either; the basis match is simply declined. */
static void test_incremental_check_basis_fifo_does_not_hang() {
tests/test_xattr.c
+2 -1
View File
@@ -210,7 +210,8 @@ static void test_xattr_receive_drops_acl_without_preserve_acls() {
/* MINOR-2: a --link-dest / -H copy fallback (linkat refused) must still apply
* the per-file xattrs and --fake-super stat. A DIRECTORY basis forces linkat
* to fail with EPERM, exercising the byte-copy fallback deterministically.
* to fail with EPERM, exercising the byte-copy fallback deterministically (the
* basis is not a regular file, so the fallback uses the caller's bytes).
* Guarded on filesystem xattr support. */
static void test_link_copy_fallback_preserves_xattrs() {
const char* dest = "test_link_xattr_dest.txt";