test: append/append-verify resume coverage
- CLI: --append/--append-verify acceptance (parse + imply --incremental, validate) and incompatibility rejection with -s and --whole-file; both removed from the unimplemented reject list. - Config: on-the-wire append/append_verify round-trip. - Unit: append_resume_eligible / append_tail_length pure resume math. - Integration (test_append.py): matching-prefix resume is byte-identical and tail-only (wire bytes << source size); --append with a wrong prefix keeps prefix+tail (rsync parity) while --append-verify detects the mismatch and falls back to a byte-exact full transfer; --append with --inplace and -m.
This commit is contained in:
@@ -0,0 +1,141 @@
|
||||
"""--append / --append-verify tail-resume integration tests.
|
||||
|
||||
A shorter existing destination file is resumed by transferring only the tail:
|
||||
--append sends it without verifying the retained prefix (rsync parity: a wrong
|
||||
prefix is kept, so the result can differ from the source), while --append-verify
|
||||
checksums the retained prefix against the source and, on a mismatch, falls back
|
||||
to a clean full transfer so the result is always a byte-identical source copy.
|
||||
"""
|
||||
import os
|
||||
import random
|
||||
import shutil
|
||||
|
||||
from common import (
|
||||
TEST_DATA_DIR,
|
||||
run_client, CountingProxy, clean_dir,
|
||||
get_dest_received_dir, CLIENT_CMD,
|
||||
)
|
||||
|
||||
REL = "sub/grow.dat"
|
||||
|
||||
|
||||
def _grow_payload(prefix_size, added_size, seed=99):
|
||||
r = random.Random(seed)
|
||||
return bytes(r.randbytes(prefix_size)), bytes(r.randbytes(added_size))
|
||||
|
||||
|
||||
class TestAppend:
|
||||
def _make(self, tag):
|
||||
source = os.path.join(TEST_DATA_DIR, f"append_{tag}_src")
|
||||
dest = os.path.join(TEST_DATA_DIR, f"append_{tag}_dst")
|
||||
clean_dir(source)
|
||||
shutil.rmtree(dest, ignore_errors=True)
|
||||
return source, dest
|
||||
|
||||
def _place(self, root, rel, data):
|
||||
p = os.path.join(root, rel)
|
||||
os.makedirs(os.path.dirname(p), exist_ok=True)
|
||||
with open(p, "wb") as fh:
|
||||
fh.write(data)
|
||||
return p
|
||||
|
||||
def _read(self, root, rel):
|
||||
with open(os.path.join(root, rel), "rb") as fh:
|
||||
return fh.read()
|
||||
|
||||
def _dest_file(self, source, dest, rel):
|
||||
return os.path.join(get_dest_received_dir(dest, source), rel)
|
||||
|
||||
def test_append_resumes_short_dest_atomically(self, shared_server):
|
||||
"""A shorter dest with a MATCHING prefix is resumed; the reconstructed
|
||||
file is byte-identical to the source."""
|
||||
source, dest = self._make("atomic")
|
||||
prefix, added = _grow_payload(1 * 1024 * 1024, 64 * 1024)
|
||||
self._place(source, REL, prefix + added)
|
||||
self._place(self._dest_file(source, dest, ""), REL, prefix)
|
||||
|
||||
result, _ = run_client(source, dest, flags=["--append"], port=shared_server.port)
|
||||
assert result.returncode == 0, \
|
||||
f"--append failed: {(result.stderr or result.stdout)[:400]}"
|
||||
assert self._read(self._dest_file(source, dest, ""), REL) == prefix + added
|
||||
|
||||
def test_append_verify_matching_prefix_succeeds(self, shared_server):
|
||||
source, dest = self._make("verify_ok")
|
||||
prefix, added = _grow_payload(512 * 1024, 32 * 1024)
|
||||
self._place(source, REL, prefix + added)
|
||||
self._place(self._dest_file(source, dest, ""), REL, prefix)
|
||||
|
||||
result, _ = run_client(source, dest, flags=["--append-verify"], port=shared_server.port)
|
||||
assert result.returncode == 0, \
|
||||
f"--append-verify failed: {(result.stderr or result.stdout)[:400]}"
|
||||
assert self._read(self._dest_file(source, dest, ""), REL) == prefix + added
|
||||
|
||||
def test_append_sends_only_tail(self, shared_server):
|
||||
"""Sorted transfer moves only the tail: wire bytes stay well below the
|
||||
full source size (incompressible payload, no -c)."""
|
||||
source, dest = self._make("tail")
|
||||
prefix, added = _grow_payload(4 * 1024 * 1024, 8 * 1024, seed=7)
|
||||
full = prefix + added
|
||||
self._place(source, REL, full)
|
||||
self._place(self._dest_file(source, dest, ""), REL, prefix)
|
||||
|
||||
proxy = CountingProxy(shared_server.port)
|
||||
cmd = (CLIENT_CMD + ["--source-dir", source, "--dest-dir", dest,
|
||||
"--save-to-disk", "--server-port", str(proxy.port), "--append"])
|
||||
result = proxy.run(cmd)
|
||||
assert result.returncode == 0, \
|
||||
f"--append failed: {(result.stderr or result.stdout)[:400]}"
|
||||
assert self._read(self._dest_file(source, dest, ""), REL) == full
|
||||
assert proxy.client_to_server < full.__len__() // 2, \
|
||||
f"expected a tail-only transfer, sent {proxy.client_to_server}B for {full.__len__()}B"
|
||||
|
||||
def test_plain_append_wrong_prefix_is_rsync_parity(self, shared_server):
|
||||
"""--append does NOT verify the retained prefix: a wrong prefix is kept,
|
||||
so the result is prefix+tail (differs from the source). This is the
|
||||
documented rsync-parity risk of plain --append."""
|
||||
source, dest = self._make("plain_wrong")
|
||||
correct_prefix, added = _grow_payload(256 * 1024, 32 * 1024, seed=1)
|
||||
wrong_prefix = bytes(b ^ 0xFF for b in correct_prefix)
|
||||
self._place(source, REL, correct_prefix + added)
|
||||
self._place(self._dest_file(source, dest, ""), REL, wrong_prefix)
|
||||
|
||||
result, _ = run_client(source, dest, flags=["--append"], port=shared_server.port)
|
||||
assert result.returncode == 0
|
||||
assert self._read(self._dest_file(source, dest, ""), REL) == wrong_prefix + added
|
||||
|
||||
def test_append_verify_wrong_prefix_never_corrupts(self, shared_server):
|
||||
"""--append-verify detects the retained prefix mismatch and falls back to
|
||||
a full transfer, so the result is a byte-identical source copy."""
|
||||
source, dest = self._make("verify_wrong")
|
||||
correct_prefix, added = _grow_payload(256 * 1024, 32 * 1024, seed=2)
|
||||
wrong_prefix = bytes(b ^ 0xFF for b in correct_prefix)
|
||||
self._place(source, REL, correct_prefix + added)
|
||||
self._place(self._dest_file(source, dest, ""), REL, wrong_prefix)
|
||||
|
||||
result, _ = run_client(source, dest, flags=["--append-verify"], port=shared_server.port)
|
||||
assert result.returncode == 0, \
|
||||
f"--append-verify mismatch fallback failed: {(result.stderr or result.stdout)[:400]}"
|
||||
assert self._read(self._dest_file(source, dest, ""), REL) == correct_prefix + added
|
||||
|
||||
def test_append_with_inplace(self, shared_server):
|
||||
source, dest = self._make("inplace")
|
||||
prefix, added = _grow_payload(128 * 1024, 16 * 1024, seed=3)
|
||||
self._place(source, REL, prefix + added)
|
||||
self._place(self._dest_file(source, dest, ""), REL, prefix)
|
||||
|
||||
result, _ = run_client(source, dest, flags=["--append", "--inplace"],
|
||||
port=shared_server.port)
|
||||
assert result.returncode == 0, \
|
||||
f"--append --inplace failed: {(result.stderr or result.stdout)[:400]}"
|
||||
assert self._read(self._dest_file(source, dest, ""), REL) == prefix + added
|
||||
|
||||
def test_append_multithreaded(self, shared_server):
|
||||
source, dest = self._make("mthread")
|
||||
prefix, added = _grow_payload(512 * 1024, 32 * 1024, seed=4)
|
||||
self._place(source, REL, prefix + added)
|
||||
self._place(self._dest_file(source, dest, ""), REL, prefix)
|
||||
|
||||
result, _ = run_client(source, dest, flags=["--append", "-m"], port=shared_server.port)
|
||||
assert result.returncode == 0, \
|
||||
f"--append -m failed: {(result.stderr or result.stdout)[:400]}"
|
||||
assert self._read(self._dest_file(source, dest, ""), REL) == prefix + added
|
||||
+96
-2
@@ -773,8 +773,6 @@ static void test_parse_args_rejects_unimplemented_options() {
|
||||
"--xattrs",
|
||||
"-D",
|
||||
"--devices",
|
||||
"--append",
|
||||
"--append-verify",
|
||||
"--delete-excluded",
|
||||
"--max-delete",
|
||||
"--prune-empty-dirs",
|
||||
@@ -1855,8 +1853,104 @@ static void test_parse_args_max_delete_inert_without_delete() {
|
||||
config_delete(cfg);
|
||||
}
|
||||
|
||||
/* --append is accepted and implies the per-file incremental check a tail resume
|
||||
* needs; it validates cleanly on its own. */
|
||||
static void test_parse_args_append() {
|
||||
Config* cfg = config_create();
|
||||
cfg->send_directory = str_dup("/src");
|
||||
cfg->receive_root_directory = str_dup("/dst");
|
||||
char* argv[] = {"fastsync", "--append", "/src", "/dst"};
|
||||
int positional_args[2];
|
||||
int positional_count = 0;
|
||||
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
|
||||
EXPECT_TRUE(cfg->append);
|
||||
EXPECT_FALSE(cfg->append_verify);
|
||||
EXPECT_TRUE(cfg->use_incremental);
|
||||
EXPECT_TRUE(validate_config(cfg));
|
||||
config_delete(cfg);
|
||||
}
|
||||
|
||||
static void test_parse_args_append_verify() {
|
||||
Config* cfg = config_create();
|
||||
cfg->send_directory = str_dup("/src");
|
||||
cfg->receive_root_directory = str_dup("/dst");
|
||||
char* argv[] = {"fastsync", "--append-verify", "/src", "/dst"};
|
||||
int positional_args[2];
|
||||
int positional_count = 0;
|
||||
EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0);
|
||||
EXPECT_TRUE(cfg->append_verify);
|
||||
EXPECT_FALSE(cfg->append);
|
||||
EXPECT_TRUE(cfg->use_incremental);
|
||||
EXPECT_TRUE(validate_config(cfg));
|
||||
config_delete(cfg);
|
||||
}
|
||||
|
||||
/* Both spellings are accepted; the safer --append-verify semantics win on the
|
||||
* wire (the sender checks append_verify first), so neither flag is silently
|
||||
* dropped but the run is still valid. */
|
||||
static void test_parse_args_append_both() {
|
||||
Config* cfg = config_create();
|
||||
cfg->send_directory = str_dup("/src");
|
||||
cfg->receive_root_directory = str_dup("/dst");
|
||||
char* argv[] = {"fastsync", "--append", "--append-verify", "/src", "/dst"};
|
||||
int positional_args[2];
|
||||
int positional_count = 0;
|
||||
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0);
|
||||
EXPECT_TRUE(cfg->append);
|
||||
EXPECT_TRUE(cfg->append_verify);
|
||||
EXPECT_TRUE(validate_config(cfg));
|
||||
config_delete(cfg);
|
||||
}
|
||||
|
||||
static void test_validate_config_append_rejects_chunk_serialization() {
|
||||
Config* cfg = config_create();
|
||||
char* argv[] = {"fastsync", "--append", "-s", "/src", "/dst"};
|
||||
int positional_args[2];
|
||||
int positional_count = 0;
|
||||
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0);
|
||||
EXPECT_FALSE(validate_config(cfg));
|
||||
config_delete(cfg);
|
||||
}
|
||||
|
||||
static void test_validate_config_append_verify_rejects_chunk_serialization() {
|
||||
Config* cfg = config_create();
|
||||
char* argv[] = {"fastsync", "--append-verify", "-s", "/src", "/dst"};
|
||||
int positional_args[2];
|
||||
int positional_count = 0;
|
||||
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0);
|
||||
EXPECT_FALSE(validate_config(cfg));
|
||||
config_delete(cfg);
|
||||
}
|
||||
|
||||
static void test_validate_config_append_rejects_whole_file() {
|
||||
Config* cfg = config_create();
|
||||
char* argv[] = {"fastsync", "--append", "-W", "/src", "/dst"};
|
||||
int positional_args[2];
|
||||
int positional_count = 0;
|
||||
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0);
|
||||
EXPECT_FALSE(validate_config(cfg));
|
||||
config_delete(cfg);
|
||||
}
|
||||
|
||||
static void test_validate_config_append_verify_rejects_whole_file() {
|
||||
Config* cfg = config_create();
|
||||
char* argv[] = {"fastsync", "--append-verify", "-W", "/src", "/dst"};
|
||||
int positional_args[2];
|
||||
int positional_count = 0;
|
||||
EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0);
|
||||
EXPECT_FALSE(validate_config(cfg));
|
||||
config_delete(cfg);
|
||||
}
|
||||
|
||||
void test_client_cli() {
|
||||
test_validate_config_required_paths();
|
||||
test_parse_args_append();
|
||||
test_parse_args_append_verify();
|
||||
test_parse_args_append_both();
|
||||
test_validate_config_append_rejects_chunk_serialization();
|
||||
test_validate_config_append_verify_rejects_chunk_serialization();
|
||||
test_validate_config_append_rejects_whole_file();
|
||||
test_validate_config_append_verify_rejects_whole_file();
|
||||
test_validate_config_incompatible_options();
|
||||
test_validate_config_tls_requirements();
|
||||
test_validate_config_delta_sendfile_constraints();
|
||||
|
||||
@@ -754,6 +754,49 @@ static void test_config_is_remote_dest() {
|
||||
EXPECT_TRUE(config_is_remote_dest("user@host:"));
|
||||
}
|
||||
|
||||
/* The append-mode fields cross the wire unchanged: --append and --append-verify
|
||||
are negotiated to the receiver so it knows to reply STATUS_APPEND on a
|
||||
shorter destination. */
|
||||
static void test_config_append_wire_roundtrip() {
|
||||
if (is_running_under_valgrind())
|
||||
return;
|
||||
struct {
|
||||
bool append, append_verify;
|
||||
} cases[] = {{true, false}, {false, true}, {true, true}, {false, false}};
|
||||
for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
|
||||
int p[2];
|
||||
EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0);
|
||||
pid_t pid = fork();
|
||||
if (pid == 0) {
|
||||
close(p[1]);
|
||||
io_set_fds(p[0], p[0]);
|
||||
Config* recv = config_receive(p[0]);
|
||||
bool ok = recv != NULL;
|
||||
if (ok)
|
||||
ok = recv->append == cases[i].append && recv->append_verify == cases[i].append_verify;
|
||||
config_delete(recv);
|
||||
close(p[0]);
|
||||
_exit(ok ? 0 : 1);
|
||||
} else {
|
||||
close(p[0]);
|
||||
io_set_fds(p[1], p[1]);
|
||||
Config* send_cfg = config_create();
|
||||
EXPECT_NOT_NULL(send_cfg);
|
||||
send_cfg->send_directory = str_dup("/src");
|
||||
send_cfg->receive_root_directory = str_dup("/dst");
|
||||
send_cfg->append = cases[i].append;
|
||||
send_cfg->append_verify = cases[i].append_verify;
|
||||
bool sent = config_send(p[1], send_cfg);
|
||||
int status;
|
||||
waitpid(pid, &status, 0);
|
||||
close(p[1]);
|
||||
config_delete(send_cfg);
|
||||
EXPECT_TRUE(sent);
|
||||
EXPECT_TRUE(WIFEXITED(status) && WEXITSTATUS(status) == 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void test_config() {
|
||||
test_config_lifecycle();
|
||||
test_config_ssh_dest();
|
||||
@@ -771,6 +814,7 @@ void test_config() {
|
||||
test_config_delete_timing_wire_roundtrip();
|
||||
test_config_delete_timing_conflict_rejected();
|
||||
test_config_delete_policy_wire_roundtrip();
|
||||
test_config_append_wire_roundtrip();
|
||||
test_config_basis_roundtrip();
|
||||
test_config_basis_wire_rejects_escaping();
|
||||
test_config_basis_normalization();
|
||||
|
||||
@@ -276,6 +276,22 @@ void test_shared_utils() {
|
||||
test_walker_unlimited_deletes_all();
|
||||
test_walker_hard_bound_all_or_nothing();
|
||||
|
||||
/* --append / --append-verify tail-resume math: a resume is eligible only for
|
||||
a shorter existing destination, and the tail length is then the difference. */
|
||||
EXPECT_TRUE(append_resume_eligible(0, 10));
|
||||
EXPECT_TRUE(append_resume_eligible(7, 10));
|
||||
EXPECT_FALSE(append_resume_eligible(10, 10));
|
||||
EXPECT_FALSE(append_resume_eligible(11, 10));
|
||||
|
||||
unsigned long long tail;
|
||||
EXPECT_TRUE(append_tail_length(0, 10, &tail));
|
||||
EXPECT_EQ_INT((int)tail, 10);
|
||||
EXPECT_TRUE(append_tail_length(7, 10, &tail));
|
||||
EXPECT_EQ_INT((int)tail, 3);
|
||||
EXPECT_FALSE(append_tail_length(10, 10, &tail));
|
||||
EXPECT_FALSE(append_tail_length(11, 10, &tail));
|
||||
EXPECT_FALSE(append_tail_length(7, 10, NULL));
|
||||
|
||||
char formatted[32];
|
||||
EXPECT_TRUE(format_human_bytes(0, formatted, sizeof(formatted)));
|
||||
EXPECT_EQ_STR(formatted, "0 B");
|
||||
|
||||
Reference in New Issue
Block a user