diff --git a/tests/integration/test_append.py b/tests/integration/test_append.py new file mode 100644 index 0000000..be84a5a --- /dev/null +++ b/tests/integration/test_append.py @@ -0,0 +1,141 @@ +"""--append / --append-verify tail-resume integration tests. + +A shorter existing destination file is resumed by transferring only the tail: +--append sends it without verifying the retained prefix (rsync parity: a wrong +prefix is kept, so the result can differ from the source), while --append-verify +checksums the retained prefix against the source and, on a mismatch, falls back +to a clean full transfer so the result is always a byte-identical source copy. +""" +import os +import random +import shutil + +from common import ( + TEST_DATA_DIR, + run_client, CountingProxy, clean_dir, + get_dest_received_dir, CLIENT_CMD, +) + +REL = "sub/grow.dat" + + +def _grow_payload(prefix_size, added_size, seed=99): + r = random.Random(seed) + return bytes(r.randbytes(prefix_size)), bytes(r.randbytes(added_size)) + + +class TestAppend: + def _make(self, tag): + source = os.path.join(TEST_DATA_DIR, f"append_{tag}_src") + dest = os.path.join(TEST_DATA_DIR, f"append_{tag}_dst") + clean_dir(source) + shutil.rmtree(dest, ignore_errors=True) + return source, dest + + def _place(self, root, rel, data): + p = os.path.join(root, rel) + os.makedirs(os.path.dirname(p), exist_ok=True) + with open(p, "wb") as fh: + fh.write(data) + return p + + def _read(self, root, rel): + with open(os.path.join(root, rel), "rb") as fh: + return fh.read() + + def _dest_file(self, source, dest, rel): + return os.path.join(get_dest_received_dir(dest, source), rel) + + def test_append_resumes_short_dest_atomically(self, shared_server): + """A shorter dest with a MATCHING prefix is resumed; the reconstructed + file is byte-identical to the source.""" + source, dest = self._make("atomic") + prefix, added = _grow_payload(1 * 1024 * 1024, 64 * 1024) + self._place(source, REL, prefix + added) + self._place(self._dest_file(source, dest, ""), REL, prefix) + + result, _ = run_client(source, dest, flags=["--append"], port=shared_server.port) + assert result.returncode == 0, \ + f"--append failed: {(result.stderr or result.stdout)[:400]}" + assert self._read(self._dest_file(source, dest, ""), REL) == prefix + added + + def test_append_verify_matching_prefix_succeeds(self, shared_server): + source, dest = self._make("verify_ok") + prefix, added = _grow_payload(512 * 1024, 32 * 1024) + self._place(source, REL, prefix + added) + self._place(self._dest_file(source, dest, ""), REL, prefix) + + result, _ = run_client(source, dest, flags=["--append-verify"], port=shared_server.port) + assert result.returncode == 0, \ + f"--append-verify failed: {(result.stderr or result.stdout)[:400]}" + assert self._read(self._dest_file(source, dest, ""), REL) == prefix + added + + def test_append_sends_only_tail(self, shared_server): + """Sorted transfer moves only the tail: wire bytes stay well below the + full source size (incompressible payload, no -c).""" + source, dest = self._make("tail") + prefix, added = _grow_payload(4 * 1024 * 1024, 8 * 1024, seed=7) + full = prefix + added + self._place(source, REL, full) + self._place(self._dest_file(source, dest, ""), REL, prefix) + + proxy = CountingProxy(shared_server.port) + cmd = (CLIENT_CMD + ["--source-dir", source, "--dest-dir", dest, + "--save-to-disk", "--server-port", str(proxy.port), "--append"]) + result = proxy.run(cmd) + assert result.returncode == 0, \ + f"--append failed: {(result.stderr or result.stdout)[:400]}" + assert self._read(self._dest_file(source, dest, ""), REL) == full + assert proxy.client_to_server < full.__len__() // 2, \ + f"expected a tail-only transfer, sent {proxy.client_to_server}B for {full.__len__()}B" + + def test_plain_append_wrong_prefix_is_rsync_parity(self, shared_server): + """--append does NOT verify the retained prefix: a wrong prefix is kept, + so the result is prefix+tail (differs from the source). This is the + documented rsync-parity risk of plain --append.""" + source, dest = self._make("plain_wrong") + correct_prefix, added = _grow_payload(256 * 1024, 32 * 1024, seed=1) + wrong_prefix = bytes(b ^ 0xFF for b in correct_prefix) + self._place(source, REL, correct_prefix + added) + self._place(self._dest_file(source, dest, ""), REL, wrong_prefix) + + result, _ = run_client(source, dest, flags=["--append"], port=shared_server.port) + assert result.returncode == 0 + assert self._read(self._dest_file(source, dest, ""), REL) == wrong_prefix + added + + def test_append_verify_wrong_prefix_never_corrupts(self, shared_server): + """--append-verify detects the retained prefix mismatch and falls back to + a full transfer, so the result is a byte-identical source copy.""" + source, dest = self._make("verify_wrong") + correct_prefix, added = _grow_payload(256 * 1024, 32 * 1024, seed=2) + wrong_prefix = bytes(b ^ 0xFF for b in correct_prefix) + self._place(source, REL, correct_prefix + added) + self._place(self._dest_file(source, dest, ""), REL, wrong_prefix) + + result, _ = run_client(source, dest, flags=["--append-verify"], port=shared_server.port) + assert result.returncode == 0, \ + f"--append-verify mismatch fallback failed: {(result.stderr or result.stdout)[:400]}" + assert self._read(self._dest_file(source, dest, ""), REL) == correct_prefix + added + + def test_append_with_inplace(self, shared_server): + source, dest = self._make("inplace") + prefix, added = _grow_payload(128 * 1024, 16 * 1024, seed=3) + self._place(source, REL, prefix + added) + self._place(self._dest_file(source, dest, ""), REL, prefix) + + result, _ = run_client(source, dest, flags=["--append", "--inplace"], + port=shared_server.port) + assert result.returncode == 0, \ + f"--append --inplace failed: {(result.stderr or result.stdout)[:400]}" + assert self._read(self._dest_file(source, dest, ""), REL) == prefix + added + + def test_append_multithreaded(self, shared_server): + source, dest = self._make("mthread") + prefix, added = _grow_payload(512 * 1024, 32 * 1024, seed=4) + self._place(source, REL, prefix + added) + self._place(self._dest_file(source, dest, ""), REL, prefix) + + result, _ = run_client(source, dest, flags=["--append", "-m"], port=shared_server.port) + assert result.returncode == 0, \ + f"--append -m failed: {(result.stderr or result.stdout)[:400]}" + assert self._read(self._dest_file(source, dest, ""), REL) == prefix + added \ No newline at end of file diff --git a/tests/test_client_cli.c b/tests/test_client_cli.c index 5882bae..cc38b5a 100644 --- a/tests/test_client_cli.c +++ b/tests/test_client_cli.c @@ -773,8 +773,6 @@ static void test_parse_args_rejects_unimplemented_options() { "--xattrs", "-D", "--devices", - "--append", - "--append-verify", "--delete-excluded", "--max-delete", "--prune-empty-dirs", @@ -1855,8 +1853,104 @@ static void test_parse_args_max_delete_inert_without_delete() { config_delete(cfg); } +/* --append is accepted and implies the per-file incremental check a tail resume + * needs; it validates cleanly on its own. */ +static void test_parse_args_append() { + Config* cfg = config_create(); + cfg->send_directory = str_dup("/src"); + cfg->receive_root_directory = str_dup("/dst"); + char* argv[] = {"fastsync", "--append", "/src", "/dst"}; + int positional_args[2]; + int positional_count = 0; + EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0); + EXPECT_TRUE(cfg->append); + EXPECT_FALSE(cfg->append_verify); + EXPECT_TRUE(cfg->use_incremental); + EXPECT_TRUE(validate_config(cfg)); + config_delete(cfg); +} + +static void test_parse_args_append_verify() { + Config* cfg = config_create(); + cfg->send_directory = str_dup("/src"); + cfg->receive_root_directory = str_dup("/dst"); + char* argv[] = {"fastsync", "--append-verify", "/src", "/dst"}; + int positional_args[2]; + int positional_count = 0; + EXPECT_EQ_INT(parse_args(cfg, 4, argv, positional_args, &positional_count), 0); + EXPECT_TRUE(cfg->append_verify); + EXPECT_FALSE(cfg->append); + EXPECT_TRUE(cfg->use_incremental); + EXPECT_TRUE(validate_config(cfg)); + config_delete(cfg); +} + +/* Both spellings are accepted; the safer --append-verify semantics win on the + * wire (the sender checks append_verify first), so neither flag is silently + * dropped but the run is still valid. */ +static void test_parse_args_append_both() { + Config* cfg = config_create(); + cfg->send_directory = str_dup("/src"); + cfg->receive_root_directory = str_dup("/dst"); + char* argv[] = {"fastsync", "--append", "--append-verify", "/src", "/dst"}; + int positional_args[2]; + int positional_count = 0; + EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0); + EXPECT_TRUE(cfg->append); + EXPECT_TRUE(cfg->append_verify); + EXPECT_TRUE(validate_config(cfg)); + config_delete(cfg); +} + +static void test_validate_config_append_rejects_chunk_serialization() { + Config* cfg = config_create(); + char* argv[] = {"fastsync", "--append", "-s", "/src", "/dst"}; + int positional_args[2]; + int positional_count = 0; + EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0); + EXPECT_FALSE(validate_config(cfg)); + config_delete(cfg); +} + +static void test_validate_config_append_verify_rejects_chunk_serialization() { + Config* cfg = config_create(); + char* argv[] = {"fastsync", "--append-verify", "-s", "/src", "/dst"}; + int positional_args[2]; + int positional_count = 0; + EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0); + EXPECT_FALSE(validate_config(cfg)); + config_delete(cfg); +} + +static void test_validate_config_append_rejects_whole_file() { + Config* cfg = config_create(); + char* argv[] = {"fastsync", "--append", "-W", "/src", "/dst"}; + int positional_args[2]; + int positional_count = 0; + EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0); + EXPECT_FALSE(validate_config(cfg)); + config_delete(cfg); +} + +static void test_validate_config_append_verify_rejects_whole_file() { + Config* cfg = config_create(); + char* argv[] = {"fastsync", "--append-verify", "-W", "/src", "/dst"}; + int positional_args[2]; + int positional_count = 0; + EXPECT_EQ_INT(parse_args(cfg, 5, argv, positional_args, &positional_count), 0); + EXPECT_FALSE(validate_config(cfg)); + config_delete(cfg); +} + void test_client_cli() { test_validate_config_required_paths(); + test_parse_args_append(); + test_parse_args_append_verify(); + test_parse_args_append_both(); + test_validate_config_append_rejects_chunk_serialization(); + test_validate_config_append_verify_rejects_chunk_serialization(); + test_validate_config_append_rejects_whole_file(); + test_validate_config_append_verify_rejects_whole_file(); test_validate_config_incompatible_options(); test_validate_config_tls_requirements(); test_validate_config_delta_sendfile_constraints(); diff --git a/tests/test_config.c b/tests/test_config.c index ced2318..b6f4b17 100644 --- a/tests/test_config.c +++ b/tests/test_config.c @@ -754,6 +754,49 @@ static void test_config_is_remote_dest() { EXPECT_TRUE(config_is_remote_dest("user@host:")); } +/* The append-mode fields cross the wire unchanged: --append and --append-verify + are negotiated to the receiver so it knows to reply STATUS_APPEND on a + shorter destination. */ +static void test_config_append_wire_roundtrip() { + if (is_running_under_valgrind()) + return; + struct { + bool append, append_verify; + } cases[] = {{true, false}, {false, true}, {true, true}, {false, false}}; + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + int p[2]; + EXPECT_EQ_INT(socketpair(AF_UNIX, SOCK_STREAM, 0, p), 0); + pid_t pid = fork(); + if (pid == 0) { + close(p[1]); + io_set_fds(p[0], p[0]); + Config* recv = config_receive(p[0]); + bool ok = recv != NULL; + if (ok) + ok = recv->append == cases[i].append && recv->append_verify == cases[i].append_verify; + config_delete(recv); + close(p[0]); + _exit(ok ? 0 : 1); + } else { + close(p[0]); + io_set_fds(p[1], p[1]); + Config* send_cfg = config_create(); + EXPECT_NOT_NULL(send_cfg); + send_cfg->send_directory = str_dup("/src"); + send_cfg->receive_root_directory = str_dup("/dst"); + send_cfg->append = cases[i].append; + send_cfg->append_verify = cases[i].append_verify; + bool sent = config_send(p[1], send_cfg); + int status; + waitpid(pid, &status, 0); + close(p[1]); + config_delete(send_cfg); + EXPECT_TRUE(sent); + EXPECT_TRUE(WIFEXITED(status) && WEXITSTATUS(status) == 0); + } + } +} + void test_config() { test_config_lifecycle(); test_config_ssh_dest(); @@ -771,6 +814,7 @@ void test_config() { test_config_delete_timing_wire_roundtrip(); test_config_delete_timing_conflict_rejected(); test_config_delete_policy_wire_roundtrip(); + test_config_append_wire_roundtrip(); test_config_basis_roundtrip(); test_config_basis_wire_rejects_escaping(); test_config_basis_normalization(); diff --git a/tests/test_shared_utils.c b/tests/test_shared_utils.c index 9828620..a2cbc1c 100644 --- a/tests/test_shared_utils.c +++ b/tests/test_shared_utils.c @@ -276,6 +276,22 @@ void test_shared_utils() { test_walker_unlimited_deletes_all(); test_walker_hard_bound_all_or_nothing(); + /* --append / --append-verify tail-resume math: a resume is eligible only for + a shorter existing destination, and the tail length is then the difference. */ + EXPECT_TRUE(append_resume_eligible(0, 10)); + EXPECT_TRUE(append_resume_eligible(7, 10)); + EXPECT_FALSE(append_resume_eligible(10, 10)); + EXPECT_FALSE(append_resume_eligible(11, 10)); + + unsigned long long tail; + EXPECT_TRUE(append_tail_length(0, 10, &tail)); + EXPECT_EQ_INT((int)tail, 10); + EXPECT_TRUE(append_tail_length(7, 10, &tail)); + EXPECT_EQ_INT((int)tail, 3); + EXPECT_FALSE(append_tail_length(10, 10, &tail)); + EXPECT_FALSE(append_tail_length(11, 10, &tail)); + EXPECT_FALSE(append_tail_length(7, 10, NULL)); + char formatted[32]; EXPECT_TRUE(format_human_bytes(0, formatted, sizeof(formatted))); EXPECT_EQ_STR(formatted, "0 B");