Release v2.29.0 #312

Merged
TapTap merged 123 commits from dev into main 2026-09-23 02:05:14 +02:00
5 changed files with 352 additions and 68 deletions
Showing only changes of commit b235721f8b - Show all commits
+27 -66
View File
@@ -1855,25 +1855,6 @@ static bool scan_paths_only(const Config* config, const ScannerOptions* options,
return ok;
}
/* Transmit any not-yet-sent per-directory delete plan needed by the entries in
* `chunk` (ancestors root-first, then the entry's own directory for --dirs
* entries) before its data frames go out, so --delete-during/--delete-delay
* clear a directory's extras (and any type conflict) before the directory's
* first write. */
static int send_chunk_delete_plans(Client* client, DeletePlanSender* plans, const Chunk* chunk) {
if (!plans)
return 0;
for (int i = 0; i < chunk->element_count; i++) {
File* f = chunk->items[i];
if (!f)
continue;
if (delete_plan_send_for_path(client->file_descriptor, plans, file_wire_path(f), f->is_dir) !=
0)
return -1;
}
return 0;
}
static int incremental_check(Client* client, File* file, const Config* config,
DeltaSignature** out_sig, unsigned long long* resume_offset) {
*out_sig = NULL;
@@ -2753,9 +2734,12 @@ static int send_chunks_multithreaded(void* pipeline_context) {
return thrd_error;
}
} else if (context->delete_plans) {
/* --delete-during/--delete-delay: transmit the receive root's plan before
any data, exactly like rsync's first generator directory. */
if (delete_plan_send_root(client->file_descriptor, context->delete_plans) != 0) {
/* --delete-during/--delete-delay: transmit the COMPLETE per-directory plan
set before any data, so a mid-transfer abort has already applied every
planned removal exactly like rsync's generator (which runs ahead of its
throttled sender). A completed run is unaffected. */
if (delete_plan_send_all(client->file_descriptor, context->delete_plans, context->plan_dirs) !=
0) {
pipeline_cancel(context);
disconnect_transfer_client(client);
mark_sender_done(context);
@@ -2802,15 +2786,6 @@ static int send_chunks_multithreaded(void* pipeline_context) {
}
break;
}
if (send_chunk_delete_plans(client, context->delete_plans, current_chunk) != 0) {
log_message(LOG_LEVEL_ERROR, "unexpected error while sending delete plan");
chunk_destroy(current_chunk);
pipeline_cancel(context);
disconnect_transfer_client(client);
mark_sender_done(context);
protocol_session_unbind();
return thrd_error;
}
if (send_chunk_with_removal(client, current_chunk, context->config,
context->remove_source_files, &context->stats) != 0) {
log_message(LOG_LEVEL_ERROR, "unexpected error while sending chunk");
@@ -2893,13 +2868,6 @@ static int send_chunks_multithreaded(void* pipeline_context) {
NULL) != 0)
goto send_fail;
}
/* Emit the plans for source directories the data stream never triggered
(empty directories): their extras are still cleared while the directory
itself is kept. */
if (!context->scan_stopped_early && context->delete_plans && context->plan_dirs &&
delete_plan_send_remaining(client->file_descriptor, context->delete_plans,
context->plan_dirs) != 0)
goto send_fail;
/* P7 Wave D: transmit the captured directory times last. The scanner thread
(and all parallel workers) has been joined before scanner_done was set, so
the list is complete and race-free; on an early stop the list may be
@@ -3232,10 +3200,12 @@ int send_files(Config* config) {
/* Traversed source directories for the per-directory delete keep set. */
ArrayList* plan_dirs = NULL;
bool delete_early = config->use_delete && config_delete_timing_early(config);
/* -d/--dirs does not recurse, so a per-directory plan would carry no child
information and could delete the contents of an untraversed directory;
fall back to the whole-tree end-of-transfer commit for that mode. */
bool delete_per_dir = config->use_delete && config_delete_timing_per_dir(config) && !config->dirs;
/* --delete-during/--delete-delay use per-directory plans for every transfer
shape. For -d/--dirs the generator records only the directories whose
direct children it actually enumerated, so the plan removes extras directly
inside a listed directory while an untraversed (kept) subdirectory is
shielded -- rsync's `-d DIR/ --delete`. */
bool delete_per_dir = config->use_delete && config_delete_timing_per_dir(config);
bool send_failed = false;
bool had_scan_io = false;
unsigned long long per_dir_non_dir_count = 0;
@@ -3287,12 +3257,12 @@ int send_files(Config* config) {
prepared.options.synced_dirs = synced_dirs;
}
}
/* The late-timing modes (--delete-after/--delete-commit and a plain --delete
that fell back from per-dir mode because of -d/--dirs) build the manifest
/* The late-timing modes (--delete-after/--delete-commit) build the manifest
while streaming and send it after the last data frame. --delete-before
sends a whole-tree keep-set up front; --delete-during/--delete-delay build a
per-directory plan set up front (paths only) and stream the plans alongside
the data, so no manifest is kept during the data pass. */
sends a whole-tree keep-set up front; --delete-during/--delete-delay build
the complete per-directory plan set up front (paths only) and transmit it
all before the first data frame, so a mid-transfer abort has already
applied every planned removal. */
if (delete_early) {
/* Pass 1: collect the complete keep-set (paths only, no data loaded) and
transmit it now, before any file data. The receiver removes extras and
@@ -3335,9 +3305,10 @@ int send_files(Config* config) {
goto send_fail;
} else if (delete_per_dir) {
/* --delete-during/--delete-delay: build one plan per source directory from a
path-only pre-scan and transmit the root plan now, before any data, so the
receive root's extras are handled exactly like rsync's first generator
directory. The remaining plans are streamed with the data below. */
path-only pre-scan and transmit the COMPLETE plan set now, before any data,
so every planned removal has already been applied when a later transfer
phase fails -- exactly like rsync's generator, whose deletion list runs
ahead of its throttled sender. A completed run is unaffected. */
plan_sender = delete_plan_sender_create();
plan_dirs = array_list_create(free);
if (!plan_sender || !plan_dirs)
@@ -3368,7 +3339,7 @@ int send_files(Config* config) {
plan_dirs = NULL;
skip_delete = true;
} else {
plans_ok = delete_plan_send_root(client->file_descriptor, plan_sender) == 0;
plans_ok = delete_plan_send_all(client->file_descriptor, plan_sender, plan_dirs) == 0;
}
}
prepared.options.excluded_paths = NULL;
@@ -3455,11 +3426,6 @@ int send_files(Config* config) {
goto send_fail;
}
}
if (send_chunk_delete_plans(client, plan_sender, current_chunk) != 0) {
chunk_destroy(current_chunk);
send_failed = true;
break;
}
if (send_chunk_with_removal(client, current_chunk, config, remove_sources, &transfer_stats) !=
0) {
log_message(LOG_LEVEL_ERROR, "Failed to send chunk");
@@ -3542,12 +3508,6 @@ int send_files(Config* config) {
}
}
}
/* Emit the plans for any source directories the data stream never triggered
(an empty directory has no file frame). Sending them now still clears that
directory's destination extras while keeping the directory itself. */
if (!scan_stopped_early && plan_sender && plan_dirs &&
delete_plan_send_remaining(client->file_descriptor, plan_sender, plan_dirs) != 0)
goto send_fail;
/* P7 Wave D: every directory has now been traversed (or the scan stopped
early), so transmit the captured directory times last. The receiver defers
applying them until after its own deletion/publication phase. */
@@ -3714,10 +3674,11 @@ int send_files_multithreaded(Config** config_ptr) {
return 1;
}
}
/* -d/--dirs does not recurse, so a per-directory plan would carry no child
information and could delete the contents of an untraversed directory;
fall back to the whole-tree end-of-transfer commit for that mode. */
bool per_dir = config_delete_timing_per_dir(config) && !config->dirs;
/* --delete-during/--delete-delay use per-directory plans for every transfer
shape. The -d/--dirs generator records only the directories whose direct
children it enumerated, so extras directly inside a listed directory are
removed while an untraversed (kept) subdirectory is shielded. */
bool per_dir = config_delete_timing_per_dir(config);
if (config_delete_timing_early(config) || per_dir) {
/* --delete-before / --delete-during / --delete-delay: build the keep-set
(paths only, nothing loaded or sent) up front so the sender thread can
+11
View File
@@ -1451,6 +1451,17 @@ static File* dirs_next_file(DirectoryScanner* scanner) {
scanner->dirs_root_emitted = true;
if (scanner->options.prune_empty_dirs && dirs_source_dir_is_empty(scanner->root_path))
return NULL;
/* The listed directory's direct children are about to be enumerated, so
its destination mirror is a synchronized directory: record it for the
per-directory delete plan. The plan keeps the enumerated children and
shields untraversed subdirectories, so --delete-during removes extras
directly inside the listed directory without descending into a kept
(but untraversed) child -- exactly rsync's `-d DIR/ --delete`. */
if (!scanner_record_synced_dir(&scanner->options, scanner->root_path, "",
scanner->relative_mode)) {
scanner->failed = true;
return NULL;
}
scanner->current_dir = opendir(scanner->root_path);
if (!scanner->current_dir) {
scanner->io_error = true;
+11
View File
@@ -432,6 +432,17 @@ int delete_plan_send_remaining(int fd, DeletePlanSender* sender, const ArrayList
return 0;
}
int delete_plan_send_all(int fd, DeletePlanSender* sender, const ArrayList* dirs) {
if (!sender)
return -1;
/* Root first: this also transmits the one-shot per-run config block on its
own carrier frame (see send_config_only), so it reaches the receiver even
when the scope permits no directory plan at all. */
if (delete_plan_send_root(fd, sender) != 0)
return -1;
return delete_plan_send_remaining(fd, sender, dirs);
}
/* ------------------------------------------------------------------ */
/* Receiver: delete session */
/* ------------------------------------------------------------------ */
+12 -2
View File
@@ -61,9 +61,19 @@ int delete_plan_send_root(int fd, DeletePlanSender* sender);
* for `path` itself; already-sent plans are skipped. */
int delete_plan_send_for_path(int fd, DeletePlanSender* sender, const char* path, bool is_dir);
/* Send the plan for every directory in `dirs` that has not been transmitted
* yet. Called after the data stream so an empty source directory's plan still
* clears its destination extras even though no file frame triggered it. */
* yet. */
int delete_plan_send_remaining(int fd, DeletePlanSender* sender, const ArrayList* dirs);
/* Transmit the COMPLETE per-directory plan set in one pass, before any data
* frame: the root plan (with the one-shot per-run config block on its carrier
* frame) followed by every directory in `dirs`. Because the whole plan set is
* known from the path-only pre-scan, sending it all up front means a
* mid-transfer abort has already applied every planned removal, matching
* rsync's generator (which runs ahead of its throttled sender). A completed
* run is unaffected. `dirs` is the set of directories whose direct children
* were enumerated (the scanner's plan_dirs sink), so a merely listed but
* untraversed directory never gets a plan and its mirror is left intact.
* Returns -1 on I/O error. */
int delete_plan_send_all(int fd, DeletePlanSender* sender, const ArrayList* dirs);
/* ---- Receiver: delete session ---- */
@@ -0,0 +1,291 @@
"""Differential coverage for the delete-timing ABORT BOUNDARY (A9/A10).
rsync's generator runs ahead of its throttled sender, so on a mid-transfer abort
it has already removed every extra it planned. FastSync now transmits the
COMPLETE per-directory plan set before the first data frame, so an abort has the
same effect. Before that change FastSync only removed the extras of the
directories its (slower) data stream had reached, and ``-d/--dirs`` used an
end-of-transfer commit that removed nothing on abort.
These tests abort both tools mid-transfer and assert the destination extras
removed match real ``rsync 3.4.1``. The rsync side is driven locally with
``--bwlimit`` and a small timing window (its generator's delete list is computed
long before the throttled payload finishes); the FastSync side uses the
byte-deterministic slicing proxy from ``test_delete_timing_parity``.
"""
import os
import shutil
import subprocess
import sys
import time
import pytest
sys.path.insert(0, os.path.dirname(__file__))
from common import ( # noqa: E402
TEST_DATA_DIR,
ServerManager,
clean_dir,
get_dest_received_dir,
run_client,
)
from test_delete_timing_parity import _SlicingProxy # noqa: E402
RSYNC = shutil.which("rsync")
requires_rsync = pytest.mark.skipif(RSYNC is None, reason="rsync 3.4.1 not installed")
# Exceeds the 10 MiB scanner chunk, so the next directory lands in a later chunk
# (still unreached when the proxy cuts the stream).
BIG_BYTES = 16 * 1024 * 1024
# Cut well past the (small) config + delete-plan frames and into the big payload,
# so the receiver has provably processed every plan before the abort.
MID_TRANSFER_BYTES = 256 * 1024
PROXY_THROTTLE = 0.001
# Throttle rsync's sender so the generator has deleted long before the payload
# finishes, then interrupt it mid-transfer.
RSYNC_BWLIMIT = 512 # KiB/s -> ~32 s for 16 MiB
RSYNC_ABORT_DELAY = 1.5
def _write(path, content):
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "wb") as fh:
fh.write(content)
def _rsync_aborted(args, delay=RSYNC_ABORT_DELAY):
"""Start rsync, let its generator run, then interrupt it mid-transfer."""
env = dict(os.environ, LC_ALL="C")
proc = subprocess.Popen([RSYNC] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
text=True, env=env)
time.sleep(delay)
proc.terminate()
try:
proc.wait(timeout=10)
except subprocess.TimeoutExpired:
proc.kill()
proc.wait(timeout=5)
return proc
class TestDeleteDuringAbortBoundary:
"""A9: on an abort, every planned removal has already been applied."""
def _seed_recursive(self, tag):
source = os.path.join(TEST_DATA_DIR, f"dab_{tag}_src")
clean_dir(source)
# ``a/keep.bin`` sorts first, so the client streams it (and the proxy
# cuts) before the data pass ever reaches ``z/deep``.
_write(os.path.join(source, "a", "keep.bin"), b"B" * BIG_BYTES)
_write(os.path.join(source, "z", "deep", "keep.txt"), b"keep\n")
return source
@requires_rsync
def test_recursive_abort_removes_all_planned_extras(self):
# ---- FastSync: abort mid ``a/keep.bin``; ``z/deep`` is never reached.
source = self._seed_recursive("rec_fs")
dest = os.path.join(TEST_DATA_DIR, "dab_rec_fs_dst")
clean_dir(dest)
received = get_dest_received_dir(dest, source)
os.makedirs(os.path.join(received, "a"), exist_ok=True)
_write(os.path.join(received, "a", "a_extra"), b"stale\n")
os.makedirs(os.path.join(received, "z", "deep"), exist_ok=True)
_write(os.path.join(received, "z", "deep", "old_extra"), b"stale\n")
with ServerManager() as server:
server.start(extra_args=["--allow-delete"])
proxy = _SlicingProxy(server.port, forward_limit=MID_TRANSFER_BYTES,
throttle=PROXY_THROTTLE)
result, _ = run_client(source, dest, flags=["--delete-during"], port=proxy.port)
proxy.finish()
assert result.returncode != 0, "truncated transfer reported success"
assert not os.path.exists(os.path.join(received, "a", "a_extra"))
assert not os.path.exists(os.path.join(received, "z", "deep", "old_extra")), (
"FastSync left an extra in a directory it never reached before the abort"
)
# ---- rsync 3.4.1: same tree, same abort, same delete outcome.
source = self._seed_recursive("rec_rs")
rsync_dst = os.path.join(TEST_DATA_DIR, "dab_rec_rs_dst")
clean_dir(rsync_dst)
os.makedirs(os.path.join(rsync_dst, "a"), exist_ok=True)
_write(os.path.join(rsync_dst, "a", "a_extra"), b"stale\n")
os.makedirs(os.path.join(rsync_dst, "z", "deep"), exist_ok=True)
_write(os.path.join(rsync_dst, "z", "deep", "old_extra"), b"stale\n")
proc = _rsync_aborted(["-a", "--delete-during", f"--bwlimit={RSYNC_BWLIMIT}",
source + "/", rsync_dst + "/"])
assert proc.returncode != 0, "rsync was not actually interrupted"
assert not os.path.exists(os.path.join(rsync_dst, "a", "a_extra"))
assert not os.path.exists(os.path.join(rsync_dst, "z", "deep", "old_extra")), (
"rsync's generator did not delete ahead of its sender"
)
class TestDirsDeleteAbortBoundary:
"""A10: ``-d/--dirs`` uses per-directory plans like rsync.
The listed directory's direct extras are removed by the up-front plan while
a kept but untraversed subdirectory (and its destination content) is
shielded.
"""
def _seed_dirs(self, tag):
source = os.path.join(TEST_DATA_DIR, f"ddb_{tag}_src")
clean_dir(source)
_write(os.path.join(source, "big.bin"), b"B" * BIG_BYTES)
_write(os.path.join(source, "subdir", "keep.txt"), b"inner\n")
return source
@pytest.mark.parametrize("fs_flag,rs_flag", [("--delete-during", "--delete-during"),
("--delete", "--delete")])
@requires_rsync
def test_dirs_abort_removes_direct_extras_only(self, fs_flag, rs_flag):
label = f"{fs_flag.lstrip('-')}_{rs_flag.lstrip('-')}"
# ---- FastSync: ``-d`` lists the immediate children; big.bin streams and
# the abort lands mid-payload.
source = self._seed_dirs(f"dirs_{label}_fs")
dest = os.path.join(TEST_DATA_DIR, f"ddb_{label}_fs_dst")
clean_dir(dest)
received = get_dest_received_dir(dest, source)
_write(os.path.join(received, "old_extra"), b"stale\n")
os.makedirs(os.path.join(received, "subdir"), exist_ok=True)
_write(os.path.join(received, "subdir", "stale.txt"), b"stale inner\n")
with ServerManager() as server:
server.start(extra_args=["--allow-delete"])
proxy = _SlicingProxy(server.port, forward_limit=MID_TRANSFER_BYTES,
throttle=PROXY_THROTTLE)
result, _ = run_client(source + "/", dest, flags=["-d", fs_flag], port=proxy.port)
proxy.finish()
assert result.returncode != 0, f"{fs_flag}: truncated transfer reported success"
assert not os.path.exists(os.path.join(received, "old_extra")), (
f"{fs_flag}: the listed directory's direct extra survived the abort"
)
assert os.path.exists(os.path.join(received, "subdir", "stale.txt")), (
f"{fs_flag}: descended into a kept, untraversed subdirectory"
)
# ---- rsync 3.4.1: same shape and same abort.
source = self._seed_dirs(f"dirs_{label}_rs")
rsync_dst = os.path.join(TEST_DATA_DIR, f"ddb_{label}_rs_dst")
clean_dir(rsync_dst)
_write(os.path.join(rsync_dst, "old_extra"), b"stale\n")
os.makedirs(os.path.join(rsync_dst, "subdir"), exist_ok=True)
_write(os.path.join(rsync_dst, "subdir", "stale.txt"), b"stale inner\n")
proc = _rsync_aborted(["-d", rs_flag, f"--bwlimit={RSYNC_BWLIMIT}",
source + "/", rsync_dst + "/"])
assert proc.returncode != 0, "rsync was not actually interrupted"
assert not os.path.exists(os.path.join(rsync_dst, "old_extra")), (
f"rsync {rs_flag}: the listed directory's direct extra survived the abort"
)
assert os.path.exists(os.path.join(rsync_dst, "subdir", "stale.txt")), (
f"rsync {rs_flag}: descended into a kept, untraversed subdirectory"
)
def _tree(root):
out = []
for dirpath, dirs, files in os.walk(root):
for name in dirs:
out.append(os.path.relpath(os.path.join(dirpath, name), root))
for name in files:
out.append(os.path.relpath(os.path.join(dirpath, name), root))
return sorted(out)
class TestDirsDeleteFinalStateParity:
"""A10 completed run: ``-d DIR/ --delete`` (during default) and
``--delete-during`` match rsync's final tree, including a kept but
untraversed subdirectory whose destination content survives."""
@pytest.mark.parametrize("flag", ["--delete", "--delete-during"])
@requires_rsync
def test_dirs_final_state_matches_rsync(self, flag):
source = os.path.join(TEST_DATA_DIR, f"ddf_{flag.lstrip('-')}_src")
clean_dir(source)
_write(os.path.join(source, "keep.txt"), b"new keep\n")
_write(os.path.join(source, "subdir", "inner.txt"), b"inner\n")
def seed_dest(root):
clean_dir(root)
_write(os.path.join(root, "keep.txt"), b"old keep\n")
_write(os.path.join(root, "extra.txt"), b"extra\n")
_write(os.path.join(root, "extrasub", "ex.txt"), b"extra sub\n")
_write(os.path.join(root, "subdir", "stale.txt"), b"stale inner\n")
rsync_dst = os.path.join(TEST_DATA_DIR, f"ddf_{flag.lstrip('-')}_rs_dst")
seed_dest(rsync_dst)
env = dict(os.environ, LC_ALL="C")
rsync_result = subprocess.run(
[RSYNC, "-d", flag, source + "/", rsync_dst + "/"],
capture_output=True, text=True, env=env, timeout=120)
assert rsync_result.returncode == 0, rsync_result.stderr
rsync_tree = _tree(rsync_dst)
dest = os.path.join(TEST_DATA_DIR, f"ddf_{flag.lstrip('-')}_fs_dst")
clean_dir(dest)
received = get_dest_received_dir(dest, source)
seed_dest(received)
with ServerManager() as server:
server.start(extra_args=["--allow-delete"])
result, _ = run_client(source + "/", dest, flags=["-d", flag], port=server.port)
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
fastsync_tree = _tree(received)
assert fastsync_tree == rsync_tree, (
f"-d {flag}: fastsync tree {fastsync_tree} != rsync tree {rsync_tree}")
class TestOneFileSystemDeleteParity:
"""A9 side effect: the per-directory plan is now emitted only for directories
whose children were enumerated, so a ``-x`` mount-point directory that is
emitted but never traversed is shielded -- its destination content survives,
exactly as rsync keeps a non-descended mount point under ``--delete``."""
@requires_rsync
def test_mountpoint_content_survives_delete(self):
local = os.stat(".")
shm = "/dev/shm"
if not os.path.isdir(shm) or os.stat(shm).st_dev == local.st_dev:
pytest.skip("no cross-device filesystem available")
probe = os.path.join(shm, f"fastsync_dofs_{os.getpid()}")
clean_dir(probe)
_write(os.path.join(probe, "inside.txt"), b"cross\n")
try:
source = os.path.join(TEST_DATA_DIR, "dofs_src")
clean_dir(source)
_write(os.path.join(source, "keep.txt"), b"keep\n")
os.symlink(probe, os.path.join(source, "nested_link"))
def seed_dest(root):
clean_dir(root)
_write(os.path.join(root, "keep.txt"), b"old\n")
_write(os.path.join(root, "nested_link", "stale.txt"), b"stale\n")
rsync_dst = os.path.join(TEST_DATA_DIR, "dofs_rs_dst")
seed_dest(rsync_dst)
env = dict(os.environ, LC_ALL="C")
rsync_result = subprocess.run(
[RSYNC, "-a", "--copy-links", "-x", "--delete-during",
source + "/", rsync_dst + "/"],
capture_output=True, text=True, env=env, timeout=120)
assert rsync_result.returncode == 0, rsync_result.stderr
assert os.path.exists(os.path.join(rsync_dst, "nested_link", "stale.txt")), (
"rsync unexpectedly descended into the mount point")
dest = os.path.join(TEST_DATA_DIR, "dofs_fs_dst")
clean_dir(dest)
received = get_dest_received_dir(dest, source)
seed_dest(received)
with ServerManager() as server:
server.start(extra_args=["--allow-delete"])
result, _ = run_client(source, dest,
flags=["-a", "--copy-links", "-x", "--delete-during"],
port=server.port)
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
assert os.path.exists(os.path.join(received, "nested_link", "stale.txt")), (
"FastSync descended into a non-traversed mount point under --delete")
finally:
clean_dir(probe)