feat(delete): default --delete to rsync delete-during; add --delete-commit
- plain --delete with no timing flag now selects delete-during (progressive deletion, matching rsync and avoiding the full old+new tree peak) - new long-only FastSync --delete-commit restores the old atomic behavior (delete only after the whole transfer succeeds); timing-identical to --delete-after, implemented via the same wire bool - timing flags are mutually exclusive; --delete-commit conflicts with other timings; --delete-before/--delete-during rows reworded per Phase-0 probes - CHANGELOG migration note; tally unchanged 116/14/27
This commit is contained in:
@@ -217,7 +217,21 @@ class _SlicingProxy:
|
||||
|
||||
|
||||
class TestDeleteTimingFinalStateParity:
|
||||
"""On a successful transfer the per-directory timings match rsync's result."""
|
||||
"""On a successful transfer the per-directory timings match rsync's result.
|
||||
|
||||
Plain ``--delete`` has no rsync-incompatible spelling: it defaults to
|
||||
delete-during on both tools, so it is compared against rsync's own default.
|
||||
``--delete-commit`` is FastSync-only and selects the late whole-tree commit,
|
||||
which is rsync's ``--delete-after`` timing.
|
||||
"""
|
||||
|
||||
# (fastsync flag, rsync flag)
|
||||
PAIRS = [
|
||||
("--delete", "--delete"),
|
||||
("--delete-during", "--delete-during"),
|
||||
("--delete-delay", "--delete-delay"),
|
||||
("--delete-commit", "--delete-after"),
|
||||
]
|
||||
|
||||
def _run_fastsync(self, tag, timing):
|
||||
source, dest, received = _seed_pair(tag)
|
||||
@@ -226,12 +240,12 @@ class TestDeleteTimingFinalStateParity:
|
||||
result, _ = run_client(source, dest, flags=[timing], port=server.port)
|
||||
return result, received
|
||||
|
||||
@pytest.mark.parametrize("timing", ["--delete-during", "--delete-delay"])
|
||||
@pytest.mark.parametrize("fs_timing,rs_timing", PAIRS)
|
||||
@requires_rsync
|
||||
def test_success_final_state_matches_rsync(self, timing):
|
||||
# Worker-safe names: xdist may run both parametrizations concurrently, so
|
||||
# the timing is part of every fixture path.
|
||||
label = timing.lstrip("-")
|
||||
def test_success_final_state_matches_rsync(self, fs_timing, rs_timing):
|
||||
# Worker-safe names: xdist may run the parametrizations concurrently, so
|
||||
# the flags are part of every fixture path.
|
||||
label = f"{fs_timing.lstrip('-')}_vs_{rs_timing.lstrip('-')}"
|
||||
# Build the rsync fixture from the same seed so both sides start equal.
|
||||
source, dest, received = _seed_pair(f"parity_rsync_{label}")
|
||||
source2 = source
|
||||
@@ -240,17 +254,18 @@ class TestDeleteTimingFinalStateParity:
|
||||
# rsync mirrors src/ into dst/; seed the same extra.
|
||||
_write(os.path.join(rsync_dst, "d", "old_extra"), b"stale extra\n")
|
||||
|
||||
rsync_result = _rsync(["-a", timing, source2 + "/", rsync_dst + "/"])
|
||||
rsync_result = _rsync(["-a", rs_timing, source2 + "/", rsync_dst + "/"])
|
||||
assert rsync_result.returncode == 0, rsync_result.stderr
|
||||
rsync_tree = _tree(rsync_dst)
|
||||
|
||||
with ServerManager() as server:
|
||||
server.start(extra_args=["--allow-delete"])
|
||||
result, _ = run_client(source, dest, flags=[timing], port=server.port)
|
||||
result, _ = run_client(source, dest, flags=[fs_timing], port=server.port)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||
fastsync_tree = _tree(received)
|
||||
assert fastsync_tree == rsync_tree, (
|
||||
f"{timing}: fastsync tree {fastsync_tree} != rsync tree {rsync_tree}"
|
||||
f"{fs_timing} vs rsync {rs_timing}: fastsync tree {fastsync_tree} != "
|
||||
f"rsync tree {rsync_tree}"
|
||||
)
|
||||
|
||||
|
||||
@@ -292,7 +307,14 @@ class TestDeleteTimingTypeConflictParity:
|
||||
|
||||
|
||||
class TestDeleteTimingFailure:
|
||||
"""A mid-transfer failure distinguishes during from delay."""
|
||||
"""A mid-transfer failure distinguishes the during timings from the late
|
||||
commit timings.
|
||||
|
||||
Plain ``--delete`` must behave like ``--delete-during`` (the rsync default),
|
||||
removing the extras of the directories already reached; ``--delete-commit``
|
||||
must behave like ``--delete-after`` and remove nothing until the transfer
|
||||
has fully succeeded.
|
||||
"""
|
||||
|
||||
@pytest.mark.parametrize("mt", [False, True])
|
||||
def test_during_removes_delay_preserves_on_failure(self, mt):
|
||||
@@ -301,8 +323,12 @@ class TestDeleteTimingFailure:
|
||||
assert os.path.exists(extra)
|
||||
with ServerManager() as server:
|
||||
server.start(extra_args=["--allow-delete"])
|
||||
for timing, expect_removed in (("--delete-during", True),
|
||||
("--delete-delay", False)):
|
||||
for timing, expect_removed in (
|
||||
("--delete-during", True),
|
||||
("--delete", True),
|
||||
("--delete-delay", False),
|
||||
("--delete-commit", False),
|
||||
("--delete-after", False)):
|
||||
# Re-seed the extra before each run.
|
||||
_write(extra, b"stale extra\n")
|
||||
proxy = _SlicingProxy(server.port, forward_limit=MID_TRANSFER_BYTES, throttle=PROXY_THROTTLE)
|
||||
@@ -449,12 +475,13 @@ class TestDeleteDelayVsAfterSnapshot:
|
||||
|
||||
|
||||
class TestDeleteAfterThreadsKeepSet:
|
||||
"""Regression: -m/--threads with the default delete-after timing (plain
|
||||
--delete) must still transmit the keep-set manifest and remove destination
|
||||
extras. PipelineContextSender.delete_suppressed was left uninitialized, so a
|
||||
garbage true silently skipped the manifest under --threads."""
|
||||
"""Regression: -j/--threads must still transmit the delete keep-set in every
|
||||
timing. PipelineContextSender.delete_suppressed was left uninitialized, so a
|
||||
garbage true silently skipped the late keep-set manifest under --threads.
|
||||
Plain --delete now uses the per-directory plans, while --delete-commit /
|
||||
--delete-after keep exercising the late whole-tree manifest."""
|
||||
|
||||
@pytest.mark.parametrize("delete_flag", ["--delete", "--delete-after"])
|
||||
@pytest.mark.parametrize("delete_flag", ["--delete", "--delete-commit", "--delete-after"])
|
||||
def test_threads_delete_after_sends_keep_set(self, delete_flag):
|
||||
source, dest, received = _seed_pair("mtkeep")
|
||||
extra = os.path.join(received, "d", "old_extra")
|
||||
@@ -465,7 +492,7 @@ class TestDeleteAfterThreadsKeepSet:
|
||||
port=server.port)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||
assert not os.path.exists(extra), (
|
||||
f"{delete_flag} --threads did not remove an extra: keep-set manifest was suppressed"
|
||||
f"{delete_flag} --threads did not remove an extra: delete keep-set was suppressed"
|
||||
)
|
||||
|
||||
class TestDeleteDelayMaxDeleteRefilledDir:
|
||||
|
||||
@@ -241,6 +241,9 @@ _CASES = [
|
||||
server_args=DELETE, ref="--delete-delay"),
|
||||
H.Case("delete_after", "basic", ["-a", "--delete-after"], seed=seed_extras,
|
||||
server_args=DELETE, ref="--delete-after"),
|
||||
H.Case("delete_commit", "basic", ["-a", "--delete-after"], seed=seed_extras,
|
||||
fastsync_flags=["-a", "--delete-commit"], server_args=DELETE,
|
||||
ref="FastSync-only --delete-commit == rsync --delete-after"),
|
||||
H.Case("delete_excluded", "filters",
|
||||
["-a", "--delete", "--delete-excluded", "--exclude=*.log"],
|
||||
seed=seed_delete_excluded, server_args=DELETE, ref="--delete-excluded"),
|
||||
@@ -255,7 +258,7 @@ _CASES = [
|
||||
H.Case("filter_protect", "filters",
|
||||
["-a", "--delete", "--filter=P *.log"],
|
||||
seed=seed_filter_protect, server_args=DELETE, ci=True,
|
||||
ref="--filter P/--protect receiver-side delete protection"),
|
||||
ref="--filter P/--protect receiver-side delete protection (default during)"),
|
||||
H.Case("filter_protect_during", "filters",
|
||||
["-a", "--delete-during", "--filter=P *.log"],
|
||||
seed=seed_filter_protect, server_args=DELETE, ci=True,
|
||||
@@ -264,6 +267,10 @@ _CASES = [
|
||||
["-a", "--delete-delay", "--filter=P *.log"],
|
||||
seed=seed_filter_protect, server_args=DELETE, ci=True,
|
||||
ref="--filter P/--protect under --delete-delay"),
|
||||
H.Case("filter_protect_after", "filters",
|
||||
["-a", "--delete-after", "--filter=P *.log"],
|
||||
seed=seed_filter_protect, server_args=DELETE, ci=True,
|
||||
ref="--filter P/--protect under the whole-tree --delete-after commit"),
|
||||
|
||||
# --- relative / dirs --------------------------------------------------
|
||||
H.Case("relative_general", "basic", ["-a", "-R"], layout=H.MIRROR_ABS,
|
||||
|
||||
@@ -3982,15 +3982,16 @@ class TestDeleteTiming:
|
||||
assert _read_file(os.path.join(received, "sub", "deep.txt")) == b"deeply nested file\n", \
|
||||
f"{flag}: nested file was not written after the early deletion"
|
||||
|
||||
@pytest.mark.parametrize("flag", ["--delete", "--delete-after"])
|
||||
@pytest.mark.parametrize("flag", ["--delete-commit", "--delete-after"])
|
||||
@pytest.mark.parametrize("mt", [False, True])
|
||||
def test_late_flags_commit_only_after_success(self, flag, mt):
|
||||
"""Plain --delete/--delete-after defer deletion until the whole transfer
|
||||
"""--delete-commit/--delete-after defer deletion until the whole transfer
|
||||
succeeds: a mid-transfer write failure must leave every extra in place
|
||||
(commit-style safety). The -m receiver must also keep the extras: the
|
||||
deferred keep-set is committed by the server only after the disk-writer
|
||||
thread has finished, and a failing writer means the manifest is freed,
|
||||
never applied."""
|
||||
(commit-style safety). Plain --delete no longer defers (it defaults to
|
||||
delete-during), so only the explicitly late timings are exercised here.
|
||||
The --threads receiver must also keep the extras: the deferred keep-set is
|
||||
committed by the server only after the disk-writer thread has finished,
|
||||
and a failing writer means the manifest is freed, never applied."""
|
||||
source = self._seed("late")
|
||||
dest = os.path.join(TEST_DATA_DIR, "deltiming_late_dst")
|
||||
clean_dir(dest)
|
||||
|
||||
Reference in New Issue
Block a user