test: delete-policy unit and integration coverage
Unit: walker all-or-nothing bounds (exceeded -> nothing removed + distinct result; exact bound -> deletes), protected-prefix skipping, config wire round-trip for force_delete/delete_excluded/prune_empty_dirs/max_delete, CLI parse/validation for the new flags. Integration (TestDeletePolicy): default delete-excluded protection and --delete-excluded opt-out (single-thread, -m, early --delete-before, excluded-dir subtrees), --max-delete all-or-nothing over and at the limit, --force file-over-nonempty-dir replacement, --prune-empty-dirs (--dirs mode + recursion-mode parity), and --ignore-errors keeping deletion active across a genuine scan I/O error (run as an unprivileged user).
This commit is contained in:
@@ -2107,6 +2107,345 @@ class TestDeleteTiming:
|
||||
assert os.path.exists(extra), "unauthorized delete removed an extra file"
|
||||
|
||||
|
||||
def _seed_delete_tree(tag, entries, dest):
|
||||
"""Create a source tree and seed a full mirror at `dest`, returning
|
||||
(source, received_mirror)."""
|
||||
source = os.path.join(TEST_DATA_DIR, f"delpol_{tag}_src")
|
||||
clean_dir(source)
|
||||
for rel, content in entries.items():
|
||||
full = os.path.join(source, rel)
|
||||
os.makedirs(os.path.dirname(full), exist_ok=True)
|
||||
with open(full, "wb") as fh:
|
||||
fh.write(content)
|
||||
clean_dir(dest)
|
||||
with ServerManager() as server:
|
||||
server.start(extra_args=["--allow-delete"])
|
||||
result, _ = run_client(source, dest, port=server.port)
|
||||
assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}"
|
||||
received = get_dest_received_dir(dest, source)
|
||||
return source, received
|
||||
|
||||
|
||||
class TestDeletePolicy:
|
||||
"""Deletion-policy family: --delete-excluded, --max-delete, --force,
|
||||
--ignore-errors and --prune-empty-dirs."""
|
||||
|
||||
def _write(self, path, content):
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
with open(path, "wb") as fh:
|
||||
fh.write(content)
|
||||
|
||||
@pytest.mark.parametrize("mt", [False, True])
|
||||
@pytest.mark.parametrize("timing", ["--delete", "--delete-before"])
|
||||
def test_delete_protects_excluded_by_default_and_delete_excluded_removes(self, mt, timing):
|
||||
"""rsync parity: with --delete a destination mirror path whose source was
|
||||
excluded survives (protected by default); --delete-excluded opts back
|
||||
into deleting it. Verified single-threaded, -m, and an early timing
|
||||
run (--delete-before) where the manifest arrives before any data."""
|
||||
source = os.path.join(TEST_DATA_DIR, f"delexcl_{timing.strip('-')}_{mt}_src")
|
||||
clean_dir(source)
|
||||
entries = {
|
||||
"keep.txt": b"kept\n",
|
||||
"secret.log": b"secret\n",
|
||||
"sub/nested.log": b"nested secret\n",
|
||||
}
|
||||
for rel, content in entries.items():
|
||||
self._write(os.path.join(source, rel), content)
|
||||
dest = os.path.join(TEST_DATA_DIR, f"delexcl_{timing.strip('-')}_{mt}_dst")
|
||||
clean_dir(dest)
|
||||
with ServerManager() as server:
|
||||
server.start(extra_args=["--allow-delete"])
|
||||
result, _ = run_client(source, dest, port=server.port)
|
||||
assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}"
|
||||
received = get_dest_received_dir(dest, source)
|
||||
self._write(os.path.join(received, "extra.txt"), b"extra\n")
|
||||
|
||||
# Default: the excluded mirrors survive --delete, genuine extras die.
|
||||
flags = ["--exclude", "*.log", timing] + (["-m"] if mt else [])
|
||||
result, _ = run_client(source, dest, flags=flags, port=server.port)
|
||||
assert result.returncode == 0, \
|
||||
f"default delete sync failed: {(result.stderr or result.stdout)[:300]}"
|
||||
assert os.path.exists(os.path.join(received, "secret.log")), \
|
||||
"excluded dest file was deleted under plain --delete (rsync protects it)"
|
||||
assert os.path.exists(os.path.join(received, "sub", "nested.log")), \
|
||||
"nested excluded dest file was deleted under plain --delete"
|
||||
assert not os.path.exists(os.path.join(received, "extra.txt")), \
|
||||
"genuine extra was not deleted"
|
||||
|
||||
# --delete-excluded: excluded mirrors are extras again and die.
|
||||
self._write(os.path.join(received, "extra.txt"), b"extra\n")
|
||||
flags = ["--exclude", "*.log", timing, "--delete-excluded"] + (["-m"] if mt else [])
|
||||
result, _ = run_client(source, dest, flags=flags, port=server.port)
|
||||
assert result.returncode == 0, \
|
||||
f"--delete-excluded sync failed: {(result.stderr or result.stdout)[:300]}"
|
||||
assert not os.path.exists(os.path.join(received, "secret.log")), \
|
||||
"--delete-excluded did not remove the excluded dest file"
|
||||
assert not os.path.exists(os.path.join(received, "sub", "nested.log")), \
|
||||
"--delete-excluded did not remove the nested excluded dest file"
|
||||
assert not os.path.exists(os.path.join(received, "extra.txt")), \
|
||||
"genuine extra survived --delete-excluded"
|
||||
assert _read_file(os.path.join(received, "keep.txt")) == b"kept\n"
|
||||
|
||||
@pytest.mark.parametrize("mt", [False, True])
|
||||
def test_delete_excluded_excluded_directory_subtree(self, mt):
|
||||
"""A whole source directory excluded by a filter rule protects its whole
|
||||
destination mirror by default; --delete-excluded removes the subtree."""
|
||||
source = os.path.join(TEST_DATA_DIR, f"delexcldir_{mt}_src")
|
||||
clean_dir(source)
|
||||
self._write(os.path.join(source, "keep.txt"), b"kept\n")
|
||||
self._write(os.path.join(source, "skipdir", "a.log"), b"a\n")
|
||||
self._write(os.path.join(source, "skipdir", "deep", "b.log"), b"b\n")
|
||||
dest = os.path.join(TEST_DATA_DIR, f"delexcldir_{mt}_dst")
|
||||
clean_dir(dest)
|
||||
with ServerManager() as server:
|
||||
server.start(extra_args=["--allow-delete"])
|
||||
result, _ = run_client(source, dest, port=server.port)
|
||||
assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}"
|
||||
received = get_dest_received_dir(dest, source)
|
||||
|
||||
flags = ["--filter=- skipdir/", "--delete"] + (["-m"] if mt else [])
|
||||
result, _ = run_client(source, dest, flags=flags, port=server.port)
|
||||
assert result.returncode == 0, \
|
||||
f"default delete sync failed: {(result.stderr or result.stdout)[:300]}"
|
||||
assert os.path.exists(os.path.join(received, "skipdir", "a.log")), \
|
||||
"excluded dir subtree was deleted under plain --delete"
|
||||
assert os.path.exists(os.path.join(received, "skipdir", "deep", "b.log")), \
|
||||
"nested excluded dir content was deleted under plain --delete"
|
||||
|
||||
flags = ["--filter=- skipdir/", "--delete", "--delete-excluded"] + (["-m"] if mt else [])
|
||||
result, _ = run_client(source, dest, flags=flags, port=server.port)
|
||||
assert result.returncode == 0, \
|
||||
f"--delete-excluded sync failed: {(result.stderr or result.stdout)[:300]}"
|
||||
assert not os.path.exists(os.path.join(received, "skipdir")), \
|
||||
"--delete-excluded did not remove the excluded dir subtree"
|
||||
|
||||
@pytest.mark.parametrize("mt", [False, True])
|
||||
@pytest.mark.parametrize("timing", ["--delete", "--delete-before"])
|
||||
def test_max_delete_exceeded_fails_without_deleting(self, mt, timing):
|
||||
"""A run that would exceed --max-delete deletes nothing and fails."""
|
||||
source = os.path.join(TEST_DATA_DIR, f"maxdel_{timing.strip('-')}_{mt}_src")
|
||||
clean_dir(source)
|
||||
self._write(os.path.join(source, "keep.txt"), b"kept\n")
|
||||
dest = os.path.join(TEST_DATA_DIR, f"maxdel_{timing.strip('-')}_{mt}_dst")
|
||||
clean_dir(dest)
|
||||
with ServerManager() as server:
|
||||
server.start(extra_args=["--allow-delete"])
|
||||
result, _ = run_client(source, dest, port=server.port)
|
||||
assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}"
|
||||
received = get_dest_received_dir(dest, source)
|
||||
extras = []
|
||||
for i in range(4):
|
||||
name = f"e{i}.txt"
|
||||
self._write(os.path.join(received, name), b"extra\n")
|
||||
extras.append(os.path.join(received, name))
|
||||
|
||||
flags = ["--max-delete=2", timing] + (["-m"] if mt else [])
|
||||
result, _ = run_client(source, dest, flags=flags, port=server.port)
|
||||
assert result.returncode != 0, \
|
||||
f"--max-delete=2 with 4 extras unexpectedly succeeded: {result.stderr[:300]}"
|
||||
for path in extras:
|
||||
assert os.path.exists(path), \
|
||||
"--max-delete overrun deleted files (must be all-or-nothing)"
|
||||
|
||||
@pytest.mark.parametrize("mt", [False, True])
|
||||
def test_max_delete_not_exceeded_deletes_exactly(self, mt):
|
||||
"""When the extras are at or below --max-delete the run succeeds and
|
||||
removes exactly the extras."""
|
||||
source = os.path.join(TEST_DATA_DIR, f"maxdelok_{mt}_src")
|
||||
clean_dir(source)
|
||||
self._write(os.path.join(source, "keep.txt"), b"kept\n")
|
||||
dest = os.path.join(TEST_DATA_DIR, f"maxdelok_{mt}_dst")
|
||||
clean_dir(dest)
|
||||
with ServerManager() as server:
|
||||
server.start(extra_args=["--allow-delete"])
|
||||
result, _ = run_client(source, dest, port=server.port)
|
||||
assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}"
|
||||
received = get_dest_received_dir(dest, source)
|
||||
for i in range(3):
|
||||
self._write(os.path.join(received, f"e{i}.txt"), b"extra\n")
|
||||
flags = ["--max-delete=3", "--delete"] + (["-m"] if mt else [])
|
||||
result, _ = run_client(source, dest, flags=flags, port=server.port)
|
||||
assert result.returncode == 0, \
|
||||
f"--max-delete=3 with 3 extras failed: {(result.stderr or result.stdout)[:300]}"
|
||||
for i in range(3):
|
||||
assert not os.path.exists(os.path.join(received, f"e{i}.txt")), \
|
||||
f"extra e{i}.txt not deleted under --max-delete=3"
|
||||
|
||||
@pytest.mark.parametrize("mt", [False, True])
|
||||
def test_force_replaces_nonempty_dir_with_file(self, mt):
|
||||
"""--force lets an incoming regular file replace a non-empty destination
|
||||
directory; without it the write (and the run) fails."""
|
||||
source = os.path.join(TEST_DATA_DIR, f"force_{mt}_src")
|
||||
clean_dir(source)
|
||||
self._write(os.path.join(source, "sub", "old.txt"), b"old\n")
|
||||
self._write(os.path.join(source, "keep.txt"), b"kept\n")
|
||||
dest = os.path.join(TEST_DATA_DIR, f"force_{mt}_dst")
|
||||
clean_dir(dest)
|
||||
with ServerManager() as server:
|
||||
server.start(extra_args=["--allow-delete"])
|
||||
result, _ = run_client(source, dest, port=server.port)
|
||||
assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}"
|
||||
received = get_dest_received_dir(dest, source)
|
||||
|
||||
# The source path `sub` becomes a regular file (the dir is gone).
|
||||
os.unlink(os.path.join(source, "sub", "old.txt"))
|
||||
os.rmdir(os.path.join(source, "sub"))
|
||||
self._write(os.path.join(source, "sub"), b"now a file\n")
|
||||
|
||||
result, _ = run_client(source, dest, port=server.port)
|
||||
assert result.returncode != 0, \
|
||||
"a file over a non-empty directory must fail without --force"
|
||||
assert os.path.isdir(os.path.join(received, "sub")), \
|
||||
"directory was destroyed although the run failed without --force"
|
||||
assert os.path.exists(os.path.join(received, "sub", "old.txt")), \
|
||||
"non-empty dir content was lost although the run failed without --force"
|
||||
|
||||
flags = ["--force"] + (["-m"] if mt else [])
|
||||
result, _ = run_client(source, dest, flags=flags, port=server.port)
|
||||
assert result.returncode == 0, \
|
||||
f"--force run failed: {(result.stderr or result.stdout)[:300]}"
|
||||
assert os.path.isfile(os.path.join(received, "sub")), \
|
||||
"--force did not replace the directory with the file"
|
||||
assert _read_file(os.path.join(received, "sub")) == b"now a file\n"
|
||||
assert not os.path.exists(os.path.join(received, "sub", "old.txt")), \
|
||||
"--force left the old directory content behind"
|
||||
|
||||
@pytest.mark.parametrize("mt", [False, True])
|
||||
def test_prune_empty_dirs_dirs_mode(self, mt):
|
||||
"""--prune-empty-dirs omits an empty source directory's explicit entry in
|
||||
--dirs mode (nothing is created, and an existing empty mirror is removed
|
||||
by --delete). Recursive transfers never emit empty dirs, so the flag is
|
||||
a no-op there (documented rsync -m parity)."""
|
||||
source = os.path.join(TEST_DATA_DIR, f"prune_{mt}_src")
|
||||
clean_dir(source)
|
||||
os.makedirs(source, exist_ok=True) # physically empty source dir
|
||||
|
||||
dest = os.path.join(TEST_DATA_DIR, f"prune_{mt}_dst")
|
||||
clean_dir(dest)
|
||||
with ServerManager() as server:
|
||||
server.start(extra_args=["--allow-delete"])
|
||||
result, _ = run_client(source, dest, flags=["--dirs"], port=server.port)
|
||||
assert result.returncode == 0, f"-d seed failed: {result.stderr[:200]}"
|
||||
received = get_dest_received_dir(dest, source)
|
||||
assert os.path.isdir(received), "-d should create the empty mirror dir"
|
||||
assert os.listdir(received) == []
|
||||
|
||||
# prune-empty-dirs: the empty mirror is pruned by --delete.
|
||||
flags = ["--dirs", "--prune-empty-dirs", "--delete"] + (["-m"] if mt else [])
|
||||
result, _ = run_client(source, dest, flags=flags, port=server.port)
|
||||
assert result.returncode == 0, \
|
||||
f"--dirs --prune-empty-dirs --delete failed: {(result.stderr or result.stdout)[:300]}"
|
||||
assert not os.path.exists(received), \
|
||||
"--prune-empty-dirs did not prune the empty dir (--delete left it)"
|
||||
|
||||
# A fresh destination: prune-empty-dirs means the empty dir is never sent.
|
||||
dest2 = os.path.join(TEST_DATA_DIR, f"prune2_{mt}_dst")
|
||||
clean_dir(dest2)
|
||||
with ServerManager() as server:
|
||||
server.start(extra_args=["--allow-delete"])
|
||||
flags = ["--dirs", "--prune-empty-dirs", "-i"] + (["-m"] if mt else [])
|
||||
result, _ = run_client(source, dest2, flags=flags, port=server.port)
|
||||
assert result.returncode == 0, \
|
||||
f"--dirs --prune-empty-dirs failed: {(result.stderr or result.stdout)[:300]}"
|
||||
received2 = get_dest_received_dir(dest2, source)
|
||||
assert not os.path.exists(received2), \
|
||||
"--prune-empty-dirs transferred the empty directory"
|
||||
assert result.stdout == "", \
|
||||
f"--prune-empty-dirs leaked an itemize line: {result.stdout[:200]}"
|
||||
|
||||
@pytest.mark.parametrize("mt", [False, True])
|
||||
def test_prune_empty_dirs_recursion_inherent(self, mt):
|
||||
"""In recursive mode FastSync never transfers empty directories (rsync
|
||||
-m parity): a truly-empty destination directory chain is removed by
|
||||
--delete whether or not --prune-empty-dirs is given (the flag has no
|
||||
additional effect there), while directories holding kept files survive.
|
||||
A filter-excluded file's mirror is protected, so a directory that still
|
||||
holds one is left intact (rsync default delete-excluded semantics)."""
|
||||
source = os.path.join(TEST_DATA_DIR, f"prunerec_{mt}_src")
|
||||
clean_dir(source)
|
||||
self._write(os.path.join(source, "keep.txt"), b"kept\n")
|
||||
self._write(os.path.join(source, "a", "keep.log"), b"a log\n")
|
||||
self._write(os.path.join(source, "b", "deep", "kept.txt"), b"deep kept\n")
|
||||
dest = os.path.join(TEST_DATA_DIR, f"prunerec_{mt}_dst")
|
||||
clean_dir(dest)
|
||||
with ServerManager() as server:
|
||||
server.start(extra_args=["--allow-delete"])
|
||||
result, _ = run_client(source, dest, port=server.port)
|
||||
assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}"
|
||||
received = get_dest_received_dir(dest, source)
|
||||
# A stray empty chain (FastSync recursion never creates such dirs, so
|
||||
# this models one left by an external tool / an earlier --dirs run).
|
||||
os.makedirs(os.path.join(received, "empty", "chain"))
|
||||
|
||||
for prune in ([], ["--prune-empty-dirs"]):
|
||||
flags = prune + ["--delete"] + (["-m"] if mt else [])
|
||||
result, _ = run_client(source, dest, flags=flags, port=server.port)
|
||||
assert result.returncode == 0, \
|
||||
f"prune recursive sync failed: {(result.stderr or result.stdout)[:300]}"
|
||||
assert not os.path.exists(os.path.join(received, "empty")), \
|
||||
"truly-empty dir chain was not removed by --delete"
|
||||
assert os.path.exists(os.path.join(received, "b", "deep", "kept.txt")), \
|
||||
"non-empty dir subtree was wrongly removed"
|
||||
assert _read_file(os.path.join(received, "keep.txt")) == b"kept\n"
|
||||
|
||||
# An excluded file's mirror is protected: the dir that holds it stays.
|
||||
flags = ["--exclude", "*.log", "--delete", "--prune-empty-dirs"] + (["-m"] if mt else [])
|
||||
result, _ = run_client(source, dest, flags=flags, port=server.port)
|
||||
assert result.returncode == 0, \
|
||||
f"prune recursive sync failed: {(result.stderr or result.stdout)[:300]}"
|
||||
assert os.path.exists(os.path.join(received, "a", "keep.log")), \
|
||||
"excluded file mirror was deleted under --delete (rsync protects it)"
|
||||
|
||||
def test_ignore_errors_keeps_deletion_active_on_scan_error(self):
|
||||
"""A source I/O error (unreadable directory) aborts the run so no
|
||||
deletion happens by default; --ignore-errors continues, still transfers
|
||||
the readable tree and still deletes. Run as an unprivileged user so the
|
||||
mode-000 directory is genuinely unreadable."""
|
||||
if os.geteuid() != 0 or shutil.which("setpriv") is None:
|
||||
pytest.skip("requires root + setpriv to drop privileges for the client")
|
||||
tag = f"ioerr_{os.getpid()}"
|
||||
source = os.path.join(TEST_DATA_DIR, f"{tag}_src")
|
||||
clean_dir(source)
|
||||
self._write(os.path.join(source, "top.txt"), b"top\n")
|
||||
self._write(os.path.join(source, "ok", "inside.txt"), b"inside\n")
|
||||
self._write(os.path.join(source, "locked", "blocked.txt"), b"blocked\n")
|
||||
dest = os.path.join(TEST_DATA_DIR, f"{tag}_dst")
|
||||
clean_dir(dest)
|
||||
with ServerManager() as server:
|
||||
server.start(extra_args=["--allow-delete"])
|
||||
# Seed as root (server is root too).
|
||||
result, _ = run_client(source, dest, port=server.port)
|
||||
assert result.returncode == 0, f"seed sync failed: {result.stderr[:200]}"
|
||||
received = get_dest_received_dir(dest, source)
|
||||
try:
|
||||
os.chmod(os.path.join(source, "locked"), 0)
|
||||
|
||||
# Default: scan error aborts the run; nothing is deleted.
|
||||
self._write(os.path.join(received, "extra.txt"), b"extra\n")
|
||||
cmd = CLIENT_CMD + ["--source-dir", source, "--dest-dir", dest,
|
||||
"--save-to-disk", "--server-port", str(server.port),
|
||||
"--delete"]
|
||||
result = subprocess.run(["setpriv", "--reuid=65534", "--regid=65534",
|
||||
"--clear-groups"] + cmd, text=True, capture_output=True)
|
||||
assert result.returncode != 0, "unreadable source dir did not fail the run"
|
||||
assert os.path.exists(os.path.join(received, "extra.txt")), \
|
||||
"default run deleted although the scan hit an I/O error"
|
||||
|
||||
# --ignore-errors: the readable tree transfers, deletion still runs.
|
||||
self._write(os.path.join(received, "extra.txt"), b"extra\n")
|
||||
cmd = CLIENT_CMD + ["--source-dir", source, "--dest-dir", dest,
|
||||
"--save-to-disk", "--server-port", str(server.port),
|
||||
"--delete", "--ignore-errors"]
|
||||
result = subprocess.run(["setpriv", "--reuid=65534", "--regid=65534",
|
||||
"--clear-groups"] + cmd, text=True, capture_output=True)
|
||||
assert not os.path.exists(os.path.join(received, "extra.txt")), \
|
||||
f"--ignore-errors did not keep deletion active: {result.stderr[:300]}"
|
||||
assert not os.path.exists(os.path.join(received, "locked")), \
|
||||
"mirror of the unreadable dir was not treated as an extra"
|
||||
finally:
|
||||
os.chmod(os.path.join(source, "locked"), 0o755)
|
||||
|
||||
|
||||
def _pin_mtime(path, ts):
|
||||
os.utime(path, (ts, ts))
|
||||
|
||||
|
||||
Reference in New Issue
Block a user