scanner: emit entries in rsync's sorted depth-first flist order
Buffer and sort each directory's inspected entries (non-directories ascending, then directories ascending) and walk them depth-first via a LIFO directory stack, so the sequential scanner's stream matches rsync 3.4.1's flist order. This makes the --info=name transfer order and the --delete-during/--delete-delay deletion sequence byte-identical to rsync (differential tests in test_parity_order.py); --threads stays unordered (no rsync analogue) and is documented as such. Adds LIFO queue_push/queue_pop over the existing ring buffer.
This commit is contained in:
@@ -0,0 +1,135 @@
|
||||
"""Differential rsync-parity coverage for FastSync's transfer/delete ORDER.
|
||||
|
||||
rsync walks a source tree in its sorted flist order: within each directory the
|
||||
non-directories come first (ascending name), then the subdirectories (ascending
|
||||
name), each subdirectory immediately followed by its own subtree (depth-first).
|
||||
The sequential scanner now reproduces that order, which makes both the
|
||||
``--info=name`` stream and the ``--delete-during`` deletion sequence match real
|
||||
``rsync 3.4.1`` exactly. ``--threads`` has no rsync analogue and is unordered.
|
||||
|
||||
Every test skips cleanly when rsync is absent.
|
||||
"""
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from common import ( # noqa: E402
|
||||
TEST_DATA_DIR,
|
||||
ServerManager,
|
||||
clean_dir,
|
||||
get_dest_received_dir,
|
||||
run_client,
|
||||
)
|
||||
|
||||
RSYNC = shutil.which("rsync")
|
||||
requires_rsync = pytest.mark.skipif(RSYNC is None, reason="rsync 3.4.1 not installed")
|
||||
|
||||
MTIME = 1_500_000_000
|
||||
|
||||
_TREE = {
|
||||
"a.txt": b"a\n",
|
||||
"b.txt": b"b\n",
|
||||
"z.txt": b"z\n",
|
||||
"a_dir/f.txt": b"f\n",
|
||||
"a_dir/deep/g.txt": b"g\n",
|
||||
"m_dir/h.txt": b"h\n",
|
||||
"Z_dir/i.txt": b"i\n",
|
||||
}
|
||||
|
||||
|
||||
def _write(path, data):
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
with open(path, "wb") as fh:
|
||||
fh.write(data)
|
||||
os.utime(path, (MTIME, MTIME))
|
||||
|
||||
|
||||
def _rsync(args):
|
||||
env = dict(os.environ, LC_ALL="C")
|
||||
return subprocess.run([RSYNC] + args, capture_output=True, text=True, env=env, timeout=120)
|
||||
|
||||
|
||||
def _deleting(text):
|
||||
out = []
|
||||
for line in text.splitlines():
|
||||
stripped = line.strip()
|
||||
if stripped.startswith("*deleting") or stripped.startswith("deleting"):
|
||||
out.append(stripped.split()[-1])
|
||||
return out
|
||||
|
||||
|
||||
class TestTransferOrderParity:
|
||||
@requires_rsync
|
||||
def test_info_name_file_order_matches_rsync(self, shared_server):
|
||||
source = os.path.join(TEST_DATA_DIR, "order_name_src")
|
||||
clean_dir(source)
|
||||
for rel, data in _TREE.items():
|
||||
_write(os.path.join(source, rel), data)
|
||||
|
||||
rdst = os.path.join(TEST_DATA_DIR, "order_name_rdst")
|
||||
clean_dir(rdst)
|
||||
rs = _rsync(["-a", "--info=name", source + "/", rdst + "/"])
|
||||
assert rs.returncode == 0, rs.stderr
|
||||
# rsync also names the directories (trailing '/'); FastSync names the
|
||||
# transferred entries. Compare the file/symlink sequence, which is what
|
||||
# the traversal order determines.
|
||||
rsync_files = [l for l in rs.stdout.splitlines() if l.strip() and not l.endswith("/")]
|
||||
|
||||
fdst = os.path.join(TEST_DATA_DIR, "order_name_fdst")
|
||||
clean_dir(fdst)
|
||||
result, _ = run_client(source, fdst, flags=["-a", "--info=name"],
|
||||
port=shared_server.port)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||
fsync_files = [
|
||||
l for l in result.stdout.splitlines()
|
||||
if l.strip() and l.strip() != "./" and not l.startswith("sending")
|
||||
]
|
||||
assert fsync_files == rsync_files, (
|
||||
f"transfer order differs\nrsync={rsync_files}\nfastsync={fsync_files}")
|
||||
|
||||
|
||||
class TestDeleteOrderParity:
|
||||
_EXTRA = {
|
||||
"a_extra.txt": b"a\n",
|
||||
"z_extra.txt": b"z\n",
|
||||
"a_extra_dir/f": b"f\n",
|
||||
"z_extra_dir/f": b"f\n",
|
||||
"a_extra_dir/sub/g": b"g\n",
|
||||
}
|
||||
|
||||
@requires_rsync
|
||||
def test_delete_during_deletion_order_matches_rsync(self):
|
||||
source = os.path.join(TEST_DATA_DIR, "order_del_src")
|
||||
clean_dir(source)
|
||||
_write(os.path.join(source, "keep.txt"), b"k\n")
|
||||
_write(os.path.join(source, "keepdir", "x.txt"), b"x\n")
|
||||
_write(os.path.join(source, "keep2", "y.txt"), b"y\n")
|
||||
|
||||
rdst = os.path.join(TEST_DATA_DIR, "order_del_rdst")
|
||||
clean_dir(rdst)
|
||||
for rel, data in self._EXTRA.items():
|
||||
_write(os.path.join(rdst, rel), data)
|
||||
rs = _rsync(["-a", "--delete-during", "--info=del", source + "/", rdst + "/"])
|
||||
assert rs.returncode == 0, rs.stderr
|
||||
|
||||
fdst = os.path.join(TEST_DATA_DIR, "order_del_fdst")
|
||||
clean_dir(fdst)
|
||||
received = get_dest_received_dir(fdst, source)
|
||||
for rel, data in self._EXTRA.items():
|
||||
_write(os.path.join(received, rel), data)
|
||||
with ServerManager() as server:
|
||||
server.start(extra_args=["--allow-delete"])
|
||||
result, _ = run_client(source, fdst, flags=["-a", "--delete-during", "--info=del"],
|
||||
port=server.port)
|
||||
assert result.returncode == 0, (result.stderr or result.stdout)[:300]
|
||||
|
||||
rsync_order = _deleting(rs.stdout)
|
||||
fsync_order = _deleting(result.stdout)
|
||||
assert sorted(fsync_order) == sorted(rsync_order), (
|
||||
f"deleted set differs\nrsync={rsync_order}\nfastsync={fsync_order}")
|
||||
assert fsync_order == rsync_order, (
|
||||
f"deletion order differs\nrsync={rsync_order}\nfastsync={fsync_order}")
|
||||
Reference in New Issue
Block a user