feat(progress): opt-in paths-only pre-count for rsync to-chk parity

- when --progress/--info=progress is requested, a metadata-only pre-scan
  builds the full file-list total and directory names so the to-chk
  denominator counts every regular/dir/link/special entry like rsync
- per-directory/symlink/special name lines emitted; sequential and --threads
- reuses the --delete-during/delay pre-scan when present; non-progress runs
  take no extra pass
- --progress stays a caveat (emission order still differs); single-file output
  remains byte-identical
This commit is contained in:
2026-09-19 12:49:56 +02:00
parent 6119e1e75c
commit c80098623f
7 changed files with 535 additions and 19 deletions
+55
View File
@@ -54,6 +54,7 @@ STDOUT_NONE = None
STDOUT_ITEMIZE = "itemize"
STDOUT_OUTFMT = "outfmt"
STDOUT_STATS = "stats"
STDOUT_PROGRESS = "progress"
# rsync --stats lines that are protocol-independent and must match exactly.
# `Number of files` and `Number of created files` carry rsync's per-type
@@ -73,6 +74,8 @@ STATS_KEYS = (
)
_ITEMIZE_RE = re.compile(r"^(<|>|c|h|\.|\*)[fdLDS][.+\-][.+\-][.+\-][.+\-]")
_PROGRESS_TOTAL_RE = re.compile(r"to-chk=\d+/(\d+)")
_PROGRESS_XFR_RE = re.compile(r"xfr#(\d+)")
@dataclass
@@ -184,6 +187,24 @@ def corpus_empty_dir(root: str) -> None:
_write(os.path.join(root, "nonempty", "f.txt"), b"f\n")
def corpus_multidir(root: str) -> None:
"""Multi-directory tree for the --progress file-list naming/denominator.
Nested files, a directory-only branch, an empty directory and a symlink
exercise every file-list entry type rsync counts in `to-chk` but FastSync's
streaming scanner never emits as a transfer entry.
"""
clean_dir(root)
_write(os.path.join(root, "a.txt"), b"alpha\n")
_write(os.path.join(root, "b.txt"), b"bravo\n")
_write(os.path.join(root, "sub1", "c.txt"), b"charlie\n")
_write(os.path.join(root, "sub1", "deep", "d.txt"), b"delta\n")
_write(os.path.join(root, "sub2", "e.txt"), b"echo\n")
os.symlink("a.txt", os.path.join(root, "link1"))
os.makedirs(os.path.join(root, "emptydir"), exist_ok=True)
os.utime(os.path.join(root, "emptydir"), (_SRC_MTIME, _SRC_MTIME))
def corpus_relative(root: str) -> None:
"""Tree for the -R/--files-from cases."""
clean_dir(root)
@@ -216,6 +237,7 @@ CORPORA: Dict[str, Callable[[str], None]] = {
"sparse": corpus_sparse,
"filters": corpus_filters,
"empty_dir": corpus_empty_dir,
"multidir": corpus_multidir,
"relative": corpus_relative,
"iconv": corpus_iconv,
}
@@ -382,6 +404,37 @@ def normalize_stdout(text: str, mode: Optional[str]) -> object:
if line.startswith(key + ":"):
found[key] = _parse_bytes(line.split(":", 1)[1])
return found
if mode == STDOUT_PROGRESS:
# rsync prints the file-list entries in sorted depth-first order while
# FastSync's streaming scan emits them in readdir/BFS order; only the
# entry set and deterministic fields are compared. The transfer-root
# `./` line's trigger condition is a separate documented residual, and
# the per-frame rate/elapsed/xfr#/to-chk numerator are wall-clock- or
# order-dependent, so only the `to-chk` denominator and the name set are
# asserted.
names = []
totals = set()
max_xfr = 0
for line in (text or "").splitlines():
line = line.rstrip()
if not line:
continue
if "%" in line:
m = _PROGRESS_TOTAL_RE.search(line)
if m:
totals.add(int(m.group(1)))
mx = _PROGRESS_XFR_RE.search(line)
if mx:
max_xfr = max(max_xfr, int(mx.group(1)))
continue
if line == "sending incremental file list":
continue
if line.startswith("created directory "):
continue
if line == "./":
continue
names.append(line)
return {"names": sorted(names), "total": sorted(totals), "xfr": max_xfr}
# raw
return sorted(l.rstrip() for l in (text or "").splitlines() if l.strip())
@@ -393,6 +446,8 @@ def stdout_diff(rsync_out: str, fs_out: str, mode: Optional[str]) -> List[str]:
return []
if mode == STDOUT_STATS:
return [f"stats rsync={r}", f"stats fastsync={f}"]
if mode == STDOUT_PROGRESS:
return [f"progress rsync={r}", f"progress fastsync={f}"]
return list(difflib.unified_diff(
[str(x) for x in r], [str(x) for x in f],
fromfile="rsync", tofile="fastsync", lineterm=""))