TapTap/FastSync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

feat(progress): opt-in paths-only pre-count for rsync to-chk parity

Browse Source 
- when --progress/--info=progress is requested, a metadata-only pre-scan
  builds the full file-list total and directory names so the to-chk
  denominator counts every regular/dir/link/special entry like rsync
- per-directory/symlink/special name lines emitted; sequential and --threads
- reuses the --delete-during/delay pre-scan when present; non-progress runs
  take no extra pass
- --progress stays a caveat (emission order still differs); single-file output
  remains byte-identical
This commit is contained in:
TapTap
2026-09-19 12:49:56 +02:00
parent 6119e1e75c
commit c80098623f
7 changed files with 535 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/integration/parity_harness.py
+55
View File
@@ -54,6 +54,7 @@ STDOUT_NONE = None
STDOUT_ITEMIZE = "itemize"
STDOUT_OUTFMT = "outfmt"
STDOUT_STATS = "stats"
STDOUT_PROGRESS = "progress"
# rsync --stats lines that are protocol-independent and must match exactly.
# `Number of files` and `Number of created files` carry rsync's per-type
@@ -73,6 +74,8 @@ STATS_KEYS = (
)
_ITEMIZE_RE = re.compile(r"^(<|>|c|h|\.|\*)[fdLDS][.+\-][.+\-][.+\-][.+\-]")
_PROGRESS_TOTAL_RE = re.compile(r"to-chk=\d+/(\d+)")
_PROGRESS_XFR_RE = re.compile(r"xfr#(\d+)")
@dataclass
@@ -184,6 +187,24 @@ def corpus_empty_dir(root: str) -> None:
_write(os.path.join(root, "nonempty", "f.txt"), b"f\n")
def corpus_multidir(root: str) -> None:
"""Multi-directory tree for the --progress file-list naming/denominator.
Nested files, a directory-only branch, an empty directory and a symlink
exercise every file-list entry type rsync counts in `to-chk` but FastSync's
streaming scanner never emits as a transfer entry.
"""
clean_dir(root)
_write(os.path.join(root, "a.txt"), b"alpha\n")
_write(os.path.join(root, "b.txt"), b"bravo\n")
_write(os.path.join(root, "sub1", "c.txt"), b"charlie\n")
_write(os.path.join(root, "sub1", "deep", "d.txt"), b"delta\n")
_write(os.path.join(root, "sub2", "e.txt"), b"echo\n")
os.symlink("a.txt", os.path.join(root, "link1"))
os.makedirs(os.path.join(root, "emptydir"), exist_ok=True)
os.utime(os.path.join(root, "emptydir"), (_SRC_MTIME, _SRC_MTIME))
def corpus_relative(root: str) -> None:
"""Tree for the -R/--files-from cases."""
clean_dir(root)
@@ -216,6 +237,7 @@ CORPORA: Dict[str, Callable[[str], None]] = {
"sparse": corpus_sparse,
"filters": corpus_filters,
"empty_dir": corpus_empty_dir,
"multidir": corpus_multidir,
"relative": corpus_relative,
"iconv": corpus_iconv,
}
@@ -382,6 +404,37 @@ def normalize_stdout(text: str, mode: Optional[str]) -> object:
if line.startswith(key + ":"):
found[key] = _parse_bytes(line.split(":", 1)[1])
return found
if mode == STDOUT_PROGRESS:
# rsync prints the file-list entries in sorted depth-first order while
# FastSync's streaming scan emits them in readdir/BFS order; only the
# entry set and deterministic fields are compared. The transfer-root
# `./` line's trigger condition is a separate documented residual, and
# the per-frame rate/elapsed/xfr#/to-chk numerator are wall-clock- or
# order-dependent, so only the `to-chk` denominator and the name set are
# asserted.
names = []
totals = set()
max_xfr = 0
for line in (text or "").splitlines():
line = line.rstrip()
if not line:
continue
if "%" in line:
m = _PROGRESS_TOTAL_RE.search(line)
if m:
totals.add(int(m.group(1)))
mx = _PROGRESS_XFR_RE.search(line)
if mx:
max_xfr = max(max_xfr, int(mx.group(1)))
continue
if line == "sending incremental file list":
continue
if line.startswith("created directory "):
continue
if line == "./":
continue
names.append(line)
return {"names": sorted(names), "total": sorted(totals), "xfr": max_xfr}
# raw
return sorted(l.rstrip() for l in (text or "").splitlines() if l.strip())
@@ -393,6 +446,8 @@ def stdout_diff(rsync_out: str, fs_out: str, mode: Optional[str]) -> List[str]:
return []
if mode == STDOUT_STATS:
return [f"stats rsync={r}", f"stats fastsync={f}"]
if mode == STDOUT_PROGRESS:
return [f"progress rsync={r}", f"progress fastsync={f}"]
return list(difflib.unified_diff(
[str(x) for x in r], [str(x) for x in f],
fromfile="rsync", tofile="fastsync", lineterm=""))
tests/integration/test_differential_parity.py
+6
View File
@@ -173,6 +173,12 @@ _CASES = [
stdout=H.STDOUT_OUTFMT, ref="--out-format %n %l"),
H.Case("out_format_i_n", "basic", ["-a", "--out-format=%i %n"],
stdout=H.STDOUT_OUTFMT, ref="--out-format %i %n"),
H.Case("progress", "multidir", ["-a", "--progress"], stdout=H.STDOUT_PROGRESS,
ci=True, ref="--progress multi-directory file list"),
H.Case("progress_threads", "multidir", ["-a", "--progress"],
fastsync_flags=["-a", "--progress", "--threads"],
stdout=H.STDOUT_PROGRESS, ci=True,
ref="--progress multi-directory file list (--threads)"),
# --- transfer modifications -------------------------------------------
H.Case("update", "basic", ["-a", "--update"], seed=seed_update,
tests/integration/test_output_parity.py
+90
View File
@@ -289,6 +289,40 @@ def _make_one_file(root, name="f.bin", size=100):
fh.write(bytes((i * 7 + 3) & 0xFF for i in range(size)))
def _make_multidir_tree(root):
"""Multi-directory corpus for the --progress file-list tests: nested files,
a directory-only branch, an empty directory and a symlink."""
clean_dir(root)
for rel, data in (("a.txt", b"alpha\n"), ("b.txt", b"bravo\n"),
("sub1/c.txt", b"charlie\n"), ("sub1/deep/d.txt", b"delta\n"),
("sub2/e.txt", b"echo\n")):
path = os.path.join(root, rel)
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "wb") as fh:
fh.write(data)
os.symlink("a.txt", os.path.join(root, "link1"))
os.makedirs(os.path.join(root, "emptydir"), exist_ok=True)
def _parse_progress(text):
"""Name lines and the `to-chk` denominators from a --progress run."""
names = []
totals = set()
for line in text.splitlines():
line = line.rstrip()
if not line or line == "sending incremental file list":
continue
if "%" in line:
match = re.search(r"to-chk=\d+/(\d+)", line)
if match:
totals.add(int(match.group(1)))
continue
if line == "./": # root-line trigger is a separate documented residual
continue
names.append(line)
return sorted(names), totals
def _pick_stats(text, keys):
out = {}
for line in text.splitlines():
@@ -493,6 +527,62 @@ class TestWireStatsParity:
assert "to-chk=0/2" in fast_lines[-1], fast_lines[-1]
assert fast_lines[-1] == rsync_lines[-1], (rsync_lines[-1], fast_lines[-1])
@requires_rsync
@pytest.mark.ci
@pytest.mark.parametrize("mt", [False, True])
def test_progress_multidir_file_list_matches_rsync(self, shared_server, mt):
"""A multi-directory tree: the paths-only pre-count must reproduce
rsync's file-list set and `to-chk` denominator. Per-directory name
lines are emitted for directories, symlinks and the empty directory; the
name set and the denominator (every entry plus the transfer root) match
rsync, while the emitted *order* remains a documented residual (rsync
sorts depth-first, FastSync streams in readdir/BFS order)."""
source = os.path.join(TEST_DATA_DIR, "wire_pgmd_src")
dest = os.path.join(TEST_DATA_DIR, "wire_pgmd_dst")
rdst = os.path.join(TEST_DATA_DIR, "wire_pgmd_rdst")
_make_multidir_tree(source)
clean_dir(dest)
clean_dir(rdst)
rsync_result = _rsync(["-a", "--progress", source + "/", rdst + "/"])
assert rsync_result.returncode == 0, rsync_result.stderr
flags = ["-a", "--progress"] + (["--threads"] if mt else [])
result, _ = run_client(source, dest, flags=flags, port=shared_server.port)
assert result.returncode == 0, result.stderr[:300]
rsync_names, rsync_totals = _parse_progress(rsync_result.stdout)
fast_names, fast_totals = _parse_progress(result.stdout)
assert sorted(rsync_names) == [
"a.txt", "b.txt", "emptydir/", "link1 -> a.txt", "sub1/",
"sub1/c.txt", "sub1/deep/", "sub1/deep/d.txt", "sub2/", "sub2/e.txt",
], rsync_names
assert fast_names == rsync_names, (rsync_names, fast_names)
# 10 entries + the transfer-root "." counted by rsync's file list.
assert rsync_totals == {11}, rsync_totals
assert fast_totals == rsync_totals, (rsync_totals, fast_totals)
@pytest.mark.ci
def test_progress_delete_during_reuses_pre_scan(self):
"""--delete-during + --progress reuses the keep-set pre-scan instead of
walking the tree a second time: the file-list total and directory name
lines are identical to a plain --progress run."""
source = os.path.join(TEST_DATA_DIR, "wire_pgdel_src")
dest = os.path.join(TEST_DATA_DIR, "wire_pgdel_dst")
_make_multidir_tree(source)
clean_dir(dest)
server = ServerManager()
server.start(extra_args=["--allow-super", "--allow-delete"])
try:
result, _ = run_client(source, dest, flags=["-a", "--progress", "--delete-during"],
port=server.port)
finally:
server.stop()
assert result.returncode == 0, result.stderr[:300]
names, totals = _parse_progress(result.stdout)
assert totals == {11}, totals
assert "sub1/" in names and "sub1/deep/" in names and "emptydir/" in names, names
assert "link1 -> a.txt" in names, names
@requires_rsync
@pytest.mark.ci
@pytest.mark.parametrize("mt", [False, True])
tests/test_scanner.c
+50
View File
@@ -1240,6 +1240,55 @@ static int collect_scan_info_parallel(ParallelScanner* scanner, const char* root
return failed ? -1 : count;
}
/* The --progress paths-only pre-count relies on `list_dirs` emitting every
* directory (including empty ones) exactly once, alongside the files and
* symlinks the streaming scanner already emits. */
static void test_scanner_list_dirs_counts_every_entry() {
const char* root = "test_scan_listdirs";
EXPECT_EQ_INT(mkdir(root, 0755), 0);
EXPECT_EQ_INT(mkdir("test_scan_listdirs/sub1", 0755), 0);
EXPECT_EQ_INT(mkdir("test_scan_listdirs/sub1/deep", 0755), 0);
EXPECT_EQ_INT(mkdir("test_scan_listdirs/sub2", 0755), 0);
EXPECT_EQ_INT(mkdir("test_scan_listdirs/emptydir", 0755), 0);
create_test_file("test_scan_listdirs/a.txt", "a");
create_test_file("test_scan_listdirs/sub1/c.txt", "c");
create_test_file("test_scan_listdirs/sub1/deep/d.txt", "d");
create_test_file("test_scan_listdirs/sub2/e.txt", "e");
EXPECT_EQ_INT(symlink("a.txt", "test_scan_listdirs/link1"), 0);
ScannerOptions options = {0};
options.list_dirs = true;
options.emit_empty_dirs = true;
options.follow_symlinks = true; /* -l/--links: carry symlinks, don't skip */
ScanInfo infos[16];
int count = collect_scan_info(root, &options, infos, 16);
EXPECT_EQ_INT(count, 9);
int dirs = 0;
for (int i = 0; i < count; i++) {
if (infos[i].is_dir)
dirs++;
}
EXPECT_EQ_INT(dirs, 4);
EXPECT_TRUE(scan_info_present(infos, count, "sub1", true, NULL));
EXPECT_TRUE(scan_info_present(infos, count, "sub1/deep", true, NULL));
EXPECT_TRUE(scan_info_present(infos, count, "sub2", true, NULL));
EXPECT_TRUE(scan_info_present(infos, count, "emptydir", true, NULL));
EXPECT_TRUE(scan_info_present(infos, count, "a.txt", false, ""));
EXPECT_TRUE(scan_info_present(infos, count, "sub1/c.txt", false, ""));
EXPECT_TRUE(scan_info_present(infos, count, "link1", false, ""));
unlink("test_scan_listdirs/a.txt");
unlink("test_scan_listdirs/sub1/c.txt");
unlink("test_scan_listdirs/sub1/deep/d.txt");
unlink("test_scan_listdirs/sub2/e.txt");
unlink("test_scan_listdirs/link1");
rmdir("test_scan_listdirs/sub1/deep");
rmdir("test_scan_listdirs/sub1");
rmdir("test_scan_listdirs/sub2");
rmdir("test_scan_listdirs/emptydir");
rmdir(root);
}
/* -d without --files-from emits exactly the source-root directory (empty) and
* never descends. */
static void test_dirs_no_descent() {
@@ -1667,6 +1716,7 @@ void test_scanner() {
test_per_dir_filter_override(true);
test_dirs_no_descent();
test_dirs_files_from();
test_scanner_list_dirs_counts_every_entry();
test_files_from_relative_send_path();
test_scanner_captures_directory_times();
test_scanner_chunk_ownership();