From 5aca91ab22eaa4124f7705eb6b4a624d070e4014 Mon Sep 17 00:00:00 2001 From: TapTap Date: Sun, 13 Sep 2026 10:14:37 +0200 Subject: [PATCH] fix(benchmark): use real FastSync flags and Release builds The benchmark tool used stale rsync-style spellings that map to different FastSync options, so it never enabled the features it claimed to measure: -c -> --checksum (not compression) -m -> --prune-empty-dirs (not multithreading) -s -> --secluded-args, a no-op (not chunk serialization) -f -> --filter, needs an argument (not sendfile) Replace them with the real flags (-z, -j, --chunk-serialization, --sendfile), force CMAKE_BUILD_TYPE=Release, route informational output to stderr so --output json emits valid JSON, surface client/rsync failures instead of silently dropping them, and widen the results table for the longer config names. Update the benchmark skill to match (correct flags, server invocation, and replace the nonexistent test.py --full with benchmark/bench.py). --- .opencode/skills/benchmark/SKILL.md | 55 ++++++++++++++++++----------- benchmark/bench.py | 50 +++++++++++++++----------- 2 files changed, 63 insertions(+), 42 deletions(-) diff --git a/.opencode/skills/benchmark/SKILL.md b/.opencode/skills/benchmark/SKILL.md index aed3533..fb91be0 100644 --- a/.opencode/skills/benchmark/SKILL.md +++ b/.opencode/skills/benchmark/SKILL.md @@ -38,50 +38,62 @@ dd if=/dev/urandom of=/tmp/fastsync_bench/src/large.bin bs=1M count=10 2>/dev/nu Test each configuration 3 times, record median: ```bash +# Real FastSync flags: -z=compression, -j=multithreading, +# --chunk-serialization, --sendfile (long form only). The old rsync-style +# spellings -c/-m/-s/-f are NOT the same options (-c=--checksum, +# -m=--prune-empty-dirs, -s=--secluded-args, -f=--filter) and must not be used. CONFIGS=( "Standard|" - "Compression|-c" - "Multithreading|-m" - "MT+Compression|-m -c" - "Chunk Serialization|-s" - "MT+Compression+Chunk|-m -c -s" - "Sendfile|-f" + "Compression|-z" + "Multithreading|-j" + "MT+Compression|-j -z" + "Chunk Serialization|-j -z --chunk-serialization" + "Sendfile|--sendfile" ) +PORT=18080 for config in "${CONFIGS[@]}"; do IFS='|' read -r name flags <<< "$config" echo "=== $name ===" for run in 1 2 3; do rm -rf /tmp/fastsync_bench/dst mkdir -p /tmp/fastsync_bench/dst - - ./build/server & + + ./build/server -p "$PORT" --allow-unauthenticated & SERVER_PID=$! sleep 0.5 - + START=$(date +%s%N) ./build/client --source-dir /tmp/fastsync_bench/src \ --dest-dir /tmp/fastsync_bench/dst \ + --server-port "$PORT" \ --save-to-disk $flags END=$(date +%s%N) - + ELAPSED=$(( (END - START) / 1000000 )) echo " Run $run: ${ELAPSED}ms" - + kill $SERVER_PID 2>/dev/null wait $SERVER_PID 2>/dev/null done done ``` -### Step 4: Full Integration Benchmark (Optional) +### Step 4: Full Benchmark Tool (Preferred) + +The maintained benchmark tool is `benchmark/bench.py`. It handles building, +data generation, network shaping (LAN/WAN profiles or custom `--delay`/`--jitter`/ +`--throughput`/`--loss`), rsync comparison, and JSON/table reporting: -For comprehensive benchmarking with network shaping: ```bash -python3 test.py --full +python3 benchmark/bench.py --help +python3 benchmark/bench.py --runs 5 --profiles unlimited +python3 benchmark/bench.py --size-mb 100 --random-ratio 0.5 --output json +python3 benchmark/bench.py --delay 50ms --jitter 10ms --throughput 100mbit ``` -This tests LAN/WAN profiles, SSH, TLS, and compares against rsync. +Network shaping needs root (`tc`/`netem` on `lo`). SSH and TLS coverage lives in +the pytest integration suite, not the benchmark tool. ### Step 5: Report Results @@ -92,13 +104,14 @@ Platform: Configuration | Run 1 | Run 2 | Run 3 | Median -----------------------|---------|---------|---------|-------- -Standard | 0.12s | 0.11s | 0.12s | 0.12s -Compression (-c) | 0.09s | 0.08s | 0.09s | 0.09s -Multithreading (-m) | 0.07s | 0.07s | 0.08s | 0.07s -MT+Compression (-m -c) | 0.05s | 0.05s | 0.06s | 0.05s -Sendfile (-f) | 0.04s | 0.04s | 0.04s | 0.04s +Standard | 0.12s | 0.11s | 0.12s | 0.12s +Compression (-z) | 0.09s | 0.08s | 0.09s | 0.09s +Multithreading (-j) | 0.07s | 0.07s | 0.08s | 0.07s +MT+Compression (-j -z) | 0.05s | 0.05s | 0.06s | 0.05s +Chunk Serialization (--chunk-serialization) | 0.05s | 0.04s | 0.05s | 0.05s +Sendfile (--sendfile) | 0.04s | 0.04s | 0.04s | 0.04s -Best configuration: MT+Compression (-m -c) +Best configuration: Sendfile (--sendfile) Throughput: MB/s ``` diff --git a/benchmark/bench.py b/benchmark/bench.py index abb1db8..d4b59b1 100644 --- a/benchmark/bench.py +++ b/benchmark/bench.py @@ -26,7 +26,7 @@ import time PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) BUILD_DIR = os.path.join(PROJECT_ROOT, "build") -SERVER_CMD = [os.path.join(BUILD_DIR, "server")] +SERVER_CMD = [os.path.join(BUILD_DIR, "server"), "--allow-unauthenticated"] CLIENT_CMD = [os.path.join(BUILD_DIR, "client")] BENCH_DIR = os.path.join(PROJECT_ROOT, "bench_data") @@ -43,11 +43,12 @@ NETWORK_PROFILES = { } FASTSYNC_CONFIGS = [ - {"name": "fastsync", "flags": [], "tool": "fastsync"}, - {"name": "fastsync -c", "flags": ["-c"], "tool": "fastsync"}, - {"name": "fastsync -m", "flags": ["-m"], "tool": "fastsync"}, - {"name": "fastsync -m -c", "flags": ["-m", "-c"], "tool": "fastsync"}, - {"name": "fastsync -m -c -s", "flags": ["-m", "-c", "-s"], "tool": "fastsync"}, + {"name": "fastsync", "flags": [], "tool": "fastsync"}, + {"name": "fastsync -z", "flags": ["-z"], "tool": "fastsync"}, + {"name": "fastsync -j", "flags": ["-j"], "tool": "fastsync"}, + {"name": "fastsync -j -z", "flags": ["-j", "-z"], "tool": "fastsync"}, + {"name": "fastsync -j -z --chunk-serialization", "flags": ["-j", "-z", "--chunk-serialization"], "tool": "fastsync"}, + {"name": "fastsync --sendfile", "flags": ["--sendfile"], "tool": "fastsync"}, ] RSYNC_CONFIGS = [ @@ -252,8 +253,10 @@ def run_fastsync(source_dir, dest_dir, flags, port): duration = time.monotonic() - start if result.returncode == 0: return duration + sys.stderr.write(f" fastsync failed (exit {result.returncode}): " + f"{result.stderr.strip()[:500]}\n") except subprocess.TimeoutExpired: - pass + sys.stderr.write(" fastsync timed out after 120s\n") return None @@ -269,8 +272,10 @@ def run_rsync(source_dir, dest_dir, flags, rsync_daemon=None): duration = time.monotonic() - start if result.returncode == 0: return duration + sys.stderr.write(f" rsync failed (exit {result.returncode}): " + f"{result.stderr.strip()[:500]}\n") except subprocess.TimeoutExpired: - pass + sys.stderr.write(" rsync timed out after 120s\n") return None @@ -367,15 +372,15 @@ def print_table(results, total_bytes, random_ratio): if fs_entries: print(f"\n FastSync:") - print(f" {'Config':<25} {'p50':>8} {'p95':>8} {'min':>8} {'max':>8} {'stdev':>8} {'runs':>5}") - print(f" {'-' * 25} {'-' * 8} {'-' * 8} {'-' * 8} {'-' * 8} {'-' * 8} {'-' * 5}") + print(f" {'Config':<38} {'p50':>8} {'p95':>8} {'min':>8} {'max':>8} {'stdev':>8} {'runs':>5}") + print(f" {'-' * 38} {'-' * 8} {'-' * 8} {'-' * 8} {'-' * 8} {'-' * 8} {'-' * 5}") for e in sorted(fs_entries, key=lambda x: x.get("p50", 999)): _print_entry(e) if rsync_entries: print(f"\n rsync:") - print(f" {'Config':<25} {'p50':>8} {'p95':>8} {'min':>8} {'max':>8} {'stdev':>8} {'runs':>5}") - print(f" {'-' * 25} {'-' * 8} {'-' * 8} {'-' * 8} {'-' * 8} {'-' * 8} {'-' * 5}") + print(f" {'Config':<38} {'p50':>8} {'p95':>8} {'min':>8} {'max':>8} {'stdev':>8} {'runs':>5}") + print(f" {'-' * 38} {'-' * 8} {'-' * 8} {'-' * 8} {'-' * 8} {'-' * 8} {'-' * 5}") for e in sorted(rsync_entries, key=lambda x: x.get("p50", 999)): _print_entry(e) @@ -392,10 +397,10 @@ def print_table(results, total_bytes, random_ratio): def _print_entry(e): if "p50" in e: - print(f" {e['config']:<25} {e['p50']:>7.4f}s {e['p95']:>7.4f}s " + print(f" {e['config']:<38} {e['p50']:>7.4f}s {e['p95']:>7.4f}s " f"{e['min']:>7.4f}s {e['max']:>7.4f}s {e['stdev']:>7.4f} {e['runs']:>5}") else: - print(f" {e['config']:<25} {'N/A':>8} {'N/A':>8} {'N/A':>8} {'N/A':>8} {'N/A':>8} {e['runs']:>5}") + print(f" {e['config']:<38} {'N/A':>8} {'N/A':>8} {'N/A':>8} {'N/A':>8} {'N/A':>8} {e['runs']:>5}") def main(): @@ -448,12 +453,14 @@ Examples: help="Don't clean up test data") args = parser.parse_args() - # Build - print("Building...") - if os.system(f"cmake -B {BUILD_DIR} -S {PROJECT_ROOT} > /dev/null 2>&1") != 0: - print("CMake configure failed"); sys.exit(1) + # Build (Release: benchmarking a debug build is meaningless) + print("Building (Release)...", file=sys.stderr) + configure = (f"cmake -B {BUILD_DIR} -S {PROJECT_ROOT} " + f"-DCMAKE_BUILD_TYPE=Release > /dev/null 2>&1") + if os.system(configure) != 0: + print("CMake configure failed", file=sys.stderr); sys.exit(1) if os.system(f"cmake --build {BUILD_DIR} -j$(nproc) > /dev/null 2>&1") != 0: - print("Build failed"); sys.exit(1) + print("Build failed", file=sys.stderr); sys.exit(1) # Determine active profile for display has_custom_net = args.delay or args.jitter or args.throughput or args.loss @@ -480,7 +487,8 @@ Examples: compressible_pct = (1 - args.random_ratio) * 100 random_pct = args.random_ratio * 100 print(f"Generated {total_bytes / (1024*1024):.1f} MB " - f"({random_pct:.0f}% random, {compressible_pct:.0f}% compressible)") + f"({random_pct:.0f}% random, {compressible_pct:.0f}% compressible)", + file=sys.stderr) # Build config list if args.configs: @@ -496,7 +504,7 @@ Examples: total_runs = len(configs) * args.runs * len(profiles_to_run) progress = Progress(total_runs, "Benchmarking") if args.progress else None if progress: - print(f"Running {total_runs} transfers...") + print(f"Running {total_runs} transfers...", file=sys.stderr) all_results = [] try: