Merge PR #282: fix(benchmark): use real FastSync flags and Release builds

.opencode/skills/benchmark/SKILL.md

@@ -38,16 +38,20 @@ dd if=/dev/urandom of=/tmp/fastsync_bench/src/large.bin bs=1M count=10 2>/dev/nu
 Test each configuration 3 times, record median:
 
 ```bash
+# Real FastSync flags: -z=compression, -j=multithreading,
+# --chunk-serialization, --sendfile (long form only).  The old rsync-style
+# spellings -c/-m/-s/-f are NOT the same options (-c=--checksum,
+# -m=--prune-empty-dirs, -s=--secluded-args, -f=--filter) and must not be used.
 CONFIGS=(
   "Standard|"
-  "Compression|-c"
-  "Multithreading|-m"
-  "MT+Compression|-m -c"
-  "Chunk Serialization|-s"
-  "MT+Compression+Chunk|-m -c -s"
-  "Sendfile|-f"
+  "Compression|-z"
+  "Multithreading|-j"
+  "MT+Compression|-j -z"
+  "Chunk Serialization|-j -z --chunk-serialization"
+  "Sendfile|--sendfile"
 )
 
+PORT=18080
 for config in "${CONFIGS[@]}"; do
   IFS='|' read -r name flags <<< "$config"
   echo "=== $name ==="
@@ -55,13 +59,14 @@ for config in "${CONFIGS[@]}"; do
     rm -rf /tmp/fastsync_bench/dst
     mkdir -p /tmp/fastsync_bench/dst
 
-    ./build/server &
+    ./build/server -p "$PORT" --allow-unauthenticated &
     SERVER_PID=$!
     sleep 0.5
 
     START=$(date +%s%N)
     ./build/client --source-dir /tmp/fastsync_bench/src \
       --dest-dir /tmp/fastsync_bench/dst \
+      --server-port "$PORT" \
       --save-to-disk $flags
     END=$(date +%s%N)
 
@@ -74,14 +79,21 @@ for config in "${CONFIGS[@]}"; do
 done
 ```
 
-### Step 4: Full Integration Benchmark (Optional)
+### Step 4: Full Benchmark Tool (Preferred)
+
+The maintained benchmark tool is `benchmark/bench.py`. It handles building,
+data generation, network shaping (LAN/WAN profiles or custom `--delay`/`--jitter`/
+`--throughput`/`--loss`), rsync comparison, and JSON/table reporting:
 
-For comprehensive benchmarking with network shaping:
 ```bash
-python3 test.py --full
+python3 benchmark/bench.py --help
+python3 benchmark/bench.py --runs 5 --profiles unlimited
+python3 benchmark/bench.py --size-mb 100 --random-ratio 0.5 --output json
+python3 benchmark/bench.py --delay 50ms --jitter 10ms --throughput 100mbit
 ```
 
-This tests LAN/WAN profiles, SSH, TLS, and compares against rsync.
+Network shaping needs root (`tc`/`netem` on `lo`). SSH and TLS coverage lives in
+the pytest integration suite, not the benchmark tool.
 
 ### Step 5: Report Results
 
@@ -93,12 +105,13 @@ Platform: <OS, CPU, network>
 Configuration          | Run 1   | Run 2   | Run 3   | Median
 -----------------------|---------|---------|---------|--------
 Standard                            | 0.12s   | 0.11s   | 0.12s   | 0.12s
-Compression (-c)       | 0.09s   | 0.08s   | 0.09s   | 0.09s
-Multithreading (-m)    | 0.07s   | 0.07s   | 0.08s   | 0.07s
-MT+Compression (-m -c) | 0.05s   | 0.05s   | 0.06s   | 0.05s
-Sendfile (-f)          | 0.04s   | 0.04s   | 0.04s   | 0.04s
+Compression (-z)                    | 0.09s   | 0.08s   | 0.09s   | 0.09s
+Multithreading (-j)                 | 0.07s   | 0.07s   | 0.08s   | 0.07s
+MT+Compression (-j -z)              | 0.05s   | 0.05s   | 0.06s   | 0.05s
+Chunk Serialization (--chunk-serialization) | 0.05s | 0.04s | 0.05s | 0.05s
+Sendfile (--sendfile)               | 0.04s   | 0.04s   | 0.04s   | 0.04s
 
-Best configuration: MT+Compression (-m -c)
+Best configuration: Sendfile (--sendfile)
 Throughput: <X> MB/s
 ```
 

benchmark/bench.py

@@ -26,7 +26,7 @@ import time
 
 PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
 BUILD_DIR = os.path.join(PROJECT_ROOT, "build")
-SERVER_CMD = [os.path.join(BUILD_DIR, "server")]
+SERVER_CMD = [os.path.join(BUILD_DIR, "server"), "--allow-unauthenticated"]
 CLIENT_CMD = [os.path.join(BUILD_DIR, "client")]
 BENCH_DIR = os.path.join(PROJECT_ROOT, "bench_data")
 
@@ -44,10 +44,11 @@ NETWORK_PROFILES = {
 
 FASTSYNC_CONFIGS = [
     {"name": "fastsync",                        "flags": [],                                     "tool": "fastsync"},
-    {"name": "fastsync -c",        "flags": ["-c"],                 "tool": "fastsync"},
-    {"name": "fastsync -m",        "flags": ["-m"],                 "tool": "fastsync"},
-    {"name": "fastsync -m -c",     "flags": ["-m", "-c"],           "tool": "fastsync"},
-    {"name": "fastsync -m -c -s",  "flags": ["-m", "-c", "-s"],     "tool": "fastsync"},
+    {"name": "fastsync -z",                     "flags": ["-z"],                                 "tool": "fastsync"},
+    {"name": "fastsync -j",                     "flags": ["-j"],                                 "tool": "fastsync"},
+    {"name": "fastsync -j -z",                  "flags": ["-j", "-z"],                           "tool": "fastsync"},
+    {"name": "fastsync -j -z --chunk-serialization", "flags": ["-j", "-z", "--chunk-serialization"], "tool": "fastsync"},
+    {"name": "fastsync --sendfile",             "flags": ["--sendfile"],                         "tool": "fastsync"},
 ]
 
 RSYNC_CONFIGS = [
@@ -252,8 +253,10 @@ def run_fastsync(source_dir, dest_dir, flags, port):
         duration = time.monotonic() - start
         if result.returncode == 0:
             return duration
+        sys.stderr.write(f"    fastsync failed (exit {result.returncode}): "
+                         f"{result.stderr.strip()[:500]}\n")
     except subprocess.TimeoutExpired:
-        pass
+        sys.stderr.write("    fastsync timed out after 120s\n")
     return None
 
 
@@ -269,8 +272,10 @@ def run_rsync(source_dir, dest_dir, flags, rsync_daemon=None):
         duration = time.monotonic() - start
         if result.returncode == 0:
             return duration
+        sys.stderr.write(f"    rsync failed (exit {result.returncode}): "
+                         f"{result.stderr.strip()[:500]}\n")
     except subprocess.TimeoutExpired:
-        pass
+        sys.stderr.write("    rsync timed out after 120s\n")
     return None
 
 
@@ -367,15 +372,15 @@ def print_table(results, total_bytes, random_ratio):
 
         if fs_entries:
             print(f"\n FastSync:")
-            print(f" {'Config':<25} {'p50':>8} {'p95':>8} {'min':>8} {'max':>8} {'stdev':>8} {'runs':>5}")
-            print(f" {'-' * 25} {'-' * 8} {'-' * 8} {'-' * 8} {'-' * 8} {'-' * 8} {'-' * 5}")
+            print(f" {'Config':<38} {'p50':>8} {'p95':>8} {'min':>8} {'max':>8} {'stdev':>8} {'runs':>5}")
+            print(f" {'-' * 38} {'-' * 8} {'-' * 8} {'-' * 8} {'-' * 8} {'-' * 8} {'-' * 5}")
             for e in sorted(fs_entries, key=lambda x: x.get("p50", 999)):
                 _print_entry(e)
 
         if rsync_entries:
             print(f"\n rsync:")
-            print(f" {'Config':<25} {'p50':>8} {'p95':>8} {'min':>8} {'max':>8} {'stdev':>8} {'runs':>5}")
-            print(f" {'-' * 25} {'-' * 8} {'-' * 8} {'-' * 8} {'-' * 8} {'-' * 8} {'-' * 5}")
+            print(f" {'Config':<38} {'p50':>8} {'p95':>8} {'min':>8} {'max':>8} {'stdev':>8} {'runs':>5}")
+            print(f" {'-' * 38} {'-' * 8} {'-' * 8} {'-' * 8} {'-' * 8} {'-' * 8} {'-' * 5}")
             for e in sorted(rsync_entries, key=lambda x: x.get("p50", 999)):
                 _print_entry(e)
 
@@ -392,10 +397,10 @@ def print_table(results, total_bytes, random_ratio):
 
 def _print_entry(e):
     if "p50" in e:
-        print(f" {e['config']:<25} {e['p50']:>7.4f}s {e['p95']:>7.4f}s "
+        print(f" {e['config']:<38} {e['p50']:>7.4f}s {e['p95']:>7.4f}s "
               f"{e['min']:>7.4f}s {e['max']:>7.4f}s {e['stdev']:>7.4f} {e['runs']:>5}")
     else:
-        print(f" {e['config']:<25} {'N/A':>8} {'N/A':>8} {'N/A':>8} {'N/A':>8} {'N/A':>8} {e['runs']:>5}")
+        print(f" {e['config']:<38} {'N/A':>8} {'N/A':>8} {'N/A':>8} {'N/A':>8} {'N/A':>8} {e['runs']:>5}")
 
 
 def main():
@@ -448,12 +453,14 @@ Examples:
                         help="Don't clean up test data")
     args = parser.parse_args()
 
-    # Build
-    print("Building...")
-    if os.system(f"cmake -B {BUILD_DIR} -S {PROJECT_ROOT} > /dev/null 2>&1") != 0:
-        print("CMake configure failed"); sys.exit(1)
+    # Build (Release: benchmarking a debug build is meaningless)
+    print("Building (Release)...", file=sys.stderr)
+    configure = (f"cmake -B {BUILD_DIR} -S {PROJECT_ROOT} "
+                 f"-DCMAKE_BUILD_TYPE=Release > /dev/null 2>&1")
+    if os.system(configure) != 0:
+        print("CMake configure failed", file=sys.stderr); sys.exit(1)
     if os.system(f"cmake --build {BUILD_DIR} -j$(nproc) > /dev/null 2>&1") != 0:
-        print("Build failed"); sys.exit(1)
+        print("Build failed", file=sys.stderr); sys.exit(1)
 
     # Determine active profile for display
     has_custom_net = args.delay or args.jitter or args.throughput or args.loss
@@ -480,7 +487,8 @@ Examples:
     compressible_pct = (1 - args.random_ratio) * 100
     random_pct = args.random_ratio * 100
     print(f"Generated {total_bytes / (1024*1024):.1f} MB  "
-          f"({random_pct:.0f}% random, {compressible_pct:.0f}% compressible)")
+          f"({random_pct:.0f}% random, {compressible_pct:.0f}% compressible)",
+          file=sys.stderr)
 
     # Build config list
     if args.configs:
@@ -496,7 +504,7 @@ Examples:
     total_runs = len(configs) * args.runs * len(profiles_to_run)
     progress = Progress(total_runs, "Benchmarking") if args.progress else None
     if progress:
-        print(f"Running {total_runs} transfers...")
+        print(f"Running {total_runs} transfers...", file=sys.stderr)
 
     all_results = []
     try: