#!/usr/bin/env python3 """Standalone benchmark tool for FastSync. Usage: python3 benchmark/bench.py # quick benchmark (unlimited, 3 runs) python3 benchmark/bench.py --runs 5 --profiles lan wan # thorough python3 benchmark/bench.py --output json # machine-readable python3 benchmark/bench.py --configs "-c" "-m" "-m -c" # custom configs """ import argparse import json import os import random import shutil import socket import statistics import subprocess import sys import tempfile import time PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) BUILD_DIR = os.path.join(PROJECT_ROOT, "build") SERVER_CMD = [os.path.join(BUILD_DIR, "server")] CLIENT_CMD = [os.path.join(BUILD_DIR, "client")] BENCH_DIR = os.path.join(PROJECT_ROOT, "bench_data") NETWORK_PROFILES = { "unlimited": {}, "lan": { "rate": "1000mbit", "delay": "20ms", "jitter": "1ms", "loss": "0.1%", "rate_bps": 1_000_000_000 / 8, }, "wan": { "rate": "100mbit", "delay": "50ms", "jitter": "10ms", "loss": "1%", "rate_bps": 100_000_000 / 8, }, } DEFAULT_CONFIGS = [ {"name": "standard", "flags": []}, {"name": "-c", "flags": ["-c"]}, {"name": "-m", "flags": ["-m"]}, {"name": "-m -c", "flags": ["-m", "-c"]}, {"name": "-s", "flags": ["-s"]}, {"name": "-m -c -s", "flags": ["-m", "-c", "-s"]}, ] def generate_bench_data(source_dir, size_mb=25): """Generate test data for benchmarking.""" if os.path.exists(source_dir): shutil.rmtree(source_dir) os.makedirs(source_dir) target = size_mb * 1024 * 1024 written = 0 # Structured files files = { "small.txt": b"hello world\n", "medium.txt": b"the quick brown fox jumps over the lazy dog\n" * 5000, "binary.bin": bytes(range(256)) * 1000, "nested/subdir/deep.txt": b"deeply nested file\n", "nested/another.txt": b"another nested file\n" * 50, } for rel_path, content in files.items(): full_path = os.path.join(source_dir, rel_path) os.makedirs(os.path.dirname(full_path), exist_ok=True) with open(full_path, "wb") as f: f.write(content) written += len(content) # Fill remaining with random data os.makedirs(os.path.join(source_dir, "bulk"), exist_ok=True) i = 0 while written < target: chunk_size = min(5 * 1024 * 1024, target - written) with open(os.path.join(source_dir, f"bulk/file_{i}.dat"), "wb") as f: f.write(random.randbytes(chunk_size)) written += chunk_size i += 1 return written def find_free_port(): with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: s.bind(("", 0)) return s.getsockname()[1] def wait_for_port(port, timeout=5): deadline = time.monotonic() + timeout while time.monotonic() < deadline: try: with socket.create_connection(("127.0.0.1", port), timeout=0.3): return except (ConnectionRefusedError, OSError): time.sleep(0.05) raise RuntimeError(f"Port {port} not ready") def wait_proc(proc, timeout=5): try: proc.wait(timeout=timeout) except subprocess.TimeoutExpired: proc.kill() proc.wait() def netem_apply(profile_name): params = NETWORK_PROFILES.get(profile_name, {}) if not params: netem_reset() return netem_reset() cmd = ["sudo", "tc", "qdisc", "add", "dev", "lo", "root", "netem"] cmd += ["rate", params["rate"]] cmd += ["delay", params["delay"], params["jitter"]] cmd += ["loss", params["loss"]] subprocess.run(cmd, check=True, capture_output=True) def netem_reset(): subprocess.run("sudo tc qdisc del dev lo root".split(), capture_output=True) def run_transfer(source_dir, dest_dir, flags, port): """Run a single transfer. Returns duration in seconds or None on failure.""" cmd = CLIENT_CMD + [ "--source-dir", source_dir, "--dest-dir", dest_dir, "--server-port", str(port), "--save-to-disk", ] + flags try: start = time.monotonic() result = subprocess.run(cmd, capture_output=True, text=True, timeout=120) duration = time.monotonic() - start if result.returncode == 0: return duration except subprocess.TimeoutExpired: pass return None def run_benchmark(source_dir, dest_dir, configs, runs, profile_name): """Run benchmark for all configs, returns list of results.""" is_limited = profile_name != "unlimited" if is_limited: netem_apply(profile_name) try: results = [] for config in configs: times = [] for run_idx in range(runs): # Clean dest for each run if os.path.exists(dest_dir): shutil.rmtree(dest_dir) os.makedirs(dest_dir, exist_ok=True) # Start fresh server port = find_free_port() server = subprocess.Popen( SERVER_CMD + ["-p", str(port)], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) try: wait_for_port(port) t = run_transfer(source_dir, dest_dir, config["flags"], port) if t is not None: times.append(t) finally: wait_proc(server) entry = { "config": config["name"], "profile": profile_name, "runs": len(times), "times": [round(t, 4) for t in times], } if times: entry["p50"] = round(statistics.median(times), 4) entry["p95"] = round(sorted(times)[int(len(times) * 0.95)], 4) if len(times) > 1 else entry["p50"] entry["min"] = round(min(times), 4) entry["max"] = round(max(times), 4) entry["stdev"] = round(statistics.stdev(times), 4) if len(times) > 1 else 0.0 results.append(entry) return results finally: if is_limited: netem_reset() def print_table(results, total_bytes): """Print results as a human-readable table.""" # Group by profile profiles = {} for r in results: profiles.setdefault(r["profile"], []).append(r) for profile, entries in profiles.items(): params = NETWORK_PROFILES.get(profile, {}) print(f"\n{'=' * 80}") print(f" Profile: {profile.upper()}") if params.get("rate"): print(f" Network: {params['rate']}, {params['delay']} +/- {params['jitter']}, loss {params['loss']}") else: print(f" Network: unlimited") print(f" Data: {total_bytes / (1024*1024):.1f} MB") print(f"{'=' * 80}") print(f" {'Config':<25} {'p50':>8} {'p95':>8} {'min':>8} {'max':>8} {'stdev':>8} {'runs':>5}") print(f" {'-' * 25} {'-' * 8} {'-' * 8} {'-' * 8} {'-' * 8} {'-' * 8} {'-' * 5}") for e in sorted(entries, key=lambda x: x.get("p50", 999)): if "p50" in e: print(f" {e['config']:<25} {e['p50']:>7.4f}s {e['p95']:>7.4f}s " f"{e['min']:>7.4f}s {e['max']:>7.4f}s {e['stdev']:>7.4f} {e['runs']:>5}") else: print(f" {e['config']:<25} {'N/A':>8} {'N/A':>8} {'N/A':>8} {'N/A':>8} {'N/A':>8} {e['runs']:>5}") if params.get("rate_bps"): best = min((e["p50"] for e in entries if "p50" in e), default=None) if best: theoretical = total_bytes / params["rate_bps"] print(f"\n Best config p50: {best:.4f}s") print(f" Theoretical max: {theoretical:.4f}s (uncompressed at line rate)") print(f" Speedup vs max: {theoretical / best:.2f}x") def main(): parser = argparse.ArgumentParser(description="FastSync benchmark tool") parser.add_argument("--runs", type=int, default=3, help="Number of runs per config (default: 3)") parser.add_argument("--profiles", nargs="+", default=["unlimited"], choices=list(NETWORK_PROFILES.keys()), help="Network profiles to test") parser.add_argument("--configs", nargs="+", default=None, help="Custom config flags (e.g. --configs '-c' '-m' '-m -c')") parser.add_argument("--size-mb", type=int, default=25, help="Test data size in MB (default: 25)") parser.add_argument("--output", choices=["table", "json"], default="table", help="Output format") parser.add_argument("--keep-data", action="store_true", help="Don't clean up test data") args = parser.parse_args() # Build print("Building...") if os.system(f"cmake -B {BUILD_DIR} -S {PROJECT_ROOT} > /dev/null 2>&1") != 0: print("CMake configure failed"); sys.exit(1) if os.system(f"cmake --build {BUILD_DIR} -j$(nproc) > /dev/null 2>&1") != 0: print("Build failed"); sys.exit(1) # Generate data source_dir = os.path.join(BENCH_DIR, "source") dest_dir = os.path.join(BENCH_DIR, "dest") total_bytes = generate_bench_data(source_dir, args.size_mb) print(f"Generated {total_bytes / (1024*1024):.1f} MB test data") # Parse configs if args.configs: configs = [{"name": c, "flags": c.split()} for c in args.configs] else: configs = DEFAULT_CONFIGS # Run benchmarks all_results = [] try: for profile in args.profiles: results = run_benchmark(source_dir, dest_dir, configs, args.runs, profile) all_results.extend(results) finally: if not args.keep_data: shutil.rmtree(BENCH_DIR, ignore_errors=True) # Output if args.output == "json": print(json.dumps(all_results, indent=2)) else: print_table(all_results, total_bytes) print() if __name__ == "__main__": main()