From f0c7bb791a20aeeafe7a2893a18981877f368bb0 Mon Sep 17 00:00:00 2001 From: TapTap Date: Sat, 18 Jul 2026 19:55:43 +0200 Subject: [PATCH] benchmark: add rsync comparison, configurable data mix, custom network limits - Add rsync and rsync+zstd as baseline comparisons - --random-ratio controls fraction of incompressible data (default 0.75) - --delay, --jitter, --throughput, --loss for custom network simulation - --no-rsync to skip rsync comparison - Grouped output: FastSync vs rsync with speedup calculations --- benchmark/bench.py | 284 ++++++++++++++++++++++++++++++++------------- 1 file changed, 206 insertions(+), 78 deletions(-) diff --git a/benchmark/bench.py b/benchmark/bench.py index b139cb6..250051c 100644 --- a/benchmark/bench.py +++ b/benchmark/bench.py @@ -1,11 +1,16 @@ #!/usr/bin/env python3 """Standalone benchmark tool for FastSync. +Compares FastSync configs against rsync (no compression) and rsync+zstd. +Data is ~75% random/incompressible and ~25% structured/compressible by default, +controllable via --random-ratio. + Usage: - python3 benchmark/bench.py # quick benchmark (unlimited, 3 runs) - python3 benchmark/bench.py --runs 5 --profiles lan wan # thorough - python3 benchmark/bench.py --output json # machine-readable - python3 benchmark/bench.py --configs "-c" "-m" "-m -c" # custom configs + python3 benchmark/bench.py + python3 benchmark/bench.py --runs 5 --profiles lan wan + python3 benchmark/bench.py --random-ratio 0.5 --size-mb 50 + python3 benchmark/bench.py --delay 50ms --jitter 10ms --throughput 100mbit + python3 benchmark/bench.py --output json """ import argparse import json @@ -16,7 +21,6 @@ import socket import statistics import subprocess import sys -import tempfile import time PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) @@ -37,41 +41,48 @@ NETWORK_PROFILES = { }, } -DEFAULT_CONFIGS = [ - {"name": "standard", "flags": []}, - {"name": "-c", "flags": ["-c"]}, - {"name": "-m", "flags": ["-m"]}, - {"name": "-m -c", "flags": ["-m", "-c"]}, - {"name": "-s", "flags": ["-s"]}, - {"name": "-m -c -s", "flags": ["-m", "-c", "-s"]}, +FASTSYNC_CONFIGS = [ + {"name": "fastsync", "flags": [], "tool": "fastsync"}, + {"name": "fastsync -c", "flags": ["-c"], "tool": "fastsync"}, + {"name": "fastsync -m", "flags": ["-m"], "tool": "fastsync"}, + {"name": "fastsync -m -c", "flags": ["-m", "-c"], "tool": "fastsync"}, + {"name": "fastsync -m -c -s", "flags": ["-m", "-c", "-s"], "tool": "fastsync"}, ] +RSYNC_CONFIGS = [ + {"name": "rsync", "flags": [], "tool": "rsync"}, + {"name": "rsync -z", "flags": ["-z"], "tool": "rsync"}, + {"name": "rsync -z --zstd", "flags": ["-z", "--zc", "zstd"],"tool": "rsync"}, +] -def generate_bench_data(source_dir, size_mb=25): - """Generate test data for benchmarking.""" +STRUCTURED_FILES = { + "small.txt": b"hello world\n", + "medium.txt": b"the quick brown fox jumps over the lazy dog\n" * 5000, + "binary.bin": bytes(range(256)) * 1000, + "nested/subdir/deep.txt": b"deeply nested file\n", + "nested/another.txt": b"another nested file\n" * 50, +} + + +def generate_bench_data(source_dir, size_mb=25, random_ratio=0.75): + """Generate test data. ~random_ratio is incompressible, rest is structured.""" if os.path.exists(source_dir): shutil.rmtree(source_dir) os.makedirs(source_dir) target = size_mb * 1024 * 1024 + structured_budget = int(target * (1 - random_ratio)) written = 0 - # Structured files - files = { - "small.txt": b"hello world\n", - "medium.txt": b"the quick brown fox jumps over the lazy dog\n" * 5000, - "binary.bin": bytes(range(256)) * 1000, - "nested/subdir/deep.txt": b"deeply nested file\n", - "nested/another.txt": b"another nested file\n" * 50, - } - for rel_path, content in files.items(): + for rel_path, content in STRUCTURED_FILES.items(): + if written >= structured_budget: + break full_path = os.path.join(source_dir, rel_path) os.makedirs(os.path.dirname(full_path), exist_ok=True) with open(full_path, "wb") as f: f.write(content) written += len(content) - # Fill remaining with random data os.makedirs(os.path.join(source_dir, "bulk"), exist_ok=True) i = 0 while written < target: @@ -109,25 +120,39 @@ def wait_proc(proc, timeout=5): proc.wait() -def netem_apply(profile_name): +def netem_apply(delay=None, jitter=None, throughput=None, loss=None): + """Apply tc/netem rules to loopback. Pass None to skip a parameter.""" + netem_reset() + cmd = ["sudo", "tc", "qdisc", "add", "dev", "lo", "root", "netem"] + if throughput: + cmd += ["rate", throughput] + if delay: + cmd += ["delay", delay, jitter or "0ms"] + if loss: + cmd += ["loss", loss] + if len(cmd) > 6: + subprocess.run(cmd, check=True, capture_output=True) + + +def netem_apply_profile(profile_name): params = NETWORK_PROFILES.get(profile_name, {}) if not params: netem_reset() return - netem_reset() - cmd = ["sudo", "tc", "qdisc", "add", "dev", "lo", "root", "netem"] - cmd += ["rate", params["rate"]] - cmd += ["delay", params["delay"], params["jitter"]] - cmd += ["loss", params["loss"]] - subprocess.run(cmd, check=True, capture_output=True) + netem_apply( + delay=params.get("delay"), + jitter=params.get("jitter"), + throughput=params.get("rate"), + loss=params.get("loss"), + ) def netem_reset(): subprocess.run("sudo tc qdisc del dev lo root".split(), capture_output=True) -def run_transfer(source_dir, dest_dir, flags, port): - """Run a single transfer. Returns duration in seconds or None on failure.""" +def run_fastsync(source_dir, dest_dir, flags, port): + """Run FastSync client. Returns duration or None.""" cmd = CLIENT_CMD + [ "--source-dir", source_dir, "--dest-dir", dest_dir, @@ -145,38 +170,64 @@ def run_transfer(source_dir, dest_dir, flags, port): return None +def run_rsync(source_dir, dest_dir, flags): + """Run rsync. Returns duration or None.""" + src = source_dir.rstrip("/") + "/" + cmd = ["rsync", "-a", "--delete"] + flags + [src, dest_dir + "/"] + try: + start = time.monotonic() + result = subprocess.run(cmd, capture_output=True, text=True, timeout=120) + duration = time.monotonic() - start + if result.returncode == 0: + return duration + except subprocess.TimeoutExpired: + pass + return None + + +def run_transfer(config, source_dir, dest_dir, port=None): + """Route to the right tool. Returns duration or None.""" + if config["tool"] == "rsync": + return run_rsync(source_dir, dest_dir, config["flags"]) + else: + return run_fastsync(source_dir, dest_dir, config["flags"], port) + + def run_benchmark(source_dir, dest_dir, configs, runs, profile_name): """Run benchmark for all configs, returns list of results.""" is_limited = profile_name != "unlimited" if is_limited: - netem_apply(profile_name) + netem_apply_profile(profile_name) try: results = [] for config in configs: times = [] for run_idx in range(runs): - # Clean dest for each run if os.path.exists(dest_dir): shutil.rmtree(dest_dir) os.makedirs(dest_dir, exist_ok=True) - # Start fresh server port = find_free_port() - server = subprocess.Popen( - SERVER_CMD + ["-p", str(port)], - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, - ) + server = None try: - wait_for_port(port) - t = run_transfer(source_dir, dest_dir, config["flags"], port) + if config["tool"] == "fastsync": + server = subprocess.Popen( + SERVER_CMD + ["-p", str(port)], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + wait_for_port(port) + + t = run_transfer(config, source_dir, dest_dir, port) if t is not None: times.append(t) finally: - wait_proc(server) + if server: + wait_proc(server) entry = { "config": config["name"], + "tool": config["tool"], "profile": profile_name, "runs": len(times), "times": [round(t, 4) for t in times], @@ -194,53 +245,105 @@ def run_benchmark(source_dir, dest_dir, configs, runs, profile_name): netem_reset() -def print_table(results, total_bytes): - """Print results as a human-readable table.""" - # Group by profile +def print_table(results, total_bytes, random_ratio): + """Print results as a human-readable table grouped by profile.""" profiles = {} for r in results: profiles.setdefault(r["profile"], []).append(r) for profile, entries in profiles.items(): params = NETWORK_PROFILES.get(profile, {}) - print(f"\n{'=' * 80}") + print(f"\n{'=' * 85}") print(f" Profile: {profile.upper()}") if params.get("rate"): print(f" Network: {params['rate']}, {params['delay']} +/- {params['jitter']}, loss {params['loss']}") else: print(f" Network: unlimited") - print(f" Data: {total_bytes / (1024*1024):.1f} MB") - print(f"{'=' * 80}") - print(f" {'Config':<25} {'p50':>8} {'p95':>8} {'min':>8} {'max':>8} {'stdev':>8} {'runs':>5}") - print(f" {'-' * 25} {'-' * 8} {'-' * 8} {'-' * 8} {'-' * 8} {'-' * 8} {'-' * 5}") - for e in sorted(entries, key=lambda x: x.get("p50", 999)): - if "p50" in e: - print(f" {e['config']:<25} {e['p50']:>7.4f}s {e['p95']:>7.4f}s " - f"{e['min']:>7.4f}s {e['max']:>7.4f}s {e['stdev']:>7.4f} {e['runs']:>5}") - else: - print(f" {e['config']:<25} {'N/A':>8} {'N/A':>8} {'N/A':>8} {'N/A':>8} {'N/A':>8} {e['runs']:>5}") + print(f" Data: {total_bytes / (1024*1024):.1f} MB ({random_ratio*100:.0f}% random, {(1-random_ratio)*100:.0f}% compressible)") + print(f"{'=' * 85}") - if params.get("rate_bps"): - best = min((e["p50"] for e in entries if "p50" in e), default=None) - if best: - theoretical = total_bytes / params["rate_bps"] - print(f"\n Best config p50: {best:.4f}s") - print(f" Theoretical max: {theoretical:.4f}s (uncompressed at line rate)") - print(f" Speedup vs max: {theoretical / best:.2f}x") + fs_entries = [e for e in entries if e.get("tool") == "fastsync"] + rsync_entries = [e for e in entries if e.get("tool") == "rsync"] + + if fs_entries: + print(f"\n FastSync:") + print(f" {'Config':<25} {'p50':>8} {'p95':>8} {'min':>8} {'max':>8} {'stdev':>8} {'runs':>5}") + print(f" {'-' * 25} {'-' * 8} {'-' * 8} {'-' * 8} {'-' * 8} {'-' * 8} {'-' * 5}") + for e in sorted(fs_entries, key=lambda x: x.get("p50", 999)): + _print_entry(e) + + if rsync_entries: + print(f"\n rsync:") + print(f" {'Config':<25} {'p50':>8} {'p95':>8} {'min':>8} {'max':>8} {'stdev':>8} {'runs':>5}") + print(f" {'-' * 25} {'-' * 8} {'-' * 8} {'-' * 8} {'-' * 8} {'-' * 8} {'-' * 5}") + for e in sorted(rsync_entries, key=lambda x: x.get("p50", 999)): + _print_entry(e) + + if params.get("rate_bps") and fs_entries and rsync_entries: + fs_best = min((e["p50"] for e in fs_entries if "p50" in e), default=None) + rsync_best = min((e["p50"] for e in rsync_entries if "p50" in e), default=None) + theoretical = total_bytes / params["rate_bps"] + if fs_best and rsync_best: + print(f"\n Theoretical max (line rate): {theoretical:.4f}s") + print(f" FastSync best: {fs_best:.4f}s ({theoretical/fs_best:.2f}x vs line rate)") + print(f" rsync best: {rsync_best:.4f}s ({theoretical/rsync_best:.2f}x vs line rate)") + print(f" FastSync vs rsync: {rsync_best/fs_best:.2f}x faster") + + +def _print_entry(e): + if "p50" in e: + print(f" {e['config']:<25} {e['p50']:>7.4f}s {e['p95']:>7.4f}s " + f"{e['min']:>7.4f}s {e['max']:>7.4f}s {e['stdev']:>7.4f} {e['runs']:>5}") + else: + print(f" {e['config']:<25} {'N/A':>8} {'N/A':>8} {'N/A':>8} {'N/A':>8} {'N/A':>8} {e['runs']:>5}") def main(): - parser = argparse.ArgumentParser(description="FastSync benchmark tool") - parser.add_argument("--runs", type=int, default=3, help="Number of runs per config (default: 3)") - parser.add_argument("--profiles", nargs="+", default=["unlimited"], + parser = argparse.ArgumentParser( + description="FastSync benchmark tool — compares FastSync vs rsync", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog="""\ +Network profiles (predefined): + unlimited No artificial limits + lan 1 Gbit, 20ms delay, 1ms jitter, 0.1%% loss + wan 100 Mbit, 50ms delay, 10ms jitter, 1%% loss + +Custom network limits (--delay/--jitter/--throughput) override profiles. + +Data mix: + Default is ~75%% random/incompressible + ~25%% structured/compressible, + reflecting typical real-world file sets. + +Examples: + %(prog)s --profiles wan --runs 5 + %(prog)s --throughput 50mbit --delay 30ms --jitter 5ms + %(prog)s --random-ratio 0.5 --size-mb 100 +""") + parser.add_argument("--runs", type=int, default=3, + help="Number of runs per config (default: 3)") + parser.add_argument("--profiles", nargs="+", default=None, choices=list(NETWORK_PROFILES.keys()), - help="Network profiles to test") + help="Predefined network profiles (default: unlimited)") parser.add_argument("--configs", nargs="+", default=None, - help="Custom config flags (e.g. --configs '-c' '-m' '-m -c')") - parser.add_argument("--size-mb", type=int, default=25, help="Test data size in MB (default: 25)") + help="Custom FastSync config flags") + parser.add_argument("--size-mb", type=int, default=25, + help="Test data size in MB (default: 25)") + parser.add_argument("--random-ratio", type=float, default=0.75, + help="Fraction of data that is random/incompressible (default: 0.75)") + parser.add_argument("--delay", default=None, + help="Custom network delay (e.g. 50ms)") + parser.add_argument("--jitter", default=None, + help="Custom network jitter (e.g. 10ms)") + parser.add_argument("--throughput", default=None, + help="Custom throughput limit (e.g. 100mbit)") + parser.add_argument("--loss", default=None, + help="Custom packet loss (e.g. 1%%)") + parser.add_argument("--no-rsync", action="store_true", + help="Skip rsync comparison") parser.add_argument("--output", choices=["table", "json"], default="table", help="Output format") - parser.add_argument("--keep-data", action="store_true", help="Don't clean up test data") + parser.add_argument("--keep-data", action="store_true", + help="Don't clean up test data") args = parser.parse_args() # Build @@ -250,22 +353,47 @@ def main(): if os.system(f"cmake --build {BUILD_DIR} -j$(nproc) > /dev/null 2>&1") != 0: print("Build failed"); sys.exit(1) + # Determine active profile for display + has_custom_net = args.delay or args.jitter or args.throughput or args.loss + if has_custom_net: + active_profile = "custom" + NETWORK_PROFILES["custom"] = { + "rate": args.throughput, "delay": args.delay or "0ms", + "jitter": args.jitter or "0ms", "loss": args.loss or "0%", + } + if args.throughput: + parts = args.throughput.replace("mbit", "").replace("mbps", "") + try: + NETWORK_PROFILES["custom"]["rate_bps"] = float(parts) * 1_000_000 / 8 + except ValueError: + pass + profiles_to_run = ["custom"] + else: + profiles_to_run = args.profiles or ["unlimited"] + # Generate data source_dir = os.path.join(BENCH_DIR, "source") dest_dir = os.path.join(BENCH_DIR, "dest") - total_bytes = generate_bench_data(source_dir, args.size_mb) - print(f"Generated {total_bytes / (1024*1024):.1f} MB test data") + total_bytes = generate_bench_data(source_dir, args.size_mb, args.random_ratio) + compressible_pct = (1 - args.random_ratio) * 100 + random_pct = args.random_ratio * 100 + print(f"Generated {total_bytes / (1024*1024):.1f} MB " + f"({random_pct:.0f}% random, {compressible_pct:.0f}% compressible)") - # Parse configs + # Build config list if args.configs: - configs = [{"name": c, "flags": c.split()} for c in args.configs] + fastsync_configs = [{"name": c, "flags": c.split(), "tool": "fastsync"} for c in args.configs] else: - configs = DEFAULT_CONFIGS + fastsync_configs = list(FASTSYNC_CONFIGS) + + configs = list(fastsync_configs) + if not args.no_rsync: + configs += RSYNC_CONFIGS # Run benchmarks all_results = [] try: - for profile in args.profiles: + for profile in profiles_to_run: results = run_benchmark(source_dir, dest_dir, configs, args.runs, profile) all_results.extend(results) finally: @@ -276,7 +404,7 @@ def main(): if args.output == "json": print(json.dumps(all_results, indent=2)) else: - print_table(all_results, total_bytes) + print_table(all_results, total_bytes, args.random_ratio) print()