#!/usr/bin/env bash # benchmark_comprehensive.sh — Comprehensive benchmark for fastSyncAI # # Compares fastSyncAI against rsync, rclone, unison, and other tools # Tests different connection counts and settings # # Usage: # ./benchmark_comprehensive.sh [DATA_SIZE_MB] [RUN_COUNT] [LATENCY_MS] ======= # ============================================================================= # benchmark_comprehensive.sh — Comprehensive benchmark for fastSyncAI # # Compares fastSyncAI against rsync, rclone, unison, and other tools # Tests different connection counts and settings # # NOTE: When LATENCY_MS > 0, only fastSyncAI (TCP-based) experiences the network # latency. rsync, rclone, unison, and cp operate locally and bypass tc netem. # For fair TCP-based comparison of all tools, use benchmark_network.sh instead. # # Usage: # ./benchmark_comprehensive.sh [DATA_SIZE_MB] [RUN_COUNT] [LATENCY_MS]============================================================================= # benchmark_comprehensive.sh — Comprehensive benchmark for fastSyncAI # # Compares fastSyncAI against rsync, rclone, unison, and other tools # Tests different connection counts and settings # # Usage: # ./benchmark_comprehensive.sh [DATA_SIZE_MB] [RUN_COUNT] [LATENCY_MS] # # Examples: # ./benchmark_comprehensive.sh # 100 MB, 3 runs, no latency (default) # ./benchmark_comprehensive.sh 500 # 500 MB, 3 runs, no latency # ./benchmark_comprehensive.sh 100 5 # 100 MB, 5 runs, no latency # ./benchmark_comprehensive.sh 100 1 20 # 100 MB, 1 run, 20ms RTT latency # ./benchmark_comprehensive.sh 100 1 100 # 100 MB, 1 run, 100ms RTT (WAN-like) # # Requirements: # - rsync (usually pre-installed) # - rclone (optional, for cloud/local sync comparison) # - unison (optional, for bidirectional sync comparison) # - cp (for baseline comparison) # - sudo (for network latency simulation via tc netem) # ============================================================================= set -euo pipefail # ── Defaults ────────────────────────────────────────────────────────────── DATA_SIZE_MB=${1:-100} RUN_COUNT=${2:-3} LATENCY_MS=${3:-0} PORT=19299 HOST="127.0.0.1" SERVER_BIN="./fastsync_server" CLIENT_BIN="./fastsync_client" # ── Directories ──────────────────────────────────────────────────────────── TEST_DIR="bench_comprehensive_$$" SRC_DIR="$TEST_DIR/src" DST_DIR_BASE="$TEST_DIR/dst" # ── Connection counts to test for fastSyncAI ─────────────────────────────── CONN_COUNTS=(1 2 4 8 16) # ── Colours ──────────────────────────────────────────────────────────────── RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m' CYAN='\033[0;36m'; BOLD='\033[1m'; RESET='\033[0m' # ── Helpers ──────────────────────────────────────────────────────────────── die() { echo -e "${RED}ERROR: $*${RESET}" >&2; exit 1; } info() { echo -e "${CYAN}▶ $*${RESET}"; } ok() { echo -e "${GREEN}✓ $*${RESET}"; } warn() { echo -e "${YELLOW}⚠ $*${RESET}"; } # ── Latency simulation (tc netem) ──────────────────────────────────────── setup_latency() { if [ "$LATENCY_MS" -gt 0 ]; then info "Adding ${LATENCY_MS}ms RTT latency to lo via tc netem..." if ! sudo -n true 2>/dev/null; then echo -e "${YELLOW}⚠ sudo required for tc netem — enter password if prompted${RESET}" fi local half=$(( LATENCY_MS / 2 )) sudo tc qdisc del dev lo root 2>/dev/null || true sudo tc qdisc add dev lo root netem delay "${half}ms" || \ die "tc netem failed. Install iproute2 or run without latency argument." ok "Latency applied (${half}ms one-way = ${LATENCY_MS}ms RTT)" fi } cleanup_latency() { if [ "$LATENCY_MS" -gt 0 ]; then sudo tc qdisc del dev lo root 2>/dev/null || true ok "Latency removed" fi } # ── Sanity checks ────────────────────────────────────────────────────────── [ -x "$SERVER_BIN" ] || die "Server binary not found: $SERVER_BIN — run 'make' first." [ -x "$CLIENT_BIN" ] || die "Client binary not found: $CLIENT_BIN — run 'make' first." command -v rsync >/dev/null 2>&1 || die "rsync not found. Install rsync first." # Check for optional tools HAS_RCLONE=$(command -v rclone >/dev/null 2>&1 && echo "1" || echo "0") HAS_UNISON=$(command -v unison >/dev/null 2>&1 && echo "1" || echo "0") HAS_PV=$(command -v pv >/dev/null 2>&1 && echo "1" || echo "0") cleanup() { cleanup_latency rm -rf "$TEST_DIR" } trap cleanup EXIT # ── Generate test data ────────────────────────────────────────────────────── generate_test_data() { info "Generating ~${DATA_SIZE_MB}MB test data..." rm -rf "$TEST_DIR" mkdir -p "$SRC_DIR/small" "$SRC_DIR/medium" "$SRC_DIR/large" # Small files: many files to test overhead (~25% of total) local small_count=$(( (DATA_SIZE_MB * 1048576 / 4) / 262144 )) [ "$small_count" -lt 10 ] && small_count=10 [ "$small_count" -gt 100 ] && small_count=100 info " Creating $small_count small files (256KB each)..." for i in $(seq 1 $small_count); do dd if=/dev/urandom bs=256K count=1 of="$SRC_DIR/small/file_$i.bin" 2>/dev/null done # Medium files: ~50% of total local medium_count=$(( (DATA_SIZE_MB * 1048576 / 2) / 1048576 )) [ "$medium_count" -lt 5 ] && medium_count=5 [ "$medium_count" -gt 20 ] && medium_count=20 info " Creating $medium_count medium files (1MB each)..." for i in $(seq 1 $medium_count); do dd if=/dev/urandom bs=1M count=1 of="$SRC_DIR/medium/file_$i.bin" 2>/dev/null done # Large files: remaining ~25% local large_size=$(( DATA_SIZE_MB - (small_count * 256 / 1024) - medium_count )) [ "$large_size" -lt 5 ] && large_size=5 local large_count=3 [ "$large_size" -lt 10 ] && large_count=1 info " Creating $large_count large files (~${large_size}MB total)..." for i in $(seq 1 $large_count); do local this_size=$(( large_size * 1048576 / large_count / 1048576 )) [ "$this_size" -lt 1 ] && this_size=1 dd if=/dev/urandom bs=1M count=$this_size of="$SRC_DIR/large/file_$i.bin" 2>/dev/null done local total_files total_size total_files=$(find "$SRC_DIR" -type f | wc -l) total_size=$(du -sh "$SRC_DIR" | cut -f1) ok "Generated $total_files files, $total_size total in $SRC_DIR" } # ── Verification helper ──────────────────────────────────────────────────── verify_sync() { local src_dir=$1 local dst_dir=$2 local src_count=$(find "$src_dir" -type f | wc -l) local dst_count=$(find "$dst_dir" -type f | wc -l) [ "$src_count" -ne "$dst_count" ] && return 1 local src_size=$(find "$src_dir" -type f -exec stat -c %s {} \; | awk '{sum+=$1} END {print sum}') local dst_size=$(find "$dst_dir" -type f -exec stat -c %s {} \; | awk '{sum+=$1} END {print sum}') [ "$src_size" -ne "$dst_size" ] && return 1 return 0 } # ── Draw bar chart ──────────────────────────────────────────────────────── draw_bar() { local val=$1 max=$2 width=30 if [ "$max" -eq 0 ]; then max=1; fi local filled filled=$(LC_ALL=C awk "BEGIN{printf \"%d\", ($val * $width / $max)}") [ "$filled" -gt "$width" ] && filled=$width local bar="" for ((i=0; i /dev/null 2>&1 & local srv_pid=$! sleep 0.2 local output output=$(timeout 120 "$CLIENT_BIN" -h "$HOST" -p "$server_port" -s "$SRC_DIR" -n "$nconn" 2>&1) || true # Wait for client to fully finish sending DONE and server to process all data # With network latency, 1 connection needs more time local wait_time=3 if [ "$LATENCY_MS" -gt 0 ] && [ "$nconn" -eq 1 ]; then wait_time=$(( 3 + LATENCY_MS / 10 + 2 )) fi sleep $wait_time # Signal server to exit (it will finish current transfers and die) kill "$srv_pid" 2>/dev/null || true timeout 10 wait "$srv_pid" 2>/dev/null || true # Force sync to ensure all disk writes are complete before verification sync # Extract from BENCH line local ms mbs ms=$(echo "$output" | grep "^BENCH:" | grep -oP 'ms=\K[0-9.]+' || echo "0") mbs=$(echo "$output" | grep "^BENCH:" | grep -oP 'throughput_mbs=\K[0-9.]+' || echo "0") if verify_sync "$SRC_DIR" "$dst_dir"; then echo "${ms}|${mbs}|1" else echo "${ms}|${mbs}|0" fi rm -rf "$dst_dir" } # ── Run rsync ─────────────────────────────────────────────────────────────── run_rsync() { local dst_dir="$DST_DIR_BASE/rsync" rm -rf "$dst_dir" mkdir -p "$dst_dir" local start end start=$(date +%s%N) rsync -a "$SRC_DIR/" "$dst_dir/" > /dev/null 2>&1 end=$(date +%s%N) local elapsed_ms=$(( (end - start) / 1000000 )) local total_bytes total_bytes=$(find "$SRC_DIR" -type f -exec stat -c %s {} \; | awk '{sum+=$1} END {print sum}') local mbs=0 if [ "$elapsed_ms" -gt 0 ]; then mbs=$(awk "BEGIN {printf \"%.2f\", $total_bytes / 1048576.0 / ($elapsed_ms / 1000.0)}") fi if verify_sync "$SRC_DIR" "$dst_dir"; then echo "${elapsed_ms}|${mbs}|1" else echo "${elapsed_ms}|${mbs}|0" fi rm -rf "$dst_dir" } # ── Run rsync with compression ────────────────────────────────────────────── run_rsync_compress() { local dst_dir="$DST_DIR_BASE/rsync_compress" rm -rf "$dst_dir" mkdir -p "$dst_dir" local start end start=$(date +%s%N) rsync -a --compress "$SRC_DIR/" "$dst_dir/" > /dev/null 2>&1 end=$(date +%s%N) local elapsed_ms=$(( (end - start) / 1000000 )) local total_bytes total_bytes=$(find "$SRC_DIR" -type f -exec stat -c %s {} \; | awk '{sum+=$1} END {print sum}') local mbs=0 if [ "$elapsed_ms" -gt 0 ]; then mbs=$(awk "BEGIN {printf \"%.2f\", $total_bytes / 1048576.0 / ($elapsed_ms / 1000.0)}") fi if verify_sync "$SRC_DIR" "$dst_dir"; then echo "${elapsed_ms}|${mbs}|1" else echo "${elapsed_ms}|${mbs}|0" fi rm -rf "$dst_dir" } # ── Run rclone (if available) ───────────────────────────────────────────── run_rclone() { if [ "$HAS_RCLONE" -eq 0 ]; then echo "0|0|0" # Skip - not installed return fi local dst_dir="$DST_DIR_BASE/rclone" rm -rf "$dst_dir" mkdir -p "$dst_dir" local start end start=$(date +%s%N) rclone copy "$SRC_DIR" "$dst_dir" -q --transfers 4 --checkers 8 2>/dev/null || true end=$(date +%s%N) local elapsed_ms=$(( (end - start) / 1000000 )) local total_bytes total_bytes=$(find "$SRC_DIR" -type f -exec stat -c %s {} \; | awk '{sum+=$1} END {print sum}') local mbs=0 if [ "$elapsed_ms" -gt 0 ]; then mbs=$(awk "BEGIN {printf \"%.2f\", $total_bytes / 1048576.0 / ($elapsed_ms / 1000.0)}") fi if verify_sync "$SRC_DIR" "$dst_dir"; then echo "${elapsed_ms}|${mbs}|1" else echo "${elapsed_ms}|${mbs}|0" fi rm -rf "$dst_dir" } # ── Run Unison (if available) ───────────────────────────────────────────── run_unison() { if [ "$HAS_UNISON" -eq 0 ]; then echo "0|0|0" # Skip - not installed return fi local dst_dir="$DST_DIR_BASE/unison" rm -rf "$dst_dir" mkdir -p "$dst_dir" local start end start=$(date +%s%N) unison "$SRC_DIR" "$dst_dir" -batch -silent -noshutdown 2>/dev/null || true end=$(date +%s%N) local elapsed_ms=$(( (end - start) / 1000000 )) local total_bytes total_bytes=$(find "$SRC_DIR" -type f -exec stat -c %s {} \; | awk '{sum+=$1} END {print sum}') local mbs=0 if [ "$elapsed_ms" -gt 0 ]; then mbs=$(awk "BEGIN {printf \"%.2f\", $total_bytes / 1048576.0 / ($elapsed_ms / 1000.0)}") fi if verify_sync "$SRC_DIR" "$dst_dir"; then echo "${elapsed_ms}|${mbs}|1" else echo "${elapsed_ms}|${mbs}|0" fi rm -rf "$dst_dir" } # ── Run cp (baseline) ───────────────────────────────────────────────────── run_cp() { local dst_dir="$DST_DIR_BASE/cp" rm -rf "$dst_dir" mkdir -p "$dst_dir" local start end start=$(date +%s%N) cp -r "$SRC_DIR"/* "$dst_dir/" 2>/dev/null end=$(date +%s%N) local elapsed_ms=$(( (end - start) / 1000000 )) local total_bytes total_bytes=$(find "$SRC_DIR" -type f -exec stat -c %s {} \; | awk '{sum+=$1} END {print sum}') local mbs=0 if [ "$elapsed_ms" -gt 0 ]; then mbs=$(awk "BEGIN {printf \"%.2f\", $total_bytes / 1048576.0 / ($elapsed_ms / 1000.0)}") fi if verify_sync "$SRC_DIR" "$dst_dir"; then echo "${elapsed_ms}|${mbs}|1" else echo "${elapsed_ms}|${mbs}|0" fi rm -rf "$dst_dir" } # ──────────────────────────────────────────────────────────────────────────── # MAIN # ──────────────────────────────────────────────────────────────────────────── echo "" echo -e "${BOLD}╔════════════════════════════════════════════════════════════════════════╗${RESET}" echo -e "${BOLD}║ fastSyncAI — Comprehensive Performance Benchmark ║${RESET}" echo -e "${BOLD}╚════════════════════════════════════════════════════════════════════════╝${RESET}" echo "" echo -e " Data size : ${CYAN}${DATA_SIZE_MB} MB${RESET}" echo -e " Runs per test : ${CYAN}${RUN_COUNT}${RESET}" echo -e " Latency : ${CYAN}${LATENCY_MS}ms RTT${RESET}" echo -e " Tools tested : fastSyncAI (${#CONN_COUNTS[@]} connection configs), rsync, rsync+compress$( [ "$HAS_RCLONE" -eq 1 ] && echo ", rclone" || echo "")$( [ "$HAS_UNISON" -eq 1 ] && echo ", unison" || echo "")${RESET}" echo "" # Initialize result arrays declare -a FS_RESULTS RS_RESULTS RSC_RESULTS RCLONE_RESULTS UNISON_RESULTS CP_RESULTS declare -a FS_LABELS=("fastSyncAI (1 conn)" "fastSyncAI (2 conn)" "fastSyncAI (4 conn)" "fastSyncAI (8 conn)" "fastSyncAI (16 conn)") # Setup latency if requested if [ "$LATENCY_MS" -gt 0 ]; then setup_latency echo "" fi info "Generating test data..." generate_test_data echo "" # ── Test fastSyncAI with different connection counts ─────────────────────── info "Testing fastSyncAI with different connection counts..." for nconn in "${CONN_COUNTS[@]}"; do echo -n " Testing ${nconn} connections... " result=$(run_fastsync $nconn) IFS='|' read -r ms mbs verified <<< "$result" FS_RESULTS+=("$ms|$mbs|$verified") if [ "$verified" -eq 1 ]; then ok "${mbs} MB/s (${ms}ms)" else warn "${mbs} MB/s (${ms}ms) - VERIFICATION FAILED" fi done echo "" # ── Test rsync ───────────────────────────────────────────────────────────── echo -n " Testing rsync... " result=$(run_rsync) IFS='|' read -r ms mbs verified <<< "$result" RS_RESULTS=("$ms|$mbs|$verified") if [ "$verified" -eq 1 ]; then ok "${mbs} MB/s (${ms}ms)" else warn "${mbs} MB/s (${ms}ms) - VERIFICATION FAILED" fi # ── Test rsync with compression ──────────────────────────────────────────── echo -n " Testing rsync + compress... " result=$(run_rsync_compress) IFS='|' read -r ms mbs verified <<< "$result" RSC_RESULTS=("$ms|$mbs|$verified") if [ "$verified" -eq 1 ]; then ok "${mbs} MB/s (${ms}ms)" else warn "${mbs} MB/s (${ms}ms) - VERIFICATION FAILED" fi # ── Test rclone (if available) ───────────────────────────────────────────── if [ "$HAS_RCLONE" -eq 1 ]; then echo -n " Testing rclone... " result=$(run_rclone) IFS='|' read -r ms mbs verified <<< "$result" RCLONE_RESULTS=("$ms|$mbs|$verified") if [ "$verified" -eq 1 ]; then ok "${mbs} MB/s (${ms}ms)" else warn "${mbs} MB/s (${ms}ms) - VERIFICATION FAILED" fi else warn "rclone not installed - skipping" fi # ── Test Unison (if available) ───────────────────────────────────────────── if [ "$HAS_UNISON" -eq 1 ]; then echo -n " Testing unison... " result=$(run_unison) IFS='|' read -r ms mbs verified <<< "$result" UNISON_RESULTS=("$ms|$mbs|$verified") if [ "$verified" -eq 1 ]; then ok "${mbs} MB/s (${ms}ms)" else warn "${mbs} MB/s (${ms}ms) - VERIFICATION FAILED" fi else warn "unison not installed - skipping" fi # ── Test cp (baseline) ───────────────────────────────────────────────────── echo -n " Testing cp (baseline)... " result=$(run_cp) IFS='|' read -r ms mbs verified <<< "$result" CP_RESULTS=("$ms|$mbs|$verified") if [ "$verified" -eq 1 ]; then ok "${mbs} MB/s (${ms}ms)" else warn "${mbs} MB/s (${ms}ms) - VERIFICATION FAILED" fi echo "" echo -e "${BOLD}════════════════════════════════════════════════════════════════════════╗${RESET}" echo -e "${BOLD} CONNECTION COUNT COMPARISON${RESET}" echo -e "${BOLD}════════════════════════════════════════════════════════════════════════╝${RESET}" echo "" # Find max MB/s for scaling calc_max_mbs() { local max=0 for res in "${FS_RESULTS[@]}" "${RS_RESULTS[@]}" "${RSC_RESULTS[@]}" "${RCLONE_RESULTS[@]}" "${UNISON_RESULTS[@]}" "${CP_RESULTS[@]}"; do local mbs=$(echo "$res" | cut -d'|' -f2) local int_mbs=$(echo "$mbs" | awk '{printf "%d", $1+0}') [ "$int_mbs" -gt "$max" ] && max=$int_mbs done echo $max } MAX_MBS=$(calc_max_mbs) [ "$MAX_MBS" -eq 0 ] && MAX_MBS=100 printf " %-25s %-10s %-10s %s\n" "Tool" "Time (ms)" "MB/s" "Throughput" printf " %-25s %-10s %-10s %s\n" "-----------------------" "----------" "----------" "-----------" # Print fastSyncAI results with different connection counts for i in "${!FS_RESULTS[@]}"; do IFS='|' read -r ms mbs verified <<< "${FS_RESULTS[$i]}" bar=$(draw_bar "$mbs" "$MAX_MBS") verified_str=$([ "$verified" -eq 1 ] && echo "✓" || echo "✗") printf " %-25s %-10s %-10s %s %s\n" "${FS_LABELS[$i]}" "$ms" "$mbs" "$bar" "$verified_str" done # Print other tool results printf "\n %-25s %-10s %-10s %s\n" "Other Tools:" "" "" "" printf " %-25s %-10s %-10s %s\n" "-----------------------" "----------" "----------" "-----------" # rsync IFS='|' read -r ms mbs verified <<< "${RS_RESULTS[0]}" bar=$(draw_bar "$mbs" "$MAX_MBS") verified_str=$([ "$verified" -eq 1 ] && echo "✓" || echo "✗") printf " %-25s %-10s %-10s %s %s\n" "rsync" "$ms" "$mbs" "$bar" "$verified_str" # rsync + compress IFS='|' read -r ms mbs verified <<< "${RSC_RESULTS[0]}" bar=$(draw_bar "$mbs" "$MAX_MBS") verified_str=$([ "$verified" -eq 1 ] && echo "✓" || echo "✗") printf " %-25s %-10s %-10s %s %s\n" "rsync + compress" "$ms" "$mbs" "$bar" "$verified_str" # rclone if [ "${RCLONE_RESULTS[0]}" != "0|0|0" ]; then IFS='|' read -r ms mbs verified <<< "${RCLONE_RESULTS[0]}" bar=$(draw_bar "$mbs" "$MAX_MBS") verified_str=$([ "$verified" -eq 1 ] && echo "✓" || echo "✗") printf " %-25s %-10s %-10s %s %s\n" "rclone" "$ms" "$mbs" "$bar" "$verified_str" fi # unison if [ "${UNISON_RESULTS[0]}" != "0|0|0" ]; then IFS='|' read -r ms mbs verified <<< "${UNISON_RESULTS[0]}" bar=$(draw_bar "$mbs" "$MAX_MBS") verified_str=$([ "$verified" -eq 1 ] && echo "✓" || echo "✗") printf " %-25s %-10s %-10s %s %s\n" "unison" "$ms" "$mbs" "$bar" "$verified_str" fi # cp IFS='|' read -r ms mbs verified <<< "${CP_RESULTS[0]}" bar=$(draw_bar "$mbs" "$MAX_MBS") verified_str=$([ "$verified" -eq 1 ] && echo "✓" || echo "✗") printf " %-25s %-10s %-10s %s %s\n" "cp (baseline)" "$ms" "$mbs" "$bar" "$verified_str" echo "" echo -e "${BOLD}════════════════════════════════════════════════════════════════════════╗${RESET}" echo -e "${BOLD} SPEEDUP ANALYSIS${RESET}" echo -e "${BOLD}════════════════════════════════════════════════════════════════════════╝${RESET}" echo "" # Calculate speedups vs rsync IFS='|' read -r rs_ms rs_mbs rs_verified <<< "${RS_RESULTS[0]}" if [ "$rs_mbs" != "0" ]; then echo " Speedup vs rsync (higher is better):" echo "" for i in "${!FS_RESULTS[@]}"; do IFS='|' read -r fs_ms fs_mbs fs_verified <<< "${FS_RESULTS[$i]}" if [ "$fs_mbs" != "0" ]; then speedup=$(awk "BEGIN {printf \"%.2f\", $fs_mbs / $rs_mbs}") # Check if speedup >= 2.0 using bc for floating point comparison if [ "$(awk "BEGIN {print ($speedup >= 2.0) ? 1 : 0}")" = "1" ]; then color="$GREEN" elif [ "$(awk "BEGIN {print ($speedup >= 1.3) ? 1 : 0}")" = "1" ]; then color="$YELLOW" else color="$RESET" fi printf " %-25s: %s%.2fx faster%s\n" "${FS_LABELS[$i]}" "$color" "$speedup" "$RESET" fi done echo "" fi # Best connection count best_idx=0 best_mbs=0 for i in "${!FS_RESULTS[@]}"; do IFS='|' read -r ms mbs verified <<< "${FS_RESULTS[$i]}" mbs_int=$(echo "$mbs" | awk '{printf "%d", $1+0}') if [ "$mbs_int" -gt "$best_mbs" ]; then best_mbs=$mbs_int best_idx=$i fi done ok "Best fastSyncAI config: ${FS_LABELS[$best_idx]} (${best_mbs} MB/s)" echo "" echo -e "${BOLD}════════════════════════════════════════════════════════════════════════╗${RESET}" echo -e "${BOLD} COMPLETE${RESET}" echo -e "${BOLD}════════════════════════════════════════════════════════════════════════╝${RESET}" echo "" ok "Benchmark complete!"