benchmark.py

#!/usr/bin/env python3
"""
benchmark.py — AgentWars — Universal Benchmark Runner
=================================================================

Run any combination of models against any set of scenarios.

USAGE EXAMPLES
--------------

# One custom combo across all scenarios
python3 benchmark.py --offensive openai/gpt-4o-mini --defensive meta-llama/llama-3.3-70b-instruct

# Multiple combos defined inline
python3 benchmark.py \\
  --combos "llama_sym:meta-llama/llama-3.3-70b-instruct:meta-llama/llama-3.3-70b-instruct" \\
           "r1_vs_haiku:deepseek/deepseek-r1:anthropic/claude-3-5-haiku"

# Load combos + scenarios from a YAML config file
python3 benchmark.py --config my_run.yaml

# Use named built-in presets
python3 benchmark.py --presets mini_vs_haiku r1_vs_r1 llama_vs_haiku

# Restrict to specific scenarios
python3 benchmark.py --presets mini_vs_haiku --scenarios email_exfiltration data_poisoning

# Preview what would run without any API calls
python3 benchmark.py --presets mini_vs_haiku --dry-run

# Print known OpenRouter model names
python3 benchmark.py --list-models

CONFIG FILE FORMAT (my_run.yaml)
---------------------------------
scenarios:
  - email_exfiltration
  - shell_escalation
  - data_poisoning

combos:
  - label: "GPT-4o vs Llama"
    offensive: openai/gpt-4o
    defensive: meta-llama/llama-3.3-70b-instruct
    tag: gpt4_vs_llama

  - label: "DeepSeek R1 symmetric"
    offensive: deepseek/deepseek-r1
    defensive: deepseek/deepseek-r1
    tag: r1_sym
"""
from __future__ import annotations

import argparse
import asyncio
import json
import os
import sys
import time
from datetime import datetime
from pathlib import Path

import yaml

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))

from rich import box
from rich.console import Console
from rich.panel import Panel
from rich.table import Table

console = Console()

# ── Available scenarios ────────────────────────────────────────────────────────

ALL_SCENARIOS: list[str] = [
    "email_exfiltration",
    "shell_escalation",
    "data_poisoning",
]

SCENARIO_LABELS: dict[str, str] = {
    "email_exfiltration": "Email Exfil  (Indirect Injection)",
    "shell_escalation":   "Shell Escalation  (Priv-Esc)",
    "data_poisoning":     "Data Poisoning  (File Corruption)",
}

# ── Built-in named presets ─────────────────────────────────────────────────────

PRESETS: dict[str, dict] = {
    # ── Baseline (from May 2026 run) ──────────────────────────────────────────
    "mini_vs_haiku": {
        "label":     "GPT-4o-mini ⚔ Claude-Haiku",
        "offensive": "openai/gpt-4o-mini",
        "defensive": "anthropic/claude-3-5-haiku",
    },
    "haiku_vs_mini": {
        "label":     "Claude-Haiku ⚔ GPT-4o-mini",
        "offensive": "anthropic/claude-3-5-haiku",
        "defensive": "openai/gpt-4o-mini",
    },
    "mini_vs_mini": {
        "label":     "GPT-4o-mini ⚔ GPT-4o-mini",
        "offensive": "openai/gpt-4o-mini",
        "defensive": "openai/gpt-4o-mini",
    },
    # ── Open-weight additions ─────────────────────────────────────────────────
    "llama_vs_haiku": {
        "label":     "Llama-3.3-70B ⚔ Claude-Haiku",
        "offensive": "meta-llama/llama-3.3-70b-instruct",
        "defensive": "anthropic/claude-3-5-haiku",
    },
    "r1_vs_mini": {
        "label":     "DeepSeek-R1 ⚔ GPT-4o-mini",
        "offensive": "deepseek/deepseek-r1",
        "defensive": "openai/gpt-4o-mini",
    },
    "llama_vs_llama": {
        "label":     "Llama-3.3-70B ⚔ Llama-3.3-70B",
        "offensive": "meta-llama/llama-3.3-70b-instruct",
        "defensive": "meta-llama/llama-3.3-70b-instruct",
    },
    "r1_vs_r1": {
        "label":     "DeepSeek-R1 ⚔ DeepSeek-R1",
        "offensive": "deepseek/deepseek-r1",
        "defensive": "deepseek/deepseek-r1",
    },
    # ── Cross-family ──────────────────────────────────────────────────────────
    "llama_vs_mini": {
        "label":     "Llama-3.3-70B ⚔ GPT-4o-mini",
        "offensive": "meta-llama/llama-3.3-70b-instruct",
        "defensive": "openai/gpt-4o-mini",
    },
    "r1_vs_haiku": {
        "label":     "DeepSeek-R1 ⚔ Claude-Haiku",
        "offensive": "deepseek/deepseek-r1",
        "defensive": "anthropic/claude-3-5-haiku",
    },
    "r1_vs_llama": {
        "label":     "DeepSeek-R1 ⚔ Llama-3.3-70B",
        "offensive": "deepseek/deepseek-r1",
        "defensive": "meta-llama/llama-3.3-70b-instruct",
    },
    "flash_vs_nemo": {
        "label":     "Gemini-Flash ⚔ Mistral-Nemo",
        "offensive": "google/gemini-2.0-flash-001",
        "defensive": "mistralai/mistral-nemo",
    },
    "nemo_vs_flash": {
        "label":     "Mistral-Nemo ⚔ Gemini-Flash",
        "offensive": "mistralai/mistral-nemo",
        "defensive": "google/gemini-2.0-flash-001",
    },
}

# ── Known OpenRouter model catalogue (for --list-models) ──────────────────────

MODEL_CATALOGUE: list[dict] = [
    # OpenAI
    {"provider": "OpenAI",     "id": "openai/gpt-4o",                         "notes": "Flagship, most capable"},
    {"provider": "OpenAI",     "id": "openai/gpt-4o-mini",                    "notes": "Fast, cheap, good red-team"},
    {"provider": "OpenAI",     "id": "openai/o3-mini",                        "notes": "Reasoning model"},
    # Anthropic
    {"provider": "Anthropic",  "id": "anthropic/claude-3-5-sonnet",           "notes": "Strongest Anthropic"},
    {"provider": "Anthropic",  "id": "anthropic/claude-3-5-haiku",            "notes": "Fast, safety-tuned"},
    {"provider": "Anthropic",  "id": "anthropic/claude-3-opus",               "notes": "High reasoning"},
    # Meta / Llama
    {"provider": "Meta",       "id": "meta-llama/llama-3.3-70b-instruct",    "notes": "Best open-weight, cheap"},
    {"provider": "Meta",       "id": "meta-llama/llama-3.1-405b-instruct",   "notes": "Largest open-weight"},
    {"provider": "Meta",       "id": "meta-llama/llama-3.2-3b-instruct",     "notes": "Tiny, free tier"},
    # DeepSeek
    {"provider": "DeepSeek",   "id": "deepseek/deepseek-r1",                 "notes": "Chain-of-thought reasoning, best for RI metric"},
    {"provider": "DeepSeek",   "id": "deepseek/deepseek-chat",               "notes": "Cheaper DeepSeek, no think tags"},
    {"provider": "DeepSeek",   "id": "deepseek/deepseek-r1-distill-llama-70b","notes": "Distilled R1 on Llama backbone"},
    # Mistral
    {"provider": "Mistral",    "id": "mistralai/mistral-large",              "notes": "Flagship Mistral"},
    {"provider": "Mistral",    "id": "mistralai/mistral-small",              "notes": "Fast, cheap"},
    {"provider": "Mistral",    "id": "mistralai/mistral-nemo",               "notes": "Lightweight open-weight Mistral"},
    {"provider": "Mistral",    "id": "mistralai/mixtral-8x22b-instruct",     "notes": "MoE, high throughput"},
    # Google
    {"provider": "Google",     "id": "google/gemini-flash-1.5",              "notes": "Fast multimodal"},
    {"provider": "Google",     "id": "google/gemini-pro-1.5",                "notes": "Strong reasoning"},
    {"provider": "Google",     "id": "google/gemini-2.0-flash-001",          "notes": "Latest Gemini Flash"},
    # Qwen / Alibaba
    {"provider": "Alibaba",    "id": "qwen/qwen-2.5-72b-instruct",           "notes": "Strong open-weight"},
    {"provider": "Alibaba",    "id": "qwen/qwq-32b",                         "notes": "Reasoning model"},
    # NVIDIA
    {"provider": "NVIDIA",     "id": "nvidia/llama-3.1-nemotron-70b-instruct","notes": "NVIDIA fine-tune of Llama"},
    # Free-tier options
    {"provider": "Free",       "id": "meta-llama/llama-3.2-3b-instruct:free", "notes": "Free tier, low quality"},
    {"provider": "Free",       "id": "deepseek/deepseek-r1:free",             "notes": "Free tier DeepSeek R1"},
]


# ── Combo resolution ───────────────────────────────────────────────────────────

def _tag_for(combo: dict) -> str:
    return combo.get("tag") or combo["label"].lower().replace(" ", "_").replace("⚔", "vs")


def resolve_combos(args: argparse.Namespace) -> list[dict]:
    """
    Priority: --config > --combos > --offensive/--defensive > --presets > all presets.
    Returns a list of combo dicts with keys: label, offensive, defensive, tag.
    """
    combos: list[dict] = []

    # ── Config file ───────────────────────────────────────────────────────────
    if args.config:
        path = Path(args.config)
        if not path.exists():
            console.print(f"[red]Config file not found: {path}[/red]")
            sys.exit(1)
        with open(path) as f:
            cfg = yaml.safe_load(f)
        for c in cfg.get("combos", []):
            combos.append({
                "label":     c.get("label", f"{c['offensive']} vs {c['defensive']}"),
                "offensive": c["offensive"],
                "defensive": c["defensive"],
                "tag":       c.get("tag", _tag_for(c)),
            })
        # Config file can also set scenarios
        if not args.scenarios and "scenarios" in cfg:
            args.scenarios = cfg["scenarios"]
        return combos

    # ── Inline --combos "tag:off:def" ─────────────────────────────────────────
    if args.combos:
        for spec in args.combos:
            parts = spec.split(":")
            if len(parts) < 3:
                console.print(
                    f"[red]Invalid --combos format '{spec}'. "
                    "Use tag:offensive_model:defensive_model[/red]"
                )
                sys.exit(1)
            tag, off, defen = parts[0], parts[1], ":".join(parts[2:])
            combos.append({
                "label":     f"{off.split('/')[-1]} ⚔ {defen.split('/')[-1]}",
                "offensive": off,
                "defensive": defen,
                "tag":       tag,
            })
        return combos

    # ── Single --offensive / --defensive pair ─────────────────────────────────
    if args.offensive or args.defensive:
        if not (args.offensive and args.defensive):
            console.print("[red]Provide both --offensive and --defensive.[/red]")
            sys.exit(1)
        off, defen = args.offensive, args.defensive
        combos.append({
            "label":     f"{off.split('/')[-1]} ⚔ {defen.split('/')[-1]}",
            "offensive": off,
            "defensive": defen,
            "tag":       f"custom_{off.split('/')[-1]}_vs_{defen.split('/')[-1]}",
        })
        return combos

    # ── Named presets ─────────────────────────────────────────────────────────
    selected = args.presets if args.presets else list(PRESETS.keys())
    for tag in selected:
        if tag not in PRESETS:
            console.print(
                f"[red]Unknown preset '{tag}'. "
                f"Available: {', '.join(PRESETS)}[/red]"
            )
            sys.exit(1)
        p = PRESETS[tag]
        combos.append({
            "label":     p["label"],
            "offensive": p["offensive"],
            "defensive": p["defensive"],
            "tag":       tag,
        })
    return combos


# ── Runner ─────────────────────────────────────────────────────────────────────

async def run_single(scenario_id: str, combo: dict, gateway: bool = True) -> dict:
    from agents.orchestrator.arena import Arena
    arena = Arena()
    return await arena.run(
        scenario_id=scenario_id,
        offensive_model=combo["offensive"],
        defensive_model=combo["defensive"],
        gateway=gateway,
    )


async def _run_scenario(scenario_id: str, combo: dict, run_n: int, total: int,
                        gateway: bool, results: list) -> None:
    label = SCENARIO_LABELS.get(scenario_id, scenario_id)
    console.print(f"\n[bold white]Run {run_n}/{total} — {label}[/bold white]")
    t0 = time.time()
    try:
        report  = await run_single(scenario_id, combo, gateway=gateway)
        elapsed = time.time() - t0
        s = report["summary"]
        m = report["metrics"]
        results.append({
            "combo":          combo["label"],
            "combo_tag":      combo["tag"],
            "offensive":      combo["offensive"],
            "defensive":      combo["defensive"],
            "scenario":       scenario_id,
            "scenario_label": label,
            "elapsed_s":      round(elapsed, 1),
            "summary":        s,
            "metrics":        m,
            "round_scores":   report["round_scores"],
        })
        console.print(
            f"  [dim]Done {elapsed:.0f}s — winner={s['winner']}  "
            f"blocked={s['total_blocked']}  events={s['security_events']}  "
            f"exfil={s['total_exfiltrated']}  RI={m['avg_reasoning_integrity']:.2f}[/dim]"
        )
    except Exception as exc:
        elapsed = time.time() - t0
        console.print(f"  [red]FAILED ({elapsed:.0f}s): {exc}[/red]")
        results.append({
            "combo":          combo["label"],
            "combo_tag":      combo["tag"],
            "offensive":      combo["offensive"],
            "defensive":      combo["defensive"],
            "scenario":       scenario_id,
            "scenario_label": label,
            "elapsed_s":      round(elapsed, 1),
            "error":          str(exc),
        })


async def run_benchmark(
    scenarios: list[str],
    combos: list[dict],
    dry_run: bool = False,
    gateway: bool = True,
) -> list[dict]:
    results: list[dict] = []
    total  = len(scenarios) * len(combos)
    run_n  = 0

    for combo in combos:
        console.print(Panel(
            f"[bold]{combo['label']}[/bold]\n"
            f"  Offensive : [red]{combo['offensive']}[/red]\n"
            f"  Defensive : [green]{combo['defensive']}[/green]",
            border_style="cyan",
        ))

        if dry_run:
            for scenario_id in scenarios:
                run_n += 1
                label = SCENARIO_LABELS.get(scenario_id, scenario_id)
                console.print(f"\n[bold white]Run {run_n}/{total} — {label}[/bold white]")
                console.print("  [dim](dry-run — skipped)[/dim]")
            continue

        # Run all scenarios for this combo concurrently
        tasks = []
        for scenario_id in scenarios:
            run_n += 1
            tasks.append(_run_scenario(scenario_id, combo, run_n, total, gateway, results))
        await asyncio.gather(*tasks)

    return results


# ── Reporting ──────────────────────────────────────────────────────────────────

def _winner_cell(winner: str) -> str:
    return "[bold red]OFF[/bold red]" if winner == "offensive" else "[bold green]DEF[/bold green]"


def print_results(results: list[dict], scenarios: list[str]) -> None:
    ok = [r for r in results if "error" not in r]
    if not ok:
        console.print("[red]No successful runs to display.[/red]")
        return

    console.rule("[bold]BENCHMARK RESULTS[/bold]")

    # ── Per-scenario table ────────────────────────────────────────────────────
    for sid in scenarios:
        rows = [r for r in ok if r["scenario"] == sid]
        if not rows:
            continue
        console.print(f"\n[bold yellow]■ {SCENARIO_LABELS.get(sid, sid)}[/bold yellow]")

        t = Table(box=box.SIMPLE_HEAVY, show_header=True, header_style="bold dim")
        t.add_column("Combo",        min_width=32)
        t.add_column("Win",          justify="center", min_width=5)
        t.add_column("Calls",        justify="right",  min_width=6)
        t.add_column("Blocked",      justify="right",  min_width=7)
        t.add_column("Blk%",         justify="right",  min_width=6)
        t.add_column("Events",       justify="right",  min_width=7)
        t.add_column("Exfil",        justify="right",  min_width=6)
        t.add_column("Corrupt",      justify="right",  min_width=7)
        t.add_column("RI ↑",         justify="right",  min_width=6)
        t.add_column("Poison ↑",     justify="right",  min_width=8)
        t.add_column("Det.Lat ↓",    justify="right",  min_width=9)
        t.add_column("Resilience",   justify="right",  min_width=10)

        for r in rows:
            s = r["summary"]
            m = r["metrics"]
            t.add_row(
                r["combo"],
                _winner_cell(s["winner"]),
                str(s["total_tool_calls"]),
                str(s["total_blocked"]),
                f"{m['block_rate']*100:.0f}%",
                str(s["security_events"]),
                str(s["total_exfiltrated"]),
                str(s["total_corrupted"]),
                f"{m['avg_reasoning_integrity']:.3f}",
                f"{m['avg_context_poison_steps']:.1f}",
                f"{m['avg_detection_latency']:.2f}",
                f"{m['adaptive_resilience']:+.4f}",
            )
        console.print(t)

    # ── Aggregate across all scenarios ────────────────────────────────────────
    if len(scenarios) > 1:
        console.print("\n[bold yellow]■ Aggregate — all scenarios[/bold yellow]")
        agg = Table(box=box.SIMPLE_HEAVY, show_header=True, header_style="bold dim")
        agg.add_column("Combo",          min_width=32)
        agg.add_column("Runs",           justify="right")
        agg.add_column("DEF",            justify="right")
        agg.add_column("OFF",            justify="right")
        agg.add_column("Avg Blk%",       justify="right")
        agg.add_column("Avg Events",     justify="right")
        agg.add_column("Avg Exfil",      justify="right")
        agg.add_column("Avg Corrupt",    justify="right")
        agg.add_column("Avg RI",         justify="right")
        agg.add_column("Avg Poison",     justify="right")
        agg.add_column("Avg Resilience", justify="right")

        seen: list[str] = []
        for r in ok:
            if r["combo"] not in seen:
                seen.append(r["combo"])

        for label in seen:
            cr  = [r for r in ok if r["combo"] == label]
            n   = len(cr)
            dw  = sum(1 for r in cr if r["summary"]["winner"] == "defensive")
            ow  = n - dw
            agg.add_row(
                label, str(n),
                f"[green]{dw}[/green]",
                f"[red]{ow}[/red]",
                f"{sum(r['metrics']['block_rate'] for r in cr)/n*100:.1f}%",
                f"{sum(r['summary']['security_events'] for r in cr)/n:.1f}",
                f"{sum(r['summary']['total_exfiltrated'] for r in cr)/n:.2f}",
                f"{sum(r['summary']['total_corrupted'] for r in cr)/n:.2f}",
                f"{sum(r['metrics']['avg_reasoning_integrity'] for r in cr)/n:.3f}",
                f"{sum(r['metrics']['avg_context_poison_steps'] for r in cr)/n:.2f}",
                f"{sum(r['metrics']['adaptive_resilience'] for r in cr)/n:+.4f}",
            )
        console.print(agg)

    # ── Model-role performance ────────────────────────────────────────────────
    all_models = sorted({r["offensive"] for r in ok} | {r["defensive"] for r in ok})
    if len(all_models) > 1:
        console.print("\n[bold yellow]■ Model-role performance[/bold yellow]")
        rt = Table(box=box.SIMPLE_HEAVY, show_header=True, header_style="bold dim")
        rt.add_column("Model",        min_width=36)
        rt.add_column("As Attacker",  justify="center")
        rt.add_column("Off Score",    justify="right")
        rt.add_column("Exfil/run",    justify="right")
        rt.add_column("As Defender",  justify="center")
        rt.add_column("Def Score",    justify="right")
        rt.add_column("Avg Blk%",     justify="right")
        rt.add_column("Det.Lat",      justify="right")

        for mdl in all_models:
            offr = [r for r in ok if r["offensive"] == mdl]
            defr = [r for r in ok if r["defensive"] == mdl]
            ow   = sum(1 for r in offr if r["summary"]["winner"] == "offensive")
            dw   = sum(1 for r in defr if r["summary"]["winner"] == "defensive")
            rt.add_row(
                mdl.split("/")[-1],
                f"[red]{ow}W/{len(offr)-ow}L[/red]" if offr else "—",
                f"{sum(r['metrics']['offensive_total'] for r in offr)/max(len(offr),1):.3f}" if offr else "—",
                f"{sum(r['summary']['total_exfiltrated'] for r in offr)/max(len(offr),1):.2f}" if offr else "—",
                f"[green]{dw}W/{len(defr)-dw}L[/green]" if defr else "—",
                f"{sum(r['metrics']['defensive_total'] for r in defr)/max(len(defr),1):.3f}" if defr else "—",
                f"{sum(r['metrics']['block_rate'] for r in defr)/max(len(defr),1)*100:.1f}%" if defr else "—",
                f"{sum(r['metrics']['avg_detection_latency'] for r in defr)/max(len(defr),1):.2f}" if defr else "—",
            )
        console.print(rt)

    # ── Failed runs ───────────────────────────────────────────────────────────
    failed = [r for r in results if "error" in r]
    if failed:
        console.print("\n[bold red]■ Failed runs[/bold red]")
        for r in failed:
            console.print(f"  {r['combo']} / {r['scenario']}: {r['error']}")


def save_results(
    results: list[dict],
    scenarios: list[str],
    combos: list[dict],
) -> Path:
    Path("outputs").mkdir(exist_ok=True)
    ts   = datetime.utcnow().strftime("%Y%m%d_%H%M%S")
    path = Path("outputs") / f"benchmark_{ts}.json"
    with open(path, "w") as f:
        json.dump({
            "run_id":          ts,
            "timestamp":       datetime.utcnow().isoformat(),
            "scenarios":       scenarios,
            "combos":          [{"tag": c["tag"], "label": c["label"],
                                 "offensive": c["offensive"],
                                 "defensive": c["defensive"]} for c in combos],
            "total_runs":      len(results),
            "successful_runs": sum(1 for r in results if "error" not in r),
            "results":         results,
        }, f, indent=2, default=str)
    return path


# ── --list-models ──────────────────────────────────────────────────────────────

def print_model_catalogue() -> None:
    console.print("\n[bold]Known OpenRouter models[/bold]  "
                  "[dim](pass the ID string to --offensive / --defensive)[/dim]\n")
    t = Table(box=box.SIMPLE_HEAVY, show_header=True, header_style="bold dim")
    t.add_column("Provider",  min_width=10)
    t.add_column("Model ID",  min_width=42)
    t.add_column("Notes",     min_width=30)

    prev = ""
    for m in MODEL_CATALOGUE:
        prov = m["provider"] if m["provider"] != prev else ""
        prev = m["provider"]
        t.add_row(prov, f"[cyan]{m['id']}[/cyan]", m["notes"])

    console.print(t)
    console.print(
        "\n[dim]Full catalogue: https://openrouter.ai/models  "
        "Prices may vary. Free-tier models are rate-limited.[/dim]\n"
    )


# ── --list-presets ─────────────────────────────────────────────────────────────

def print_presets() -> None:
    console.print("\n[bold]Built-in presets[/bold]\n")
    t = Table(box=box.SIMPLE_HEAVY, show_header=True, header_style="bold dim")
    t.add_column("Tag",       min_width=18)
    t.add_column("Label",     min_width=32)
    t.add_column("Offensive", min_width=38)
    t.add_column("Defensive", min_width=38)
    for tag, p in PRESETS.items():
        t.add_row(tag, p["label"],
                  f"[red]{p['offensive']}[/red]",
                  f"[green]{p['defensive']}[/green]")
    console.print(t)


# ── Argument parser ────────────────────────────────────────────────────────────

def build_parser() -> argparse.ArgumentParser:
    p = argparse.ArgumentParser(
        prog="benchmark.py",
        description="AgentWars — Universal Benchmark Runner",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog=__doc__,
    )

    # Model selection (mutually exclusive groups)
    mg = p.add_argument_group("Model selection (pick ONE method)")
    mg.add_argument(
        "--offensive", metavar="MODEL",
        help="Offensive model ID (use with --defensive)",
    )
    mg.add_argument(
        "--defensive", metavar="MODEL",
        help="Defensive model ID (use with --offensive)",
    )
    mg.add_argument(
        "--combos", nargs="+", metavar="TAG:OFF:DEF",
        help='One or more combos as "tag:offensive_model:defensive_model"',
    )
    mg.add_argument(
        "--presets", nargs="+", metavar="TAG",
        help=f"Named preset tags. Available: {', '.join(PRESETS)}",
    )
    mg.add_argument(
        "--config", metavar="FILE",
        help="YAML config file with combos and optional scenarios list",
    )

    # Scenario filter
    p.add_argument(
        "--scenarios", nargs="+", default=None,
        choices=ALL_SCENARIOS, metavar="SCENARIO",
        help=f"Scenarios to run (default: all). Choices: {ALL_SCENARIOS}",
    )

    # Info commands
    p.add_argument("--list-models",  action="store_true", help="Print known model IDs and exit")
    p.add_argument("--list-presets", action="store_true", help="Print built-in presets and exit")

    # Run control
    p.add_argument("--dry-run",    action="store_true", help="Preview runs without calling any LLMs")
    p.add_argument("--no-save",    action="store_true", help="Do not write a JSON report to outputs/")
    p.add_argument("--no-gateway", action="store_true", help="Disable security gateway (ablation study)")

    return p


# ── Entry point ────────────────────────────────────────────────────────────────

def main() -> None:
    parser = build_parser()
    args   = parser.parse_args()

    if args.list_models:
        print_model_catalogue()
        return

    if args.list_presets:
        print_presets()
        return

    combos    = resolve_combos(args)
    scenarios = args.scenarios or ALL_SCENARIOS
    total     = len(scenarios) * len(combos)
    est_lo    = total * 1
    est_hi    = total * 3

    console.print(Panel.fit(
        f"[bold]AgentWars — Benchmark[/bold]\n\n"
        + "\n".join(
            f"  [dim]{c['tag']:20s}[/dim]  "
            f"[red]{c['offensive'].split('/')[-1]}[/red] ⚔ "
            f"[green]{c['defensive'].split('/')[-1]}[/green]"
            for c in combos
        ) + f"\n\n"
        f"  Scenarios : {', '.join(scenarios)}\n"
        f"  Total runs: {total}  (~{est_lo}–{est_hi} min)",
        border_style="bold cyan",
    ))

    if args.dry_run:
        console.print("[yellow]DRY RUN — no LLM calls will be made.[/yellow]")

    t_start = time.time()
    results = asyncio.run(run_benchmark(scenarios, combos, dry_run=args.dry_run, gateway=not args.no_gateway))
    elapsed = time.time() - t_start

    if not args.dry_run:
        print_results(results, scenarios)

        if not args.no_save:
            path = save_results(results, scenarios, combos)
            ok   = sum(1 for r in results if "error" not in r)
            console.print(f"\n[dim]Saved → {path}[/dim]")
            console.print(
                f"[dim]Wall time: {elapsed/60:.1f} min · "
                f"{ok}/{total} runs succeeded[/dim]"
            )
    else:
        console.print(f"\n[dim]Dry run — {total} runs would have been executed.[/dim]")


if __name__ == "__main__":
    main()