chore(testing): add extended E2E benchmark session runner (#106)

Author: saschabuehrle

docs/guides/extended-testing.md

@@ -0,0 +1,108 @@
+# Extended Testing And Benchmarking (Real APIs)
+
+This guide describes how to run an extended end-to-end (E2E) validation session for cascadeflow using **real provider APIs**.
+
+Goals:
+- Validate correctness, routing logic, and cost optimization in the ways developers actually use cascadeflow:
+  - Apps (Next.js API routes, Vercel AI SDK `useChat`)
+  - Agents (tools, multi-turn, structured outputs)
+  - Routers/proxies (OpenAI-compatible HTTP, existing SDKs)
+- Produce numbers you can share: **accuracy**, **drafter acceptance**, **cost reduction**, latency.
+
+## Setup
+
+1. Load provider keys (repo root `.env`):
+```bash
+set -a && source .env && set +a
+```
+
+2. Install deps:
+```bash
+pnpm install
+python3 -m pip install -r requirements-dev.txt
+```
+
+## What We Measure
+
+- **Accuracy**: dataset-specific correctness (e.g. GSM8K exact match, MMLU multiple-choice, tool-call correctness).
+- **Drafter acceptance**: how often the cheap model is accepted without escalation.
+- **Cost reduction**: savings vs a verifier-only baseline.
+- **Latency**: end-to-end time per request (where available).
+
+## Benchmark Coverage Map
+
+Python benchmark suite (see `tests/benchmarks/`):
+- `run_benchmarks.py`: GSM8K + MMLU + MT-Bench (cost reduction + quality retention targets).
+- `run_all.py`: broad coverage:
+  - HumanEval (code)
+  - GSM8K (math)
+  - MT-Bench (multi-turn)
+  - TruthfulQA (factual)
+  - Banking77 (classification)
+  - Customer support (real-world Q&A)
+  - BFCL agentic tool calling (multi-turn + dependencies)
+  - Tool calling (single + multi-turn tool selection correctness)
+  - Agentic multi-agent (router + tool call correctness)
+  - Provider comparison (quality engine consistency across providers)
+
+TypeScript coverage (monorepo tests):
+- `pnpm test`: builds + tests all TS packages and the Next.js `useChat` example.
+- Vercel AI SDK handler E2E tests:
+  - `packages/core/src/vercel-ai/__tests__/e2e.test.ts`
+  - `packages/core/src/__tests__/vercel-ai-chat-handler.e2e.test.ts`
+- Optional real API smoke:
+  - `pnpm -C packages/core run real-api:smoke`
+
+## Recommended Sessions
+
+### 1) Smoke (Fast Signal, Low Spend)
+
+One-command runner (writes logs/results under `benchmark_results/sessions/`):
+```bash
+set -a && source .env && set +a
+./scripts/extended-e2e-session.sh smoke
+```
+
+Manual steps:
+```bash
+pnpm test
+python3 -m pytest
+
+pnpm -C packages/core exec vitest run \
+  src/vercel-ai/__tests__/e2e.test.ts \
+  src/__tests__/vercel-ai-chat-handler.e2e.test.ts
+
+python3 tests/benchmarks/run_benchmarks.py --quick --output benchmark_results/e2e_quick.json || true
+python3 -m tests.benchmarks.run_all --profile smoke --output-dir benchmark_results/smoke
+```
+
+### 2) Standard (Shareable Numbers)
+
+```bash
+set -a && source .env && set +a
+./scripts/extended-e2e-session.sh standard
+```
+
+### 3) Overnight (Stress + Agentic)
+
+```bash
+python3 -m tests.benchmarks.run_all --profile overnight --output-dir benchmark_results/overnight
+```
+
+## Developer DX Validation (Out Of The Box)
+
+Minimal “does it work for users tomorrow” checks:
+1. `docs/guides/integrate_fast.md` paths:
+   - Vercel AI SDK `useChat` drop-in: build the example `examples/vercel-ai-nextjs/`
+   - Proxy: validate OpenAI-compatible endpoint behavior (see `docs/guides/proxy.md`)
+2. Agent tools:
+   - Tool-call generation correctness: `python3 -m tests.benchmarks.tool_calls`
+   - Multi-turn tool history: `python3 -m tests.benchmarks.tool_calls_agentic`
+
+## Notes / Current Limits
+
+- Tool-call *generation* is benchmarked heavily.
+- Full tool *execution loops* depend on the integration path:
+  - Streaming tool execution exists via the streaming tool manager.
+  - Non-streaming multi-step tool execution is supported for direct routing; cascade tool paths currently focus on tool-call correctness and verification.
+

packages/core/package.json

@@ -34,6 +34,7 @@
     "lint": "eslint src --ext .ts",
     "typecheck": "tsc --noEmit",
     "typecheck:examples": "tsc --noEmit -p examples/nodejs/tsconfig.json",
+    "real-api:smoke": "tsx scripts/real-api-smoke.ts",
     "docs": "typedoc",
     "docs:watch": "typedoc --watch",
     "clean": "rm -rf dist"

packages/core/scripts/real-api-smoke.ts

@@ -0,0 +1,116 @@
+/* Real API smoke checks (developer-run).
+ *
+ * Runs a small set of calls against configured providers to validate:
+ * - basic text completion
+ * - tool-call generation + tool execution loop (direct tool loop)
+ * - cost/acceptance fields are populated
+ *
+ * Usage (from repo root):
+ *   set -a && source .env && set +a
+ *   pnpm -C packages/core run real-api:smoke
+ */
+
+import { CascadeAgent } from '../src/index.ts';
+import type { Tool } from '../src/types.ts';
+
+type Env = Record<string, string | undefined>;
+
+function requireAny(env: Env, keys: string[]): void {
+  if (!keys.some((k) => Boolean(env[k]))) {
+    throw new Error(`Missing API keys: expected at least one of: ${keys.join(', ')}`);
+  }
+}
+
+const env = process.env as Env;
+requireAny(env, ['OPENAI_API_KEY', 'ANTHROPIC_API_KEY']);
+
+const getWeatherTool: Tool = {
+  type: 'function',
+  function: {
+    name: 'get_weather',
+    description: 'Get the weather for a location.',
+    parameters: {
+      type: 'object',
+      properties: {
+        location: { type: 'string' },
+      },
+      required: ['location'],
+    },
+  },
+};
+
+async function main(): Promise<void> {
+  const openaiKey = env.OPENAI_API_KEY;
+  const anthropicKey = env.ANTHROPIC_API_KEY;
+
+  if (openaiKey) {
+    const agent = new CascadeAgent({
+      models: [
+        { name: 'gpt-4o-mini', provider: 'openai', cost: 0.00015, apiKey: openaiKey },
+        { name: 'gpt-4o', provider: 'openai', cost: 0.00625, apiKey: openaiKey },
+      ],
+    });
+
+    const r1 = await agent.run('Return exactly: OK');
+    if (!r1.content || !r1.content.includes('OK')) {
+      throw new Error(`OpenAI smoke failed: unexpected content: ${JSON.stringify(r1.content)}`);
+    }
+
+    // Tool loop: direct path only (keeps this deterministic).
+    const r2 = await agent.run("Call get_weather for location 'Paris'. Then respond with the result.", {
+      tools: [getWeatherTool],
+      forceDirect: true,
+      maxSteps: 3,
+      toolExecutor: async (call) => {
+        const name = call.function?.name ?? call.name;
+        if (name !== 'get_weather') return { ok: false, error: 'unknown_tool' };
+        return { location: 'Paris', forecast: 'sunny' };
+      },
+    });
+    if (!r2.content || !/sunny/i.test(r2.content)) {
+      throw new Error(`OpenAI tool-loop smoke failed: content=${JSON.stringify(r2.content)}`);
+    }
+
+    console.log(
+      JSON.stringify(
+        {
+          provider: 'openai',
+          text: { accepted: r1.draftAccepted, model: r1.modelUsed, cost: r1.totalCost },
+          toolLoop: { accepted: r2.draftAccepted, model: r2.modelUsed, cost: r2.totalCost },
+        },
+        null,
+        2
+      )
+    );
+  }
+
+  if (anthropicKey) {
+    const agent = new CascadeAgent({
+      models: [
+        { name: 'claude-haiku-4-5-20251001', provider: 'anthropic', cost: 0.003, apiKey: anthropicKey },
+        { name: 'claude-opus-4-5-20251101', provider: 'anthropic', cost: 0.045, apiKey: anthropicKey },
+      ],
+    });
+
+    const r1 = await agent.run('Return exactly: OK');
+    if (!r1.content || !r1.content.includes('OK')) {
+      throw new Error(`Anthropic smoke failed: unexpected content: ${JSON.stringify(r1.content)}`);
+    }
+
+    console.log(
+      JSON.stringify(
+        {
+          provider: 'anthropic',
+          text: { accepted: r1.draftAccepted, model: r1.modelUsed, cost: r1.totalCost },
+        },
+        null,
+        2
+      )
+    );
+  }
+}
+
+main().catch((err) => {
+  console.error(String(err?.stack ?? err));
+  process.exit(1);
+});

scripts/extended-e2e-session.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Extended E2E validation session (real APIs).
+#
+# Usage:
+#   set -a && source .env && set +a
+#   ./scripts/extended-e2e-session.sh smoke
+#
+# Profiles: smoke | standard | overnight | full
+
+PROFILE="${1:-smoke}"
+TS="$(date +%Y%m%d_%H%M%S)"
+OUTDIR="benchmark_results/sessions/${TS}_${PROFILE}"
+
+mkdir -p "${OUTDIR}"
+
+{
+  echo "timestamp=${TS}"
+  echo "profile=${PROFILE}"
+  echo "git_sha=$(git rev-parse HEAD)"
+  echo "git_branch=$(git rev-parse --abbrev-ref HEAD)"
+  echo "node=$(node -v 2>/dev/null || true)"
+  echo "pnpm=$(pnpm -v 2>/dev/null || true)"
+  echo "python=$(python3 -V 2>/dev/null || true)"
+} >"${OUTDIR}/meta.txt"
+
+echo "Output: ${OUTDIR}"
+
+echo
+echo "== TS + Python Unit/Integration Tests =="
+pnpm test 2>&1 | tee "${OUTDIR}/pnpm_test.log"
+python3 -m pytest 2>&1 | tee "${OUTDIR}/pytest.log"
+
+echo
+echo "== Vercel AI Handler E2E (TS) =="
+pnpm -C packages/core exec vitest run \
+  src/vercel-ai/__tests__/e2e.test.ts \
+  src/__tests__/vercel-ai-chat-handler.e2e.test.ts 2>&1 | tee "${OUTDIR}/vercel_ai_e2e_vitest.log"
+
+echo
+echo "== TS Real API Smoke (optional) =="
+pnpm -C packages/core run real-api:smoke 2>&1 | tee "${OUTDIR}/ts_real_api_smoke.log" || true
+
+echo
+echo "== Python Benchmarks (triad) =="
+python3 tests/benchmarks/run_benchmarks.py \
+  $([[ "${PROFILE}" == "smoke" ]] && echo "--quick" || true) \
+  --output "${OUTDIR}/triad.json" 2>&1 | tee "${OUTDIR}/triad.log" || true
+
+echo
+echo "== Python Benchmarks (broad suite) =="
+python3 -m tests.benchmarks.run_all \
+  --profile "${PROFILE}" \
+  --output-dir "${OUTDIR}/run_all" 2>&1 | tee "${OUTDIR}/run_all.log"
+
+echo
+echo "Completed. Results in: ${OUTDIR}"
+

tests/benchmarks/README.md

@@ -8,6 +8,10 @@ Professional benchmarks to validate CascadeFlow performance across real-world us
 2. **Bitext Customer Support** - Customer service Q&A (27,000+ examples)
 3. **Banking77** - Banking intent classification (13,000+ examples)
 4. **GSM8K** - Grade school math reasoning (8,500+ problems)
+5. **MT-Bench** - Multi-turn chat quality / routing behavior (sampled)
+6. **TruthfulQA** - Factual correctness (sampled)
+7. **Tool Calling** - Structured tool selection correctness (single + multi-turn)
+8. **BFCL Agentic** - Agentic/multi-turn tool-calling patterns (dependencies, chaining)
 
 #### Metrics
 
@@ -20,14 +24,14 @@ Each benchmark measures:
 #### Running Benchmarks
 
 ```bash
-# Run a single benchmark
-python -m benchmarks.datasets.humaneval
+# Load API keys (repo root .env)
+set -a && source .env && set +a
 
-# Run all benchmarks
-python -m benchmarks.run_all
+# Run quick triad suite (GSM8K + MMLU + MT-Bench)
+python3 tests/benchmarks/run_benchmarks.py --quick --output benchmark_results/e2e_quick.json || true
 
-# View results
-ls benchmarks/results/
+# Run broad suite (HumanEval, GSM8K, MT-Bench, TruthfulQA, Banking77, tool calling, etc.)
+python3 -m tests.benchmarks.run_all --profile smoke --output-dir benchmark_results/smoke
 ```
 
 #### Output
@@ -39,15 +43,19 @@ ls benchmarks/results/
 #### Structure
 
 ```
-benchmarks/
-├── base.py          # Abstract benchmark class
-├── metrics.py       # Cost/latency/quality calculations
-├── reporter.py      # Report generation
-├── humaneval.py     # Code generation benchmark
-├── customer_support.py  # Customer service Q&A
-├── banking77.py     # Banking intent classification
-├── gsm8k.py         # Math reasoning
-└── results/         # Output directory
+tests/benchmarks/
+├── base.py              # Benchmark base class + summary metrics
+├── run_benchmarks.py    # GSM8K + MMLU + MT-Bench runner (targets)
+├── run_all.py           # Broad suite runner (reports to benchmark_results/)
+├── humaneval/           # HumanEval benchmark implementation
+├── gsm8k/               # GSM8K benchmark implementation
+├── mmlu/                # MMLU benchmark implementation
+├── mtbench/             # MT-Bench benchmark implementation
+├── truthfulqa.py        # TruthfulQA benchmark implementation
+├── banking77_benchmark.py
+├── customer_support.py
+├── tool_calls.py
+└── tool_calls_agentic.py
 ```
 
 All benchmarks extend the `Benchmark` base class.