From 720eca118bad27082f5446b0a0c641aff4ec239e Mon Sep 17 00:00:00 2001 From: saschabuehrle Date: Mon, 20 Apr 2026 20:14:06 +0200 Subject: [PATCH 1/8] docs(skills): add Claude Code / Codex skill for cascadeflow users MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds skills/cascadeflow/SKILL.md — a distributable skill that gives Claude Code and Codex agents up-to-date guidance on using cascadeflow: the three-tier API (init / run / @agent), runtime intelligence harness, drafter+verifier selection, tool loops, framework integrations, and the common pitfalls students and contributors hit. Also carves a narrow exception in .gitignore so this one published skill file is committed while personal/local skill files under skills/ remain ignored. Install paths for consumers: Claude Code: ~/.claude/skills/cascadeflow/SKILL.md Codex: ~/.agents/skills/cascadeflow/SKILL.md --- .gitignore | 7 +- skills/cascadeflow/SKILL.md | 295 ++++++++++++++++++++++++++++++++++++ 2 files changed, 300 insertions(+), 2 deletions(-) create mode 100644 skills/cascadeflow/SKILL.md diff --git a/.gitignore b/.gitignore index a05d9124..0970e904 100644 --- a/.gitignore +++ b/.gitignore @@ -49,8 +49,11 @@ CLAUDE.md AGENTS.md CODEX.md -# AI assistant skill definitions -skills/ +# AI assistant skill definitions (personal/local only) +skills/* +# Exception: distributable skills published with the project +!skills/cascadeflow/ +!skills/cascadeflow/** # ============================================================================ # INTERNAL PLANNING (local only, not for GitHub) diff --git a/skills/cascadeflow/SKILL.md b/skills/cascadeflow/SKILL.md new file mode 100644 index 00000000..0524556a --- /dev/null +++ b/skills/cascadeflow/SKILL.md @@ -0,0 +1,295 @@ +--- +name: cascadeflow +description: Use when building, extending, or debugging AI agents with cascadeflow (the agent runtime intelligence layer) — installing `cascadeflow` (Python) or `@cascadeflow/core`/`@cascadeflow/langchain` (TypeScript); using `CascadeAgent`/`ModelConfig`; harness APIs `cascadeflow.init`, `cascadeflow.run`, `@cascadeflow.agent`, `simulate`; `withCascade`/`CascadeFlow`; picking drafter+verifier pairs; per-step budget/compliance/KPI enforcement; quality validation; pre-routing by complexity; tool execution and multi-turn agent loops; presets (`auto_agent`, `get_cost_optimized_agent`); user profiles and tiers; decision traces; or wiring cascadeflow into LangChain, OpenAI Agents SDK, CrewAI, PydanticAI, Google ADK, n8n, or Vercel AI SDK. Also use when a user mentions "cascade", "drafter/verifier", "runtime intelligence", "in-process harness", "cost-optimized agent", "agent loop with cost control", or is working inside the lemony-ai/cascadeflow repo. +--- + +# cascadeflow + +## What it is + +**Agent runtime intelligence layer.** An in-process harness that sits *inside* the agent execution loop (not at the HTTP boundary) and makes per-step decisions on cost, latency, quality, budget, compliance, and energy. Sub-5ms overhead. Works alongside LangChain, OpenAI Agents SDK, CrewAI, PydanticAI, Google ADK, n8n, and Vercel AI SDK. + +Two complementary pieces: + +1. **Cascading** — try a cheap "drafter" model first, validate quality, escalate to a "verifier" model only when needed (40–85% cost savings). +2. **Runtime intelligence (harness)** — instrument the agent loop with budget caps, KPI weights, compliance gates, and a full per-step decision trace. + +Python (`pip install cascadeflow`) and TypeScript (`@cascadeflow/core`). Docs: https://docs.cascadeflow.ai + +## When to use this skill + +- User is building an AI agent and wants cost/latency/quality control *inside* the loop +- Code imports `cascadeflow`, `@cascadeflow/core`, `@cascadeflow/langchain`, `@cascadeflow/vercel-ai`, or `@cascadeflow/n8n-nodes-cascadeflow` +- Mentions budgets, compliance (GDPR/HIPAA/PCI), KPI weights, tool-call routing, decision traces, drafter/verifier +- Working inside `lemony-ai/cascadeflow` (examples, integrations, gateway server) + +## Pick the right entry point (30-second decision) + +| Situation | Use | File/pattern | +|---|---|---| +| Existing OpenAI/Anthropic app, want instant observability | `cascadeflow.init(mode="observe")` | Auto-patches the SDKs. Zero code changes in the app. | +| Existing app, no code changes at all, want gateway | `python -m cascadeflow.server` | Drop-in OpenAI/Anthropic-compatible proxy; point client at `http://127.0.0.1:/v1` | +| New agent, want the default "just works" cascade | `auto_agent()` or `get_cost_optimized_agent()` | Presets — fastest path; no model picking required | +| New agent, custom drafter+verifier | `CascadeAgent(models=[drafter, verifier])` | Both languages | +| Agent function with budget + policy metadata | `@cascadeflow.agent(budget=..., compliance=..., kpi_weights=...)` | Attaches metadata; combine with `cascadeflow.run()` for enforcement | +| Scoped run with budget and full trace | `with cascadeflow.run(budget=0.50, max_tool_calls=10) as session:` | Primary harness pattern | +| Inside LangChain / OpenAI Agents / CrewAI / PydanticAI / Google ADK / Vercel AI / n8n | Use the integration package | Don't reinvent — the integrations preserve tool calling, streaming, callbacks | + +## Minimum viable cascade + +**Python:** + +```python +from cascadeflow import CascadeAgent, ModelConfig + +agent = CascadeAgent(models=[ + ModelConfig(name="gpt-4o-mini", provider="openai", cost=0.000375), # drafter + ModelConfig(name="gpt-4o", provider="openai", cost=0.00625), # verifier +]) + +result = await agent.run("What's the capital of France?") +print(result.content, result.model_used, result.total_cost, result.cost_saved) +``` + +**TypeScript:** + +```ts +import { CascadeAgent } from '@cascadeflow/core'; + +const agent = new CascadeAgent({ + models: [ + { name: 'gpt-4o-mini', provider: 'openai', cost: 0.000375 }, + { name: 'gpt-4o', provider: 'openai', cost: 0.00625 }, + ], +}); + +const r = await agent.run('What is TypeScript?'); +console.log(r.modelUsed, r.totalCost, r.savingsPercentage); +``` + +**Even faster — presets (Python):** + +```python +from cascadeflow import auto_agent, get_cost_optimized_agent + +agent = auto_agent() # picks a sensible pair +# or: get_cost_optimized_agent(), get_balanced_agent(), +# get_quality_optimized_agent(), get_speed_optimized_agent(), +# get_development_agent() +``` + +## Runtime intelligence — the harness + +This is what makes cascadeflow different from a proxy or a model router. The harness runs **inside** the agent loop and decides per step. + +### Three modes, safe rollout + +- `off` — no instrumentation (default) +- `observe` — patches OpenAI + Anthropic SDKs, records cost/tokens/decisions, enforces nothing +- `enforce` — same, plus applies actions (see below) + +### Per-step actions the harness can take + +`allow` · `switch_model` · `deny_tool` · `stop` + +Every LLM call, tool call, and sub-agent handoff is a decision point. The harness reads the current run state (cost so far, budget remaining, compliance flag, KPI weights) and chooses one of the four actions. + +### Scoped runs with budget + trace (the demo-worthy pattern) + +```python +import cascadeflow + +cascadeflow.init(mode="enforce") # or "observe" while you tune + +with cascadeflow.run( + budget=0.50, # hard USD cap + max_tool_calls=10, + max_latency_ms=15000, + max_energy=None, + kpi_weights={"quality": 0.6, "cost": 0.3, "latency": 0.1}, + compliance="gdpr", # blocks non-compliant models +) as session: + result = await agent.run("Analyze this dataset") + print(session.summary()) # cost, tokens, steps, tool_calls, last_action, budget_remaining + for entry in session.trace(): # per-step decision audit + print(entry) + session.save("run.jsonl") # exportable trace — great for demos / submissions +``` + +### Policy metadata on agent functions + +```python +@cascadeflow.agent( + budget=0.20, + kpi_weights={"quality": 0.6, "cost": 0.3, "latency": 0.1}, + compliance="gdpr", +) +async def my_agent(query: str): ... +``` + +`@cascadeflow.agent` **attaches metadata** — it doesn't change the function's runtime by itself. Combine with `cascadeflow.init(mode="enforce")` and/or `cascadeflow.run(...)` to enforce. Works on sync or async functions. + +### Zero-code config (env + file) + +All harness settings also read from env vars and a config file — so students can demo `observe → enforce` rollout without touching code. + +```bash +export CASCADEFLOW_HARNESS_MODE=enforce +export CASCADEFLOW_HARNESS_BUDGET=0.50 +export CASCADEFLOW_HARNESS_MAX_TOOL_CALLS=10 +export CASCADEFLOW_HARNESS_KPI_WEIGHTS='{"quality":0.6,"cost":0.3,"latency":0.1}' +# or point at a file: +export CASCADEFLOW_CONFIG=./cascadeflow.yaml +``` + +Precedence: explicit kwargs > env > config file > defaults. `HarnessInitReport.config_sources` tells you which source won. + +### Simulate before running (for tuning and pitch slides) + +```python +from cascadeflow.harness import simulate +report = simulate(...) # model a run against historical traces without calling providers +``` + +## Agent loops — tools, multi-turn, multi-agent + +cascadeflow's harness is built for multi-step agents, not just single calls. + +- **Tool calling** — universal tool format across providers; drafter can be pinned for simple tool calls while verifier handles complex reasoning. +- **Multi-turn loops** — automatic tool call → result → re-prompt with full history preservation (`tool_calls`, `tool_call_id` preserved across turns). +- **Per-tool-call gating** — block or re-route tools based on risk/complexity (TS: `tool-risk.ts`, `ToolRouter`). +- **Agent-as-a-tool / multi-agent** — delegate sub-tasks to other agents; each sub-call runs through the same harness. +- **Hooks & callbacks** — telemetry, cost events, streaming events. + +**Starter examples in the repo** (all exist — verified): + +| Pattern | Python | TypeScript | +|---|---|---| +| Tool execution | `examples/tool_execution.py` | `packages/core/examples/nodejs/tool-execution.ts` | +| Multi-turn tool loop | `examples/multi_step_cascade.py` | `packages/core/examples/nodejs/agentic-multi-agent.ts` | +| Streaming tools | `examples/streaming_tools.py` | `packages/core/examples/nodejs/streaming-tools.ts` | +| Multi-agent / agent-as-a-tool | `examples/agentic_multi_agent.py` | `packages/core/examples/nodejs/agentic-multi-agent.ts` | +| Harness + budget enforcement | `examples/enforcement/basic_enforcement.py` | — | +| User budget tracking | `examples/user_budget_tracking.py` | — | +| Guardrails | `examples/guardrails_usage.py` | — | +| Rate limiting | `examples/rate_limiting_usage.py` | — | + +## Picking drafter + verifier (the decision that decides savings) + +The drafter should be ~8–20× cheaper than the verifier and actually able to answer the common case. If the drafter is too weak, escalation rate climbs and savings collapse. + +| Use case | Drafter | Verifier | +|---|---|---| +| General chat (OpenAI) | `gpt-4o-mini` | `gpt-4o` or `gpt-5` | +| Cross-provider | `claude-haiku` / `gpt-4o-mini` | `claude-sonnet-4-5` / `gpt-5` | +| Code / reasoning | `gpt-4o-mini` | Reasoning model (o-series, `claude-sonnet-4-5`, `deepseek-r1`) | +| Local / edge | Ollama small (`llama3.1:8b`, `qwen2.5:7b`) | Local large or cloud fallback | + +**TS helpers to pick from your configured LangChain models** (all real — exported from `@cascadeflow/langchain`): + +```ts +import { + findBestCascadePair, discoverCascadePairs, analyzeModel, + validateCascadePair, analyzeCascadePair, suggestCascadePairs, +} from '@cascadeflow/langchain'; +``` + +## Pre-routing by complexity (TS) + +For agents where most queries are simple and a few are hard, pre-route so HARD queries skip the drafter entirely and go straight to the verifier. + +```ts +import { PreRouter, ComplexityDetector } from '@cascadeflow/langchain'; +// PreRouter config uses ComplexityDetector to classify SIMPLE / MEDIUM / HARD +``` + +Python equivalent: `ComplexityDetector`, `QueryComplexity` from `cascadeflow.quality.complexity`. + +## Quality validation + +Default: length + confidence (logprobs) + format checks. Opt in to ML-based semantic similarity for better escalation decisions: + +- Python: `pip install cascadeflow[semantic]` → `from cascadeflow.quality.semantic import SemanticQualityChecker` +- TS: `npm install @cascadeflow/ml @huggingface/transformers`, then `quality: { useSemanticValidation: true, semanticThreshold: 0.5 }` on `CascadeAgent` + +Tune `qualityThreshold` (TS) / `quality_threshold` (Py) to hit a target drafter-handled rate. 0.6–0.8 is a reasonable hackathon default. Higher threshold → more escalations → less savings. + +## Multi-tenant demos — user profiles & tiers + +```python +from cascadeflow import UserProfile, UserProfileManager, TierLevel, TIER_PRESETS +# Per-user budget enforcement, tier-aware routing (FREE/STARTER/PRO/BUSINESS/ENTERPRISE) +``` + +See `examples/user_profile_usage.py` and `examples/user_budget_tracking.py`. Great for SaaS-style hackathon submissions. + +## Framework integrations (pick one, don't reinvent) + +All of the following exist in the repo — verified on current main: + +| Framework | Package / module | Entry point | +|---|---|---| +| LangChain (TS) | `@cascadeflow/langchain` | `withCascade({ drafter, verifier, qualityThreshold })` | +| LangChain (Py) | `cascadeflow.integrations.langchain` | `CascadeFlow(drafter=..., verifier=..., quality_threshold=...)` | +| LangChain callbacks (Py) | `cascadeflow.integrations.langchain.langchain_callbacks` | `get_cascade_callback()` | +| OpenAI Agents SDK | `cascadeflow.integrations.openai_agents` | See `examples/integrations/openai_agents_harness.py` | +| CrewAI | `cascadeflow.integrations.crewai` | See `examples/integrations/crewai_harness.py` | +| PydanticAI | `cascadeflow.integrations.pydantic_ai` | See `examples/integrations/pydantic_ai_harness.py` | +| Google ADK | `cascadeflow.integrations.google_adk` | See `examples/integrations/google_adk_harness.py` | +| n8n | `@cascadeflow/n8n-nodes-cascadeflow` | CascadeFlow Model + CascadeFlow Agent nodes | +| Vercel AI SDK | `@cascadeflow/vercel-ai` | Middleware for `ai` package; 17+ extra providers | +| OTel / Grafana | `cascadeflow.integrations.otel` | See `examples/integrations/opentelemetry_grafana.py` | +| LiteLLM | `cascadeflow.integrations.litellm` | See `examples/integrations/litellm_providers.py` | + +When adding cascadeflow to a project already using one of these, prefer the integration package over raw `CascadeAgent` — keeps tool calling, streaming, and callbacks working. + +## Common pitfalls + +- **`@cascadeflow.agent` alone does nothing at runtime.** It attaches metadata. Pair with `cascadeflow.init(mode="enforce")` and/or `cascadeflow.run(...)` to actually enforce budgets/compliance. +- **`observe` mode does not stop on overrun.** Switch to `enforce` (or wrap in `cascadeflow.run(budget=...)`) to actually cut off. +- **Drafter too weak → escalation rate ~100%.** Log `result.model_used` on a sample; if the drafter is never "accepted", lower `quality_threshold` or upgrade the drafter. +- **Pairing two models of similar price.** No meaningful savings. Pick drafter and verifier from different tiers. +- **Per-provider auth.** cascadeflow does not proxy auth. Each provider still needs its own `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, etc. +- **GPT-5 streaming requires org verification.** Non-streaming works for all users. If streaming breaks during a demo, flip to non-streaming or pick a different verifier. +- **Forgetting `[all]` extras.** `pip install cascadeflow[all]` pulls every provider + semantic validation. Otherwise install per-provider extras (`[openai]`, `[anthropic]`, `[groq]`, `[together]`, `[vllm]`, `[huggingface]`, `[local]`, `[semantic]`, `[langchain]`, `[crewai]`). +- **Expecting local clones to match docs.** The GitHub README and PyPI package are authoritative. Check `cascadeflow.__version__` and compare against [latest release](https://github.com/lemony-ai/cascadeflow/releases). + +## Prove the savings in your demo + +```python +print(f"Model used: {result.model_used}") +print(f"Cost: ${result.total_cost:.6f}") +print(f"Saved: ${result.cost_saved:.6f} ({result.cost_saved_percentage():.1f}%)") +print(f"Draft/verifier breakdown: ${result.draft_cost:.6f} / ${result.verifier_cost:.6f}") +``` + +For aggregate across a run: `session.summary()` (harness) or the LangChain callback: + +```python +from cascadeflow.integrations.langchain.langchain_callbacks import get_cascade_callback +with get_cascade_callback() as cb: + await cascade.ainvoke("...") + print(cb.total_cost, cb.drafter_cost, cb.verifier_cost, cb.total_tokens) +``` + +TS: `result.savingsPercentage` directly — use it in the UI. + +## Where to look next + +- Docs: https://docs.cascadeflow.ai +- Python API: https://docs.cascadeflow.ai/api-reference/python/overview +- TypeScript API: https://docs.cascadeflow.ai/api-reference/typescript/overview +- Agent harness: https://docs.cascadeflow.ai/get-started/agent-harness +- Rollout guide (observe → enforce): https://docs.cascadeflow.ai/get-started/rollout-guide +- Providers + presets: https://docs.cascadeflow.ai/developers/providers-and-presets +- Python examples: `./examples/` — start with `basic_usage.py`, `multi_step_cascade.py`, `tool_execution.py`, `enforcement/basic_enforcement.py` +- TS examples: `./packages/core/examples/nodejs/` — start with `basic-usage.ts`, `tool-calling.ts`, `agentic-multi-agent.ts`, `cost-tracking.ts` + +## Red flags — stop and re-check + +- Writing your own retry/escalation loop around two model calls → use `CascadeAgent` or a preset. +- Hand-rolling budget tracking on top of OpenAI/Anthropic calls → use `cascadeflow.init(mode="enforce")` + `cascadeflow.run(budget=...)`. +- Computing cost savings manually by subtracting hardcoded prices → use `result.total_cost` / `result.cost_saved` / `result.savings_percentage`, or the LangChain callback. +- Drafter and verifier from the same tier (e.g. `gpt-4o` + `gpt-4o`) → no meaningful savings. +- Treating `@cascadeflow.agent` as enforcement — it's metadata only. +- Demoing `observe` mode and claiming "budget enforced" — observe doesn't stop calls. Use `enforce` or `run(budget=...)`. From 68a2b7bbe263a0148443c898e12e0395f66f4a21 Mon Sep 17 00:00:00 2001 From: saschabuehrle Date: Tue, 28 Apr 2026 21:27:27 +0200 Subject: [PATCH 2/8] docs(skills): cover full agent-in-loop API surface Validated the skill the way a hackathon developer will use it: dispatched a subagent with SKILL.md as their only reference and asked them to build a cost-optimized agent with budget cap, tool calling, graceful stop handling, trace export, and a callback hook. RED-phase report identified the gaps that block a 30-minute demo: - no proxy-vs-in-process comparison (the core "why agent-in-loop" pitch) - stop-action contract undocumented: which exception, which attribute - session.trace() / session.summary() field shapes not shown - specific stop reason strings (budget_exceeded, max_tool_calls_reached, compliance_no_approved_model, latency_limit_exceeded, energy_limit_exceeded) absent - no Python tool-registration example (ToolConfig + ToolExecutor + tools= on agent.run) - callback hook for cost/decision streaming was a one-liner with no API - max_latency_ms semantics (cumulative vs per-step) unspecified - self-improving aspect not mentioned GREEN-phase patch: - Add "Why in-the-loop matters" comparison table (proxy vs harness) - Document HarnessStopError + BudgetExceededError handling with try/except - Document the five verbatim stop reason strings - Show session.summary() and session.trace() dict shapes inline - Add ToolConfig + ToolExecutor + tools= example for CascadeAgent - Add CallbackManager + CallbackEvent + callback_manager= wiring example - Clarify max_latency_ms is cumulative across the run - Add self-improving paragraph to agent loops section GREEN re-test with the same hackathon brief moved every previously NOT-COVERED in-loop advantage to COVERED, and the verdict from "Partially" to "Yes, ships in under 30 minutes." Frontmatter still 984 / 1024 chars. --- skills/cascadeflow/SKILL.md | 143 +++++++++++++++++++++++++++++++++++- 1 file changed, 139 insertions(+), 4 deletions(-) diff --git a/skills/cascadeflow/SKILL.md b/skills/cascadeflow/SKILL.md index 0524556a..60fab0f3 100644 --- a/skills/cascadeflow/SKILL.md +++ b/skills/cascadeflow/SKILL.md @@ -16,6 +16,23 @@ Two complementary pieces: Python (`pip install cascadeflow`) and TypeScript (`@cascadeflow/core`). Docs: https://docs.cascadeflow.ai +## Why "in the loop" matters (the core pitch) + +cascadeflow is **not a proxy or a gateway**. It runs inside the agent's process and sees every model call, tool call, and sub-agent handoff as it happens — so it can act on running state (cost so far, tool calls used, compliance flag) at *each step*, not just per HTTP request. + +| Dimension | External proxy | cascadeflow harness | +|---|---|---| +| Scope | HTTP request boundary | Inside the agent loop | +| What it can see | One request at a time | Full run state (cost-so-far, step #, tool-calls used, budget remaining) | +| Optimization axes | Cost only | Cost · latency · quality · budget · compliance · energy — simultaneously | +| Latency overhead | 10–50 ms network RTT per call | <5 ms in-process per call | +| 10-step agent loop | +400–600 ms avoidable | negligible | +| Enforcement | Observe only | `allow` · `switch_model` · `deny_tool` · `stop` | +| Auditability | Request logs | Per-step decision trace (one entry per LLM/tool/handoff decision) | +| Business logic | None | Live KPI weights + targets injected at runtime | + +This is what unlocks: stop-after-step-7 budget enforcement, deny-this-tool-mid-loop, switch-models-on-this-call, and a full audit trail of *why* every step did what it did. None of that is possible from outside the loop. + ## When to use this skill - User is building an AI agent and wants cost/latency/quality control *inside* the loop @@ -94,6 +111,30 @@ This is what makes cascadeflow different from a proxy or a model router. The har Every LLM call, tool call, and sub-agent handoff is a decision point. The harness reads the current run state (cost so far, budget remaining, compliance flag, KPI weights) and chooses one of the four actions. +**Stop reasons (verbatim strings on the trace + on `HarnessStopError.reason`):** + +`budget_exceeded` · `max_tool_calls_reached` · `compliance_no_approved_model` · `latency_limit_exceeded` · `energy_limit_exceeded` + +### Handling stops gracefully (don't crash the demo) + +In `enforce` mode the harness raises a typed exception when it stops a run. Catch them so the agent can summarize and exit cleanly: + +```python +from cascadeflow.schema.exceptions import BudgetExceededError, HarnessStopError + +try: + result = await agent.run(query) +except BudgetExceededError as e: + print(f"Stopped: budget exceeded. Remaining: ${e.remaining:.4f}") +except HarnessStopError as e: + print(f"Stopped: {e.reason}") # e.g. "max_tool_calls_reached" +finally: + print(session.summary()) # cost/steps/tool_calls captured up to the stop + session.save("run.jsonl") # full trace still exportable +``` + +`max_latency_ms` is **cumulative across the run** (not per step) — `latency_used_ms` accumulates and triggers `latency_limit_exceeded` when it crosses the cap. + ### Scoped runs with budget + trace (the demo-worthy pattern) ```python @@ -104,18 +145,54 @@ cascadeflow.init(mode="enforce") # or "observe" while you tune with cascadeflow.run( budget=0.50, # hard USD cap max_tool_calls=10, - max_latency_ms=15000, + max_latency_ms=15000, # cumulative across the run max_energy=None, kpi_weights={"quality": 0.6, "cost": 0.3, "latency": 0.1}, compliance="gdpr", # blocks non-compliant models ) as session: result = await agent.run("Analyze this dataset") - print(session.summary()) # cost, tokens, steps, tool_calls, last_action, budget_remaining + print(session.summary()) # see shape below for entry in session.trace(): # per-step decision audit print(entry) session.save("run.jsonl") # exportable trace — great for demos / submissions ``` +### Shapes you'll actually print + +`session.summary()` → dict: + +```python +{ + "run_id": "ab12cd34ef56", "mode": "enforce", "step_count": 7, "tool_calls": 3, + "cost": 0.0421, "savings": 0.0118, "latency_used_ms": 4820.4, "energy_used": 0.0, + "budget_max": 0.50, "budget_remaining": 0.4579, + "last_action": "allow", "model_used": "gpt-4o-mini", "duration_ms": 5103.2, +} +``` + +`session.trace()` → list of dicts, one per decision: + +```python +{ + "action": "switch_model", # allow | switch_model | deny_tool | stop + "reason": "budget_pressure", # human-readable; on stop it's the reason code + "model": "gpt-4o-mini", + "run_id": "ab12cd34ef56", + "mode": "enforce", + "step": 4, + "timestamp_ms": 1730000123456.0, + "tool_calls_total": 2, + "cost_total": 0.0312, + "latency_used_ms": 2400.1, + "energy_used": 0.0, + "budget_state": {"max": 0.50, "remaining": 0.4688}, + "applied": true, # false for observe-mode "would have" + "decision_mode": "pre_call", # optional +} +``` + +`session.save("run.jsonl")` writes one session-header line + one trace line per decision. `HarnessRunContext.load("run.jsonl")` reads it back as `{"session": ..., "traces": [...]}`. + ### Policy metadata on agent functions ```python @@ -158,8 +235,66 @@ cascadeflow's harness is built for multi-step agents, not just single calls. - **Tool calling** — universal tool format across providers; drafter can be pinned for simple tool calls while verifier handles complex reasoning. - **Multi-turn loops** — automatic tool call → result → re-prompt with full history preservation (`tool_calls`, `tool_call_id` preserved across turns). - **Per-tool-call gating** — block or re-route tools based on risk/complexity (TS: `tool-risk.ts`, `ToolRouter`). -- **Agent-as-a-tool / multi-agent** — delegate sub-tasks to other agents; each sub-call runs through the same harness. -- **Hooks & callbacks** — telemetry, cost events, streaming events. +- **Agent-as-a-tool / multi-agent** — delegate sub-tasks to other agents; each sub-call runs through the same harness (sub-call decisions show up on the parent's trace). +- **Hooks & callbacks** — register a `CallbackManager` to stream cost/decision events to a dashboard. +- **Self-improving** — because the harness sees every step, every tool result, and every quality score over time, it accumulates the data needed to tune routing strategies and escalation thresholds. Long-lived agents get smarter the more they run. + +### Wiring tools to the agent (Python) + +```python +from cascadeflow import CascadeAgent, ModelConfig +from cascadeflow.tools import ToolConfig, ToolExecutor + +def get_weather(city: str) -> str: + return f"{city}: 18°C, cloudy" # mock + +tool_configs = [ + ToolConfig( + name="get_weather", + description="Get current weather for a city.", + parameters={ + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + function=get_weather, + ), +] + +executor = ToolExecutor(tool_configs) +agent = CascadeAgent( + models=[ + ModelConfig(name="gpt-4o-mini", provider="openai", cost=0.000375), + ModelConfig(name="gpt-4o", provider="openai", cost=0.00625), + ], + tool_executor=executor, # executor goes on the agent +) + +# Schemas (no function ref) go on the call: +schemas = [{"name": t.name, "description": t.description, "parameters": t.parameters} + for t in tool_configs] +result = await agent.run("What's the weather in Paris?", tools=schemas) +``` + +### Streaming decision events to a dashboard + +```python +import cascadeflow +from cascadeflow.telemetry.callbacks import CallbackManager, CallbackEvent + +manager = CallbackManager() + +def on_decision(data): + # data.event, data.query, data.data — push to your dashboard / Slack / OTel + print(data.event.value, data.data) + +manager.register(CallbackEvent.CASCADE_DECISION, on_decision) +manager.register(CallbackEvent.MODEL_CALL_COMPLETE, on_decision) + +cascadeflow.init(mode="enforce", callback_manager=manager) +``` + +Available events: `QUERY_START`, `COMPLEXITY_DETECTED`, `MODEL_CALL_START`, `MODEL_CALL_COMPLETE`, `MODEL_CALL_ERROR`, `CASCADE_DECISION`, `CACHE_HIT`/`MISS`, `QUERY_COMPLETE`, `QUERY_ERROR`. For LangChain, prefer `get_cascade_callback()` (covered below). **Starter examples in the repo** (all exist — verified): From b763c52e3168c766d01da754bb4bb924f1ffd1a3 Mon Sep 17 00:00:00 2001 From: saschabuehrle Date: Tue, 28 Apr 2026 22:16:58 +0200 Subject: [PATCH 3/8] docs(skills): add upstream bug-fix / PR contribution workflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a "Found a bug? Contribute the fix back" section so that when a hackathon dev (or the agent assisting them) discovers a bug in cascadeflow itself or in any of its integrations, the skill instructs the canonical fork → fix branch → commit → push → upstream PR flow without making the dev read CONTRIBUTING.md or guess paths. Includes: - Scope rule: "Bug in cascadeflow or an integration → upstream PR. Bug in your own hackathon app → skill has no opinion." - Path map for every place a bug can live (Python core, TS core, each Python integration, each TS integration package, ml/, etc.) so the agent doesn't guess where withCascade or harness/instrument lives. - Canonical command sequence: gh repo fork --clone --remote, pre-commit install, branch off main, conventional-commit area tags, pnpm --filter test vs pytest, gh pr create against lemony-ai/cascadeflow. - "Unblock the demo while the PR is in review" with both pip install -e (Python) and pnpm pack + npm install (TS), plus a warning that npm link is flaky with pnpm workspaces. - Don't list: no main pushes, no force-pushes to shared branches, no --no-verify, no PR without a regression test, no committed secrets. Validated RED → GREEN with the same 3 scenarios: - A) bug in cascadeflow.harness.instrument - B) bug in @cascadeflow/langchain withCascade - C) bug in dev's own app RED (without section): agent produced a sensible plan but flagged the contributor flow, monorepo layout, paths, and test commands as guesses. GREEN (with section): every one of 12 audited facts came directly from the skill with quotable lines, including the Scenario-C disclaimer. Verdict moved to "Yes — PR-able fix without reading source or CONTRIBUTING.md." Frontmatter trimmed to 960 / 1024 chars to stay within the agentskills.io spec after adding the new bugfix trigger. --- skills/cascadeflow/SKILL.md | 81 ++++++++++++++++++++++++++++++++++++- 1 file changed, 80 insertions(+), 1 deletion(-) diff --git a/skills/cascadeflow/SKILL.md b/skills/cascadeflow/SKILL.md index 60fab0f3..18f2f4ca 100644 --- a/skills/cascadeflow/SKILL.md +++ b/skills/cascadeflow/SKILL.md @@ -1,6 +1,6 @@ --- name: cascadeflow -description: Use when building, extending, or debugging AI agents with cascadeflow (the agent runtime intelligence layer) — installing `cascadeflow` (Python) or `@cascadeflow/core`/`@cascadeflow/langchain` (TypeScript); using `CascadeAgent`/`ModelConfig`; harness APIs `cascadeflow.init`, `cascadeflow.run`, `@cascadeflow.agent`, `simulate`; `withCascade`/`CascadeFlow`; picking drafter+verifier pairs; per-step budget/compliance/KPI enforcement; quality validation; pre-routing by complexity; tool execution and multi-turn agent loops; presets (`auto_agent`, `get_cost_optimized_agent`); user profiles and tiers; decision traces; or wiring cascadeflow into LangChain, OpenAI Agents SDK, CrewAI, PydanticAI, Google ADK, n8n, or Vercel AI SDK. Also use when a user mentions "cascade", "drafter/verifier", "runtime intelligence", "in-process harness", "cost-optimized agent", "agent loop with cost control", or is working inside the lemony-ai/cascadeflow repo. +description: Use when building, extending, or debugging AI agents with cascadeflow (agent runtime intelligence layer) — installing `cascadeflow` (Python) or `@cascadeflow/core`/`@cascadeflow/langchain` (TypeScript); using `CascadeAgent`, `ModelConfig`, harness APIs (`cascadeflow.init`, `cascadeflow.run`, `@cascadeflow.agent`, `simulate`), `withCascade`/`CascadeFlow`; picking drafter+verifier pairs; per-step budget/compliance/KPI enforcement; quality validation; complexity pre-routing; tool execution and multi-turn agent loops; presets; decision traces; or wiring cascadeflow into LangChain, OpenAI Agents, CrewAI, PydanticAI, Google ADK, n8n, or Vercel AI SDK. Also when a user mentions "cascade", "drafter/verifier", "runtime intelligence", "in-process harness", "cost-optimized agent", "agent loop with cost control", is in the lemony-ai/cascadeflow repo, or found a bug in cascadeflow/integrations needing an upstream fix/PR. --- # cascadeflow @@ -39,6 +39,7 @@ This is what unlocks: stop-after-step-7 budget enforcement, deny-this-tool-mid-l - Code imports `cascadeflow`, `@cascadeflow/core`, `@cascadeflow/langchain`, `@cascadeflow/vercel-ai`, or `@cascadeflow/n8n-nodes-cascadeflow` - Mentions budgets, compliance (GDPR/HIPAA/PCI), KPI weights, tool-call routing, decision traces, drafter/verifier - Working inside `lemony-ai/cascadeflow` (examples, integrations, gateway server) +- A bug is discovered in cascadeflow itself or any of its integrations and needs to be fixed upstream ## Pick the right entry point (30-second decision) @@ -409,6 +410,84 @@ with get_cascade_callback() as cb: TS: `result.savingsPercentage` directly — use it in the UI. +## Found a bug? Contribute the fix back + +If you discover a bug **inside cascadeflow itself** (the `cascadeflow` Python package, `@cascadeflow/core`, or any integration package), the skill expects you to fix it upstream — fork, patch, push, open a PR — not paper over it locally. Everything ships from one monorepo: `lemony-ai/cascadeflow`. + +If the bug is in **your own hackathon app**, this skill has no opinion — follow your project's normal workflow. The flow below is for upstream fixes only. + +### Where the code lives (so the agent doesn't guess) + +| Where the bug is | Path in the monorepo | +|---|---| +| Python core | `cascadeflow/` (e.g. `cascadeflow/harness/instrument.py`, `cascadeflow/agent.py`) | +| TypeScript core | `packages/core/src/` | +| LangChain (TS) | `packages/langchain-cascadeflow/src/` | +| LangChain (Py) | `cascadeflow/integrations/langchain/` | +| OpenAI Agents (Py) | `cascadeflow/integrations/openai_agents.py` | +| CrewAI (Py) | `cascadeflow/integrations/crewai.py` | +| PydanticAI (Py) | `cascadeflow/integrations/pydantic_ai/` | +| Google ADK (Py) | `cascadeflow/integrations/google_adk.py` | +| LiteLLM (Py) | `cascadeflow/integrations/litellm.py` | +| OTel (Py) | `cascadeflow/integrations/otel.py` | +| n8n | `packages/integrations/n8n/` | +| Vercel AI SDK | `packages/integrations/vercel-ai/` | +| ML (semantic quality) | `packages/ml/` | + +### Upstream-fix workflow + +```bash +# 0. Pin & verify it's not already fixed in latest +python -c "import cascadeflow; print(cascadeflow.__version__)" +gh release list --repo lemony-ai/cascadeflow --limit 5 +gh issue list --repo lemony-ai/cascadeflow --search "" + +# 1. Fork + clone (creates origin = your fork, upstream = lemony-ai) +gh repo fork lemony-ai/cascadeflow --clone --remote +cd cascadeflow +pre-commit install # repo enforces hooks + +# 2. Branch off main — never push fixes to main +git checkout main && git pull upstream main +git checkout -b fix/ # e.g. fix/harness-max-energy-none + +# 3. Patch + add a regression test next to existing tests for that area + +# 4. Run the right test suite +pytest # Python core / Python integrations +pnpm --filter @cascadeflow/core test # TS core +pnpm --filter @cascadeflow/langchain test # TS LangChain integration +# (substitute the package for whichever folder you touched) + +# 5. Conventional-commit message — required by the repo +git commit -am "fix(): " +# areas: harness, langchain, crewai, pydantic-ai, openai-agents, +# google-adk, n8n, vercel-ai, core, docs, etc. + +# 6. Push to your fork and open the PR upstream +git push -u origin fix/ +gh pr create --repo lemony-ai/cascadeflow --base main \ + --title "fix(): " \ + --body "Fixes #. " +``` + +### Unblock the demo while the PR is in review + +Don't wait for the merge — install your patched fork into the hackathon app: + +- **Python:** `pip install -e /path/to/your/cascadeflow-fork` +- **TypeScript:** `pnpm pack` inside the patched package, then `npm install /path/to/cascadeflow--x.y.z.tgz` in the hackathon app. (`npm link` works but is flaky with pnpm workspaces.) + +After the PR merges and a release ships, swap back to the published package. + +### Don't + +- Don't push fixes directly to `main` (your fork or upstream). +- Don't `--force-push` to a shared/upstream branch. +- Don't bypass `pre-commit` with `--no-verify` — fix the lint/format issue instead. +- Don't open a PR without a regression test for non-trivial fixes. +- Don't commit API keys, `.env` files, or local config. + ## Where to look next - Docs: https://docs.cascadeflow.ai From ec6e68c8721d37d209ddd87b415b2361a6855af2 Mon Sep 17 00:00:00 2001 From: saschabuehrle Date: Tue, 28 Apr 2026 22:33:27 +0200 Subject: [PATCH 4/8] docs(skills): fix two real-dev frictions found by ground-truth dry-run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ran the upstream-fix workflow end-to-end on a real working copy (real bug, real branch, real pytest, real commit) and hit two gotchas that directly violate the skill's own rules. Fixed them. 1. Bare `pytest` fails on a fresh `pip install -e .` because the repo's pyproject pytest config injects `--cov=cascadeflow ... --asyncio-mode=auto`, and `pytest-cov` / `pytest-asyncio` aren't pulled by the base install — they live in the `[dev]` extra. Skill now says `pip install -e ".[dev]"` before `pytest`. Same gotcha added to the "Don't" list. 2. The skill's commit example was `git commit -am "fix(...)"`. The `-a` flag stages tracked changes only; the new regression test file is untracked, so it gets silently dropped. A copy-paste user would commit the fix without the test, directly violating the skill's own "Don't open a PR without a regression test" rule. Replaced with explicit `git add ` + `git commit -m`. Also added to the "Don't" list with the rationale. Plus two minor improvements from the integration-package dry-run: - Mention `pnpm install --filter @cascadeflow/... --frozen-lockfile` for faster iteration when only one package was touched. - Note that Step 0 (`gh release list` / `gh issue list`) requires `gh auth login`; suggest a web search + `git log upstream/main` as fallback for unauthed contributors. - Clarified the regression-test rule applies to "non-trivial fixes" (single-line comment/typo fixes are fine without one — confirmed by the integration dry-run, which fixed a one-line JSDoc with no test). Method: spawned three parallel real-dev subagents — (a) hackathon demo build with imports/agent-construction validated against installed cascadeflow 1.1.0, all 13 documented session.summary() keys verified; (b) core bug-fix on a worktree, real `pip install -e .`, real `pytest`, real `git commit`, real `cascadeflow.__version__` regression test that passes; (c) integration bug-fix on a worktree, real `pnpm install --frozen-lockfile`, real `pnpm --filter @cascadeflow/langchain test` (75 passed), real `git commit`. None pushed; would-be `gh pr create` commands rendered. Frontmatter still 960 / 1024 chars. --- skills/cascadeflow/SKILL.md | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/skills/cascadeflow/SKILL.md b/skills/cascadeflow/SKILL.md index 18f2f4ca..398c3a90 100644 --- a/skills/cascadeflow/SKILL.md +++ b/skills/cascadeflow/SKILL.md @@ -445,32 +445,44 @@ gh issue list --repo lemony-ai/cascadeflow --search "" # 1. Fork + clone (creates origin = your fork, upstream = lemony-ai) gh repo fork lemony-ai/cascadeflow --clone --remote cd cascadeflow + +# 2. Install dev deps + hooks. THIS IS NOT OPTIONAL. +# Python: the repo's pyproject pytest config injects --cov / --asyncio-mode=auto, +# so bare `pytest` fails on a fresh `pip install -e .` until you pull the dev extra. +pip install -e ".[dev]" # pulls pytest, pytest-cov, pytest-asyncio, pre-commit, ruff, black, mypy pre-commit install # repo enforces hooks -# 2. Branch off main — never push fixes to main +# 3. Branch off main — never push fixes to main git checkout main && git pull upstream main git checkout -b fix/ # e.g. fix/harness-max-energy-none -# 3. Patch + add a regression test next to existing tests for that area +# 4. Patch + add a regression test next to existing tests for that area -# 4. Run the right test suite +# 5. Run the right test suite pytest # Python core / Python integrations pnpm --filter @cascadeflow/core test # TS core pnpm --filter @cascadeflow/langchain test # TS LangChain integration # (substitute the package for whichever folder you touched) - -# 5. Conventional-commit message — required by the repo -git commit -am "fix(): " +# Faster iteration on a single TS package: `pnpm install --filter @cascadeflow/... --frozen-lockfile` + +# 6. Stage everything (including the new test file) and commit. DO NOT use +# `git commit -am` — `-a` skips untracked files, so your regression test +# silently won't be in the commit and the PR will fail review. +git status # confirm new test file is listed under "Untracked" +git add +git commit -m "fix(): " # areas: harness, langchain, crewai, pydantic-ai, openai-agents, # google-adk, n8n, vercel-ai, core, docs, etc. -# 6. Push to your fork and open the PR upstream +# 7. Push to your fork and open the PR upstream git push -u origin fix/ gh pr create --repo lemony-ai/cascadeflow --base main \ --title "fix(): " \ --body "Fixes #. " ``` +> **Step 0 (`gh release list`/`gh issue list`) requires `gh auth login` against your GitHub account.** If unauthed, substitute a quick web search of `github.com/lemony-ai/cascadeflow/issues` and `git log upstream/main -- ` to check for prior fixes. + ### Unblock the demo while the PR is in review Don't wait for the merge — install your patched fork into the hackathon app: @@ -485,7 +497,9 @@ After the PR merges and a release ships, swap back to the published package. - Don't push fixes directly to `main` (your fork or upstream). - Don't `--force-push` to a shared/upstream branch. - Don't bypass `pre-commit` with `--no-verify` — fix the lint/format issue instead. -- Don't open a PR without a regression test for non-trivial fixes. +- Don't `git commit -am` when you've added a new test file — `-a` skips untracked files. Use `git add` then `git commit -m`. +- Don't run bare `pytest` after `pip install -e .` — the repo's pyproject injects `--cov` and `--asyncio-mode=auto`. Install `".[dev]"` first. +- Don't open a PR without a regression test for non-trivial fixes (single-line comment/typo fixes are fine without one). - Don't commit API keys, `.env` files, or local config. ## Where to look next From 1aa5de7656e43780947f79eb97950b5550d14786 Mon Sep 17 00:00:00 2001 From: saschabuehrle Date: Tue, 28 Apr 2026 23:03:51 +0200 Subject: [PATCH 5/8] docs(skills): fix Python savings API names --- skills/cascadeflow/SKILL.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/skills/cascadeflow/SKILL.md b/skills/cascadeflow/SKILL.md index 398c3a90..ffb4df91 100644 --- a/skills/cascadeflow/SKILL.md +++ b/skills/cascadeflow/SKILL.md @@ -395,7 +395,7 @@ When adding cascadeflow to a project already using one of these, prefer the inte ```python print(f"Model used: {result.model_used}") print(f"Cost: ${result.total_cost:.6f}") -print(f"Saved: ${result.cost_saved:.6f} ({result.cost_saved_percentage():.1f}%)") +print(f"Saved: ${result.cost_saved:.6f} ({result.cost_saved_percentage:.1f}%)") print(f"Draft/verifier breakdown: ${result.draft_cost:.6f} / ${result.verifier_cost:.6f}") ``` @@ -517,7 +517,7 @@ After the PR merges and a release ships, swap back to the published package. - Writing your own retry/escalation loop around two model calls → use `CascadeAgent` or a preset. - Hand-rolling budget tracking on top of OpenAI/Anthropic calls → use `cascadeflow.init(mode="enforce")` + `cascadeflow.run(budget=...)`. -- Computing cost savings manually by subtracting hardcoded prices → use `result.total_cost` / `result.cost_saved` / `result.savings_percentage`, or the LangChain callback. +- Computing cost savings manually by subtracting hardcoded prices → use `result.total_cost` / `result.cost_saved` / `result.cost_saved_percentage`, or the LangChain callback. - Drafter and verifier from the same tier (e.g. `gpt-4o` + `gpt-4o`) → no meaningful savings. - Treating `@cascadeflow.agent` as enforcement — it's metadata only. - Demoing `observe` mode and claiming "budget enforced" — observe doesn't stop calls. Use `enforce` or `run(budget=...)`. From 189750be82dbe3f92232f1e4f59ff53ff4bd3ebb Mon Sep 17 00:00:00 2001 From: saschabuehrle Date: Tue, 28 Apr 2026 23:20:25 +0200 Subject: [PATCH 6/8] docs(skills): fix critical @cascadeflow.agent decorator bug + reviewer items MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Independent code review surfaced a critical accuracy bug plus four non-trivial issues. All confirmed against the installed package and fixed. CRITICAL — `@cascadeflow.agent(...)` raises TypeError ===================================================== `cascadeflow/__init__.py` registers two colliding names: - lazy import "agent" → the module file `cascadeflow/agent.py` - eager `harness_agent` → the actual decorator `cascadeflow.agent` resolves to the module. The skill recommended `@cascadeflow.agent(...)` in five places (frontmatter description, entry-point table, the headline policy-metadata code example, Common pitfalls, Red flags). A hackathon dev who copy-pastes hits: TypeError: 'module' object is not callable Verified empirically against pip-installed cascadeflow. Fix: switch all five sites to the working pattern: from cascadeflow.harness import agent @agent(budget=..., kpi_weights=..., compliance=...) Also note `cascadeflow.harness_agent` as the eager top-level alias. Added an explicit "Don't write @cascadeflow.agent(...)" entry to the pitfalls list with the exact TypeError so devs recognize it. Verified the patched example works against pip-installed cascadeflow: from cascadeflow.harness import agent @agent(budget=0.20, kpi_weights={...}, compliance="gdpr") async def my_agent(query: str): ... # callable: True (The root-cause collision should also be fixed upstream — drop the `"agent"` lazy alias in `cascadeflow/__init__.py`. Out of scope for this skill PR. Note left for the project README, which has the same bug at line 180.) OTHER REVIEWER ITEMS ==================== I1 — Stop-handling snippet referenced `session` without showing where it came from. Wrapped the try/except in an explicit `with cascadeflow.run(...) as session:` block. I2 — `pre-commit install` is documented but the repo has no `.pre-commit-config.yaml`. Made it conditional ("if a config file exists at the root, also run pre-commit install") and softened the related "Don't" entry. (CONTRIBUTING.md has the same stale doc bug; out of scope here.) I3 — "Mentions budgets / compliance / KPI weights" was an over-broad trigger for a globally-installed skill — could mis-fire on healthcare or fintech apps that don't use cascadeflow. Tightened to require a cascadeflow signal (import / repo / explicit mention) alongside. I4 — `simulate(...)` was framed as "model a run against historical traces". Real signature is `simulate(queries, models, quality_threshold, domain_detection)` — a query replay through the deterministic routing pipeline. Rewrote the example with the real signature and the actual return fields (projected_cost, escalation_rate, model_distribution). M1 — Moved `gh auth login` requirement to step 0 prerequisite (every gh command needs it, not just gh release/issue list). M2 — Dropped the misleading "faster iteration on a single TS package" hint — `pnpm install --filter ... --frozen-lockfile` is a CI pattern, not a speed-up. Replaced with `pnpm --filter test:watch` suggestion for vitest watch mode. Frontmatter still under cap: 975 / 1024 chars. --- skills/cascadeflow/SKILL.md | 70 +++++++++++++++++++++++-------------- 1 file changed, 44 insertions(+), 26 deletions(-) diff --git a/skills/cascadeflow/SKILL.md b/skills/cascadeflow/SKILL.md index ffb4df91..972bb4be 100644 --- a/skills/cascadeflow/SKILL.md +++ b/skills/cascadeflow/SKILL.md @@ -1,6 +1,6 @@ --- name: cascadeflow -description: Use when building, extending, or debugging AI agents with cascadeflow (agent runtime intelligence layer) — installing `cascadeflow` (Python) or `@cascadeflow/core`/`@cascadeflow/langchain` (TypeScript); using `CascadeAgent`, `ModelConfig`, harness APIs (`cascadeflow.init`, `cascadeflow.run`, `@cascadeflow.agent`, `simulate`), `withCascade`/`CascadeFlow`; picking drafter+verifier pairs; per-step budget/compliance/KPI enforcement; quality validation; complexity pre-routing; tool execution and multi-turn agent loops; presets; decision traces; or wiring cascadeflow into LangChain, OpenAI Agents, CrewAI, PydanticAI, Google ADK, n8n, or Vercel AI SDK. Also when a user mentions "cascade", "drafter/verifier", "runtime intelligence", "in-process harness", "cost-optimized agent", "agent loop with cost control", is in the lemony-ai/cascadeflow repo, or found a bug in cascadeflow/integrations needing an upstream fix/PR. +description: Use when building, extending, or debugging AI agents with cascadeflow (agent runtime intelligence layer) — installing `cascadeflow` (Python) or `@cascadeflow/core`/`@cascadeflow/langchain` (TypeScript); using `CascadeAgent`, `ModelConfig`, harness APIs (`cascadeflow.init`, `cascadeflow.run`, `@agent` from `cascadeflow.harness`, `simulate`), `withCascade`/`CascadeFlow`; picking drafter+verifier pairs; per-step budget/compliance/KPI enforcement; quality validation; complexity pre-routing; tool execution and multi-turn agent loops; presets; decision traces; or wiring cascadeflow into LangChain, OpenAI Agents, CrewAI, PydanticAI, Google ADK, n8n, or Vercel AI SDK. Also when a user mentions "cascade", "drafter/verifier", "runtime intelligence", "in-process harness", "cost-optimized agent", "agent loop with cost control", is in the lemony-ai/cascadeflow repo, or found a bug in cascadeflow/integrations needing an upstream fix/PR. --- # cascadeflow @@ -37,7 +37,7 @@ This is what unlocks: stop-after-step-7 budget enforcement, deny-this-tool-mid-l - User is building an AI agent and wants cost/latency/quality control *inside* the loop - Code imports `cascadeflow`, `@cascadeflow/core`, `@cascadeflow/langchain`, `@cascadeflow/vercel-ai`, or `@cascadeflow/n8n-nodes-cascadeflow` -- Mentions budgets, compliance (GDPR/HIPAA/PCI), KPI weights, tool-call routing, decision traces, drafter/verifier +- Mentions budgets, compliance (GDPR/HIPAA/PCI), KPI weights, tool-call routing, decision traces, drafter/verifier — *together with* a cascadeflow signal (import, repo path, or explicit cascadeflow mention). Don't fire on unrelated compliance/budget conversations in user code. - Working inside `lemony-ai/cascadeflow` (examples, integrations, gateway server) - A bug is discovered in cascadeflow itself or any of its integrations and needs to be fixed upstream @@ -49,7 +49,7 @@ This is what unlocks: stop-after-step-7 budget enforcement, deny-this-tool-mid-l | Existing app, no code changes at all, want gateway | `python -m cascadeflow.server` | Drop-in OpenAI/Anthropic-compatible proxy; point client at `http://127.0.0.1:/v1` | | New agent, want the default "just works" cascade | `auto_agent()` or `get_cost_optimized_agent()` | Presets — fastest path; no model picking required | | New agent, custom drafter+verifier | `CascadeAgent(models=[drafter, verifier])` | Both languages | -| Agent function with budget + policy metadata | `@cascadeflow.agent(budget=..., compliance=..., kpi_weights=...)` | Attaches metadata; combine with `cascadeflow.run()` for enforcement | +| Agent function with budget + policy metadata | `from cascadeflow.harness import agent` then `@agent(budget=..., compliance=..., kpi_weights=...)` | Attaches metadata; combine with `cascadeflow.run()` for enforcement. Note: import the decorator from `cascadeflow.harness` — `cascadeflow.agent` resolves to the module, not the decorator. | | Scoped run with budget and full trace | `with cascadeflow.run(budget=0.50, max_tool_calls=10) as session:` | Primary harness pattern | | Inside LangChain / OpenAI Agents / CrewAI / PydanticAI / Google ADK / Vercel AI / n8n | Use the integration package | Don't reinvent — the integrations preserve tool calling, streaming, callbacks | @@ -118,20 +118,21 @@ Every LLM call, tool call, and sub-agent handoff is a decision point. The harnes ### Handling stops gracefully (don't crash the demo) -In `enforce` mode the harness raises a typed exception when it stops a run. Catch them so the agent can summarize and exit cleanly: +In `enforce` mode the harness raises a typed exception when it stops a run. Catch them inside a `with cascadeflow.run(...) as session:` block so the agent can summarize and exit cleanly: ```python from cascadeflow.schema.exceptions import BudgetExceededError, HarnessStopError -try: - result = await agent.run(query) -except BudgetExceededError as e: - print(f"Stopped: budget exceeded. Remaining: ${e.remaining:.4f}") -except HarnessStopError as e: - print(f"Stopped: {e.reason}") # e.g. "max_tool_calls_reached" -finally: - print(session.summary()) # cost/steps/tool_calls captured up to the stop - session.save("run.jsonl") # full trace still exportable +with cascadeflow.run(budget=0.10, max_tool_calls=5) as session: + try: + result = await agent.run(query) + except BudgetExceededError as e: + print(f"Stopped: budget exceeded. Remaining: ${e.remaining:.4f}") + except HarnessStopError as e: + print(f"Stopped: {e.reason}") # e.g. "max_tool_calls_reached" + finally: + print(session.summary()) # cost/steps/tool_calls captured up to the stop + session.save("run.jsonl") # full trace still exportable ``` `max_latency_ms` is **cumulative across the run** (not per step) — `latency_used_ms` accumulates and triggers `latency_limit_exceeded` when it crosses the cap. @@ -197,7 +198,9 @@ with cascadeflow.run( ### Policy metadata on agent functions ```python -@cascadeflow.agent( +from cascadeflow.harness import agent # NOT `cascadeflow.agent` — that resolves to the module + +@agent( budget=0.20, kpi_weights={"quality": 0.6, "cost": 0.3, "latency": 0.1}, compliance="gdpr", @@ -205,7 +208,7 @@ with cascadeflow.run( async def my_agent(query: str): ... ``` -`@cascadeflow.agent` **attaches metadata** — it doesn't change the function's runtime by itself. Combine with `cascadeflow.init(mode="enforce")` and/or `cascadeflow.run(...)` to enforce. Works on sync or async functions. +The `@agent` decorator **attaches metadata** — it doesn't change the function's runtime by itself. Combine with `cascadeflow.init(mode="enforce")` and/or `cascadeflow.run(...)` to enforce. Works on sync or async functions. (`cascadeflow.harness_agent` is the same decorator re-exported at the top level if you prefer not to import from `cascadeflow.harness`.) ### Zero-code config (env + file) @@ -224,11 +227,21 @@ Precedence: explicit kwargs > env > config file > defaults. `HarnessInitReport.c ### Simulate before running (for tuning and pitch slides) +`simulate(queries, models, quality_threshold=0.7, domain_detection=True)` replays a list of queries through the deterministic complexity + domain routing pipeline — projecting which model would handle each query and the resulting cost/escalation rate — **without making any provider calls**. + ```python from cascadeflow.harness import simulate -report = simulate(...) # model a run against historical traces without calling providers + +report = simulate( + queries=["What's 2+2?", "Write a poem about Paris", "Refactor this Python loop"], + models=[drafter_config, verifier_config], + quality_threshold=0.7, +) +print(report.projected_cost, report.escalation_rate, report.model_distribution) ``` +`queries` accepts a list of strings or a path to a JSONL file with `{"query": ...}` lines (so a previously-saved `session.save("run.jsonl")` can also be replayed by extracting the queries from it). Use this to tune `quality_threshold` against representative traffic before turning on `enforce` mode. + ## Agent loops — tools, multi-turn, multi-agent cascadeflow's harness is built for multi-step agents, not just single calls. @@ -381,7 +394,8 @@ When adding cascadeflow to a project already using one of these, prefer the inte ## Common pitfalls -- **`@cascadeflow.agent` alone does nothing at runtime.** It attaches metadata. Pair with `cascadeflow.init(mode="enforce")` and/or `cascadeflow.run(...)` to actually enforce budgets/compliance. +- **The `@agent` decorator alone does nothing at runtime.** It attaches metadata. Pair with `cascadeflow.init(mode="enforce")` and/or `cascadeflow.run(...)` to actually enforce budgets/compliance. +- **Don't write `@cascadeflow.agent(...)` — it raises `TypeError: 'module' object is not callable`.** `cascadeflow.agent` is the module file, not the decorator. Use `from cascadeflow.harness import agent` and `@agent(...)`, or `@cascadeflow.harness_agent(...)`. - **`observe` mode does not stop on overrun.** Switch to `enforce` (or wrap in `cascadeflow.run(budget=...)`) to actually cut off. - **Drafter too weak → escalation rate ~100%.** Log `result.model_used` on a sample; if the drafter is never "accepted", lower `quality_threshold` or upgrade the drafter. - **Pairing two models of similar price.** No meaningful savings. Pick drafter and verifier from different tiers. @@ -437,7 +451,8 @@ If the bug is in **your own hackathon app**, this skill has no opinion — follo ### Upstream-fix workflow ```bash -# 0. Pin & verify it's not already fixed in latest +# 0. Prerequisite: `gh auth login` (every gh command below needs it). +# Pin & verify it's not already fixed in latest: python -c "import cascadeflow; print(cascadeflow.__version__)" gh release list --repo lemony-ai/cascadeflow --limit 5 gh issue list --repo lemony-ai/cascadeflow --search "" @@ -446,11 +461,13 @@ gh issue list --repo lemony-ai/cascadeflow --search "" gh repo fork lemony-ai/cascadeflow --clone --remote cd cascadeflow -# 2. Install dev deps + hooks. THIS IS NOT OPTIONAL. -# Python: the repo's pyproject pytest config injects --cov / --asyncio-mode=auto, +# 2. Install dev deps. THIS IS NOT OPTIONAL. +# The repo's pyproject pytest config injects --cov / --asyncio-mode=auto, # so bare `pytest` fails on a fresh `pip install -e .` until you pull the dev extra. -pip install -e ".[dev]" # pulls pytest, pytest-cov, pytest-asyncio, pre-commit, ruff, black, mypy -pre-commit install # repo enforces hooks +pip install -e ".[dev]" # pulls pytest, pytest-cov, pytest-asyncio, ruff, black, mypy +# If the repo has a `.pre-commit-config.yaml` at the root, also run: +# pre-commit install +# CONTRIBUTING.md mentions this; check whether the config file exists first. # 3. Branch off main — never push fixes to main git checkout main && git pull upstream main @@ -463,7 +480,7 @@ pytest # Python core / Python integrations pnpm --filter @cascadeflow/core test # TS core pnpm --filter @cascadeflow/langchain test # TS LangChain integration # (substitute the package for whichever folder you touched) -# Faster iteration on a single TS package: `pnpm install --filter @cascadeflow/... --frozen-lockfile` +# For watch mode during iteration: `pnpm --filter @cascadeflow/ test:watch` (if defined) # 6. Stage everything (including the new test file) and commit. DO NOT use # `git commit -am` — `-a` skips untracked files, so your regression test @@ -481,7 +498,7 @@ gh pr create --repo lemony-ai/cascadeflow --base main \ --body "Fixes #. " ``` -> **Step 0 (`gh release list`/`gh issue list`) requires `gh auth login` against your GitHub account.** If unauthed, substitute a quick web search of `github.com/lemony-ai/cascadeflow/issues` and `git log upstream/main -- ` to check for prior fixes. +> **Every `gh ...` command above requires `gh auth login`.** If unauthed, run that first, or substitute a web search of `github.com/lemony-ai/cascadeflow/issues` and `git log upstream/main -- ` for the prior-fix check. ### Unblock the demo while the PR is in review @@ -496,7 +513,7 @@ After the PR merges and a release ships, swap back to the published package. - Don't push fixes directly to `main` (your fork or upstream). - Don't `--force-push` to a shared/upstream branch. -- Don't bypass `pre-commit` with `--no-verify` — fix the lint/format issue instead. +- Don't bypass `pre-commit` with `--no-verify` if a `.pre-commit-config.yaml` exists — fix the lint/format issue instead. - Don't `git commit -am` when you've added a new test file — `-a` skips untracked files. Use `git add` then `git commit -m`. - Don't run bare `pytest` after `pip install -e .` — the repo's pyproject injects `--cov` and `--asyncio-mode=auto`. Install `".[dev]"` first. - Don't open a PR without a regression test for non-trivial fixes (single-line comment/typo fixes are fine without one). @@ -519,5 +536,6 @@ After the PR merges and a release ships, swap back to the published package. - Hand-rolling budget tracking on top of OpenAI/Anthropic calls → use `cascadeflow.init(mode="enforce")` + `cascadeflow.run(budget=...)`. - Computing cost savings manually by subtracting hardcoded prices → use `result.total_cost` / `result.cost_saved` / `result.cost_saved_percentage`, or the LangChain callback. - Drafter and verifier from the same tier (e.g. `gpt-4o` + `gpt-4o`) → no meaningful savings. -- Treating `@cascadeflow.agent` as enforcement — it's metadata only. +- Treating the `@agent` decorator as enforcement — it's metadata only. +- Writing `@cascadeflow.agent(...)` — that's the module, not the decorator. See the `@agent` import note above. - Demoing `observe` mode and claiming "budget enforced" — observe doesn't stop calls. Use `enforce` or `run(budget=...)`. From 7c4241bc1227bd7f2988b43e43b58bce4abc60fa Mon Sep 17 00:00:00 2001 From: saschabuehrle Date: Tue, 28 Apr 2026 23:34:16 +0200 Subject: [PATCH 7/8] fix(agent): make `cascadeflow.agent` callable as the harness decorator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `cascadeflow.agent` is the module file `cascadeflow/agent.py`, and many internal imports rely on that: from cascadeflow.agent import CascadeAgent cascadeflow.agent.PROVIDER_REGISTRY cascadeflow/integrations/openclaw/wrapper.py: from cascadeflow.agent import CascadeAgent But README/docs/llms.txt all show the policy decorator as: @cascadeflow.agent(budget=0.20, compliance="gdpr", kpi_weights={...}) async def my_agent(query: str): ... That historically raised `TypeError: 'module' object is not callable`, because modules aren't callable. Every README/docs reader who copy-pasted the headline decorator example crashed on the first line. We can't drop the `agent` lazy alias without breaking all the existing module imports. Instead, subclass the module's type and add `__call__` so calling `cascadeflow.agent(...)` delegates to the harness `agent` decorator. Module-attribute access is unaffected. Verified all four call paths: 1. @cascadeflow.agent(budget=0.20, ...) → works (the fix) 2. from cascadeflow.agent import CascadeAgent → still works 3. cascadeflow.agent.PROVIDER_REGISTRY → still works 4. from cascadeflow.harness import agent → still works 5. @cascadeflow.harness_agent(...) → still works Regression tests in `tests/test_agent_module_callable.py` cover all five paths so this can't silently regress. Discovered while validating the new Claude Code / Codex skill: an independent code review (`superpowers:code-reviewer`) flagged the TypeError on the headline decorator example. This commit fixes the root cause in the package; the skill keeps recommending the version-agnostic `from cascadeflow.harness import agent` pattern because pip-installed cascadeflow ≤ 1.2.0 still has the old behavior until a release ships with this fix. Test plan: - `pytest tests/test_agent_module_callable.py` — 5 new tests pass - `pytest tests/test_agent_p0_tool_loop.py tests/test_harness_api.py` — 46 existing tests still pass (no regression on module-attr access) - Manual: `import cascadeflow; @cascadeflow.agent(budget=0.20)` no longer raises --- cascadeflow/agent.py | 27 ++++++++++ tests/test_agent_module_callable.py | 78 +++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 tests/test_agent_module_callable.py diff --git a/cascadeflow/agent.py b/cascadeflow/agent.py index c8faab44..7e48273d 100644 --- a/cascadeflow/agent.py +++ b/cascadeflow/agent.py @@ -3229,3 +3229,30 @@ def from_profile( __all__ = ["CascadeAgent", "CascadeResult"] + + +# ───────────────────────────────────────────────────────────────────────────── +# Make `cascadeflow.agent` callable as the harness decorator. +# +# `cascadeflow.agent` is the module file (this file). Many internal imports rely +# on that — `from cascadeflow.agent import CascadeAgent`, `cascadeflow.agent.PROVIDER_REGISTRY`, +# etc. But README/docs/llms.txt also use `@cascadeflow.agent(budget=..., ...)` as a +# decorator, which historically raised `TypeError: 'module' object is not callable`. +# +# Subclass the module's type and add `__call__` so `cascadeflow.agent(...)` returns +# the harness `agent` decorator. Module-attribute access is unaffected. +# ───────────────────────────────────────────────────────────────────────────── + +import sys as _sys + + +class _CallableAgentModule(type(_sys.modules[__name__])): + """Module subclass that delegates calls to `cascadeflow.harness.agent`.""" + + def __call__(self, *args, **kwargs): + from cascadeflow.harness import agent as _harness_agent_decorator + + return _harness_agent_decorator(*args, **kwargs) + + +_sys.modules[__name__].__class__ = _CallableAgentModule diff --git a/tests/test_agent_module_callable.py b/tests/test_agent_module_callable.py new file mode 100644 index 00000000..b0d4aa94 --- /dev/null +++ b/tests/test_agent_module_callable.py @@ -0,0 +1,78 @@ +"""Regression test: `cascadeflow.agent(...)` works as the harness decorator. + +`cascadeflow.agent` is the module file `cascadeflow/agent.py`, and many +internal imports rely on that (`from cascadeflow.agent import CascadeAgent`, +`cascadeflow.agent.PROVIDER_REGISTRY`, etc.). But README/docs/llms.txt also +use `@cascadeflow.agent(budget=..., ...)` as a decorator, which historically +raised `TypeError: 'module' object is not callable`. + +The fix in `cascadeflow/agent.py` subclasses the module's type so that +`cascadeflow.agent(...)` delegates to `cascadeflow.harness.agent`, while +module-attribute access continues to work. This test guards both paths. +""" + +import asyncio + +import cascadeflow +import cascadeflow.agent as agent_module + + +def test_module_attribute_access_still_works() -> None: + """`from cascadeflow.agent import CascadeAgent` and friends must keep working.""" + from cascadeflow.agent import CascadeAgent, CascadeResult + + assert CascadeAgent.__name__ == "CascadeAgent" + assert CascadeResult.__name__ == "CascadeResult" + # Module attribute (used by tests + the openclaw integration) + assert hasattr(agent_module, "PROVIDER_REGISTRY") + + +def test_cascadeflow_agent_callable_as_decorator_factory() -> None: + """`@cascadeflow.agent(budget=..., ...)` must return a decorator.""" + decorator = cascadeflow.agent( + budget=0.20, + kpi_weights={"quality": 0.6, "cost": 0.3, "latency": 0.1}, + compliance="gdpr", + ) + assert callable(decorator) + + @decorator + async def my_agent(query: str) -> str: + return query + + assert callable(my_agent) + assert asyncio.run(my_agent("hello")) == "hello" + + +def test_cascadeflow_agent_decorator_attaches_metadata() -> None: + """The decorator returned by `cascadeflow.agent(...)` should attach policy metadata.""" + + @cascadeflow.agent(budget=0.10, compliance="hipaa") + async def f(q: str) -> str: + return q + + # The harness `agent` decorator stores metadata on the wrapped function. + # We don't assert on the exact attribute name here — only that the + # decorator returned something callable and didn't raise. + assert callable(f) + + +def test_harness_agent_alias_still_works() -> None: + """`cascadeflow.harness_agent` is the eager top-level alias.""" + + @cascadeflow.harness_agent(budget=0.10) + async def g(q: str) -> str: + return q + + assert callable(g) + + +def test_explicit_import_from_harness_still_works() -> None: + """`from cascadeflow.harness import agent` is the explicit import path.""" + from cascadeflow.harness import agent + + @agent(budget=0.10) + async def h(q: str) -> str: + return q + + assert callable(h) From d6e2408c8e8bc6eb6dc76db4c24dda3a87cc1967 Mon Sep 17 00:00:00 2001 From: saschabuehrle Date: Tue, 28 Apr 2026 23:38:08 +0200 Subject: [PATCH 8/8] style(agent): satisfy ruff E402 (use existing top-level sys import) + black formatting CI Python Code Quality flagged the bottom-of-file 'import sys as _sys' as E402 (module-level import not at top). sys is already imported at line 58, so just use it directly. Plus black asked for one extra blank line before the class. Verified locally: - ruff check: All checks passed! - black --check: 2 files would be left unchanged - mypy --ignore-missing-imports: Success --- cascadeflow/agent.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/cascadeflow/agent.py b/cascadeflow/agent.py index 7e48273d..f8c3ab38 100644 --- a/cascadeflow/agent.py +++ b/cascadeflow/agent.py @@ -3243,10 +3243,8 @@ def from_profile( # the harness `agent` decorator. Module-attribute access is unaffected. # ───────────────────────────────────────────────────────────────────────────── -import sys as _sys - -class _CallableAgentModule(type(_sys.modules[__name__])): +class _CallableAgentModule(type(sys.modules[__name__])): """Module subclass that delegates calls to `cascadeflow.harness.agent`.""" def __call__(self, *args, **kwargs): @@ -3255,4 +3253,4 @@ def __call__(self, *args, **kwargs): return _harness_agent_decorator(*args, **kwargs) -_sys.modules[__name__].__class__ = _CallableAgentModule +sys.modules[__name__].__class__ = _CallableAgentModule