|
| 1 | +# configs/smoke-no-mcp.yaml -- no-MCP empty-tools live smoke (~$0.30, 2 trials) |
| 2 | +# |
| 3 | +# PURPOSE: validate the ONE remaining unproven live path before Stage 1 turns the |
| 4 | +# baseline on -- the N=0 "no-MCP" arm, where _invoke_api must omit `tools` entirely and |
| 5 | +# the live API must accept a no-tools request. If the API 400s on empty tools or the N=0 |
| 6 | +# trial mis-records, we learn it here for ~$0.30, not mid-Stage-1. |
| 7 | +# |
| 8 | +# WHY 2 TRIALS (not 1): the no-MCP arm emits 1 cell per query at N=0, but enumerate_cells |
| 9 | +# always also emits the normal N cells, and a real task carries a required primary_server |
| 10 | +# (so N=[] would HALT, not isolate the arm). So N=[1] yields exactly: |
| 11 | +# - 1 GROUNDED trial (N=1, github_mcp): the model retrieves with tools (~$0.20, needs Docker) |
| 12 | +# - 1 NO-MCP trial (N=0, empty tools): the unproven path being validated (~$0.10, no servers) |
| 13 | +# The grounded trial is the cheapest the enumerator allows AND a useful contrast. |
| 14 | +# |
| 15 | +# WHAT THE N=0 TRIAL PROVES (the uncontaminated floor): the model is asked the post-cutoff |
| 16 | +# pylint symbol `visit_call` (introduced 2026-05-22) with NO tools. It should NOT be able to |
| 17 | +# name it from memory -> pass=False is EXPECTED and GOOD (that is the 0/N floor that makes any |
| 18 | +# tool-sourced pass meaningful). The grounded trial should retrieve it (pass=True). |
| 19 | +# |
| 20 | +# GREEN (path healthy): two trial rows. The N=0 row has error_type=none, an empty server_set, |
| 21 | +# and a recorded cost_usd -- i.e. the empty-tools API call SUCCEEDED. pass=False on the N=0 |
| 22 | +# row is the expected floor, not a failure. |
| 23 | +# RED (fix before Stage 1 enables no-MCP): the N=0 row shows error_type in {api_fault, |
| 24 | +# harness_bug} (the empty-tools request was rejected or mishandled), or the run crashed. |
| 25 | +# |
| 26 | +# NOTE: deepwiki is not in this pool, so the deepwiki-index hook does not fire here |
| 27 | +# (smoke-pipeline.yaml covers that path). cost-cap 1 = ~3x backstop. |
| 28 | +# |
| 29 | +# RUN (after Docker up + key confirmed): |
| 30 | +# tcrun validate -c configs/smoke-no-mcp.yaml # $0 offline config/artifact check |
| 31 | +# tcrun run -c configs/smoke-no-mcp.yaml --cost-cap 1 # ~$0.30 live (1 grounded + 1 no-MCP) |
| 32 | + |
| 33 | +# Path-typed fields (content-hashed into run_id) -- identical artifacts to the other configs. |
| 34 | +task_set: tasks/github_smoke.jsonl # post-cutoff pylint symbol; the no-MCP trial tests "can the model name it without tools" (expected: no) |
| 35 | +oracle: tcrun/oracles/pass_v1.py |
| 36 | +servers_pinned: tcrun/servers_pinned.yaml |
| 37 | +descriptions: pool/descriptions.json |
| 38 | +endpoints: models/endpoints.json |
| 39 | +environment: environment.lock |
| 40 | +padding_corpus: ../design/fake_tool_corpus.jsonl |
| 41 | +embedder: models/embedder.json |
| 42 | + |
| 43 | +# Non-path fields (value-hashed) |
| 44 | +primary_servers: |
| 45 | + - github_mcp |
| 46 | + |
| 47 | +distractors: [] # N=1 grounded cell needs only the primary; the N=0 arm spawns nothing |
| 48 | + |
| 49 | +N: [1] |
| 50 | +runs_per_cell: 1 |
| 51 | +orderings: 1 # 1 grounded trial; the no-MCP arm adds 1 (N=0) -> 2 trials total |
| 52 | +model: claude-sonnet-4-6 |
| 53 | +host: claude-desktop |
| 54 | +seed: 42 |
| 55 | +temperature: 1.0 # validate the empty-tools path at the Stage-1 sampling |
| 56 | + |
| 57 | +tool_listing_strategy: full |
| 58 | +include_padded_n1_control: false |
| 59 | +include_no_mcp_baseline: true # THE POINT: emits the N=0 empty-tools arm |
| 60 | +include_random_tool_call_baseline: false |
| 61 | +ragmcp_replication: false |
| 62 | + |
| 63 | +out: results/smoke-no-mcp/ |
0 commit comments