fix(benchmarks): Validate session defaults during load

Validate Claude UI benchmark sessionDefaults while reading the suite config so unknown keys and invalid value types fail before simulator setup starts.

Co-Authored-By: OpenAI Codex <codex@openai.com>

Author: cameroncooke

CHANGELOG.md

@@ -24,7 +24,7 @@
 
 - Fixed Claude UI benchmark preflight so transient malformed or still-loading UI snapshots no longer crash the harness or finish before app UI is observable.
 - Fixed Claude UI benchmark preflight so configured first-run dismissals require a concrete simulator ID and suite-provided simulator IDs are recorded in command logs.
-- Fixed Claude UI benchmark config handling so invalid `failurePatterns` regexes fail before a suite starts and partial `allowedVariance` overrides preserve defaults for omitted metrics.
+- Fixed Claude UI benchmark config handling so invalid `failurePatterns` regexes and `sessionDefaults` fail before a suite starts and partial `allowedVariance` overrides preserve defaults for omitted metrics.
 - Fixed Claude UI benchmark temporary simulator cleanup so simulators created by the harness are deleted even when post-creation setup fails.
 - Fixed UI action snapshot refreshes so timeout while waiting for a settled post-action snapshot returns a recoverable warning instead of unstable element refs.
 - Fixed Claude UI benchmark suite runs so temporary simulators are applied through an isolated per-run MCP config instead of being overridden by repo or example-project config defaults.

src/benchmarks/claude-ui/__tests__/claude-ui-benchmark.test.ts

@@ -213,6 +213,34 @@ describe('Claude UI benchmark analysis', () => {
     ).toThrow('weather.yml.failurePatterns[1]: invalid regular expression');
   });
 
+  it('rejects invalid session defaults when loading config', () => {
+    expect(() =>
+      readConfig(
+        {
+          name: 'weather',
+          prompt: 'prompt.md',
+          sessionDefaults: {
+            simulatorTypo: 'iPhone 17 Pro Max',
+          },
+        },
+        'weather.yml',
+      ),
+    ).toThrow("unknown sessionDefaults key 'simulatorTypo'");
+
+    expect(() =>
+      readConfig(
+        {
+          name: 'weather',
+          prompt: 'prompt.md',
+          sessionDefaults: {
+            simulatorId: 42,
+          },
+        },
+        'weather.yml',
+      ),
+    ).toThrow('sessionDefaults.simulatorId must be a string or boolean');
+  });
+
   it('warns by default when tool sequences drift', () => {
     const config: BenchmarkConfig = {
       name: 'weather',

src/benchmarks/claude-ui/config.ts

@@ -185,7 +185,12 @@ export function readConfig(raw: unknown, source: string): BenchmarkConfig {
     ),
   };
 
-  if (isRecord(raw.sessionDefaults)) config.sessionDefaults = raw.sessionDefaults;
+  if (raw.sessionDefaults !== undefined) {
+    if (!isRecord(raw.sessionDefaults)) {
+      throw new Error(`${source}.sessionDefaults: expected object`);
+    }
+    config.sessionDefaults = validateSessionDefaults(raw.sessionDefaults);
+  }
   config.allowedVariance = readAllowedVariance(raw.allowedVariance, `${source}.allowedVariance`);
 
   if (raw.baseline !== undefined) {