Fuzz testing for chat completions (#84)

- use fuzz to generate random payloads
- verify openai => universal => openai works
- snapshot fuzz cases, and keep the fuzz generator ignored

---------

Co-authored-by: Matt Perpick <matt@braintrustdata.com>
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>

Author: 3 people

.claude/commands/fuzz-fix.md

@@ -0,0 +1,3 @@
+# Fix a fuzz roundtrip issue
+
+Run `make -C crates/lingua/tests/fuzz run` to get a minimal failing case. Show the issue to the user, validate it with a real request, and fix with TDD.

.claude/settings.json

@@ -0,0 +1,15 @@
+{
+  "hooks": {
+    "PostToolUse": [
+      {
+        "matcher": "Edit|Write|NotebookEdit",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "cargo fmt"
+          }
+        ]
+      }
+    ]
+  }
+}

.github/workflows/ci.yml

@@ -10,6 +10,26 @@ env:
   CARGO_TERM_COLOR: always
 
 jobs:
+  typed-boundary:
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+
+    - name: Fetch base branch
+      run: git fetch --no-tags --prune --depth=1 origin "${{ github.base_ref }}"
+
+    - name: Install ripgrep
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y ripgrep
+
+    - name: Enforce typed boundary invariant
+      run: make typed-boundary-check-branch BASE=origin/${{ github.base_ref }}
+
   test:
     runs-on: ubuntu-latest
 

.gitignore

@@ -11,4 +11,4 @@ dev-notes/
 *.node
 .turbo
 .env
-.claude
+.claude/*local*.json

CLAUDE.md AGENTS.mdCLAUDE.md renamed to AGENTS.md

@@ -13,6 +13,13 @@ Lingua is a universal message format that compiles to provider-specific formats
 - **Type safety**: Full TypeScript and Rust type generation with bidirectional validation
 - **No network calls**: This is a message format library, not an API client
 - **Explicit error handling**: All errors must be properly handled, never silently swallowed
+- **No hidden marker fields**: Do not encode provider semantics via internal marker keys (for example in `provider_options`) to fake lossless roundtrips.
+- **Ask when non-lossy mapping is unclear**: If the universal type cannot represent a provider feature non-lossily, stop and ask for clarification on the intended canonical representation before implementing a workaround.
+- **No unapproved fallback logic**: Do not add ad-hoc fallback parsing/translation paths (for example `fallback_*` helpers) without checking with the programmer first.
+- **Typed boundaries only**: At provider boundaries, parse into well-defined typed structs/enums. Do not add lenient raw-JSON parsing that guesses defaults for required fields (for example defaulting missing `role` to `user`, lowercasing unknown roles, or inventing empty `content`).
+- **Fix via types or explicit errors**: If fuzzing finds unsupported/ambiguous shapes, either model them explicitly in types/converters or return a clear error. Do not silently coerce invalid input into a "best effort" shape.
+- **Typed-boundary CI gate**: CI enforces `make typed-boundary-check-branch BASE=origin/<base-branch>` on pull requests. Running `make typed-boundary-check` locally is recommended for faster feedback, but not required as a pre-commit hook.
+- **Typed extras views over raw map access**: If provider extras must be read, deserialize extras into a typed view struct first; do not pluck fields ad-hoc with `map.get(...)`.
 
 ## Documentation style guide
 
@@ -428,4 +435,4 @@ Update README.md:
 - **Document differences**: Note any provider-specific quirks or limitations
 - **Consider streaming**: Many providers support streaming responses
 
-This process ensures consistent provider integration while maintaining type safety and zero-runtime overhead.
+This process ensures consistent provider integration while maintaining type safety and zero-runtime overhead.

CONTRIBUTING.md

@@ -0,0 +1,35 @@
+# Contributing
+
+## Fuzz Snapshot Workflow
+
+Use the top-level make target to refresh fuzz snapshots:
+
+```bash
+make fuzz-snapshots
+```
+
+What it does:
+
+1. Runs ignored fuzz tests that generate/update snapshots:
+   `openai_roundtrip`, `openai_roundtrip_stats`
+   `responses_roundtrip`, `responses_roundtrip_stats`
+   `anthropic_roundtrip`, `anthropic_roundtrip_stats`
+   `chat_anthropic_two_arm`, `chat_anthropic_two_arm_stats`
+   `chat_responses_anthropic_three_arm`, `chat_responses_anthropic_three_arm_stats`
+2. Runs snapshot prune/dedupe:
+   `openai_roundtrip_prune_snapshots`
+   `responses_roundtrip_prune_snapshots`
+   `anthropic_roundtrip_prune_snapshots`
+   `chat_anthropic_two_arm_prune_snapshots`
+   `chat_responses_anthropic_three_arm_prune_snapshots`
+
+Notes:
+
+- The generation tests may fail while still producing useful snapshots; this is expected in this workflow.
+- Prune is conservative: it removes malformed/orphan files and dedupes failures by normalized reason.
+
+If you only want to dedupe/prune existing fuzz snapshots:
+
+```bash
+make fuzz-snapshots-prune
+```

Cargo.lock

@@ -1,4 +1,4 @@
-.PHONY: all lingua-wasm typescript python test test-payloads capture capture-transforms clean help generate-types generate-all-providers install-hooks install-wasm-tools setup precommit
+.PHONY: all lingua-wasm typescript python test test-payloads capture capture-transforms clean help generate-types generate-all-providers install-hooks install-wasm-tools setup precommit fuzz-snapshots fuzz-snapshots-prune typed-boundary-check typed-boundary-check-branch
 
 all: typescript python ## Build all bindings
 
@@ -58,6 +58,38 @@ test-payloads: lingua-wasm ## Run payload transform tests (REGENERATE=1 to auto-
 	@cd payloads && pnpm vitest run scripts/transforms $(if $(REGENERATE),|| pnpm tsx scripts/regenerate-failed.ts) \
 		|| (echo "\n❌ Tests failed! Run 'make regenerate-failed-transforms' to regenerate with real API calls" && exit 1)
 
+fuzz-snapshots: ## Run ignored fuzz tests to generate/update snapshots, then prune duplicate failures
+	@$(MAKE) -C crates/lingua/tests/fuzz refresh-snapshots
+
+fuzz-snapshots-prune: ## Prune fuzz snapshots (dedupe by normalized failure reason)
+	@$(MAKE) -C crates/lingua/tests/fuzz prune
+
+typed-boundary-check: ## Fail if local provider/universal edits add direct Value field access
+	@echo "Checking typed boundary regressions in local changes..."; \
+	RESULTS=$$(git diff --unified=0 -- crates/lingua/src/providers crates/lingua/src/universal | \
+		rg -N '^\+.*(\.get\(\"|\.as_object\(|\.as_array\(|\.as_str\(|\.as_i64\(|\.as_u64\(|\.as_bool\(|\.as_f64\()' || true); \
+	if [ -n "$$RESULTS" ]; then \
+		echo "Found direct Value access in added lines:"; \
+		printf '%s\n' "$$RESULTS"; \
+		exit 1; \
+	fi; \
+	echo "No new direct Value field access detected."
+
+typed-boundary-check-branch: ## Fail if committed branch diff adds direct Value field access (usage: make typed-boundary-check-branch BASE=fuzz-test-anthropic)
+	@if [ -z "$(BASE)" ]; then \
+		echo "Usage: make typed-boundary-check-branch BASE=<comparison-branch>"; \
+		exit 1; \
+	fi; \
+	echo "Checking typed boundary regressions in committed diff against $(BASE)..."; \
+	RESULTS=$$(git diff --unified=0 "$(BASE)"...HEAD -- crates/lingua/src/providers crates/lingua/src/universal | \
+		rg -N '^\+.*(\.get\(\"|\.as_object\(|\.as_array\(|\.as_str\(|\.as_i64\(|\.as_u64\(|\.as_bool\(|\.as_f64\()' || true); \
+	if [ -n "$$RESULTS" ]; then \
+		echo "Found direct Value access in added lines:"; \
+		printf '%s\n' "$$RESULTS"; \
+		exit 1; \
+	fi; \
+	echo "No new direct Value field access detected in committed diff."
+
 capture: lingua-wasm ## Capture payloads (snapshots + transforms + vitest snapshots)
 	cd payloads && pnpm capture $(if $(FILTER),--filter $(FILTER)) $(if $(CASES),--cases $(CASES)) $(if $(FORCE),--force)
 
@@ -67,10 +99,10 @@ capture-transforms: lingua-wasm ## Re-capture only transforms (e.g. make capture
 regenerate-failed-transforms: lingua-wasm ## Auto-regenerate failed transform payloads
 	cd payloads && pnpm tsx scripts/regenerate-failed.ts
 
-test-python: ## Run Python tests
+test-python: python ## Run Python tests
 	@echo "Running Python tests..."
-	cd bindings/python && uv run maturin develop --features python
-	cd bindings/python && uv run pytest tests/ -v
+	cd bindings/python && uv run --extra dev --group dev maturin develop --features python
+	cd bindings/python && uv run --with pytest python -m pytest tests/ -v
 
 clean: ## Clean build artifacts
 	@echo "Cleaning build artifacts..."

Makefile

@@ -0,0 +1,6 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+/**
+ * Provider identity for built-in tool passthrough.
+ */
+export type BuiltinToolProvider = "anthropic" | "responses" | "google" | "converse";

bindings/typescript/src/generated/BuiltinToolProvider.ts

@@ -3,4 +3,4 @@ import type { AssistantContent } from "./AssistantContent";
 import type { ToolContentPart } from "./ToolContentPart";
 import type { UserContent } from "./UserContent";
 
-export type Message = { "role": "system", content: UserContent, } | { "role": "user", content: UserContent, } | { "role": "assistant", content: AssistantContent, id?: string, } | { "role": "tool", content: Array<ToolContentPart>, };
+export type Message = { "role": "system", content: UserContent, } | { "role": "developer", content: UserContent, } | { "role": "user", content: UserContent, } | { "role": "assistant", content: AssistantContent, id?: string, } | { "role": "tool", content: Array<ToolContentPart>, };