scorer_calibration_test.go

//go:build calibration

package llmtest_test

import (
	"context"
	"encoding/json"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"github.com/adamwoolhether/llmtest"

	"github.com/adamwoolhether/llmtest/provider"
)

func TestMain(m *testing.M) {
	code := m.Run()
	llmtest.Flush() // writes JSON summary if LLMTEST_OUTPUT is set
	os.Exit(code)
}

type calibrationCase struct {
	ID               string   `json:"id"`
	Description      string   `json:"description"`
	Criterion        string   `json:"criterion"`
	Input            string   `json:"input"`
	ActualOutput     string   `json:"actual_output"`
	ExpectedOutput   string   `json:"expected_output"`
	Context          []string `json:"context"`
	RetrievalContext []string `json:"retrieval_context"`
	ExpectedVerdict  string   `json:"expected_verdict"`
	Tags             []string `json:"tags"`
}

type providerConfig struct {
	name     string
	model    string
	provider provider.Provider
}

var (
	ollamaModels    = []string{"llama3.2", "qwen2.5"}
	openaiModels    = []string{"gpt-4.1-mini", "o4-mini"}
	anthropicModels = []string{"claude-sonnet-4-5-20250929"}
)

func modelsForProvider(providerName string) []string {
	envKey := "LLMTEST_CALIBRATION_MODELS_" + strings.ToUpper(providerName)
	if v := os.Getenv(envKey); v != "" {
		return strings.Split(v, ",")
	}

	switch providerName {
	case "ollama":
		return ollamaModels
	case "openai":
		return openaiModels
	case "anthropic":
		return anthropicModels
	default:
		return nil
	}
}

func calibrationProviders(t *testing.T) []providerConfig {
	t.Helper()

	var configs []providerConfig

	if os.Getenv("OPENAI_API_KEY") != "" {
		p, err := provider.OpenAI()
		if err != nil {
			t.Logf("skipping openai: %v", err)
		} else {
			for _, m := range modelsForProvider("openai") {
				configs = append(configs, providerConfig{name: "openai", model: m, provider: p})
			}
		}
	}

	if os.Getenv("ANTHROPIC_API_KEY") != "" {
		p, err := provider.Anthropic()
		if err != nil {
			t.Logf("skipping anthropic: %v", err)
		} else {
			for _, m := range modelsForProvider("anthropic") {
				configs = append(configs, providerConfig{name: "anthropic", model: m, provider: p})
			}
		}
	}

	if calibrationOllamaReachable() {
		p, err := provider.Ollama()
		if err != nil {
			t.Logf("skipping ollama: %v", err)
		} else {
			for _, m := range modelsForProvider("ollama") {
				configs = append(configs, providerConfig{name: "ollama", model: m, provider: p})
			}
		}
	}

	if len(configs) == 0 {
		t.Fatal("no providers available: set OPENAI_API_KEY, ANTHROPIC_API_KEY, or run Ollama")
	}

	return configs
}

func calibrationOllamaReachable() bool {
	u := os.Getenv("LLMTEST_OLLAMA_URL")
	if u == "" {
		u = "http://localhost:11434"
	}

	resp, err := http.Get(u)
	if err != nil {
		return false
	}
	defer func() {
		io.Copy(io.Discard, resp.Body)
		resp.Body.Close()
	}()

	return resp.StatusCode == 200
}

func loadCases(t *testing.T, filename string) []calibrationCase {
	t.Helper()

	data, err := os.ReadFile(filepath.Join("testdata", "calibration", filename))
	if err != nil {
		t.Fatalf("loading %s: %v", filename, err)
	}

	var cases []calibrationCase
	if err := json.Unmarshal(data, &cases); err != nil {
		t.Fatalf("parsing %s: %v", filename, err)
	}

	return cases
}

func toTestCase(cc calibrationCase) llmtest.TestCase {
	return llmtest.TestCase{
		Input:            cc.Input,
		ActualOutput:     cc.ActualOutput,
		ExpectedOutput:   cc.ExpectedOutput,
		Context:          cc.Context,
		RetrievalContext: cc.RetrievalContext,
	}
}

// groupedProviders returns provider configs grouped by name with stable ordering.
func groupedProviders(t *testing.T) ([]string, map[string][]providerConfig) {
	t.Helper()

	configs := calibrationProviders(t)
	grouped := make(map[string][]providerConfig)
	var order []string
	for _, pc := range configs {
		if _, ok := grouped[pc.name]; !ok {
			order = append(order, pc.name)
		}
		grouped[pc.name] = append(grouped[pc.name], pc)
	}

	return order, grouped
}

// runCalibration is a shared helper for calibration tests. It loads the
// given fixture file, iterates over all provider/model/case combinations,
// creates a Rubric scorer with the given options, and calls assertFn with
// the result for each case.
func runCalibration(
	t *testing.T,
	filename string,
	scorerOpts func(cc calibrationCase, pc providerConfig) []llmtest.ScorerOption,
	assertFn func(t *testing.T, cc calibrationCase, result llmtest.Result),
) {
	t.Helper()

	order, grouped := groupedProviders(t)
	cases := loadCases(t, filename)

	for _, name := range order {
		t.Run(name, func(t *testing.T) {
			t.Parallel()
			for _, pc := range grouped[name] {
				pc := pc
				t.Run(pc.model, func(t *testing.T) {
					t.Parallel()
					for _, cc := range cases {
						cc := cc
						t.Run(cc.ID, func(t *testing.T) {
							t.Parallel()
							opts := scorerOpts(cc, pc)
							scorer := llmtest.Rubric(cc.Criterion, opts...)
							result, err := scorer.Score(context.Background(), toTestCase(cc))
							if err != nil {
								t.Fatalf("score error: %v", err)
							}

							t.Logf("verdict=%s score=%.1f reason=%s tokens=%d latency=%dms",
								result.Verdict, result.Score, result.Reason, result.Tokens, result.LatencyMS)

							assertFn(t, cc, result)
						})
					}
				})
			}
		})
	}
}

func defaultScorerOpts(cc calibrationCase, pc providerConfig) []llmtest.ScorerOption {
	return []llmtest.ScorerOption{llmtest.Provider(pc.provider), llmtest.Model(pc.model)}
}

// TestCalibrationPass verifies that obvious-pass cases return PASS.
func TestCalibrationPass(t *testing.T) {
	runCalibration(t, "obvious_pass.json", defaultScorerOpts, func(t *testing.T, cc calibrationCase, result llmtest.Result) {
		if result.Verdict != llmtest.Pass {
			t.Errorf("[%s] expected PASS, got %s: %s", cc.ID, result.Verdict, result.Reason)
		}
	})
}

// TestCalibrationFail verifies that obvious-fail cases return FAIL.
func TestCalibrationFail(t *testing.T) {
	runCalibration(t, "obvious_fail.json", defaultScorerOpts, func(t *testing.T, cc calibrationCase, result llmtest.Result) {
		if result.Verdict != llmtest.Fail {
			t.Errorf("[%s] expected FAIL, got %s: %s", cc.ID, result.Verdict, result.Reason)
		}
	})
}

// TestCalibrationAmbiguous runs each case 3 times and expects at least 2/3 agreement.
func TestCalibrationAmbiguous(t *testing.T) {
	order, grouped := groupedProviders(t)
	cases := loadCases(t, "ambiguous.json")

	for _, name := range order {
		t.Run(name, func(t *testing.T) {
			t.Parallel()
			for _, pc := range grouped[name] {
				pc := pc
				t.Run(pc.model, func(t *testing.T) {
					t.Parallel()
					for _, cc := range cases {
						cc := cc
						t.Run(cc.ID, func(t *testing.T) {
							t.Parallel()
							scorer := llmtest.Rubric(cc.Criterion, llmtest.Provider(pc.provider), llmtest.Model(pc.model))
							counts := make(map[llmtest.Verdict]int)

							for i := 0; i < 3; i++ {
								result, err := scorer.Score(context.Background(), toTestCase(cc))
								if err != nil {
									t.Fatalf("score error (run %d): %v", i, err)
								}

								counts[result.Verdict]++
								t.Logf("run %d: verdict=%s reason=%s", i, result.Verdict, result.Reason)
							}

							var maxCount int
							var maxVerdict llmtest.Verdict
							for v, c := range counts {
								if c > maxCount {
									maxCount = c
									maxVerdict = v
								}
							}

							if maxCount < 2 {
								t.Errorf("[%s] no 2/3 agreement: %v", cc.ID, counts)
							} else {
								t.Logf("[%s] agreement: %s (%d/3)", cc.ID, maxVerdict, maxCount)
							}
						})
					}
				})
			}
		})
	}
}

// TestCalibrationThreshold verifies that the Threshold option correctly
// promotes verdicts: Threshold(0.5) should promote PARTIAL to PASS
// while leaving FAIL unchanged.
func TestCalibrationThreshold(t *testing.T) {
	thresholdOpts := func(cc calibrationCase, pc providerConfig) []llmtest.ScorerOption {
		return []llmtest.ScorerOption{llmtest.Provider(pc.provider), llmtest.Model(pc.model), llmtest.Threshold(0.5)}
	}

	t.Run("pass_unaffected", func(t *testing.T) {
		runCalibration(t, "obvious_pass.json", thresholdOpts, func(t *testing.T, cc calibrationCase, result llmtest.Result) {
			if result.Verdict != llmtest.Pass {
				t.Errorf("[%s] expected PASS with Threshold(0.5), got %s: %s", cc.ID, result.Verdict, result.Reason)
			}
		})
	})

	t.Run("fail_unaffected", func(t *testing.T) {
		runCalibration(t, "obvious_fail.json", thresholdOpts, func(t *testing.T, cc calibrationCase, result llmtest.Result) {
			if result.Verdict != llmtest.Fail {
				t.Errorf("[%s] expected FAIL with Threshold(0.5), got %s: %s", cc.ID, result.Verdict, result.Reason)
			}
		})
	})
}

// TestCalibrationAdversarial verifies that adversarial cases do NOT return PASS.
func TestCalibrationAdversarial(t *testing.T) {
	runCalibration(t, "adversarial.json", defaultScorerOpts, func(t *testing.T, cc calibrationCase, result llmtest.Result) {
		if result.Verdict == llmtest.Pass {
			t.Errorf("[%s] expected NOT PASS, got PASS: %s", cc.ID, result.Reason)
		}
	})
}