rubric-evaluator/config.sample.yaml at main · spChalk/rubric-evaluator · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
# ─── Rubric Evaluator — Sample Configuration ────────────────────────
#
# Quick start:
#   1. make setup          (copies this to config.yaml and .env.sample to .env)
#   2. Edit .env with your API keys
#   3. make run


judges:
  # --- OpenAI ---
  - id: "judge_1"
    llm:
      model: "gpt-5.4"
      api_key: "${OPENAI_API_KEY}"
      temperature: 0.3
      reasoning_effort: "medium"
      max_completion_tokens: 8192

  # --- Anthropic ---
  - id: "judge_2"
    llm:
      model: "anthropic/claude-sonnet-4-6"
      api_key: "${ANTHROPIC_API_KEY}"
      temperature: 0.1
      max_completion_tokens: 8192

  # --- Anthropic with extended thinking (uncomment and configure) ---
  # - id: "judge_3"
  #   llm:
  #     model: "anthropic/claude-opus-4-6"
  #     api_key: "${ANTHROPIC_API_KEY}"
  #     temperature: 1.0
  #     max_completion_tokens: 8192
  #     thinking_budget: 5000

  # --- Azure OpenAI (uncomment and configure) ---
  # - id: "judge_4"
  #   llm:
  #     model: "azure/<your-deployment-name>"
  #     api_key: "${AZURE_API_KEY}"
  #     api_base: "${AZURE_API_BASE}"
  #     api_version: "2024-12-01-preview"
  #     temperature: 0.3
  #     max_completion_tokens: 8192

  # --- Google Gemini (uncomment and configure) ---
  # - id: "judge_5"
  #   llm:
  #     model: "gemini/gemini-3.1-pro-preview"
  #     api_key: "${GOOGLE_API_KEY}"
  #     temperature: 0.3
  #     max_completion_tokens: 8192


# ─── Council ─────────────────────────────────────────────────────────
# Controls how judges reach agreement.

council:
  consensus_threshold: 0.5        # fraction of votes needed to accept a proposal
  max_synthesis_attempts: 2       # retries with synthesis before dropping a round


# ─── Synthesizer ─────────────────────────────────────────────────────
# Merges proposals when judges cannot reach consensus.

synthesizer:
  model: "gpt-5.4"
  api_key: "${OPENAI_API_KEY}"
  temperature: 0.3
  max_completion_tokens: 8192


# ─── Diff Explainer ──────────────────────────────────────────────────
# Generates the final explanation comparing original vs improved rubric.

diff_explainer:
  model: "anthropic/claude-sonnet-4-6"
  api_key: "${ANTHROPIC_API_KEY}"
  temperature: 0.3
  max_completion_tokens: 8192


# ─── Inputs ──────────────────────────────────────────────────────────

inputs:
  exam_question: "data/transcribed/exam_question.txt"
  grading_rubric: "data/transcribed/rubric.txt"
  teaching_resource: "data/transcribed/teaching_resource.txt"
  optimization_dimensions: "data/transcribed/optimization_dimensions.txt"
  student_answers_dir: "data/transcribed/student_answers"  # all .txt files in this directory


# ─── Output ──────────────────────────────────────────────────────────
# Each run creates a timestamped subdirectory with result.json and
# metadata.json inside.

output:
  dir: "output"


# ─── Cache ───────────────────────────────────────────────────────────
# Caches LLM responses to avoid duplicate API calls across retries
# and re-runs. Set to null to disable.

cache_path: ".cache"