Skip to content

Commit 22280b9

Browse files
ibolmoclaude
andcommitted
Add verbose Python diagnostics and fix braintrust version for canary bt
- Add bt --version, PYTHONUNBUFFERED=1, and python3 version check to help diagnose Python eval failures in CI - Update test-eval-py to require braintrust>=0.8.0 (was 0.0.160) so bt's canary eval-runner.py can import braintrust.parameters - Regenerate uv.lock with braintrust 0.8.0 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent 2ac6979 commit 22280b9

6 files changed

Lines changed: 313 additions & 226 deletions

File tree

eval/dist/index.js

Lines changed: 32 additions & 32 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

eval/dist/index.js.map

Lines changed: 2 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

eval/src/braintrust.ts

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,18 @@ export async function runEval(args: Params, onSummary: OnSummaryFn) {
134134

135135
process.chdir(path.resolve(root));
136136

137+
await runCommand("bt --version", () => {});
138+
139+
// Ensure Python output is not buffered so bt captures it in real time.
140+
process.env.PYTHONUNBUFFERED = "1";
141+
142+
if (args.runtime === "python" || args.runner?.includes("python")) {
143+
await runCommand(
144+
"python3 -c \"import sys; print('python:', sys.executable, sys.version)\"",
145+
() => {},
146+
);
147+
}
148+
137149
// Build bt eval flags
138150
const flags: string[] = ["--jsonl", "--verbose"];
139151

test-eval-py/pyproject.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,5 +6,5 @@ readme = "README.md"
66
requires-python = ">=3.12"
77
dependencies = [
88
"autoevals>=0.0.92",
9-
"braintrust>=0.0.160",
9+
"braintrust>=0.8.0",
1010
]

test-eval-py/requirements.txt

Lines changed: 2 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,2 @@
1-
attrs==24.2.0
2-
autoevals==0.0.92
3-
braintrust==0.0.160
4-
braintrust-core==0.0.54
5-
certifi==2024.8.30
6-
charset-normalizer==3.3.2
7-
chevron==0.14.0
8-
exceptiongroup==1.2.0
9-
gitdb==4.0.11
10-
gitpython==3.1.43
11-
idna==3.10
12-
jsonschema==4.23.0
13-
jsonschema-specifications==2023.12.1
14-
levenshtein==0.26.0
15-
python-dotenv==1.0.1
16-
pyyaml==6.0.2
17-
rapidfuzz==3.10.0
18-
referencing==0.35.1
19-
requests==2.32.4
20-
rpds-py==0.20.0
21-
smmap==5.0.1
22-
sseclient-py==1.8.0
23-
tqdm==4.66.5
24-
urllib3==2.5.0
1+
braintrust>=0.8.0
2+
autoevals

0 commit comments

Comments
 (0)