From f03aba209b318532c0a599e25d639351ce77a8af Mon Sep 17 00:00:00 2001 From: Sean McGuire Date: Tue, 25 Feb 2025 15:41:40 -0800 Subject: [PATCH 1/4] useAPI: true --- evals/initStagehand.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/evals/initStagehand.ts b/evals/initStagehand.ts index f818ff5f7..fda2b9c51 100644 --- a/evals/initStagehand.ts +++ b/evals/initStagehand.ts @@ -77,6 +77,7 @@ export const initStagehand = async ({ logger.log(logLine); }, ...configOverrides, + useAPI: true, }; const stagehand = new Stagehand(config); From 11c11ed5faa0664fdf56953239b875ce401ceff0 Mon Sep 17 00:00:00 2001 From: Sean McGuire Date: Tue, 25 Feb 2025 15:41:50 -0800 Subject: [PATCH 2/4] start with gpt --- evals/taskConfig.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evals/taskConfig.ts b/evals/taskConfig.ts index 0031b8a80..231424fe0 100644 --- a/evals/taskConfig.ts +++ b/evals/taskConfig.ts @@ -49,7 +49,7 @@ if (filterByEvalName && !tasksByName[filterByEvalName]) { */ const DEFAULT_EVAL_MODELS = process.env.EVAL_MODELS ? process.env.EVAL_MODELS.split(",") - : ["gpt-4o", "claude-3-5-sonnet-latest"]; + : ["gpt-4o"]; /** * getModelList: From c89aa409db4f83c9131c69e3e59f53dfca4f6059 Mon Sep 17 00:00:00 2001 From: Sean McGuire Date: Tue, 25 Feb 2025 15:42:09 -0800 Subject: [PATCH 3/4] start with lower trial count & concurrency --- evals/index.eval.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/evals/index.eval.ts b/evals/index.eval.ts index 578ab3235..58a002185 100644 --- a/evals/index.eval.ts +++ b/evals/index.eval.ts @@ -32,8 +32,8 @@ import { AvailableModel } from "@/dist"; import dotenv from "dotenv"; dotenv.config(); -const MAX_CONCURRENCY = 20; -const TRIAL_COUNT = 5; +const MAX_CONCURRENCY = 5; +const TRIAL_COUNT = 1; /** * generateSummary: From a8d1c6db795c4eafe107b8ad5c344a5f87c7c357 Mon Sep 17 00:00:00 2001 From: Sean McGuire Date: Tue, 25 Feb 2025 16:38:41 -0800 Subject: [PATCH 4/4] env: BB --- evals/env.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/evals/env.ts b/evals/env.ts index 45f877bd1..4f49517e1 100644 --- a/evals/env.ts +++ b/evals/env.ts @@ -4,10 +4,11 @@ * * The environment is read from the EVAL_ENV environment variable. */ -export const env: "BROWSERBASE" | "LOCAL" = - process.env.EVAL_ENV?.toLowerCase() === "browserbase" - ? "BROWSERBASE" - : "LOCAL"; +// export const env: "BROWSERBASE" | "LOCAL" = +// process.env.EVAL_ENV?.toLowerCase() === "browserbase" +// ? "BROWSERBASE" +// : "LOCAL"; +export const env = "BROWSERBASE" as const; /** * Enable or disable caching based on the EVAL_ENABLE_CACHING environment variable.