Skip to content

Commit 13039c5

Browse files
realarkrealark
committed
support devserver experiment snapshotting
1 parent 568f9f3 commit 13039c5

4 files changed

Lines changed: 349 additions & 49 deletions

File tree

‎braintrust-sdk/src/main/java/dev/braintrust/devserver/Devserver.java‎

Lines changed: 160 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -377,10 +377,31 @@ private <I, O> void handleStreamingEval(
377377
BraintrustUtils.createProjectURI(
378378
braintrust.config().appUrl(), orgName, projectName)
379379
.toASCIIString();
380-
final var experimentUrl = projectUrl + "/experiments/" + experimentName;
381380

382381
var tracer = BraintrustTracing.getTracer();
383382

383+
// A remote eval can be triggered two ways:
384+
// 1. Playground run: the request carries a `parent` object (playground_id). We
385+
// stream per-case progress under that parent (handled by the loop below).
386+
// 2. Experiment "snapshot": no such parent (instead experiment_name + project).
387+
// We run a standard Eval, which creates a real experiment and emits standard
388+
// spans, then send a summary + done with the experiment link.
389+
final var playgroundParent = extractPlaygroundParent(request);
390+
if (playgroundParent.isEmpty()) {
391+
handleExperimentSnapshot(
392+
os,
393+
eval,
394+
request,
395+
braintrust,
396+
apiClient,
397+
projectId,
398+
projectName,
399+
projectUrl,
400+
experimentName,
401+
remoteScorers);
402+
return;
403+
}
404+
384405
// Merge parameters: evaluator defaults + request overrides
385406
final Parameters mergedParameters =
386407
new Parameters(
@@ -391,7 +412,7 @@ private <I, O> void handleStreamingEval(
391412

392413
// Execute task and scorers for each case
393414
final Map<String, List<Double>> scoresByName = new ConcurrentHashMap<>();
394-
final var parentInfo = extractParentInfo(request);
415+
final var parentInfo = playgroundParent.get();
395416
final var braintrustParent = parentInfo.braintrustParent();
396417
final var braintrustGeneration = parentInfo.generation();
397418

@@ -547,13 +568,7 @@ private <I, O> void handleStreamingEval(
547568
}
548569

549570
sendSummaryEvent(
550-
os,
551-
projectName,
552-
projectId,
553-
experimentName,
554-
projectUrl,
555-
experimentUrl,
556-
scoreSummaries);
571+
os, projectName, projectId, experimentName, projectUrl, scoreSummaries);
557572
sendDoneEvent(os);
558573
} catch (Exception e) {
559574
// Send error event via SSE
@@ -577,6 +592,73 @@ private <I, O> void handleStreamingEval(
577592
}
578593
}
579594

595+
/**
596+
* Handles an experiment "snapshot" run: a remote eval triggered as an Experiment from the UI
597+
* (no playground parent). Rather than re-implementing experiment creation and span emission, it
598+
* builds a first-class {@link Eval} and runs it synchronously — so snapshots get the exact same
599+
* behavior as a normal {@code Eval.run()} (experiment creation with {@code ensure_new}, dataset
600+
* id/version linkage for Braintrust-backed datasets, standard span shape). When the run
601+
* completes it streams a single {@code summary} (with the created experiment's id/name/url) and
602+
* a {@code done} event.
603+
*
604+
* <p>Unlike playground runs, snapshots do not stream per-case {@code progress} events: the user
605+
* is handed the experiment link and views results in the experiment UI.
606+
*/
607+
@SuppressWarnings({"unchecked", "rawtypes"})
608+
private <I, O> void handleExperimentSnapshot(
609+
OutputStream os,
610+
RemoteEval<I, O> eval,
611+
EvalRequest request,
612+
Braintrust braintrust,
613+
BraintrustOpenApiClient apiClient,
614+
String projectId,
615+
String projectName,
616+
String projectUrl,
617+
String experimentName,
618+
List<Scorer<I, O>> remoteScorers)
619+
throws IOException {
620+
// Combine local scorers (from the RemoteEval) with remote scorers (from the request).
621+
List<Scorer<I, O>> allScorers = new ArrayList<>(eval.getScorers());
622+
allScorers.addAll(remoteScorers);
623+
624+
// TODO: when async evals are supported, simply begin the eval and hand back the link to the
625+
// user and finish eval in the background
626+
627+
var evalResult =
628+
Eval.<I, O>builder()
629+
.name(experimentName)
630+
.config(braintrust.config())
631+
.apiClient(apiClient)
632+
.projectId(projectId)
633+
.dataset((Dataset<I, O>) extractDataset(request, apiClient))
634+
.task(eval.getTask())
635+
.scorers(allScorers.toArray(new Scorer[0]))
636+
.parameters(eval.getParameters())
637+
.parameterValues(
638+
request.getParameters() == null
639+
? Map.of()
640+
: request.getParameters())
641+
// Each snapshot run should produce a distinct experiment even if a prior
642+
// run used the same name (the backend dedupes the name on conflict).
643+
.ensureNew(true)
644+
.build()
645+
.run();
646+
647+
// Snapshots don't stream per-scorer progress. The scores are recorded on the experiment
648+
// and visible via the experiment link.
649+
sendExperimentSnapshotSummaryEvent(
650+
os,
651+
projectName,
652+
projectId,
653+
evalResult.getExperimentId(),
654+
evalResult.getExperimentName() != null
655+
? evalResult.getExperimentName()
656+
: experimentName,
657+
projectUrl,
658+
evalResult.getExperimentUrl());
659+
sendExperimentSnapshotDoneEvent(os);
660+
}
661+
580662
private void setEvalSpanAttributes(
581663
Span evalSpan,
582664
BraintrustUtils.Parent braintrustParent,
@@ -802,7 +884,6 @@ private void sendSummaryEvent(
802884
String projectId,
803885
String experimentName,
804886
String projectUrl,
805-
String experimentUrl,
806887
Map<String, EvalResponse.ScoreSummary> scoreSummaries)
807888
throws IOException {
808889
Map<String, Object> summary = new LinkedHashMap<>();
@@ -835,6 +916,40 @@ private void sendDoneEvent(OutputStream os) throws IOException {
835916
sendSSEEvent(os, "done", "");
836917
}
837918

919+
/**
920+
* Sends the {@code summary} event for an experiment snapshot run. Unlike {@link
921+
* #sendSummaryEvent} (playground), this references the created experiment via {@code
922+
* experimentId}/{@code experimentUrl} so the UI can link straight to it, and carries no
923+
* streamed per-scorer scores (results live on the experiment itself).
924+
*/
925+
private void sendExperimentSnapshotSummaryEvent(
926+
OutputStream os,
927+
String projectName,
928+
String projectId,
929+
@Nullable String experimentId,
930+
String experimentName,
931+
String projectUrl,
932+
@Nullable String experimentUrl)
933+
throws IOException {
934+
Map<String, Object> summary = new LinkedHashMap<>();
935+
summary.put("projectName", projectName);
936+
summary.put("projectId", projectId);
937+
summary.put("experimentId", experimentId);
938+
summary.put("experimentName", experimentName);
939+
summary.put("projectUrl", projectUrl);
940+
summary.put("experimentUrl", experimentUrl);
941+
summary.put("comparisonExperimentName", null);
942+
summary.put("scores", Map.of());
943+
summary.put("metrics", Map.of());
944+
945+
sendSSEEvent(os, "summary", toJson(summary));
946+
}
947+
948+
/** Sends the terminating {@code done} event for an experiment snapshot run. */
949+
private void sendExperimentSnapshotDoneEvent(OutputStream os) throws IOException {
950+
sendSSEEvent(os, "done", "");
951+
}
952+
838953
private void sendResponse(
839954
HttpExchange exchange, int statusCode, String contentType, String body)
840955
throws IOException {
@@ -1105,44 +1220,50 @@ private record ParentInfo(
11051220
@Nonnull BraintrustUtils.Parent braintrustParent, @Nullable String generation) {}
11061221

11071222
/**
1108-
* Extracts parent information from the eval request.
1223+
* Extracts the playground parent (and generation) from the eval request, if present.
1224+
*
1225+
* <p>Playground runs send a {@code parent} object carrying {@code object_type}/{@code
1226+
* object_id}; experiment ("snapshot") runs send no such parent (instead {@code experiment_name}
1227+
* + project via headers) and are handled by {@link #handleExperimentSnapshot}. Returns empty
1228+
* for the latter case.
11091229
*
11101230
* @param request The eval request
1111-
* @return ParentInfo containing braintrustParent and generation
1231+
* @return the playground {@link ParentInfo} if this is a playground run, else empty
11121232
*/
1113-
private static ParentInfo extractParentInfo(EvalRequest request) {
1114-
String parentSpec = null;
1115-
String generation = null;
1233+
private static Optional<ParentInfo> extractPlaygroundParent(EvalRequest request) {
1234+
if (!(request.getParent() instanceof Map)) {
1235+
return Optional.empty();
1236+
}
1237+
@SuppressWarnings("unchecked")
1238+
Map<String, Object> parentMap = (Map<String, Object>) request.getParent();
1239+
String objectType = (String) parentMap.get("object_type");
1240+
String objectId = (String) parentMap.get("object_id");
1241+
if (objectType == null && objectId == null) {
1242+
return Optional.empty();
1243+
}
11161244

1117-
// Extract parent spec and generation from request
1118-
if (request.getParent() != null && request.getParent() instanceof Map) {
1119-
@SuppressWarnings("unchecked")
1120-
Map<String, Object> parentMap = (Map<String, Object>) request.getParent();
1121-
String objectType = (String) parentMap.get("object_type");
1122-
String objectId = (String) parentMap.get("object_id");
1245+
if (objectType == null || objectId == null) {
1246+
throw new IllegalArgumentException(
1247+
"malformed braintrust parent: %s, %s".formatted(objectType, objectId));
1248+
}
11231249

1124-
// Extract generation from propagated_event.span_attributes.generation
1125-
Object propEventObj = parentMap.get("propagated_event");
1126-
if (propEventObj instanceof Map) {
1250+
// Extract generation from propagated_event.span_attributes.generation
1251+
String generation = null;
1252+
Object propEventObj = parentMap.get("propagated_event");
1253+
if (propEventObj instanceof Map) {
1254+
@SuppressWarnings("unchecked")
1255+
Map<String, Object> propEvent = (Map<String, Object>) propEventObj;
1256+
Object spanAttrsObj = propEvent.get("span_attributes");
1257+
if (spanAttrsObj instanceof Map) {
11271258
@SuppressWarnings("unchecked")
1128-
Map<String, Object> propEvent = (Map<String, Object>) propEventObj;
1129-
Object spanAttrsObj = propEvent.get("span_attributes");
1130-
if (spanAttrsObj instanceof Map) {
1131-
@SuppressWarnings("unchecked")
1132-
Map<String, Object> spanAttrs = (Map<String, Object>) spanAttrsObj;
1133-
generation = (String) spanAttrs.get("generation");
1134-
}
1135-
}
1136-
1137-
if (objectType != null && objectId != null) {
1138-
parentSpec = "playground_id:" + objectId;
1259+
Map<String, Object> spanAttrs = (Map<String, Object>) spanAttrsObj;
1260+
generation = (String) spanAttrs.get("generation");
11391261
}
11401262
}
11411263

1142-
if (parentSpec == null) {
1143-
throw new IllegalArgumentException("braintrust parent (playground_id) not found");
1144-
}
1145-
return new ParentInfo(BraintrustUtils.parseParent(parentSpec), generation);
1264+
return Optional.of(
1265+
new ParentInfo(
1266+
BraintrustUtils.parseParent("playground_id:" + objectId), generation));
11461267
}
11471268

11481269
/**

‎braintrust-sdk/src/main/java/dev/braintrust/eval/Eval.java‎

Lines changed: 32 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ public final class Eval<INPUT, OUTPUT> {
4646
private final @Nonnull List<String> tags;
4747
private final @Nonnull Map<String, Object> metadata;
4848
private final @Nonnull Parameters parameters;
49+
private final boolean ensureNew;
4950

5051
private Eval(Builder<INPUT, OUTPUT> builder) {
5152
this.experimentName = builder.experimentName;
@@ -63,6 +64,7 @@ private Eval(Builder<INPUT, OUTPUT> builder) {
6364
this.tags = List.copyOf(builder.tags);
6465
this.metadata = Map.copyOf(builder.metadata);
6566
this.parameters = builder.buildParameters();
67+
this.ensureNew = builder.ensureNew;
6668
}
6769

6870
/** Runs the evaluation and returns results. */
@@ -78,6 +80,9 @@ public EvalResult run() {
7880
var createExperiment =
7981
new CreateExperiment().projectId(project.getId()).name(experimentName);
8082

83+
if (ensureNew) {
84+
createExperiment.ensureNew(true);
85+
}
8186
if (!tags.isEmpty()) {
8287
createExperiment.tags(tags);
8388
}
@@ -90,16 +95,22 @@ public EvalResult run() {
9095
var experiment = new ExperimentsApi(client).postExperiment(createExperiment);
9196

9297
cursor.forEach(datasetCase -> evalOne(experiment.getId().toString(), datasetCase));
93-
}
9498

95-
var experimentUrl =
96-
"%s/experiments/%s"
97-
.formatted(
98-
BraintrustUtils.createProjectURI(
99-
config.appUrl(), orgInfo.name(), project.getName())
100-
.toASCIIString(),
101-
experimentName);
102-
return new EvalResult(experimentUrl);
99+
// Use the experiment's actual name from the response: with ensure_new the backend may
100+
// dedupe a conflicting name (e.g. "foo" -> "foo-2f8ca776"), and the URL must point at
101+
// the real, created experiment.
102+
var resolvedName = experiment.getName() != null ? experiment.getName() : experimentName;
103+
var experimentUrl =
104+
"%s/experiments/%s"
105+
.formatted(
106+
BraintrustUtils.createProjectURI(
107+
config.appUrl(),
108+
orgInfo.name(),
109+
project.getName())
110+
.toASCIIString(),
111+
resolvedName);
112+
return new EvalResult(experiment.getId().toString(), resolvedName, experimentUrl);
113+
}
103114
}
104115

105116
private void evalOne(String experimentId, DatasetCase<INPUT, OUTPUT> datasetCase) {
@@ -397,6 +408,7 @@ public static final class Builder<INPUT, OUTPUT> {
397408
private @Nonnull Map<String, Object> parameterValues = Map.of();
398409
private @Nonnull List<String> tags = List.of();
399410
private @Nonnull Map<String, Object> metadata = Map.of();
411+
private boolean ensureNew = false;
400412

401413
public Eval<INPUT, OUTPUT> build() {
402414
if (config == null) {
@@ -521,6 +533,17 @@ public Builder<INPUT, OUTPUT> metadata(Map<String, Object> metadata) {
521533
return this;
522534
}
523535

536+
/**
537+
* When {@code true}, sets {@code ensure_new} on the create-experiment request so a new
538+
* experiment is always created even if one with the same name already exists (the backend
539+
* dedupes the name on conflict). Useful for repeated runs (e.g. UI/remote snapshots) that
540+
* should each produce a distinct experiment. Defaults to {@code false}.
541+
*/
542+
public Builder<INPUT, OUTPUT> ensureNew(boolean ensureNew) {
543+
this.ensureNew = ensureNew;
544+
return this;
545+
}
546+
524547
/**
525548
* Sets parameter definitions for this eval. Default values from the definitions are used
526549
* unless overridden via {@link #parameterValues(Map)}.

‎braintrust-sdk/src/main/java/dev/braintrust/eval/EvalResult.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,20 @@
11
package dev.braintrust.eval;
22

3+
import javax.annotation.Nullable;
34
import lombok.Getter;
45
import lombok.SneakyThrows;
56

67
/** Results of all eval cases of an experiment. */
78
public class EvalResult {
9+
@Getter private final @Nullable String experimentId;
10+
@Getter private final @Nullable String experimentName;
811
@Getter private final String experimentUrl;
912

1013
@SneakyThrows
11-
EvalResult(String experimentUrl) {
14+
EvalResult(
15+
@Nullable String experimentId, @Nullable String experimentName, String experimentUrl) {
16+
this.experimentId = experimentId;
17+
this.experimentName = experimentName;
1218
this.experimentUrl = experimentUrl;
1319
}
1420

0 commit comments

Comments
 (0)