diff --git a/api/client.go b/api/client.go index 49f9afa7..be3f63fb 100644 --- a/api/client.go +++ b/api/client.go @@ -5,6 +5,7 @@ import ( "github.com/braintrustdata/braintrust-sdk-go/api/datasets" "github.com/braintrustdata/braintrust-sdk-go/api/experiments" "github.com/braintrustdata/braintrust-sdk-go/api/functions" + "github.com/braintrustdata/braintrust-sdk-go/api/objects" "github.com/braintrustdata/braintrust-sdk-go/api/projects" "github.com/braintrustdata/braintrust-sdk-go/internal/https" "github.com/braintrustdata/braintrust-sdk-go/logger" @@ -86,3 +87,8 @@ func (a *API) Datasets() *datasets.API { func (a *API) Functions() *functions.API { return functions.New(a.client) } + +// Objects is used to access generic object APIs (e.g. /v1/{object_type}/{id}/fetch). +func (a *API) Objects() *objects.API { + return objects.New(a.client) +} diff --git a/api/functions/functions.go b/api/functions/functions.go index 8806ff98..cfc689f3 100644 --- a/api/functions/functions.go +++ b/api/functions/functions.go @@ -102,35 +102,46 @@ func (a *API) Invoke(ctx context.Context, functionID string, input any) (any, er } path := fmt.Sprintf("/v1/function/%s/invoke", functionID) + return a.invokePath(ctx, path, req) +} + +// InvokeGlobal calls a global function by slug/type and returns the output. +func (a *API) InvokeGlobal(ctx context.Context, req InvokeGlobalParams) (any, error) { + if req.GlobalFunction == "" { + return nil, fmt.Errorf("global function is required") + } + + return a.invokePath(ctx, "/function/invoke", req) +} + +func (a *API) invokePath(ctx context.Context, path string, req any) (any, error) { resp, err := a.client.POST(ctx, path, req) if err != nil { return nil, err } defer func() { _ = resp.Body.Close() }() - // Read the entire response body so we can parse it multiple ways body, err := io.ReadAll(resp.Body) if err != nil { return nil, fmt.Errorf("failed to read response body: %w", err) } - // Parse response - try as object first, then as raw value + return decodeInvokeResponse(body) +} + +func decodeInvokeResponse(body []byte) (any, error) { var response map[string]any if err := json.Unmarshal(body, &response); err == nil { - // Response is an object, extract output field if present if output, ok := response["output"]; ok { return output, nil } - // If no output field, return the whole object return response, nil } - // Response is not an object, try parsing as raw JSON value (string, number, etc.) var output any if err := json.Unmarshal(body, &output); err != nil { return nil, fmt.Errorf("failed to decode response: %w", err) } - return output, nil } diff --git a/api/functions/functions_test.go b/api/functions/functions_test.go index 4810db42..867a04c4 100644 --- a/api/functions/functions_test.go +++ b/api/functions/functions_test.go @@ -10,6 +10,7 @@ import ( "github.com/braintrustdata/braintrust-sdk-go/api/projects" "github.com/braintrustdata/braintrust-sdk-go/internal/https" "github.com/braintrustdata/braintrust-sdk-go/internal/vcr" + "github.com/braintrustdata/braintrust-sdk-go/logger" ) const integrationTestProject = "go-sdk-tests" @@ -318,3 +319,14 @@ func TestFunctions_Invoke_Validation(t *testing.T) { require.Error(t, err) assert.Contains(t, err.Error(), "required") } + +func TestFunctions_InvokeGlobal_Validation(t *testing.T) { + t.Parallel() + + ctx := context.Background() + client := New(https.NewClient("test-key", "https://example.com", logger.Discard())) + + _, err := client.InvokeGlobal(ctx, InvokeGlobalParams{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "required") +} diff --git a/api/functions/types.go b/api/functions/types.go index f2866ee6..4e24d462 100644 --- a/api/functions/types.go +++ b/api/functions/types.go @@ -52,6 +52,14 @@ type InvokeParams struct { Input any `json:"input"` } +// InvokeGlobalParams represents the request payload for invoking a global function. +type InvokeGlobalParams struct { + GlobalFunction string `json:"global_function"` + FunctionType string `json:"function_type,omitempty"` + Mode string `json:"mode,omitempty"` + Input any `json:"input,omitempty"` +} + // QueryResponse represents the response from querying functions. type QueryResponse struct { Objects []Function `json:"objects"` diff --git a/api/objects/objects.go b/api/objects/objects.go new file mode 100644 index 00000000..fcfd489f --- /dev/null +++ b/api/objects/objects.go @@ -0,0 +1,42 @@ +package objects + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/braintrustdata/braintrust-sdk-go/internal/https" +) + +// API provides methods for generic object operations. +type API struct { + client *https.Client +} + +// New creates a new objects API client. +func New(client *https.Client) *API { + return &API{client: client} +} + +// Fetch retrieves rows from a given object type and ID. +func (a *API) Fetch(ctx context.Context, objectType, objectID string, params FetchParams) (*FetchResponse, error) { + if objectType == "" { + return nil, fmt.Errorf("object type is required") + } + if objectID == "" { + return nil, fmt.Errorf("object ID is required") + } + + path := fmt.Sprintf("/v1/%s/%s/fetch", objectType, objectID) + resp, err := a.client.POST(ctx, path, params) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + + var out FetchResponse + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return nil, fmt.Errorf("error decoding response: %w", err) + } + return &out, nil +} diff --git a/api/objects/objects_test.go b/api/objects/objects_test.go new file mode 100644 index 00000000..0c0c327f --- /dev/null +++ b/api/objects/objects_test.go @@ -0,0 +1,77 @@ +package objects + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/braintrustdata/braintrust-sdk-go/api/datasets" + "github.com/braintrustdata/braintrust-sdk-go/api/projects" + "github.com/braintrustdata/braintrust-sdk-go/internal/https" + "github.com/braintrustdata/braintrust-sdk-go/internal/vcr" + "github.com/braintrustdata/braintrust-sdk-go/logger" +) + +const integrationTestProject = "go-sdk-tests" + +func TestObjects_Fetch_Integration(t *testing.T) { + t.Parallel() + + ctx := context.Background() + client := vcr.GetHTTPSClient(t) + api := New(client) + + // Create a project and dataset with events + projectsAPI := projects.New(client) + project, err := projectsAPI.Create(ctx, projects.CreateParams{Name: integrationTestProject}) + require.NoError(t, err) + + datasetsAPI := datasets.New(client) + dataset, err := datasetsAPI.Create(ctx, datasets.CreateParams{ + ProjectID: project.ID, + Name: "test-objects-fetch", + }) + require.NoError(t, err) + defer func() { _ = datasetsAPI.Delete(ctx, dataset.ID) }() + + err = datasetsAPI.InsertEvents(ctx, dataset.ID, []datasets.Event{ + {Input: map[string]any{"q": "1"}, Expected: map[string]any{"a": "1"}}, + {Input: map[string]any{"q": "2"}, Expected: map[string]any{"a": "2"}}, + }) + require.NoError(t, err) + + // Fetch via the generic objects API (retry for eventual consistency) + var rows []map[string]any + for i := 0; i < 3; i++ { + resp, err := api.Fetch(ctx, "dataset", dataset.ID, FetchParams{Limit: 10}) + require.NoError(t, err) + require.NotNil(t, resp) + + rows = resp.Events + if len(rows) == 0 { + rows = resp.Rows + } + if len(rows) >= 2 { + break + } + time.Sleep(500 * time.Millisecond) + } + assert.GreaterOrEqual(t, len(rows), 2) +} + +func TestObjects_Fetch_Validation(t *testing.T) { + t.Parallel() + + api := New(https.NewClient("test-key", "https://example.com", logger.Discard())) + + _, err := api.Fetch(context.Background(), "", "obj-1", FetchParams{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "object type is required") + + _, err = api.Fetch(context.Background(), "experiment", "", FetchParams{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "object ID is required") +} diff --git a/api/objects/testdata/cassettes/TestObjects_Fetch_Integration.yaml b/api/objects/testdata/cassettes/TestObjects_Fetch_Integration.yaml new file mode 100644 index 00000000..55977bde --- /dev/null +++ b/api/objects/testdata/cassettes/TestObjects_Fetch_Integration.yaml @@ -0,0 +1,381 @@ +--- +version: 2 +interactions: + - id: 0 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 23 + transfer_encoding: [] + trailer: {} + host: api.braintrust.dev + remote_addr: "" + request_uri: "" + body: '{"name":"go-sdk-tests"}' + form: {} + headers: + Accept: + - application/json + Content-Type: + - application/json + url: https://api.braintrust.dev/v1/project + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: '{"id":"842c2063-b9c7-4c40-9fb9-fa95f810c57f","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"go-sdk-tests","description":null,"created":"2025-11-04T13:38:56.532Z","deleted_at":null,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","settings":null}' + headers: + Access-Control-Allow-Credentials: + - "true" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms + Content-Type: + - application/json; charset=utf-8 + Date: + - Thu, 05 Mar 2026 00:05:43 GMT + Etag: + - W/"fe-rfCZvNlcUhLHZTl4/HHBhOEJFv4" + Vary: + - Origin, Accept-Encoding + Via: + - 1.1 4894bef31db1c311602a51393339af0a.cloudfront.net (CloudFront), 1.1 9b38ff4b39c6c0a269c601916dab060e.cloudfront.net (CloudFront) + X-Amz-Apigw-Id: + - ZuRRxGLyIAMEQQg= + X-Amz-Cf-Id: + - NtBwyeB08Y-ItpW1HDvPMWbMOmv_FBRYMy9xHjSReqVWlDXpRyIUNA== + X-Amz-Cf-Pop: + - HIO52-P2 + - HIO52-P4 + X-Amzn-Requestid: + - 159f6093-569b-4354-b9dc-73ef6bb9f39f + X-Amzn-Trace-Id: + - Root=1-69a8c8d7-5d22036c0ec1651c690bcc27;Parent=2fa26f4153aa1552;Sampled=0;Lineage=1:24be3d11:0 + X-Bt-Found-Existing: + - "true" + X-Bt-Internal-Trace-Id: + - 69a8c8d70000000068ada928481af99a + X-Cache: + - Miss from cloudfront + status: 200 OK + code: 200 + duration: 309.069292ms + - id: 1 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 81 + transfer_encoding: [] + trailer: {} + host: api.braintrust.dev + remote_addr: "" + request_uri: "" + body: '{"project_id":"842c2063-b9c7-4c40-9fb9-fa95f810c57f","name":"test-objects-fetch"}' + form: {} + headers: + Accept: + - application/json + Content-Type: + - application/json + url: https://api.braintrust.dev/v1/dataset + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: '{"id":"42c8e29a-cb66-48bb-b096-231ef67ee693","project_id":"842c2063-b9c7-4c40-9fb9-fa95f810c57f","name":"test-objects-fetch","description":null,"created":"2026-03-05T00:05:44.048Z","deleted_at":null,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","metadata":null,"url_slug":"test-objects-fetch"}' + headers: + Access-Control-Allow-Credentials: + - "true" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms + Content-Type: + - application/json; charset=utf-8 + Date: + - Thu, 05 Mar 2026 00:05:44 GMT + Etag: + - W/"128-9dZLrIF+s7Enz6YOFwnL3vpsTXc" + Vary: + - Origin, Accept-Encoding + Via: + - 1.1 ac695892d6ed07904483819bdb88134e.cloudfront.net (CloudFront), 1.1 9b38ff4b39c6c0a269c601916dab060e.cloudfront.net (CloudFront) + X-Amz-Apigw-Id: + - ZuRRzHd4IAMEacw= + X-Amz-Cf-Id: + - Yrzhrg7VhMkScb55u7DCB1oLDIS-xb1wik8zT-oZUACQn9uvHDg8Qw== + X-Amz-Cf-Pop: + - HIO52-P2 + - HIO52-P4 + X-Amzn-Requestid: + - a4f176f0-a12a-4c08-b0f2-731d6e00b343 + X-Amzn-Trace-Id: + - Root=1-69a8c8d7-2c52d7f4730e93202c195117;Parent=6becba8bd3da90cf;Sampled=0;Lineage=1:24be3d11:0 + X-Bt-Internal-Trace-Id: + - 69a8c8d700000000199c4daf38897fc2 + X-Cache: + - Miss from cloudfront + status: 200 OK + code: 200 + duration: 207.960541ms + - id: 2 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 94 + transfer_encoding: [] + trailer: {} + host: api.braintrust.dev + remote_addr: "" + request_uri: "" + body: '{"events":[{"input":{"q":"1"},"expected":{"a":"1"}},{"input":{"q":"2"},"expected":{"a":"2"}}]}' + form: {} + headers: + Accept: + - application/json + Content-Type: + - application/json + url: https://api.braintrust.dev/v1/dataset/42c8e29a-cb66-48bb-b096-231ef67ee693/insert + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: '{"row_ids":["12cbf61f-e2ca-4230-89c6-2160b78849c1","575f3273-9fdd-4f49-800f-a547cb47b56f"]}' + headers: + Access-Control-Allow-Credentials: + - "true" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms + Content-Type: + - application/json; charset=utf-8 + Date: + - Thu, 05 Mar 2026 00:05:44 GMT + Etag: + - W/"5b-gNVplGTUVPDxEk/AfzY8tvQvPOI" + Vary: + - Origin, Accept-Encoding + Via: + - 1.1 1f941fcf288b6d0259a0f708c955afae.cloudfront.net (CloudFront), 1.1 9b38ff4b39c6c0a269c601916dab060e.cloudfront.net (CloudFront) + X-Amz-Apigw-Id: + - ZuRR1Ha5oAMEB0w= + X-Amz-Cf-Id: + - yeLfCHQO4arThyvdAJEQrDHVijcnfh_QaxhDlMqG_0HDrgKrAMCvdg== + X-Amz-Cf-Pop: + - HIO52-P2 + - HIO52-P4 + X-Amzn-Requestid: + - 30c48abb-b9ca-4a2b-bc52-9e7c5880a6a7 + X-Amzn-Trace-Id: + - Root=1-69a8c8d8-76d732a45576b7ef455759c6;Parent=7b066d8767d21069;Sampled=0;Lineage=1:24be3d11:0 + X-Bt-Internal-Trace-Id: + - 69a8c8d8000000001d21567b603fc11f + X-Cache: + - Miss from cloudfront + status: 200 OK + code: 200 + duration: 487.507666ms + - id: 3 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 12 + transfer_encoding: [] + trailer: {} + host: api.braintrust.dev + remote_addr: "" + request_uri: "" + body: '{"limit":10}' + form: {} + headers: + Accept: + - application/json + Content-Type: + - application/json + url: https://api.braintrust.dev/v1/dataset/42c8e29a-cb66-48bb-b096-231ef67ee693/fetch + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: '{"events":[]}' + headers: + Access-Control-Allow-Credentials: + - "true" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms + Content-Type: + - application/json + Date: + - Thu, 05 Mar 2026 00:05:54 GMT + Vary: + - Origin + Via: + - 1.1 f9cbfbc3568832d017c09dbd4649932c.cloudfront.net (CloudFront), 1.1 9b38ff4b39c6c0a269c601916dab060e.cloudfront.net (CloudFront) + X-Amz-Apigw-Id: + - ZuRR6E2ZoAMEB3A= + X-Amz-Cf-Id: + - 7kZYUDCQBe6m-b8nSMUDwsmt57_U-ic9Jjk4VzM69j0RtP6YcbXwQA== + X-Amz-Cf-Pop: + - HIO52-P2 + - HIO52-P4 + X-Amzn-Requestid: + - 6c5b3f9f-f095-4f4d-b21d-92880ca539e2 + X-Amzn-Trace-Id: + - Root=1-69a8c8d8-13d31e21678d56ae2609f81b;Parent=5995c5e98fce254b;Sampled=0;Lineage=1:24be3d11:0 + X-Bt-Api-Duration-Ms: + - "10021" + X-Bt-Brainstore-Duration-Ms: + - "10014" + X-Bt-Internal-Trace-Id: + - 69a8c8d80000000071fdc7c63e3f47c0 + X-Cache: + - Miss from cloudfront + status: 200 OK + code: 200 + duration: 10.149685667s + - id: 4 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 12 + transfer_encoding: [] + trailer: {} + host: api.braintrust.dev + remote_addr: "" + request_uri: "" + body: '{"limit":10}' + form: {} + headers: + Accept: + - application/json + Content-Type: + - application/json + url: https://api.braintrust.dev/v1/dataset/42c8e29a-cb66-48bb-b096-231ef67ee693/fetch + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: '{"events":[{"_pagination_key":"p07613556004171874305","_xact_id":"1000196765898043957","audit_data":[{"_xact_id":"1000196765898043957","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"created":"2026-03-05T00:05:44.185Z","dataset_id":"42c8e29a-cb66-48bb-b096-231ef67ee693","expected":{"a":"2"},"facets":null,"id":"575f3273-9fdd-4f49-800f-a547cb47b56f","input":{"q":"2"},"is_root":true,"metadata":null,"origin":null,"project_id":"842c2063-b9c7-4c40-9fb9-fa95f810c57f","root_span_id":"1f320e46-2283-4b4e-9cad-cc67913655fb","span_id":"1f320e46-2283-4b4e-9cad-cc67913655fb","tags":null},{"_pagination_key":"p07613556004171874304","_xact_id":"1000196765898043957","audit_data":[{"_xact_id":"1000196765898043957","audit_data":{"action":"upsert"},"metadata":{},"source":"api"}],"classifications":null,"comments":null,"created":"2026-03-05T00:05:44.185Z","dataset_id":"42c8e29a-cb66-48bb-b096-231ef67ee693","expected":{"a":"1"},"facets":null,"id":"12cbf61f-e2ca-4230-89c6-2160b78849c1","input":{"q":"1"},"is_root":true,"metadata":null,"origin":null,"project_id":"842c2063-b9c7-4c40-9fb9-fa95f810c57f","root_span_id":"413c27c2-037c-4d73-8683-a141624db7ce","span_id":"413c27c2-037c-4d73-8683-a141624db7ce","tags":null}],"cursor":"aajI2PI1AAA"}' + headers: + Access-Control-Allow-Credentials: + - "true" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms + Content-Type: + - application/json + Date: + - Thu, 05 Mar 2026 00:05:55 GMT + Vary: + - Origin + Via: + - 1.1 f9cbfbc3568832d017c09dbd4649932c.cloudfront.net (CloudFront), 1.1 9b38ff4b39c6c0a269c601916dab060e.cloudfront.net (CloudFront) + X-Amz-Apigw-Id: + - ZuRTlGI9oAMEVLw= + X-Amz-Cf-Id: + - 3Yq-Do-eXArDGVLEJTgMjkZ7C4ENHC7ZndH91GAKFx9Rz9Ccxjk_aA== + X-Amz-Cf-Pop: + - HIO52-P2 + - HIO52-P4 + X-Amzn-Requestid: + - 57a8aa34-5e33-4702-894e-89d84cf58b5d + X-Amzn-Trace-Id: + - Root=1-69a8c8e3-5acc56e165bdccc2735a345b;Parent=23befe53b999f942;Sampled=0;Lineage=1:24be3d11:0 + X-Bt-Api-Duration-Ms: + - "182" + X-Bt-Brainstore-Duration-Ms: + - "176" + X-Bt-Cursor: + - aajI2PI1AAA + X-Bt-Internal-Trace-Id: + - 69a8c8e300000000180e18ff0d85357f + X-Cache: + - Miss from cloudfront + status: 200 OK + code: 200 + duration: 322.868542ms + - id: 5 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: api.braintrust.dev + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Accept: + - application/json + url: https://api.braintrust.dev/v1/dataset/42c8e29a-cb66-48bb-b096-231ef67ee693 + method: DELETE + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: '{"id":"42c8e29a-cb66-48bb-b096-231ef67ee693","project_id":"842c2063-b9c7-4c40-9fb9-fa95f810c57f","name":"test-objects-fetch","description":null,"created":"2026-03-05T00:05:44.048Z","deleted_at":"2026-03-05T00:05:55.772Z","user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","metadata":null,"url_slug":"test-objects-fetch"}' + headers: + Access-Control-Allow-Credentials: + - "true" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms + Content-Type: + - application/json; charset=utf-8 + Date: + - Thu, 05 Mar 2026 00:05:55 GMT + Etag: + - W/"13e-pm9nW/A6zwttzgo7ca8oGurjkb8" + Vary: + - Origin, Accept-Encoding + Via: + - 1.1 e2ad8d56b8dbdb69144113ad1c008e02.cloudfront.net (CloudFront), 1.1 9b38ff4b39c6c0a269c601916dab060e.cloudfront.net (CloudFront) + X-Amz-Apigw-Id: + - ZuRToEcPoAMEvDg= + X-Amz-Cf-Id: + - jBBVVlYMA19Tav5xEVdiVUcpnX_lq4G05piS3jyflZGAW3pX5B9vOw== + X-Amz-Cf-Pop: + - HIO52-P2 + - HIO52-P4 + X-Amzn-Requestid: + - a50493f3-8dca-4e87-b882-823fe0023540 + X-Amzn-Trace-Id: + - Root=1-69a8c8e3-565cf3126a0d7e2e07594ff7;Parent=6f9cc76e0c721e0a;Sampled=0;Lineage=1:24be3d11:0 + X-Bt-Internal-Trace-Id: + - 69a8c8e3000000003a48e11d1467a996 + X-Cache: + - Miss from cloudfront + status: 200 OK + code: 200 + duration: 289.345792ms diff --git a/api/objects/types.go b/api/objects/types.go new file mode 100644 index 00000000..f15dc65a --- /dev/null +++ b/api/objects/types.go @@ -0,0 +1,17 @@ +// Package objects provides generic object operations across Braintrust object types. +package objects + +// FetchParams represents request parameters for object fetch. +type FetchParams struct { + Limit int `json:"limit,omitempty"` + Filter map[string]any `json:"filter,omitempty"` + Cursor string `json:"cursor,omitempty"` +} + +// FetchResponse is the response from an object fetch request. +type FetchResponse struct { + Events []map[string]any `json:"events"` + Rows []map[string]any `json:"rows"` + Objects []map[string]any `json:"objects"` + Cursor string `json:"cursor"` +} diff --git a/eval/eval.go b/eval/eval.go index 2d4f008c..e10c6fc0 100644 --- a/eval/eval.go +++ b/eval/eval.go @@ -230,8 +230,10 @@ type eval[I, R any] struct { datasetID string // For origin.object_id task TaskFunc[I, R] scorers []Scorer[I, R] + apiClient *api.API tracer oteltrace.Tracer startSpanOpt oteltrace.SpanStartOption + ensureFlush func() error goroutines int quiet bool } @@ -244,9 +246,11 @@ type nextCase[I, R any] struct { // newEval creates a new eval executor from concrete parameters (low-level constructor). // This is the shared code path used by both newEvalOpts (production) and testNewEval (tests). +// FIXME: we shouldn't pass session and API() — collapse into a single dependency. func newEval[I, R any]( s *auth.Session, tracer oteltrace.Tracer, + apiClient *api.API, experimentID string, experimentName string, projectID string, @@ -254,6 +258,7 @@ func newEval[I, R any]( dataset Dataset[I, R], task TaskFunc[I, R], scorers []Scorer[I, R], + ensureFlush func() error, parallelism int, quiet bool, ) *eval[I, R] { @@ -281,8 +286,10 @@ func newEval[I, R any]( datasetID: datasetID, task: task, scorers: scorers, + apiClient: apiClient, tracer: tracer, startSpanOpt: startSpanOpt, + ensureFlush: ensureFlush, goroutines: goroutines, quiet: quiet, } @@ -313,6 +320,7 @@ func newEvalOpts[I, R any](ctx context.Context, s *auth.Session, tp *trace.Trace return newEval( s, tracer, + apiClient, exp.ID, exp.Name, projectID, @@ -320,6 +328,11 @@ func newEvalOpts[I, R any](ctx context.Context, s *auth.Session, tp *trace.Trace opts.Dataset, opts.Task, opts.Scorers, + func() error { + flushCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + return tp.ForceFlush(flushCtx) + }, opts.Parallelism, opts.Quiet, ), nil @@ -425,6 +438,13 @@ func (e *eval[I, R]) runCase(ctx context.Context, span oteltrace.Span, c Case[I, return err } output := taskResult.Output + taskResult.fetcher = newSpanFetcher( + e.apiClient, + "experiment", + e.experimentID, + rootSpanIDFromSpan(span), + e.ensureFlush, + ) _, err = e.runScorers(ctx, taskResult) if err != nil { @@ -459,6 +479,13 @@ func (e *eval[I, R]) runCase(ctx context.Context, span oteltrace.Span, c Case[I, return setJSONAttrs(span, meta) } +func rootSpanIDFromSpan(span oteltrace.Span) string { + if span == nil { + return "" + } + return span.SpanContext().TraceID().String() +} + // runTask executes the task function and creates a task span. // Returns a TaskResult containing all task execution data. func (e *eval[I, R]) runTask(ctx context.Context, evalSpan oteltrace.Span, c Case[I, R]) (TaskResult[I, R], error) { @@ -523,7 +550,6 @@ func (e *eval[I, R]) runScorers(ctx context.Context, taskResult TaskResult[I, R] if err := setJSONAttr(span, "braintrust.span_attributes", scoreSpanAttrs); err != nil { return nil, err } - var scores []Score var errs []error @@ -710,6 +736,7 @@ func minInt(a, b int) int { func testNewEval[I, R any]( s *auth.Session, tracer oteltrace.Tracer, + apiClient *api.API, experimentID string, experimentName string, projectID string, @@ -723,6 +750,7 @@ func testNewEval[I, R any]( return newEval( s, tracer, + apiClient, experimentID, experimentName, projectID, @@ -730,6 +758,7 @@ func testNewEval[I, R any]( dataset, task, scorers, + nil, parallelism, true, // quiet=true for tests ) diff --git a/eval/eval_test.go b/eval/eval_test.go index 8bac1d41..f6173d38 100644 --- a/eval/eval_test.go +++ b/eval/eval_test.go @@ -47,6 +47,7 @@ func newUnitTestEval[I, R any](t *testing.T, dataset Dataset[I, R], task TaskFun e := testNewEval( session, tracer, + nil, // no apiClient for unit tests "exp-12345678", // fake experiment ID "test-experiment", // fake experiment name "proj-87654321", // fake project ID diff --git a/eval/functions_api.go b/eval/functions_api.go index 3dc11263..bd33deb9 100644 --- a/eval/functions_api.go +++ b/eval/functions_api.go @@ -115,11 +115,24 @@ func (f *FunctionsAPI[I, R]) Scorer(ctx context.Context, opts FunctionOpts) (Sco // Create a scorer that invokes the function scorerFunc := func(ctx context.Context, result TaskResult[I, R]) (Scores, error) { + spans, err := result.Spans(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get spans: %w", err) + } + thread, err := result.Thread(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get thread: %w", err) + } + // Build scorer input scorerInput := map[string]any{ "input": result.Input, "output": result.Output, "expected": result.Expected, + "trace": map[string]any{ + "spans": spans, + "thread": thread, + }, } // Invoke the scorer function diff --git a/eval/spans.go b/eval/spans.go new file mode 100644 index 00000000..a26dc199 --- /dev/null +++ b/eval/spans.go @@ -0,0 +1,252 @@ +package eval + +import ( + "context" + "encoding/json" + "sync" + + "github.com/braintrustdata/braintrust-sdk-go/api" + functionsapi "github.com/braintrustdata/braintrust-sdk-go/api/functions" + "github.com/braintrustdata/braintrust-sdk-go/api/objects" +) + +// Span represents a single span from a Braintrust trace. +type Span struct { + ID string `json:"id"` + SpanID string `json:"span_id"` + RootSpanID string `json:"root_span_id"` + SpanParents []string `json:"span_parents"` + SpanAttributes map[string]any `json:"span_attributes"` + Input any `json:"input"` + Output any `json:"output"` + Metadata map[string]any `json:"metadata"` +} + +// flushState wraps sync.Once so that spanFetcher (and therefore TaskResult) +// remains safe to copy by value. +type flushState struct { + once sync.Once + err error +} + +// spanFetcher retrieves span and thread data from the Braintrust API. +// It is unexported and attached to TaskResult as a pointer so that +// TaskResult stays copyable by value. +type spanFetcher struct { + apiClient *api.API + objectType string + objectID string + rootSpanID string + flush *flushState + flushFn func() error +} + +func newSpanFetcher( + apiClient *api.API, + objectType string, + objectID string, + rootSpanID string, + ensureSpansFlushed func() error, +) *spanFetcher { + return &spanFetcher{ + apiClient: apiClient, + objectType: objectType, + objectID: objectID, + rootSpanID: rootSpanID, + flush: &flushState{}, + flushFn: ensureSpansFlushed, + } +} + +// Spans returns spans for the provided span types. +func (f *spanFetcher) Spans(ctx context.Context, spanTypes []string) ([]Span, error) { + if f.objectType == "" || f.objectID == "" || f.rootSpanID == "" || f.apiClient == nil { + return nil, nil + } + + if err := f.ensureSpansReady(); err != nil { + return nil, err + } + + return f.fetchSpans(ctx, spanTypes) +} + +// Thread returns thread entries associated with the case. +func (f *spanFetcher) Thread(ctx context.Context) ([]map[string]any, error) { + if f.objectType == "" || f.objectID == "" || f.rootSpanID == "" || f.apiClient == nil { + return nil, nil + } + + if err := f.ensureSpansReady(); err != nil { + return nil, err + } + + return f.fetchThread(ctx) +} + +func (f *spanFetcher) ensureSpansReady() error { + f.flush.once.Do(func() { + if f.flushFn == nil { + return + } + f.flush.err = f.flushFn() + }) + return f.flush.err +} + +func (f *spanFetcher) fetchSpans(ctx context.Context, spanTypes []string) ([]Span, error) { + var all []Span + cursor := "" + + for { + req := objects.FetchParams{ + Limit: 1000, + Filter: buildSpanFilter(f.rootSpanID, spanTypes), + } + if cursor != "" { + req.Cursor = cursor + } + + payload, err := f.apiClient.Objects().Fetch(ctx, f.objectType, f.objectID, req) + if err != nil { + return nil, err + } + + rows := payload.Events + if len(rows) == 0 { + rows = payload.Rows + } + if len(rows) == 0 { + rows = payload.Objects + } + + for _, row := range rows { + if isScorerPurpose(row) { + continue + } + span, err := rowToSpan(row) + if err != nil { + return nil, err + } + all = append(all, span) + } + + if payload.Cursor == "" { + break + } + cursor = payload.Cursor + } + + return all, nil +} + +func (f *spanFetcher) fetchThread(ctx context.Context) ([]map[string]any, error) { + payload, err := f.apiClient.Functions().InvokeGlobal(ctx, functionsapi.InvokeGlobalParams{ + GlobalFunction: "project_default", + FunctionType: "preprocessor", + Mode: "json", + Input: map[string]any{ + "trace_ref": map[string]any{ + "object_type": f.objectType, + "object_id": f.objectID, + "root_span_id": f.rootSpanID, + }, + }, + }) + if err != nil { + return nil, err + } + + values, ok := payload.([]any) + if !ok { + return nil, nil + } + + thread := make([]map[string]any, 0, len(values)) + for _, value := range values { + if item, ok := value.(map[string]any); ok { + thread = append(thread, item) + } + } + return thread, nil +} + +// rowToSpan converts a raw API row into a typed Span via JSON round-trip. +func rowToSpan(row map[string]any) (Span, error) { + b, err := json.Marshal(row) + if err != nil { + return Span{}, err + } + var s Span + if err := json.Unmarshal(b, &s); err != nil { + return Span{}, err + } + return s, nil +} + +func buildSpanFilter(rootSpanID string, spanTypeFilter []string) map[string]any { + children := []map[string]any{ + { + "op": "eq", + "left": map[string]any{ + "op": "ident", + "name": []string{"root_span_id"}, + }, + "right": map[string]any{ + "op": "literal", + "value": rootSpanID, + }, + }, + { + "op": "or", + "children": []map[string]any{ + { + "op": "isnull", + "expr": map[string]any{ + "op": "ident", + "name": []string{"span_attributes", "purpose"}, + }, + }, + { + "op": "ne", + "left": map[string]any{ + "op": "ident", + "name": []string{"span_attributes", "purpose"}, + }, + "right": map[string]any{ + "op": "literal", + "value": "scorer", + }, + }, + }, + }, + } + + if len(spanTypeFilter) > 0 { + children = append(children, map[string]any{ + "op": "in", + "left": map[string]any{ + "op": "ident", + "name": []string{"span_attributes", "type"}, + }, + "right": map[string]any{ + "op": "literal", + "value": spanTypeFilter, + }, + }) + } + + return map[string]any{ + "op": "and", + "children": children, + } +} + +func isScorerPurpose(row map[string]any) bool { + attrs, ok := row["span_attributes"].(map[string]any) + if !ok || attrs == nil { + return false + } + purpose, ok := attrs["purpose"].(string) + return ok && purpose == "scorer" +} diff --git a/eval/spans_test.go b/eval/spans_test.go new file mode 100644 index 00000000..57105ed6 --- /dev/null +++ b/eval/spans_test.go @@ -0,0 +1,80 @@ +package eval + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/sdk/trace" + + "github.com/braintrustdata/braintrust-sdk-go/api/projects" +) + +func TestSpans_Integration(t *testing.T) { + session, apiClient := setupIntegrationTest(t) + t.Parallel() + + ctx := context.Background() + + _, err := apiClient.Projects().Create(ctx, projects.CreateParams{Name: integrationTestProject}) + require.NoError(t, err) + + tp := trace.NewTracerProvider() + defer func() { _ = tp.Shutdown(ctx) }() + + evaluator := NewEvaluator[string, string](session, tp, apiClient, integrationTestProject) + result, err := evaluator.Run(ctx, Opts[string, string]{ + Experiment: "test-spans", + Dataset: NewDataset([]Case[string, string]{ + {Input: "hello", Expected: "hello"}, + }), + Task: T(func(ctx context.Context, input string) (string, error) { + return input, nil + }), + Scorers: []Scorer[string, string]{ + NewScorer("spans-test", func(ctx context.Context, tr TaskResult[string, string]) (Scores, error) { + // Fetch all spans + spans, err := tr.Spans(ctx) + require.NoError(t, err) + + // Fetch thread (will be nil for simple task, but should not error) + thread, err := tr.Thread(ctx) + require.NoError(t, err) + + t.Logf("spans=%d thread=%d", len(spans), len(thread)) + + // Filter by nonexistent type — should return empty + filtered, err := tr.Spans(ctx, WithSpanTypes("nonexistent")) + require.NoError(t, err) + assert.Empty(t, filtered) + + return S(1.0), nil + }), + }, + Quiet: true, + }) + + require.NoError(t, err) + require.NotNil(t, result) +} + +func TestSpans_NilFetcher(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + // TaskResult with no fetcher should return nil, nil + tr := TaskResult[string, string]{ + Input: "hello", + Output: "hello", + } + + spans, err := tr.Spans(ctx) + assert.NoError(t, err) + assert.Nil(t, spans) + + thread, err := tr.Thread(ctx) + assert.NoError(t, err) + assert.Nil(t, thread) +} diff --git a/eval/task.go b/eval/task.go index 77e96e24..8018ba2a 100644 --- a/eval/task.go +++ b/eval/task.go @@ -45,6 +45,45 @@ type TaskResult[I, R any] struct { // UserData is custom application context from the task. // This field is NOT logged and isn't supported outside the context of the Go SDK. UserData any + + fetcher *spanFetcher // unexported, pointer for nil-check + safe copy +} + +// SpanQueryOpt is a functional option for configuring a Spans query. +type SpanQueryOpt func(*spansQuery) + +type spansQuery struct { + types []string +} + +// WithSpanTypes filters spans by span_attributes.type (e.g. "llm", "function", "custom"). +// Multiple types are OR'd together. Omit to get all spans. +func WithSpanTypes(types ...string) SpanQueryOpt { + return func(q *spansQuery) { + q.types = types + } +} + +// Spans returns spans from the trace. +// Returns nil, nil if no trace data is available (e.g. no API client configured). +func (r TaskResult[I, R]) Spans(ctx context.Context, opts ...SpanQueryOpt) ([]Span, error) { + if r.fetcher == nil { + return nil, nil + } + var q spansQuery + for _, opt := range opts { + opt(&q) + } + return r.fetcher.Spans(ctx, q.types) +} + +// Thread returns thread entries associated with this case's trace. +// Returns nil, nil if no trace data is available (e.g. no API client configured). +func (r TaskResult[I, R]) Thread(ctx context.Context) ([]map[string]any, error) { + if r.fetcher == nil { + return nil, nil + } + return r.fetcher.Thread(ctx) } // T is a convenience function for writing short task functions ([TaskFunc]) that only diff --git a/eval/testdata/cassettes/TestFunctionsAPI_EndToEnd_MixedTypes.yaml b/eval/testdata/cassettes/TestFunctionsAPI_EndToEnd_MixedTypes.yaml index f875cd82..09455dfa 100644 --- a/eval/testdata/cassettes/TestFunctionsAPI_EndToEnd_MixedTypes.yaml +++ b/eval/testdata/cassettes/TestFunctionsAPI_EndToEnd_MixedTypes.yaml @@ -6,18 +6,18 @@ interactions: proto: HTTP/1.1 proto_major: 1 proto_minor: 1 - content_length: 23 + content_length: 0 transfer_encoding: [] trailer: {} host: api.braintrust.dev remote_addr: "" request_uri: "" - body: '{"name":"go-sdk-tests"}' + body: "" form: {} headers: - Content-Type: + Accept: - application/json - url: https://api.braintrust.dev/v1/project + url: https://api.braintrust.dev/api/apikey/login method: POST response: proto: HTTP/2.0 @@ -27,58 +27,60 @@ interactions: trailer: {} content_length: -1 uncompressed: true - body: '{"id":"842c2063-b9c7-4c40-9fb9-fa95f810c57f","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"go-sdk-tests","created":"2025-11-04T13:38:56.532Z","deleted_at":null,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","settings":null}' + body: '{"org_info":[{"id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"matt-test-org","api_url":"https://api.braintrust.dev","git_metadata":null,"is_universal_api":null,"proxy_url":"https://api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' headers: Access-Control-Allow-Credentials: - "true" Access-Control-Expose-Headers: - - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms Content-Type: - application/json; charset=utf-8 Date: - - Wed, 12 Nov 2025 15:00:25 GMT + - Thu, 05 Mar 2026 00:49:35 GMT Etag: - - W/"eb-poL8yUftdUFahhL/Peyi6gN8N/E" + - W/"101-EqXzt+vlRFUt5HCzQY7qxp9rZU0" Vary: - Origin, Accept-Encoding Via: - - 1.1 3b0649a8bee506c1d7498462d39e6c44.cloudfront.net (CloudFront), 1.1 f458ab1245bb4f257969c1da8e708f88.cloudfront.net (CloudFront) + - 1.1 2e87eef03ab555daefa684d946e111b4.cloudfront.net (CloudFront), 1.1 7ad3d6571deff4c3c83d7e4476fcc6d0.cloudfront.net (CloudFront) X-Amz-Apigw-Id: - - T74ZkESnIAMEPdw= + - ZuXs7GIdIAMEchQ= X-Amz-Cf-Id: - - bq6E0LTeZeS6CjqqKRuVB06w81pRQd3dIkmhwK_p91JliQSGkLJ7rA== + - V8BFdU20tZ-OtNCT8IR0Cjh0HOAHxZ6DixO_t_xx05s4zyyuhZoUBw== X-Amz-Cf-Pop: - - JFK50-P5 - - JFK50-P2 + - HIO52-P2 + - HIO52-P4 X-Amzn-Requestid: - - 58587d56-8adf-42cb-9708-ab6b36db4da5 + - ce478b10-34cf-436c-a070-693a814104c0 X-Amzn-Trace-Id: - - Root=1-6914a109-0a8d601052e6e10c4220a61f;Parent=170ad496f1585e98;Sampled=0;Lineage=1:24be3d11:0 - X-Bt-Found-Existing: - - "true" + - Root=1-69a8d31f-4acc0edc52f76cbb3b5712fe;Parent=2e4f989f8fd73461;Sampled=0;Lineage=1:24be3d11:0 X-Bt-Internal-Trace-Id: - - 6914a109000000001d4b1e1cfdec0f4a + - 69a8d31f000000000a7548177a572e2c X-Cache: - Miss from cloudfront status: 200 OK code: 200 - duration: 160.601542ms + duration: 380.176208ms - id: 1 request: proto: HTTP/1.1 proto_major: 1 proto_minor: 1 - content_length: 0 + content_length: 23 transfer_encoding: [] trailer: {} host: api.braintrust.dev remote_addr: "" request_uri: "" - body: "" + body: '{"name":"go-sdk-tests"}' form: {} - headers: {} - url: https://api.braintrust.dev/v1/function?limit=1&project_name=go-sdk-tests&slug=TestFunctionsAPI_EndToEnd_MixedTypes-task - method: GET + headers: + Accept: + - application/json + Content-Type: + - application/json + url: https://api.braintrust.dev/v1/project + method: POST response: proto: HTTP/2.0 proto_major: 2 @@ -87,40 +89,42 @@ interactions: trailer: {} content_length: -1 uncompressed: true - body: '{"objects":[]}' + body: '{"id":"842c2063-b9c7-4c40-9fb9-fa95f810c57f","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"go-sdk-tests","description":null,"created":"2025-11-04T13:38:56.532Z","deleted_at":null,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","settings":null}' headers: Access-Control-Allow-Credentials: - "true" Access-Control-Expose-Headers: - - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms Content-Type: - application/json; charset=utf-8 Date: - - Wed, 12 Nov 2025 15:00:25 GMT + - Thu, 05 Mar 2026 00:49:35 GMT Etag: - - W/"e-xZKibKAiOxxBbzTm2byfFNRkvtA" + - W/"fe-rfCZvNlcUhLHZTl4/HHBhOEJFv4" Vary: - Origin, Accept-Encoding Via: - - 1.1 f9aa0e4086fcbefc20f307d96a8e3b44.cloudfront.net (CloudFront), 1.1 f458ab1245bb4f257969c1da8e708f88.cloudfront.net (CloudFront) + - 1.1 7f51caabae8141bdcde4283a42be2a56.cloudfront.net (CloudFront), 1.1 7ad3d6571deff4c3c83d7e4476fcc6d0.cloudfront.net (CloudFront) X-Amz-Apigw-Id: - - T74ZlGNGIAMERPw= + - ZuXs7HlSIAMEVig= X-Amz-Cf-Id: - - T-kQF4EqoQcJNC6XpTwzujkD0Eb_t9OstjxxI5FZrvWIS5sx6I7rLg== + - 4yapvkcSyfJEzfYb8LVqsbRsixdYeI8jRbJXQo7UN8dFvnMb47iGWg== X-Amz-Cf-Pop: - - JFK50-P5 - - JFK50-P2 + - HIO52-P2 + - HIO52-P4 X-Amzn-Requestid: - - 8457ba24-fcb7-4fe5-9982-3d5668220f05 + - d06d5ac0-e676-4987-a21f-fccc52a4af3a X-Amzn-Trace-Id: - - Root=1-6914a109-4ba7da5643c0654b248e24f4;Parent=2c4929839e1a788a;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-69a8d31f-34e52b9112481fec2f1595c1;Parent=1105d6a35631322e;Sampled=0;Lineage=1:24be3d11:0 + X-Bt-Found-Existing: + - "true" X-Bt-Internal-Trace-Id: - - 6914a109000000007e1a8d75bcd327d1 + - 69a8d31f000000003c1b407edee5ddc4 X-Cache: - Miss from cloudfront status: 200 OK code: 200 - duration: 138.443417ms + duration: 400.695125ms - id: 2 request: proto: HTTP/1.1 @@ -134,9 +138,11 @@ interactions: request_uri: "" body: "" form: {} - headers: {} - url: https://api.braintrust.dev/api/apikey/login - method: POST + headers: + Accept: + - application/json + url: https://api.braintrust.dev/v1/function?limit=1&project_name=go-sdk-tests&slug=TestFunctionsAPI_EndToEnd_MixedTypes-task + method: GET response: proto: HTTP/2.0 proto_major: 2 @@ -145,40 +151,40 @@ interactions: trailer: {} content_length: -1 uncompressed: true - body: '{"org_info":[{"id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"matt-test-org","api_url":"https://api.braintrust.dev","git_metadata":null,"is_universal_api":null,"proxy_url":"https://api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' + body: '{"objects":[]}' headers: Access-Control-Allow-Credentials: - "true" Access-Control-Expose-Headers: - - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms Content-Type: - application/json; charset=utf-8 Date: - - Wed, 12 Nov 2025 15:00:25 GMT + - Thu, 05 Mar 2026 00:49:35 GMT Etag: - - W/"101-EqXzt+vlRFUt5HCzQY7qxp9rZU0" + - W/"e-xZKibKAiOxxBbzTm2byfFNRkvtA" Vary: - Origin, Accept-Encoding Via: - - 1.1 f8debc28b6c73eb3dc7540e2ac2f0e18.cloudfront.net (CloudFront), 1.1 f458ab1245bb4f257969c1da8e708f88.cloudfront.net (CloudFront) + - 1.1 fd441d5d42c4e243bf0b88902034e302.cloudfront.net (CloudFront), 1.1 7ad3d6571deff4c3c83d7e4476fcc6d0.cloudfront.net (CloudFront) X-Amz-Apigw-Id: - - T74ZkHBJoAMEPBg= + - ZuXs9FUboAMEGgA= X-Amz-Cf-Id: - - NkjLZjEVuk42hwd7EsbR07NSziPr4Mhwy8_YQWIyWDjb3U2N0kLP4A== + - GUVQ6ds4Ahj4OccHimN5-a9m0qYiYrpDdmHV4oIykOYUs-n9arMx2w== X-Amz-Cf-Pop: - - JFK50-P5 - - JFK50-P2 + - HIO52-P2 + - HIO52-P4 X-Amzn-Requestid: - - ce1757d6-f991-4c45-94a4-1fc76a7abd4d + - 76ec94a1-b2ed-451b-94d5-457c1f33d631 X-Amzn-Trace-Id: - - Root=1-6914a109-2c0f14125abbdc713a3f7a6f;Parent=4384fe985ff6ac96;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-69a8d31f-770acd6e4d3ce64f09aca409;Parent=49940aabbe447108;Sampled=0;Lineage=1:24be3d11:0 X-Bt-Internal-Trace-Id: - - 6914a1090000000072cc3183fcea9ee1 + - 69a8d31f0000000055a5e274c9362d08 X-Cache: - Miss from cloudfront status: 200 OK code: 200 - duration: 299.309167ms + duration: 255.454416ms - id: 3 request: proto: HTTP/1.1 @@ -193,6 +199,8 @@ interactions: body: '{"project_id":"842c2063-b9c7-4c40-9fb9-fa95f810c57f","name":"E2E Task","slug":"TestFunctionsAPI_EndToEnd_MixedTypes-task","function_data":{"type":"prompt"},"prompt_data":{"options":{"model":"gpt-4o-mini","params":{"max_tokens":100,"response_format":{"type":"json_object"},"temperature":0}},"prompt":{"messages":[{"content":"You answer questions with JSON. Return ONLY the JSON object, no markdown, no code blocks, no backticks.","role":"system"},{"content":"Question: {{input.question}}. Return ONLY JSON like {\"answer\": \"your answer\", \"confidence\": 0.9}. No markdown formatting.","role":"user"}],"type":"chat"}}}' form: {} headers: + Accept: + - application/json Content-Type: - application/json url: https://api.braintrust.dev/v1/function @@ -205,40 +213,40 @@ interactions: trailer: {} content_length: -1 uncompressed: true - body: '{"log_id":"p","project_id":"842c2063-b9c7-4c40-9fb9-fa95f810c57f","slug":"TestFunctionsAPI_EndToEnd_MixedTypes-task","name":"E2E Task","function_data":{"type":"prompt"},"prompt_data":{"options":{"model":"gpt-4o-mini","params":{"max_tokens":100,"response_format":{"type":"json_object"},"temperature":0}},"prompt":{"messages":[{"content":"You answer questions with JSON. Return ONLY the JSON object, no markdown, no code blocks, no backticks.","role":"system"},{"content":"Question: {{input.question}}. Return ONLY JSON like {\"answer\": \"your answer\", \"confidence\": 0.9}. No markdown formatting.","role":"user"}],"type":"chat"}},"id":"29072339-6591-46f1-a865-3fcce6c44f40","created":"2025-11-12T15:00:26.063Z","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","_xact_id":"1000196129575022738"}' + body: '{"log_id":"p","project_id":"842c2063-b9c7-4c40-9fb9-fa95f810c57f","slug":"TestFunctionsAPI_EndToEnd_MixedTypes-task","name":"E2E Task","function_data":{"type":"prompt"},"prompt_data":{"options":{"model":"gpt-4o-mini","params":{"max_tokens":100,"response_format":{"type":"json_object"},"temperature":0}},"prompt":{"messages":[{"content":"You answer questions with JSON. Return ONLY the JSON object, no markdown, no code blocks, no backticks.","role":"system"},{"content":"Question: {{input.question}}. Return ONLY JSON like {\"answer\": \"your answer\", \"confidence\": 0.9}. No markdown formatting.","role":"user"}],"type":"chat"}},"id":"aeb4d39b-f38b-4a20-98d3-6e5adfe64af6","created":"2026-03-05T00:49:35.784Z","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","_xact_id":"1000196766070421934"}' headers: Access-Control-Allow-Credentials: - "true" Access-Control-Expose-Headers: - - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms Content-Type: - application/json; charset=utf-8 Date: - - Wed, 12 Nov 2025 15:00:26 GMT + - Thu, 05 Mar 2026 00:49:36 GMT Etag: - - W/"31a-2sPpzg0Qu9SZPTYx3IyIzx1xF5E" + - W/"31a-u63TMcnl+2zDildgbw2PNcNhP0U" Vary: - Origin, Accept-Encoding Via: - - 1.1 98bc8180e0431e8f05afc9802305f1d2.cloudfront.net (CloudFront), 1.1 f458ab1245bb4f257969c1da8e708f88.cloudfront.net (CloudFront) + - 1.1 89b24af8db05335e68292856e0a53668.cloudfront.net (CloudFront), 1.1 7ad3d6571deff4c3c83d7e4476fcc6d0.cloudfront.net (CloudFront) X-Amz-Apigw-Id: - - T74ZnGZkIAMENaQ= + - ZuXtAE2cIAMEmWQ= X-Amz-Cf-Id: - - KRoRqECIY27YqIwiuE_LbGUAPaZ6RmkV2S_M7rH11Fwk7C4i3RTIOg== + - 6UUWmKmzXQe3OsZV81oR13W8Hncj4tWJss-3tD2sifgy3sc3bmlDOw== X-Amz-Cf-Pop: - - JFK50-P5 - - JFK50-P2 + - HIO52-P2 + - HIO52-P4 X-Amzn-Requestid: - - 2b4bb93b-05d0-43e0-8bba-dd670625e4a8 + - 3b42dc12-0c4d-42f8-bc3a-8e290491fdb7 X-Amzn-Trace-Id: - - Root=1-6914a109-6ffe8a575ced2c766fbb325b;Parent=51dd41033bedff9c;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-69a8d31f-06784b973e1189dc09902857;Parent=3173e56c2fd25c97;Sampled=0;Lineage=1:24be3d11:0 X-Bt-Internal-Trace-Id: - - 6914a1090000000005877586dd2100fa + - 69a8d31f00000000028bd8cf775380e5 X-Cache: - Miss from cloudfront status: 200 OK code: 200 - duration: 360.401958ms + duration: 464.592875ms - id: 4 request: proto: HTTP/1.1 @@ -252,7 +260,9 @@ interactions: request_uri: "" body: "" form: {} - headers: {} + headers: + Accept: + - application/json url: https://api.braintrust.dev/v1/function?limit=1&project_name=go-sdk-tests&slug=TestFunctionsAPI_EndToEnd_MixedTypes-scorer method: GET response: @@ -268,35 +278,35 @@ interactions: Access-Control-Allow-Credentials: - "true" Access-Control-Expose-Headers: - - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms Content-Type: - application/json; charset=utf-8 Date: - - Wed, 12 Nov 2025 15:00:26 GMT + - Thu, 05 Mar 2026 00:49:36 GMT Etag: - W/"e-xZKibKAiOxxBbzTm2byfFNRkvtA" Vary: - Origin, Accept-Encoding Via: - - 1.1 c4d0da6268789cfda9bb5da1f3f8fc58.cloudfront.net (CloudFront), 1.1 f458ab1245bb4f257969c1da8e708f88.cloudfront.net (CloudFront) + - 1.1 250b49a977a2df6676d3fbf2508fc16e.cloudfront.net (CloudFront), 1.1 7ad3d6571deff4c3c83d7e4476fcc6d0.cloudfront.net (CloudFront) X-Amz-Apigw-Id: - - T74ZqGTdoAMEHhw= + - ZuXtGGvFIAMES0Q= X-Amz-Cf-Id: - - hEqXcSW2c4EF1D2r7kGMSbF-_IfaxDmv-X6qu3mLW6KzgIGz5sdCEw== + - qVmNtkX4-RJXRI9KlxOJN5gHwqWTqP-v0dn-kPrlLEsRwwI8gEpp7A== X-Amz-Cf-Pop: - - JFK50-P5 - - JFK50-P2 + - HIO52-P2 + - HIO52-P4 X-Amzn-Requestid: - - 2a4c0f5e-8d2b-416f-82c5-7032aeecc557 + - 653ec07c-d886-4f53-bb0e-43fc9812c710 X-Amzn-Trace-Id: - - Root=1-6914a10a-4b7fa5e172b2d71169091733;Parent=7060252e74711618;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-69a8d320-2723f5b1518c7f98388eacc4;Parent=7684b9b0f688ee60;Sampled=0;Lineage=1:24be3d11:0 X-Bt-Internal-Trace-Id: - - 6914a10a000000007a91e819becd7717 + - 69a8d320000000000fa0909979f17aac X-Cache: - Miss from cloudfront status: 200 OK code: 200 - duration: 171.598958ms + duration: 385.217042ms - id: 5 request: proto: HTTP/1.1 @@ -311,6 +321,8 @@ interactions: body: '{"project_id":"842c2063-b9c7-4c40-9fb9-fa95f810c57f","name":"E2E Scorer","slug":"TestFunctionsAPI_EndToEnd_MixedTypes-scorer","function_type":"scorer","function_data":{"type":"prompt"},"prompt_data":{"options":{"model":"gpt-4o-mini","params":{"temperature":0}},"parser":{"choice_scores":{"correct":1,"incorrect":0},"type":"llm_classifier","use_cot":false},"prompt":{"messages":[{"content":"You are a scorer.","role":"system"},{"content":"Is the output.answer field non-empty? Choose ''correct'' if yes, ''incorrect'' if no.","role":"user"}],"type":"chat"}}}' form: {} headers: + Accept: + - application/json Content-Type: - application/json url: https://api.braintrust.dev/v1/function @@ -323,40 +335,40 @@ interactions: trailer: {} content_length: -1 uncompressed: true - body: '{"log_id":"p","project_id":"842c2063-b9c7-4c40-9fb9-fa95f810c57f","slug":"TestFunctionsAPI_EndToEnd_MixedTypes-scorer","name":"E2E Scorer","function_type":"scorer","function_data":{"type":"prompt"},"prompt_data":{"options":{"model":"gpt-4o-mini","params":{"temperature":0}},"parser":{"choice_scores":{"correct":1,"incorrect":0},"type":"llm_classifier","use_cot":false},"prompt":{"messages":[{"content":"You are a scorer.","role":"system"},{"content":"Is the output.answer field non-empty? Choose ''correct'' if yes, ''incorrect'' if no.","role":"user"}],"type":"chat"}},"id":"063f91ca-beb0-42dd-8f39-d34bebea6d98","created":"2025-11-12T15:00:26.612Z","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","_xact_id":"1000196129575023135"}' + body: '{"log_id":"p","project_id":"842c2063-b9c7-4c40-9fb9-fa95f810c57f","slug":"TestFunctionsAPI_EndToEnd_MixedTypes-scorer","name":"E2E Scorer","function_type":"scorer","function_data":{"type":"prompt"},"prompt_data":{"options":{"model":"gpt-4o-mini","params":{"temperature":0}},"parser":{"choice_scores":{"correct":1,"incorrect":0},"type":"llm_classifier","use_cot":false},"prompt":{"messages":[{"content":"You are a scorer.","role":"system"},{"content":"Is the output.answer field non-empty? Choose ''correct'' if yes, ''incorrect'' if no.","role":"user"}],"type":"chat"}},"id":"c81984ec-a11b-4de4-86bd-6256985f82ab","created":"2026-03-05T00:49:36.599Z","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","_xact_id":"1000196766070488487"}' headers: Access-Control-Allow-Credentials: - "true" Access-Control-Expose-Headers: - - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms Content-Type: - application/json; charset=utf-8 Date: - - Wed, 12 Nov 2025 15:00:26 GMT + - Thu, 05 Mar 2026 00:49:36 GMT Etag: - - W/"2d8-93Sx/tzEYoS276h1cUOf6qrdEvM" + - W/"2d8-B2EIoXqQeewNJn70u+y9hMRF4tQ" Vary: - Origin, Accept-Encoding Via: - - 1.1 b5fe18267507cb61755963d8928a60f4.cloudfront.net (CloudFront), 1.1 f458ab1245bb4f257969c1da8e708f88.cloudfront.net (CloudFront) + - 1.1 8502ceae0080b3523f89d1a518a99726.cloudfront.net (CloudFront), 1.1 7ad3d6571deff4c3c83d7e4476fcc6d0.cloudfront.net (CloudFront) X-Amz-Apigw-Id: - - T74ZsE7jIAMETeQ= + - ZuXtIE6woAMEmWQ= X-Amz-Cf-Id: - - IaxeYDwGKAKbDCYNco4K7FSWX22C9IQ3j1L85nTsPyWtJo6jTgAETw== + - cE-fY0vCxOVVFNZHAN-FTwc6qQfOoWTaVzxjCft7i_ITnUaxcyxxjQ== X-Amz-Cf-Pop: - - JFK50-P5 - - JFK50-P2 + - HIO52-P2 + - HIO52-P4 X-Amzn-Requestid: - - be90eaa1-31f7-49ea-a5c5-0e3140ad5aa2 + - a0d74814-bab1-44f4-b04f-dd596c8f2573 X-Amzn-Trace-Id: - - Root=1-6914a10a-103b89b5424920f4364c4d58;Parent=61cc9c1bc2f71c31;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-69a8d320-60e184231085f5a16ec41b17;Parent=36d8413b312ae1fb;Sampled=0;Lineage=1:24be3d11:0 X-Bt-Internal-Trace-Id: - - 6914a10a0000000018968c383f8d7194 + - 69a8d320000000001c1afabe800f4d16 X-Cache: - Miss from cloudfront status: 200 OK code: 200 - duration: 442.612292ms + duration: 490.491708ms - id: 6 request: proto: HTTP/1.1 @@ -370,7 +382,9 @@ interactions: request_uri: "" body: "" form: {} - headers: {} + headers: + Accept: + - application/json url: https://api.braintrust.dev/v1/function?limit=1&project_name=go-sdk-tests&slug=TestFunctionsAPI_EndToEnd_MixedTypes-task method: GET response: @@ -381,40 +395,40 @@ interactions: trailer: {} content_length: -1 uncompressed: true - body: '{"objects":[{"id":"29072339-6591-46f1-a865-3fcce6c44f40","_xact_id":"1000196129575022738","project_id":"842c2063-b9c7-4c40-9fb9-fa95f810c57f","log_id":"p","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"E2E Task","slug":"TestFunctionsAPI_EndToEnd_MixedTypes-task","description":null,"created":"2025-11-12T15:00:26.063Z","prompt_data":{"prompt":{"type":"chat","messages":[{"role":"system","content":"You answer questions with JSON. Return ONLY the JSON object, no markdown, no code blocks, no backticks."},{"role":"user","content":"Question: {{input.question}}. Return ONLY JSON like {\"answer\": \"your answer\", \"confidence\": 0.9}. No markdown formatting."}]},"options":{"model":"gpt-4o-mini","params":{"max_tokens":100,"temperature":0,"response_format":{"type":"json_object"}}}},"tags":null,"metadata":null,"function_type":null,"function_data":{"type":"prompt"},"origin":null,"function_schema":null}]}' + body: '{"objects":[{"id":"aeb4d39b-f38b-4a20-98d3-6e5adfe64af6","_xact_id":"1000196766070421934","project_id":"842c2063-b9c7-4c40-9fb9-fa95f810c57f","log_id":"p","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"E2E Task","slug":"TestFunctionsAPI_EndToEnd_MixedTypes-task","description":null,"created":"2026-03-05T00:49:35.784Z","prompt_data":{"prompt":{"type":"chat","messages":[{"role":"system","content":"You answer questions with JSON. Return ONLY the JSON object, no markdown, no code blocks, no backticks."},{"role":"user","content":"Question: {{input.question}}. Return ONLY JSON like {\"answer\": \"your answer\", \"confidence\": 0.9}. No markdown formatting."}]},"options":{"model":"gpt-4o-mini","params":{"max_tokens":100,"temperature":0,"response_format":{"type":"json_object"}}}},"tags":null,"metadata":null,"function_type":null,"function_data":{"type":"prompt"},"origin":null,"function_schema":null}]}' headers: Access-Control-Allow-Credentials: - "true" Access-Control-Expose-Headers: - - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms Content-Type: - application/json; charset=utf-8 Date: - - Wed, 12 Nov 2025 15:00:27 GMT + - Thu, 05 Mar 2026 00:49:37 GMT Etag: - - W/"391-e7g1AM+1gReVj71N32+mVYVwDuE" + - W/"391-h4a/5l/DbhX6bRvfOtLQO3uXUOw" Vary: - Origin, Accept-Encoding Via: - - 1.1 ab734ad5d81cc9d470b6176a05dd968e.cloudfront.net (CloudFront), 1.1 f458ab1245bb4f257969c1da8e708f88.cloudfront.net (CloudFront) + - 1.1 5b69cd230a06f482da15abd9c53bb694.cloudfront.net (CloudFront), 1.1 7ad3d6571deff4c3c83d7e4476fcc6d0.cloudfront.net (CloudFront) X-Amz-Apigw-Id: - - T74ZwGwpoAMEtrg= + - ZuXtNGy0IAMElHQ= X-Amz-Cf-Id: - - jDGU3joTBIFHc2tfXwVbwZdLpnzfKH-qwfY4wXxkBfP3BrN5_L8dwg== + - qGJipj6FaKThoF9UtKBkahQ7YlUTrisaI4-nYB8rcBTruXt1oOg9Dg== X-Amz-Cf-Pop: - - JFK50-P5 - - JFK50-P2 + - HIO52-P2 + - HIO52-P4 X-Amzn-Requestid: - - ca78451f-3b60-465f-a2a0-0e2afffaf555 + - 2c0107a5-0670-45f6-bc5d-aa30061a05df X-Amzn-Trace-Id: - - Root=1-6914a10a-43f1a2ed147c8e9b2aa29e00;Parent=39b94375a3b02854;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-69a8d320-51a17bba7a615bcd511aec1b;Parent=635b6fc9fd247e2e;Sampled=0;Lineage=1:24be3d11:0 X-Bt-Internal-Trace-Id: - - 6914a10a00000000398c274d02a41e0b + - 69a8d32100000000403013d9aeddd3b5 X-Cache: - Miss from cloudfront status: 200 OK code: 200 - duration: 516.321125ms + duration: 761.63ms - id: 7 request: proto: HTTP/1.1 @@ -428,7 +442,9 @@ interactions: request_uri: "" body: "" form: {} - headers: {} + headers: + Accept: + - application/json url: https://api.braintrust.dev/v1/function?limit=1&project_name=go-sdk-tests&slug=TestFunctionsAPI_EndToEnd_MixedTypes-scorer method: GET response: @@ -439,40 +455,40 @@ interactions: trailer: {} content_length: -1 uncompressed: true - body: '{"objects":[{"id":"063f91ca-beb0-42dd-8f39-d34bebea6d98","_xact_id":"1000196129575023135","project_id":"842c2063-b9c7-4c40-9fb9-fa95f810c57f","log_id":"p","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"E2E Scorer","slug":"TestFunctionsAPI_EndToEnd_MixedTypes-scorer","description":null,"created":"2025-11-12T15:00:26.612Z","prompt_data":{"parser":{"type":"llm_classifier","use_cot":false,"choice_scores":{"correct":1,"incorrect":0}},"prompt":{"type":"chat","messages":[{"role":"system","content":"You are a scorer."},{"role":"user","content":"Is the output.answer field non-empty? Choose ''correct'' if yes, ''incorrect'' if no."}]},"options":{"model":"gpt-4o-mini","params":{"temperature":0}}},"tags":null,"metadata":null,"function_type":"scorer","function_data":{"type":"prompt"},"origin":null,"function_schema":null}]}' + body: '{"objects":[{"id":"c81984ec-a11b-4de4-86bd-6256985f82ab","_xact_id":"1000196766070488487","project_id":"842c2063-b9c7-4c40-9fb9-fa95f810c57f","log_id":"p","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"E2E Scorer","slug":"TestFunctionsAPI_EndToEnd_MixedTypes-scorer","description":null,"created":"2026-03-05T00:49:36.599Z","prompt_data":{"parser":{"type":"llm_classifier","use_cot":false,"choice_scores":{"correct":1,"incorrect":0}},"prompt":{"type":"chat","messages":[{"role":"system","content":"You are a scorer."},{"role":"user","content":"Is the output.answer field non-empty? Choose ''correct'' if yes, ''incorrect'' if no."}]},"options":{"model":"gpt-4o-mini","params":{"temperature":0}}},"tags":null,"metadata":null,"function_type":"scorer","function_data":{"type":"prompt"},"origin":null,"function_schema":null}]}' headers: Access-Control-Allow-Credentials: - "true" Access-Control-Expose-Headers: - - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms Content-Type: - application/json; charset=utf-8 Date: - - Wed, 12 Nov 2025 15:00:27 GMT + - Thu, 05 Mar 2026 00:49:38 GMT Etag: - - W/"33a-5OmlqaB1nSGvD+Kl+SBxtCIA+PI" + - W/"33a-H0msJQDiJn46t+m73Iy7uiLCpsY" Vary: - Origin, Accept-Encoding Via: - - 1.1 8a9cdb228e33f8d52a4b42c56ca26590.cloudfront.net (CloudFront), 1.1 f458ab1245bb4f257969c1da8e708f88.cloudfront.net (CloudFront) + - 1.1 49798ef4b8dd64fece36e067d09f69ec.cloudfront.net (CloudFront), 1.1 7ad3d6571deff4c3c83d7e4476fcc6d0.cloudfront.net (CloudFront) X-Amz-Apigw-Id: - - T74Z2FYeoAMEp7Q= + - ZuXtVEIZIAMEVLw= X-Amz-Cf-Id: - - RxOZRJE8PdEN4BGUY1SbmOPZe7F3HXX0nwKynn2HNtKJfdzTrnWg0A== + - A0mRWWyNZ7WYTt3okSZdWDDUG5MCR7nb9ajX9NZMS8vk1Qw0wkzWIw== X-Amz-Cf-Pop: - - JFK50-P5 - - JFK50-P2 + - HIO52-P2 + - HIO52-P4 X-Amzn-Requestid: - - 2e590e88-06e4-4ccd-9979-1b7f88d769e6 + - 35e30c44-dc20-4d93-9565-37d20c99577c X-Amzn-Trace-Id: - - Root=1-6914a10b-31ac38366013571e2face76b;Parent=243c6f347d1cef8a;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-69a8d321-4c374a286eff9c654d9969cf;Parent=30855eddda5adf37;Sampled=0;Lineage=1:24be3d11:0 X-Bt-Internal-Trace-Id: - - 6914a10b000000003ad3068347d3e5ce + - 69a8d3210000000007fdcd5bf0c22773 X-Cache: - Miss from cloudfront status: 200 OK code: 200 - duration: 503.660916ms + duration: 560.324541ms - id: 8 request: proto: HTTP/1.1 @@ -487,6 +503,8 @@ interactions: body: '{"name":"go-sdk-tests"}' form: {} headers: + Accept: + - application/json Content-Type: - application/json url: https://api.braintrust.dev/v1/project @@ -499,42 +517,42 @@ interactions: trailer: {} content_length: -1 uncompressed: true - body: '{"id":"842c2063-b9c7-4c40-9fb9-fa95f810c57f","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"go-sdk-tests","created":"2025-11-04T13:38:56.532Z","deleted_at":null,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","settings":null}' + body: '{"id":"842c2063-b9c7-4c40-9fb9-fa95f810c57f","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"go-sdk-tests","description":null,"created":"2025-11-04T13:38:56.532Z","deleted_at":null,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","settings":null}' headers: Access-Control-Allow-Credentials: - "true" Access-Control-Expose-Headers: - - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms Content-Type: - application/json; charset=utf-8 Date: - - Wed, 12 Nov 2025 15:00:28 GMT + - Thu, 05 Mar 2026 00:49:38 GMT Etag: - - W/"eb-poL8yUftdUFahhL/Peyi6gN8N/E" + - W/"fe-rfCZvNlcUhLHZTl4/HHBhOEJFv4" Vary: - Origin, Accept-Encoding Via: - - 1.1 98bc8180e0431e8f05afc9802305f1d2.cloudfront.net (CloudFront), 1.1 f458ab1245bb4f257969c1da8e708f88.cloudfront.net (CloudFront) + - 1.1 049ca50de603d43d8c9d0f7716efb414.cloudfront.net (CloudFront), 1.1 7ad3d6571deff4c3c83d7e4476fcc6d0.cloudfront.net (CloudFront) X-Amz-Apigw-Id: - - T74Z7EH5IAMEbjQ= + - ZuXtbHzIoAMEvDg= X-Amz-Cf-Id: - - 1fiCjYEQxXwVKKgmJsnE_vuRpmqWc8CvN9okoPxHPb_pZorSKpvc9w== + - klgxSRri84hGF9kapjaqIg6i1Ri2EEp_SWMpaLSpNRpjXCJcebPLHQ== X-Amz-Cf-Pop: - - JFK50-P5 - - JFK50-P2 + - HIO52-P2 + - HIO52-P4 X-Amzn-Requestid: - - f27c857a-5f1d-40c7-b587-cfd807827e3e + - 2f235ae0-c0e7-4b2a-8495-7c30d63f5711 X-Amzn-Trace-Id: - - Root=1-6914a10b-4eb6ea5f3c632efd109ca197;Parent=0b8bae4768799a79;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-69a8d322-396cd87b4272cc4d4f168dee;Parent=79819f16d5be5f3c;Sampled=0;Lineage=1:24be3d11:0 X-Bt-Found-Existing: - "true" X-Bt-Internal-Trace-Id: - - 6914a10b00000000577c5d0dc7cfe5ab + - 69a8d322000000003291e2aa28d7234d X-Cache: - Miss from cloudfront status: 200 OK code: 200 - duration: 422.061917ms + duration: 251.787541ms - id: 9 request: proto: HTTP/1.1 @@ -549,6 +567,8 @@ interactions: body: '{"project_id":"842c2063-b9c7-4c40-9fb9-fa95f810c57f","name":"test-e2e-exp","ensure_new":true}' form: {} headers: + Accept: + - application/json Content-Type: - application/json url: https://api.braintrust.dev/v1/experiment @@ -561,40 +581,40 @@ interactions: trailer: {} content_length: -1 uncompressed: true - body: '{"id":"fb3f7727-c60f-48e4-ad30-2a2befaf85f8","project_id":"842c2063-b9c7-4c40-9fb9-fa95f810c57f","name":"test-e2e-exp-20b5b9c1","description":null,"created":"2025-11-12T15:00:28.439Z","repo_info":{},"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"public":false,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","metadata":null,"tags":null}' + body: '{"id":"ff367ddf-d12b-46c9-a6f2-8f5d82b29a37","project_id":"842c2063-b9c7-4c40-9fb9-fa95f810c57f","name":"test-e2e-exp-340b67ab","description":null,"created":"2026-03-05T00:49:38.648Z","repo_info":{},"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"public":false,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","metadata":null,"tags":null}' headers: Access-Control-Allow-Credentials: - "true" Access-Control-Expose-Headers: - - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms Content-Type: - application/json; charset=utf-8 Date: - - Wed, 12 Nov 2025 15:00:28 GMT + - Thu, 05 Mar 2026 00:49:38 GMT Etag: - - W/"17f-L7Es4ehBG+vrDKjUmJB1DsxDWq4" + - W/"17f-wrYluRYq+KO7HXrtgUiusBTRuZA" Vary: - Origin, Accept-Encoding Via: - - 1.1 68f2eed06d7ecb02b863cacb0da2fc28.cloudfront.net (CloudFront), 1.1 f458ab1245bb4f257969c1da8e708f88.cloudfront.net (CloudFront) + - 1.1 8bf233dd8a97bd754666b427b6d19d34.cloudfront.net (CloudFront), 1.1 7ad3d6571deff4c3c83d7e4476fcc6d0.cloudfront.net (CloudFront) X-Amz-Apigw-Id: - - T74Z_Ez-oAMET0w= + - ZuXtdEchIAMEbdQ= X-Amz-Cf-Id: - - UGhQcSE7IuKz2_tj0tf_rcYpZ0QDw3lvdiQ1N0RPkDxdWNaztSC20w== + - D1JdbYpTvzAQF5X16NKGW3LTkTXuCvWXYsvgUFOjIDMckiLNUxm1YA== X-Amz-Cf-Pop: - - JFK50-P5 - - JFK50-P2 + - HIO52-P2 + - HIO52-P4 X-Amzn-Requestid: - - 16de2b29-27de-476c-a409-4e06087ba8a7 + - 2a9e8d0c-a1b9-402c-9aa9-86f51bed9984 X-Amzn-Trace-Id: - - Root=1-6914a10c-5ed59e9b120cb3ac32c7627d;Parent=762376db910c8dc6;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-69a8d322-041b88d171c632c83d37680f;Parent=5f3803e9f29308fb;Sampled=0;Lineage=1:24be3d11:0 X-Bt-Internal-Trace-Id: - - 6914a10c000000006f99ae55efce4c51 + - 69a8d3220000000058ae9162354e6eb9 X-Cache: - Miss from cloudfront status: 200 OK code: 200 - duration: 191.883375ms + duration: 213.041709ms - id: 10 request: proto: HTTP/1.1 @@ -609,9 +629,11 @@ interactions: body: '{"input":{"question":"What is the capital of France?"}}' form: {} headers: + Accept: + - application/json Content-Type: - application/json - url: https://api.braintrust.dev/v1/function/29072339-6591-46f1-a865-3fcce6c44f40/invoke + url: https://api.braintrust.dev/v1/function/aeb4d39b-f38b-4a20-98d3-6e5adfe64af6/invoke method: POST response: proto: HTTP/2.0 @@ -624,41 +646,41 @@ interactions: body: '{"answer":"Paris","confidence":0.95}' headers: Age: - - "0" + - "801" Cache-Control: - max-age=604800 Cf-Cache-Status: - DYNAMIC Cf-Ray: - - 99d6e6329a156905-IAD + - 9d750b66cc02916d-IAD Content-Type: - application/json Date: - - Wed, 12 Nov 2025 15:00:32 GMT + - Thu, 05 Mar 2026 00:49:39 GMT Openai-Organization: - braintrust-data Openai-Processing-Ms: - - "2381" + - "10236" Openai-Project: - proj_wMRY6YpEiASXMxPIIcI9nQRi Openai-Version: - "2020-10-01" Set-Cookie: - - _cfuvid=NE7TD4ro6b4V.0U1ROSVW.E3d2j30m6rR3dBRFbIKko-1762959631814-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + - __cf_bm=i5UVzxSetLUyp4DcqK6LjSXC_RQJLn2SO5snbn5oV3c-1772670966.8465185-1.0.1.1-Z._8XEIIfPy94NKxdDbozAp6NSONRaR_lK.7WfdrdosGJRXJEI6tESeTpkUBUEh2HjCkKYoB8RZJJjv4luMByyxubZ7GKEJ4jyvxnFOj8xVh7dhFi7.C8eujQCsobL7d; HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Thu, 05 Mar 2026 01:06:17 GMT Strict-Transport-Security: - max-age=31536000; includeSubDomains; preload Via: - - 1.1 f458ab1245bb4f257969c1da8e708f88.cloudfront.net (CloudFront) + - 1.1 7ad3d6571deff4c3c83d7e4476fcc6d0.cloudfront.net (CloudFront) X-Amz-Cf-Id: - - E67Jp_C0n039UteDcOg19wp3zDtbgeMBYC3e-6FyVvyv1ziwymE2fg== + - bnlp3bUPgrMngQB4dSBn7NfuYvL741IoyTks0qzrWgk_lVD5uWVITQ== X-Amz-Cf-Pop: - - JFK50-P2 + - HIO52-P4 X-Amzn-Requestid: - - 71841cc7-fce1-4e39-b685-8634737e6ecb + - 9b4138e7-b0b8-420e-bdf8-e4c2b8c6bdcd X-Amzn-Trace-Id: - - Root=1-6914a10c-06a5a96a7bb5516b6a74c8fe;Parent=767671b96874afb8;Sampled=0;Lineage=1:8be8f50d:0 + - Root=1-69a8d322-50a0b91752d55b4b5623109b;Parent=183deb43e1a08726;Sampled=0;Lineage=1:8be8f50d:0 X-Bt-Cached: - - MISS + - HIT X-Bt-Function-Creds-Cached: - HIT X-Bt-Function-Meta-Cached: @@ -669,8 +691,6 @@ interactions: - Miss from cloudfront X-Content-Type-Options: - nosniff - X-Envoy-Upstream-Service-Time: - - "2448" X-Openai-Proxy-Wasm: - v0.1 X-Ratelimit-Limit-Requests: @@ -686,27 +706,146 @@ interactions: X-Ratelimit-Reset-Tokens: - 0s X-Request-Id: - - req_e27adf5c9f95417f9db6e20812f8fe97 + - req_31e7ebac94ef49d084548a5880b2cfcc status: 200 OK code: 200 - duration: 4.09908225s + duration: 864.584166ms - id: 11 request: proto: HTTP/1.1 proto_major: 1 proto_minor: 1 - content_length: 156 + content_length: 388 transfer_encoding: [] trailer: {} host: api.braintrust.dev remote_addr: "" request_uri: "" - body: '{"input":{"expected":{"answer":"Paris","confidence":1},"input":{"question":"What is the capital of France?"},"output":{"answer":"Paris","confidence":0.95}}}' + body: '{"limit":1000,"filter":{"children":[{"left":{"name":["root_span_id"],"op":"ident"},"op":"eq","right":{"op":"literal","value":"bb8d12c3758f8f0a1a46ce6864a322bb"}},{"children":[{"expr":{"name":["span_attributes","purpose"],"op":"ident"},"op":"isnull"},{"left":{"name":["span_attributes","purpose"],"op":"ident"},"op":"ne","right":{"op":"literal","value":"scorer"}}],"op":"or"}],"op":"and"}}' form: {} headers: + Accept: + - application/json Content-Type: - application/json - url: https://api.braintrust.dev/v1/function/063f91ca-beb0-42dd-8f39-d34bebea6d98/invoke + url: https://api.braintrust.dev/v1/experiment/ff367ddf-d12b-46c9-a6f2-8f5d82b29a37/fetch + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: '{"events":[]}' + headers: + Access-Control-Allow-Credentials: + - "true" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms + Content-Type: + - application/json + Date: + - Thu, 05 Mar 2026 00:49:39 GMT + Vary: + - Origin + Via: + - 1.1 fc36d22b58a363b02ecdd852a2e51610.cloudfront.net (CloudFront), 1.1 7ad3d6571deff4c3c83d7e4476fcc6d0.cloudfront.net (CloudFront) + X-Amz-Apigw-Id: + - ZuXtoGt4IAMEK3Q= + X-Amz-Cf-Id: + - IgsfCIPp3YYyw1jj-GSTd2hloqzqx8Wh9W2-Px9mzX1d3OiBmWF1IQ== + X-Amz-Cf-Pop: + - HIO52-P2 + - HIO52-P4 + X-Amzn-Requestid: + - e4ed36bf-b431-4eef-b91c-29c3e48c2193 + X-Amzn-Trace-Id: + - Root=1-69a8d323-2020b1d5475f4915696bc98e;Parent=6dd93d8eb085d7ac;Sampled=0;Lineage=1:24be3d11:0 + X-Bt-Api-Duration-Ms: + - "206" + X-Bt-Brainstore-Duration-Ms: + - "76" + X-Bt-Internal-Trace-Id: + - 69a8d3230000000069140f55d034267f + X-Cache: + - Miss from cloudfront + status: 200 OK + code: 200 + duration: 409.460875ms + - id: 12 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 234 + transfer_encoding: [] + trailer: {} + host: api.braintrust.dev + remote_addr: "" + request_uri: "" + body: '{"global_function":"project_default","function_type":"preprocessor","mode":"json","input":{"trace_ref":{"object_id":"ff367ddf-d12b-46c9-a6f2-8f5d82b29a37","object_type":"experiment","root_span_id":"bb8d12c3758f8f0a1a46ce6864a322bb"}}}' + form: {} + headers: + Accept: + - application/json + Content-Type: + - application/json + url: https://api.braintrust.dev/function/invoke + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: false + body: "null" + headers: + Content-Type: + - application/json + Date: + - Thu, 05 Mar 2026 00:49:40 GMT + Via: + - 1.1 7ad3d6571deff4c3c83d7e4476fcc6d0.cloudfront.net (CloudFront) + X-Amz-Cf-Id: + - rZmnLLqq8SPahtJKCvc7m2JWs505ZH_lG0RB8ka8CqTqyc0psMCa4w== + X-Amz-Cf-Pop: + - HIO52-P4 + X-Amzn-Requestid: + - ef2b3ee3-c9fa-412d-8004-d1ef25cdef72 + X-Amzn-Trace-Id: + - Root=1-69a8d324-79f8b3b90101b34263d073a3;Parent=6738d2e5d888a5c5;Sampled=0;Lineage=1:8be8f50d:0 + X-Bt-Function-Creds-Cached: + - HIT + X-Bt-Function-Meta-Cached: + - HIT + X-Cache: + - Miss from cloudfront + status: 200 OK + code: 200 + duration: 409.483083ms + - id: 13 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 193 + transfer_encoding: [] + trailer: {} + host: api.braintrust.dev + remote_addr: "" + request_uri: "" + body: '{"input":{"expected":{"answer":"Paris","confidence":1},"input":{"question":"What is the capital of France?"},"output":{"answer":"Paris","confidence":0.95},"trace":{"spans":null,"thread":null}}}' + form: {} + headers: + Accept: + - application/json + Content-Type: + - application/json + url: https://api.braintrust.dev/v1/function/c81984ec-a11b-4de4-86bd-6256985f82ab/invoke method: POST response: proto: HTTP/2.0 @@ -719,41 +858,41 @@ interactions: body: '{"name":"E2E Scorer","score":0,"metadata":{"choice":"incorrect"}}' headers: Age: - - "0" + - "838" Cache-Control: - max-age=604800 Cf-Cache-Status: - DYNAMIC Cf-Ray: - - 99d6e64aea71ca3a-IAD + - 9d750a9bff6f9c43-IAD Content-Type: - application/json Date: - - Wed, 12 Nov 2025 15:00:33 GMT + - Thu, 05 Mar 2026 00:49:41 GMT Openai-Organization: - braintrust-data Openai-Processing-Ms: - - "454" + - "8900" Openai-Project: - proj_wMRY6YpEiASXMxPIIcI9nQRi Openai-Version: - "2020-10-01" Set-Cookie: - - _cfuvid=zyE3rObDpfQwUEIP.8RY1X_2uU.vlZAnGI6rvvJAzMU-1762959633692-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + - __cf_bm=tx7xmc6tvsqv23DbDHziKzCYiKvZp8RylWd.q7yIwmw-1772670934.3944066-1.0.1.1-zXNNcHGbsyy5UMRIHorhUwgZ6.v52SoXtP5fqMgKVrNtWXSTyuLoQFGFJCIGRFO6HEmBbmsO8yj4bGTr50mkmG8i0yy88QJgjnOj8ZQE3o1HRBHGFTTUzNYOnpFl_USJ; HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Thu, 05 Mar 2026 01:05:43 GMT Strict-Transport-Security: - max-age=31536000; includeSubDomains; preload Via: - - 1.1 f458ab1245bb4f257969c1da8e708f88.cloudfront.net (CloudFront) + - 1.1 7ad3d6571deff4c3c83d7e4476fcc6d0.cloudfront.net (CloudFront) X-Amz-Cf-Id: - - hDZ0MULEIZgOhWsrhJXQvId1nQvhVDfjNNQtsvsv6FzVrLY_bxyVDQ== + - 7kg0XX5XT4LG-YTmZDBhpQRn2WpvGcuHvuD4XmyXULv4EhKeej-F5w== X-Amz-Cf-Pop: - - JFK50-P2 + - HIO52-P4 X-Amzn-Requestid: - - 44068fc3-d8a6-4c5a-b017-fdcdeffcbf9e + - 80ef3e2e-4906-4940-8880-4403a524807f X-Amzn-Trace-Id: - - Root=1-6914a110-362efa2713f9ddaa5b185c18;Parent=0f8f33e95e5897f9;Sampled=0;Lineage=1:8be8f50d:0 + - Root=1-69a8d324-7a207d4723e63e36047c6c9b;Parent=0dafe645b757b75d;Sampled=0;Lineage=1:8be8f50d:0 X-Bt-Cached: - - MISS + - HIT X-Bt-Function-Creds-Cached: - HIT X-Bt-Function-Meta-Cached: @@ -764,8 +903,6 @@ interactions: - Miss from cloudfront X-Content-Type-Options: - nosniff - X-Envoy-Upstream-Service-Time: - - "492" X-Openai-Proxy-Wasm: - v0.1 X-Ratelimit-Limit-Requests: @@ -781,11 +918,11 @@ interactions: X-Ratelimit-Reset-Tokens: - 0s X-Request-Id: - - req_ad28be45002847c28e945e4ee86af84e + - req_a365cda4d32449309ae473b2f6ff3280 status: 200 OK code: 200 - duration: 1.349148834s - - id: 12 + duration: 1.368409541s + - id: 14 request: proto: HTTP/1.1 proto_major: 1 @@ -799,9 +936,11 @@ interactions: body: '{"input":{"question":"What is 2+2?"}}' form: {} headers: + Accept: + - application/json Content-Type: - application/json - url: https://api.braintrust.dev/v1/function/29072339-6591-46f1-a865-3fcce6c44f40/invoke + url: https://api.braintrust.dev/v1/function/aeb4d39b-f38b-4a20-98d3-6e5adfe64af6/invoke method: POST response: proto: HTTP/2.0 @@ -814,41 +953,41 @@ interactions: body: '{"answer":"4","confidence":0.9}' headers: Age: - - "0" + - "849" Cache-Control: - max-age=604800 Cf-Cache-Status: - DYNAMIC Cf-Ray: - - 99d6e650aa80ca3a-IAD + - 9d750a43398191fa-IAD Content-Type: - application/json Date: - - Wed, 12 Nov 2025 15:00:35 GMT + - Thu, 05 Mar 2026 00:49:41 GMT Openai-Organization: - braintrust-data Openai-Processing-Ms: - - "344" + - "12004" Openai-Project: - proj_wMRY6YpEiASXMxPIIcI9nQRi Openai-Version: - "2020-10-01" Set-Cookie: - - _cfuvid=JxkU8ffxPlbTo.Y1dlirjKsZNWdrgMMaXLvJdQDT8dA-1762959634507-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + - __cf_bm=V248iJ3jN39AeublOy_9hFDHDRHQ80jHT3v2t7UvkjE-1772670920.1976304-1.0.1.1-YQiVpaP0YcdOHfuTgV4BXam6DkhcVozpymhppntp.8mP_il_AeCQ3SM5_pNlh0D0uyGa6HZx62LZ3.XiNPve9AxJHQ_Gvl1tYIQhoaxz4lSIOUwsTe14ZDU5OXZayoc5; HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Thu, 05 Mar 2026 01:05:32 GMT Strict-Transport-Security: - max-age=31536000; includeSubDomains; preload Via: - - 1.1 f458ab1245bb4f257969c1da8e708f88.cloudfront.net (CloudFront) + - 1.1 7ad3d6571deff4c3c83d7e4476fcc6d0.cloudfront.net (CloudFront) X-Amz-Cf-Id: - - iD530VSXpcaD5KeIWZi43EMVLosCgVwFvA7fR0l4joV5Z_og0pVqoQ== + - VviYFJY0KoRTETtTee3hZyAlHDKGHrVnNtRYQO6TspE_oNOLdua4aQ== X-Amz-Cf-Pop: - - JFK50-P2 + - HIO52-P4 X-Amzn-Requestid: - - ff69756f-81db-43aa-a27a-d59ba9af4ed2 + - be620b07-5736-4b73-a653-5210236da28d X-Amzn-Trace-Id: - - Root=1-6914a111-4884681227fd631c76e27993;Parent=4ecb9c56aa491a66;Sampled=0;Lineage=1:8be8f50d:0 + - Root=1-69a8d325-01226901060205a92278f04b;Parent=57c0d1f3928e192d;Sampled=0;Lineage=1:8be8f50d:0 X-Bt-Cached: - - MISS + - HIT X-Bt-Function-Creds-Cached: - HIT X-Bt-Function-Meta-Cached: @@ -859,8 +998,6 @@ interactions: - Miss from cloudfront X-Content-Type-Options: - nosniff - X-Envoy-Upstream-Service-Time: - - "410" X-Openai-Proxy-Wasm: - v0.1 X-Ratelimit-Limit-Requests: @@ -876,27 +1013,146 @@ interactions: X-Ratelimit-Reset-Tokens: - 0s X-Request-Id: - - req_10a6daacecde47339f732766d7c53f51 + - req_d39fc3a5188f4da48078153da7bc8ae2 status: 200 OK code: 200 - duration: 1.036700792s - - id: 13 + duration: 268.10975ms + - id: 15 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 388 + transfer_encoding: [] + trailer: {} + host: api.braintrust.dev + remote_addr: "" + request_uri: "" + body: '{"limit":1000,"filter":{"children":[{"left":{"name":["root_span_id"],"op":"ident"},"op":"eq","right":{"op":"literal","value":"b1395ed95cb69fd003ffbf2a4ca40d8f"}},{"children":[{"expr":{"name":["span_attributes","purpose"],"op":"ident"},"op":"isnull"},{"left":{"name":["span_attributes","purpose"],"op":"ident"},"op":"ne","right":{"op":"literal","value":"scorer"}}],"op":"or"}],"op":"and"}}' + form: {} + headers: + Accept: + - application/json + Content-Type: + - application/json + url: https://api.braintrust.dev/v1/experiment/ff367ddf-d12b-46c9-a6f2-8f5d82b29a37/fetch + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: '{"events":[]}' + headers: + Access-Control-Allow-Credentials: + - "true" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms + Content-Type: + - application/json + Date: + - Thu, 05 Mar 2026 00:49:42 GMT + Vary: + - Origin + Via: + - 1.1 d220e3f3d93439a8c69225156c6ae800.cloudfront.net (CloudFront), 1.1 7ad3d6571deff4c3c83d7e4476fcc6d0.cloudfront.net (CloudFront) + X-Amz-Apigw-Id: + - ZuXuAHleoAMEFxw= + X-Amz-Cf-Id: + - 1Y-VICml3l2C3WxLlFmgiYI6m-AFj8l0HDBIyv4MXblpAE6UkQRGgw== + X-Amz-Cf-Pop: + - HIO52-P2 + - HIO52-P4 + X-Amzn-Requestid: + - 2cd3a378-79ee-404a-a23b-d6542b9db8e5 + X-Amzn-Trace-Id: + - Root=1-69a8d326-52cdc9832bad47f03d504789;Parent=57b4c3969bc6f512;Sampled=0;Lineage=1:24be3d11:0 + X-Bt-Api-Duration-Ms: + - "66" + X-Bt-Brainstore-Duration-Ms: + - "55" + X-Bt-Internal-Trace-Id: + - 69a8d326000000000dcaccad3c5e5d38 + X-Cache: + - Miss from cloudfront + status: 200 OK + code: 200 + duration: 195.153708ms + - id: 16 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 234 + transfer_encoding: [] + trailer: {} + host: api.braintrust.dev + remote_addr: "" + request_uri: "" + body: '{"global_function":"project_default","function_type":"preprocessor","mode":"json","input":{"trace_ref":{"object_id":"ff367ddf-d12b-46c9-a6f2-8f5d82b29a37","object_type":"experiment","root_span_id":"b1395ed95cb69fd003ffbf2a4ca40d8f"}}}' + form: {} + headers: + Accept: + - application/json + Content-Type: + - application/json + url: https://api.braintrust.dev/function/invoke + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: false + body: "null" + headers: + Content-Type: + - application/json + Date: + - Thu, 05 Mar 2026 00:49:42 GMT + Via: + - 1.1 7ad3d6571deff4c3c83d7e4476fcc6d0.cloudfront.net (CloudFront) + X-Amz-Cf-Id: + - zV2YJypge8uJzYMP5hfSqqhZAXEdZmJ0t-WL1J61qIvhMeaj-dZC3Q== + X-Amz-Cf-Pop: + - HIO52-P4 + X-Amzn-Requestid: + - d02aa017-c25e-43cd-9dd6-590484c7fb26 + X-Amzn-Trace-Id: + - Root=1-69a8d326-192c16b601fcf37f4a221e38;Parent=55c71e5158321430;Sampled=0;Lineage=1:8be8f50d:0 + X-Bt-Function-Creds-Cached: + - HIT + X-Bt-Function-Meta-Cached: + - HIT + X-Cache: + - Miss from cloudfront + status: 200 OK + code: 200 + duration: 336.941583ms + - id: 17 request: proto: HTTP/1.1 proto_major: 1 proto_minor: 1 - content_length: 129 + content_length: 166 transfer_encoding: [] trailer: {} host: api.braintrust.dev remote_addr: "" request_uri: "" - body: '{"input":{"expected":{"answer":"4","confidence":1},"input":{"question":"What is 2+2?"},"output":{"answer":"4","confidence":0.9}}}' + body: '{"input":{"expected":{"answer":"4","confidence":1},"input":{"question":"What is 2+2?"},"output":{"answer":"4","confidence":0.9},"trace":{"spans":null,"thread":null}}}' form: {} headers: + Accept: + - application/json Content-Type: - application/json - url: https://api.braintrust.dev/v1/function/063f91ca-beb0-42dd-8f39-d34bebea6d98/invoke + url: https://api.braintrust.dev/v1/function/c81984ec-a11b-4de4-86bd-6256985f82ab/invoke method: POST response: proto: HTTP/2.0 @@ -909,39 +1165,39 @@ interactions: body: '{"name":"E2E Scorer","score":0,"metadata":{"choice":"incorrect"}}' headers: Age: - - "1" + - "839" Cache-Control: - max-age=604800 Cf-Cache-Status: - DYNAMIC Cf-Ray: - - 99d6e64aea71ca3a-IAD + - 9d750a9bff6f9c43-IAD Content-Type: - application/json Date: - - Wed, 12 Nov 2025 15:00:35 GMT + - Thu, 05 Mar 2026 00:49:42 GMT Openai-Organization: - braintrust-data Openai-Processing-Ms: - - "454" + - "8900" Openai-Project: - proj_wMRY6YpEiASXMxPIIcI9nQRi Openai-Version: - "2020-10-01" Set-Cookie: - - _cfuvid=zyE3rObDpfQwUEIP.8RY1X_2uU.vlZAnGI6rvvJAzMU-1762959633692-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + - __cf_bm=tx7xmc6tvsqv23DbDHziKzCYiKvZp8RylWd.q7yIwmw-1772670934.3944066-1.0.1.1-zXNNcHGbsyy5UMRIHorhUwgZ6.v52SoXtP5fqMgKVrNtWXSTyuLoQFGFJCIGRFO6HEmBbmsO8yj4bGTr50mkmG8i0yy88QJgjnOj8ZQE3o1HRBHGFTTUzNYOnpFl_USJ; HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Thu, 05 Mar 2026 01:05:43 GMT Strict-Transport-Security: - max-age=31536000; includeSubDomains; preload Via: - - 1.1 f458ab1245bb4f257969c1da8e708f88.cloudfront.net (CloudFront) + - 1.1 7ad3d6571deff4c3c83d7e4476fcc6d0.cloudfront.net (CloudFront) X-Amz-Cf-Id: - - yPg3IbXp-0FmfY1gkHSnxO3A8E-IFTUpNe2gCE_L92yNuOAHeU-lpg== + - rINawjlY_Qcgcq9unlilEIqT7Iza1JU0i8zcXvPDg02Ixg0JYMn80Q== X-Amz-Cf-Pop: - - JFK50-P2 + - HIO52-P4 X-Amzn-Requestid: - - cc91aca2-62bd-4001-a951-7682b978f1c3 + - 1a79acc5-be7b-49c5-acfb-48eeb0b0f63c X-Amzn-Trace-Id: - - Root=1-6914a113-364b3c292d0830400435f7a7;Parent=3694160fa7c7879b;Sampled=0;Lineage=1:8be8f50d:0 + - Root=1-69a8d326-295306367dfb2b0815fc53e4;Parent=61ede213a067d874;Sampled=0;Lineage=1:8be8f50d:0 X-Bt-Cached: - HIT X-Bt-Function-Creds-Cached: @@ -954,8 +1210,6 @@ interactions: - Miss from cloudfront X-Content-Type-Options: - nosniff - X-Envoy-Upstream-Service-Time: - - "492" X-Openai-Proxy-Wasm: - v0.1 X-Ratelimit-Limit-Requests: @@ -971,11 +1225,11 @@ interactions: X-Ratelimit-Reset-Tokens: - 0s X-Request-Id: - - req_ad28be45002847c28e945e4ee86af84e + - req_a365cda4d32449309ae473b2f6ff3280 status: 200 OK code: 200 - duration: 78.633375ms - - id: 14 + duration: 131.369417ms + - id: 18 request: proto: HTTP/1.1 proto_major: 1 @@ -988,8 +1242,10 @@ interactions: request_uri: "" body: "" form: {} - headers: {} - url: https://api.braintrust.dev/v1/function/063f91ca-beb0-42dd-8f39-d34bebea6d98 + headers: + Accept: + - application/json + url: https://api.braintrust.dev/v1/function/c81984ec-a11b-4de4-86bd-6256985f82ab method: DELETE response: proto: HTTP/2.0 @@ -999,41 +1255,41 @@ interactions: trailer: {} content_length: -1 uncompressed: true - body: '{"log_id":"p","project_id":"842c2063-b9c7-4c40-9fb9-fa95f810c57f","slug":"TestFunctionsAPI_EndToEnd_MixedTypes-scorer","id":"063f91ca-beb0-42dd-8f39-d34bebea6d98","created":"2025-11-12T15:00:35.359Z","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","_object_delete":true,"_xact_id":"1000196129575620471"}' + body: '{"log_id":"p","project_id":"842c2063-b9c7-4c40-9fb9-fa95f810c57f","slug":"TestFunctionsAPI_EndToEnd_MixedTypes-scorer","id":"c81984ec-a11b-4de4-86bd-6256985f82ab","created":"2026-03-05T00:49:42.865Z","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","_object_delete":true,"_xact_id":"1000196766070890268"}' headers: Access-Control-Allow-Credentials: - "true" Access-Control-Expose-Headers: - - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms Content-Type: - application/json; charset=utf-8 Date: - - Wed, 12 Nov 2025 15:00:35 GMT + - Thu, 05 Mar 2026 00:49:43 GMT Etag: - - W/"12f-GRQ22KEM0pPNmGtvOoGOw13UPS0" + - W/"12f-eyOPkORMDSf9v1c8fBMBAQ4+S4Y" Vary: - Origin, Accept-Encoding Via: - - 1.1 241db89625f6ef70a00b0e19e0cfc332.cloudfront.net (CloudFront), 1.1 f458ab1245bb4f257969c1da8e708f88.cloudfront.net (CloudFront) + - 1.1 4e7012bff211fc1604763d0935533d32.cloudfront.net (CloudFront), 1.1 7ad3d6571deff4c3c83d7e4476fcc6d0.cloudfront.net (CloudFront) X-Amz-Apigw-Id: - - T74bDFEvIAMEOmw= + - ZuXuHEm_IAMEf1A= X-Amz-Cf-Id: - - jNxtKHFW2FjfMdYjHCocTRn60nBxn8tcePbnrrNr3wZQhaIwJHyeSg== + - -IAc9zrisQ6N2qE9islsmtyxSXwsS2Pus4epZzZXH0WNlHnRTYnk3g== X-Amz-Cf-Pop: - - JFK50-P5 - - JFK50-P2 + - HIO52-P2 + - HIO52-P4 X-Amzn-Requestid: - - be21dd98-89a8-43b0-aa04-5d559e20c2af + - 9132e99a-0d5e-4ea0-9f29-fcde8e2da9c8 X-Amzn-Trace-Id: - - Root=1-6914a113-3325bc4f5643cc641ab3f0f7;Parent=0d732b51a32cb479;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-69a8d326-2ae634ed4a4cd77b6b2bbc4c;Parent=7bf21c0633367639;Sampled=0;Lineage=1:24be3d11:0 X-Bt-Internal-Trace-Id: - - 6914a113000000004489d85ebd9e608b + - 69a8d326000000004881e5345c2f29e1 X-Cache: - Miss from cloudfront status: 200 OK code: 200 - duration: 444.713459ms - - id: 15 + duration: 461.6655ms + - id: 19 request: proto: HTTP/1.1 proto_major: 1 @@ -1046,8 +1302,10 @@ interactions: request_uri: "" body: "" form: {} - headers: {} - url: https://api.braintrust.dev/v1/function/29072339-6591-46f1-a865-3fcce6c44f40 + headers: + Accept: + - application/json + url: https://api.braintrust.dev/v1/function/aeb4d39b-f38b-4a20-98d3-6e5adfe64af6 method: DELETE response: proto: HTTP/2.0 @@ -1057,37 +1315,37 @@ interactions: trailer: {} content_length: -1 uncompressed: true - body: '{"log_id":"p","project_id":"842c2063-b9c7-4c40-9fb9-fa95f810c57f","slug":"TestFunctionsAPI_EndToEnd_MixedTypes-task","id":"29072339-6591-46f1-a865-3fcce6c44f40","created":"2025-11-12T15:00:35.677Z","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","_object_delete":true,"_xact_id":"1000196129575620769"}' + body: '{"log_id":"p","project_id":"842c2063-b9c7-4c40-9fb9-fa95f810c57f","slug":"TestFunctionsAPI_EndToEnd_MixedTypes-task","id":"aeb4d39b-f38b-4a20-98d3-6e5adfe64af6","created":"2026-03-05T00:49:43.349Z","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","_object_delete":true,"_xact_id":"1000196766070956283"}' headers: Access-Control-Allow-Credentials: - "true" Access-Control-Expose-Headers: - - x-bt-cursor,x-bt-found-existing,x-bt-query-plan + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms Content-Type: - application/json; charset=utf-8 Date: - - Wed, 12 Nov 2025 15:00:35 GMT + - Thu, 05 Mar 2026 00:49:43 GMT Etag: - - W/"12d-2FirTT1wW4vV/jHPRbLESlnaqbk" + - W/"12d-sgKO/VJ7Qubf8mQ3P5Oc8xq9IbM" Vary: - Origin, Accept-Encoding Via: - - 1.1 95708ab75ec6181aa75086df530332d6.cloudfront.net (CloudFront), 1.1 f458ab1245bb4f257969c1da8e708f88.cloudfront.net (CloudFront) + - 1.1 49798ef4b8dd64fece36e067d09f69ec.cloudfront.net (CloudFront), 1.1 7ad3d6571deff4c3c83d7e4476fcc6d0.cloudfront.net (CloudFront) X-Amz-Apigw-Id: - - T74bHEb8oAMEAXQ= + - ZuXuMGLdoAMEXUA= X-Amz-Cf-Id: - - Q0lt3Z8AyXxzsTadS3pWziJIyiM-64121IeiC67dXQILQbwbC3Es6g== + - cZoAR92WxJGaRhMBwtEg03uYFMHqbl-GrmRE8LxwkOawMoWp1UCSww== X-Amz-Cf-Pop: - - JFK50-P5 - - JFK50-P2 + - HIO52-P2 + - HIO52-P4 X-Amzn-Requestid: - - d55ab3a0-ceca-40c4-b4b5-2acc03f8f424 + - 22085cbb-9999-4414-87c2-0fde47ec694a X-Amzn-Trace-Id: - - Root=1-6914a113-4fa9083f7568fc8b6d9d309e;Parent=70453b5d4c7330f1;Sampled=0;Lineage=1:24be3d11:0 + - Root=1-69a8d327-6d6fb38815df43ce30dc377e;Parent=1bee60d8d66b9597;Sampled=0;Lineage=1:24be3d11:0 X-Bt-Internal-Trace-Id: - - 6914a1130000000050092a3a6c82a87b + - 69a8d32700000000210fcaf1f12ae174 X-Cache: - Miss from cloudfront status: 200 OK code: 200 - duration: 374.398208ms + duration: 442.361291ms diff --git a/eval/testdata/cassettes/TestSpans_Integration.yaml b/eval/testdata/cassettes/TestSpans_Integration.yaml new file mode 100644 index 00000000..91128355 --- /dev/null +++ b/eval/testdata/cassettes/TestSpans_Integration.yaml @@ -0,0 +1,434 @@ +--- +version: 2 +interactions: + - id: 0 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 23 + transfer_encoding: [] + trailer: {} + host: api.braintrust.dev + remote_addr: "" + request_uri: "" + body: '{"name":"go-sdk-tests"}' + form: {} + headers: + Accept: + - application/json + Content-Type: + - application/json + url: https://api.braintrust.dev/v1/project + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: '{"id":"842c2063-b9c7-4c40-9fb9-fa95f810c57f","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"go-sdk-tests","description":null,"created":"2025-11-04T13:38:56.532Z","deleted_at":null,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","settings":null}' + headers: + Access-Control-Allow-Credentials: + - "true" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms + Content-Type: + - application/json; charset=utf-8 + Date: + - Thu, 05 Mar 2026 00:50:09 GMT + Etag: + - W/"fe-rfCZvNlcUhLHZTl4/HHBhOEJFv4" + Vary: + - Origin, Accept-Encoding + Via: + - 1.1 05cf67c96e96cd376921ba5b65795a56.cloudfront.net (CloudFront), 1.1 41d11417b7470cfb79cc5b299692369a.cloudfront.net (CloudFront) + X-Amz-Apigw-Id: + - ZuXyTFmwIAMEuzQ= + X-Amz-Cf-Id: + - NYo2tTxvnT1Xbbi-EFd0Gj4BGbea2FJ4PPfCa1He5bplt2PReKgeuA== + X-Amz-Cf-Pop: + - HIO52-P2 + - HIO52-P4 + X-Amzn-Requestid: + - cd462757-3e29-48a4-a2f4-7bf6cc1f55e9 + X-Amzn-Trace-Id: + - Root=1-69a8d341-29c17b58763b99ce076be82c;Parent=4a4bb0124fc8d829;Sampled=0;Lineage=1:24be3d11:0 + X-Bt-Found-Existing: + - "true" + X-Bt-Internal-Trace-Id: + - 69a8d3410000000031a3a70e3e47fd68 + X-Cache: + - Miss from cloudfront + status: 200 OK + code: 200 + duration: 251.65825ms + - id: 1 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: api.braintrust.dev + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Accept: + - application/json + url: https://api.braintrust.dev/api/apikey/login + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: '{"org_info":[{"id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"matt-test-org","api_url":"https://api.braintrust.dev","git_metadata":null,"is_universal_api":null,"proxy_url":"https://api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' + headers: + Access-Control-Allow-Credentials: + - "true" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms + Content-Type: + - application/json; charset=utf-8 + Date: + - Thu, 05 Mar 2026 00:50:09 GMT + Etag: + - W/"101-EqXzt+vlRFUt5HCzQY7qxp9rZU0" + Vary: + - Origin, Accept-Encoding + Via: + - 1.1 6a52d37737133b0b8a09947e5c586ec4.cloudfront.net (CloudFront), 1.1 41d11417b7470cfb79cc5b299692369a.cloudfront.net (CloudFront) + X-Amz-Apigw-Id: + - ZuXyVET4oAMEYtQ= + X-Amz-Cf-Id: + - r1DO6WNeJtDZyPPCupp27LaFhDufAErAwaARkZnfShZmF7iRE30BAA== + X-Amz-Cf-Pop: + - HIO52-P2 + - HIO52-P4 + X-Amzn-Requestid: + - a406de4e-b6b2-43f0-ba75-d02b6ebd8b96 + X-Amzn-Trace-Id: + - Root=1-69a8d341-6b51e5791b9f54f473e53a79;Parent=45ea512dbb0d7c7c;Sampled=0;Lineage=1:24be3d11:0 + X-Bt-Internal-Trace-Id: + - 69a8d3410000000004bd74d6d772a7c2 + X-Cache: + - Miss from cloudfront + status: 200 OK + code: 200 + duration: 396.15425ms + - id: 2 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 23 + transfer_encoding: [] + trailer: {} + host: api.braintrust.dev + remote_addr: "" + request_uri: "" + body: '{"name":"go-sdk-tests"}' + form: {} + headers: + Accept: + - application/json + Content-Type: + - application/json + url: https://api.braintrust.dev/v1/project + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: '{"id":"842c2063-b9c7-4c40-9fb9-fa95f810c57f","org_id":"5ba6d482-b475-4c66-8cd2-5815694764e3","name":"go-sdk-tests","description":null,"created":"2025-11-04T13:38:56.532Z","deleted_at":null,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","settings":null}' + headers: + Access-Control-Allow-Credentials: + - "true" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms + Content-Type: + - application/json; charset=utf-8 + Date: + - Thu, 05 Mar 2026 00:50:09 GMT + Etag: + - W/"fe-rfCZvNlcUhLHZTl4/HHBhOEJFv4" + Vary: + - Origin, Accept-Encoding + Via: + - 1.1 f36cc119cb86b2f70c315ca53fd1b4ee.cloudfront.net (CloudFront), 1.1 41d11417b7470cfb79cc5b299692369a.cloudfront.net (CloudFront) + X-Amz-Apigw-Id: + - ZuXyVGDroAMEezQ= + X-Amz-Cf-Id: + - 6UIuEM9SOkx0pJ_5cykpVwX4urAT6Bp21zzX1JmpWwUZIQ-kEJlPYA== + X-Amz-Cf-Pop: + - HIO52-P2 + - HIO52-P4 + X-Amzn-Requestid: + - a6671dab-832b-4c09-b63c-b55bb83244d1 + X-Amzn-Trace-Id: + - Root=1-69a8d341-3cff57025b1a423c04d2e163;Parent=5e8a7e6172aec3f9;Sampled=0;Lineage=1:24be3d11:0 + X-Bt-Found-Existing: + - "true" + X-Bt-Internal-Trace-Id: + - 69a8d3410000000017ee2b0340f0378a + X-Cache: + - Miss from cloudfront + status: 200 OK + code: 200 + duration: 234.860959ms + - id: 3 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 91 + transfer_encoding: [] + trailer: {} + host: api.braintrust.dev + remote_addr: "" + request_uri: "" + body: '{"project_id":"842c2063-b9c7-4c40-9fb9-fa95f810c57f","name":"test-spans","ensure_new":true}' + form: {} + headers: + Accept: + - application/json + Content-Type: + - application/json + url: https://api.braintrust.dev/v1/experiment + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: '{"id":"1ca73994-ead0-49be-92d4-8f8916477dcd","project_id":"842c2063-b9c7-4c40-9fb9-fa95f810c57f","name":"test-spans-5d390dc0","description":null,"created":"2026-03-05T00:50:10.051Z","repo_info":{},"commit":null,"base_exp_id":null,"deleted_at":null,"dataset_id":null,"dataset_version":null,"public":false,"user_id":"855483c6-68f0-4df4-a147-df9b4ea32e0c","metadata":null,"tags":null}' + headers: + Access-Control-Allow-Credentials: + - "true" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms + Content-Type: + - application/json; charset=utf-8 + Date: + - Thu, 05 Mar 2026 00:50:10 GMT + Etag: + - W/"17d-cG0lelZAcNlJL1NknFljLpci0GY" + Vary: + - Origin, Accept-Encoding + Via: + - 1.1 f36cc119cb86b2f70c315ca53fd1b4ee.cloudfront.net (CloudFront), 1.1 41d11417b7470cfb79cc5b299692369a.cloudfront.net (CloudFront) + X-Amz-Apigw-Id: + - ZuXyXFguoAMEA6A= + X-Amz-Cf-Id: + - __z4NnnsWyqf6H5FKVIbLt66D2-JdsklpRvUx4lq4xSTfXnDHlrCTg== + X-Amz-Cf-Pop: + - HIO52-P2 + - HIO52-P4 + X-Amzn-Requestid: + - e32ed372-3fa8-4432-96c1-fdeb5d934a54 + X-Amzn-Trace-Id: + - Root=1-69a8d341-0f93b9c13fddcc1b36ac2295;Parent=139d89c07bf7b4a4;Sampled=0;Lineage=1:24be3d11:0 + X-Bt-Internal-Trace-Id: + - 69a8d3410000000053849cd8d26108c9 + X-Cache: + - Miss from cloudfront + status: 200 OK + code: 200 + duration: 214.578166ms + - id: 4 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 388 + transfer_encoding: [] + trailer: {} + host: api.braintrust.dev + remote_addr: "" + request_uri: "" + body: '{"limit":1000,"filter":{"children":[{"left":{"name":["root_span_id"],"op":"ident"},"op":"eq","right":{"op":"literal","value":"1faf54c2b37cad5df7ba1ece3757b934"}},{"children":[{"expr":{"name":["span_attributes","purpose"],"op":"ident"},"op":"isnull"},{"left":{"name":["span_attributes","purpose"],"op":"ident"},"op":"ne","right":{"op":"literal","value":"scorer"}}],"op":"or"}],"op":"and"}}' + form: {} + headers: + Accept: + - application/json + Content-Type: + - application/json + url: https://api.braintrust.dev/v1/experiment/1ca73994-ead0-49be-92d4-8f8916477dcd/fetch + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: '{"events":[]}' + headers: + Access-Control-Allow-Credentials: + - "true" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms + Content-Type: + - application/json + Date: + - Thu, 05 Mar 2026 00:50:10 GMT + Vary: + - Origin + Via: + - 1.1 44e3ef26e727fc044d711ef45aefcd72.cloudfront.net (CloudFront), 1.1 41d11417b7470cfb79cc5b299692369a.cloudfront.net (CloudFront) + X-Amz-Apigw-Id: + - ZuXyZE4GoAMEumw= + X-Amz-Cf-Id: + - eNAmkvqN5X3OGxr5eH3HxYEEcLEWWrBiHdDOqhiiKlUjzDyO6jOb8g== + X-Amz-Cf-Pop: + - HIO52-P2 + - HIO52-P4 + X-Amzn-Requestid: + - 5498c09d-c1a6-4595-aaba-0f76f4ffb01a + X-Amzn-Trace-Id: + - Root=1-69a8d342-0c6fdc4b635f440171233a03;Parent=755a00c73a89c003;Sampled=0;Lineage=1:24be3d11:0 + X-Bt-Api-Duration-Ms: + - "159" + X-Bt-Brainstore-Duration-Ms: + - "54" + X-Bt-Internal-Trace-Id: + - 69a8d3420000000006f38e1bc16804fe + X-Cache: + - Miss from cloudfront + status: 200 OK + code: 200 + duration: 287.293459ms + - id: 5 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 234 + transfer_encoding: [] + trailer: {} + host: api.braintrust.dev + remote_addr: "" + request_uri: "" + body: '{"global_function":"project_default","function_type":"preprocessor","mode":"json","input":{"trace_ref":{"object_id":"1ca73994-ead0-49be-92d4-8f8916477dcd","object_type":"experiment","root_span_id":"1faf54c2b37cad5df7ba1ece3757b934"}}}' + form: {} + headers: + Accept: + - application/json + Content-Type: + - application/json + url: https://api.braintrust.dev/function/invoke + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: false + body: "null" + headers: + Content-Type: + - application/json + Date: + - Thu, 05 Mar 2026 00:50:11 GMT + Via: + - 1.1 41d11417b7470cfb79cc5b299692369a.cloudfront.net (CloudFront) + X-Amz-Cf-Id: + - 1LvfwvIBefmZIy5-99ct3WL1oCdyR4NUFuaWJ-UMH7u4x9dTU9aZzA== + X-Amz-Cf-Pop: + - HIO52-P4 + X-Amzn-Requestid: + - 9749feee-9c37-4c42-8976-77daa0f5e043 + X-Amzn-Trace-Id: + - Root=1-69a8d342-7e4961f7070b84e71c8dbad1;Parent=45afc57b330e1497;Sampled=0;Lineage=1:8be8f50d:0 + X-Bt-Function-Creds-Cached: + - MISS + X-Bt-Function-Meta-Cached: + - MISS + X-Cache: + - Miss from cloudfront + status: 200 OK + code: 200 + duration: 706.176375ms + - id: 6 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 505 + transfer_encoding: [] + trailer: {} + host: api.braintrust.dev + remote_addr: "" + request_uri: "" + body: '{"limit":1000,"filter":{"children":[{"left":{"name":["root_span_id"],"op":"ident"},"op":"eq","right":{"op":"literal","value":"1faf54c2b37cad5df7ba1ece3757b934"}},{"children":[{"expr":{"name":["span_attributes","purpose"],"op":"ident"},"op":"isnull"},{"left":{"name":["span_attributes","purpose"],"op":"ident"},"op":"ne","right":{"op":"literal","value":"scorer"}}],"op":"or"},{"left":{"name":["span_attributes","type"],"op":"ident"},"op":"in","right":{"op":"literal","value":["nonexistent"]}}],"op":"and"}}' + form: {} + headers: + Accept: + - application/json + Content-Type: + - application/json + url: https://api.braintrust.dev/v1/experiment/1ca73994-ead0-49be-92d4-8f8916477dcd/fetch + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: '{"events":[]}' + headers: + Access-Control-Allow-Credentials: + - "true" + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms + Content-Type: + - application/json + Date: + - Thu, 05 Mar 2026 00:50:11 GMT + Vary: + - Origin + Via: + - 1.1 f36cc119cb86b2f70c315ca53fd1b4ee.cloudfront.net (CloudFront), 1.1 41d11417b7470cfb79cc5b299692369a.cloudfront.net (CloudFront) + X-Amz-Apigw-Id: + - ZuXyjFyvoAMEK3g= + X-Amz-Cf-Id: + - OgFeXtXEvKJd0XUFKY6G3UPWY3i8fZk4S9cbdHpWBtpgW-4_lYJh9w== + X-Amz-Cf-Pop: + - HIO52-P2 + - HIO52-P4 + X-Amzn-Requestid: + - f3febb2b-e915-40ab-9085-0606c22d687e + X-Amzn-Trace-Id: + - Root=1-69a8d343-2aa8f6773604a56854850deb;Parent=36e358b31427137e;Sampled=0;Lineage=1:24be3d11:0 + X-Bt-Api-Duration-Ms: + - "66" + X-Bt-Brainstore-Duration-Ms: + - "54" + X-Bt-Internal-Trace-Id: + - 69a8d3430000000039aee098bfdc19aa + X-Cache: + - Miss from cloudfront + status: 200 OK + code: 200 + duration: 196.673416ms diff --git a/examples/internal/README.md b/examples/internal/README.md index dfa4a5bf..91c13a6e 100644 --- a/examples/internal/README.md +++ b/examples/internal/README.md @@ -26,6 +26,7 @@ Comprehensive examples testing all features for each AI provider: - **[langchaingo-anthropic/](langchaingo-anthropic/)** - LangChainGo with Anthropic provider (uses forked langchaingo) - **[functions/](functions/)** - Functions API usage (loading tasks/scorers with FunctionOpts) +- **[trace-scorer/](trace-scorer/)** - Accessing per-case trace spans/thread in scorer logic - **[rewrite/](rewrite/)** - Manual tracing and evaluator API testing - **[email-evals/](email-evals/)** - Realistic eval example with complex scoring - **[eval-updates/](eval-updates/)** - Testing Update option for appending to experiments diff --git a/examples/internal/trace-scorer/main.go b/examples/internal/trace-scorer/main.go new file mode 100644 index 00000000..62304ad9 --- /dev/null +++ b/examples/internal/trace-scorer/main.go @@ -0,0 +1,89 @@ +package main + +import ( + "context" + "log" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/sdk/trace" + + braintrust "github.com/braintrustdata/braintrust-sdk-go" + "github.com/braintrustdata/braintrust-sdk-go/eval" +) + +func main() { + ctx := context.Background() + + tp := trace.NewTracerProvider() + defer tp.Shutdown(ctx) //nolint:errcheck + otel.SetTracerProvider(tp) + + client, err := braintrust.New(tp, + braintrust.WithProject("go-sdk-examples"), + braintrust.WithBlockingLogin(true), + ) + if err != nil { + log.Fatalf("failed to create Braintrust client: %v", err) + } + + evaluator := braintrust.NewEvaluator[string, string](client) + + task := eval.T(func(ctx context.Context, input string) (string, error) { + tracer := otel.Tracer("trace-scorer-example") + _, span := tracer.Start(ctx, "task-work") + defer span.End() + span.SetAttributes( + attribute.String("span_attributes.type", "custom"), + attribute.String("example.input", input), + ) + return "hello " + input, nil + }) + + traceAwareScorer := eval.NewScorer("trace_aware", func(ctx context.Context, tr eval.TaskResult[string, string]) (eval.Scores, error) { + allSpans, err := tr.Spans(ctx) + if err != nil { + return nil, err + } + customSpans, err := tr.Spans(ctx, eval.WithSpanTypes("custom")) + if err != nil { + return nil, err + } + thread, err := tr.Thread(ctx) + if err != nil { + return nil, err + } + + log.Printf("trace info: spans=%d custom_spans=%d thread=%d", len(allSpans), len(customSpans), len(thread)) + + score := 0.0 + if len(allSpans) > 0 { + score = 1.0 + } + + return eval.Scores{{ + Name: "trace_aware", + Score: score, + Metadata: map[string]any{ + "span_count": len(allSpans), + "custom_span_count": len(customSpans), + "thread_count": len(thread), + }, + }}, nil + }) + + _, err = evaluator.Run(ctx, eval.Opts[string, string]{ + Experiment: "internal-trace-scorer-demo", + Dataset: eval.NewDataset([]eval.Case[string, string]{ + {Input: "world", Expected: "hello world"}, + {Input: "team", Expected: "hello team"}, + }), + Task: task, + Scorers: []eval.Scorer[string, string]{traceAwareScorer}, + }) + if err != nil { + log.Fatalf("eval failed: %v", err) + } + + log.Println("trace-scorer example completed") +} diff --git a/internal/https/client.go b/internal/https/client.go index 6d69fad6..13ad0edb 100644 --- a/internal/https/client.go +++ b/internal/https/client.go @@ -157,6 +157,9 @@ func (c *Client) Client() *http.Client { func (c *Client) doRequest(req *http.Request) (*http.Response, error) { // Add auth header req.Header.Set("Authorization", "Bearer "+c.apiKey) + if req.Header.Get("Accept") == "" { + req.Header.Set("Accept", "application/json") + } // Log request start := time.Now()