From 881b07c8782facd7ec03a7890afd4c6395fff430 Mon Sep 17 00:00:00 2001 From: Ankur Goyal Date: Fri, 8 May 2026 15:46:54 -0700 Subject: [PATCH 1/6] opt out by default --- js/src/gitutil.ts | 8 +++++--- js/src/logger.ts | 20 ++++++++++---------- js/util/git_fields.ts | 20 +++++++++++++++++++- js/util/index.ts | 5 ++++- 4 files changed, 38 insertions(+), 15 deletions(-) diff --git a/js/src/gitutil.ts b/js/src/gitutil.ts index ef7a26bcb..b82d177c0 100644 --- a/js/src/gitutil.ts +++ b/js/src/gitutil.ts @@ -4,6 +4,7 @@ import { } from "./generated_types"; import { debugLogger } from "./debug-logger"; import { simpleGit } from "simple-git"; +import { defaultGitMetadataSettings } from "../util/git_fields"; const COMMON_BASE_BRANCHES = ["main", "master", "develop"]; @@ -149,17 +150,18 @@ function truncateToByteLimit(s: string, byteLimit: number = 65536): string { } export async function getRepoInfo(settings?: GitMetadataSettings) { - if (settings && settings.collect === "none") { + const resolvedSettings = settings ?? defaultGitMetadataSettings(); + if (resolvedSettings.collect === "none") { return undefined; } const repo = await repoInfo(); - if (!repo || !settings || settings.collect === "all") { + if (!repo || resolvedSettings.collect === "all") { return repo; } let sanitized: RepoInfo = {}; - settings.fields?.forEach((field) => { + resolvedSettings.fields?.forEach((field) => { sanitized = { ...sanitized, [field]: repo[field] }; }); diff --git a/js/src/logger.ts b/js/src/logger.ts index 81da807fe..dee2c3342 100644 --- a/js/src/logger.ts +++ b/js/src/logger.ts @@ -31,6 +31,7 @@ import { IdField, IS_MERGE_FIELD, LogFeedbackFullArgs, + defaultGitMetadataSettings, mergeDicts, mergeGitMetadataSettings, mergeRowBatch, @@ -3535,7 +3536,7 @@ type InitializedExperiment = * @param options.apiKey The API key to use. If the parameter is not specified, will try to use the `BRAINTRUST_API_KEY` environment variable. If no API key is specified, will prompt the user to login. * @param options.orgName (Optional) The name of a specific organization to connect to. This is useful if you belong to multiple. * @param options.metadata (Optional) A dictionary with additional data about the test example, model outputs, or just about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any JSON-serializable type, but its keys must be strings. - * @param options.gitMetadataSettings (Optional) Settings for collecting git metadata. By default, will collect all git metadata fields allowed in org-level settings. + * @param options.gitMetadataSettings (Optional) Settings for collecting git metadata. By default, will collect git metadata fields allowed in org-level settings, excluding diff content unless the org opts in. * @param setCurrent If true (the default), set the global current-experiment to the newly-created one. * @param options.open If the experiment already exists, open it in read-only mode. Throws an error if the experiment does not already exist. * @param options.projectId The id of the project to create the experiment in. This takes precedence over `project` if specified. @@ -3689,9 +3690,7 @@ export function init( return repoInfo; } let mergedGitMetadataSettings = { - ...(state.gitMetadataSettings || { - collect: "all", - }), + ...(state.gitMetadataSettings || defaultGitMetadataSettings()), }; if (gitMetadataSettings) { mergedGitMetadataSettings = mergeGitMetadataSettings( @@ -4263,8 +4262,8 @@ export function initDataset< legacy, _internal_btql, resolvedVersion instanceof LazyValue || - normalizedEnvironment !== undefined || - normalizedSnapshotName !== undefined + normalizedEnvironment !== undefined || + normalizedSnapshotName !== undefined ? { ...(resolvedVersion instanceof LazyValue ? { @@ -5699,7 +5698,8 @@ function _saveOrgInfo( state.orgName = org.name; state.apiUrl = iso.getEnv("BRAINTRUST_API_URL") ?? org.api_url; state.proxyUrl = iso.getEnv("BRAINTRUST_PROXY_URL") ?? org.proxy_url; - state.gitMetadataSettings = org.git_metadata || undefined; + state.gitMetadataSettings = + org.git_metadata || defaultGitMetadataSettings(); break; } } @@ -6041,9 +6041,9 @@ export type WithTransactionId = R & { export const DEFAULT_FETCH_BATCH_SIZE = 1000; export const MAX_BTQL_ITERATIONS = 10000; -export class ObjectFetcher implements AsyncIterable< - WithTransactionId -> { +export class ObjectFetcher + implements AsyncIterable> +{ private _fetchedData: WithTransactionId[] | undefined = undefined; constructor( diff --git a/js/util/git_fields.ts b/js/util/git_fields.ts index 8eb06d625..0ea592ab8 100644 --- a/js/util/git_fields.ts +++ b/js/util/git_fields.ts @@ -1,4 +1,22 @@ -import { GitMetadataSettingsType as GitMetadataSettings } from "./generated_types"; +import { type GitMetadataSettingsType as GitMetadataSettings } from "./generated_types"; + +export const DEFAULT_GIT_METADATA_FIELDS = [ + "commit", + "branch", + "tag", + "dirty", + "author_name", + "author_email", + "commit_message", + "commit_time", +]; + +export function defaultGitMetadataSettings(): GitMetadataSettings { + return { + collect: "some", + fields: [...DEFAULT_GIT_METADATA_FIELDS], + }; +} export function mergeGitMetadataSettings( s1: GitMetadataSettings, diff --git a/js/util/index.ts b/js/util/index.ts index 0d1d0b1a4..83d2b76c5 100644 --- a/js/util/index.ts +++ b/js/util/index.ts @@ -136,7 +136,10 @@ export { spanTypeAttributeValues, } from "./span_types"; -export { mergeGitMetadataSettings } from "./git_fields"; +export { + defaultGitMetadataSettings, + mergeGitMetadataSettings, +} from "./git_fields"; export { loadPrettyXact, prettifyXact } from "./xact-ids"; From d361db1c567ae56f831d9d657fcc1cc590563623 Mon Sep 17 00:00:00 2001 From: Ankur Goyal Date: Fri, 8 May 2026 16:12:52 -0700 Subject: [PATCH 2/6] bump --- js/util/git_fields.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/js/util/git_fields.ts b/js/util/git_fields.ts index 0ea592ab8..6001516f4 100644 --- a/js/util/git_fields.ts +++ b/js/util/git_fields.ts @@ -1,6 +1,8 @@ import { type GitMetadataSettingsType as GitMetadataSettings } from "./generated_types"; -export const DEFAULT_GIT_METADATA_FIELDS = [ +export const DEFAULT_GIT_METADATA_FIELDS: NonNullable< + GitMetadataSettings["fields"] +> = [ "commit", "branch", "tag", From c863e2e0e477efba92203376865f394c3157b727 Mon Sep 17 00:00:00 2001 From: Alex Rhee Date: Thu, 21 May 2026 01:22:44 -0700 Subject: [PATCH 3/6] cleanup --- .changeset/soft-eagles-protect.md | 5 +++++ js/src/framework.ts | 2 +- js/src/logger.ts | 10 +++++----- 3 files changed, 11 insertions(+), 6 deletions(-) create mode 100644 .changeset/soft-eagles-protect.md diff --git a/.changeset/soft-eagles-protect.md b/.changeset/soft-eagles-protect.md new file mode 100644 index 000000000..16c60f222 --- /dev/null +++ b/.changeset/soft-eagles-protect.md @@ -0,0 +1,5 @@ +--- +"braintrust": patch +--- + +Make the default git metadata settings exclude diff content unless the org opts in. diff --git a/js/src/framework.ts b/js/src/framework.ts index eb22fbb8b..7f00ea055 100644 --- a/js/src/framework.ts +++ b/js/src/framework.ts @@ -344,7 +344,7 @@ export interface Evaluator< baseExperimentId?: string; /** - * Optional settings for collecting git metadata. By default, will collect all git metadata fields allowed in org-level settings. + * Optional settings for collecting git metadata. By default, will collect git metadata fields allowed in org-level settings, excluding diff content unless the org opts in. */ gitMetadataSettings?: GitMetadataSettings; diff --git a/js/src/logger.ts b/js/src/logger.ts index 5d83cc509..91c4e4e61 100644 --- a/js/src/logger.ts +++ b/js/src/logger.ts @@ -4272,8 +4272,8 @@ export function initDataset< legacy, _internal_btql, resolvedVersion instanceof LazyValue || - normalizedEnvironment !== undefined || - normalizedSnapshotName !== undefined + normalizedEnvironment !== undefined || + normalizedSnapshotName !== undefined ? { ...(resolvedVersion instanceof LazyValue ? { @@ -6051,9 +6051,9 @@ export type WithTransactionId = R & { export const DEFAULT_FETCH_BATCH_SIZE = 1000; export const MAX_BTQL_ITERATIONS = 10000; -export class ObjectFetcher - implements AsyncIterable> -{ +export class ObjectFetcher implements AsyncIterable< + WithTransactionId +> { private _fetchedData: WithTransactionId[] | undefined = undefined; constructor( From 966f324fe5b9c1d8c827165a8bee27dd6413aaf2 Mon Sep 17 00:00:00 2001 From: Alex Rhee Date: Thu, 21 May 2026 10:27:46 -0700 Subject: [PATCH 4/6] remove git default metadata settings and keep source of truth in control plane --- .changeset/soft-eagles-protect.md | 2 +- js/src/framework.ts | 2 +- js/src/gitutil.ts | 8 +++----- js/src/logger.ts | 8 +++----- js/util/git_fields.ts | 22 +--------------------- js/util/index.ts | 5 +---- 6 files changed, 10 insertions(+), 37 deletions(-) diff --git a/.changeset/soft-eagles-protect.md b/.changeset/soft-eagles-protect.md index 16c60f222..1f72dad5a 100644 --- a/.changeset/soft-eagles-protect.md +++ b/.changeset/soft-eagles-protect.md @@ -2,4 +2,4 @@ "braintrust": patch --- -Make the default git metadata settings exclude diff content unless the org opts in. +Do not collect git metadata when org-level metadata settings are absent. diff --git a/js/src/framework.ts b/js/src/framework.ts index 7f00ea055..c9a834ebd 100644 --- a/js/src/framework.ts +++ b/js/src/framework.ts @@ -344,7 +344,7 @@ export interface Evaluator< baseExperimentId?: string; /** - * Optional settings for collecting git metadata. By default, will collect git metadata fields allowed in org-level settings, excluding diff content unless the org opts in. + * Optional settings for collecting git metadata. By default, will collect git metadata fields allowed in org-level settings. If org settings are absent, git metadata is not collected. */ gitMetadataSettings?: GitMetadataSettings; diff --git a/js/src/gitutil.ts b/js/src/gitutil.ts index b82d177c0..43a6d8005 100644 --- a/js/src/gitutil.ts +++ b/js/src/gitutil.ts @@ -4,7 +4,6 @@ import { } from "./generated_types"; import { debugLogger } from "./debug-logger"; import { simpleGit } from "simple-git"; -import { defaultGitMetadataSettings } from "../util/git_fields"; const COMMON_BASE_BRANCHES = ["main", "master", "develop"]; @@ -150,18 +149,17 @@ function truncateToByteLimit(s: string, byteLimit: number = 65536): string { } export async function getRepoInfo(settings?: GitMetadataSettings) { - const resolvedSettings = settings ?? defaultGitMetadataSettings(); - if (resolvedSettings.collect === "none") { + if (!settings || settings.collect === "none") { return undefined; } const repo = await repoInfo(); - if (!repo || resolvedSettings.collect === "all") { + if (!repo || settings.collect === "all") { return repo; } let sanitized: RepoInfo = {}; - resolvedSettings.fields?.forEach((field) => { + settings.fields?.forEach((field) => { sanitized = { ...sanitized, [field]: repo[field] }; }); diff --git a/js/src/logger.ts b/js/src/logger.ts index 91c4e4e61..2fe305dec 100644 --- a/js/src/logger.ts +++ b/js/src/logger.ts @@ -31,7 +31,6 @@ import { IdField, IS_MERGE_FIELD, LogFeedbackFullArgs, - defaultGitMetadataSettings, mergeDicts, mergeGitMetadataSettings, mergeRowBatch, @@ -3546,7 +3545,7 @@ type InitializedExperiment = * @param options.apiKey The API key to use. If the parameter is not specified, will try to use the `BRAINTRUST_API_KEY` environment variable. If no API key is specified, will prompt the user to login. * @param options.orgName (Optional) The name of a specific organization to connect to. This is useful if you belong to multiple. * @param options.metadata (Optional) A dictionary with additional data about the test example, model outputs, or just about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any JSON-serializable type, but its keys must be strings. - * @param options.gitMetadataSettings (Optional) Settings for collecting git metadata. By default, will collect git metadata fields allowed in org-level settings, excluding diff content unless the org opts in. + * @param options.gitMetadataSettings (Optional) Settings for collecting git metadata. By default, will collect git metadata fields allowed in org-level settings. If org settings are absent, git metadata is not collected. * @param setCurrent If true (the default), set the global current-experiment to the newly-created one. * @param options.open If the experiment already exists, open it in read-only mode. Throws an error if the experiment does not already exist. * @param options.projectId The id of the project to create the experiment in. This takes precedence over `project` if specified. @@ -3700,7 +3699,7 @@ export function init( return repoInfo; } let mergedGitMetadataSettings = { - ...(state.gitMetadataSettings || defaultGitMetadataSettings()), + ...(state.gitMetadataSettings || { collect: "none" as const }), }; if (gitMetadataSettings) { mergedGitMetadataSettings = mergeGitMetadataSettings( @@ -5708,8 +5707,7 @@ function _saveOrgInfo( state.orgName = org.name; state.apiUrl = iso.getEnv("BRAINTRUST_API_URL") ?? org.api_url; state.proxyUrl = iso.getEnv("BRAINTRUST_PROXY_URL") ?? org.proxy_url; - state.gitMetadataSettings = - org.git_metadata || defaultGitMetadataSettings(); + state.gitMetadataSettings = org.git_metadata || { collect: "none" }; break; } } diff --git a/js/util/git_fields.ts b/js/util/git_fields.ts index 6001516f4..8eb06d625 100644 --- a/js/util/git_fields.ts +++ b/js/util/git_fields.ts @@ -1,24 +1,4 @@ -import { type GitMetadataSettingsType as GitMetadataSettings } from "./generated_types"; - -export const DEFAULT_GIT_METADATA_FIELDS: NonNullable< - GitMetadataSettings["fields"] -> = [ - "commit", - "branch", - "tag", - "dirty", - "author_name", - "author_email", - "commit_message", - "commit_time", -]; - -export function defaultGitMetadataSettings(): GitMetadataSettings { - return { - collect: "some", - fields: [...DEFAULT_GIT_METADATA_FIELDS], - }; -} +import { GitMetadataSettingsType as GitMetadataSettings } from "./generated_types"; export function mergeGitMetadataSettings( s1: GitMetadataSettings, diff --git a/js/util/index.ts b/js/util/index.ts index 83d2b76c5..0d1d0b1a4 100644 --- a/js/util/index.ts +++ b/js/util/index.ts @@ -136,10 +136,7 @@ export { spanTypeAttributeValues, } from "./span_types"; -export { - defaultGitMetadataSettings, - mergeGitMetadataSettings, -} from "./git_fields"; +export { mergeGitMetadataSettings } from "./git_fields"; export { loadPrettyXact, prettifyXact } from "./xact-ids"; From 799bf0f77750375a3bc5bf04ce561e3aa9e3a95e Mon Sep 17 00:00:00 2001 From: Alex Rhee Date: Sat, 23 May 2026 15:05:43 -0700 Subject: [PATCH 5/6] fix: preserve user opt-in when org git metadata policy is absent Using `{ collect: "none" }` as the fallback for a missing org setting caused it to win in mergeGitMetadataSettings, silently ignoring any explicit gitMetadataSettings passed to init(). Using `{ collect: "all" }` as the fallback instead acts as a passthrough, so the user's setting is respected when the org has no policy. --- js/src/logger.ts | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/js/src/logger.ts b/js/src/logger.ts index 2fe305dec..42766bf6f 100644 --- a/js/src/logger.ts +++ b/js/src/logger.ts @@ -3698,15 +3698,10 @@ export function init( if (repoInfo) { return repoInfo; } - let mergedGitMetadataSettings = { - ...(state.gitMetadataSettings || { collect: "none" as const }), - }; - if (gitMetadataSettings) { - mergedGitMetadataSettings = mergeGitMetadataSettings( - mergedGitMetadataSettings, - gitMetadataSettings, - ); - } + const mergedGitMetadataSettings = mergeGitMetadataSettings( + state.gitMetadataSettings ?? { collect: "all" as const }, + gitMetadataSettings ?? { collect: "none" as const }, + ); return await iso.getRepoInfo(mergedGitMetadataSettings); })(); From 25f1c86cc1fdb94b206bcac159eadfd29f5b66d8 Mon Sep 17 00:00:00 2001 From: Luca Forstner Date: Tue, 26 May 2026 11:27:18 +0200 Subject: [PATCH 6/6] wording, fix, tests --- .changeset/soft-eagles-protect.md | 2 +- js/src/framework.ts | 2 +- js/src/logger.test.ts | 81 +++++++++++++++++++++++++++++++ js/src/logger.ts | 15 +++--- 4 files changed, 92 insertions(+), 8 deletions(-) diff --git a/.changeset/soft-eagles-protect.md b/.changeset/soft-eagles-protect.md index 1f72dad5a..930d936b6 100644 --- a/.changeset/soft-eagles-protect.md +++ b/.changeset/soft-eagles-protect.md @@ -2,4 +2,4 @@ "braintrust": patch --- -Do not collect git metadata when org-level metadata settings are absent. +feat: Do not collect git metadata by default when organization-level git metadata settings are absent diff --git a/js/src/framework.ts b/js/src/framework.ts index c9a834ebd..26a2bb515 100644 --- a/js/src/framework.ts +++ b/js/src/framework.ts @@ -344,7 +344,7 @@ export interface Evaluator< baseExperimentId?: string; /** - * Optional settings for collecting git metadata. By default, will collect git metadata fields allowed in org-level settings. If org settings are absent, git metadata is not collected. + * Optional settings for collecting git metadata. By default, Braintrust collects the git metadata fields allowed by your organization's git metadata settings. If those settings are absent, git metadata is not collected unless this option is set. */ gitMetadataSettings?: GitMetadataSettings; diff --git a/js/src/logger.test.ts b/js/src/logger.test.ts index d9c89ad48..166cecaaf 100644 --- a/js/src/logger.test.ts +++ b/js/src/logger.test.ts @@ -21,6 +21,7 @@ import { } from "./logger"; import { configureNode } from "./node/config"; +import { type GitMetadataSettingsType as GitMetadataSettings } from "./generated_types"; import { writeFile, unlink } from "node:fs/promises"; import { join } from "node:path"; import { tmpdir } from "node:os"; @@ -484,6 +485,86 @@ function mockInitGitMetadata() { ).mockResolvedValue([]); } +const initGitMetadataSettingsCases: Array<{ + name: string; + orgSettings?: GitMetadataSettings; + initSettings?: GitMetadataSettings; + expected: GitMetadataSettings; +}> = [ + { + name: "uses organization settings by default", + orgSettings: { collect: "some", fields: ["commit", "branch"] }, + expected: { collect: "some", fields: ["commit", "branch"] }, + }, + { + name: "does not collect by default when organization settings are absent", + expected: { collect: "none" }, + }, + { + name: "preserves explicit opt-in when organization settings are absent", + initSettings: { collect: "all" }, + expected: { collect: "all" }, + }, + { + name: "intersects explicit settings with organization settings", + orgSettings: { collect: "some", fields: ["commit", "branch"] }, + initSettings: { collect: "some", fields: ["branch", "git_diff"] }, + expected: { collect: "some", fields: ["branch"] }, + }, + { + name: "lets organization settings constrain explicit all", + orgSettings: { collect: "some", fields: ["commit"] }, + initSettings: { collect: "all" }, + expected: { collect: "some", fields: ["commit"] }, + }, +]; + +test.each(initGitMetadataSettingsCases)( + "init applies git metadata settings: $name", + async ({ orgSettings, initSettings, expected }) => { + const state = await _exportsForTestingOnly.simulateLoginForTests(); + + try { + state.gitMetadataSettings = orgSettings; + vi.spyOn(state, "login").mockResolvedValue(state); + const getRepoInfo = vi + .spyOn(_exportsForTestingOnly.isomorph, "getRepoInfo") + .mockResolvedValue(undefined); + vi.spyOn( + _exportsForTestingOnly.isomorph, + "getPastNAncestors", + ).mockResolvedValue([]); + vi.spyOn(state.appConn(), "post_json").mockResolvedValue({ + project: { + id: "00000000-0000-0000-0000-000000000001", + name: "test-project", + }, + experiment: { + id: "00000000-0000-0000-0000-000000000003", + project_id: "00000000-0000-0000-0000-000000000001", + name: "test-experiment", + public: false, + }, + }); + + const experiment = init({ + project: "test-project", + experiment: "test-experiment", + gitMetadataSettings: initSettings, + setCurrent: false, + state, + }); + + await experiment.id; + + expect(getRepoInfo).toHaveBeenCalledWith(expected); + } finally { + _exportsForTestingOnly.simulateLogoutForTests(); + vi.restoreAllMocks(); + } + }, +); + test("init forwards dataset _internal_btql to experiment register", async () => { const state = await _exportsForTestingOnly.simulateLoginForTests(); diff --git a/js/src/logger.ts b/js/src/logger.ts index 42766bf6f..c0ba74c36 100644 --- a/js/src/logger.ts +++ b/js/src/logger.ts @@ -3545,7 +3545,7 @@ type InitializedExperiment = * @param options.apiKey The API key to use. If the parameter is not specified, will try to use the `BRAINTRUST_API_KEY` environment variable. If no API key is specified, will prompt the user to login. * @param options.orgName (Optional) The name of a specific organization to connect to. This is useful if you belong to multiple. * @param options.metadata (Optional) A dictionary with additional data about the test example, model outputs, or just about anything else that's relevant, that you can use to help find and analyze examples later. For example, you could log the `prompt`, example's `id`, or anything else that would be useful to slice/dice later. The values in `metadata` can be any JSON-serializable type, but its keys must be strings. - * @param options.gitMetadataSettings (Optional) Settings for collecting git metadata. By default, will collect git metadata fields allowed in org-level settings. If org settings are absent, git metadata is not collected. + * @param options.gitMetadataSettings (Optional) Settings for collecting git metadata. By default, Braintrust collects the git metadata fields allowed by your organization's git metadata settings. If those settings are absent, git metadata is not collected unless this option is set. * @param setCurrent If true (the default), set the global current-experiment to the newly-created one. * @param options.open If the experiment already exists, open it in read-only mode. Throws an error if the experiment does not already exist. * @param options.projectId The id of the project to create the experiment in. This takes precedence over `project` if specified. @@ -3698,10 +3698,13 @@ export function init( if (repoInfo) { return repoInfo; } - const mergedGitMetadataSettings = mergeGitMetadataSettings( - state.gitMetadataSettings ?? { collect: "all" as const }, - gitMetadataSettings ?? { collect: "none" as const }, - ); + const mergedGitMetadataSettings = + state.gitMetadataSettings == null + ? (gitMetadataSettings ?? { collect: "none" as const }) + : mergeGitMetadataSettings( + state.gitMetadataSettings, + gitMetadataSettings ?? { collect: "all" as const }, + ); return await iso.getRepoInfo(mergedGitMetadataSettings); })(); @@ -5702,7 +5705,7 @@ function _saveOrgInfo( state.orgName = org.name; state.apiUrl = iso.getEnv("BRAINTRUST_API_URL") ?? org.api_url; state.proxyUrl = iso.getEnv("BRAINTRUST_PROXY_URL") ?? org.proxy_url; - state.gitMetadataSettings = org.git_metadata || { collect: "none" }; + state.gitMetadataSettings = org.git_metadata || undefined; break; } }