Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions apps/framework/harness/run-eval.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import { bootPlatformBackend } from './platform-backend.js';
import { viteBuild, vitestRun } from './project-runner.js';
import { buildSystemPrompt } from './system-prompt.js';
import {
applyAgentInstructions,
buildDocsResult,
buildSkillResult,
createJudgeRecorder,
Expand Down Expand Up @@ -447,6 +448,14 @@ async function runOne(
})
);

if (exp.instructions) {
await applyAgentInstructions(
session.sandbox,
exp.agent.id,
exp.instructions
);
}

const systemPrompt = buildSystemPrompt({
agent: exp.agent.id,
addendum: session.promptAddendum,
Expand Down Expand Up @@ -538,6 +547,19 @@ async function runOne(
)
: undefined;

if (exp.instructions) {
if (!cliSandbox) {
throw new Error(
`${expName}: instructions need an agent that runs in a sandbox, but ${exp.agent.id} runs in-process`
);
}
await applyAgentInstructions(
cliSandbox.sandbox,
exp.agent.id,
exp.instructions
);
}

// A CLI agent discovers its installed skills itself. An in-process agent has
// no filesystem, so its skills are advertised in the prompt and pulled on
// demand via the load_skill tool instead.
Expand Down
2 changes: 2 additions & 0 deletions apps/web/src/components/results/eval-details.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ const DOCS_CALL_SOURCE_LABEL: Record<DocsCall["source"], string> = {
web_fetch: "Web Fetch",
web_search: "Web Search",
shell_fetch: "Shell",
supabase_sh: "supabase.sh",
}

// Cool color for MCP (our own docs tool), warm for the agent going around it onto the open web.
Expand All @@ -34,6 +35,7 @@ const DOCS_CALL_SOURCE_CHIP_CLASS: Record<DocsCall["source"], string> = {
web_fetch: "bg-amber-500/10 text-amber-700 dark:text-amber-400",
web_search: "bg-amber-500/10 text-amber-700 dark:text-amber-400",
shell_fetch: "bg-orange-500/10 text-orange-700 dark:text-orange-400",
supabase_sh: "bg-indigo-500/10 text-indigo-700 dark:text-indigo-400",
}

const docsSourceChipClassName =
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
import { defineExperiment } from '@supabase-evals/core';
import { claudeCodeSonnet55 } from '../presets.js';
import { supabaseShInstructions } from '../supabase-sh.js';

export default defineExperiment({
...claudeCodeSonnet55,
skills: [],
instructions: supabaseShInstructions,
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
import { defineExperiment } from '@supabase-evals/core';
import { claudeCodeSonnet55 } from '../presets.js';
import { supabaseShInstructions } from '../supabase-sh.js';

export default defineExperiment({
...claudeCodeSonnet55,
instructions: supabaseShInstructions,
});
29 changes: 29 additions & 0 deletions experiments/supabase-sh.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
/**
* Project instructions for supabase.sh, Supabase's docs served over SSH
* (https://supabase.com/blog/supabase-docs-over-ssh). This is the output of
* `ssh supabase.sh agents`, the snippet the post tells users to add to
* AGENTS.md or CLAUDE.md. Experiments pass it as `instructions`, so the agent
* gets the same setup a user would have while the prompt stays unchanged.
*
* Captured 2026-11-20. Re-run `ssh supabase.sh agents` to refresh it.
*/
export const supabaseShInstructions = `## Supabase Docs

Before working on a Supabase feature, check the docs via \`ssh supabase.sh <command>\`.

\`\`\`bash
# Search for a topic
ssh supabase.sh grep -rl 'auth' /supabase/docs/

# Read a specific guide
ssh supabase.sh cat /supabase/docs/guides/auth/passwords.md

# Find all guides in a section
ssh supabase.sh find /supabase/docs/guides/database -name '*.md'

# Search with context
ssh supabase.sh grep -r 'RLS' /supabase/docs/guides/auth --include='*.md' -l
\`\`\`

All docs live under \`/supabase/docs/\` as markdown files. You can use any standard Unix tools (grep, find, cat, etc.) to search and read them.
`;
80 changes: 80 additions & 0 deletions packages/core/src/agent-instructions.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
import { execSync } from 'node:child_process';
import { mkdtempSync, readFileSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { describe, expect, it } from 'vitest';
import {
agentInstructionsFile,
applyAgentInstructions,
} from './agent-instructions.js';
import type { AgentSandbox } from './agents/types.js';

/** A sandbox that runs commands with the host shell in a temp workspace. */
function localSandbox(): AgentSandbox {
const workspace = mkdtempSync(join(tmpdir(), 'agent-instructions-'));
return {
workspace,
async exec(command) {
try {
const stdout = execSync(command, {
cwd: workspace,
shell: '/bin/bash',
encoding: 'utf8',
});
return { ok: true, exitCode: 0, stdout, stderr: '' };
} catch (error) {
const e = error as { status?: number; stderr?: string };
return {
ok: false,
exitCode: e.status ?? 1,
stdout: '',
stderr: e.stderr ?? '',
};
}
},
async readFile(path) {
return readFileSync(join(workspace, path), 'utf8');
},
async copyToHost() {},
};
}

describe('agentInstructionsFile', () => {
it('uses CLAUDE.md for Claude Code and AGENTS.md for the other CLI agents', () => {
expect(agentInstructionsFile('claude-code')).toBe('CLAUDE.md');
expect(agentInstructionsFile('codex')).toBe('AGENTS.md');
expect(agentInstructionsFile('opencode')).toBe('AGENTS.md');
expect(agentInstructionsFile('grok')).toBe('AGENTS.md');
});
});

describe('applyAgentInstructions', () => {
it('creates the file with the instructions, keeping shell characters intact', async () => {
const sandbox = localSandbox();
const instructions =
"## Supabase Docs\n\nRun `ssh supabase.sh grep -rl 'auth' /supabase/docs/` and check $HOME.";

await applyAgentInstructions(sandbox, 'claude-code', instructions);

expect(await sandbox.readFile('CLAUDE.md')).toBe(`${instructions}\n`);
});

it('appends to an instructions file the eval ships, without touching its content', async () => {
const sandbox = localSandbox();
writeFileSync(join(sandbox.workspace, 'AGENTS.md'), '# Project rules\n');

await applyAgentInstructions(sandbox, 'codex', '## Supabase Docs');

expect(await sandbox.readFile('AGENTS.md')).toBe(
'# Project rules\n\n## Supabase Docs\n'
);
});

it('does nothing for empty instructions', async () => {
const sandbox = localSandbox();

await applyAgentInstructions(sandbox, 'claude-code', ' \n');

await expect(sandbox.readFile('CLAUDE.md')).rejects.toThrow();
});
});
37 changes: 37 additions & 0 deletions packages/core/src/agent-instructions.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
import type { AgentSandbox } from './agents/types.js';
import type { AgentHarnessId } from './eval-metadata.js';

/**
* The project instructions file each CLI agent reads from its working
* directory. Claude Code reads CLAUDE.md; Codex, OpenCode and Grok read
* AGENTS.md.
*/
export function agentInstructionsFile(agent: AgentHarnessId): string {
return agent === 'claude-code' ? 'CLAUDE.md' : 'AGENTS.md';
}

/**
* Adds an experiment's instructions to the project instructions file in the
* agent's workspace, the way a user sets up a tool like supabase.sh. Appends
* rather than replaces, so an AGENTS.md or CLAUDE.md the eval ships with
* keeps its own content.
*/
export async function applyAgentInstructions(
sandbox: AgentSandbox,
agent: AgentHarnessId,
instructions: string
): Promise<void> {
const text = instructions.trim();
if (!text) return;
const file = agentInstructionsFile(agent);
// Base64 keeps quotes, backticks and `$` in the instructions out of the shell.
const encoded = Buffer.from(`${text}\n`, 'utf8').toString('base64');
const result = await sandbox.exec(
`if [ -s ${file} ]; then printf '\\n' >> ${file}; fi; printf '%s' '${encoded}' | base64 -d >> ${file}`
);
if (!result.ok) {
throw new Error(
`failed to write experiment instructions to ${file}: ${result.stderr || result.stdout}`
);
}
}
153 changes: 153 additions & 0 deletions packages/core/src/docs-results.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1109,3 +1109,156 @@ describe('rehydrateTruncatedDocsResults', () => {
expect(readFile).not.toHaveBeenCalled();
});
});

describe('buildDocsResult with supabase.sh', () => {
/** A Claude Code Bash call, the way agents reach supabase.sh. */
function bash(
command: string,
result: unknown,
error?: string
): ToolCallRecord {
return toolCall(
'Bash',
{ command },
{ name: 'shell', command, result, ...(error ? { error } : {}) }
);
}

it('counts a page printed over ssh and maps it to its supabase.com url', () => {
const command =
'ssh supabase.sh cat /supabase/docs/guides/auth/server-side/creating-a-client.md';
const result = buildDocsResult([bash(command, '# Creating a client')]);

expect(result.calls).toEqual([
{
source: 'supabase_sh',
query: command,
hasContent: true,
pages: [
{
url: 'https://supabase.com/docs/guides/auth/server-side/creating-a-client',
},
],
resultChars: '# Creating a client'.length,
},
]);
});

it('keeps a page read whose output is trimmed by a local pipe', () => {
const command =
"ssh supabase.sh sed -n '40,80p' /supabase/docs/guides/local-development/cli/migrating-to-v3.md | head -40";
const [call] = buildDocsResult([bash(command, 'db push --remote')]).calls;

expect(call?.hasContent).toBe(true);
expect(call?.pages).toEqual([
{
url: 'https://supabase.com/docs/guides/local-development/cli/migrating-to-v3',
},
]);
});

it('counts a search without claiming it delivered page text', () => {
const command =
"ssh supabase.sh grep -rl 'gen types' /supabase/docs/ | head -20";
const result = buildDocsResult([
bash(command, '/supabase/docs/guides/api/rest/generating-types.md'),
]);

expect(result.calls).toEqual([
{
source: 'supabase_sh',
query: command,
pages: [],
resultChars: expect.any(Number),
},
]);
expect(result.calls[0]?.hasContent).toBeUndefined();
});

it('collects every page from several ssh calls in one command, mixed with local commands', () => {
const command =
'ssh supabase.sh ls /supabase/docs/guides/auth/server-side; ssh supabase.sh cat /supabase/docs/guides/auth/server-side/nextjs.md; cat package.json';
const [call] = buildDocsResult([bash(command, 'mixed output')]).calls;

expect(call?.source).toBe('supabase_sh');
expect(call?.pages).toEqual([
{ url: 'https://supabase.com/docs/guides/auth/server-side/nextjs' },
]);
// ls is a listing and cat package.json shares the output, so page text isn't proven.
expect(call?.hasContent).toBeUndefined();
});

it('reads a quoted command and ssh options before the host', () => {
const command =
'ssh -o BatchMode=yes supabase.sh "cat /supabase/docs/guides/database/views.md"';
const [call] = buildDocsResult([bash(command, '# Views')]).calls;

expect(call?.hasContent).toBe(true);
expect(call?.pages).toEqual([
{ url: 'https://supabase.com/docs/guides/database/views' },
]);
});

it('unwraps a bash -lc wrapper, the way Codex runs commands', () => {
const command =
"/bin/bash -lc 'ssh supabase.sh cat /supabase/docs/reference/cli.md'";
const [call] = buildDocsResult([bash(command, '# CLI reference')]).calls;

expect(call?.pages).toEqual([
{ url: 'https://supabase.com/docs/reference/cli' },
]);
});

it('counts a call inside a shell loop without inventing a page from the loop variable', () => {
const command =
'for f in auth auth-headers; do ssh supabase.sh cat /supabase/docs/guides/functions/$f.md; done';
const result = buildDocsResult([bash(command, '# Auth\n# Auth headers')]);

expect(result.calls).toEqual([
{
source: 'supabase_sh',
query: command,
pages: [],
resultChars: '# Auth\n# Auth headers'.length,
},
]);
});

it('ignores the setup command, an interactive session, and other hosts', () => {
const result = buildDocsResult([
bash('ssh supabase.sh agents', '## Supabase docs over SSH'),
bash('ssh supabase.sh', 'Welcome'),
bash('ssh example.com cat supabase.sh', 'unrelated'),
bash('echo ssh supabase.sh cat /supabase/docs/a.md', 'unrelated'),
]);

expect(result.calls).toEqual([]);
});

it('drops a failed call and one with no output, like a shell fetch', () => {
const result = buildDocsResult([
bash(
"ssh supabase.sh grep -rl 'no such thing' /supabase/docs/",
'',
'exit code 1'
),
bash('ssh supabase.sh cat /supabase/docs/guides/missing.md', ''),
]);

expect(result.calls).toEqual([]);
});

it('rehydrates a truncated supabase.sh read', async () => {
const path = '/home/node/.claude/projects/x/tool-results/abc.txt';
const call = bash(
'ssh supabase.sh cat /supabase/docs/reference/api.md',
`<persisted-output>\nOutput too large (80.1KB). Full output saved to: ${path}`
);
const readFile = vi.fn().mockResolvedValue('# Management API');

await rehydrateTruncatedDocsResults({ readFile }, [call]);

expect(readFile).toHaveBeenCalledWith(path);
expect(call.result).toBe('# Management API');
});
});
Loading