Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 33 additions & 9 deletions web/src/realtime/hooks/contextFormatters.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ describe('formatReadyEvent', () => {
})

describe('formatMessage', () => {
it('formats codex stream-json assistant messages for voice context', () => {
it('uses the supplied agent label as the assistant prefix', () => {
const formatted = formatMessage(msg({
id: '1',
seq: 1,
Expand All @@ -139,12 +139,33 @@ describe('formatMessage', () => {
}
}
}
}))
}), 'Codex')

expect(formatted).toContain('Claude Code:')
expect(formatted).toContain('Codex:')
expect(formatted).not.toContain('Claude Code')
expect(formatted).toContain('<text>Indexed 5,018 items in the search database.</text>')
})

it('threads agentLabel for a different flavor (regression for #680)', () => {
const formatted = formatMessage(msg({
id: '1',
seq: 1,
content: {
role: 'agent',
content: {
type: 'codex',
data: {
type: 'message',
message: 'Cursor is generating a plan.'
}
}
}
}), 'Cursor')

expect(formatted).toContain('Cursor:')
expect(formatted).not.toContain('Claude Code')
})

it('ignores codex ready and tool-call payloads', () => {
expect(formatMessage(msg({
id: '1',
Expand All @@ -156,7 +177,7 @@ describe('formatMessage', () => {
data: { type: 'ready' }
}
}
}))).toBeNull()
}), 'Codex')).toBeNull()
})

it('does not treat session status events as speakable assistant text', () => {
Expand All @@ -171,10 +192,10 @@ describe('formatMessage', () => {
data: { type: 'message', message: 'Aborting task.' }
}
}
}))).toBeNull()
}), 'Codex')).toBeNull()
})

it('preserves tool-call context for mixed text+tool_use content array', () => {
it('preserves tool-call context for mixed text+tool_use content array and uses the agent label', () => {
const formatted = formatMessage(msg({
id: '1',
seq: 1,
Expand All @@ -185,10 +206,11 @@ describe('formatMessage', () => {
{ type: 'tool_use', name: 'Bash', input: { command: 'ls' } }
]
}
}))
}), 'Claude')

expect(formatted).toContain('Here is the result.')
expect(formatted).toContain('Claude Code is using Bash')
expect(formatted).toContain('Claude is using Bash')
expect(formatted).not.toContain('Claude Code is using')
})
})

Expand All @@ -209,9 +231,11 @@ describe('formatNewMessages', () => {
}
}
})
])
], 'Codex')

expect(update).toContain('New messages in session: session-1')
expect(update).toContain('Local database file size is 2.43 GiB.')
expect(update).toContain('Codex:')
expect(update).not.toContain('Claude Code')
})
})
45 changes: 26 additions & 19 deletions web/src/realtime/hooks/contextFormatters.ts
Original file line number Diff line number Diff line change
Expand Up @@ -66,32 +66,39 @@ function unwrapOutputContent(content: unknown): { roleOverride: NormalizedRole |
return { roleOverride, content: messageContent }
}

function formatPlainText(role: NormalizedRole | null, text: string): string {
function formatPlainText(role: NormalizedRole | null, text: string, agentLabel: string): string {
if (role === 'assistant') {
return `Claude Code: \n<text>${text}</text>`
return `${agentLabel}: \n<text>${text}</text>`
}
return `User sent message: \n<text>${text}</text>`
}

/**
* Format a permission request for natural language context
* Format a permission request for natural language context.
*
* `agentLabel` is the display label for the session's agent flavor
* (e.g. "Claude", "Cursor", "Codex"); voiceHooks computes it once per call.
*/
export function formatPermissionRequest(
sessionId: string,
requestId: string,
toolName: string,
toolArgs: unknown
toolArgs: unknown,
agentLabel: string
): string {
return `Claude Code is requesting permission to use ${toolName} (session ${sessionId}):
return `${agentLabel} is requesting permission to use ${toolName} (session ${sessionId}):
<request_id>${requestId}</request_id>
<tool_name>${toolName}</tool_name>
<tool_args>${JSON.stringify(toolArgs)}</tool_args>`
}

/**
* Format a single message for voice context
* Format a single message for voice context.
*
* `agentLabel` is the display label for the session's agent flavor
* (e.g. "Claude", "Cursor", "Codex"); voiceHooks computes it once per call.
*/
export function formatMessage(message: DecryptedMessage): string | null {
export function formatMessage(message: DecryptedMessage, agentLabel: string): string | null {
const { role, content: wrappedContent } = unwrapRoleWrappedContent(message)
const { roleOverride, content } = unwrapOutputContent(wrappedContent)
const normalizedRole = roleOverride ?? role
Expand All @@ -103,7 +110,7 @@ export function formatMessage(message: DecryptedMessage): string | null {
const speakable = !isContentArray(content) ? extractSpeakableFromContent(content) : null
if (speakable) {
const roleForFormat = normalizedRole === 'user' ? 'user' : 'assistant'
return formatPlainText(roleForFormat, speakable)
return formatPlainText(roleForFormat, speakable, agentLabel)
}

if (!isContentArray(content)) {
Expand All @@ -122,13 +129,13 @@ export function formatMessage(message: DecryptedMessage): string | null {

for (const item of content) {
if (item.type === 'text' && item.text) {
lines.push(formatPlainText(isAssistant ? 'assistant' : 'user', item.text))
lines.push(formatPlainText(isAssistant ? 'assistant' : 'user', item.text, agentLabel))
} else if (item.type === 'tool_use' && !VOICE_CONFIG.DISABLE_TOOL_CALLS) {
const name = item.name || 'unknown'
if (VOICE_CONFIG.LIMITED_TOOL_CALLS) {
lines.push(`Claude Code is using ${name}`)
lines.push(`${agentLabel} is using ${name}`)
} else {
lines.push(`Claude Code is using ${name} with arguments: <arguments>${JSON.stringify(item.input)}</arguments>`)
lines.push(`${agentLabel} is using ${name} with arguments: <arguments>${JSON.stringify(item.input)}</arguments>`)
}
}
}
Expand Down Expand Up @@ -214,34 +221,34 @@ export function extractLastAssistantSpeakable(messages: DecryptedMessage[]): str
return null
}

export function formatNewSingleMessage(sessionId: string, message: DecryptedMessage): string | null {
const formatted = formatMessage(message)
export function formatNewSingleMessage(sessionId: string, message: DecryptedMessage, agentLabel: string): string | null {
const formatted = formatMessage(message, agentLabel)
if (!formatted) {
return null
}
return 'New message in session: ' + sessionId + '\n\n' + formatted
}

export function formatNewMessages(sessionId: string, messages: DecryptedMessage[]): string | null {
export function formatNewMessages(sessionId: string, messages: DecryptedMessage[], agentLabel: string): string | null {
const formatted = [...messages]
.sort((a, b) => (a.seq ?? 0) - (b.seq ?? 0))
.map(formatMessage)
.map((m) => formatMessage(m, agentLabel))
.filter(Boolean)
if (formatted.length === 0) {
return null
}
return 'New messages in session: ' + sessionId + '\n\n' + formatted.join('\n\n')
}

export function formatHistory(sessionId: string, messages: DecryptedMessage[]): string {
export function formatHistory(sessionId: string, messages: DecryptedMessage[], agentLabel: string): string {
const messagesToFormat = VOICE_CONFIG.MAX_HISTORY_MESSAGES > 0
? messages.slice(-VOICE_CONFIG.MAX_HISTORY_MESSAGES)
: messages
const formatted = messagesToFormat.map(formatMessage).filter(Boolean)
const formatted = messagesToFormat.map((m) => formatMessage(m, agentLabel)).filter(Boolean)
return 'History of messages in session: ' + sessionId + '\n\n' + formatted.join('\n\n')
}

export function formatSessionFull(session: Session | null, messages: DecryptedMessage[]): string {
export function formatSessionFull(session: Session | null, messages: DecryptedMessage[], agentLabel: string): string {
if (!session) {
return 'Session not available'
}
Expand All @@ -262,7 +269,7 @@ export function formatSessionFull(session: Session | null, messages: DecryptedMe

lines.push('## Our interaction history so far')
lines.push('')
lines.push(formatHistory(session.id, messages))
lines.push(formatHistory(session.id, messages, agentLabel))

return lines.join('\n\n')
}
Expand Down
5 changes: 3 additions & 2 deletions web/src/realtime/hooks/voiceContextPlan.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,17 +25,18 @@ describe('buildSessionVoiceContextPlan', () => {
const session = makeSession('sess-1')
const messages = Array.from({ length: 10 }, (_, i) => makeMessage(i + 1, `line ${i + 1}`))

const plan = buildSessionVoiceContextPlan(session, messages)
const plan = buildSessionVoiceContextPlan(session, messages, 'Codex')

expect(plan.bootstrap).toContain('sess-1')
expect(plan.bootstrap).toContain('Auth refactor')
expect(utf8ByteLength(plan.bootstrap)).toBeLessThanOrEqual(ELEVENLABS_WEBRTC_CONTEXT_MAX_BYTES)
expect(plan.streamChunks.length).toBeGreaterThan(0)
expect(plan.messagesInBootstrap).toBeLessThanOrEqual(2)
expect(plan.bootstrap).not.toContain('Claude Code')
})

test('handles missing session', () => {
const plan = buildSessionVoiceContextPlan(null, [])
const plan = buildSessionVoiceContextPlan(null, [], 'Claude')
expect(plan.bootstrap).toBe('Session not available')
expect(plan.streamChunks).toEqual([])
})
Expand Down
8 changes: 6 additions & 2 deletions web/src/realtime/hooks/voiceContextPlan.ts
Original file line number Diff line number Diff line change
Expand Up @@ -66,10 +66,14 @@ function chunkTextByBytes(parts: string[], maxBytes: number): string[] {

/**
* Small handshake context for startSession; remainder is streamed after connect.
*
* `agentLabel` is the display label for the session's agent flavor
* (e.g. "Claude", "Cursor", "Codex"); voiceHooks computes it once per call.
*/
export function buildSessionVoiceContextPlan(
session: Session | null,
messages: DecryptedMessage[]
messages: DecryptedMessage[],
agentLabel: string
): SessionVoiceContextPlan {
if (!session) {
return {
Expand All @@ -89,7 +93,7 @@ export function buildSessionVoiceContextPlan(
: all

const formatted = capped
.map((m) => formatMessage(m))
.map((m) => formatMessage(m, agentLabel))
.filter((line): line is string => Boolean(line))

const recentCount = Math.min(BOOTSTRAP_RECENT_MESSAGES, formatted.length)
Expand Down
26 changes: 20 additions & 6 deletions web/src/realtime/hooks/voiceHooks.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,26 @@ import {
} from './contextFormatters'
import { VOICE_CONFIG } from '../voiceConfig'
import { buildSessionVoiceContextPlan, type SessionVoiceContextPlan } from './voiceContextPlan'
import type { DecryptedMessage, Session } from '@/types/api'
import { getFlavorLabel, isKnownFlavor } from '@hapi/protocol'
import type { DecryptedMessage, Session, SessionMetadataSummary } from '@/types/api'

interface SessionMetadata {
summary?: { text?: string }
path?: string
machineId?: string
}

/**
* Resolve the display label for the session's agent flavor. Falls back to a
* generic "coding agent" string for unknown or missing flavors so the voice
* context never bottoms out with a literal "undefined" or the old hardcoded
* "Claude Code" (closes #680).
*/
function getAgentLabel(session: Session | null): string {
const flavor = (session?.metadata as SessionMetadataSummary | undefined)?.flavor
return isKnownFlavor(flavor) ? getFlavorLabel(flavor) : 'coding agent'
}

// Track which sessions have been reported
const shownSessions = new Set<string>()
let lastFocusSession: string | null = null
Expand Down Expand Up @@ -66,7 +78,7 @@ function reportSession(sessionId: string) {
if (!session) return

const messages = messagesGetter?.(sessionId) ?? []
const contextUpdate = formatSessionFull(session, messages)
const contextUpdate = formatSessionFull(session, messages, getAgentLabel(session))
reportContextualUpdate(contextUpdate)
}

Expand Down Expand Up @@ -106,13 +118,14 @@ export const voiceHooks = {
},

/**
* Called when Claude requests permission for a tool use
* Called when the active agent requests permission for a tool use
*/
onPermissionRequested(sessionId: string, requestId: string, toolName: string, toolArgs: unknown) {
if (VOICE_CONFIG.DISABLE_PERMISSION_REQUESTS) return

const session = sessionGetter?.(sessionId) ?? null
reportSession(sessionId)
reportTextUpdate(formatPermissionRequest(sessionId, requestId, toolName, toolArgs))
reportTextUpdate(formatPermissionRequest(sessionId, requestId, toolName, toolArgs, getAgentLabel(session)))
},

/**
Expand All @@ -121,8 +134,9 @@ export const voiceHooks = {
onMessages(sessionId: string, messages: DecryptedMessage[]) {
if (VOICE_CONFIG.DISABLE_MESSAGES) return

const session = sessionGetter?.(sessionId) ?? null
reportSession(sessionId)
reportContextualUpdate(formatNewMessages(sessionId, messages))
reportContextualUpdate(formatNewMessages(sessionId, messages, getAgentLabel(session)))
},

/**
Expand All @@ -136,7 +150,7 @@ export const voiceHooks = {

const session = sessionGetter?.(sessionId) ?? null
const messages = messagesGetter?.(sessionId) ?? []
const plan = buildSessionVoiceContextPlan(session, messages)
const plan = buildSessionVoiceContextPlan(session, messages, getAgentLabel(session))
shownSessions.add(sessionId)
return plan
},
Expand Down
Loading