Skip to content

Commit d2ff57a

Browse files
cephalizationcephalization
authored
feat: Support Response Format in Playground (#5259)
* fix: Prevent page crash when parsing invocation parameters with unexpected key types * feat: parse response_format from span into playground store * Fix responseFormat issues in tests Still need to figure out why unrelated fields are failing * Fix rest of tests * feat: Render response_format/Output Schema editor * Parse response format from invocation parameters, and write back on edits * prettier * Rename OutputContent card title back to Output * Always add a tool choice invocation param to chat completion params if tools are defined * Rename openai vars to openAI * Add test to ensure response format errors do not appear when not applicable * Improve comments, remove dead code * lint
1 parent b6e7499 commit d2ff57a

12 files changed

+464
-43
lines changed

‎.pre-commit-config.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ repos:
2929
files: \.(jsx?|tsx?|css|.md)$
3030
exclude: \.*__generated__.*$
3131
additional_dependencies:
32-
- prettier@3.2.4
32+
- prettier@3.3.3
3333
- repo: https://github.com/pre-commit/mirrors-eslint
3434
rev: "8ddcbd412c0b348841f0f82c837702f432539652"
3535
hooks:

‎app/src/pages/playground/PlaygroundChatTemplate.tsx

+34-1
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ import { usePlaygroundContext } from "@phoenix/contexts/PlaygroundContext";
3636
import { useChatMessageStyles } from "@phoenix/hooks/useChatMessageStyles";
3737
import {
3838
ChatMessage,
39+
createOpenAIResponseFormat,
3940
generateMessageId,
4041
PlaygroundChatTemplate as PlaygroundChatTemplateType,
4142
PlaygroundInstance,
@@ -44,11 +45,16 @@ import { assertUnreachable } from "@phoenix/typeUtils";
4445
import { safelyParseJSON } from "@phoenix/utils/jsonUtils";
4546

4647
import { ChatMessageToolCallsEditor } from "./ChatMessageToolCallsEditor";
48+
import {
49+
RESPONSE_FORMAT_PARAM_CANONICAL_NAME,
50+
RESPONSE_FORMAT_PARAM_NAME,
51+
} from "./constants";
4752
import {
4853
MessageContentRadioGroup,
4954
MessageMode,
5055
} from "./MessageContentRadioGroup";
5156
import { MessageRolePicker } from "./MessageRolePicker";
57+
import { PlaygroundResponseFormat } from "./PlaygroundResponseFormat";
5258
import { PlaygroundTools } from "./PlaygroundTools";
5359
import {
5460
createToolCallForProvider,
@@ -74,11 +80,18 @@ export function PlaygroundChatTemplate(props: PlaygroundChatTemplateProps) {
7480
);
7581
const instances = usePlaygroundContext((state) => state.instances);
7682
const updateInstance = usePlaygroundContext((state) => state.updateInstance);
83+
const upsertInvocationParameterInput = usePlaygroundContext(
84+
(state) => state.upsertInvocationParameterInput
85+
);
7786
const playgroundInstance = instances.find((instance) => instance.id === id);
7887
if (!playgroundInstance) {
7988
throw new Error(`Playground instance ${id} not found`);
8089
}
8190
const hasTools = playgroundInstance.tools.length > 0;
91+
const hasResponseFormat =
92+
playgroundInstance.model.invocationParameters.find(
93+
(p) => p.canonicalName === RESPONSE_FORMAT_PARAM_CANONICAL_NAME
94+
) != null;
8295
const { template } = playgroundInstance;
8396
if (template.__type !== "chat") {
8497
throw new Error(`Invalid template type ${template.__type}`);
@@ -151,9 +164,28 @@ export function PlaygroundChatTemplate(props: PlaygroundChatTemplateProps) {
151164
paddingBottom="size-100"
152165
borderColor="dark"
153166
borderTopWidth="thin"
154-
borderBottomWidth={hasTools ? "thin" : undefined}
167+
borderBottomWidth={hasTools || hasResponseFormat ? "thin" : undefined}
155168
>
156169
<Flex direction="row" justifyContent="end" gap="size-100">
170+
<Button
171+
variant="default"
172+
size="compact"
173+
aria-label="output schema"
174+
icon={<Icon svg={<Icons.PlusOutline />} />}
175+
disabled={hasResponseFormat}
176+
onClick={() => {
177+
upsertInvocationParameterInput({
178+
instanceId: id,
179+
invocationParameterInput: {
180+
valueJson: createOpenAIResponseFormat(),
181+
invocationName: RESPONSE_FORMAT_PARAM_NAME,
182+
canonicalName: RESPONSE_FORMAT_PARAM_CANONICAL_NAME,
183+
},
184+
});
185+
}}
186+
>
187+
Output Schema
188+
</Button>
157189
<Button
158190
variant="default"
159191
aria-label="add tool"
@@ -217,6 +249,7 @@ export function PlaygroundChatTemplate(props: PlaygroundChatTemplateProps) {
217249
</Flex>
218250
</View>
219251
{hasTools ? <PlaygroundTools {...props} /> : null}
252+
{hasResponseFormat ? <PlaygroundResponseFormat {...props} /> : null}
220253
</DndContext>
221254
);
222255
}

‎app/src/pages/playground/PlaygroundOutput.tsx

+1-1
Original file line numberDiff line numberDiff line change
@@ -351,7 +351,7 @@ export function PlaygroundOutput(props: PlaygroundOutputProps) {
351351

352352
return (
353353
<Card
354-
title={<TitleWithAlphabeticIndex index={index} title="OutputContent" />}
354+
title={<TitleWithAlphabeticIndex index={index} title="Output" />}
355355
collapsible
356356
variant="compact"
357357
bodyStyle={{ padding: 0 }}

‎app/src/pages/playground/PlaygroundResponseFormat.tsx

+134
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
import React, { useCallback, useState } from "react";
2+
import { JSONSchema7 } from "json-schema";
3+
4+
import {
5+
Accordion,
6+
AccordionItem,
7+
Button,
8+
Card,
9+
Flex,
10+
Icon,
11+
Icons,
12+
Text,
13+
View,
14+
} from "@arizeai/components";
15+
16+
import { CopyToClipboardButton } from "@phoenix/components";
17+
import { JSONEditor } from "@phoenix/components/code";
18+
import { LazyEditorWrapper } from "@phoenix/components/code/LazyEditorWrapper";
19+
import { usePlaygroundContext } from "@phoenix/contexts/PlaygroundContext";
20+
import { safelyParseJSON } from "@phoenix/utils/jsonUtils";
21+
22+
import {
23+
RESPONSE_FORMAT_PARAM_CANONICAL_NAME,
24+
RESPONSE_FORMAT_PARAM_NAME,
25+
} from "./constants";
26+
import { jsonObjectSchema, openAIResponseFormatJSONSchema } from "./schemas";
27+
import { PlaygroundInstanceProps } from "./types";
28+
29+
/**
30+
* The minimum height for the editor before it is initialized.
31+
* This is to ensure that the editor is properly initialized when it is rendered outside of the viewport.
32+
*/
33+
const RESPONSE_FORMAT_EDITOR_PRE_INIT_HEIGHT = 400;
34+
35+
export function PlaygroundResponseFormat({
36+
playgroundInstanceId,
37+
}: PlaygroundInstanceProps) {
38+
const deleteInvocationParameterInput = usePlaygroundContext(
39+
(state) => state.deleteInvocationParameterInput
40+
);
41+
const instance = usePlaygroundContext((state) =>
42+
state.instances.find((i) => i.id === playgroundInstanceId)
43+
);
44+
const upsertInvocationParameterInput = usePlaygroundContext(
45+
(state) => state.upsertInvocationParameterInput
46+
);
47+
48+
if (!instance) {
49+
throw new Error(`Instance ${playgroundInstanceId} not found`);
50+
}
51+
52+
const responseFormat = instance.model.invocationParameters.find(
53+
(p) => p.invocationName === RESPONSE_FORMAT_PARAM_NAME
54+
);
55+
56+
const [responseFormatDefinition, setResponseFormatDefinition] = useState(
57+
JSON.stringify(responseFormat?.valueJson ?? {}, null, 2)
58+
);
59+
60+
const onChange = useCallback(
61+
(value: string) => {
62+
setResponseFormatDefinition(value);
63+
const { json: format } = safelyParseJSON(value);
64+
if (format == null) {
65+
return;
66+
}
67+
// Don't use data here returned by safeParse, as we want to allow for extra keys,
68+
// there is no "deepPassthrough" to allow for extra keys
69+
// at all levels of the schema, so we just use the json parsed value here,
70+
// knowing that it is valid with potentially extra keys
71+
const { success } = jsonObjectSchema.safeParse(format);
72+
if (!success) {
73+
return;
74+
}
75+
upsertInvocationParameterInput({
76+
instanceId: playgroundInstanceId,
77+
invocationParameterInput: {
78+
invocationName: RESPONSE_FORMAT_PARAM_NAME,
79+
valueJson: format,
80+
canonicalName: RESPONSE_FORMAT_PARAM_CANONICAL_NAME,
81+
},
82+
});
83+
},
84+
[playgroundInstanceId, upsertInvocationParameterInput]
85+
);
86+
87+
return (
88+
<Accordion arrowPosition="start">
89+
<AccordionItem id="response-format" title="Output Schema">
90+
<View padding="size-200">
91+
<Card
92+
variant="compact"
93+
title={
94+
<Flex direction="row" gap="size-100">
95+
<Text>Schema</Text>
96+
</Flex>
97+
}
98+
bodyStyle={{ padding: 0 }}
99+
extra={
100+
<Flex direction="row" gap="size-100">
101+
<CopyToClipboardButton text={responseFormatDefinition} />
102+
<Button
103+
aria-label="Delete Output Schema"
104+
icon={<Icon svg={<Icons.TrashOutline />} />}
105+
variant="default"
106+
size="compact"
107+
onClick={() => {
108+
deleteInvocationParameterInput({
109+
instanceId: playgroundInstanceId,
110+
invocationParameterInputInvocationName:
111+
RESPONSE_FORMAT_PARAM_NAME,
112+
});
113+
}}
114+
/>
115+
</Flex>
116+
}
117+
>
118+
<LazyEditorWrapper
119+
preInitializationMinHeight={
120+
RESPONSE_FORMAT_EDITOR_PRE_INIT_HEIGHT
121+
}
122+
>
123+
<JSONEditor
124+
value={responseFormatDefinition}
125+
onChange={onChange}
126+
jsonSchema={openAIResponseFormatJSONSchema as JSONSchema7}
127+
/>
128+
</LazyEditorWrapper>
129+
</Card>
130+
</View>
131+
</AccordionItem>
132+
</Accordion>
133+
);
134+
}

‎app/src/pages/playground/__tests__/fixtures.ts

+6
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,12 @@ export const basePlaygroundSpan: PlaygroundSpan = {
4747
invocationInputField: "value_int",
4848
invocationName: "seed",
4949
},
50+
{
51+
__typename: "JsonInvocationParameter",
52+
canonicalName: "RESPONSE_FORMAT",
53+
invocationInputField: "value_json",
54+
invocationName: "response_format",
55+
},
5056
],
5157
};
5258
export const spanAttributesWithInputMessages = {

‎app/src/pages/playground/__tests__/playgroundUtils.test.ts

+36-5
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import { LlmProviderToolCall } from "@phoenix/schemas/toolCallSchemas";
55
import {
66
_resetInstanceId,
77
_resetMessageId,
8+
createOpenAIResponseFormat,
89
PlaygroundInput,
910
PlaygroundInstance,
1011
} from "@phoenix/store";
@@ -13,6 +14,7 @@ import {
1314
INPUT_MESSAGES_PARSING_ERROR,
1415
MODEL_CONFIG_PARSING_ERROR,
1516
MODEL_CONFIG_WITH_INVOCATION_PARAMETERS_PARSING_ERROR,
17+
MODEL_CONFIG_WITH_RESPONSE_FORMAT_PARSING_ERROR,
1618
OUTPUT_MESSAGES_PARSING_ERROR,
1719
OUTPUT_VALUE_PARSING_ERROR,
1820
SPAN_ATTRIBUTES_PARSING_ERROR,
@@ -144,7 +146,6 @@ describe("transformSpanAttributesToPlaygroundInstance", () => {
144146
modelName: "gpt-4o",
145147
},
146148
template: defaultTemplate,
147-
148149
output: undefined,
149150
},
150151
parsingErrors: [
@@ -153,6 +154,7 @@ describe("transformSpanAttributesToPlaygroundInstance", () => {
153154
OUTPUT_VALUE_PARSING_ERROR,
154155
MODEL_CONFIG_PARSING_ERROR,
155156
MODEL_CONFIG_WITH_INVOCATION_PARAMETERS_PARSING_ERROR,
157+
MODEL_CONFIG_WITH_RESPONSE_FORMAT_PARSING_ERROR,
156158
],
157159
});
158160
});
@@ -455,8 +457,7 @@ describe("transformSpanAttributesToPlaygroundInstance", () => {
455457
...spanAttributesWithInputMessages.llm,
456458
// only parameters defined on the span InvocationParameter[] field are parsed
457459
// note that snake case keys are automatically converted to camel case
458-
invocation_parameters:
459-
'{"top_p": 0.5, "max_tokens": 100, "seed": 12345, "stop": ["stop", "me"]}',
460+
invocation_parameters: `{"top_p": 0.5, "max_tokens": 100, "seed": 12345, "stop": ["stop", "me"], "response_format": ${JSON.stringify(createOpenAIResponseFormat())}}`,
460461
},
461462
}),
462463
};
@@ -486,6 +487,11 @@ describe("transformSpanAttributesToPlaygroundInstance", () => {
486487
invocationName: "stop",
487488
valueStringList: ["stop", "me"],
488489
},
490+
{
491+
canonicalName: "RESPONSE_FORMAT",
492+
invocationName: "response_format",
493+
valueJson: createOpenAIResponseFormat(),
494+
},
489495
],
490496
},
491497
} satisfies PlaygroundInstance,
@@ -548,7 +554,10 @@ describe("transformSpanAttributesToPlaygroundInstance", () => {
548554
playgroundInstance: {
549555
...expectedPlaygroundInstanceWithIO,
550556
},
551-
parsingErrors: [MODEL_CONFIG_WITH_INVOCATION_PARAMETERS_PARSING_ERROR],
557+
parsingErrors: [
558+
MODEL_CONFIG_WITH_INVOCATION_PARAMETERS_PARSING_ERROR,
559+
MODEL_CONFIG_WITH_RESPONSE_FORMAT_PARSING_ERROR,
560+
],
552561
});
553562
});
554563

@@ -568,7 +577,10 @@ describe("transformSpanAttributesToPlaygroundInstance", () => {
568577
playgroundInstance: {
569578
...expectedPlaygroundInstanceWithIO,
570579
},
571-
parsingErrors: [MODEL_CONFIG_WITH_INVOCATION_PARAMETERS_PARSING_ERROR],
580+
parsingErrors: [
581+
MODEL_CONFIG_WITH_INVOCATION_PARAMETERS_PARSING_ERROR,
582+
MODEL_CONFIG_WITH_RESPONSE_FORMAT_PARSING_ERROR,
583+
],
572584
});
573585
});
574586

@@ -600,6 +612,25 @@ describe("transformSpanAttributesToPlaygroundInstance", () => {
600612
parsingErrors: [MODEL_CONFIG_WITH_INVOCATION_PARAMETERS_PARSING_ERROR],
601613
});
602614
});
615+
616+
it("should only return response format parsing errors if response format is defined AND malformed", () => {
617+
const span = {
618+
...basePlaygroundSpan,
619+
attributes: JSON.stringify({
620+
...spanAttributesWithInputMessages,
621+
llm: {
622+
...spanAttributesWithInputMessages.llm,
623+
invocation_parameters: `{"response_format": 1234}`,
624+
},
625+
}),
626+
};
627+
expect(transformSpanAttributesToPlaygroundInstance(span)).toEqual({
628+
playgroundInstance: {
629+
...expectedPlaygroundInstanceWithIO,
630+
},
631+
parsingErrors: [MODEL_CONFIG_WITH_RESPONSE_FORMAT_PARSING_ERROR],
632+
});
633+
});
603634
});
604635

605636
describe("getChatRole", () => {

‎app/src/pages/playground/constants.tsx

+11-3
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,10 @@ export const MODEL_CONFIG_PARSING_ERROR =
2828
"Unable to parse model config, expected llm.model_name to be present.";
2929
export const MODEL_CONFIG_WITH_INVOCATION_PARAMETERS_PARSING_ERROR =
3030
"Unable to parse model config, expected llm.invocation_parameters json string to be present.";
31-
// TODO(parker / apowell) - adjust this error message with anthropic support https://github.com/Arize-ai/phoenix/issues/5100
31+
export const MODEL_CONFIG_WITH_RESPONSE_FORMAT_PARSING_ERROR =
32+
"Unable to parse invocation parameters response_format, expected llm.invocation_parameters.response_format to be a well formed json object or undefined.";
3233
export const TOOLS_PARSING_ERROR =
33-
"Unable to parse tools, expected tools to be an array of valid OpenAI tools.";
34+
"Unable to parse tools, expected tools to be an array of valid tools.";
3435

3536
export const modelProviderToModelPrefixMap: Record<ModelProvider, string[]> = {
3637
AZURE_OPENAI: [],
@@ -45,9 +46,16 @@ export const TOOL_CHOICE_PARAM_CANONICAL_NAME: Extract<
4546

4647
export const TOOL_CHOICE_PARAM_NAME = "tool_choice";
4748

49+
export const RESPONSE_FORMAT_PARAM_CANONICAL_NAME: Extract<
50+
CanonicalParameterName,
51+
"RESPONSE_FORMAT"
52+
> = "RESPONSE_FORMAT";
53+
54+
export const RESPONSE_FORMAT_PARAM_NAME = "response_format";
55+
4856
/**
4957
* List of parameter canonical names to ignore in the invocation parameters form
5058
* These parameters are rendered else where on the page
5159
*/
5260
export const paramsToIgnoreInInvocationParametersForm: CanonicalParameterName[] =
53-
[TOOL_CHOICE_PARAM_CANONICAL_NAME];
61+
[TOOL_CHOICE_PARAM_CANONICAL_NAME, RESPONSE_FORMAT_PARAM_CANONICAL_NAME];

0 commit comments

Comments
 (0)