refactor(prompts): unify template format and language (#6403)

* refactor(prompts): unify template format

* migrate the frontend

* build openapi

* refactor python client

* refactor JS

* fix db

* run ruff

* fix tests

* cleanup

* more cleanup

* run ci on prompts

* ruff again

* more fixes to python tests

* restore PromptTemplateFormat

* ruff

* build openapi

* build client

* cleanup

* fix tests

* cleanup

Author: mikeldking

.github/workflows/typescript-packages-CI.yml

@@ -2,7 +2,7 @@ name: Typescript Packages CI
 
 on:
   push:
-    branches: [main, js]
+    branches: [main, prompts]
   pull_request:
     paths:
       - "js/**"

app/schema.graphql

@@ -109,7 +109,7 @@ input ChatCompletionInput {
   invocationParameters: [InvocationParameterInput!]! = []
   tools: [JSON!]
   apiKey: String = null
-  template: TemplateOptions
+  template: PromptTemplateOptions
   promptName: Identifier = null
 }
 
@@ -154,7 +154,7 @@ input ChatCompletionOverDatasetInput {
   invocationParameters: [InvocationParameterInput!]! = []
   tools: [JSON!]
   apiKey: String = null
-  templateLanguage: TemplateLanguage!
+  templateFormat: PromptTemplateFormat! = MUSTACHE
   datasetId: GlobalID!
   datasetVersionId: GlobalID = null
   experimentName: String = null
@@ -1573,10 +1573,15 @@ union PromptTemplate = PromptStringTemplate | PromptChatTemplate
 
 enum PromptTemplateFormat {
   MUSTACHE
-  FSTRING
+  F_STRING
   NONE
 }
 
+input PromptTemplateOptions {
+  variables: JSON!
+  format: PromptTemplateFormat!
+}
+
 enum PromptTemplateType {
   STRING
   CHAT
@@ -1979,17 +1984,6 @@ type SystemApiKey implements ApiKey & Node {
   id: GlobalID!
 }
 
-enum TemplateLanguage {
-  NONE
-  MUSTACHE
-  F_STRING
-}
-
-input TemplateOptions {
-  variables: JSON!
-  language: TemplateLanguage!
-}
-
 type TextChunk implements ChatCompletionSubscriptionPayload {
   datasetExampleId: GlobalID
   content: String!

app/src/components/templateEditor/TemplateEditor.tsx

@@ -15,11 +15,11 @@ import { assertUnreachable } from "@phoenix/typeUtils";
 
 import { FStringTemplating } from "./language/fString";
 import { MustacheLikeTemplating } from "./language/mustacheLike";
-import { TemplateLanguages } from "./constants";
-import { TemplateLanguage } from "./types";
+import { TemplateFormats } from "./constants";
+import { TemplateFormat } from "./types";
 
 type TemplateEditorProps = Omit<ReactCodeMirrorProps, "value"> & {
-  templateLanguage: TemplateLanguage;
+  templateFormat: TemplateFormat;
   defaultValue: string;
 };
 
@@ -49,7 +49,7 @@ const baseExtensions = [
  * cursor position when value is updated.
  */
 export const TemplateEditor = ({
-  templateLanguage,
+  templateFormat,
   defaultValue,
   readOnly,
   ...props
@@ -59,20 +59,20 @@ export const TemplateEditor = ({
   const codeMirrorTheme = theme === "light" ? githubLight : nord;
   const extensions = useMemo(() => {
     const ext: TemplateEditorProps["extensions"] = [...baseExtensions];
-    switch (templateLanguage) {
-      case TemplateLanguages.FString:
+    switch (templateFormat) {
+      case TemplateFormats.FString:
         ext.push(FStringTemplating());
         break;
-      case TemplateLanguages.Mustache:
+      case TemplateFormats.Mustache:
         ext.push(MustacheLikeTemplating());
         break;
-      case TemplateLanguages.NONE:
+      case TemplateFormats.NONE:
         break;
       default:
-        assertUnreachable(templateLanguage);
+        assertUnreachable(templateFormat);
     }
     return ext;
-  }, [templateLanguage]);
+  }, [templateFormat]);
 
   useEffect(() => {
     if (readOnly) {

app/src/components/templateEditor/constants.ts

@@ -1,15 +1,15 @@
 /**
- * Enum for the different template languages supported by the template editor
+ * Enum for the different template formats supported by the template editor
  *
  * - FString: `variables look like {variable}`
  * - Mustache: `variables look like {{variable}}`
  *
  * @example
  * ```tsx
- * <TemplateEditor language={TemplateLanguages.Mustache} />
+ * <TemplateEditor format={TemplateFormats.Mustache} />
  * ```
  */
-export const TemplateLanguages = {
+export const TemplateFormats = {
   NONE: "NONE", // No templating
   FString: "F_STRING", // {variable}
   Mustache: "MUSTACHE", // {{variable}}

app/src/components/templateEditor/templateEditorUtils.ts

@@ -5,8 +5,8 @@ import {
   extractVariablesFromMustacheLike,
   formatMustacheLike,
 } from "./language/mustacheLike";
-import { TemplateLanguages } from "./constants";
-import { TemplateLanguage } from "./types";
+import { TemplateFormats } from "./constants";
+import { TemplateFormat } from "./types";
 
 /**
  * A function that formats a template with the given variables
@@ -24,35 +24,35 @@ export type ExtractVariablesFn = (template: string) => string[];
 /**
  * Get an object of isomorphic functions for processing templates of the given language
  *
- * @param templateLanguage - The language of the template to process
+ * @param templateFormat - The format of the template to process
  *
  * @returns An object containing the `format` and `extractVariables` functions.
  * These functions share the same signature despite the different underlying
  * templating languages.
  */
-export const getTemplateLanguageUtils = (
-  templateLanguage: TemplateLanguage
+export const getTemplateFormatUtils = (
+  templateFormat: TemplateFormat
 ): {
   format: FormatFn;
   extractVariables: ExtractVariablesFn;
 } => {
-  switch (templateLanguage) {
-    case TemplateLanguages.FString:
+  switch (templateFormat) {
+    case TemplateFormats.FString:
       return {
         format: formatFString,
         extractVariables: extractVariablesFromFString,
       };
-    case TemplateLanguages.Mustache:
+    case TemplateFormats.Mustache:
       return {
         format: formatMustacheLike,
         extractVariables: extractVariablesFromMustacheLike,
       };
-    case TemplateLanguages.NONE:
+    case TemplateFormats.NONE:
       return {
         format: ({ text }) => text,
         extractVariables: () => [],
       };
     default:
-      assertUnreachable(templateLanguage);
+      assertUnreachable(templateFormat);
   }
 };

app/src/components/templateEditor/types.ts

@@ -1,11 +1,11 @@
-import { TemplateLanguages } from "./constants";
+import { TemplateFormats } from "./constants";
 
-export type TemplateLanguage =
-  (typeof TemplateLanguages)[keyof typeof TemplateLanguages];
+export type TemplateFormat =
+  (typeof TemplateFormats)[keyof typeof TemplateFormats];
 
 /**
- * Type guard for the TemplateLanguage type
+ * Type guard for the TemplateFormat type
  */
-export function isTemplateLanguage(v: string): v is TemplateLanguage {
-  return Object.values(TemplateLanguages).includes(v as TemplateLanguage);
+export function isTemplateFormat(v: string): v is TemplateFormat {
+  return Object.values(TemplateFormats).includes(v as TemplateFormat);
 }

app/src/pages/playground/Playground.tsx

@@ -20,7 +20,7 @@ import {
 import { ConfirmNavigationDialog } from "@phoenix/components/ConfirmNavigation";
 import { resizeHandleCSS } from "@phoenix/components/resize";
 import { StopPropagation } from "@phoenix/components/StopPropagation";
-import { TemplateLanguages } from "@phoenix/components/templateEditor/constants";
+import { TemplateFormats } from "@phoenix/components/templateEditor/constants";
 import {
   PlaygroundProvider,
   usePlaygroundContext,
@@ -40,7 +40,7 @@ import { PlaygroundOutput } from "./PlaygroundOutput";
 import { PlaygroundRunButton } from "./PlaygroundRunButton";
 import { PlaygroundStreamToggle } from "./PlaygroundStreamToggle";
 import { PlaygroundTemplate } from "./PlaygroundTemplate";
-import { TemplateLanguageRadioGroup } from "./TemplateLanguageRadioGroup";
+import { TemplateFormatRadioGroup } from "./TemplateFormatRadioGroup";
 
 const playgroundWrapCSS = css`
   display: flex;
@@ -209,9 +209,7 @@ const DEFAULT_EXPANDED_PARAMS = ["input", "output"];
 
 function PlaygroundContent() {
   const instances = usePlaygroundContext((state) => state.instances);
-  const templateLanguage = usePlaygroundContext(
-    (state) => state.templateLanguage
-  );
+  const templateFormat = usePlaygroundContext((state) => state.templateFormat);
   const [searchParams] = useSearchParams();
   const datasetId = searchParams.get("datasetId");
   const isDatasetMode = datasetId != null;
@@ -272,7 +270,7 @@ function PlaygroundContent() {
                   Prompts
                   <StopPropagation>
                     <Flex direction="row" gap="size-100" alignItems="center">
-                      <TemplateLanguageRadioGroup size="M" />
+                      <TemplateFormatRadioGroup size="M" />
                       <AddPromptButton />
                     </Flex>
                   </StopPropagation>
@@ -307,7 +305,7 @@ function PlaygroundContent() {
           ) : (
             <div css={playgroundInputOutputPanelContentCSS}>
               <DisclosureGroup defaultExpandedKeys={DEFAULT_EXPANDED_PARAMS}>
-                {templateLanguage !== TemplateLanguages.NONE ? (
+                {templateFormat !== TemplateFormats.NONE ? (
                   <Disclosure id="input" size="L">
                     <DisclosureTrigger arrowPosition="start">
                       Inputs

app/src/pages/playground/PlaygroundChatTemplate.tsx

@@ -39,7 +39,7 @@ import {
   TemplateEditor,
   TemplateEditorWrap,
 } from "@phoenix/components/templateEditor";
-import { TemplateLanguage } from "@phoenix/components/templateEditor/types";
+import { TemplateFormat } from "@phoenix/components/templateEditor/types";
 import { usePlaygroundContext } from "@phoenix/contexts/PlaygroundContext";
 import { useChatMessageStyles } from "@phoenix/hooks/useChatMessageStyles";
 import { ChatMessage, PlaygroundState } from "@phoenix/store";
@@ -85,9 +85,7 @@ interface PlaygroundChatTemplateProps extends PlaygroundInstanceProps {}
 export function PlaygroundChatTemplate(props: PlaygroundChatTemplateProps) {
   const id = props.playgroundInstanceId;
 
-  const templateLanguage = usePlaygroundContext(
-    (state) => state.templateLanguage
-  );
+  const templateFormat = usePlaygroundContext((state) => state.templateFormat);
   const updateInstance = usePlaygroundContext((state) => state.updateInstance);
   const instanceSelector = useMemo(() => selectPlaygroundInstance(id), [id]);
   const playgroundInstance = usePlaygroundContext(instanceSelector);
@@ -156,7 +154,7 @@ export function PlaygroundChatTemplate(props: PlaygroundChatTemplateProps) {
             return (
               <SortableMessageItem
                 playgroundInstanceId={id}
-                templateLanguage={templateLanguage}
+                templateFormat={templateFormat}
                 key={messageId}
                 messageId={messageId}
               />
@@ -191,13 +189,13 @@ export function PlaygroundChatTemplate(props: PlaygroundChatTemplateProps) {
 function MessageEditor({
   message,
   updateMessage,
-  templateLanguage,
+  templateFormat,
   playgroundInstanceId,
   messageMode,
 }: {
   playgroundInstanceId: number;
   message: ChatMessage;
-  templateLanguage: TemplateLanguage;
+  templateFormat: TemplateFormat;
   updateMessage: (patch: Partial<ChatMessage>) => void;
   messageMode: MessageMode;
 }) {
@@ -277,7 +275,7 @@ function MessageEditor({
         height="100%"
         defaultValue={message.content || ""}
         aria-label="Message content"
-        templateLanguage={templateLanguage}
+        templateFormat={templateFormat}
         onChange={onChange}
         placeholder={
           message.role === "system"
@@ -293,12 +291,12 @@ function MessageEditor({
 
 function SortableMessageItem({
   playgroundInstanceId,
-  templateLanguage,
+  templateFormat,
   messageId,
 }: PropsWithChildren<{
   playgroundInstanceId: number;
   messageId: number;
-  templateLanguage: TemplateLanguage;
+  templateFormat: TemplateFormat;
 }>) {
   const updateMessage = usePlaygroundContext((state) => state.updateMessage);
   const deleteMessage = usePlaygroundContext((state) => state.deleteMessage);
@@ -482,7 +480,7 @@ function SortableMessageItem({
             message={message}
             messageMode={aiMessageMode}
             playgroundInstanceId={playgroundInstanceId}
-            templateLanguage={templateLanguage}
+            templateFormat={templateFormat}
             updateMessage={onMessageUpdate}
           />
         </div>

app/src/pages/playground/PlaygroundDatasetExamplesTable.tsx

@@ -421,9 +421,7 @@ export function PlaygroundDatasetExamplesTable({
   const allInstanceMessages = usePlaygroundContext(
     (state) => state.allInstanceMessages
   );
-  const templateLanguage = usePlaygroundContext(
-    (state) => state.templateLanguage
-  );
+  const templateFormat = usePlaygroundContext((state) => state.templateFormat);
 
   const updateInstance = usePlaygroundContext((state) => state.updateInstance);
   const updateExampleData = usePlaygroundDatasetExamplesTableContext(
@@ -766,7 +764,7 @@ export function PlaygroundDatasetExamplesTable({
       );
       const instanceVariables = extractVariablesFromInstance({
         instance: enrichedInstance,
-        templateLanguage,
+        templateFormat,
       });
       return {
         id: `instance-${instance.id}`,
@@ -792,7 +790,7 @@ export function PlaygroundDatasetExamplesTable({
         size: 500,
       };
     });
-  }, [hasSomeRunIds, instances, templateLanguage, allInstanceMessages]);
+  }, [hasSomeRunIds, instances, templateFormat, allInstanceMessages]);
 
   const columns: ColumnDef<TableRow>[] = [
     {

app/src/pages/playground/PlaygroundInput.tsx

@@ -1,6 +1,7 @@
 import React from "react";
 
 import { Flex, Text, View } from "@phoenix/components";
+import { TemplateFormats } from "@phoenix/components/templateEditor/constants";
 import { usePlaygroundContext } from "@phoenix/contexts/PlaygroundContext";
 import { assertUnreachable } from "@phoenix/typeUtils";
 
@@ -12,25 +13,23 @@ export function PlaygroundInput() {
   const setVariableValue = usePlaygroundContext(
     (state) => state.setVariableValue
   );
-  const templateLanguage = usePlaygroundContext(
-    (state) => state.templateLanguage
-  );
+  const templateFormat = usePlaygroundContext((state) => state.templateFormat);
   if (variableKeys.length === 0) {
     let templateSyntax = "";
-    switch (templateLanguage) {
-      case "F_STRING": {
+    switch (templateFormat) {
+      case TemplateFormats.FString: {
         templateSyntax = "{input name}";
         break;
       }
-      case "MUSTACHE": {
+      case TemplateFormats.Mustache: {
         templateSyntax = "{{input name}}";
         break;
       }
-      case "NONE": {
+      case TemplateFormats.NONE: {
         return null;
       }
       default:
-        assertUnreachable(templateLanguage);
+        assertUnreachable(templateFormat);
     }
     return (
       <View padding="size-100">