From 1309ac212d538cfe54c7863e16df8aa9cfb874a3 Mon Sep 17 00:00:00 2001
From: Federico Villa <federico.villa@epfl.ch>
Date: Tue, 5 Aug 2025 19:22:06 +0200
Subject: [PATCH 1/4] Configure external models in autogen studio via
 model_info parameter

---
 .../component-editor/fields/model-fields.tsx  | 175 +++++++++++++++++-
 1 file changed, 171 insertions(+), 4 deletions(-)

diff --git a/python/packages/autogen-studio/frontend/src/components/views/teambuilder/builder/component-editor/fields/model-fields.tsx b/python/packages/autogen-studio/frontend/src/components/views/teambuilder/builder/component-editor/fields/model-fields.tsx
index 4126f79585cd..ad75dd8e9142 100644
--- a/python/packages/autogen-studio/frontend/src/components/views/teambuilder/builder/component-editor/fields/model-fields.tsx
+++ b/python/packages/autogen-studio/frontend/src/components/views/teambuilder/builder/component-editor/fields/model-fields.tsx
@@ -1,5 +1,5 @@
 import React, { useCallback } from "react";
-import { Input, InputNumber, Select, Tooltip, Collapse } from "antd";
+import { Input, InputNumber, Select, Tooltip, Collapse, Switch } from "antd";
 import TextArea from "antd/es/input/TextArea";
 import { HelpCircle, Settings, User, Wrench } from "lucide-react";
 import {
@@ -56,7 +56,8 @@ type FieldName =
   | "azure_ad_token"
   | "tools"
   | "tool_choice"
-  | "metadata";
+  | "metadata"
+  | "model_info";
 
 // Define the field specification type
 interface FieldSpec {
@@ -70,7 +71,127 @@ interface FieldSpec {
   };
 }
 
-// Field specifications for all possible model parameters
+const ModelInfoEditor: React.FC<{
+  value: any;
+  onChange: (value: any) => void;
+}> = ({ value, onChange }) => {
+  const modelInfo = value || {
+    vision: false,
+    function_calling: false,
+    json_output: false,
+    structured_output: false,
+    family: "unknown",
+    multiple_system_messages: false,
+  };
+
+  const updateField = (field: string, newValue: any) => {
+    onChange({
+      ...modelInfo,
+      [field]: newValue,
+    });
+  };
+
+  return (
+    <div className="space-y-4 p-4 border rounded-lg">
+      <div className="grid grid-cols-2 gap-4">
+        <label className="flex items-center gap-2">
+          <Switch
+            checked={modelInfo.vision}
+            onChange={(checked) => updateField("vision", checked)}
+          />
+          <span>Vision Support</span>
+        </label>
+
+        <label className="flex items-center gap-2">
+          <Switch
+            checked={modelInfo.function_calling}
+            onChange={(checked) => updateField("function_calling", checked)}
+          />
+          <span>Function Calling</span>
+        </label>
+
+        <label className="flex items-center gap-2">
+          <Switch
+            checked={modelInfo.json_output}
+            onChange={(checked) => updateField("json_output", checked)}
+          />
+          <span>JSON Output</span>
+        </label>
+
+        <label className="flex items-center gap-2">
+          <Switch
+            checked={modelInfo.structured_output}
+            onChange={(checked) => updateField("structured_output", checked)}
+          />
+          <span>Structured Output</span>
+        </label>
+
+        <label className="flex items-center gap-2">
+          <Switch
+            checked={modelInfo.multiple_system_messages}
+            onChange={(checked) => updateField("multiple_system_messages", checked)}
+          />
+          <span>Multiple System Messages</span>
+        </label>
+      </div>
+
+      <label className="block">
+        <span className="text-sm font-medium">Model Family</span>
+        <Select
+          value={modelInfo.family}
+          onChange={(value) => updateField("family", value)}
+          className="w-full mt-1"
+          options={[
+            { label: "GPT-4.1", value: "gpt-41" },
+            { label: "GPT-4.5", value: "gpt-45" },
+            { label: "GPT-4o", value: "gpt-4o" },
+            { label: "GPT-4", value: "gpt-4" },
+            { label: "GPT-3.5", value: "gpt-35" },
+            { label: "o1", value: "o1" },
+            { label: "o3", value: "o3" },
+            { label: "o4", value: "o4" },
+            { label: "r1", value: "r1" },
+            
+            { label: "Gemini 1.5 Flash", value: "gemini-1.5-flash" },
+            { label: "Gemini 1.5 Pro", value: "gemini-1.5-pro" },
+            { label: "Gemini 2.0 Flash", value: "gemini-2.0-flash" },
+            { label: "Gemini 2.5 Pro", value: "gemini-2.5-pro" },
+            { label: "Gemini 2.5 Flash", value: "gemini-2.5-flash" },
+            
+            { label: "Claude 3 Haiku", value: "claude-3-haiku" },
+            { label: "Claude 3 Sonnet", value: "claude-3-sonnet" },
+            { label: "Claude 3 Opus", value: "claude-3-opus" },
+            { label: "Claude 3.5 Haiku", value: "claude-3-5-haiku" },
+            { label: "Claude 3.5 Sonnet", value: "claude-3-5-sonnet" },
+            { label: "Claude 3.7 Sonnet", value: "claude-3-7-sonnet" },
+            { label: "Claude 4 Opus", value: "claude-4-opus" },
+            { label: "Claude 4 Sonnet", value: "claude-4-sonnet" },
+            
+            { label: "Llama 3.3 8B", value: "llama-3.3-8b" },
+            { label: "Llama 3.3 70B", value: "llama-3.3-70b" },
+            { label: "Llama 4 Scout", value: "llama-4-scout" },
+            { label: "Llama 4 Maverick", value: "llama-4-maverick" },
+            
+            { label: "Codestral", value: "codestral" },
+            { label: "Open Codestral Mamba", value: "open-codestral-mamba" },
+            { label: "Mistral", value: "mistral" },
+            { label: "Ministral", value: "ministral" },
+            { label: "Pixtral", value: "pixtral" },
+            
+            { label: "Unknown", value: "unknown" },
+          ]}
+          showSearch
+          filterOption={(input, option) =>
+            (option?.label ?? '').toLowerCase().includes(input.toLowerCase())
+          }
+          placeholder="Select model family"
+        />
+      </label>
+    </div>
+  );
+};
+
+// Field specifications for each possible model parameters
 const fieldSpecs: Record<FieldName, FieldSpec> = {
   // Common fields
   temperature: {
@@ -258,6 +379,38 @@ const fieldSpecs: Record<FieldName, FieldSpec> = {
       },
     },
   },
+  model_info: {
+    label: "Model Information",
+    tooltip: "Model capabilities and features",
+    component: ModelInfoEditor,
+    props: {},
+    transform: {
+      fromConfig: (value: any) => {
+        const defaultModelInfo = {
+          vision: false,
+          function_calling: true,
+          json_output: false,
+          structured_output: false,
+          family: "unknown",
+          multiple_system_messages: false,
+        };
+        
+        return value ? { ...defaultModelInfo, ...value } : defaultModelInfo;
+      },
+      toConfig: (value: any) => {
+        const defaultModelInfo = {
+          vision: false,
+          function_calling: true,
+          json_output: false,
+          structured_output: false,
+          family: "unknown",
+          multiple_system_messages: false,
+        };
+        
+        return { ...defaultModelInfo, ...value };
+      },
+    },
+  },
 };
 
 // Define provider field mapping type
@@ -266,6 +419,7 @@ type ProviderType = "openai" | "azure" | "anthropic";
 interface ProviderFields {
   modelConfig: FieldName[];
   modelParams: FieldName[];
+  modelInfo: FieldName[];
 }
 
 // Define which fields each provider uses
@@ -287,6 +441,7 @@ const providerFields: Record<ProviderType, ProviderFields> = {
       "presence_penalty",
       "stop",
     ],
+    modelInfo: ["model_info"],
   },
   azure: {
     modelConfig: [
@@ -307,6 +462,7 @@ const providerFields: Record<ProviderType, ProviderFields> = {
       "presence_penalty",
       "stop",
     ],
+    modelInfo: ["model_info"],
   },
   anthropic: {
     modelConfig: ["model", "api_key", "base_url", "timeout", "max_retries"],
@@ -320,6 +476,7 @@ const providerFields: Record<ProviderType, ProviderFields> = {
       "tool_choice",
       "metadata",
     ],
+    modelInfo: ["model_info"],
   },
 };
 
@@ -412,7 +569,7 @@ export const ModelFields: React.FC<ModelFieldsProps> = ({
 
   return (
     <Collapse
-      defaultActiveKey={["details", "configuration", "parameters"]}
+      defaultActiveKey={["details", "configuration", "parameters", "model_info"]}
       className="border-0"
       expandIconPosition="end"
       items={[
@@ -479,6 +636,16 @@ export const ModelFields: React.FC<ModelFieldsProps> = ({
           ),
           children: renderFieldGroup(providerFields[providerType].modelParams),
         },
+        {
+          key: "model_info",
+          label: (
+            <div className="flex items-center gap-2">
+              <HelpCircle className="w-4 h-4 text-purple-500" />
+              <span className="font-medium">Model Information</span>
+            </div>
+          ),
+          children: renderFieldGroup(providerFields[providerType].modelInfo),
+        },
         // Only render tool configuration if it's an Anthropic model and has tools
         ...(providerType === "anthropic" &&
         (component.config as any).tool_choice === "custom"

From 9b93c53e1030c9da7df5d5ed162eaa913f4b744f Mon Sep 17 00:00:00 2001
From: Federico Villa <federico.villa@epfl.ch>
Date: Wed, 6 Aug 2025 19:21:26 +0200
Subject: [PATCH 2/4] Included model info tooltips, fixed Vision Support button
 to not be triggered when the general Model Information div is clicked and
 added string value for Model Family

---
 .../component-editor/fields/model-fields.tsx  | 170 +++++++++---------
 1 file changed, 89 insertions(+), 81 deletions(-)

diff --git a/python/packages/autogen-studio/frontend/src/components/views/teambuilder/builder/component-editor/fields/model-fields.tsx b/python/packages/autogen-studio/frontend/src/components/views/teambuilder/builder/component-editor/fields/model-fields.tsx
index ad75dd8e9142..85326edf9805 100644
--- a/python/packages/autogen-studio/frontend/src/components/views/teambuilder/builder/component-editor/fields/model-fields.tsx
+++ b/python/packages/autogen-studio/frontend/src/components/views/teambuilder/builder/component-editor/fields/model-fields.tsx
@@ -1,5 +1,5 @@
 import React, { useCallback } from "react";
-import { Input, InputNumber, Select, Tooltip, Collapse, Switch } from "antd";
+import { Input, InputNumber, Select, Tooltip, Collapse, Switch, AutoComplete } from "antd";
 import TextArea from "antd/es/input/TextArea";
 import { HelpCircle, Settings, User, Wrench } from "lucide-react";
 import {
@@ -23,7 +23,7 @@ const InputWithTooltip: React.FC<{
   tooltip: string;
   children: React.ReactNode;
 }> = ({ label, tooltip, children }) => (
-  <label className="block">
+  <div className="block">
     <div className="flex items-center gap-2 mb-1">
       <span className="text-sm font-medium text-primary">{label}</span>
       <Tooltip title={tooltip}>
@@ -31,7 +31,7 @@ const InputWithTooltip: React.FC<{
       </Tooltip>
     </div>
     {children}
-  </label>
+  </div>
 );
 
 // Define possible field names to ensure type safety
@@ -71,18 +71,20 @@ interface FieldSpec {
   };
 }
 
+const defaultModelInfo = {
+  vision: false,
+  function_calling: false,
+  json_output: false,
+  structured_output: false,
+  family: "",
+  multiple_system_messages: false,
+};
+
 const ModelInfoEditor: React.FC<{
   value: any;
   onChange: (value: any) => void;
 }> = ({ value, onChange }) => {
-  const modelInfo = value || {
-    vision: false,
-    function_calling: false,
-    json_output: false,
-    structured_output: false,
-    family: "unknown",
-    multiple_system_messages: false,
-  };
+  const modelInfo = value || defaultModelInfo;
 
   const updateField = (field: string, newValue: any) => {
     onChange({
@@ -94,53 +96,81 @@ const ModelInfoEditor: React.FC<{
   return (
     <div className="space-y-4 p-4 border rounded-lg">
       <div className="grid grid-cols-2 gap-4">
-        <label className="flex items-center gap-2">
-          <Switch
-            checked={modelInfo.vision}
-            onChange={(checked) => updateField("vision", checked)}
-          />
-          <span>Vision Support</span>
-        </label>
-
-        <label className="flex items-center gap-2">
-          <Switch
-            checked={modelInfo.function_calling}
-            onChange={(checked) => updateField("function_calling", checked)}
-          />
-          <span>Function Calling</span>
-        </label>
-
-        <label className="flex items-center gap-2">
-          <Switch
-            checked={modelInfo.json_output}
-            onChange={(checked) => updateField("json_output", checked)}
-          />
-          <span>JSON Output</span>
-        </label>
-
-        <label className="flex items-center gap-2">
-          <Switch
-            checked={modelInfo.structured_output}
-            onChange={(checked) => updateField("structured_output", checked)}
-          />
-          <span>Structured Output</span>
-        </label>
-
-        <label className="flex items-center gap-2">
-          <Switch
-            checked={modelInfo.multiple_system_messages}
-            onChange={(checked) => updateField("multiple_system_messages", checked)}
-          />
-          <span>Multiple System Messages</span>
-        </label>
+        <InputWithTooltip
+          label="Vision Support"
+          tooltip="Enable vision support (image input capability)."
+        >
+          <div className="w-16 pointer-events-none">
+            <Switch
+              checked={modelInfo.vision}
+              onChange={(checked) => updateField("vision", checked)}
+              className="pointer-events-auto"
+            />
+          </div>
+        </InputWithTooltip>
+
+        <InputWithTooltip
+          label="Function Calling"
+          tooltip="Enable function calling feature for external integrations."
+        >
+          <div className="w-16 pointer-events-none">
+            <Switch
+              checked={modelInfo.function_calling}
+              onChange={(checked) => updateField("function_calling", checked)}
+              className="pointer-events-auto"
+            />
+          </div>
+        </InputWithTooltip>
+
+        <InputWithTooltip
+          label="JSON Output"
+          tooltip="Enable JSON output mode (distinct from structured output)."
+        >
+          <div className="w-16 pointer-events-none">
+            <Switch
+              checked={modelInfo.json_output}
+              onChange={(checked) => updateField("json_output", checked)}
+              className="pointer-events-auto"
+            />
+          </div>
+        </InputWithTooltip>
+
+        <InputWithTooltip
+          label="Structured Output"
+          tooltip="Enable structured output mode (distinct from JSON output)."
+        >
+          <div className="w-16 pointer-events-none">
+            <Switch
+              checked={modelInfo.structured_output}
+              onChange={(checked) => updateField("structured_output", checked)}
+              className="pointer-events-auto"
+            />
+          </div>
+        </InputWithTooltip>
+
+        <InputWithTooltip
+          label="Multiple System Messages"
+          tooltip="Enable support for multiple, non-consecutive system messages."
+        >
+          <div className="w-16 pointer-events-none">
+            <Switch
+              checked={modelInfo.multiple_system_messages}
+              onChange={(checked) => updateField("multiple_system_messages", checked)}
+              className="pointer-events-auto"
+            />
+          </div>
+        </InputWithTooltip>
       </div>
 
-      <label className="block">
-        <span className="text-sm font-medium">Model Family</span>
-        <Select
+      <InputWithTooltip
+        label="Model Family"
+        tooltip="Model family should be one of the constants from ModelFamily or a string for unknown families."
+      >
+        <AutoComplete
           value={modelInfo.family}
           onChange={(value) => updateField("family", value)}
           className="w-full mt-1"
+          placeholder="Select or enter model family"
           options={[
             { label: "GPT-4.1", value: "gpt-41" },
             { label: "GPT-4.5", value: "gpt-45" },
@@ -180,13 +210,12 @@ const ModelInfoEditor: React.FC<{
             
             { label: "Unknown", value: "unknown" },
           ]}
-          showSearch
           filterOption={(input, option) =>
-            (option?.label ?? '').toLowerCase().includes(input.toLowerCase())
+            (option?.label ?? "").toLowerCase().includes(input.toLowerCase())
           }
-          placeholder="Select model family"
+          allowClear
         />
-      </label>
+      </InputWithTooltip>
     </div>
   );
 };
@@ -385,30 +414,9 @@ const fieldSpecs: Record<FieldName, FieldSpec> = {
     component: ModelInfoEditor,
     props: {},
     transform: {
-      fromConfig: (value: any) => {
-        const defaultModelInfo = {
-          vision: false,
-          function_calling: true,
-          json_output: false,
-          structured_output: false,
-          family: "unknown",
-          multiple_system_messages: false,
-        };
-        
-        return value ? { ...defaultModelInfo, ...value } : defaultModelInfo;
-      },
-      toConfig: (value: any) => {
-        const defaultModelInfo = {
-          vision: false,
-          function_calling: true,
-          json_output: false,
-          structured_output: false,
-          family: "unknown",
-          multiple_system_messages: false,
-        };
-        
-        return { ...defaultModelInfo, ...value };
-      },
+      fromConfig: (value: any) => 
+        value ? { ...defaultModelInfo, ...value } : defaultModelInfo,
+      toConfig: (value: any) => ({ ...defaultModelInfo, ...value }),
     },
   },
 };

From 263a72e1fe26707058a7de529253204d5a1b984e Mon Sep 17 00:00:00 2001
From: Federico Villa <federico.villa@epfl.ch>
Date: Wed, 6 Aug 2025 22:15:06 +0200
Subject: [PATCH 3/4] Added template for OpenAI-compatible model

---
 .../components/types/component-templates.ts   | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/python/packages/autogen-studio/frontend/src/components/types/component-templates.ts b/python/packages/autogen-studio/frontend/src/components/types/component-templates.ts
index 4372ad20bd69..3ca2b5728c1c 100644
--- a/python/packages/autogen-studio/frontend/src/components/types/component-templates.ts
+++ b/python/packages/autogen-studio/frontend/src/components/types/component-templates.ts
@@ -226,6 +226,27 @@ export const MODEL_TEMPLATES: ComponentTemplate<ModelConfig>[] = [
       temperature: 0.7,
     } as AnthropicClientConfig,
   },
+  {
+    id: "external-custom-model",
+    label: "External Model",
+    description: "Template for a model using an OpenAI-compatible endpoint",
+    provider: PROVIDERS.OPENAI,
+    component_type: "model",
+    version: 1,
+    component_version: 1,
+    config: {
+      model: "your-model-name",
+      base_url: "https://example.com/",
+      model_info: {
+        vision: false,
+        function_calling: true,
+        json_output: false,
+        structured_output: false,
+        family: "unknown",
+        multiple_system_messages: false,
+      },
+    } as OpenAIClientConfig,
+  },
 ];
 
 // Tool Templates

From 49d40ed913f4cb439a6f3e4132c4b144190aabca Mon Sep 17 00:00:00 2001
From: Federico Villa <61495946+federicovilla55@users.noreply.github.com>
Date: Fri, 12 Sep 2025 21:31:44 +0200
Subject: [PATCH 4/4] Update
 python/packages/autogen-studio/frontend/src/components/views/teambuilder/builder/component-editor/fields/model-fields.tsx

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
---
 .../component-editor/fields/model-fields.tsx  | 60 ++++++++-----------
 1 file changed, 25 insertions(+), 35 deletions(-)

diff --git a/python/packages/autogen-studio/frontend/src/components/views/teambuilder/builder/component-editor/fields/model-fields.tsx b/python/packages/autogen-studio/frontend/src/components/views/teambuilder/builder/component-editor/fields/model-fields.tsx
index 85326edf9805..d5e1ec21b3fa 100644
--- a/python/packages/autogen-studio/frontend/src/components/views/teambuilder/builder/component-editor/fields/model-fields.tsx
+++ b/python/packages/autogen-studio/frontend/src/components/views/teambuilder/builder/component-editor/fields/model-fields.tsx
@@ -100,65 +100,55 @@ const ModelInfoEditor: React.FC<{
           label="Vision Support"
           tooltip="Enable vision support (image input capability)."
         >
-          <div className="w-16 pointer-events-none">
-            <Switch
-              checked={modelInfo.vision}
-              onChange={(checked) => updateField("vision", checked)}
-              className="pointer-events-auto"
-            />
-          </div>
+          <Switch
+            checked={modelInfo.vision}
+            onChange={(checked) => updateField("vision", checked)}
+            className="w-16"
+          />
         </InputWithTooltip>
 
         <InputWithTooltip
           label="Function Calling"
           tooltip="Enable function calling feature for external integrations."
         >
-          <div className="w-16 pointer-events-none">
-            <Switch
-              checked={modelInfo.function_calling}
-              onChange={(checked) => updateField("function_calling", checked)}
-              className="pointer-events-auto"
-            />
-          </div>
+          <Switch
+            checked={modelInfo.function_calling}
+            onChange={(checked) => updateField("function_calling", checked)}
+            className="w-16"
+          />
         </InputWithTooltip>
 
         <InputWithTooltip
           label="JSON Output"
           tooltip="Enable JSON output mode (distinct from structured output)."
         >
-          <div className="w-16 pointer-events-none">
-            <Switch
-              checked={modelInfo.json_output}
-              onChange={(checked) => updateField("json_output", checked)}
-              className="pointer-events-auto"
-            />
-          </div>
+          <Switch
+            checked={modelInfo.json_output}
+            onChange={(checked) => updateField("json_output", checked)}
+            className="w-16"
+          />
         </InputWithTooltip>
 
         <InputWithTooltip
           label="Structured Output"
           tooltip="Enable structured output mode (distinct from JSON output)."
         >
-          <div className="w-16 pointer-events-none">
-            <Switch
-              checked={modelInfo.structured_output}
-              onChange={(checked) => updateField("structured_output", checked)}
-              className="pointer-events-auto"
-            />
-          </div>
+          <Switch
+            checked={modelInfo.structured_output}
+            onChange={(checked) => updateField("structured_output", checked)}
+            className="w-16"
+          />
         </InputWithTooltip>
 
         <InputWithTooltip
           label="Multiple System Messages"
           tooltip="Enable support for multiple, non-consecutive system messages."
         >
-          <div className="w-16 pointer-events-none">
-            <Switch
-              checked={modelInfo.multiple_system_messages}
-              onChange={(checked) => updateField("multiple_system_messages", checked)}
-              className="pointer-events-auto"
-            />
-          </div>
+          <Switch
+            checked={modelInfo.multiple_system_messages}
+            onChange={(checked) => updateField("multiple_system_messages", checked)}
+            className="w-16"
+          />
         </InputWithTooltip>
       </div>