Adding options for streaming and temperature for models

tristandostaler · Sep 24, 2024 · 01bba3f · 01bba3f
1 parent 2c853d5
commit 01bba3f
Show file tree

Hide file tree

Showing 2 changed files with 19 additions and 2 deletions.
diff --git a/src/api/api.ts b/src/api/api.ts
@@ -26,6 +26,10 @@ async function prepareStreamAndGetResponse(customHeaders: Record<string, string>
     tempConfig.max_tokens -= functionsSchemaTokenLength;
   }
 
+  if(!modelCost[tempConfig.model].supportTemperature) {
+    config.temperature = 1;
+  }
+
   const adjustedMessagesTuple = limitMessageTokens(
     messagesToSend,
     config.max_tokens,
@@ -51,20 +55,21 @@ async function prepareStreamAndGetResponse(customHeaders: Record<string, string>
   }
 
   var bodyToSend = "";
+
   if(modelCost[tempConfig.model].supportFunctions) {
     bodyToSend = JSON.stringify({
       messages,
       ...tempConfig,
       max_tokens: undefined,
-      stream: stream,
+      stream: (stream && modelCost[tempConfig.model].supportStreaming),
       functions: functionsSchemas
     });
   } else {
     bodyToSend = JSON.stringify({
       messages,
       ...tempConfig,
       max_tokens: undefined,
-      stream: stream
+      stream: (stream && modelCost[tempConfig.model].supportStreaming)
     });
   }
 

diff --git a/src/constants/chat.ts b/src/constants/chat.ts
@@ -21,36 +21,48 @@ export const modelCost = {
     completion: { price: 0.06, unit: 1000 },
     modelMaxToken: 32768,
     supportFunctions: false,
+    supportTemperature: false,
+    supportStreaming: false,
   },
   'o1-mini': {
     prompt: { price: 0.003, unit: 1000 },
     completion: { price: 0.012, unit: 1000 },
     modelMaxToken: 65536,
     supportFunctions: false,
+    supportTemperature: false,
+    supportStreaming: false,
   },
   'gpt-4': {
     prompt: { price: 0.03, unit: 1000 },
     completion: { price: 0.06, unit: 1000 },
     modelMaxToken: 8192,
     supportFunctions: true,
+    supportTemperature: true,
+    supportStreaming: true,
   },
   'gpt-4-turbo': {
     prompt: { price: 0.01, unit: 1000 },
     completion: { price: 0.03, unit: 1000 },
     modelMaxToken: 128000,
     supportFunctions: true,
+    supportTemperature: true,
+    supportStreaming: true,
   },
   'gpt-4o': {
     prompt: { price: 0.005, unit: 1000 },
     completion: { price: 0.015, unit: 1000 },
     modelMaxToken: 128000,
     supportFunctions: true,
+    supportTemperature: true,
+    supportStreaming: true,
   },
   'gpt-4o-mini': {
     prompt: { price: 0.00015, unit: 1000 },
     completion: { price: 0.006, unit: 1000 },
     modelMaxToken: 128000,
     supportFunctions: true,
+    supportTemperature: true,
+    supportStreaming: true,
   }
 };