Merge pull request #8122 from sagemathinc/npm-llm-updates-20251114

npm: updating a couple of server packages and langchain

src/packages/package.json

@@ -20,8 +20,8 @@
       "undici@<5.28.3": "^5.28.4",
       "postcss@<8.4.31": "^8.4.31",
       "retry-request@<7.0.1": "^7.0.2",
-      "@langchain/core": "^0.3.17",
-      "langchain": "^0.2.19",
+      "@langchain/core": "^0.3.30",
+      "langchain": "^0.3.11",
       "katex@<0.16.9": "^0.16.10",
       "nanoid@<3.3.8": "^3.3.8"
     }

src/packages/server/llm/index.ts

@@ -26,6 +26,7 @@ import {
   LanguageModel,
   LanguageServiceCore,
   OpenAIMessages,
+  OpenAIModel,
   getLLMCost,
   isAnthropicModel,
   isCoreLanguageModel,
@@ -50,6 +51,7 @@ import { evaluateCustomOpenAI } from "./custom-openai";
 import { GoogleGenAIClient } from "./google-genai-client";
 import { evaluateMistral } from "./mistral";
 import { evaluateOllama } from "./ollama";
+import { evaluateOpenAILC } from "./openai-lc";
 import { saveResponse } from "./save-response";
 import { evaluateUserDefinedLLM } from "./user-defined";
 
@@ -176,12 +178,15 @@ async function evaluateImpl({
           throw new Error("Wrong client. This should never happen. [GenAI]");
         }
         return await evaluateGoogleGenAI({ ...params, client });
+      } else if (isOpenAIModel(model)) {
+        return await evaluateOpenAILC(params);
       } else {
-        const client = await getClient(model);
-        if (!(client instanceof OpenAI)) {
-          throw new Error("Wrong client. This should never happen. [OpenAI]");
-        }
-        return await evaluateOpenAI({ ...params, client });
+        throw new Error(`Unable to handel model '${model}'.`);
+        // const client = await getClient(model);
+        // if (!(client instanceof OpenAI)) {
+        //   throw new Error("Wrong client. This should never happen. [OpenAI]");
+        // }
+        // return await evaluateOpenAI({ ...params, client });
       }
     })();
 
@@ -327,15 +332,7 @@ export async function evaluateOpenAI({
     throw new Error(`Model "${model}" not an OpenAI model.`);
   }
 
-  // the *-8k variants are artificial – the input is already limited/truncated to 8k
-  // convert *-preview and all *-8k to "gpt-4-turbo"
-  if (model.startsWith("gpt-4-turbo")) {
-    model = "gpt-4-turbo";
-  } else if (model.startsWith("gpt-4o-mini")) {
-    model = "gpt-4o-mini";
-  } else if (model.startsWith("gpt-4o")) {
-    model = "gpt-4o";
-  }
+  model = normalizeOpenAIModel(model);
 
   const messages: OpenAIMessages = [];
   if (system) {
@@ -356,3 +353,23 @@ export async function evaluateOpenAI({
     stream,
   });
 }
+
+export function normalizeOpenAIModel(model): OpenAIModel {
+  // the *-8k variants are artificial – the input is already limited/truncated to 8k
+  // convert *-preview and all *-8k to "gpt-4-turbo"
+  if (model.startsWith("gpt-4-turbo")) {
+    model = "gpt-4-turbo";
+  } else if (model.startsWith("gpt-4o-mini")) {
+    model = "gpt-4o-mini";
+  } else if (model.startsWith("gpt-4o")) {
+    model = "gpt-4o";
+  } else if (model.startsWith("o1-mini")) {
+    model = "o1-mini";
+  } else if (model.startsWith("o1")) {
+    model = "o1";
+  }
+  if (!isOpenAIModel(model)) {
+    throw new Error(`Internal problem normalizing OpenAI model name: ${model}`);
+  }
+  return model;
+}

src/packages/server/llm/openai-lc.ts

@@ -1,4 +1,4 @@
-import { AIMessageChunk } from "@langchain/core/messages";
+import { AIMessageChunk, MessageContent } from "@langchain/core/messages";
 import {
   ChatPromptTemplate,
   MessagesPlaceholder,
@@ -11,6 +11,7 @@ import getLogger from "@cocalc/backend/logger";
 import { getServerSettings } from "@cocalc/database/settings";
 import { isOpenAIModel } from "@cocalc/util/db-schema/llm-utils";
 import { ChatOutput, History } from "@cocalc/util/types/llm";
+import { normalizeOpenAIModel } from ".";
 import { transformHistoryToMessages } from "./chat-history";
 import { numTokens } from "./chatgpt-numtokens";
 
@@ -42,20 +43,28 @@ async function getParams(model: string) {
 }
 
 export async function evaluateOpenAILC(
-  opts: Readonly<OpenAIOpts>,
+  opts: OpenAIOpts,
   mode: "cocalc" | "user" = "cocalc",
 ): Promise<ChatOutput> {
+  if (mode === "cocalc") {
+    opts.model = normalizeOpenAIModel(opts.model);
+  }
   if (mode === "cocalc" && !isOpenAIModel(opts.model)) {
     throw new Error(`model ${opts.model} not supported`);
   }
   const { system, history, input, maxTokens, stream, model } = opts;
 
+  // As of Jan 2025: reasoning models (o1) do not support streaming
+  // https://platform.openai.com/docs/guides/reasoning/
+  const isO1 = model != "o1-mini" && model != "o1";
+  const streaming = stream != null && isO1;
+
   log.debug("evaluateOpenAILC", {
     input,
     history,
     system,
     model,
-    stream: stream != null,
+    stream: streaming,
     maxTokens,
   });
 
@@ -65,13 +74,11 @@ export async function evaluateOpenAILC(
   const openai = new ChatOpenAI({
     ...params,
     maxTokens,
-    streaming: stream != null,
-  }).bind({
-    stream_options: { include_usage: true },
-  });
+    streaming,
+  }).bind(isO1 ? {} : { stream_options: { include_usage: true } });
 
   const prompt = ChatPromptTemplate.fromMessages([
-    ["system", system ?? ""],
+    [isO1 ? "developer" : "system", system ?? ""],
     new MessagesPlaceholder("history"),
     ["human", "{input}"],
   ]);
@@ -94,25 +101,34 @@ export async function evaluateOpenAILC(
     },
   });
 
-  const chunks = await chainWithHistory.stream({
-    input,
-  });
-
   let finalResult: AIMessageChunk | undefined;
   let output = "";
-  for await (const chunk of chunks) {
-    const { content } = chunk;
-    if (typeof content !== "string") continue;
-    output += content;
-    opts.stream?.(content);
-
-    if (finalResult) {
-      finalResult = concat(finalResult, chunk);
-    } else {
-      finalResult = chunk;
+
+  if (streaming) {
+    const chunks = await chainWithHistory.stream({
+      input,
+    });
+
+    for await (const chunk of chunks) {
+      const { content } = chunk;
+      const contentStr = content2string(content);
+      output += contentStr;
+      opts.stream?.(contentStr);
+
+      if (finalResult) {
+        finalResult = concat(finalResult, chunk);
+      } else {
+        finalResult = chunk;
+      }
     }
+  } else {
+    finalResult = await chainWithHistory.invoke({ input });
+    const { content } = finalResult;
+    output = content2string(content);
   }
 
+  log.debug("finalResult", finalResult);
+
   // and an empty call when done
   opts.stream?.();
 
@@ -142,3 +158,17 @@ export async function evaluateOpenAILC(
     };
   }
 }
+
+function content2string(content: MessageContent): string {
+  if (typeof content === "string") {
+    return content;
+  } else {
+    const output0 = content[0];
+    if (output0.type === "text") {
+      return output0.text;
+    } else {
+      log.debug("content2string unable to process", content);
+      return "Problem processing returned message content.";
+    }
+  }
+}

src/packages/server/llm/test/00.test.ts

@@ -12,12 +12,13 @@ import {
   isMistralModel,
   isOpenAIModel,
 } from "@cocalc/util/db-schema/llm-utils";
-import { evaluateGoogleGenAI, evaluateOpenAI } from "..";
+import { evaluateGoogleGenAI } from "..";
 import { getClient } from "../client";
 // import { evaluateMistral } from "../mistral";
 import { evaluateAnthropic } from "../anthropic";
 import { GoogleGenAIClient } from "../google-genai-client";
 import { evaluateMistral } from "../mistral";
+import { evaluateOpenAILC } from "../openai-lc";
 import { enableModels, setupAPIKeys, test_llm } from "./shared";
 
 beforeAll(async () => {
@@ -48,12 +49,7 @@ async function llmOpenAI(model: LanguageModelCore) {
     throw new Error(`model: ${model} is not an OpenAI model`);
   }
 
-  const client = await getClient(model);
-  if (client == null) {
-    throw new Error(`model: ${model} not found`);
-  }
-  const answer = await evaluateOpenAI({
-    client: client as any,
+  const answer = await evaluateOpenAILC({
     model,
     ...QUERY,
   });
@@ -78,6 +74,13 @@ test_llm("openai")("OpenAI", () => {
   test("gpt 4o mini works", async () => {
     llmOpenAI("gpt-4o-mini-8k");
   });
+
+  // test("gpt o1", async () => {
+  //   llmOpenAI("o1-8k");
+  // });
+  // test("gpt o1 mini works", async () => {
+  //   llmOpenAI("o1-mini-8k");
+  // });
 });
 
 // ATTN: does not work everywhere around, geolocation matters

src/packages/server/package.json

@@ -50,18 +50,18 @@
     "@google-cloud/storage-transfer": "^3.3.0",
     "@google/generative-ai": "^0.14.0",
     "@isaacs/ttlcache": "^1.2.1",
-    "@langchain/anthropic": "^0.3.3",
-    "@langchain/community": "^0.3.11",
-    "@langchain/core": "^0.3.17",
-    "@langchain/google-genai": "^0.1.0",
-    "@langchain/mistralai": "^0.1.1",
-    "@langchain/openai": "^0.3.7",
+    "@langchain/anthropic": "^0.3.11",
+    "@langchain/community": "^0.3.24",
+    "@langchain/core": "^0.3.30",
+    "@langchain/google-genai": "^0.1.6",
+    "@langchain/mistralai": "^0.2.0",
+    "@langchain/openai": "^0.3.17",
     "@node-saml/passport-saml": "^4.0.4",
     "@passport-js/passport-twitter": "^1.0.8",
     "@passport-next/passport-google-oauth2": "^1.0.0",
     "@passport-next/passport-oauth2": "^2.1.4",
-    "@sendgrid/client": "^8.1.3",
-    "@sendgrid/mail": "^8.1.3",
+    "@sendgrid/client": "^8.1.4",
+    "@sendgrid/mail": "^8.1.4",
     "@types/async": "^2.0.43",
     "@types/cloudflare": "^2.7.11",
     "@types/dot-object": "^2.1.6",
@@ -100,8 +100,8 @@
     "ms": "2.1.2",
     "nanoid": "^3.3.8",
     "node-zendesk": "^5.0.13",
-    "nodemailer": "^6.9.14",
-    "openai": "^4.52.1",
+    "nodemailer": "^6.9.16",
+    "openai": "^4.78.1",
     "parse-domain": "^5.0.0",
     "passport": "^0.6.0",
     "passport-activedirectory": "^1.0.4",