feat: Support Google Gemini AI (#1805)

YousefED · web-flow · commit 9d3a54257f07 · 2025-07-03T10:45:21.000+02:00
* fix google ai

* fix lint
diff --git a/examples/05-interoperability/08-converting-blocks-to-react-email/package.json b/examples/05-interoperability/08-converting-blocks-to-react-email/package.json
@@ -26,4 +26,4 @@
     "@vitejs/plugin-react": "^4.3.1",
     "vite": "^5.3.4"
   }
-}
+}
diff --git a/examples/06-custom-schema/06-toggleable-blocks/index.html b/examples/06-custom-schema/06-toggleable-blocks/index.html
@@ -5,7 +5,7 @@
     </script>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>Toggleable Blocks</title>
+    <title>Toggleable Custom Blocks</title>
   </head>
   <body>
     <div id="root"></div>
diff --git a/examples/06-custom-schema/06-toggleable-blocks/package.json b/examples/06-custom-schema/06-toggleable-blocks/package.json
@@ -1,5 +1,5 @@
 {
-  "name": "@blocknote/example-toggleable-blocks",
+  "name": "@blocknote/example-custom-schema-toggleable-blocks",
   "description": "AUTO-GENERATED FILE, DO NOT EDIT DIRECTLY",
   "private": true,
   "version": "0.12.4",
diff --git a/examples/09-ai/02-playground/.bnexample.json b/examples/09-ai/02-playground/.bnexample.json
@@ -7,6 +7,7 @@
     "@blocknote/xl-ai": "latest",
     "@mantine/core": "^7.10.1",
     "ai": "^4.3.15",
+    "@ai-sdk/google": "^1.2.20",
     "@ai-sdk/openai": "^1.3.22",
     "@ai-sdk/openai-compatible": "^0.2.14",
     "@ai-sdk/groq": "^1.2.9",
diff --git a/examples/09-ai/02-playground/App.tsx b/examples/09-ai/02-playground/App.tsx
@@ -1,4 +1,5 @@
 import { createAnthropic } from "@ai-sdk/anthropic";
+import { createGoogleGenerativeAI } from "@ai-sdk/google";
 import { createGroq } from "@ai-sdk/groq";
 import { createMistral } from "@ai-sdk/mistral";
 import { createOpenAI } from "@ai-sdk/openai";
@@ -32,6 +33,7 @@ import { en as aiEn } from "@blocknote/xl-ai/locales";
 import "@blocknote/xl-ai/style.css";
 import { Fieldset, MantineProvider, Switch } from "@mantine/core";
 
+
 import { LanguageModelV1 } from "ai";
 import { useEffect, useMemo, useState } from "react";
 import { useStore } from "zustand";
@@ -72,6 +74,12 @@ function getModel(aiModelString: string) {
     return createAnthropic({
       ...client.getProviderSettings("anthropic"),
     })(modelName);
+  } else if (provider === "google.generative-ai") {
+    return createGoogleGenerativeAI({
+      ...client.getProviderSettings("google"),
+    })(modelName, {
+      structuredOutputs: false,
+    });
   } else {
     return "unknown-model" as const;
   }
diff --git a/examples/09-ai/02-playground/data/aimodels.ts b/examples/09-ai/02-playground/data/aimodels.ts
@@ -14,4 +14,8 @@ export const AI_MODELS = [
   "anthropic.chat/claude-3-7-sonnet-latest",
   "anthropic.chat/claude-3-5-haiku-latest",
   "albert-etalab.chat/neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8",
+  "google.generative-ai/gemini-1.5-pro",
+  "google.generative-ai/gemini-1.5-flash",
+  "google.generative-ai/gemini-2.5-pro",
+  "google.generative-ai/gemini-2.5-flash",
 ];
diff --git a/examples/09-ai/02-playground/package.json b/examples/09-ai/02-playground/package.json
@@ -20,6 +20,7 @@
     "@blocknote/xl-ai": "latest",
     "@mantine/core": "^7.10.1",
     "ai": "^4.3.15",
+    "@ai-sdk/google": "^1.2.20",
     "@ai-sdk/openai": "^1.3.22",
     "@ai-sdk/openai-compatible": "^0.2.14",
     "@ai-sdk/groq": "^1.2.9",
diff --git a/packages/xl-ai-server/.env.example b/packages/xl-ai-server/.env.example
@@ -2,4 +2,5 @@ TOKEN=<Token to secure the /ai endpoint of the proxy server>
 GROQ_API_KEY=<API Key for Groq>
 MISTRAL_API_KEY=<API Key for Mistral>
 OPENAI_API_KEY=<API Key for OpenAI>
+GOOGLE_API_KEY=<API Key for Google Gemini>
 MY_PROVIDER_API_KEY=<You can support any provider by setting the *_API_KEY pattern>
diff --git a/packages/xl-ai-server/src/index.ts b/packages/xl-ai-server/src/index.ts
@@ -61,9 +61,22 @@ function getProviderInfo(provider: string) {
   if (!key || !key.length) {
     return "not-found";
   }
+  if (provider === "google") {
+    return {
+      key,
+      header: "x-goog-api-key",
+    };
+  }
+  if (provider === "anthropic") {
+    return {
+      key,
+      header: "x-api-key",
+    };
+  }
+
   return {
     key,
-    header: provider === "anthropic" ? "x-api-key" : "Authorization",
+    header: "Authorization",
   };
 }
 
@@ -111,7 +124,6 @@ app.use("/ai", cors(), async (c) => {
     request.headers.set(providerInfo.header, `${providerInfo.key}`);
   }
 
-  request.headers.set("x-api-key", `${providerInfo.key}`);
   return proxyFetch(request);
 });
 
diff --git a/packages/xl-ai/package.json b/packages/xl-ai/package.json
@@ -93,6 +93,7 @@
     "@ai-sdk/openai": "^1.3.22",
     "@ai-sdk/openai-compatible": "^0.2.14",
     "@ai-sdk/anthropic": "^1.2.12",
+    "@ai-sdk/google": "^1.2.20",
     "@mswjs/interceptors": "^0.37.5",
     "@types/diff": "^6.0.0",
     "@types/json-diff": "^1.0.3",
diff --git a/packages/xl-ai/src/api/LLMRequest.ts b/packages/xl-ai/src/api/LLMRequest.ts
@@ -177,6 +177,7 @@ export async function doLLMRequest(
           content: `USER_MESSAGE: ${m.content}`,
         };
       }
+
       return m;
     });
     /*
@@ -192,9 +193,9 @@ export async function doLLMRequest(
     For now, this approach works ok.
     */
     previousMessages.push({
-      role: "assistant",
+      role: "system", // using "assistant" here doesn't work with gemini because we can't mix system / assistant messages
       content:
-        "These are the operations returned by a previous LLM call: \n" +
+        "ASSISTANT_MESSAGE: These are the operations returned by a previous LLM call: \n" +
         JSON.stringify(
           await previousResponse.llmResult.getGeneratedOperations(),
         ),
diff --git a/packages/xl-ai/src/streamTool/callLLMWithStreamTools.ts b/packages/xl-ai/src/streamTool/callLLMWithStreamTools.ts
@@ -105,9 +105,14 @@ export async function generateOperations<T extends StreamTool<any>[]>(
     // - optional, with defaults
 
     // mistral somehow needs "auto", while groq/llama needs "tool"
+    // google needs "auto" because https://github.com/vercel/ai/issues/6959
     // TODO: further research this and / or make configurable
     // for now stick to "tool" by default as this has been tested mostly
-    mode: rest.model.provider === "mistral.chat" ? "auto" : "tool",
+    mode:
+      rest.model.provider === "mistral.chat" ||
+      rest.model.provider === "google.generative-ai"
+        ? "auto"
+        : "tool",
     //  - mandatory ones:
     ...rest,
 
@@ -215,9 +220,14 @@ export async function streamOperations<T extends StreamTool<any>[]>(
 
     // - optional, with defaults
     // mistral somehow needs "auto", while groq/llama needs "tool"
+    // google needs "auto" because https://github.com/vercel/ai/issues/6959
     // TODO: further research this and / or make configurable
     // for now stick to "tool" by default as this has been tested mostly
-    mode: rest.model.provider === "mistral.chat" ? "auto" : "tool",
+    mode:
+      rest.model.provider === "mistral.chat" ||
+      rest.model.provider === "google.generative-ai"
+        ? "auto"
+        : "tool",
     //  - mandatory ones:
     ...rest,
 
diff --git a/playground/package.json b/playground/package.json
@@ -12,6 +12,7 @@
   },
   "dependencies": {
     "@ai-sdk/anthropic": "^1.2.11",
+    "@ai-sdk/google": "^1.2.20",
     "@ai-sdk/groq": "^1.2.9",
     "@ai-sdk/mistral": "^1.2.8",
     "@ai-sdk/openai": "^1.3.22",
diff --git a/playground/src/examples.gen.tsx b/playground/src/examples.gen.tsx
@@ -1185,7 +1185,7 @@
             "Basic"
           ]
         },
-        "title": "Toggleable Blocks",
+        "title": "Toggleable Custom Blocks",
         "group": {
           "pathFromRoot": "examples/06-custom-schema",
           "slug": "custom-schema"
@@ -1492,6 +1492,7 @@
             "@blocknote/xl-ai": "latest",
             "@mantine/core": "^7.10.1",
             "ai": "^4.3.15",
+            "@ai-sdk/google": "^1.2.20",
             "@ai-sdk/openai": "^1.3.22",
             "@ai-sdk/openai-compatible": "^0.2.14",
             "@ai-sdk/groq": "^1.2.9",
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml

Original file line number	Diff line number	Diff line change
`@@ -26,4 +26,4 @@`
`26`	`26`	`"@vitejs/plugin-react": "^4.3.1",`
`27`	`27`	`"vite": "^5.3.4"`
`28`	`28`	`}`
`29`		`-}`
	`29`	`+}`
Original file line number	Diff line number	Diff line change
`@@ -1,5 +1,5 @@`
`1`	`1`	`{`
`2`		`- "name": "@blocknote/example-toggleable-blocks",`
	`2`	`+ "name": "@blocknote/example-custom-schema-toggleable-blocks",`
`3`	`3`	`"description": "AUTO-GENERATED FILE, DO NOT EDIT DIRECTLY",`
`4`	`4`	`"private": true,`
`5`	`5`	`"version": "0.12.4",`