feat(provider/google): Url support for pdfs and images in tool results

.changeset/beige-monkeys-complain.md

@@ -0,0 +1,5 @@
+---
+'@ai-sdk/google': patch
+---
+
+Adds url-based pdf and image support for google tool results

examples/ai-core/src/generate-text/google-image-tool-result-url.ts

@@ -0,0 +1,50 @@
+import { google } from '@ai-sdk/google';
+import { generateText, stepCountIs, tool } from 'ai';
+import { run } from '../lib/run';
+import { z } from 'zod';
+
+run(async () => {
+  const readImage = tool({
+    description: `Read and return an image`,
+    inputSchema: z.object({}),
+    execute: async () => {
+      try {
+        return {
+          success: true,
+          description: 'Successfully loaded image',
+          imageUrl:
+            'https://github.com/vercel/ai/blob/main/examples/ai-core/data/comic-cat.png?raw=true',
+        };
+      } catch (error) {
+        throw new Error(`Failed to analyze image: ${error}`);
+      }
+    },
+    toModelOutput(result) {
+      return {
+        type: 'content',
+        value: [
+          {
+            type: 'text',
+            text: result.description,
+          },
+          {
+            type: 'image-url',
+            url: result.imageUrl,
+          },
+        ],
+      };
+    },
+  });
+
+  const result = await generateText({
+    model: google('gemini-2.5-flash'),
+    prompt:
+      'Please read the image using the tool provided and return the summary of that image',
+    tools: {
+      readImage,
+    },
+    stopWhen: stepCountIs(4),
+  });
+
+  console.log(`Assistant response: ${JSON.stringify(result.text, null, 2)}`);
+});

examples/ai-core/src/generate-text/google-pdf-tool-results-base64.ts

@@ -0,0 +1,60 @@
+import { google } from '@ai-sdk/google';
+import { generateText, stepCountIs, tool } from 'ai';
+import { run } from '../lib/run';
+import { z } from 'zod';
+import path from 'path';
+import fs from 'fs/promises';
+
+run(async () => {
+  const readPDFDocument = tool({
+    description: `Read and return a PDF document`,
+    inputSchema: z.object({}),
+    execute: async () => {
+      try {
+        const pdfPath = path.join(__dirname, '../../data/ai.pdf');
+        const pdfData = await fs.readFile(pdfPath);
+
+        const base64Data = pdfData.toString('base64');
+
+        console.log(`PDF document read successfully`);
+
+        return {
+          success: true,
+          description: 'Successfully loaded PDF document',
+          pdfData: base64Data,
+        };
+      } catch (error) {
+        throw new Error(`Failed to analyze PDF: ${error}`);
+      }
+    },
+    toModelOutput(result) {
+      return {
+        type: 'content',
+        value: [
+          {
+            type: 'text',
+            text: result.description,
+          },
+          {
+            type: 'file-data',
+            data: result.pdfData,
+            mediaType: 'application/pdf',
+            filename: 'ai.pdf',
+          },
+        ],
+      };
+    },
+  });
+
+  const result = await generateText({
+    model: google('gemini-2.5-flash'),
+    prompt:
+      'Please read the pdf document using the tool provided and return the summary of that pdf',
+    tools: {
+      readPDFDocument,
+    },
+    stopWhen: stepCountIs(4),
+  });
+
+  console.log(`Assistant response: ${JSON.stringify(result.text, null, 2)}`);
+});

examples/ai-core/src/generate-text/google-pdf-tool-results-url.ts

@@ -0,0 +1,50 @@
+import { google } from '@ai-sdk/google';
+import { generateText, stepCountIs, tool } from 'ai';
+import { run } from '../lib/run';
+import { z } from 'zod';
+
+run(async () => {
+  const readPDFDocument = tool({
+    description: `Read and return a PDF document`,
+    inputSchema: z.object({}),
+    execute: async () => {
+      try {
+        return {
+          success: true,
+          description: 'Successfully loaded PDF document',
+          pdfUrl:
+            'https://github.com/vercel/ai/blob/main/examples/ai-core/data/ai.pdf?raw=true',
+        };
+      } catch (error) {
+        throw new Error(`Failed to analyze PDF: ${error}`);
+      }
+    },
+    toModelOutput(result) {
+      return {
+        type: 'content',
+        value: [
+          {
+            type: 'text',
+            text: result.description,
+          },
+          {
+            type: 'file-url',
+            url: result.pdfUrl,
+          },
+        ],
+      };
+    },
+  });
+
+  const result = await generateText({
+    model: google('gemini-2.5-flash'),
+    prompt:
+      'Please read the pdf document using the tool provided and return the summary of that pdf',
+    tools: {
+      readPDFDocument,
+    },
+    stopWhen: stepCountIs(4),
+  });
+
+  console.log(`Assistant response: ${JSON.stringify(result.text, null, 2)}`);
+});

packages/google/src/convert-to-google-generative-ai-messages.test.ts

@@ -410,4 +410,92 @@ describe('assistant messages', () => {
       ],
     });
   });
+
+  it('should handle tool result with url-based pdf content', async () => {
+    const result = convertToGoogleGenerativeAIMessages([
+      {
+        role: 'tool',
+        content: [
+          {
+            type: 'tool-result',
+            toolName: 'pdf-fetcher',
+            toolCallId: 'get-pdf-1',
+            output: {
+              type: 'content',
+              value: [
+                {
+                  type: 'file-url',
+                  url: 'https://example.com/document.pdf',
+                },
+              ],
+            },
+          },
+        ],
+      },
+    ]);
+
+    expect(result).toEqual({
+      systemInstruction: undefined,
+      contents: [
+        {
+          role: 'user',
+          parts: [
+            {
+              fileData: {
+                mimeType: 'application/pdf',
+                fileUri: 'https://example.com/document.pdf',
+              },
+            },
+            {
+              text: 'Tool executed successfully and returned this file as a response',
+            },
+          ],
+        },
+      ],
+    });
+  });
+
+  it('should handle tool result with url-based image content', async () => {
+    const result = convertToGoogleGenerativeAIMessages([
+      {
+        role: 'tool',
+        content: [
+          {
+            type: 'tool-result',
+            toolName: 'image-generator',
+            toolCallId: 'image-gen-1',
+            output: {
+              type: 'content',
+              value: [
+                {
+                  type: 'image-url',
+                  url: 'https://example.com/image.png',
+                },
+              ],
+            },
+          },
+        ],
+      },
+    ]);
+
+    expect(result).toEqual({
+      systemInstruction: undefined,
+      contents: [
+        {
+          role: 'user',
+          parts: [
+            {
+              fileData: {
+                mimeType: 'image/png',
+                fileUri: 'https://example.com/image.png',
+              },
+            },
+            {
+              text: 'Tool executed successfully and returned this image as a response',
+            },
+          ],
+        },
+      ],
+    });
+  });
 });

packages/google/src/convert-to-google-generative-ai-messages.ts

@@ -9,6 +9,51 @@ import {
 } from './google-generative-ai-prompt';
 import { convertToBase64 } from '@ai-sdk/provider-utils';
 
+const imageExtensionMimeTypes: Record<string, string> = {
+  png: 'image/png',
+  jpg: 'image/jpeg',
+  jpeg: 'image/jpeg',
+  webp: 'image/webp',
+  gif: 'image/gif',
+  bmp: 'image/bmp',
+  svg: 'image/svg+xml',
+  tif: 'image/tiff',
+  tiff: 'image/tiff',
+  heic: 'image/heic',
+  heif: 'image/heif',
+};
+
+const fileExtensionMimeTypes: Record<string, string> = {
+  pdf: 'application/pdf',
+};
+
+function inferMimeTypeFromUrl(
+  urlString: string,
+  options: {
+    extensionMap: Record<string, string>;
+    defaultMimeType: string;
+  },
+): string {
+  try {
+    const url = new URL(urlString);
+    const pathname = url.pathname.toLowerCase();
+    const lastDotIndex = pathname.lastIndexOf('.');
+
+    if (lastDotIndex !== -1 && lastDotIndex < pathname.length - 1) {
+      const extension = pathname.slice(lastDotIndex + 1);
+      const mappedMimeType = options.extensionMap[extension];
+
+      if (mappedMimeType) {
+        return mappedMimeType;
+      }
+    }
+  } catch {
+    // ignore invalid URLs and fall back to default mime type
+  }
+
+  return options.defaultMimeType;
+}
+
 export function convertToGoogleGenerativeAIMessages(
   prompt: LanguageModelV3Prompt,
   options?: { isGemmaModel?: boolean },
@@ -178,6 +223,44 @@ export function convertToGoogleGenerativeAIMessages(
                     },
                   );
                   break;
+                case 'image-url': {
+                  const mimeType = inferMimeTypeFromUrl(contentPart.url, {
+                    extensionMap: imageExtensionMimeTypes,
+                    defaultMimeType: 'image/*',
+                  });
+
+                  parts.push(
+                    {
+                      fileData: {
+                        mimeType,
+                        fileUri: contentPart.url,
+                      },
+                    },
+                    {
+                      text: 'Tool executed successfully and returned this image as a response',
+                    },
+                  );
+                  break;
+                }
+                case 'file-url': {
+                  const mimeType = inferMimeTypeFromUrl(contentPart.url, {
+                    extensionMap: fileExtensionMimeTypes,
+                    defaultMimeType: 'application/octet-stream',
+                  });
+
+                  parts.push(
+                    {
+                      fileData: {
+                        mimeType,
+                        fileUri: contentPart.url,
+                      },
+                    },
+                    {
+                      text: 'Tool executed successfully and returned this file as a response',
+                    },
+                  );
+                  break;
+                }
                 default:
                   parts.push({ text: JSON.stringify(contentPart) });
                   break;