openrouter: add image, video, music, and speech generation providers

extensions/openrouter/image-generation-provider.test.ts

@@ -0,0 +1,195 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { buildOpenrouterImageGenerationProvider } from "./image-generation-provider.js";
+
+const {
+  resolveApiKeyForProviderMock,
+  postJsonRequestMock,
+  assertOkOrThrowHttpErrorMock,
+  resolveProviderHttpRequestConfigMock,
+} = vi.hoisted(() => ({
+  resolveApiKeyForProviderMock: vi.fn(async () => ({ apiKey: "openrouter-key" })),
+  postJsonRequestMock: vi.fn(),
+  assertOkOrThrowHttpErrorMock: vi.fn(async () => {}),
+  resolveProviderHttpRequestConfigMock: vi.fn((params) => ({
+    baseUrl: params.baseUrl ?? params.defaultBaseUrl,
+    allowPrivateNetwork: Boolean(params.allowPrivateNetwork),
+    headers: new Headers(params.defaultHeaders),
+    dispatcherPolicy: undefined,
+  })),
+}));
+
+vi.mock("openclaw/plugin-sdk/provider-auth-runtime", () => ({
+  resolveApiKeyForProvider: resolveApiKeyForProviderMock,
+}));
+
+vi.mock("openclaw/plugin-sdk/provider-http", () => ({
+  assertOkOrThrowHttpError: assertOkOrThrowHttpErrorMock,
+  postJsonRequest: postJsonRequestMock,
+  resolveProviderHttpRequestConfig: resolveProviderHttpRequestConfigMock,
+}));
+
+function makeImageDataUrl(content: string): string {
+  return `data:image/png;base64,${Buffer.from(content).toString("base64")}`;
+}
+
+describe("openrouter image generation provider", () => {
+  afterEach(() => {
+    resolveApiKeyForProviderMock.mockClear();
+    postJsonRequestMock.mockReset();
+    assertOkOrThrowHttpErrorMock.mockClear();
+    resolveProviderHttpRequestConfigMock.mockClear();
+  });
+
+  it("exposes correct provider metadata", () => {
+    const provider = buildOpenrouterImageGenerationProvider();
+    expect(provider.id).toBe("openrouter");
+    expect(provider.label).toBe("OpenRouter");
+    expect(provider.defaultModel).toBe("google/gemini-2.5-flash-image");
+    expect(provider.models).toContain("google/gemini-2.5-flash-image");
+    expect(provider.models).toContain("black-forest-labs/flux.2-pro");
+    expect(provider.capabilities.generate.supportsAspectRatio).toBe(true);
+    expect(provider.capabilities.generate.supportsResolution).toBe(true);
+    expect(provider.capabilities.edit.enabled).toBe(false);
+  });
+
+  it("generates an image and parses base64 data URL response", async () => {
+    const imageUrl = makeImageDataUrl("png-bytes");
+    postJsonRequestMock.mockResolvedValue({
+      response: {
+        json: async () => ({
+          choices: [
+            {
+              message: {
+                role: "assistant",
+                content: "Here is your image",
+                images: [
+                  {
+                    type: "image_url",
+                    image_url: { url: imageUrl },
+                  },
+                ],
+              },
+            },
+          ],
+        }),
+      },
+      release: vi.fn(async () => {}),
+    });
+
+    const provider = buildOpenrouterImageGenerationProvider();
+    const result = await provider.generateImage({
+      provider: "openrouter",
+      model: "google/gemini-2.5-flash-image",
+      prompt: "Draw a lobster",
+      cfg: {},
+    });
+
+    expect(result.images).toHaveLength(1);
+    expect(result.images[0]?.mimeType).toBe("image/png");
+    expect(result.images[0]?.buffer.toString()).toBe("png-bytes");
+    expect(result.images[0]?.fileName).toBe("image-1.png"); // PNG from data URL MIME type
+    expect(result.model).toBe("google/gemini-2.5-flash-image");
+
+    expect(postJsonRequestMock).toHaveBeenCalledWith(
+      expect.objectContaining({
+        url: "https://openrouter.ai/api/v1/chat/completions",
+        body: expect.objectContaining({
+          model: "google/gemini-2.5-flash-image",
+          modalities: ["image"],
+          messages: [{ role: "user", content: "Draw a lobster" }],
+        }),
+      }),
+    );
+  });
+
+  it("passes aspect_ratio and image_size via image_config", async () => {
+    postJsonRequestMock.mockResolvedValue({
+      response: {
+        json: async () => ({
+          choices: [
+            {
+              message: {
+                images: [
+                  {
+                    type: "image_url",
+                    image_url: { url: makeImageDataUrl("bytes") },
+                  },
+                ],
+              },
+            },
+          ],
+        }),
+      },
+      release: vi.fn(async () => {}),
+    });
+
+    const provider = buildOpenrouterImageGenerationProvider();
+    await provider.generateImage({
+      provider: "openrouter",
+      model: "google/gemini-2.5-flash-image",
+      prompt: "A sunset",
+      cfg: {},
+      aspectRatio: "16:9",
+      resolution: "2K",
+    });
+
+    expect(postJsonRequestMock).toHaveBeenCalledWith(
+      expect.objectContaining({
+        body: expect.objectContaining({
+          image_config: {
+            aspect_ratio: "16:9",
+            image_size: "2K",
+          },
+        }),
+      }),
+    );
+  });
+
+  it("throws when response contains no images", async () => {
+    postJsonRequestMock.mockResolvedValue({
+      response: {
+        json: async () => ({
+          choices: [{ message: { content: "Sorry, no image" } }],
+        }),
+      },
+      release: vi.fn(async () => {}),
+    });
+
+    const provider = buildOpenrouterImageGenerationProvider();
+    await expect(
+      provider.generateImage({
+        provider: "openrouter",
+        model: "google/gemini-2.5-flash-image",
+        prompt: "a tree",
+        cfg: {},
+      }),
+    ).rejects.toThrow("OpenRouter image generation response missing image data");
+  });
+
+  it("rejects input images since edit mode is not supported", async () => {
+    const provider = buildOpenrouterImageGenerationProvider();
+    await expect(
+      provider.generateImage({
+        provider: "openrouter",
+        model: "google/gemini-2.5-flash-image",
+        prompt: "edit this",
+        cfg: {},
+        inputImages: [{ buffer: Buffer.from("img"), mimeType: "image/png" }],
+      }),
+    ).rejects.toThrow("OpenRouter image generation does not support image editing");
+  });
+
+  it("throws when API key is missing", async () => {
+    resolveApiKeyForProviderMock.mockResolvedValueOnce({ apiKey: "" });
+
+    const provider = buildOpenrouterImageGenerationProvider();
+    await expect(
+      provider.generateImage({
+        provider: "openrouter",
+        model: "google/gemini-2.5-flash-image",
+        prompt: "a tree",
+        cfg: {},
+      }),
+    ).rejects.toThrow("OpenRouter API key missing");
+  });
+});

extensions/openrouter/image-generation-provider.ts

@@ -0,0 +1,203 @@
+import type {
+  GeneratedImageAsset,
+  ImageGenerationProvider,
+  ImageGenerationResolution,
+} from "openclaw/plugin-sdk/image-generation";
+import { isProviderApiKeyConfigured } from "openclaw/plugin-sdk/provider-auth";
+import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime";
+import {
+  assertOkOrThrowHttpError,
+  postJsonRequest,
+  resolveProviderHttpRequestConfig,
+} from "openclaw/plugin-sdk/provider-http";
+import { normalizeOptionalString } from "openclaw/plugin-sdk/text-runtime";
+import { OPENROUTER_BASE_URL, resolveConfiguredBaseUrl } from "./openrouter-config.js";
+const DEFAULT_OPENROUTER_IMAGE_MODEL = "google/gemini-2.5-flash-image";
+const OPENROUTER_IMAGE_MODELS = [
+  "google/gemini-2.5-flash-image",
+  "google/gemini-3.1-flash-image-preview",
+  "black-forest-labs/flux.2-pro",
+] as const;
+const OPENROUTER_IMAGE_ASPECT_RATIOS = [
+  "1:1",
+  "2:3",
+  "3:2",
+  "3:4",
+  "4:3",
+  "9:16",
+  "16:9",
+] as const;
+const OPENROUTER_IMAGE_RESOLUTIONS: readonly ImageGenerationResolution[] = ["1K", "2K", "4K"];
+
+type OpenRouterImageMessage = {
+  role: string;
+  content?: string;
+  images?: Array<{
+    type?: string;
+    image_url?: {
+      url?: string;
+    };
+  }>;
+};
+
+type OpenRouterImageApiResponse = {
+  choices?: Array<{
+    message?: OpenRouterImageMessage;
+  }>;
+};
+
+function extractBase64FromDataUrl(dataUrl: string): { buffer: Buffer; mimeType: string } | null {
+  const match = /^data:([^;]+);base64,(.+)$/u.exec(dataUrl);
+  if (!match?.[1] || !match[2]) {
+    return null;
+  }
+  return {
+    buffer: Buffer.from(match[2], "base64"),
+    mimeType: match[1],
+  };
+}
+
+function resolveFileExtension(mimeType: string): string {
+  switch (mimeType) {
+    case "image/jpeg":
+    case "image/jpg":
+      return "jpg";
+    case "image/webp":
+      return "webp";
+    case "image/gif":
+      return "gif";
+    case "image/png":
+      return "png";
+    default: {
+      // Derive extension from MIME subtype for unknown formats (e.g. image/avif -> avif).
+      const subtype = mimeType.split("/")[1]?.split(";")[0]?.trim();
+      if (subtype && /^[a-z0-9+-]+$/u.test(subtype)) return subtype;
+      return "png";
+    }
+  }
+}
+
+export function buildOpenrouterImageGenerationProvider(): ImageGenerationProvider {
+  return {
+    id: "openrouter",
+    label: "OpenRouter",
+    defaultModel: DEFAULT_OPENROUTER_IMAGE_MODEL,
+    models: [...OPENROUTER_IMAGE_MODELS],
+    isConfigured: ({ agentDir }) =>
+      isProviderApiKeyConfigured({
+        provider: "openrouter",
+        agentDir,
+      }),
+    capabilities: {
+      generate: {
+        maxCount: 1,
+        supportsSize: false,
+        supportsAspectRatio: true,
+        supportsResolution: true,
+      },
+      edit: {
+        enabled: false,
+      },
+      geometry: {
+        aspectRatios: [...OPENROUTER_IMAGE_ASPECT_RATIOS],
+        resolutions: [...OPENROUTER_IMAGE_RESOLUTIONS],
+      },
+    },
+    async generateImage(req) {
+      if ((req.inputImages?.length ?? 0) > 0) {
+        throw new Error("OpenRouter image generation does not support image editing");
+      }
+
+      const auth = await resolveApiKeyForProvider({
+        provider: "openrouter",
+        cfg: req.cfg,
+        agentDir: req.agentDir,
+        store: req.authStore,
+      });
+      if (!auth.apiKey) {
+        throw new Error("OpenRouter API key missing");
+      }
+
+      const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } =
+        resolveProviderHttpRequestConfig({
+          baseUrl: resolveConfiguredBaseUrl(req.cfg),
+          defaultBaseUrl: OPENROUTER_BASE_URL,
+          allowPrivateNetwork: false,
+          defaultHeaders: {
+            Authorization: `Bearer ${auth.apiKey}`,
+          },
+          provider: "openrouter",
+          capability: "image",
+          transport: "http",
+        });
+
+      const model = normalizeOptionalString(req.model) ?? DEFAULT_OPENROUTER_IMAGE_MODEL;
+      const aspectRatio = normalizeOptionalString(req.aspectRatio);
+      const resolution = normalizeOptionalString(req.resolution);
+
+      const imageConfig: Record<string, string> = {};
+      if (aspectRatio) {
+        imageConfig.aspect_ratio = aspectRatio;
+      }
+      if (resolution) {
+        imageConfig.image_size = resolution;
+      }
+
+      const jsonHeaders = new Headers(headers);
+      jsonHeaders.set("Content-Type", "application/json");
+      const { response, release } = await postJsonRequest({
+        url: `${baseUrl}/chat/completions`,
+        headers: jsonHeaders,
+        body: {
+          model,
+          messages: [{ role: "user", content: req.prompt }],
+          modalities: ["image"],
+          ...(Object.keys(imageConfig).length > 0 ? { image_config: imageConfig } : {}),
+        },
+        timeoutMs: req.timeoutMs,
+        fetchFn: fetch,
+        allowPrivateNetwork,
+        dispatcherPolicy,
+      });
+
+      try {
+        await assertOkOrThrowHttpError(response, "OpenRouter image generation failed");
+        const data = (await response.json()) as OpenRouterImageApiResponse;
+        const rawImages = data.choices?.[0]?.message?.images ?? [];
+
+        const images: GeneratedImageAsset[] = rawImages
+          .map((entry, index) => {
+            const url = normalizeOptionalString(entry.image_url?.url);
+            if (!url) {
+              return null;
+            }
+            const parsed = extractBase64FromDataUrl(url);
+            if (!parsed) {
+              return null;
+            }
+            const mimeType = parsed.mimeType.toLowerCase();
+            if (!mimeType.startsWith("image/")) {
+              return null;
+            }
+            return {
+              buffer: parsed.buffer,
+              mimeType,
+              fileName: `image-${index + 1}.${resolveFileExtension(mimeType)}`,
+            };
+          })
+          .filter((entry): entry is NonNullable<typeof entry> => entry !== null);
+
+        if (images.length === 0) {
+          throw new Error("OpenRouter image generation response missing image data");
+        }
+
+        return {
+          images,
+          model,
+        };
+      } finally {
+        await release();
+      }
+    },
+  };
+}