From d09c947a9b1956e5356335c0a7ea295cdf7f0898 Mon Sep 17 00:00:00 2001 From: mingming-ma <133393905+mingming-ma@users.noreply.github.com> Date: Fri, 15 Mar 2024 17:16:55 -0400 Subject: [PATCH] =?UTF-8?q?Add:=20Generate=20Image=20by=20DALL=C2=B7E=203?= =?UTF-8?q?=20(#493)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add generateImage function * remove debug comment * change command to image * add image command instructions * add n parameter instructions * use the currentProvider.createClient method * display no supported models error * refactor for same key value names --- src/components/Message/AppMessage/Help.tsx | 4 +- src/lib/ai.ts | 48 ++++++++++++++++++++++ src/lib/commands/ImageCommand.ts | 30 ++++++++++++++ src/lib/commands/index.ts | 2 + 4 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 src/lib/commands/ImageCommand.ts diff --git a/src/components/Message/AppMessage/Help.tsx b/src/components/Message/AppMessage/Help.tsx index 3850f7af..b6cb4cab 100644 --- a/src/components/Message/AppMessage/Help.tsx +++ b/src/components/Message/AppMessage/Help.tsx @@ -17,7 +17,9 @@ Some commands accept arguments as well. | /new | Creates a new chat. | | /clear | Erases all messages in the current chat. | | /summary [max-length] | Uses ChatGPT to create a summary of the current chat. Optionally takes a maximum word length (defaults to 500). | -| /import  | Loads the provided URL and imports the text. Where possible, ChatCraft will try to get raw text vs. HTML from sites like GitHub. NOTE: to prevent abuse, you must be logged into use the import command. |`; +| /import  | Loads the provided URL and imports the text. Where possible, ChatCraft will try to get raw text vs. HTML from sites like GitHub. NOTE: to prevent abuse, you must be logged into use the import command. | +| /image  | Creates an image using the provided prompt.| +`; const helpText = `## ChatCraft.org Help diff --git a/src/lib/ai.ts b/src/lib/ai.ts index 5bed37ff..96d05d56 100644 --- a/src/lib/ai.ts +++ b/src/lib/ai.ts @@ -396,3 +396,51 @@ export const textToSpeech = async (message: string): Promise => { return objectUrl; }; + +/** + * Only meant to be used outside components or hooks + * where useModels cannot be used. + */ +export async function isGenerateImageSupported() { + const { currentProvider } = getSettings(); + if (!currentProvider.apiKey) { + throw new Error("Missing API Key"); + } + + return ( + (await currentProvider.queryModels(currentProvider.apiKey)).filter((model: string) => + model.includes("dall-e-3") + )?.length > 0 + ); +} + +type dalle3ImageSize = "1024x1024" | "1792x1024" | "1024x1792"; + +export const generateImage = async ( + prompt: string, + //You can request 1 image at a time with DALL·E 3 (request more by making parallel requests) or up to 10 images at a time using DALL·E 2 with the n parameter. + //https://platform.openai.com/docs/guides/images/generations + n: number = 1, + size: dalle3ImageSize = "1024x1024" +): Promise => { + const { currentProvider } = getSettings(); + if (!currentProvider.apiKey) { + throw new Error("Missing OpenAI API Key"); + } + + const { openai } = currentProvider.createClient(currentProvider.apiKey); + + try { + const response = await openai.images.generate({ + model: "dall-e-3", + prompt, + n, + size, + }); + + const imageUrls = response.data.map((img: any) => img.url); + return imageUrls; + } catch (error: any) { + throw new Error(error); + } +}; diff --git a/src/lib/commands/ImageCommand.ts b/src/lib/commands/ImageCommand.ts new file mode 100644 index 00000000..3404c52d --- /dev/null +++ b/src/lib/commands/ImageCommand.ts @@ -0,0 +1,30 @@ +import { ChatCraftCommand } from "../ChatCraftCommand"; +import { ChatCraftChat } from "../ChatCraftChat"; +import { ChatCraftHumanMessage } from "../ChatCraftMessage"; +import { generateImage, isGenerateImageSupported } from "../../lib/ai"; + +export class ImageCommand extends ChatCraftCommand { + constructor() { + super("image"); + } + + async execute(chat: ChatCraftChat, user: User | undefined, args?: string[]) { + if (!(await isGenerateImageSupported())) { + throw new Error("Failed to generate image, no image generation models available"); + } + if (!(args && args[0])) { + throw new Error("must include a prompt"); + } + const prompt = args.join(" "); + let imageUrls: string[] = []; + const text = `(DALL·E 3 result of the prompt: ${prompt})`; + + try { + imageUrls = await generateImage(prompt); + } catch (error: any) { + console.error(`Failed to generate image: ${error.message}`); + throw new Error(`Failed to generate image: ${error.message}`); + } + return chat.addMessage(new ChatCraftHumanMessage({ user, text, imageUrls })); + } +} diff --git a/src/lib/commands/index.ts b/src/lib/commands/index.ts index 562c3725..37349b20 100644 --- a/src/lib/commands/index.ts +++ b/src/lib/commands/index.ts @@ -7,6 +7,7 @@ import { SummaryCommand } from "./SummaryCommand"; import { HelpCommand } from "./HelpCommand"; import { ImportCommand } from "./ImportCommand"; import { CommandsHelpCommand } from "./CommandsHelpCommand"; +import { ImageCommand } from "./ImageCommand"; // Register all our commands ChatCraftCommandRegistry.registerCommand(new NewCommand()); @@ -15,3 +16,4 @@ ChatCraftCommandRegistry.registerCommand(new SummaryCommand()); ChatCraftCommandRegistry.registerCommand(new HelpCommand()); ChatCraftCommandRegistry.registerCommand(new CommandsHelpCommand()); ChatCraftCommandRegistry.registerCommand(new ImportCommand()); +ChatCraftCommandRegistry.registerCommand(new ImageCommand());