Merge PR TheoKanning#413 Add support for gpt-4-vision by @JanCong

Ran-Mewo · Ran-Mewo · commit a502d83a3341 · 2024-01-03T18:26:35.000+06:00
with a few modifications that removes parameterized `ChatMessage`
diff --git a/api/src/main/java/com/theokanning/openai/completion/chat/ChatCompletionChoice.java b/api/src/main/java/com/theokanning/openai/completion/chat/ChatCompletionChoice.java
@@ -15,7 +15,7 @@ public class ChatCompletionChoice {
     Integer index;
 
     /**
-     * The {@link ChatMessageRole#assistant} message or delta (when streaming) which was generated
+     * The {@link ChatMessageRole#ASSISTANT} message or delta (when streaming) which was generated
      */
     @JsonAlias("delta")
     ChatMessage message;
@@ -25,4 +25,10 @@ public class ChatCompletionChoice {
      */
     @JsonProperty("finish_reason")
     String finishReason;
+
+    /**
+     * When use the GPT-4V model, this will be return, for example {"type":"stop","stop":"<|fim_suffix|>"}.
+     */
+    @JsonProperty("finish_details")
+    FinishDetails finishDetails;
 }
diff --git a/api/src/main/java/com/theokanning/openai/completion/chat/ChatMessage.java b/api/src/main/java/com/theokanning/openai/completion/chat/ChatMessage.java
@@ -1,8 +1,12 @@
 package com.theokanning.openai.completion.chat;
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
-import lombok.*;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
 
 import java.util.List;
 
@@ -28,7 +32,7 @@ public class ChatMessage {
 	 */
 	String role;
 	@JsonInclude() // content should always exist in the call, even if it is null
-	String content;
+	Object content;
 	//name is optional, The name of the author of this message. May contain a-z, A-Z, 0-9, and underscores, with a maximum length of 64 characters.
 	String name;
 
@@ -38,6 +42,17 @@ public class ChatMessage {
 	@JsonProperty("function_call")
 	ChatFunctionCall functionCall;
 
+	public ChatMessage(String role, List<ChatMessageContent> content) {
+		this.role = role == null ? "assistant" : role;
+		this.content = content;
+	}
+
+	public ChatMessage(String role, List<ChatMessageContent> content, String name) {
+		this.role = role == null ? "assistant" : role;
+		this.content = content;
+		this.name = name;
+	}
+
 	public ChatMessage(String role, String content) {
 		this.role = role == null ? "assistant" : role;
 		this.content = content;
@@ -49,4 +64,9 @@ public ChatMessage(String role, String content, String name) {
 		this.name = name;
 	}
 
+	@JsonIgnore
+	public String getStringContent() {
+		return content instanceof String ? (String) content : null;
+	}
+
 }
diff --git a/api/src/main/java/com/theokanning/openai/completion/chat/ChatMessageContent.java b/api/src/main/java/com/theokanning/openai/completion/chat/ChatMessageContent.java
@@ -0,0 +1,38 @@
+package com.theokanning.openai.completion.chat;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@NoArgsConstructor
+public class ChatMessageContent {
+
+    /**
+     * The type of the content part
+     *
+     * @see ChatMessageContentType
+     */
+    private String type;
+
+    /**
+     * The text content.
+     */
+    private String text;
+
+    /**
+     * Image input is only supported when using the gpt-4-visual-preview model.
+     */
+    @JsonProperty("image_url")
+    private ImageUrl imageUrl;
+
+    public ChatMessageContent(String text) {
+        this.type = ChatMessageContentType.TEXT.value();
+        this.text = text;
+    }
+
+    public ChatMessageContent(ImageUrl imageUrl) {
+        this.type = ChatMessageContentType.IMAGE_URL.value();
+        this.imageUrl = imageUrl;
+    }
+}
diff --git a/api/src/main/java/com/theokanning/openai/completion/chat/ChatMessageContentType.java b/api/src/main/java/com/theokanning/openai/completion/chat/ChatMessageContentType.java
@@ -0,0 +1,20 @@
+package com.theokanning.openai.completion.chat;
+
+/**
+ * see {@link ChatMessage} documentation.
+ */
+public enum ChatMessageContentType {
+
+    TEXT("text"),
+    IMAGE_URL("image_url");
+
+    private final String value;
+
+    ChatMessageContentType(final String value) {
+        this.value = value;
+    }
+
+    public String value() {
+        return value;
+    }
+}
diff --git a/api/src/main/java/com/theokanning/openai/completion/chat/Content.java b/api/src/main/java/com/theokanning/openai/completion/chat/Content.java
@@ -0,0 +1,4 @@
+package com.theokanning.openai.completion.chat;
+
+public interface Content {
+}
diff --git a/api/src/main/java/com/theokanning/openai/completion/chat/FinishDetails.java b/api/src/main/java/com/theokanning/openai/completion/chat/FinishDetails.java
@@ -0,0 +1,23 @@
+package com.theokanning.openai.completion.chat;
+
+import lombok.Data;
+
+/**
+ * finish_details
+ *
+ * @author cong
+ * @since 2023/12/3
+ */
+@Data
+public class FinishDetails {
+
+    /**
+     * The reason why GPT stopped generating, for example "stop", "max_tokens".
+     */
+    private String type;
+
+    /**
+     * For example "<|fim_suffix|>"
+     */
+    private String stop;
+}
diff --git a/api/src/main/java/com/theokanning/openai/completion/chat/ImageUrl.java b/api/src/main/java/com/theokanning/openai/completion/chat/ImageUrl.java
@@ -0,0 +1,23 @@
+package com.theokanning.openai.completion.chat;
+
+import lombok.*;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+@RequiredArgsConstructor
+public class ImageUrl {
+
+    /**
+     * Either a URL of the image or the base64 encoded image data.
+     */
+    @NonNull
+    private String url;
+
+    /**
+     * Specifies the detail level of the image. Learn more in the
+     * <a href="https://platform.openai.com/docs/guides/vision/low-or-high-fidelity-image-understanding">
+     * Vision guide</a>.
+     */
+    private String detail;
+}
diff --git a/api/src/main/java/com/theokanning/openai/utils/TikTokensUtil.java b/api/src/main/java/com/theokanning/openai/utils/TikTokensUtil.java
@@ -186,7 +186,7 @@ public static int tokens(String modelName, List<ChatMessage> messages) {
         int sum = 0;
         for (ChatMessage msg : messages) {
             sum += tokensPerMessage;
-            sum += tokens(encoding, msg.getContent());
+            if (msg.getContent() instanceof String) sum += tokens(encoding, (String) msg.getContent());
             sum += tokens(encoding, msg.getRole());
             sum += tokens(encoding, msg.getName());
             if (isNotBlank(msg.getName())) {
diff --git a/api/src/main/java/com/theokanning/openai/utils/VisionUtil.java b/api/src/main/java/com/theokanning/openai/utils/VisionUtil.java
@@ -0,0 +1,46 @@
+package com.theokanning.openai.utils;
+
+import com.theokanning.openai.completion.chat.ChatMessage;
+import com.theokanning.openai.completion.chat.ChatMessageContent;
+import com.theokanning.openai.completion.chat.ImageUrl;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Vision tool class
+ *
+ * @author cong
+ * @since 2023/11/17
+ */
+public class VisionUtil {
+
+    private static final Pattern pattern = Pattern.compile("(https?://\\S+)");
+
+    public static ChatMessage convertForVision(ChatMessage msg) {
+        List<ChatMessageContent> content = new ArrayList<>();
+        String sourceText = msg.getStringContent();
+        // Regular expression to match image URLs
+        Matcher matcher = pattern.matcher(sourceText);
+        // Find image URLs and split the string
+        int lastIndex = 0;
+        while (matcher.find()) {
+            // Add the text before the image URL
+            if (matcher.start() > lastIndex) {
+                content.add(new ChatMessageContent(sourceText.substring(lastIndex, matcher.start()).trim()));
+            }
+            // Add the image URL
+            ImageUrl imageUrl = new ImageUrl();
+            imageUrl.setUrl(matcher.group());
+            content.add(new ChatMessageContent(imageUrl));
+            lastIndex = matcher.end();
+        }
+        // Add the remaining text
+        if (lastIndex < sourceText.length()) {
+            content.add(new ChatMessageContent(sourceText.substring(lastIndex).trim()));
+        }
+        return new ChatMessage(msg.getRole(), content, msg.getName());
+    }
+}
diff --git a/example/src/main/java/example/OpenAiApiVisionExample.java b/example/src/main/java/example/OpenAiApiVisionExample.java
@@ -0,0 +1,42 @@
+package example;
+
+import com.theokanning.openai.completion.chat.*;
+import com.theokanning.openai.service.OpenAiService;
+import com.theokanning.openai.utils.VisionUtil;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+
+class OpenAiApiVisionExample {
+    public static void main(String... args) {
+        String token = System.getenv("OPENAI_TOKEN");
+        OpenAiService service = new OpenAiService(token, Duration.ofSeconds(30));
+
+        System.out.println("Streaming chat completion...");
+        final List<ChatMessage> messages = new ArrayList<>();
+        List<ChatMessageContent> content = new ArrayList<>();
+        content.add(new ChatMessageContent("What’s in this image?"));
+        content.add(new ChatMessageContent(new ImageUrl(
+                "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg")));
+        messages.add(new ChatMessage(ChatMessageRole.USER.value(), content));
+
+        // use VisionUtil to convert image prompt to OpenAI format
+        System.out.println("Converting image to OpenAI format...");
+        ChatMessage visionChatMessage = VisionUtil.convertForVision(
+                new ChatMessage(ChatMessageRole.USER.value(),
+                        "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg "
+                                + "What are in these images? Is there any difference between them?"));
+        messages.add(visionChatMessage);
+
+        ChatCompletionRequest chatCompletionRequest = ChatCompletionRequest
+                .builder()
+                .model("gpt-4-vision-preview")
+                .messages(messages)
+                .maxTokens(300)
+                .build();
+
+        service.streamChatCompletion(chatCompletionRequest).blockingForEach(System.out::println);
+        service.shutdownExecutor();
+    }
+}
diff --git a/service/src/main/java/com/theokanning/openai/service/ChatMessageMixIn.java b/service/src/main/java/com/theokanning/openai/service/ChatMessageMixIn.java
@@ -0,0 +1,16 @@
+package com.theokanning.openai.service;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+
+/**
+ * @author cong
+ * @since 2023/11/17
+ */
+public abstract class ChatMessageMixIn {
+    @JsonProperty("content")
+    @JsonSerialize(using = ChatMessageSerializerAndDeserializer.ChatMessageContentSerializer.class)
+    @JsonDeserialize(using = ChatMessageSerializerAndDeserializer.ChatMessageContentDeserializer.class)
+    abstract Object getContent();
+}
diff --git a/service/src/main/java/com/theokanning/openai/service/ChatMessageSerializerAndDeserializer.java b/service/src/main/java/com/theokanning/openai/service/ChatMessageSerializerAndDeserializer.java
@@ -0,0 +1,79 @@
+package com.theokanning.openai.service;
+
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.*;
+import com.theokanning.openai.completion.chat.*;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+public class ChatMessageSerializerAndDeserializer {
+
+    public static class ChatMessageContentSerializer extends JsonSerializer<Object> {
+        @Override
+        public void serialize(Object content, JsonGenerator gen, SerializerProvider serializers) throws IOException {
+            if (content == null) {
+                gen.writeNull();
+                return;
+            }
+            if (content instanceof String) {
+                gen.writeString((String) content);
+                return;
+            }
+            if (content instanceof List) {
+                gen.writeStartArray();
+                List<?> contentList = (List<?>)content;
+                for (Object item : contentList) {
+                    if (item instanceof ChatMessageContent) {
+                        ChatMessageContent contentItem = (ChatMessageContent)item;
+                        gen.writeStartObject();
+                        gen.writeStringField("type", contentItem.getType());
+                        if (ChatMessageContentType.TEXT.value().equals(contentItem.getType())) {
+                            gen.writeStringField("text", contentItem.getText());
+                        } else if (ChatMessageContentType.IMAGE_URL.value().equals(contentItem.getType())) {
+                            gen.writeObjectFieldStart("image_url");
+                            gen.writeStringField("url", contentItem.getImageUrl().getUrl());
+                            gen.writeStringField("detail", contentItem.getImageUrl().getDetail());
+                            gen.writeEndObject();
+                        }
+                        gen.writeEndObject();
+                    }
+                }
+                gen.writeEndArray();
+            }
+        }
+    }
+
+    public static class ChatMessageContentDeserializer extends JsonDeserializer<Object> {
+        @Override
+        public Object deserialize(JsonParser p, DeserializationContext ctxt) throws IOException {
+            JsonNode contentNode = p.readValueAsTree();
+            if (contentNode.isTextual()) {
+                return contentNode.asText();
+            }
+            if (contentNode.isArray()) {
+                List<Object> contentList = new ArrayList<>();
+                for (JsonNode itemNode : contentNode) {
+                    String type = itemNode.get("type").asText();
+                    if (ChatMessageContentType.TEXT.value().equals(type)) {
+                        contentList.add(new ChatMessageContent(itemNode.get("text").asText()));
+                    } else if (ChatMessageContentType.IMAGE_URL.value().equals(type)) {
+                        JsonNode imageUrlJsonNode = itemNode.get("image_url");
+                        ImageUrl imageUrl = new ImageUrl();
+                        imageUrl.setUrl(Optional.ofNullable(imageUrlJsonNode.get("url"))
+                                .map(JsonNode::asText).orElse(null));
+                        imageUrl.setDetail(Optional.ofNullable(imageUrlJsonNode.get("detail"))
+                                .map(JsonNode::asText).orElse(null));
+                        contentList.add(new ChatMessageContent(imageUrl));
+                    }
+                }
+                return contentList;
+            }
+            return null;
+        }
+    }
+
+}
diff --git a/service/src/main/java/com/theokanning/openai/service/OpenAiService.java b/service/src/main/java/com/theokanning/openai/service/OpenAiService.java
diff --git a/service/src/test/java/com/theokanning/openai/service/ChatCompletionTest.java b/service/src/test/java/com/theokanning/openai/service/ChatCompletionTest.java

-Original file line number
+Diff line change
@@ @@ -0,0 +1,4 @@ @@
 +package com.theokanning.openai.completion.chat;
++
 +public interface Content {
 +}