From d7dfaa13da7e7eeca62fcde3d69c3449dbe9ffdd Mon Sep 17 00:00:00 2001 From: Wenxiao Date: Sun, 16 Feb 2025 22:39:25 +0800 Subject: [PATCH 1/2] fix moviepy version issue --- API/hardware/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/API/hardware/views.py b/API/hardware/views.py index 8c382a1e..373611ea 100644 --- a/API/hardware/views.py +++ b/API/hardware/views.py @@ -7,7 +7,7 @@ from django.conf import settings from django.http import HttpResponse from drf_yasg.utils import swagger_auto_schema -from moviepy.editor import VideoFileClip, concatenate_videoclips +from moviepy import VideoFileClip, concatenate_videoclips from rest_framework import status, viewsets from rest_framework.decorators import action from rest_framework.response import Response From 365dd2c7ceb10868f0e60623f5ffe5a2fc6380e0 Mon Sep 17 00:00:00 2001 From: Wenxiao Date: Sun, 16 Feb 2025 22:52:12 +0800 Subject: [PATCH 2/2] try integrating openai realtime and fix formatting --- API/orchestrator/chain/completed_task.py | 7 ++++ API/orchestrator/models.py | 2 ++ Agent/main.py | 1 + Agent/models/parameters.py | 6 ++++ Agent/models/task.py | 1 + Agent/modules/openai/handler.py | 43 ++++++++++++++++++++++++ Agent/storage.py | 2 +- 7 files changed, 61 insertions(+), 1 deletion(-) diff --git a/API/orchestrator/chain/completed_task.py b/API/orchestrator/chain/completed_task.py index 5a358a7f..50fd9d0a 100644 --- a/API/orchestrator/chain/completed_task.py +++ b/API/orchestrator/chain/completed_task.py @@ -78,6 +78,13 @@ def trigger_completed_task(sender, **kwargs): return completed_openai_gpt_4o_text_only.send( sender=sender, data=data, track_id=task_data.track_id ) + + if task_data.task_name == "openai_gpt_4o_realtime": + logger.info("OpenAI GPT4O Realtime task completed") + return completed_openai_gpt_4o_text_only.send( + sender=sender, data=data, track_id=task_data.track_id + ) + if task_data.task_name == "rag": logger.info("RAG task completed") return completed_rag.send(sender=sender, data=data, track_id=task_data.track_id) diff --git a/API/orchestrator/models.py b/API/orchestrator/models.py index 98d118f4..ce93312c 100644 --- a/API/orchestrator/models.py +++ b/API/orchestrator/models.py @@ -138,6 +138,7 @@ def get_task_name_choices(): ("openai_text2speech", "OpenAI Text2Speech"), ("openai_gpt_4o_text_and_image", "OpenAI GPT4o Text and Image"), ("openai_gpt_4o_text_only", "OpenAI GPT4o Text"), + ("openai_gpt_4o_realtime", "OpenAI GPT4o Realtime"), ("rag", "RAG"), ] @@ -156,6 +157,7 @@ def task_ml_task_mapping() -> dict: "openai_text2speech": "text2speech", "openai_gpt_4o_text_and_image": "text_generation", "openai_gpt_4o_text_only": "text_generation", + "openai_gpt_4o_realtime": "text_generation", "rag": "rag", } diff --git a/Agent/main.py b/Agent/main.py index 544e2180..b073e781 100644 --- a/Agent/main.py +++ b/Agent/main.py @@ -294,6 +294,7 @@ def handle_rag_task(self, task: Task): TaskName.text2speech.value, TaskName.openai_gpt4o_text_only.value, TaskName.openai_gpt_4o_text_and_image.value, + TaskName.openai_gpt_4o_realtime.value, TaskName.rag.value, ] else: diff --git a/Agent/models/parameters.py b/Agent/models/parameters.py index 956b8084..f26b74f7 100644 --- a/Agent/models/parameters.py +++ b/Agent/models/parameters.py @@ -55,3 +55,9 @@ class OpenAIGPT4OParameters(BaseModel): class OpenAIGPT4OTextOnlyParameters(BaseModel): text: str = Field(..., description="The text to analyze for GPT-4o text only") prompt_template: str = Field(description="The prompt template") + +class OpenAIGPT4ORealTimeParameters(BaseModel): + text: str = Field(..., description="The text to analyze for GPT-4o real time") + audio_file: str = Field(..., description="The audio data for GPT-4o real time") + prompt_template: str = Field(description="The prompt template") + sample_ratio: int = Field(1, description="The sample ratio") \ No newline at end of file diff --git a/Agent/models/task.py b/Agent/models/task.py index 9e7e9a53..56078d4e 100644 --- a/Agent/models/task.py +++ b/Agent/models/task.py @@ -17,6 +17,7 @@ class TaskName(str, Enum): openai_gpt_35 = "openai_gpt_35" openai_gpt4o_text_only = "openai_gpt_4o_text_only" openai_gpt_4o_text_and_image = "openai_gpt_4o_text_and_image" + openai_gpt_4o_realtime = "openai_gpt_4o_realtime" rag = "rag" diff --git a/Agent/modules/openai/handler.py b/Agent/modules/openai/handler.py index 3ddcee62..bd17a80a 100644 --- a/Agent/modules/openai/handler.py +++ b/Agent/modules/openai/handler.py @@ -6,6 +6,7 @@ from models.parameters import ( OpenAIGPT4OParameters, + OpenAIGPT4ORealTimeParameters, OpenAIGPT4OTextOnlyParameters, Speech2TextParameters, Text2SpeechParameters, @@ -57,6 +58,11 @@ def handle_task(self, task: Task) -> Task: text = self.gpt_35(task) TimeLogger.log(latency_profile, "end_openai_gpt_35") result_profile["text"] = text + if "openai_gpt_4o_realtime" in task.task_name: + TimeLogger.log(latency_profile, "start_openai_gpt_4o_realtime") + text = self.gpt_4o_realtime(task) + TimeLogger.log(latency_profile, "end_openai_gpt_4o_realtime") + result_profile["text"] = text if "text2speech" in task.task_name: TimeLogger.log(latency_profile, "start_openai_text2speech") audio_file_path = self.text2speech(task) @@ -147,6 +153,43 @@ def gpt_4o_text_only(self, task: Task) -> str: ) return res.choices[0].message.content + def gpt_4o_realtime(self, task: Task) -> Optional[str]: + """ + Get the text and audio in real time + Args: + task: + + Returns: + + """ + params = OpenAIGPT4ORealTimeParameters(**task.parameters) + text = params.text + audio = params.audio_file + prompt_template = params.prompt_template + logger.info(f"Text: {text}") + prompt = prompt_template.format(text=text) + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": prompt}, + {"type": "audio", "audio": audio}, + ], + } + ] + + with time_tracker( + "gpt-4o-realtime-call", + task.result_json.latency_profile, + track_type=TrackType.MODEL.value, + ): + res = self.client.chat.completions.create( + model="gpt-4o-realtime-preview-2024-12-17", + modalities=["text", "audio"], + messages=messages, + ) + return res.choices[0].message.content + def gpt_35(self, task: Task) -> Optional[str]: """ Call OpenAI endpoints to convert speech to text diff --git a/Agent/storage.py b/Agent/storage.py index cc586b13..36c3bb84 100644 --- a/Agent/storage.py +++ b/Agent/storage.py @@ -9,11 +9,11 @@ import argparse import multiprocessing +import os import time from pathlib import Path import boto3 -import os import requests from watchdog.observers import Observer