Feature/realtime

API/hardware/views.py

@@ -7,7 +7,7 @@
 from django.conf import settings
 from django.http import HttpResponse
 from drf_yasg.utils import swagger_auto_schema
-from moviepy.editor import VideoFileClip, concatenate_videoclips
+from moviepy import VideoFileClip, concatenate_videoclips
 from rest_framework import status, viewsets
 from rest_framework.decorators import action
 from rest_framework.response import Response

API/orchestrator/chain/completed_task.py

@@ -78,6 +78,13 @@ def trigger_completed_task(sender, **kwargs):
         return completed_openai_gpt_4o_text_only.send(
             sender=sender, data=data, track_id=task_data.track_id
         )
+
+    if task_data.task_name == "openai_gpt_4o_realtime":
+        logger.info("OpenAI GPT4O Realtime task completed")
+        return completed_openai_gpt_4o_text_only.send(
+            sender=sender, data=data, track_id=task_data.track_id
+        )
+
     if task_data.task_name == "rag":
         logger.info("RAG task completed")
         return completed_rag.send(sender=sender, data=data, track_id=task_data.track_id)

API/orchestrator/models.py

@@ -138,6 +138,7 @@ def get_task_name_choices():
             ("openai_text2speech", "OpenAI Text2Speech"),
             ("openai_gpt_4o_text_and_image", "OpenAI GPT4o Text and Image"),
             ("openai_gpt_4o_text_only", "OpenAI GPT4o Text"),
+            ("openai_gpt_4o_realtime", "OpenAI GPT4o Realtime"),
             ("rag", "RAG"),
         ]
 
@@ -156,6 +157,7 @@ def task_ml_task_mapping() -> dict:
             "openai_text2speech": "text2speech",
             "openai_gpt_4o_text_and_image": "text_generation",
             "openai_gpt_4o_text_only": "text_generation",
+            "openai_gpt_4o_realtime": "text_generation",
             "rag": "rag",
         }
 

Agent/main.py

@@ -294,6 +294,7 @@ def handle_rag_task(self, task: Task):
                 TaskName.text2speech.value,
                 TaskName.openai_gpt4o_text_only.value,
                 TaskName.openai_gpt_4o_text_and_image.value,
+                TaskName.openai_gpt_4o_realtime.value,
                 TaskName.rag.value,
             ]
         else:

Agent/models/parameters.py

@@ -55,3 +55,9 @@ class OpenAIGPT4OParameters(BaseModel):
 class OpenAIGPT4OTextOnlyParameters(BaseModel):
     text: str = Field(..., description="The text to analyze for GPT-4o text only")
     prompt_template: str = Field(description="The prompt template")
+
+class OpenAIGPT4ORealTimeParameters(BaseModel):
+    text: str = Field(..., description="The text to analyze for GPT-4o real time")
+    audio_file: str = Field(..., description="The audio data for GPT-4o real time")
+    prompt_template: str = Field(description="The prompt template")
+    sample_ratio: int = Field(1, description="The sample ratio")

Agent/models/task.py

@@ -17,6 +17,7 @@ class TaskName(str, Enum):
     openai_gpt_35 = "openai_gpt_35"
     openai_gpt4o_text_only = "openai_gpt_4o_text_only"
     openai_gpt_4o_text_and_image = "openai_gpt_4o_text_and_image"
+    openai_gpt_4o_realtime = "openai_gpt_4o_realtime"
     rag = "rag"
 
 

Agent/modules/openai/handler.py

@@ -6,6 +6,7 @@
 
 from models.parameters import (
     OpenAIGPT4OParameters,
+    OpenAIGPT4ORealTimeParameters,
     OpenAIGPT4OTextOnlyParameters,
     Speech2TextParameters,
     Text2SpeechParameters,
@@ -57,6 +58,11 @@ def handle_task(self, task: Task) -> Task:
             text = self.gpt_35(task)
             TimeLogger.log(latency_profile, "end_openai_gpt_35")
             result_profile["text"] = text
+        if "openai_gpt_4o_realtime" in task.task_name:
+            TimeLogger.log(latency_profile, "start_openai_gpt_4o_realtime")
+            text = self.gpt_4o_realtime(task)
+            TimeLogger.log(latency_profile, "end_openai_gpt_4o_realtime")
+            result_profile["text"] = text
         if "text2speech" in task.task_name:
             TimeLogger.log(latency_profile, "start_openai_text2speech")
             audio_file_path = self.text2speech(task)
@@ -147,6 +153,43 @@ def gpt_4o_text_only(self, task: Task) -> str:
             )
         return res.choices[0].message.content
 
+    def gpt_4o_realtime(self, task: Task) -> Optional[str]:
+        """
+        Get the text and audio in real time
+        Args:
+            task:
+
+        Returns:
+
+        """
+        params = OpenAIGPT4ORealTimeParameters(**task.parameters)
+        text = params.text
+        audio = params.audio_file
+        prompt_template = params.prompt_template
+        logger.info(f"Text: {text}")
+        prompt = prompt_template.format(text=text)
+        messages = [
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": prompt},
+                    {"type": "audio", "audio": audio},
+                ],
+            }
+        ]
+
+        with time_tracker(
+                "gpt-4o-realtime-call",
+                task.result_json.latency_profile,
+                track_type=TrackType.MODEL.value,
+        ):
+            res = self.client.chat.completions.create(
+                model="gpt-4o-realtime-preview-2024-12-17",
+                modalities=["text", "audio"],
+                messages=messages,
+            )
+        return res.choices[0].message.content
+
     def gpt_35(self, task: Task) -> Optional[str]:
         """
         Call OpenAI endpoints to convert speech to text

Agent/storage.py

@@ -9,11 +9,11 @@
 
 import argparse
 import multiprocessing
+import os
 import time
 from pathlib import Path
 
 import boto3
-import os
 import requests
 from watchdog.observers import Observer