Merge branch 'main' into dependabot/pip/src/backend/h11-0.16.0

Author: pamelafox

.env.sample

@@ -5,20 +5,20 @@ POSTGRES_PASSWORD=postgres
 POSTGRES_DATABASE=postgres
 POSTGRES_SSL=disable
 
-# OPENAI_CHAT_HOST can be either azure, openai, or ollama:
+# OPENAI_CHAT_HOST can be either azure, openai, ollama, or github:
 OPENAI_CHAT_HOST=azure
-# OPENAI_EMBED_HOST can be either azure or openai:
+# OPENAI_EMBED_HOST can be either azure, openai, ollama, or github:
 OPENAI_EMBED_HOST=azure
 # Needed for Azure:
 # You also need to `azd auth login` if running this locally
 AZURE_OPENAI_ENDPOINT=https://YOUR-AZURE-OPENAI-SERVICE-NAME.openai.azure.com
 AZURE_OPENAI_VERSION=2024-03-01-preview
 AZURE_OPENAI_CHAT_DEPLOYMENT=gpt-4o-mini
 AZURE_OPENAI_CHAT_MODEL=gpt-4o-mini
-AZURE_OPENAI_EMBED_DEPLOYMENT=text-embedding-ada-002
-AZURE_OPENAI_EMBED_MODEL=text-embedding-ada-002
-AZURE_OPENAI_EMBED_DIMENSIONS=1536
-AZURE_OPENAI_EMBEDDING_COLUMN=embedding_ada002
+AZURE_OPENAI_EMBED_DEPLOYMENT=text-embedding-3-large
+AZURE_OPENAI_EMBED_MODEL=text-embedding-3-large
+AZURE_OPENAI_EMBED_DIMENSIONS=1024
+AZURE_OPENAI_EMBEDDING_COLUMN=embedding_3l
 AZURE_OPENAI_EVAL_DEPLOYMENT=gpt-4
 AZURE_OPENAI_EVAL_MODEL=gpt-4
 AZURE_TENANT_ID=
@@ -27,11 +27,18 @@ AZURE_OPENAI_KEY=
 # Needed for OpenAI.com:
 OPENAICOM_KEY=YOUR-OPENAI-API-KEY
 OPENAICOM_CHAT_MODEL=gpt-3.5-turbo
-OPENAICOM_EMBED_MODEL=text-embedding-ada-002
-OPENAICOM_EMBED_MODEL_DIMENSIONS=1536
-OPENAICOM_EMBEDDING_COLUMN=embedding_ada002
+OPENAICOM_EMBED_MODEL=text-embedding-3-large
+OPENAICOM_EMBED_DIMENSIONS=1024
+OPENAICOM_EMBEDDING_COLUMN=embedding_3l
 # Needed for Ollama:
 OLLAMA_ENDPOINT=http://host.docker.internal:11434/v1
 OLLAMA_CHAT_MODEL=llama3.1
 OLLAMA_EMBED_MODEL=nomic-embed-text
 OLLAMA_EMBEDDING_COLUMN=embedding_nomic
+# Needed for GitHub Models:
+GITHUB_TOKEN=YOUR-GITHUB-TOKEN
+GITHUB_BASE_URL=https://models.inference.ai.azure.com
+GITHUB_MODEL=gpt-4o
+GITHUB_EMBED_MODEL=text-embedding-3-large
+GITHUB_EMBED_DIMENSIONS=1024
+GITHUB_EMBEDDING_COLUMN=embedding_3l

.github/workflows/app-tests.yaml

@@ -85,7 +85,7 @@ jobs:
             architecture: x64
 
         - name: Install uv
-          uses: astral-sh/setup-uv@v5
+          uses: astral-sh/setup-uv@v6
           with:
             enable-cache: true
             version: "0.4.20"

.github/workflows/evaluate.yaml

@@ -82,7 +82,7 @@ jobs:
           python-version: '3.12'
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v5
+        uses: astral-sh/setup-uv@v6
         with:
           enable-cache: true
           version: "0.4.20"

README.md

@@ -123,7 +123,7 @@ Once you've opened the project in [Codespaces](#github-codespaces), [Dev Contain
     azd up
     ```
 
-    You will be asked to select two locations, first a region for most of the resources (Container Apps, PostgreSQL), then a region specifically for the Azure OpenAI models. This project uses the gpt-4o-mini and text-embedding-ada-002 models which may not be available in all Azure regions. Check for [up-to-date region availability](https://learn.microsoft.com/azure/ai-services/openai/concepts/models#standard-deployment-model-availability) and select a region accordingly.
+    You will be asked to select two locations, first a region for most of the resources (Container Apps, PostgreSQL), then a region specifically for the Azure OpenAI models. This project uses the gpt-4o-mini and text-embedding-3-large models which may not be available in all Azure regions. Check for [up-to-date region availability](https://learn.microsoft.com/azure/ai-services/openai/concepts/models#standard-deployment-model-availability) and select a region accordingly.
 
 ## Local Development
 

evals/evaluate.py

@@ -66,6 +66,10 @@ def get_openai_config() -> dict:
             # azure-ai-evaluate will call DefaultAzureCredential behind the scenes,
             # so we must be logged in to Azure CLI with the correct tenant
         openai_config["model"] = os.environ["AZURE_OPENAI_EVAL_MODEL"]
+    elif os.environ.get("OPENAI_CHAT_HOST") == "ollama":
+        raise NotImplementedError("Ollama is not supported. Switch to Azure or OpenAI.com")
+    elif os.environ.get("OPENAI_CHAT_HOST") == "github":
+        raise NotImplementedError("GitHub Models is not supported. Switch to Azure or OpenAI.com")
     else:
         logger.info("Using OpenAI Service with API Key from OPENAICOM_KEY")
         openai_config = {"api_key": os.environ["OPENAICOM_KEY"], "model": "gpt-4"}

evals/generate_ground_truth.py

@@ -101,7 +101,9 @@ def get_openai_client() -> tuple[Union[AzureOpenAI, OpenAI], str]:
             )
         model = os.environ["AZURE_OPENAI_CHAT_DEPLOYMENT"]
     elif OPENAI_CHAT_HOST == "ollama":
-        raise NotImplementedError("Ollama OpenAI Service is not supported. Switch to Azure or OpenAI.com")
+        raise NotImplementedError("Ollama is not supported. Switch to Azure or OpenAI.com")
+    elif OPENAI_CHAT_HOST == "github":
+        raise NotImplementedError("GitHub Models is not supported. Switch to Azure or OpenAI.com")
     else:
         logger.info("Using OpenAI Service with API Key from OPENAICOM_KEY")
         openai_client = OpenAI(api_key=os.environ["OPENAICOM_KEY"])

evals/safety_evaluation.py

@@ -106,15 +106,13 @@ async def run_simulator(target_url: str, max_simulations: int):
             else:
                 logger.info(f"Failing score from:\nQ: {query}\nA: {answer}\n{evaluator} score: {eval_score}")
             numeric_severity_score = eval_score[f"{evaluator}_score"]
-            if isinstance(numeric_severity_score, float):
+            if isinstance(numeric_severity_score, float) or isinstance(numeric_severity_score, int):
                 summary_scores[evaluator]["score_total"] += numeric_severity_score
 
     # Compute the overall statistics
     for evaluator in evaluators:
         if len(outputs) > 0:
-            summary_scores[evaluator]["mean_score"] = (
-                summary_scores[evaluator]["score_total"] / summary_scores[evaluator]["low_count"]
-            )
+            summary_scores[evaluator]["mean_score"] = summary_scores[evaluator]["score_total"] / len(outputs)
             summary_scores[evaluator]["low_rate"] = summary_scores[evaluator]["low_count"] / len(outputs)
 
     # Save summary scores

infra/main.bicep

@@ -302,12 +302,12 @@ var webAppEnv = union(azureOpenAIKeyEnv, openAIComKeyEnv, [
     value: openAIEmbedHost
   }
   {
-    name: 'OPENAICOM_EMBED_MODEL_DIMENSIONS'
-    value: openAIEmbedHost == 'openaicom' ? '1536' : ''
+    name: 'OPENAICOM_EMBED_DIMENSIONS'
+    value: openAIEmbedHost == 'openaicom' ? '1024' : ''
   }
   {
     name: 'OPENAICOM_EMBED_MODEL'
-    value: openAIEmbedHost == 'openaicom' ? 'text-embedding-ada-002' : ''
+    value: openAIEmbedHost == 'openaicom' ? 'text-embedding-3-large' : ''
   }
   {
     name: 'AZURE_OPENAI_EMBED_MODEL'

infra/main.parameters.json

@@ -69,22 +69,22 @@
         "value": "${OPENAI_EMBED_HOST=azure}"
       },
       "embedModelName":{
-        "value": "${AZURE_OPENAI_EMBED_MODEL=text-embedding-ada-002}"
+        "value": "${AZURE_OPENAI_EMBED_MODEL=text-embedding-3-large}"
       },
       "embedDeploymentName": {
-        "value": "${AZURE_OPENAI_EMBED_DEPLOYMENT=text-embedding-ada-002}"
+        "value": "${AZURE_OPENAI_EMBED_DEPLOYMENT=text-embedding-3-large}"
       },
       "embedDeploymentVersion":{
-        "value": "${AZURE_OPENAI_EMBED_DEPLOYMENT_VERSION=2}"
+        "value": "${AZURE_OPENAI_EMBED_DEPLOYMENT_VERSION=1}"
       },
       "embedDeploymentSku": {
-        "value": "${AZURE_OPENAI_EMBED_DEPLOYMENT_SKU=Standard}"
+        "value": "${AZURE_OPENAI_EMBED_DEPLOYMENT_SKU=GlobalStandard}"
       },
       "embedDeploymentCapacity":{
         "value": "${AZURE_OPENAI_EMBED_DEPLOYMENT_CAPACITY=30}"
       },
       "embedDimensions": {
-        "value": "${AZURE_OPENAI_EMBED_DIMENSIONS=1536}"
+        "value": "${AZURE_OPENAI_EMBED_DIMENSIONS=1024}"
       },
       "openAIComKey": {
         "value": "${OPENAICOM_KEY}"

src/backend/fastapi_app/dependencies.py

@@ -42,27 +42,35 @@ async def common_parameters():
     OPENAI_EMBED_HOST = os.getenv("OPENAI_EMBED_HOST")
     OPENAI_CHAT_HOST = os.getenv("OPENAI_CHAT_HOST")
     if OPENAI_EMBED_HOST == "azure":
-        openai_embed_deployment = os.getenv("AZURE_OPENAI_EMBED_DEPLOYMENT") or "text-embedding-ada-002"
-        openai_embed_model = os.getenv("AZURE_OPENAI_EMBED_MODEL") or "text-embedding-ada-002"
-        openai_embed_dimensions = int(os.getenv("AZURE_OPENAI_EMBED_DIMENSIONS") or 1536)
-        embedding_column = os.getenv("AZURE_OPENAI_EMBEDDING_COLUMN") or "embedding_ada002"
+        openai_embed_deployment = os.getenv("AZURE_OPENAI_EMBED_DEPLOYMENT") or "text-embedding-3-large"
+        openai_embed_model = os.getenv("AZURE_OPENAI_EMBED_MODEL") or "text-embedding-3-large"
+        openai_embed_dimensions = int(os.getenv("AZURE_OPENAI_EMBED_DIMENSIONS") or 1024)
+        embedding_column = os.getenv("AZURE_OPENAI_EMBEDDING_COLUMN") or "embedding_3l"
     elif OPENAI_EMBED_HOST == "ollama":
         openai_embed_deployment = None
         openai_embed_model = os.getenv("OLLAMA_EMBED_MODEL") or "nomic-embed-text"
         openai_embed_dimensions = None
         embedding_column = os.getenv("OLLAMA_EMBEDDING_COLUMN") or "embedding_nomic"
+    elif OPENAI_EMBED_HOST == "github":
+        openai_embed_deployment = None
+        openai_embed_model = os.getenv("GITHUB_EMBED_MODEL") or "text-embedding-3-large"
+        openai_embed_dimensions = int(os.getenv("GITHUB_EMBED_DIMENSIONS", 1024))
+        embedding_column = os.getenv("GITHUB_EMBEDDING_COLUMN") or "embedding_3l"
     else:
         openai_embed_deployment = None
-        openai_embed_model = os.getenv("OPENAICOM_EMBED_MODEL") or "text-embedding-ada-002"
-        openai_embed_dimensions = int(os.getenv("OPENAICOM_EMBED_DIMENSIONS", 1536))
-        embedding_column = os.getenv("OPENAICOM_EMBEDDING_COLUMN") or "embedding_ada002"
+        openai_embed_model = os.getenv("OPENAICOM_EMBED_MODEL") or "text-embedding-3-large"
+        openai_embed_dimensions = int(os.getenv("OPENAICOM_EMBED_DIMENSIONS", 1024))
+        embedding_column = os.getenv("OPENAICOM_EMBEDDING_COLUMN") or "embedding_3l"
     if OPENAI_CHAT_HOST == "azure":
         openai_chat_deployment = os.getenv("AZURE_OPENAI_CHAT_DEPLOYMENT") or "gpt-4o-mini"
         openai_chat_model = os.getenv("AZURE_OPENAI_CHAT_MODEL") or "gpt-4o-mini"
     elif OPENAI_CHAT_HOST == "ollama":
         openai_chat_deployment = None
         openai_chat_model = os.getenv("OLLAMA_CHAT_MODEL") or "phi3:3.8b"
         openai_embed_model = os.getenv("OLLAMA_EMBED_MODEL") or "nomic-embed-text"
+    elif OPENAI_CHAT_HOST == "github":
+        openai_chat_deployment = None
+        openai_chat_model = os.getenv("GITHUB_MODEL") or "gpt-4o"
     else:
         openai_chat_deployment = None
         openai_chat_model = os.getenv("OPENAICOM_CHAT_MODEL") or "gpt-3.5-turbo"

src/backend/fastapi_app/openai_clients.py

@@ -50,6 +50,15 @@ async def create_openai_chat_client(
             base_url=os.getenv("OLLAMA_ENDPOINT"),
             api_key="nokeyneeded",
         )
+    elif OPENAI_CHAT_HOST == "github":
+        logger.info("Setting up OpenAI client for chat completions using GitHub Models")
+        github_base_url = os.getenv("GITHUB_BASE_URL", "https://models.inference.ai.azure.com")
+        github_model = os.getenv("GITHUB_MODEL", "gpt-4o")
+        logger.info(f"Using GitHub Models with base URL: {github_base_url}, model: {github_model}")
+        openai_chat_client = openai.AsyncOpenAI(
+            base_url=github_base_url,
+            api_key=os.getenv("GITHUB_TOKEN"),
+        )
     else:
         logger.info("Setting up OpenAI client for chat completions using OpenAI.com API key")
         openai_chat_client = openai.AsyncOpenAI(api_key=os.getenv("OPENAICOM_KEY"))
@@ -99,6 +108,15 @@ async def create_openai_embed_client(
             base_url=os.getenv("OLLAMA_ENDPOINT"),
             api_key="nokeyneeded",
         )
+    elif OPENAI_EMBED_HOST == "github":
+        logger.info("Setting up OpenAI client for embeddings using GitHub Models")
+        github_base_url = os.getenv("GITHUB_BASE_URL", "https://models.inference.ai.azure.com")
+        github_embed_model = os.getenv("GITHUB_EMBED_MODEL", "text-embedding-3-small")
+        logger.info(f"Using GitHub Models with base URL: {github_base_url}, embedding model: {github_embed_model}")
+        openai_embed_client = openai.AsyncOpenAI(
+            base_url=github_base_url,
+            api_key=os.getenv("GITHUB_TOKEN"),
+        )
     else:
         logger.info("Setting up OpenAI client for embeddings using OpenAI.com API key")
         openai_embed_client = openai.AsyncOpenAI(api_key=os.getenv("OPENAICOM_KEY"))

src/backend/fastapi_app/postgres_models.py

@@ -19,16 +19,16 @@ class Item(Base):
     description: Mapped[str] = mapped_column()
     price: Mapped[float] = mapped_column()
     # Embeddings for different models:
-    embedding_ada002: Mapped[Vector] = mapped_column(Vector(1536), nullable=True)  # ada-002
+    embedding_3l: Mapped[Vector] = mapped_column(Vector(1024), nullable=True)  # text-embedding-3-large
     embedding_nomic: Mapped[Vector] = mapped_column(Vector(768), nullable=True)  # nomic-embed-text
 
     def to_dict(self, include_embedding: bool = False):
         model_dict = {column.name: getattr(self, column.name) for column in self.__table__.columns}
         if include_embedding:
-            model_dict["embedding_ada002"] = model_dict.get("embedding_ada002", [])
+            model_dict["embedding_3l"] = model_dict.get("embedding_3l", [])
             model_dict["embedding_nomic"] = model_dict.get("embedding_nomic", [])
         else:
-            del model_dict["embedding_ada002"]
+            del model_dict["embedding_3l"]
             del model_dict["embedding_nomic"]
         return model_dict
 
@@ -51,12 +51,12 @@ def to_str_for_embedding(self):
 
 table_name = Item.__tablename__
 
-index_ada002 = Index(
-    "hnsw_index_for_cosine_{table_name}_embedding_ada002",
-    Item.embedding_ada002,
+index_3l = Index(
+    f"hnsw_index_for_cosine_{table_name}_embedding_3l",
+    Item.embedding_3l,
     postgresql_using="hnsw",
     postgresql_with={"m": 16, "ef_construction": 64},
-    postgresql_ops={"embedding_ada002": "vector_cosine_ops"},
+    postgresql_ops={"embedding_3l": "vector_cosine_ops"},
 )
 
 index_nomic = Index(

src/backend/fastapi_app/query_rewriter.py

@@ -69,7 +69,7 @@ def extract_search_arguments(original_user_query: str, chat_completion: ChatComp
                 arg = json.loads(function.arguments)
                 # Even though its required, search_query is not always specified
                 search_query = arg.get("search_query", original_user_query)
-                if "price_filter" in arg and arg["price_filter"]:
+                if "price_filter" in arg and arg["price_filter"] and isinstance(arg["price_filter"], dict):
                     price_filter = arg["price_filter"]
                     filters.append(
                         {
@@ -78,7 +78,7 @@ def extract_search_arguments(original_user_query: str, chat_completion: ChatComp
                             "value": price_filter["value"],
                         }
                     )
-                if "brand_filter" in arg and arg["brand_filter"]:
+                if "brand_filter" in arg and arg["brand_filter"] and isinstance(arg["brand_filter"], dict):
                     brand_filter = arg["brand_filter"]
                     filters.append(
                         {

src/backend/fastapi_app/routes/api_routes.py

@@ -68,7 +68,7 @@ async def similar_handler(
                 f"SELECT *, {context.embedding_column} <=> :embedding as DISTANCE FROM {Item.__tablename__} "
                 "WHERE id <> :item_id ORDER BY distance LIMIT :n"
             ),
-            {"embedding": item.embedding_ada002, "n": n, "item_id": id},
+            {"embedding": getattr(item, context.embedding_column), "n": n, "item_id": id},
         )
     ).fetchall()
 
@@ -145,6 +145,7 @@ async def chat_handler(
         if isinstance(e, APIError) and e.code == "content_filter":
             return ERROR_FILTER
         else:
+            logging.exception("Exception while generating response: %s", e)
             return {"error": str(e)}