Updates the baseline evals with embedded 3 large (#2533)

Author: pamelafox

evals/results/baseline/README.md

@@ -0,0 +1,8 @@
+# Baseline Evaluation
+
+This evaluation was done with the application using the following models:
+
+* Chat completion: gpt-4o-mini
+* Embedding: text-embedding-3-large (with binary quantization, 1024 dimension reducation, and oversampling)
+
+These are the default models and settings as of May 8, 2025.

evals/results/baseline/config.json

@@ -1,6 +1,6 @@
 {
     "testdata_path": "ground_truth.jsonl",
-    "results_dir": "results/gpt-4o-mini",
+    "results_dir": "results/experiment<TIMESTAMP>",
     "requested_metrics": ["gpt_groundedness", "gpt_relevance", "answer_length", "latency", "citations_matched", "any_citation"],
     "target_url": "http://localhost:50505/chat",
     "target_parameters": {

evals/results/baseline/eval_results.jsonl

@@ -1,6 +1,6 @@
 {
     "evaluation_gpt_model": "gpt-4o",
-    "evaluation_timestamp": 1744920281,
+    "evaluation_timestamp": 1746818372,
     "testdata_path": "/Users/pamelafox/azure-search-openai-demo/evals/ground_truth.jsonl",
     "target_url": "http://localhost:50505/chat",
     "target_parameters": {

evals/results/baseline/evaluate_parameters.json

@@ -1,27 +1,27 @@
 {
     "gpt_groundedness": {
-        "pass_count": 44,
-        "pass_rate": 0.88,
-        "mean_rating": 4.62
+        "pass_count": 43,
+        "pass_rate": 0.86,
+        "mean_rating": 4.5
     },
     "gpt_relevance": {
         "pass_count": 42,
         "pass_rate": 0.84,
-        "mean_rating": 4.12
+        "mean_rating": 4.22
     },
     "answer_length": {
-        "mean": 922.42,
-        "max": 1616,
+        "mean": 919.26,
+        "max": 1647,
         "min": 193
     },
     "latency": {
-        "mean": 3.14,
-        "max": 7.583068,
-        "min": 1.598833
+        "mean": 4.46,
+        "max": 15.129978,
+        "min": 2.465542
     },
     "citations_matched": {
-        "total": 25,
-        "rate": 0.5
+        "total": 24,
+        "rate": 0.49
     },
     "any_citation": {
         "total": 50,

evals/results/baseline/summary.json

@@ -0,0 +1,28 @@
+{
+    "testdata_path": "ground_truth.jsonl",
+    "results_dir": "results/gpt-4o-mini",
+    "requested_metrics": ["gpt_groundedness", "gpt_relevance", "answer_length", "latency", "citations_matched", "any_citation"],
+    "target_url": "http://localhost:50505/chat",
+    "target_parameters": {
+        "overrides": {
+            "top": 3,
+            "temperature": 0.3,
+            "minimum_reranker_score": 0,
+            "minimum_search_score": 0,
+            "retrieval_mode": "hybrid",
+            "semantic_ranker": true,
+            "semantic_captions": false,
+            "suggest_followup_questions": false,
+            "use_oid_security_filter": false,
+            "use_groups_security_filter": false,
+            "vector_fields": [
+                "embedding"
+            ],
+            "use_gpt4v": false,
+            "gpt4v_input": "textAndImages",
+            "seed": 1
+        }
+    },
+    "target_response_answer_jmespath": "message.content",
+    "target_response_context_jmespath": "context.data_points.text"
+}

evals/results/gpt-35-turbo/config.json renamed to evals/results/gpt35turbo-ada002/config.json

@@ -0,0 +1,27 @@
+{
+    "evaluation_gpt_model": "gpt-4o",
+    "evaluation_timestamp": 1744920281,
+    "testdata_path": "/Users/pamelafox/azure-search-openai-demo/evals/ground_truth.jsonl",
+    "target_url": "http://localhost:50505/chat",
+    "target_parameters": {
+        "overrides": {
+            "top": 3,
+            "temperature": 0.3,
+            "minimum_reranker_score": 0,
+            "minimum_search_score": 0,
+            "retrieval_mode": "hybrid",
+            "semantic_ranker": true,
+            "semantic_captions": false,
+            "suggest_followup_questions": false,
+            "use_oid_security_filter": false,
+            "use_groups_security_filter": false,
+            "vector_fields": [
+                "embedding"
+            ],
+            "use_gpt4v": false,
+            "gpt4v_input": "textAndImages",
+            "seed": 1
+        }
+    },
+    "num_questions": null
+}

evals/results/gpt-35-turbo/eval_results.jsonl renamed to evals/results/gpt35turbo-ada002/eval_results.jsonl

@@ -0,0 +1,33 @@
+{
+    "gpt_groundedness": {
+        "pass_count": 44,
+        "pass_rate": 0.88,
+        "mean_rating": 4.62
+    },
+    "gpt_relevance": {
+        "pass_count": 42,
+        "pass_rate": 0.84,
+        "mean_rating": 4.12
+    },
+    "answer_length": {
+        "mean": 922.42,
+        "max": 1616,
+        "min": 193
+    },
+    "latency": {
+        "mean": 3.14,
+        "max": 7.583068,
+        "min": 1.598833
+    },
+    "citations_matched": {
+        "total": 25,
+        "rate": 0.5
+    },
+    "any_citation": {
+        "total": 50,
+        "rate": 1.0
+    },
+    "num_questions": {
+        "total": 50
+    }
+}