Add interactive checks for Q3VL submission checker

tools/submission/submission_checker/checks/accuracy_check.py

@@ -106,14 +106,37 @@ def accuracy_result_check(self):
         """
 
         if self.is_endpoints:
-            if self.mlperf_log["accuracy_score"] is not None:
-                self.submission_logs.loader_data["accuracy_metrics"] = self.mlperf_log["accuracy_score"]
+            if self.mlperf_log["accuracy_score"] is None:
+                self.log.error("%s accuracy score not found", self.path)
+                return False
+            score = self.mlperf_log["accuracy_score"]
+            self.submission_logs.loader_data["accuracy_metrics"] = score
+            if self.division.lower() == "open":
                 return True
-            self.log.error("%s accuracy score not found", self.path)
-            return False
+            if self.model == "qwen3-vl-235b-a22b":
+                _, acc_targets, acc_types, *_ = self.config.get_accuracy_values(
+                    self.model, self.scenario_fixed
+                )
+                numeric = self._numeric_accuracy_score(score, acc_types)
+                if numeric is None:
+                    self.log.error(
+                        "%s could not parse accuracy score %s",
+                        self.path,
+                        score,
+                    )
+                    return False
+                if numeric < acc_targets[0]:
+                    self.log.warning(
+                        "%s accuracy not met: expected=%f, found=%s",
+                        self.path,
+                        acc_targets[0],
+                        numeric,
+                    )
+                    return False
+            return True
 
         patterns, acc_targets, acc_types, acc_limits, up_patterns, acc_upper_limit = self.config.get_accuracy_values(
-            self.model
+            self.model, self.scenario_fixed
         )
         acc = None
         hash_val = None
@@ -181,6 +204,37 @@ def accuracy_result_check(self):
             return True
         return is_valid
 
+    def _numeric_accuracy_score(self, score, acc_types):
+        """Extract a comparable numeric accuracy from an endpoints score."""
+        if isinstance(score, (int, float)):
+            return float(score)
+        if isinstance(score, str):
+            try:
+                return float(score)
+            except ValueError:
+                return None
+        if isinstance(score, dict):
+            keys = []
+            for acc_type in acc_types:
+                keys.extend(
+                    [
+                        acc_type,
+                        acc_type.lower(),
+                        acc_type.split("_")[-1].lower(),
+                    ]
+                )
+            keys.extend(["f1", "score", "accuracy"])
+            for key in keys:
+                value = score.get(key)
+                if isinstance(value, (int, float)):
+                    return float(value)
+                if isinstance(value, str):
+                    try:
+                        return float(value)
+                    except ValueError:
+                        continue
+        return None
+
     def accuracy_json_check(self):
         """Check that the accuracy JSON exists and is within size limits.
 

tools/submission/submission_checker/checks/compliance_check.py

@@ -51,6 +51,7 @@ def __init__(self, log, path, config: Config,
         self.division = self.submission_logs.loader_data.get("division", "")
         self.model = self.config.get_mlperf_model(
             self.model, self.model_mapping)
+        self.scenario = self.submission_logs.loader_data.get("scenario", "")
         self.test_list = self.get_test_list(self.model)
         self.is_endpoints = self.submission_logs.loader_data.get(
             "is_endpoints_submission", False)
@@ -258,9 +259,9 @@ def accuracy_check(self):
                             is_valid = False
                         else:
                             target = self.config.get_accuracy_target(
-                                self.model)
+                                self.model, self.scenario)
                             patterns, acc_targets, acc_types, acc_limits, up_patterns, acc_upper_limit = self.config.get_accuracy_values(
-                                self.model)
+                                self.model, self.scenario)
                             acc_limit_check = True
 
                             acc_seen = [False for _ in acc_targets]

tools/submission/submission_checker/checks/performance_check.py

@@ -1,3 +1,3 @@
 from .base import BaseCheck
 from ..constants import *
 from ..loader import SubmissionLogs
@@ -64,6 +64,8 @@
     def map_endpoints_scenario(self, scenario, scenario_fixed):
         if scenario_fixed.lower() == "singlestream":
             return "SingleStream"
+        if scenario_fixed.lower() == "interactive" or str(scenario).lower() == "interactive":
+            return "Interactive"
         if scenario.lower() == "online":
             return "Server"
         return scenario
@@ -300,8 +302,12 @@
             # Qwen3VL falls in this category, it has scenario specific e2e
             # latency constraints
             latency_99_percentile = self.mlperf_log["result_99.00_percentile_latency_ns"]
+            constraint_scenario = SCENARIO_MAPPING.get(
+                (self.scenario_fixed or self.scenario).lower(),
+                self.scenario_fixed or self.scenario,
+            )
             target_latency = self.config.latency_constraint.get(
-                self.model, dict()).get(self.scenario)
+                self.model, dict()).get(constraint_scenario)
             self.log.info(
                 "Target latency: %s, Latency: %s, Scenario: %s",
                 target_latency,

tools/submission/submission_checker/configuration/configuration.py

@@ -1,4 +1,4 @@
-from ..constants import MODEL_CONFIG, ACC_PATTERN
+from ..constants import MODEL_CONFIG, ACC_PATTERN, SCENARIO_MAPPING
 
 
 class Config:
@@ -120,23 +120,39 @@ def get_optional(self, model):
             return set()
         return set(self.optional[model])
 
-    def get_accuracy_target(self, model):
+    def get_accuracy_target(self, model, scenario=None):
         if model not in self.accuracy_target:
             raise ValueError("model not known: " + model)
-        return self.accuracy_target[model]
+        target = self.accuracy_target[model]
+        if not isinstance(target, dict):
+            return target
+        if scenario is not None:
+            mapped = SCENARIO_MAPPING.get(str(scenario).lower(), scenario)
+            if mapped in target:
+                return target[mapped]
+            if scenario in target:
+                return target[scenario]
+        if "default" in target:
+            return target["default"]
+        for fallback in ("Offline", "Server"):
+            if fallback in target:
+                return target[fallback]
+        raise ValueError(
+            "no accuracy target for model=%s scenario=%s" % (model, scenario)
+        )
 
     def get_accuracy_upper_limit(self, model):
         return self.accuracy_upper_limit.get(model, None)
 
-    def get_accuracy_values(self, model):
+    def get_accuracy_values(self, model, scenario=None):
         patterns = []
         acc_targets = []
         acc_types = []
         acc_limits = []
         up_patterns = []
         acc_limit_check = False
 
-        target = self.get_accuracy_target(model)
+        target = self.get_accuracy_target(model, scenario)
         acc_upper_limit = self.get_accuracy_upper_limit(model)
         if acc_upper_limit is not None:
             for i in range(0, len(acc_upper_limit), 2):

tools/submission/submission_checker/constants.py

@@ -136,7 +136,12 @@
             "deepseek-r1": ("exact_match", 0.99 * 81.3582, "TOKENS_PER_SAMPLE", 0.9 * 3886.2274),
             "whisper": ("ACCURACY", (100.0 - 2.0671) * 0.99),
             "gpt-oss-120b": ("exact_match", 83.13 * 0.99),
-            "qwen3-vl-235b-a22b": ("F1_HIERARCHICAL", 0.7903 * 0.99),
+            "qwen3-vl-235b-a22b": {
+                "Offline": ("F1_HIERARCHICAL", 0.7903 * 0.99),
+                "Server": ("F1_HIERARCHICAL", 0.7903 * 0.99),
+                # Interactive uses the 8k subset; 0.7878 * 0.99 = 0.779922
+                "Interactive": ("F1_HIERARCHICAL", 0.7878 * 0.99),
+            },
             "dlrm-v3": (
                 "DLRM_NE",
                 0.86687 * 0.999,
@@ -248,7 +253,10 @@
             "llama2-70b-99.9": {"Server": 20000000000},
             "deepseek-r1": {"Server": 60000000000},
             "gpt-oss-120b": {"Server": 60000000000},
-            "qwen3-vl-235b-a22b": {"Server": 12000000000},
+            "qwen3-vl-235b-a22b": {
+                "Server": 12000000000,
+                "Interactive": 1500000000,
+            },
             "dlrm-v3": {"Server": 60000000000},
         },
         "min-queries": {