diff --git a/tools/submission/submission_checker/checks/accuracy_check.py b/tools/submission/submission_checker/checks/accuracy_check.py index 4d8fb00d4e..c18c9d10e0 100644 --- a/tools/submission/submission_checker/checks/accuracy_check.py +++ b/tools/submission/submission_checker/checks/accuracy_check.py @@ -106,14 +106,37 @@ def accuracy_result_check(self): """ if self.is_endpoints: - if self.mlperf_log["accuracy_score"] is not None: - self.submission_logs.loader_data["accuracy_metrics"] = self.mlperf_log["accuracy_score"] + if self.mlperf_log["accuracy_score"] is None: + self.log.error("%s accuracy score not found", self.path) + return False + score = self.mlperf_log["accuracy_score"] + self.submission_logs.loader_data["accuracy_metrics"] = score + if self.division.lower() == "open": return True - self.log.error("%s accuracy score not found", self.path) - return False + if self.model == "qwen3-vl-235b-a22b": + _, acc_targets, acc_types, *_ = self.config.get_accuracy_values( + self.model, self.scenario_fixed + ) + numeric = self._numeric_accuracy_score(score, acc_types) + if numeric is None: + self.log.error( + "%s could not parse accuracy score %s", + self.path, + score, + ) + return False + if numeric < acc_targets[0]: + self.log.warning( + "%s accuracy not met: expected=%f, found=%s", + self.path, + acc_targets[0], + numeric, + ) + return False + return True patterns, acc_targets, acc_types, acc_limits, up_patterns, acc_upper_limit = self.config.get_accuracy_values( - self.model + self.model, self.scenario_fixed ) acc = None hash_val = None @@ -181,6 +204,37 @@ def accuracy_result_check(self): return True return is_valid + def _numeric_accuracy_score(self, score, acc_types): + """Extract a comparable numeric accuracy from an endpoints score.""" + if isinstance(score, (int, float)): + return float(score) + if isinstance(score, str): + try: + return float(score) + except ValueError: + return None + if isinstance(score, dict): + keys = [] + for acc_type in acc_types: + keys.extend( + [ + acc_type, + acc_type.lower(), + acc_type.split("_")[-1].lower(), + ] + ) + keys.extend(["f1", "score", "accuracy"]) + for key in keys: + value = score.get(key) + if isinstance(value, (int, float)): + return float(value) + if isinstance(value, str): + try: + return float(value) + except ValueError: + continue + return None + def accuracy_json_check(self): """Check that the accuracy JSON exists and is within size limits. diff --git a/tools/submission/submission_checker/checks/compliance_check.py b/tools/submission/submission_checker/checks/compliance_check.py index f18d98f000..b8d42d69c7 100644 --- a/tools/submission/submission_checker/checks/compliance_check.py +++ b/tools/submission/submission_checker/checks/compliance_check.py @@ -51,6 +51,7 @@ def __init__(self, log, path, config: Config, self.division = self.submission_logs.loader_data.get("division", "") self.model = self.config.get_mlperf_model( self.model, self.model_mapping) + self.scenario = self.submission_logs.loader_data.get("scenario", "") self.test_list = self.get_test_list(self.model) self.is_endpoints = self.submission_logs.loader_data.get( "is_endpoints_submission", False) @@ -258,9 +259,9 @@ def accuracy_check(self): is_valid = False else: target = self.config.get_accuracy_target( - self.model) + self.model, self.scenario) patterns, acc_targets, acc_types, acc_limits, up_patterns, acc_upper_limit = self.config.get_accuracy_values( - self.model) + self.model, self.scenario) acc_limit_check = True acc_seen = [False for _ in acc_targets] diff --git a/tools/submission/submission_checker/checks/performance_check.py b/tools/submission/submission_checker/checks/performance_check.py index 7ec573d29f..fef9cb055e 100644 --- a/tools/submission/submission_checker/checks/performance_check.py +++ b/tools/submission/submission_checker/checks/performance_check.py @@ -64,6 +64,8 @@ def __init__(self, log, path, config: Config, def map_endpoints_scenario(self, scenario, scenario_fixed): if scenario_fixed.lower() == "singlestream": return "SingleStream" + if scenario_fixed.lower() == "interactive" or str(scenario).lower() == "interactive": + return "Interactive" if scenario.lower() == "online": return "Server" return scenario @@ -300,8 +302,12 @@ def latency_check(self): # Qwen3VL falls in this category, it has scenario specific e2e # latency constraints latency_99_percentile = self.mlperf_log["result_99.00_percentile_latency_ns"] + constraint_scenario = SCENARIO_MAPPING.get( + (self.scenario_fixed or self.scenario).lower(), + self.scenario_fixed or self.scenario, + ) target_latency = self.config.latency_constraint.get( - self.model, dict()).get(self.scenario) + self.model, dict()).get(constraint_scenario) self.log.info( "Target latency: %s, Latency: %s, Scenario: %s", target_latency, diff --git a/tools/submission/submission_checker/configuration/configuration.py b/tools/submission/submission_checker/configuration/configuration.py index 31665c7499..952ce7159d 100644 --- a/tools/submission/submission_checker/configuration/configuration.py +++ b/tools/submission/submission_checker/configuration/configuration.py @@ -1,4 +1,4 @@ -from ..constants import MODEL_CONFIG, ACC_PATTERN +from ..constants import MODEL_CONFIG, ACC_PATTERN, SCENARIO_MAPPING class Config: @@ -120,15 +120,31 @@ def get_optional(self, model): return set() return set(self.optional[model]) - def get_accuracy_target(self, model): + def get_accuracy_target(self, model, scenario=None): if model not in self.accuracy_target: raise ValueError("model not known: " + model) - return self.accuracy_target[model] + target = self.accuracy_target[model] + if not isinstance(target, dict): + return target + if scenario is not None: + mapped = SCENARIO_MAPPING.get(str(scenario).lower(), scenario) + if mapped in target: + return target[mapped] + if scenario in target: + return target[scenario] + if "default" in target: + return target["default"] + for fallback in ("Offline", "Server"): + if fallback in target: + return target[fallback] + raise ValueError( + "no accuracy target for model=%s scenario=%s" % (model, scenario) + ) def get_accuracy_upper_limit(self, model): return self.accuracy_upper_limit.get(model, None) - def get_accuracy_values(self, model): + def get_accuracy_values(self, model, scenario=None): patterns = [] acc_targets = [] acc_types = [] @@ -136,7 +152,7 @@ def get_accuracy_values(self, model): up_patterns = [] acc_limit_check = False - target = self.get_accuracy_target(model) + target = self.get_accuracy_target(model, scenario) acc_upper_limit = self.get_accuracy_upper_limit(model) if acc_upper_limit is not None: for i in range(0, len(acc_upper_limit), 2): diff --git a/tools/submission/submission_checker/constants.py b/tools/submission/submission_checker/constants.py index 3aa5c93b21..a0531aff7e 100644 --- a/tools/submission/submission_checker/constants.py +++ b/tools/submission/submission_checker/constants.py @@ -136,7 +136,12 @@ "deepseek-r1": ("exact_match", 0.99 * 81.3582, "TOKENS_PER_SAMPLE", 0.9 * 3886.2274), "whisper": ("ACCURACY", (100.0 - 2.0671) * 0.99), "gpt-oss-120b": ("exact_match", 83.13 * 0.99), - "qwen3-vl-235b-a22b": ("F1_HIERARCHICAL", 0.7903 * 0.99), + "qwen3-vl-235b-a22b": { + "Offline": ("F1_HIERARCHICAL", 0.7903 * 0.99), + "Server": ("F1_HIERARCHICAL", 0.7903 * 0.99), + # Interactive uses the 8k subset; 0.7878 * 0.99 = 0.779922 + "Interactive": ("F1_HIERARCHICAL", 0.7878 * 0.99), + }, "dlrm-v3": ( "DLRM_NE", 0.86687 * 0.999, @@ -248,7 +253,10 @@ "llama2-70b-99.9": {"Server": 20000000000}, "deepseek-r1": {"Server": 60000000000}, "gpt-oss-120b": {"Server": 60000000000}, - "qwen3-vl-235b-a22b": {"Server": 12000000000}, + "qwen3-vl-235b-a22b": { + "Server": 12000000000, + "Interactive": 1500000000, + }, "dlrm-v3": {"Server": 60000000000}, }, "min-queries": {