From 2d2e220f9c06d36fa746b9392b679c1aed9d4206 Mon Sep 17 00:00:00 2001 From: Brian Dellabetta Date: Fri, 7 Mar 2025 20:27:17 +0000 Subject: [PATCH] change setting of random seed to configuration Signed-off-by: Brian Dellabetta --- tests/lmeval/configs/vl_fp8_dynamic_per_token.yaml | 1 + tests/lmeval/configs/vl_int8_w8a8_dynamic_per_token.yaml | 1 + tests/lmeval/configs/vl_w4a16_actorder_weight.yaml | 1 + tests/lmeval/test_lmeval.py | 9 +++++++-- 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/lmeval/configs/vl_fp8_dynamic_per_token.yaml b/tests/lmeval/configs/vl_fp8_dynamic_per_token.yaml index 223549995..2b2bb6601 100644 --- a/tests/lmeval/configs/vl_fp8_dynamic_per_token.yaml +++ b/tests/lmeval/configs/vl_fp8_dynamic_per_token.yaml @@ -2,6 +2,7 @@ cadence: weekly model: Qwen/Qwen2-VL-2B-Instruct model_class: TraceableQwen2VLForConditionalGeneration scheme: FP8_DYNAMIC +seed: 42 #compressed model is sensitive to random seed lmeval: model: "hf-multimodal" model_args: diff --git a/tests/lmeval/configs/vl_int8_w8a8_dynamic_per_token.yaml b/tests/lmeval/configs/vl_int8_w8a8_dynamic_per_token.yaml index 19544acdd..aa53c52c4 100644 --- a/tests/lmeval/configs/vl_int8_w8a8_dynamic_per_token.yaml +++ b/tests/lmeval/configs/vl_int8_w8a8_dynamic_per_token.yaml @@ -5,6 +5,7 @@ scheme: INT8_dyn_per_token recipe: tests/e2e/vLLM/recipes/INT8/recipe_int8_channel_weight_dynamic_per_token.yaml dataset_id: lmms-lab/flickr30k dataset_split: "test[:512]" +seed: 42 #compressed model is sensitive to random seed lmeval: model: "hf-multimodal" model_args: diff --git a/tests/lmeval/configs/vl_w4a16_actorder_weight.yaml b/tests/lmeval/configs/vl_w4a16_actorder_weight.yaml index 692c0f0eb..b28cf3dd7 100644 --- a/tests/lmeval/configs/vl_w4a16_actorder_weight.yaml +++ b/tests/lmeval/configs/vl_w4a16_actorder_weight.yaml @@ -5,6 +5,7 @@ recipe: tests/e2e/vLLM/recipes/actorder/recipe_w4a16_actorder_weight.yaml dataset_id: lmms-lab/flickr30k dataset_split: "test[:512]" scheme: W4A16_actorder_group +seed: 42 #compressed model is sensitive to random seed lmeval: model: "hf-multimodal" model_args: diff --git a/tests/lmeval/test_lmeval.py b/tests/lmeval/test_lmeval.py index 1b53333e5..34db1356d 100644 --- a/tests/lmeval/test_lmeval.py +++ b/tests/lmeval/test_lmeval.py @@ -1,4 +1,5 @@ import os +import random import shutil from pathlib import Path @@ -58,8 +59,6 @@ class TestLMEval: """ # noqa: E501 def set_up(self): - torch.manual_seed(42) - eval_config = yaml.safe_load(Path(TEST_DATA_FILE).read_text(encoding="utf-8")) if os.environ.get("CADENCE", "commit") != eval_config.get("cadence"): @@ -76,6 +75,12 @@ def set_up(self): self.quant_type = eval_config.get("quant_type") self.save_dir = eval_config.get("save_dir") + seed = eval_config.get("seed", None) + if seed is not None: + random.seed(seed) + numpy.random.seed(seed) + torch.manual_seed(seed) + logger.info("========== RUNNING ==============") logger.info(self.scheme)