From b91cd7b9947e7155cba8d71c2a7135cd4170b839 Mon Sep 17 00:00:00 2001 From: Brian Dellabetta Date: Fri, 7 Mar 2025 19:59:08 +0000 Subject: [PATCH] change setting fo random seed to configuration Signed-off-by: Brian Dellabetta --- src/llmcompressor/metrics/logger.py | 4 ++-- tests/lmeval/configs/vl_fp8_dynamic_per_token.yaml | 1 + tests/lmeval/configs/vl_int8_w8a8_dynamic_per_token.yaml | 1 + tests/lmeval/configs/vl_w4a16_actorder_weight.yaml | 1 + tests/lmeval/test_lmeval.py | 9 +++++++-- 5 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/llmcompressor/metrics/logger.py b/src/llmcompressor/metrics/logger.py index dc7aabc73..b4c2f9505 100644 --- a/src/llmcompressor/metrics/logger.py +++ b/src/llmcompressor/metrics/logger.py @@ -829,8 +829,8 @@ def __init__( or SparsificationGroupLogger( python=log_python, name=name, - tensorboard=True, - wandb_=True, + tensorboard=False, + wandb_=False, ).loggers ) diff --git a/tests/lmeval/configs/vl_fp8_dynamic_per_token.yaml b/tests/lmeval/configs/vl_fp8_dynamic_per_token.yaml index 223549995..2b2bb6601 100644 --- a/tests/lmeval/configs/vl_fp8_dynamic_per_token.yaml +++ b/tests/lmeval/configs/vl_fp8_dynamic_per_token.yaml @@ -2,6 +2,7 @@ cadence: weekly model: Qwen/Qwen2-VL-2B-Instruct model_class: TraceableQwen2VLForConditionalGeneration scheme: FP8_DYNAMIC +seed: 42 #compressed model is sensitive to random seed lmeval: model: "hf-multimodal" model_args: diff --git a/tests/lmeval/configs/vl_int8_w8a8_dynamic_per_token.yaml b/tests/lmeval/configs/vl_int8_w8a8_dynamic_per_token.yaml index 19544acdd..aa53c52c4 100644 --- a/tests/lmeval/configs/vl_int8_w8a8_dynamic_per_token.yaml +++ b/tests/lmeval/configs/vl_int8_w8a8_dynamic_per_token.yaml @@ -5,6 +5,7 @@ scheme: INT8_dyn_per_token recipe: tests/e2e/vLLM/recipes/INT8/recipe_int8_channel_weight_dynamic_per_token.yaml dataset_id: lmms-lab/flickr30k dataset_split: "test[:512]" +seed: 42 #compressed model is sensitive to random seed lmeval: model: "hf-multimodal" model_args: diff --git a/tests/lmeval/configs/vl_w4a16_actorder_weight.yaml b/tests/lmeval/configs/vl_w4a16_actorder_weight.yaml index 692c0f0eb..b28cf3dd7 100644 --- a/tests/lmeval/configs/vl_w4a16_actorder_weight.yaml +++ b/tests/lmeval/configs/vl_w4a16_actorder_weight.yaml @@ -5,6 +5,7 @@ recipe: tests/e2e/vLLM/recipes/actorder/recipe_w4a16_actorder_weight.yaml dataset_id: lmms-lab/flickr30k dataset_split: "test[:512]" scheme: W4A16_actorder_group +seed: 42 #compressed model is sensitive to random seed lmeval: model: "hf-multimodal" model_args: diff --git a/tests/lmeval/test_lmeval.py b/tests/lmeval/test_lmeval.py index 1b53333e5..34db1356d 100644 --- a/tests/lmeval/test_lmeval.py +++ b/tests/lmeval/test_lmeval.py @@ -1,4 +1,5 @@ import os +import random import shutil from pathlib import Path @@ -58,8 +59,6 @@ class TestLMEval: """ # noqa: E501 def set_up(self): - torch.manual_seed(42) - eval_config = yaml.safe_load(Path(TEST_DATA_FILE).read_text(encoding="utf-8")) if os.environ.get("CADENCE", "commit") != eval_config.get("cadence"): @@ -76,6 +75,12 @@ def set_up(self): self.quant_type = eval_config.get("quant_type") self.save_dir = eval_config.get("save_dir") + seed = eval_config.get("seed", None) + if seed is not None: + random.seed(seed) + numpy.random.seed(seed) + torch.manual_seed(seed) + logger.info("========== RUNNING ==============") logger.info(self.scheme)