attempts to fix lmeval failing tests

Signed-off-by: Brian Dellabetta <[email protected]>
vllm-project · Mar 4, 2025 · ca5a6ba · ca5a6ba
1 parent 7bb517f
commit ca5a6ba
Show file tree

Hide file tree

Showing 4 changed files with 8 additions and 8 deletions.
diff --git a/setup.py b/setup.py
@@ -60,9 +60,12 @@
         "datasets",
         "accelerate>=0.20.3,!=1.1.0",
         "pynvml",
-        "compressed-tensors"
-        if version_info.build_type == "release"
-        else "compressed-tensors-nightly",
+        "pillow",
+        (
+            "compressed-tensors"
+            if version_info.build_type == "release"
+            else "compressed-tensors-nightly"
+        ),
     ],
     extras_require={
         "dev": [

diff --git a/tests/lmeval/configs/vl_fp8_dynamic_per_token.yaml b/tests/lmeval/configs/vl_fp8_dynamic_per_token.yaml
@@ -10,7 +10,6 @@ lmeval:
     convert_img_format: True
   task: mmmu_val_economics
   num_fewshot: 0
-  limit: 1000
   batch_size: 8
   metrics:
-    acc,none: 0.333
+    acc,none: 0.266
diff --git a/tests/lmeval/configs/vl_int8_w8a8_dynamic_per_token.yaml b/tests/lmeval/configs/vl_int8_w8a8_dynamic_per_token.yaml
@@ -13,7 +13,6 @@ lmeval:
     convert_img_format: True
   task: mmmu_val_economics
   num_fewshot: 0
-  limit: 1000
   metrics:
     acc,none: 0.233
   batch_size: 8
diff --git a/tests/lmeval/configs/vl_w4a16_actorder_weight.yaml b/tests/lmeval/configs/vl_w4a16_actorder_weight.yaml
@@ -13,7 +13,6 @@ lmeval:
     convert_img_format: True
   task: mmmu_val_economics
   num_fewshot: 0
-  limit: 1000
   metrics:
-    acc,none: 0.4
+    acc,none: 0.433
   batch_size: 4