fixing reproducibility of lmeval tests

setup.py

@@ -60,9 +60,12 @@
         "datasets",
         "accelerate>=0.20.3,!=1.1.0",
         "pynvml",
-        "compressed-tensors"
-        if version_info.build_type == "release"
-        else "compressed-tensors-nightly",
+        "pillow",
+        (
+            "compressed-tensors"
+            if version_info.build_type == "release"
+            else "compressed-tensors-nightly"
+        ),
     ],
     extras_require={
         "dev": [

tests/lmeval/configs/vl_fp8_dynamic_per_token.yaml

@@ -2,6 +2,7 @@ cadence: weekly
 model: Qwen/Qwen2-VL-2B-Instruct
 model_class: TraceableQwen2VLForConditionalGeneration
 scheme: FP8_DYNAMIC
+seed: 42  # compressed model is sensitive to random seed
 lmeval:
   model: "hf-multimodal"
   model_args:
@@ -10,7 +11,6 @@ lmeval:
     convert_img_format: True
   task: mmmu_val_economics
   num_fewshot: 0
-  limit: 1000
   batch_size: 8
   metrics:
     acc,none: 0.333

tests/lmeval/configs/vl_int8_w8a8_dynamic_per_token.yaml

@@ -5,6 +5,7 @@ scheme: INT8_dyn_per_token
 recipe: tests/e2e/vLLM/recipes/INT8/recipe_int8_channel_weight_dynamic_per_token.yaml
 dataset_id: lmms-lab/flickr30k
 dataset_split: "test[:512]"
+seed: 42 #compressed model is sensitive to random seed
 lmeval:
   model: "hf-multimodal"
   model_args:
@@ -13,7 +14,6 @@ lmeval:
     convert_img_format: True
   task: mmmu_val_economics
   num_fewshot: 0
-  limit: 1000
   metrics:
     acc,none: 0.233
   batch_size: 8

tests/lmeval/configs/vl_w4a16_actorder_weight.yaml

@@ -5,6 +5,7 @@ recipe: tests/e2e/vLLM/recipes/actorder/recipe_w4a16_actorder_weight.yaml
 dataset_id: lmms-lab/flickr30k
 dataset_split: "test[:512]"
 scheme: W4A16_actorder_group
+seed: 42 #compressed model is sensitive to random seed
 lmeval:
   model: "hf-multimodal"
   model_args:
@@ -13,7 +14,6 @@ lmeval:
     convert_img_format: True
   task: mmmu_val_economics
   num_fewshot: 0
-  limit: 1000
   metrics:
-    acc,none: 0.4
+    acc,none: 0.366
   batch_size: 4

tests/lmeval/test_lmeval.py

@@ -1,9 +1,11 @@
 import os
+import random
 import shutil
 from pathlib import Path
 
 import numpy
 import pytest
+import torch
 import yaml
 from loguru import logger
 from pydantic import BaseModel
@@ -73,6 +75,12 @@ def set_up(self):
         self.quant_type = eval_config.get("quant_type")
         self.save_dir = eval_config.get("save_dir")
 
+        seed = eval_config.get("seed", None)
+        if seed is not None:
+            random.seed(seed)
+            numpy.random.seed(seed)
+            torch.manual_seed(seed)
+
         logger.info("========== RUNNING ==============")
         logger.info(self.scheme)