Update transformers tests to speed-up execution

tests/llmcompressor/transformers/compression/configs/channelwise_15m.yaml

@@ -1,5 +1,4 @@
 cadence: "commit"
 test_type: "regression"
 model_stub: "Xenova/llama2.c-stories15M"
-new_recipe: "tests/llmcompressor/transformers/compression/recipes/new_quant_channel.yaml"
-ppl_threshold: 30000
+new_recipe: "tests/llmcompressor/transformers/compression/recipes/new_quant_channel.yaml"

tests/llmcompressor/transformers/compression/configs/fp8_15m.yaml

@@ -1,5 +1,4 @@
 cadence: "commit"
 test_type: "regression"
 model_stub: "Xenova/llama2.c-stories15M"
-new_recipe: "tests/llmcompressor/transformers/compression/recipes/new_quant_fp8.yaml"
-ppl_threshold: 30000
+new_recipe: "tests/llmcompressor/transformers/compression/recipes/new_quant_fp8.yaml"

tests/llmcompressor/transformers/compression/configs/inputs_15m.yaml

@@ -1,5 +1,4 @@
 cadence: "commit"
 test_type: "regression"
 model_stub: "Xenova/llama2.c-stories15M"
-new_recipe: "tests/llmcompressor/transformers/compression/recipes/new_quant_full.yaml"
-ppl_threshold: 30000
+new_recipe: "tests/llmcompressor/transformers/compression/recipes/new_quant_full.yaml"

tests/llmcompressor/transformers/compression/configs/weights_only_1.1b.yaml

@@ -1,5 +1,4 @@
 cadence: "nightly"
 test_type: "regression"
 model_stub: "TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T"
-new_recipe: "tests/llmcompressor/transformers/compression/recipes/new_quant_weight.yaml"
-ppl_threshold: 20
+new_recipe: "tests/llmcompressor/transformers/compression/recipes/new_quant_weight.yaml"

tests/llmcompressor/transformers/compression/configs/weights_only_15m.yaml

@@ -1,5 +1,4 @@
 cadence: "commit"
 test_type: "regression"
 model_stub: "Xenova/llama2.c-stories15M"
-new_recipe: "tests/llmcompressor/transformers/compression/recipes/new_quant_weight.yaml"
-ppl_threshold: 30000
+new_recipe: "tests/llmcompressor/transformers/compression/recipes/new_quant_weight.yaml"

tests/llmcompressor/transformers/compression/decompression_configs/w4a16.yaml

@@ -1,4 +1,4 @@
-cadence: "commit"
+cadence: "nightly"
 test_type: "regression"
 compressed_model_stub: "nm-testing/tinyllama-w4a16-compressed"
 skeleton_model_stub: "TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T"

tests/llmcompressor/transformers/compression/decompression_configs/w8a16_dense.yaml

@@ -1,4 +1,4 @@
-cadence: "commit"
+cadence: "nightly"
 test_type: "regression"
 compressed_model_stub: "nm-testing/tinyllama-w8a16-dense"
 skeleton_model_stub: "TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T"

tests/llmcompressor/transformers/compression/test_decompress.py

@@ -3,6 +3,7 @@
 import tempfile
 import unittest
 
+import torch
 from compressed_tensors import QUANTIZATION_CONFIG_NAME
 from compressed_tensors.compressors import ModelCompressor
 from compressed_tensors.quantization import QuantizationStatus
@@ -113,16 +114,16 @@ def test_hf_quantizer_decompress_match_manual_decompress(self):
             )
             inputs = inputs.to(self.decompressed_model_manual.device)
 
-            decompressed_model_manual_output = self.tokenizer.batch_decode(
-                self.decompressed_model_manual.generate(**inputs, max_length=50)
+            decompressed_model_manual_output = self.decompressed_model_manual.generate(
+                **inputs, max_length=50
             )
 
-            decompressed_model_hf_quantizer_out = self.tokenizer.batch_decode(
+            decompressed_model_hf_quantizer_out = (
                 self.decompressed_model_hf_quantizer.generate(**inputs, max_length=50)
             )
 
-            assert (
-                decompressed_model_hf_quantizer_out == decompressed_model_manual_output
+            assert torch.equal(
+                decompressed_model_hf_quantizer_out, decompressed_model_manual_output
             )
 
     @classmethod

tests/llmcompressor/transformers/compression/test_quantization.py

@@ -55,7 +55,7 @@ def tearDownClass(cls):
 
     @staticmethod
     def _run_oneshot(model, recipe, dataset, output_dir):
-        num_calibration_samples = 512
+        num_calibration_samples = 64
         max_seq_length = 512
         pad_to_max_length = False
 
@@ -68,7 +68,7 @@ def _run_oneshot(model, recipe, dataset, output_dir):
             recipe=recipe,
             pad_to_max_length=pad_to_max_length,
             clear_sparse_session=False,
-            splits={"calibration": "train_gen[:5%]"},
+            splits={"calibration": "train_gen[:1%]"},
             save_compressed=False,
         )
         return model
@@ -142,6 +142,8 @@ def _get_dataloader(self, data_args, tokenizer):
 
     @torch.no_grad()
     def test_perplexity(self):
+        if self.ppl_threshold is None:
+            pytest.skip("Skipping perplexity calculation.")
         tokenizer = AutoTokenizer.from_pretrained(self.model_stub)
         data_args = DatasetArguments(
             dataset="ultrachat-200k",

tests/llmcompressor/transformers/compression/test_run_compressed.py

@@ -3,6 +3,7 @@
 import tempfile
 import unittest
 
+import torch
 from compressed_tensors import QUANTIZATION_CONFIG_NAME
 from compressed_tensors.compressors import ModelCompressor
 from compressed_tensors.quantization import QuantizationStatus
@@ -113,16 +114,16 @@ def test_hf_quantizer_decompress_match_manual_decompress(self):
             )
             inputs = inputs.to(self.decompressed_model_manual.device)
 
-            decompressed_model_manual_output = self.tokenizer.batch_decode(
-                self.decompressed_model_manual.generate(**inputs, max_length=50)
+            decompressed_model_manual_output = self.decompressed_model_manual.generate(
+                **inputs, max_length=50
             )
 
-            decompressed_model_hf_quantizer_out = self.tokenizer.batch_decode(
+            decompressed_model_hf_quantizer_out = (
                 self.decompressed_model_hf_quantizer.generate(**inputs, max_length=50)
             )
 
-            assert (
-                decompressed_model_hf_quantizer_out == decompressed_model_manual_output
+            assert torch.equal(
+                decompressed_model_hf_quantizer_out, decompressed_model_manual_output
             )
 
     @classmethod