quality

Signed-off-by: shanjiaz <[email protected]>

Author: shanjiaz

nvfp4_decompress.py

@@ -0,0 +1,15 @@
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from llmcompressor.utils import dispatch_for_generation
+
+#MODEL_ID = "nm-testing/TinyLlama-1.1B-Chat-v1.0-w4a16-asym-awq-e2e"
+MODEL_ID = "nm-testing/TinyLlama-1.1B-Chat-v1.0-NVFP4"
+
+model = AutoModelForCausalLM.from_pretrained(MODEL_ID, torch_dtype="auto")
+tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
+
+print("========== SAMPLE GENERATION ==============")
+dispatch_for_generation(model)
+input_ids = tokenizer("Hello my name is", return_tensors="pt").input_ids.to(model.device)
+output = model.generate(input_ids, max_new_tokens=100)
+print(tokenizer.decode(output[0]))''
+print("==========================================\n\n")

src/compressed_tensors/compressors/model_compressors/model_compressor.py

@@ -338,10 +338,10 @@ def __init__(
 
             self.quantization_compressor = {}
             for format in self.compression_formats:
-                self.quantization_compressor[
-                    format
-                ] = BaseCompressor.load_from_registry(
-                    format, config=quantization_config
+                self.quantization_compressor[format] = (
+                    BaseCompressor.load_from_registry(
+                        format, config=quantization_config
+                    )
                 )
 
     def get_missing_module_keys(self, model: Module) -> List[str]:

src/compressed_tensors/quantization/utils/helpers.py

@@ -116,9 +116,11 @@ def calculate_qparams(
     # 4. Update any 0s with small values to
     # prevent div by 0
     eps = _get_dtype_eps(
-        dtype=quantization_args.scale_dtype
-        if quantization_args.scale_dtype is not None
-        else scales.dtype
+        dtype=(
+            quantization_args.scale_dtype
+            if quantization_args.scale_dtype is not None
+            else scales.dtype
+        )
     )
     scales = torch.where(
         scales == 0,