Add FP8 dynamic scheme for latest Llama3.1 meta models and fix an issue with the W4A8 representation to have dynamic token for activations (#114)

Author: markurtz

src/compressed_tensors/quantization/quant_scheme.py

@@ -165,7 +165,7 @@ def is_preset_scheme(name: str) -> bool:
     input_activations=QuantizationArgs(
         num_bits=8,
         type=QuantizationType.INT,
-        strategy=QuantizationStrategy.TENSOR,
+        strategy=QuantizationStrategy.TOKEN,
         symmetric=True,
         dynamic=True,
     ),
@@ -189,6 +189,24 @@ def is_preset_scheme(name: str) -> bool:
     ),
 )
 
+# FP8 weights and FP8 dynamic activations quantization
+FP8_DYNAMIC = dict(
+    weights=QuantizationArgs(
+        num_bits=8,
+        type=QuantizationType.FLOAT,
+        strategy=QuantizationStrategy.CHANNEL,
+        symmetric=True,
+        dynamic=False,
+    ),
+    input_activations=QuantizationArgs(
+        num_bits=8,
+        type=QuantizationType.FLOAT,
+        strategy=QuantizationStrategy.TOKEN,
+        symmetric=True,
+        dynamic=True,
+    ),
+)
+
 PRESET_SCHEMES = {
     # Integer weight only schemes
     "W8A16": W8A16,
@@ -198,4 +216,5 @@ def is_preset_scheme(name: str) -> bool:
     "W4A8": W4A8,
     # Float weight and activation schemes
     "FP8": FP8,
+    "FP8_DYNAMIC": FP8_DYNAMIC,
 }