[WC][OV] NVFP4 support

Author: daniil-lyakhov

.ci/cspell_dict.txt

@@ -310,6 +310,7 @@ nsamples
 nsga
 numels
 nvcc
+nvfp
 objwalk
 odict
 oimp

docs/Algorithms.md

@@ -14,6 +14,7 @@
   - Arbitrary look-up table (CODEBOOK) or predefined lookup table based on NF4 (CB4)
   - MX-compliant types - MXFP4 and MXFP8_E4M3
   - FP types - FP8_E4M3 and FP4
+  - NVFP4 type
   - Mixed precision weights compression
   - Grouped weights compression
 

docs/usage/post_training_compression/weights_compression/Usage.md

@@ -55,6 +55,7 @@ NNCF can automatically distribute precision assignments based on quantization se
 | MXFP8_E4M3       | E4M3         | E8M0       | Group-wise (32)          | [MX-compliant FP8](https://www.opencompute.org/documents/ocp-microscaling-formats-mx-v1-0-spec-final-pdf) |
 |   FP8_E4M3       | E4M3         | FP16       | Per-channel / Group-wise | [FP8](https://www.opencompute.org/documents/ocp-microscaling-formats-mx-v1-0-spec-final-pdf) |
 |   FP4            | E2M1         | FP16       | Per-channel / Group-wise | [FP4](https://www.opencompute.org/documents/ocp-microscaling-formats-mx-v1-0-spec-final-pdf) |
+| NVFP4            | E2M1         | E4M3 per group / FP32 per weight | Group-wise (16)          | [NVFP4](https://www.arxiv.org/pdf/2602.14582) |
 
 **Note**: Granularity refers to the scope of elements sharing quantization parameters. "Per-channel" applies different parameters for each output channel, while "Group-wise" divides weights into groups (e.g., group_size=128) that share the same parameters.
 
@@ -706,7 +707,7 @@ Accuracy/footprint trade-off for `microsoft/Phi-3-mini-4k-instruct`:
 - The compression applies in-place.
 - The compressed model is not trainable.
 - INT4_SYM, INT4_ASYM, NF4 and E2M1 modes, grouped quantization and mixed precision selection is available for OpenVINO backend only.
-- NF4, MXFP4, MXFP8_E4M3 support is experimental on GPU and NPU - models quantized to nf4/mxfp4/mxfp8_e4m3 should not be faster models quantized to 8-bit integer.
+- NF4, MXFP4, MXFP8_E4M3, NVFP4 support is experimental on GPU and NPU - models quantized to nf4/mxfp4/mxfp8_e4m3/nvfp4 should not be faster models quantized to 8-bit integer.
 
 ### Additional resources
 

src/nncf/openvino/optimized_functions/functions.py

@@ -23,6 +23,7 @@
 from nncf.quantization.algorithms.weight_compression.config import WeightCompressionConfig
 from nncf.quantization.algorithms.weight_compression.constants import OPTIMIZED_COMPRESSION_COMPATIBLE_FLOAT_MODES
 from nncf.quantization.algorithms.weight_compression.constants import OPTIMIZED_COMPRESSION_COMPATIBLE_INT_MODES
+from nncf.quantization.algorithms.weight_compression.parameters import CompressedWeight
 from nncf.quantization.algorithms.weight_compression.weight_lowering import reshape_weight_for_grouped_quantization
 from nncf.tensor import Tensor
 from nncf.tensor import TensorBackend
@@ -37,7 +38,7 @@ def do_integer_quantization(
     reduction_axes: ReductionAxes | None = None,
     precomputed_scale: Tensor = None,
     precomputed_zero_point: Tensor = None,
-) -> tuple[Tensor, Tensor, Tensor]:
+) -> CompressedWeight:
     """
     Quantizes the given weight tensor to an integer data type.
 
@@ -47,7 +48,7 @@ def do_integer_quantization(
         precomputed scale (and zero point) are provided.
     :param precomputed_scale: Optional precomputed scale tensor.
     :param precomputed_zero_point: Optional precomputed zero point tensor.
-    :return: A tuple containing the compressed weights, scale, and zero point tensors.
+    :return: A CompressedWeight object containing the compressed weights, scale, and zero point tensors.
     """
     assert config.mode in OPTIMIZED_COMPRESSION_COMPATIBLE_INT_MODES
 
@@ -100,15 +101,15 @@ def do_integer_quantization(
         compressed_weight = model(inputs)[0]
         scale, zero_point = precomputed_scale, precomputed_zero_point
 
-    return compressed_weight, scale, zero_point
+    return CompressedWeight(compressed_weight, scale, zero_point=zero_point)
 
 
 def do_float_quantization(
     weight: Tensor,
     config: WeightCompressionConfig,
     reduction_axes: ReductionAxes | None = None,
     precomputed_scale: Tensor | None = None,
-) -> tuple[Tensor, Tensor, Tensor]:
+) -> CompressedWeight:
     """
     Computes quantization scale if not provided, and performs corresponding float weight quantization.
     NF4 format uses 16 levels in [-1, 1] range, while FP4/MXFP4 uses 16 levels in [-6, 6].
@@ -117,7 +118,7 @@ def do_float_quantization(
     :param config: Weight compression configuration.
     :param reduction_axes: Axes, along which to reduce (collect) different statistics.
     :param precomputed_scale: Optional precomputed scale.
-    :return: Returns quantized weight tensor and corresponding scale tensor.
+    :return: A CompressedWeight object containing the quantized weights and scale.
     """
     assert config.mode in OPTIMIZED_COMPRESSION_COMPATIBLE_FLOAT_MODES
 
@@ -153,7 +154,7 @@ def do_float_quantization(
         compressed_weight = model([weight, precomputed_scale])[0]
         scale = precomputed_scale
 
-    return compressed_weight, scale, None
+    return CompressedWeight(compressed_weight, scale)
 
 
 def integer_quantize_dequantize_weight(
@@ -163,7 +164,7 @@ def integer_quantize_dequantize_weight(
     precomputed_scale: Tensor | None = None,
     precomputed_zero_point: Tensor | None = None,
     return_compressed_weight: bool | None = False,
-) -> Tensor | tuple[Tensor, Tensor, Tensor, Tensor]:
+) -> Tensor | tuple[Tensor, CompressedWeight]:
     """
     Quantizes the given weight tensor to an integer data type and then dequantizes it back to obtain float32 values.
 
@@ -175,8 +176,7 @@ def integer_quantize_dequantize_weight(
     :param precomputed_zero_point: Optional precomputed zero point tensor.
     :param return_compressed_weight: If True, besides decompressed weight will also return compressed weight, scale,
         (and zero point).
-    :return: Dequantized weight tensor or a tuple containing the decompressed weight, compressed weight, scale,
-        (and zero point).
+    :return: Dequantized weight tensor or a tuple containing the decompressed weight and a CompressedWeight object.
     """
     assert config.mode in OPTIMIZED_COMPRESSION_COMPATIBLE_INT_MODES
 
@@ -217,7 +217,7 @@ def integer_quantize_dequantize_weight(
     else:
         decompressed_weight, compressed_weight, scale, zero_point = results
     if return_compressed_weight:
-        return decompressed_weight, compressed_weight, scale, zero_point
+        return decompressed_weight, CompressedWeight(compressed_weight, scale, zero_point=zero_point)
     return decompressed_weight
 
 
@@ -227,7 +227,7 @@ def float_quantize_dequantize_weight(
     reduction_axes: ReductionAxes | None = None,
     precomputed_scale: Tensor | None = None,
     return_compressed_weight: bool | None = False,
-) -> Tensor | tuple[Tensor, Tensor, Tensor]:
+) -> Tensor | tuple[Tensor, CompressedWeight]:
     """
     First quantizes the given weight tensor and then dequantizes it back to obtain float32 values.
 
@@ -236,7 +236,7 @@ def float_quantize_dequantize_weight(
     :param reduction_axes: Axes along which to reduce statistics. Not required if precomputed scale are provided.
     :param precomputed_scale: Optional precomputed scale tensor.
     :param return_compressed_weight: If True, besides decompressed weight will also return compressed weight and scale.
-    :return: Dequantized weight tensor or a tuple containing the decompressed weight, compressed weight and scale.
+    :return: Dequantized weight tensor or a tuple containing the decompressed weight and a CompressedWeight object.
     """
     assert config.mode in OPTIMIZED_COMPRESSION_COMPATIBLE_FLOAT_MODES
 
@@ -270,7 +270,7 @@ def float_quantize_dequantize_weight(
     else:
         decompressed_weight, compressed_weight, scale = results
     if return_compressed_weight:
-        return decompressed_weight, compressed_weight, scale
+        return decompressed_weight, CompressedWeight(compressed_weight, scale)
     return decompressed_weight
 
 

src/nncf/parameters.py

@@ -94,6 +94,8 @@ class CompressWeightsMode(StrEnum):
     :param MXFP8_E4M3: MX-compliant FP8 format with E4M3 values sharing group-level E8M0 scale. The size of group is 32.
     :param FP8_E4M3: A FP8 format with E4M3 values sharing group-level fp16 scale.
     :param FP4: A FP4 format with E2M1 values sharing group-level fp16 scale.
+    :param NVFP4: A FP4 format with E2M1 values sharing group-level E4M3 scale and FP32 per weight scale.
+        The size of group is 16.
     :param CODEBOOK: Codebook (LUT) quantization format.
     :param ADAPTIVE_CODEBOOK: Adaptive codebook (LUT) quantization format.
     :param CB4: Codebook (LUT) format with 16 fixed fp8 values in E4M3 format.
@@ -110,6 +112,7 @@ class CompressWeightsMode(StrEnum):
     MXFP8_E4M3 = "mxfp8_e4m3"
     FP8_E4M3 = "fp8_e4m3"
     FP4 = "fp4"
+    NVFP4 = "nvfp4"
     CODEBOOK = "codebook"
     ADAPTIVE_CODEBOOK = "adaptive_codebook"
 

src/nncf/quantization/algorithms/weight_compression/algorithm.py

@@ -94,6 +94,8 @@ def get_weight_compression_configuration(
     elif group_size is None and mode in NON_INT8_MODES:
         if mode in [CompressWeightsMode.MXFP4, CompressWeightsMode.MXFP8_E4M3]:
             group_size = 32
+        elif mode == CompressWeightsMode.NVFP4:
+            group_size = 16
         elif mode in [
             CompressWeightsMode.CODEBOOK,
             CompressWeightsMode.CB4,
@@ -275,14 +277,18 @@ def check_user_compression_configuration(
             f"Supported modes are: {[e.value for e in GroupSizeFallbackMode]}."
         )
         raise nncf.ValidationError(msg)
-    if mode in [CompressWeightsMode.MXFP4, CompressWeightsMode.MXFP8_E4M3]:
-        if group_size not in [None, 32]:
+    if mode in [CompressWeightsMode.MXFP4, CompressWeightsMode.MXFP8_E4M3, CompressWeightsMode.NVFP4]:
+        if mode in [CompressWeightsMode.MXFP4, CompressWeightsMode.MXFP8_E4M3] and group_size not in [None, 32]:
             msg = f"MXFP4 and MXFP8_E4M3 types only support group size of 32, group size of {group_size} is given"
             raise nncf.ValidationError(msg)
 
+        if mode == CompressWeightsMode.NVFP4 and group_size not in [None, 16]:
+            msg = f"NVFP4 type only supports group size of 16, group size of {group_size} is given"
+            raise nncf.ValidationError(msg)
+
         if advanced_parameters and advanced_parameters.group_size_fallback_mode is GroupSizeFallbackMode.ADJUST:
             msg = (
-                "MXFP4 and MXFP8_E4M3 types do not support the group size"
+                "MXFP4, MXFP8_E4M3 and NVFP4 types do not support the group size"
                 f" fallback mode {advanced_parameters.group_size_fallback_mode.value}."
                 " Please use other group size fallback mode."
             )
@@ -332,6 +338,8 @@ def __init__(
             MXFP8_E4M3 is MX-compliant FP8 with E4M3 values sharing group-level E8M0 scale. The size of group is 32.
             FP8_E4M3 is FP8 with E4M3 values sharing group-level FP16 scale.
             FP4 is FP4 with E2M1 values sharing group-level FP16 scale.
+            NVFP4 is FP4 format with E2M1 values sharing group-level E4M3 scale and FP32 per weight scale.
+                The size of group is 16.
         :param ratio: the ratio between primary and backup precisions (e.g. 0.9 means 90% of layers quantized to NF4
             and the rest to backup_mode).
         :param group_size: number of weights (e.g. 128) in the channel dimension

src/nncf/quantization/algorithms/weight_compression/config.py

@@ -70,6 +70,7 @@ def is_integer(self):
             CompressWeightsMode.MXFP8_E4M3,
             CompressWeightsMode.FP8_E4M3,
             CompressWeightsMode.FP4,
+            CompressWeightsMode.NVFP4,
             CompressWeightsMode.CODEBOOK,
             CompressWeightsMode.ADAPTIVE_CODEBOOK,
             CompressWeightsMode.CB4,
@@ -106,6 +107,7 @@ def compression_dtype(self) -> TensorDataType:
             CompressWeightsMode.NF4: TensorDataType.nf4,
             CompressWeightsMode.FP4: TensorDataType.f4e2m1,
             CompressWeightsMode.MXFP4: TensorDataType.f4e2m1,
+            CompressWeightsMode.NVFP4: TensorDataType.f4e2m1,
             CompressWeightsMode.FP8_E4M3: TensorDataType.f8e4m3,
             CompressWeightsMode.MXFP8_E4M3: TensorDataType.f8e4m3,
         }

src/nncf/quantization/algorithms/weight_compression/lora_correction.py

@@ -172,13 +172,11 @@ def calculate_low_rank_matrices(
         reduction_axis = reduction_axes[0] if compression_config.group_size != -1 else -1
         if mode in (CompressWeightsMode.INT4_SYM, CompressWeightsMode.INT4_ASYM):
             fq_weights = do_integer_dequantization(
-                compressed_weight.tensor,
-                compressed_weight.scale,
-                compressed_weight.zero_point,
+                compressed_weight,
                 reduction_axis,
             )
         elif mode == CompressWeightsMode.NF4:
-            fq_weights = do_float_dequantization(compressed_weight.tensor, compressed_weight.scale, reduction_axis)
+            fq_weights = do_float_dequantization(compressed_weight, reduction_axis)
         else:
             msg = (
                 f"{mode.value} mode is invalid for Lora Correction algorithm. Supported modes: INT4_SYM, INT4_ASYM, NF4"

src/nncf/quantization/algorithms/weight_compression/openvino_backend.py

@@ -224,13 +224,6 @@ def _create_compression_subgraph(
         should_add_convert_node: bool,
         precomputed_compressed_weight: CompressedWeight | None = None,
     ):
-        compression_dtype = DTYPE_MAP[compression_config.compression_dtype]
-        scale_dtype = (
-            ov.Type.f8e8m0
-            if compression_config.mode in [CompressWeightsMode.MXFP4, CompressWeightsMode.MXFP8_E4M3]
-            else ov.Type.f16
-        )
-
         original_shape = weight.shape
 
         with disable_results_caching(OV_MODEL_CACHE):
@@ -241,6 +234,14 @@ def _create_compression_subgraph(
                 precomputed_compressed_weight,
             )
 
+        compression_dtype = DTYPE_MAP[compression_config.compression_dtype]
+
+        scale_dtype = ov.Type.f16
+        if compression_config.mode in [CompressWeightsMode.MXFP4, CompressWeightsMode.MXFP8_E4M3]:
+            scale_dtype = ov.Type.f8e8m0
+        elif compression_config.mode == CompressWeightsMode.NVFP4:
+            scale_dtype = ov.Type.f8e4m3
+
         if compression_config.is_codebook:
             converted_const = create_ov_codebook_subgraph(
                 codebook=compressed_weight.codebook
@@ -266,6 +267,19 @@ def _create_compression_subgraph(
                 )
 
         scale_const = create_ov_const_from_tensor(compressed_weight.scale, scale_dtype, name=f"{const_node_name}/scale")
+
+        if compressed_weight.second_degree_scale is not None:
+            sec_order_scale = create_ov_const_from_tensor(
+                compressed_weight.second_degree_scale, ov.Type.f32, name=f"{const_node_name}/second_order_scale"
+            )
+            scale_const = convert_op(scale_const, ov.Type.f32)
+
+            scale_const = opset.multiply(
+                scale_const,
+                sec_order_scale,
+                name=f"{const_node_name}/dequantized_scale_{weight_port_id}",
+            )
+
         scale_const = convert_op(scale_const, ov.Type.f16)
 
         mul = opset.multiply(

src/nncf/quantization/algorithms/weight_compression/parameters.py

@@ -30,11 +30,4 @@ class CompressedWeight:
     scale: Tensor | None = None
     zero_point: Tensor | None = None
     codebook: Tensor | None = None
-
-    def is_codebook(self):
-        """
-        Check if the compressed weight is a codebook.
-
-        :return: True if the compressed weight is a codebook, False otherwise.
-        """
-        return self.codebook is not None and self.tensor is not None and self.scale is not None
+    second_degree_scale: Tensor | None = None