Introduce OVPipelineQuantizationConfig (#1310)

* Initial commit

* Tweaks

* Fix test

* Fix config for phi4mm

* Update ignored scope

* Update docs

* Revert disabling tests

* Update optimization.mdx

* Update test_quantization.py

* Revert "Update test_quantization.py"

This reverts commit 1ae7374.

* Fix test

* Add support for VLM ptq

* Update docs

* Update docs

* Rename pipeline_quantization_configs

* Also rename in docs

* Update phi4mm config

Author: nikita-savelyevv

docs/source/openvino/optimization.mdx

@@ -133,8 +133,18 @@ Click on a ✅ to copy the command/code for the corresponding optimization case.
             </td>
             <td style="text-align: center; vertical-align: middle;">–</td>
             <td style="text-align: center; vertical-align: middle;">–</td>
-            <td style="text-align: center; vertical-align: middle;">–</td>
-            <td style="text-align: center; vertical-align: middle;">–</td>
+            <td style="text-align: center; vertical-align: middle;">
+                <button
+                    onclick="navigator.clipboard.writeText('optimum-cli export openvino --task image-text-to-text -m OpenGVLab/InternVL2-1B --trust-remote-code --quant-mode int8 --dataset contextual ./save_dir')">
+                    ✅
+                </button>
+            </td>
+            <td style="text-align: center; vertical-align: middle;">
+                <button
+                    onclick="navigator.clipboard.writeText('OVModelForVisualCausalLM.from_pretrained(\'OpenGVLab/InternVL2-1B\', trust_remote_code=True, quantization_config=OVQuantizationConfig(bits=8, dataset=\'contextual\', trust_remote_code=True)).save_pretrained(\'save_dir\')')">
+                    ✅
+                </button>
+            </td>
             <td style="text-align: center; vertical-align: middle;">–</td>
             <td style="text-align: center; vertical-align: middle;">–</td>
         </tr>
@@ -636,4 +646,31 @@ To apply mixed quantization through CLI, the `--quant-mode` argument should be u
 optimum-cli export openvino -m TinyLlama/TinyLlama-1.1B-Chat-v1.0 --quant-mode nf4_f8e4m3 --dataset wikitext2 ./save_dir
 ```
 
-Don't forget to provide a dataset since it is required for the calibration procedure during full quantization.
+Don't forget to provide a dataset since it is required for the calibration procedure during full quantization.
+
+
+## Pipeline Quantization
+
+There are multimodal pipelines that consist of multiple components, such as Stable Diffusion or Visual Language models. In these cases, there may be a need to apply different quantization methods to different components of the pipeline. For example, you may want to apply int4 data-aware weight-only quantization to a language model in visual-language pipeline, while applying int8 weight-only quantization to other components. In this case you can use the `OVPipelineQuantizationConfig` class to specify the quantization configuration for each component of the pipeline.
+
+For example, the code below quantizes weights and activations of a language model inside InternVL2-1B, compresses weights of a text embedding model and skips any quantization for vision embedding model.
+
+```python
+from optimum.intel import OVModelForVisualCausalLM
+from optimum.intel import OVPipelineQuantizationConfig, OVQuantizationConfig, OVWeightQuantizationConfig
+
+model_id = "OpenGVLab/InternVL2-1B"
+model = OVModelForVisualCausalLM.from_pretrained(
+    model_id,
+    export=True,
+    trust_remote_code=True,
+    quantization_config=OVPipelineQuantizationConfig(
+        quantization_configs={
+            "lm_model": OVQuantizationConfig(bits=8),
+            "text_embeddings_model": OVWeightQuantizationConfig(bits=8),
+        },
+        dataset="contextual",
+        trust_remote_code=True,
+    )
+)
+```

optimum/commands/export/openvino.py

@@ -410,8 +410,7 @@ def run(self):
                         }
                     else:
                         quantization_config = prepare_q_config(self.args)
-            if quantization_config.get("dataset", None):
-                quantization_config["trust_remote_code"] = self.args.trust_remote_code
+            quantization_config["trust_remote_code"] = self.args.trust_remote_code
             ov_config = OVConfig(quantization_config=quantization_config)
 
         quantization_config = ov_config.quantization_config if ov_config else None

optimum/intel/__init__.py

@@ -81,6 +81,7 @@
         [
             "OVQuantizer",
             "OVCalibrationDataset",
+            "OVPipelineQuantizationConfig",
             "OVQuantizationConfig",
             "OVWeightQuantizationConfig",
             "OVDynamicQuantizationConfig",
@@ -92,6 +93,7 @@
         [
             "OVQuantizer",
             "OVCalibrationDataset",
+            "OVPipelineQuantizationConfig",
             "OVQuantizationConfig",
             "OVWeightQuantizationConfig",
             "OVDynamicQuantizationConfig",
@@ -275,6 +277,7 @@
             OVCalibrationDataset,
             OVDynamicQuantizationConfig,
             OVMixedQuantizationConfig,
+            OVPipelineQuantizationConfig,
             OVQuantizationConfig,
             OVQuantizer,
             OVWeightQuantizationConfig,
@@ -284,6 +287,7 @@
             OVCalibrationDataset,
             OVDynamicQuantizationConfig,
             OVMixedQuantizationConfig,
+            OVPipelineQuantizationConfig,
             OVQuantizationConfig,
             OVQuantizer,
             OVWeightQuantizationConfig,

optimum/intel/openvino/__init__.py

@@ -54,6 +54,7 @@
     OVConfig,
     OVDynamicQuantizationConfig,
     OVMixedQuantizationConfig,
+    OVPipelineQuantizationConfig,
     OVQuantizationConfig,
     OVWeightQuantizationConfig,
 )