[Backend Tester] Add additional quantized test flows for XNNPACK and Vulkan (#13534)

Wire up the PT2E Vulkan quantizer and add additional test flows for
XNNPACK PT2E (8-bit static per-tensor and 8-bit dynamic per-channel).

Author: GregoryComer

.github/workflows/nightly.yml

@@ -42,7 +42,11 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        flow: [qnn, qnn_16a16w, qnn_16a8w, qnn_16a4w, qnn_16a4w_block, qnn_8a8w, vulkan, xnnpack, xnnpack_static_int8_per_channel]
+        flow: [
+          qnn, qnn_16a16w, qnn_16a8w, qnn_16a4w, qnn_16a4w_block, qnn_8a8w,
+          vulkan, vulkan_static_int8_per_channel,
+          xnnpack, xnnpack_dynamic_int8_per_channel, xnnpack_static_int8_per_channel, xnnpack_static_int8_per_tensor
+        ]
         suite: [models, operators]
     with:
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}

backends/test/suite/flow.py

@@ -47,13 +47,17 @@ def all_flows() -> dict[str, TestFlow]:
 
     try:
         from executorch.backends.test.suite.flows.xnnpack import (
+            XNNPACK_DYNAMIC_INT8_PER_CHANNEL_TEST_FLOW,
             XNNPACK_STATIC_INT8_PER_CHANNEL_TEST_FLOW,
+            XNNPACK_STATIC_INT8_PER_TENSOR_TEST_FLOW,
             XNNPACK_TEST_FLOW,
         )
 
         flows += [
             XNNPACK_TEST_FLOW,
+            XNNPACK_DYNAMIC_INT8_PER_CHANNEL_TEST_FLOW,
             XNNPACK_STATIC_INT8_PER_CHANNEL_TEST_FLOW,
+            XNNPACK_STATIC_INT8_PER_TENSOR_TEST_FLOW,
         ]
     except Exception as e:
         logger.info(f"Skipping XNNPACK flow registration: {e}")
@@ -72,10 +76,14 @@ def all_flows() -> dict[str, TestFlow]:
         logger.info(f"Skipping Core ML flow registration: {e}")
 
     try:
-        from executorch.backends.test.suite.flows.vulkan import VULKAN_TEST_FLOW
+        from executorch.backends.test.suite.flows.vulkan import (
+            VULKAN_STATIC_INT8_PER_CHANNEL_TEST_FLOW,
+            VULKAN_TEST_FLOW,
+        )
 
         flows += [
             VULKAN_TEST_FLOW,
+            VULKAN_STATIC_INT8_PER_CHANNEL_TEST_FLOW,
         ]
     except Exception as e:
         logger.info(f"Skipping Vulkan flow registration: {e}")

backends/test/suite/flows/vulkan.py

@@ -1,17 +1,43 @@
+from typing import Callable
+
+from executorch.backends.test.harness.stages import Quantize
 from executorch.backends.test.suite.flow import TestFlow
-from executorch.backends.vulkan.test.tester import VulkanTester
+from executorch.backends.vulkan.quantizer.vulkan_quantizer import (
+    get_symmetric_quantization_config as get_symmetric_quantization_config_vulkan,
+)
+from executorch.backends.vulkan.test.tester import (
+    Quantize as VulkanQuantize,
+    VulkanTester,
+)
 
 
-def _create_vulkan_flow(
-    name: str,
-    quantize: bool = False,
+def _create_vulkan_flow_base(
+    name: str, quantize_stage_factory: Callable[..., Quantize] | None = None
 ) -> TestFlow:
     return TestFlow(
         name,
         backend="vulkan",
         tester_factory=VulkanTester,
-        quantize=quantize,
+        quantize=quantize_stage_factory is not None,
+        quantize_stage_factory=quantize_stage_factory,
+    )
+
+
+def _create_vulkan_flow() -> TestFlow:
+    return _create_vulkan_flow_base("vulkan")
+
+
+def _create_vulkan_static_int8_per_channel_flow() -> TestFlow:
+    def create_quantize_stage() -> Quantize:
+        qparams = get_symmetric_quantization_config_vulkan()
+        return VulkanQuantize(
+            quantization_config=qparams,
+        )
+
+    return _create_vulkan_flow_base(
+        "vulkan_static_int8_per_channel", create_quantize_stage
     )
 
 
-VULKAN_TEST_FLOW = _create_vulkan_flow("vulkan")
+VULKAN_TEST_FLOW = _create_vulkan_flow()
+VULKAN_STATIC_INT8_PER_CHANNEL_TEST_FLOW = _create_vulkan_static_int8_per_channel_flow()

backends/test/suite/flows/xnnpack.py

@@ -31,6 +31,20 @@ def _create_xnnpack_flow() -> TestFlow:
     return _create_xnnpack_flow_base("xnnpack")
 
 
+def _create_xnnpack_dynamic_int8_per_channel_flow() -> TestFlow:
+    def create_quantize_stage() -> Quantize:
+        qparams = get_symmetric_quantization_config(
+            is_per_channel=True, is_dynamic=True
+        )
+        return XnnpackQuantize(
+            quantization_config=qparams,
+        )
+
+    return _create_xnnpack_flow_base(
+        "xnnpack_dynamic_int8_per_channel", create_quantize_stage
+    )
+
+
 def _create_xnnpack_static_int8_per_channel_flow() -> TestFlow:
     def create_quantize_stage() -> Quantize:
         qparams = get_symmetric_quantization_config(is_per_channel=True)
@@ -43,7 +57,23 @@ def create_quantize_stage() -> Quantize:
     )
 
 
+def _create_xnnpack_static_int8_per_tensor_flow() -> TestFlow:
+    def create_quantize_stage() -> Quantize:
+        qparams = get_symmetric_quantization_config(is_per_channel=False)
+        return XnnpackQuantize(
+            quantization_config=qparams,
+        )
+
+    return _create_xnnpack_flow_base(
+        "xnnpack_static_int8_per_tensor", create_quantize_stage
+    )
+
+
 XNNPACK_TEST_FLOW = _create_xnnpack_flow()
+XNNPACK_DYNAMIC_INT8_PER_CHANNEL_TEST_FLOW = (
+    _create_xnnpack_dynamic_int8_per_channel_flow()
+)
 XNNPACK_STATIC_INT8_PER_CHANNEL_TEST_FLOW = (
     _create_xnnpack_static_int8_per_channel_flow()
 )
+XNNPACK_STATIC_INT8_PER_TENSOR_TEST_FLOW = _create_xnnpack_static_int8_per_tensor_flow()

backends/vulkan/test/tester.py

@@ -4,7 +4,7 @@
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
-from typing import Any, List, Optional, Tuple
+from typing import Any, List, Optional, Sequence, Tuple
 
 import executorch
 import executorch.backends.test.harness.stages as BaseStages
@@ -13,8 +13,33 @@
 from executorch.backends.test.harness import Tester as TesterBase
 from executorch.backends.test.harness.stages import StageType
 from executorch.backends.vulkan.partitioner.vulkan_partitioner import VulkanPartitioner
+from executorch.backends.vulkan.quantizer.vulkan_quantizer import (
+    get_symmetric_quantization_config as get_symmetric_quantization_config_vulkan,
+    VulkanQuantizer,
+)
 from executorch.exir import EdgeCompileConfig
 from executorch.exir.backend.partitioner import Partitioner
+from torchao.quantization.pt2e.quantizer import Quantizer
+
+
+class Quantize(BaseStages.Quantize):
+    def __init__(
+        self,
+        quantizer: Optional[Quantizer] = None,
+        quantization_config: Any | None = None,
+        calibrate: bool = True,
+        calibration_samples: Optional[Sequence[Any]] = None,
+        is_qat: Optional[bool] = False,
+    ):
+        super().__init__(
+            quantizer=quantizer or VulkanQuantizer(),
+            quantization_config=(
+                quantization_config or get_symmetric_quantization_config_vulkan()
+            ),
+            calibrate=calibrate,
+            calibration_samples=calibration_samples,
+            is_qat=is_qat,
+        )
 
 
 class Partition(BaseStages.Partition):