diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 24bf86bf441..1ef89c2ed6d 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -42,7 +42,11 @@ jobs: strategy: fail-fast: false matrix: - flow: [qnn, qnn_16a16w, qnn_16a8w, qnn_16a4w, qnn_16a4w_block, qnn_8a8w, vulkan, xnnpack, xnnpack_static_int8_per_channel] + flow: [ + qnn, qnn_16a16w, qnn_16a8w, qnn_16a4w, qnn_16a4w_block, qnn_8a8w, + vulkan, vulkan_static_int8_per_channel, + xnnpack, xnnpack_dynamic_int8_per_channel, xnnpack_static_int8_per_channel, xnnpack_static_int8_per_tensor + ] suite: [models, operators] with: ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} diff --git a/backends/test/suite/flow.py b/backends/test/suite/flow.py index fbc5552d7d8..b7a126eaf35 100644 --- a/backends/test/suite/flow.py +++ b/backends/test/suite/flow.py @@ -47,13 +47,17 @@ def all_flows() -> dict[str, TestFlow]: try: from executorch.backends.test.suite.flows.xnnpack import ( + XNNPACK_DYNAMIC_INT8_PER_CHANNEL_TEST_FLOW, XNNPACK_STATIC_INT8_PER_CHANNEL_TEST_FLOW, + XNNPACK_STATIC_INT8_PER_TENSOR_TEST_FLOW, XNNPACK_TEST_FLOW, ) flows += [ XNNPACK_TEST_FLOW, + XNNPACK_DYNAMIC_INT8_PER_CHANNEL_TEST_FLOW, XNNPACK_STATIC_INT8_PER_CHANNEL_TEST_FLOW, + XNNPACK_STATIC_INT8_PER_TENSOR_TEST_FLOW, ] except Exception as e: logger.info(f"Skipping XNNPACK flow registration: {e}") @@ -72,10 +76,14 @@ def all_flows() -> dict[str, TestFlow]: logger.info(f"Skipping Core ML flow registration: {e}") try: - from executorch.backends.test.suite.flows.vulkan import VULKAN_TEST_FLOW + from executorch.backends.test.suite.flows.vulkan import ( + VULKAN_STATIC_INT8_PER_CHANNEL_TEST_FLOW, + VULKAN_TEST_FLOW, + ) flows += [ VULKAN_TEST_FLOW, + VULKAN_STATIC_INT8_PER_CHANNEL_TEST_FLOW, ] except Exception as e: logger.info(f"Skipping Vulkan flow registration: {e}") diff --git a/backends/test/suite/flows/vulkan.py b/backends/test/suite/flows/vulkan.py index 4d661efe3c7..2a8c4e506fa 100644 --- a/backends/test/suite/flows/vulkan.py +++ b/backends/test/suite/flows/vulkan.py @@ -1,17 +1,43 @@ +from typing import Callable + +from executorch.backends.test.harness.stages import Quantize from executorch.backends.test.suite.flow import TestFlow -from executorch.backends.vulkan.test.tester import VulkanTester +from executorch.backends.vulkan.quantizer.vulkan_quantizer import ( + get_symmetric_quantization_config as get_symmetric_quantization_config_vulkan, +) +from executorch.backends.vulkan.test.tester import ( + Quantize as VulkanQuantize, + VulkanTester, +) -def _create_vulkan_flow( - name: str, - quantize: bool = False, +def _create_vulkan_flow_base( + name: str, quantize_stage_factory: Callable[..., Quantize] | None = None ) -> TestFlow: return TestFlow( name, backend="vulkan", tester_factory=VulkanTester, - quantize=quantize, + quantize=quantize_stage_factory is not None, + quantize_stage_factory=quantize_stage_factory, + ) + + +def _create_vulkan_flow() -> TestFlow: + return _create_vulkan_flow_base("vulkan") + + +def _create_vulkan_static_int8_per_channel_flow() -> TestFlow: + def create_quantize_stage() -> Quantize: + qparams = get_symmetric_quantization_config_vulkan() + return VulkanQuantize( + quantization_config=qparams, + ) + + return _create_vulkan_flow_base( + "vulkan_static_int8_per_channel", create_quantize_stage ) -VULKAN_TEST_FLOW = _create_vulkan_flow("vulkan") +VULKAN_TEST_FLOW = _create_vulkan_flow() +VULKAN_STATIC_INT8_PER_CHANNEL_TEST_FLOW = _create_vulkan_static_int8_per_channel_flow() diff --git a/backends/test/suite/flows/xnnpack.py b/backends/test/suite/flows/xnnpack.py index 9de071377ff..a181e2de711 100644 --- a/backends/test/suite/flows/xnnpack.py +++ b/backends/test/suite/flows/xnnpack.py @@ -31,6 +31,20 @@ def _create_xnnpack_flow() -> TestFlow: return _create_xnnpack_flow_base("xnnpack") +def _create_xnnpack_dynamic_int8_per_channel_flow() -> TestFlow: + def create_quantize_stage() -> Quantize: + qparams = get_symmetric_quantization_config( + is_per_channel=True, is_dynamic=True + ) + return XnnpackQuantize( + quantization_config=qparams, + ) + + return _create_xnnpack_flow_base( + "xnnpack_dynamic_int8_per_channel", create_quantize_stage + ) + + def _create_xnnpack_static_int8_per_channel_flow() -> TestFlow: def create_quantize_stage() -> Quantize: qparams = get_symmetric_quantization_config(is_per_channel=True) @@ -43,7 +57,23 @@ def create_quantize_stage() -> Quantize: ) +def _create_xnnpack_static_int8_per_tensor_flow() -> TestFlow: + def create_quantize_stage() -> Quantize: + qparams = get_symmetric_quantization_config(is_per_channel=False) + return XnnpackQuantize( + quantization_config=qparams, + ) + + return _create_xnnpack_flow_base( + "xnnpack_static_int8_per_tensor", create_quantize_stage + ) + + XNNPACK_TEST_FLOW = _create_xnnpack_flow() +XNNPACK_DYNAMIC_INT8_PER_CHANNEL_TEST_FLOW = ( + _create_xnnpack_dynamic_int8_per_channel_flow() +) XNNPACK_STATIC_INT8_PER_CHANNEL_TEST_FLOW = ( _create_xnnpack_static_int8_per_channel_flow() ) +XNNPACK_STATIC_INT8_PER_TENSOR_TEST_FLOW = _create_xnnpack_static_int8_per_tensor_flow() diff --git a/backends/vulkan/test/tester.py b/backends/vulkan/test/tester.py index def5aa05e5f..b2066a06ec0 100644 --- a/backends/vulkan/test/tester.py +++ b/backends/vulkan/test/tester.py @@ -4,7 +4,7 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. -from typing import Any, List, Optional, Tuple +from typing import Any, List, Optional, Sequence, Tuple import executorch import executorch.backends.test.harness.stages as BaseStages @@ -13,8 +13,33 @@ from executorch.backends.test.harness import Tester as TesterBase from executorch.backends.test.harness.stages import StageType from executorch.backends.vulkan.partitioner.vulkan_partitioner import VulkanPartitioner +from executorch.backends.vulkan.quantizer.vulkan_quantizer import ( + get_symmetric_quantization_config as get_symmetric_quantization_config_vulkan, + VulkanQuantizer, +) from executorch.exir import EdgeCompileConfig from executorch.exir.backend.partitioner import Partitioner +from torchao.quantization.pt2e.quantizer import Quantizer + + +class Quantize(BaseStages.Quantize): + def __init__( + self, + quantizer: Optional[Quantizer] = None, + quantization_config: Any | None = None, + calibrate: bool = True, + calibration_samples: Optional[Sequence[Any]] = None, + is_qat: Optional[bool] = False, + ): + super().__init__( + quantizer=quantizer or VulkanQuantizer(), + quantization_config=( + quantization_config or get_symmetric_quantization_config_vulkan() + ), + calibrate=calibrate, + calibration_samples=calibration_samples, + is_qat=is_qat, + ) class Partition(BaseStages.Partition):