Update

[ghstack-poisoned]

Author: GregoryComer

backends/apple/coreml/test/tester.py

@@ -4,12 +4,12 @@
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
+import functools
 from typing import Any, List, Optional, Sequence, Tuple
 
 import coremltools as ct
 import executorch
 import executorch.backends.test.harness.stages as BaseStages
-import functools
 import torch
 
 from executorch.backends.apple.coreml.compiler import CoreMLBackend
@@ -21,7 +21,7 @@
 from executorch.exir.backend.partitioner import Partitioner
 
 
-def _get_static_int8_qconfig():
+def _get_static_int8_linear_qconfig():
     return ct.optimize.torch.quantization.LinearQuantizerConfig(
         global_config=ct.optimize.torch.quantization.ModuleLinearQuantizerConfig(
             quantization_scheme="symmetric",
@@ -42,22 +42,23 @@ def __init__(
         is_qat: Optional[bool] = False,
     ):
         super().__init__(
-            quantizer=quantizer or CoreMLQuantizer(quantization_config or _get_static_int8_qconfig()),
+            quantizer=quantizer
+            or CoreMLQuantizer(quantization_config or _get_static_int8_linear_qconfig()),
             calibrate=calibrate,
             calibration_samples=calibration_samples,
             is_qat=is_qat,
         )
 
 
-
 class Partition(BaseStages.Partition):
     def __init__(
-        self, 
+        self,
         partitioner: Optional[Partitioner] = None,
         minimum_deployment_target: Optional[Any] = ct.target.iOS15,
     ):
         super().__init__(
-            partitioner=partitioner or CoreMLPartitioner(
+            partitioner=partitioner
+            or CoreMLPartitioner(
                 compile_specs=CoreMLBackend.generate_compile_specs(
                     minimum_deployment_target=minimum_deployment_target
                 )
@@ -74,9 +75,9 @@ def __init__(
     ):
         super().__init__(
             default_partitioner_cls=lambda: CoreMLPartitioner(
-               compile_specs=CoreMLBackend.generate_compile_specs(
+                compile_specs=CoreMLBackend.generate_compile_specs(
                     minimum_deployment_target=minimum_deployment_target
-                ) 
+                )
             ),
             partitioners=partitioners,
             edge_compile_config=edge_compile_config,
@@ -96,8 +97,13 @@ def __init__(
             executorch.backends.test.harness.Tester.default_stage_classes()
             | {
                 StageType.QUANTIZE: Quantize,
-                StageType.PARTITION: functools.partial(Partition, minimum_deployment_target=minimum_deployment_target),
-                StageType.TO_EDGE_TRANSFORM_AND_LOWER: functools.partial(ToEdgeTransformAndLower, minimum_deployment_target=minimum_deployment_target),
+                StageType.PARTITION: functools.partial(
+                    Partition, minimum_deployment_target=minimum_deployment_target
+                ),
+                StageType.TO_EDGE_TRANSFORM_AND_LOWER: functools.partial(
+                    ToEdgeTransformAndLower,
+                    minimum_deployment_target=minimum_deployment_target,
+                ),
             }
         )
 

backends/test/harness/stages/quantize.py

@@ -25,13 +25,14 @@ def __init__(
         calibrate: bool = True,
         calibration_samples: Optional[Sequence[Any]] = None,
         is_qat: Optional[bool] = False,
+        set_global: bool = True,
     ):
         self.quantizer = quantizer
         self.quantization_config = quantization_config
         self.calibrate = calibrate
         self.calibration_samples = calibration_samples
 
-        if self.quantization_config is not None:
+        if self.quantization_config is not None and set_global:
             self.quantizer.set_global(self.quantization_config)
 
         self.converted_graph = None

backends/test/suite/flow.py

@@ -22,30 +22,39 @@ class TestFlow:
 
     backend: str
     """ The name of the target backend. """
-    
+
     tester_factory: Callable[..., Tester]
     """ A factory function that returns a Tester instance for this lowering flow. """
 
     quantize: bool = field(default=False)
     """ Whether to tester should run the quantize stage on the model. """
-    
+
     quantize_stage_factory: Callable[..., Quantize] | None = None
     """ A factory function which instantiates a Quantize stage. Can be None to use the tester's default. """
 
+
 def all_flows() -> dict[str, TestFlow]:
     flows = []
-    
+
     try:
-        from executorch.backends.test.suite.flows.xnnpack import XNNPACK_TEST_FLOW, XNNPACK_STATIC_INT8_TEST_FLOW
+        from executorch.backends.test.suite.flows.xnnpack import (
+            XNNPACK_STATIC_INT8_PER_CHANNEL_TEST_FLOW,
+            XNNPACK_TEST_FLOW,
+        )
+
         flows += [
             XNNPACK_TEST_FLOW,
-            XNNPACK_STATIC_INT8_TEST_FLOW,
+            XNNPACK_STATIC_INT8_PER_CHANNEL_TEST_FLOW,
         ]
     except Exception as e:
         logger.info(f"Skipping XNNPACK flow registration: {e}")
 
     try:
-        from executorch.backends.test.suite.flows.coreml import COREML_TEST_FLOW, COREML_STATIC_INT8_TEST_FLOW
+        from executorch.backends.test.suite.flows.coreml import (
+            COREML_STATIC_INT8_TEST_FLOW,
+            COREML_TEST_FLOW,
+        )
+
         flows += [
             COREML_TEST_FLOW,
             COREML_STATIC_INT8_TEST_FLOW,

backends/test/suite/flows/coreml.py

@@ -1,24 +1,30 @@
-import coremltools
 import functools
+from typing import Any
+
+import coremltools
 
 from executorch.backends.apple.coreml.test.tester import CoreMLTester
 from executorch.backends.test.suite.flow import TestFlow
-from typing import Any
+
 
 def _create_coreml_flow(
-    name: str, 
-    quantize: bool = False, 
-    minimum_deployment_target: Any = coremltools.target.iOS15
+    name: str,
+    quantize: bool = False,
+    minimum_deployment_target: Any = coremltools.target.iOS15,
 ) -> TestFlow:
     return TestFlow(
         name,
         backend="coreml",
-        tester_factory=functools.partial(CoreMLTester, minimum_deployment_target=minimum_deployment_target),
+        tester_factory=functools.partial(
+            CoreMLTester, minimum_deployment_target=minimum_deployment_target
+        ),
         quantize=quantize,
     )
 
+
 COREML_TEST_FLOW = _create_coreml_flow("coreml")
 COREML_STATIC_INT8_TEST_FLOW = _create_coreml_flow(
-    "coreml_static_int8", 
+    "coreml_static_int8",
     quantize=True,
-    minimum_deployment_target=coremltools.target.iOS17)
+    minimum_deployment_target=coremltools.target.iOS17,
+)

backends/test/suite/flows/xnnpack.py

@@ -1,36 +1,45 @@
+import logging
+from typing import Callable
+
 from executorch.backends.test.harness.stages import Quantize
 from executorch.backends.test.suite.flow import TestFlow
-from executorch.backends.xnnpack.quantizer.xnnpack_quantizer import get_symmetric_quantization_config
+from executorch.backends.xnnpack.quantizer.xnnpack_quantizer import (
+    get_symmetric_quantization_config,
+)
 from executorch.backends.xnnpack.test.tester import (
     Quantize as XnnpackQuantize,
-    Tester as XnnpackTester
+    Tester as XnnpackTester,
 )
-from typing import Callable
-
-import logging
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
 
-def _create_xnnpack_flow_base(name: str, quantize_stage_factory: Callable[..., Quantize] | None = None) -> TestFlow:
+
+def _create_xnnpack_flow_base(
+    name: str, quantize_stage_factory: Callable[..., Quantize] | None = None
+) -> TestFlow:
     return TestFlow(
         name,
         backend="xnnpack",
         tester_factory=XnnpackTester,
         quantize=quantize_stage_factory is not None,
         quantize_stage_factory=quantize_stage_factory,
     )
-    
+
+
 def _create_xnnpack_flow() -> TestFlow:
     return _create_xnnpack_flow_base("xnnpack")
 
-def _create_xnnpack_static_int8_flow() -> TestFlow:
+
+def _create_xnnpack_static_int8_per_channel_flow() -> TestFlow:
     def create_quantize_stage() -> Quantize:
-        qparams = get_symmetric_quantization_config(is_per_channel=True) 
+        qparams = get_symmetric_quantization_config(is_per_channel=True)
         return XnnpackQuantize(
             quantization_config=qparams,
         )
-    return _create_xnnpack_flow_base("xnnpack_static_int8", create_quantize_stage)
+
+    return _create_xnnpack_flow_base("xnnpack_static_int8_per_channel", create_quantize_stage)
+
 
 XNNPACK_TEST_FLOW = _create_xnnpack_flow()
-XNNPACK_STATIC_INT8_TEST_FLOW = _create_xnnpack_static_int8_flow()
+XNNPACK_STATIC_INT8_PER_CHANNEL_TEST_FLOW = _create_xnnpack_static_int8_per_channel_flow()

backends/test/suite/models/__init__.py

@@ -12,7 +12,6 @@
 from typing import Any, Callable
 
 import torch
-from executorch.backends.test.harness import Tester
 from executorch.backends.test.suite import get_test_flows
 from executorch.backends.test.suite.context import get_active_test_context, TestContext
 from executorch.backends.test.suite.flow import TestFlow

backends/test/suite/models/test_torchaudio.py

@@ -7,7 +7,7 @@
 # pyre-unsafe
 
 import unittest
-from typing import Callable, Tuple
+from typing import Tuple
 
 import torch
 import torchaudio
@@ -92,7 +92,10 @@ def test_wav2letter(
 
     @unittest.skip("This model times out on all backends.")
     def test_wavernn(
-        self, flow: TestFlow, dtype: torch.dtype, use_dynamic_shapes: bool,
+        self,
+        flow: TestFlow,
+        dtype: torch.dtype,
+        use_dynamic_shapes: bool,
     ):
         model = torchaudio.models.WaveRNN(
             upsample_scales=[5, 5, 8], n_classes=512, hop_length=200

backends/test/suite/models/test_torchvision.py

@@ -154,9 +154,7 @@ def test_swin_v2_t(
         model = torchvision.models.swin_v2_t()
         self._test_cv_model(model, flow, dtype, use_dynamic_shapes)
 
-    def test_vgg11(
-        self, flow: TestFlow, dtype: torch.dtype, use_dynamic_shapes: bool
-    ):
+    def test_vgg11(self, flow: TestFlow, dtype: torch.dtype, use_dynamic_shapes: bool):
         model = torchvision.models.vgg11()
         self._test_cv_model(model, flow, dtype, use_dynamic_shapes)
 

backends/test/suite/operators/test_gelu.py

@@ -26,9 +26,7 @@ def forward(self, x):
 class TestGELU(OperatorTest):
     @dtype_test
     def test_gelu_dtype(self, flow: TestFlow, dtype) -> None:
-        self._test_op(
-            Model(), ((torch.rand(2, 10) * 10 - 5).to(dtype),), flow
-        )
+        self._test_op(Model(), ((torch.rand(2, 10) * 10 - 5).to(dtype),), flow)
 
     def test_gelu_f32_single_dim(self, flow: TestFlow) -> None:
         self._test_op(Model(), (torch.randn(20),), flow)
@@ -37,9 +35,7 @@ def test_gelu_f32_multi_dim(self, flow: TestFlow) -> None:
         self._test_op(Model(), (torch.randn(2, 3, 4, 5),), flow)
 
     def test_gelu_f32_tanh_approximation(self, flow: TestFlow) -> None:
-        self._test_op(
-            Model(approximate="tanh"), (torch.randn(3, 4, 5),), flow
-        )
+        self._test_op(Model(approximate="tanh"), (torch.randn(3, 4, 5),), flow)
 
     def test_gelu_f32_boundary_values(self, flow: TestFlow) -> None:
         # Test with specific values spanning negative and positive ranges

backends/test/suite/operators/test_glu.py

@@ -27,9 +27,7 @@ class TestGLU(OperatorTest):
     @dtype_test
     def test_glu_dtype(self, flow: TestFlow, dtype) -> None:
         # Input must have even number of elements in the specified dimension
-        self._test_op(
-            Model(), ((torch.rand(2, 10) * 10 - 5).to(dtype),), flow
-        )
+        self._test_op(Model(), ((torch.rand(2, 10) * 10 - 5).to(dtype),), flow)
 
     def test_glu_f32_dim_last(self, flow: TestFlow) -> None:
         # Default dim is -1 (last dimension)