Merge branch 'main' into refactor-bitwise-logical-tests

Author: zingo

backends/arm/operator_support/tosa_supported_operators.py

@@ -208,26 +208,41 @@ def is_node_supported(
 class EthosU55NotSupported(OperatorSupportBase):
     """
     Certain operators are not supported on U55. These are listed in `unsupported_ops`.
+    The comment mentions the unsupported TOSA operator that the aten operator maps to where it is not obvious.
+    For unimplemented operators, this is the anticipated mapping, and it might be incorrect.
     """
 
     unsupported_ops = [
-        exir_ops.edge.aten.any.default,
-        exir_ops.edge.aten.any.dim,
-        exir_ops.edge.aten.any.dims,
+        exir_ops.edge.aten.any.default,  # REDUCE_ANY
+        exir_ops.edge.aten.any.dim,  # REDUCE_ANY
+        exir_ops.edge.aten.any.dims,  # REDUCE_ANY
         exir_ops.edge.aten.bitwise_and.Tensor,
         exir_ops.edge.aten.bitwise_or.Tensor,
         exir_ops.edge.aten.bitwise_xor.Tensor,
+        exir_ops.edge.aten.bitwise_not,
         exir_ops.edge.aten.logical_and.default,
         exir_ops.edge.aten.logical_or.default,
         exir_ops.edge.aten.logical_xor.default,
         exir_ops.edge.aten.logical_not.default,
-        exir_ops.edge.aten.amax.default,
-        exir_ops.edge.aten.amin.default,
+        exir_ops.edge.aten.amax.default,  # REDUCE_MAX
+        exir_ops.edge.aten.amin.default,  # REDUCE_MIN
         exir_ops.edge.aten.eq.Tensor,
         exir_ops.edge.aten.ge.Tensor,
         exir_ops.edge.aten.gt.Tensor,
         exir_ops.edge.aten.le.Tensor,
         exir_ops.edge.aten.lt.Tensor,
+        exir_ops.edge.aten.flip.default,  # REVERSE
+        exir_ops.edge.aten.grid_sampler_2d,  # GATHER
+        exir_ops.edge.aten.scatter.src,
+        exir_ops.edge.aten.scatter.value,
+        exir_ops.edge.aten.select_scatter.default,
+        exir_ops.edge.aten.scatter_reduce.two,
+        exir_ops.edge.aten.scatter_add.default,
+        exir_ops.edge.aten.upsample_nearest2d.vec,  # RESIZE
+        exir_ops.edge.aten.upsample_bilinear2d.vec,  # RESIZE
+        exir_ops.edge.aten.reflection_pad1d.default,  # REVERSE
+        exir_ops.edge.aten.reflection_pad2d.default,  # REVERSE
+        exir_ops.edge.aten.reflection_pad3d.default,  # REVERSE
     ]
 
     def __init__(self, reporter: WhyNoPartitionReporter):

backends/arm/test/common.py

@@ -231,6 +231,7 @@ def parametrize(
     arg_name: str,
     test_data: dict[str, Any],
     xfails: dict[str, xfail_type] | None = None,
+    strict: bool = True,
 ):
     """
     Custom version of pytest.mark.parametrize with some syntatic sugar and added xfail functionality
@@ -261,7 +262,9 @@ def decorator_func(func):
                 pytest_param = pytest.param(
                     test_parameters,
                     id=id,
-                    marks=pytest.mark.xfail(reason=reason, raises=raises, strict=True),
+                    marks=pytest.mark.xfail(
+                        reason=reason, raises=raises, strict=strict
+                    ),
                 )
             else:
                 pytest_param = pytest.param(test_parameters, id=id)

backends/arm/test/ops/test_sigmoid_16bit.py

@@ -3,6 +3,8 @@
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
+import pytest
+
 import torch
 from executorch.backends.arm.quantizer.arm_quantizer import (
     get_symmetric_quantization_config,
@@ -48,12 +50,12 @@ def get_16bit_sigmoid_quantizer(tosa_str: str):
 
 input_t = tuple[torch.Tensor]
 test_data_suite = {
-    "ones": (torch.ones(10, 10, 10),),
-    "rand": (torch.rand(10, 10) - 0.5,),
-    "rand_4d": (torch.rand(1, 1, 5, 10),),
-    "randn_pos": (torch.randn(10) + 10,),
-    "randn_neg": (torch.randn(10) - 10,),
-    "ramp": (torch.arange(-16, 16, 0.02),),
+    "ones": lambda: torch.ones(10, 10, 10),
+    "rand": lambda: torch.rand(10, 10) - 0.5,
+    "rand_4d": lambda: torch.rand(1, 1, 5, 10),
+    "randn_pos": lambda: torch.randn(10) + 10,
+    "randn_neg": lambda: torch.randn(10) - 10,
+    "ramp": lambda: torch.arange(-16, 16, 0.02),
 }
 
 
@@ -79,8 +81,11 @@ def forward(self, x):
 
 
 @common.parametrize("test_data", test_data_suite)
+@pytest.mark.flaky(reruns=5)
 def test_sigmoid_tosa_BI(test_data):
-    pipeline = TosaPipelineBI(Sigmoid(), test_data, Sigmoid.aten_op, Sigmoid.exir_op)
+    pipeline = TosaPipelineBI(
+        Sigmoid(), (test_data(),), Sigmoid.aten_op, Sigmoid.exir_op
+    )
     pipeline.change_args("quantize", get_16bit_sigmoid_quantizer("TOSA-0.80+BI"))
     pipeline.run()
 
@@ -89,12 +94,13 @@ def test_sigmoid_tosa_BI(test_data):
     "test_data",
     test_data_suite,
     xfails={
-        "ramp": "AssertionError: Output 0 does not match reference output. Passes with qtol=2. MLETORCH-787"
+        "ramp": "AssertionError: Output 0 does not match reference output. MLETORCH-787"
     },
 )
+@pytest.mark.flaky(reruns=5)
 def test_sigmoid_add_sigmoid_tosa_BI(test_data):
     pipeline = TosaPipelineBI(
-        SigmoidAddSigmoid(), test_data, Sigmoid.aten_op, Sigmoid.exir_op
+        SigmoidAddSigmoid(), (test_data(),), Sigmoid.aten_op, Sigmoid.exir_op
     )
     pipeline.change_args("quantize", get_16bit_sigmoid_quantizer("TOSA-0.80+BI"))
     pipeline.run()
@@ -107,13 +113,18 @@ def test_sigmoid_add_sigmoid_tosa_BI(test_data):
         "ones": "AssertionError: Output 0 does not match reference output. MLBEDSW-9770",
         "rand": "AssertionError: Output 0 does not match reference output. MLBEDSW-9770",
         "rand_4d": "AssertionError: Output 0 does not match reference output. MLBEDSW-9770",
+        "randn_pos": "AssertionError: Output 0 does not match reference output. MLBEDSW-9770",
+        "randn_neg": "AssertionError: Output 0 does not match reference output. MLBEDSW-9770",
         "ramp": "AssertionError: Output 0 does not match reference output. MLBEDSW-9770",
     },
+    # int16 tables are not supported, but some tests happen to pass regardless.
+    # Set them to xfail but strict=False -> ok if they pass.
+    strict=False,
 )
 @common.XfailIfNoCorstone300
 def test_sigmoid_tosa_u55(test_data):
     pipeline = EthosU55PipelineBI(
-        Sigmoid(), test_data, Sigmoid.aten_op, Sigmoid.exir_op, run_on_fvp=True
+        Sigmoid(), (test_data(),), Sigmoid.aten_op, Sigmoid.exir_op, run_on_fvp=True
     )
     pipeline.change_args("quantize", get_16bit_sigmoid_quantizer("TOSA-0.80+BI+u55"))
     pipeline.run()
@@ -127,14 +138,18 @@ def test_sigmoid_tosa_u55(test_data):
         "rand": "AssertionError: Output 0 does not match reference output. MLBEDSW-9770",
         "rand_4d": "AssertionError: Output 0 does not match reference output. MLBEDSW-9770",
         "randn_neg": "AssertionError: Output 0 does not match reference output. MLBEDSW-9770",
-        "ramp": "AssertionError: Output 0 does not match reference output. MLBEDSW-9770",
+        "randn_pos": "AssertionError: Output 0 does not match reference output. MLBEDSW-9770",
+        "ramp": "AsssertionError: Output 0 does not match reference output. MLBEDSW-9770",
     },
+    # int16 tables are not supported, but some tests happen to pass regardless.
+    # Set them to xfail but strict=False -> ok if they pass.
+    strict=False,
 )
 @common.XfailIfNoCorstone300
 def test_sigmoid_add_sigmoid_tosa_u55(test_data):
     pipeline = EthosU55PipelineBI(
         SigmoidAddSigmoid(),
-        test_data,
+        (test_data(),),
         Sigmoid.aten_op,
         Sigmoid.exir_op,
         run_on_fvp=True,
@@ -144,10 +159,11 @@ def test_sigmoid_add_sigmoid_tosa_u55(test_data):
 
 
 @common.parametrize("test_data", test_data_suite)
+@pytest.mark.flaky(reruns=5)
 @common.XfailIfNoCorstone320
 def test_sigmoid_tosa_u85(test_data):
     pipeline = EthosU85PipelineBI(
-        Sigmoid(), test_data, Sigmoid.aten_op, Sigmoid.exir_op, run_on_fvp=True
+        Sigmoid(), (test_data(),), Sigmoid.aten_op, Sigmoid.exir_op, run_on_fvp=True
     )
     pipeline.change_args("quantize", get_16bit_sigmoid_quantizer("TOSA-0.80+BI"))
     pipeline.run()
@@ -160,11 +176,12 @@ def test_sigmoid_tosa_u85(test_data):
         "ramp": "AssertionError: Output 0 does not match reference output.",
     },
 )
+@pytest.mark.flaky(reruns=5)
 @common.XfailIfNoCorstone320
 def test_sigmoid_add_sigmoid_tosa_u85(test_data):
     pipeline = EthosU85PipelineBI(
         SigmoidAddSigmoid(),
-        test_data,
+        (test_data(),),
         Sigmoid.aten_op,
         Sigmoid.exir_op,
         run_on_fvp=True,

backends/arm/test/ops/test_sigmoid_32bit.py

@@ -3,6 +3,7 @@
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
+import pytest
 import torch
 from executorch.backends.arm.quantizer.arm_quantizer import TOSAQuantizer
 from executorch.backends.arm.quantizer.quantization_config import QuantizationConfig
@@ -52,7 +53,7 @@ def _get_32_bit_quant_config():
     return qconfig
 
 
-def get_16bit_sigmoid_quantizer(tosa_str: str):
+def get_32bit_sigmoid_quantizer(tosa_str: str):
     tosa_spec = common.TosaSpecification.create_from_string(tosa_str)
     quantizer = TOSAQuantizer(tosa_spec)
     quantizer.set_global(_get_32_bit_quant_config())
@@ -65,12 +66,12 @@ def get_16bit_sigmoid_quantizer(tosa_str: str):
 
 input_t = tuple[torch.Tensor]
 test_data_suite = {
-    "ones": (torch.ones(10, 10, 10),),
-    "rand": (torch.rand(10, 10) - 0.5,),
-    "rand_4d": (torch.rand(1, 10, 10, 10),),
-    "randn_pos": (torch.randn(10) + 10,),
-    "randn_neg": (torch.randn(10) - 10,),
-    "ramp": (torch.arange(-16, 16, 0.2),),
+    "ones": lambda: torch.ones(10, 10, 10),
+    "rand": lambda: torch.rand(10, 10) - 0.5,
+    "rand_4d": lambda: torch.rand(1, 10, 10, 10),
+    "randn_pos": lambda: torch.randn(10) + 10,
+    "randn_neg": lambda: torch.randn(10) - 10,
+    "ramp": lambda: torch.arange(-16, 16, 0.2),
 }
 
 
@@ -96,28 +97,28 @@ def forward(self, x):
 
 
 @common.parametrize("test_data", test_data_suite)
+@pytest.mark.flaky(reruns=5)
 def test_sigmoid_tosa_BI(test_data):
     pipeline = TosaPipelineBI(
         Sigmoid(),
-        test_data,
+        (test_data(),),
         Sigmoid.aten_op,
         Sigmoid.exir_op,
     )
-    pipeline.change_args("quantize", get_16bit_sigmoid_quantizer("TOSA-0.80+BI"))
+    pipeline.change_args("quantize", get_32bit_sigmoid_quantizer("TOSA-0.80+BI"))
     pipeline.run()
 
 
 @common.parametrize("test_data", test_data_suite)
+@pytest.mark.flaky(reruns=5)
 def test_sigmoid_add_sigmoid_tosa_BI(test_data):
     pipeline = TosaPipelineBI(
         SigmoidAddSigmoid(),
-        test_data,
+        (test_data(),),
         Sigmoid.aten_op,
         Sigmoid.exir_op,
     )
-    pipeline.change_args("quantize", get_16bit_sigmoid_quantizer("TOSA-0.80+BI"))
-    pipeline.change_args("run_method_and_compare_outputs", test_data, qtol=1)
-
+    pipeline.change_args("quantize", get_32bit_sigmoid_quantizer("TOSA-0.80+BI"))
     pipeline.run()
 
 
@@ -129,16 +130,19 @@ def test_sigmoid_add_sigmoid_tosa_BI(test_data):
         "rand": "AssertionError: Output 0 does not match reference output. MLBEDSW-9770",
         "rand_4d": "AssertionError: Output 0 does not match reference output. MLBEDSW-9770",
         "randn_pos": "AssertionError: Output 0 does not match reference output. MLBEDSW-9770",
+        "randn_neg": "AssertionError: Output 0 does not match reference output. MLBEDSW-9770",
         "ramp": "AssertionError: Output 0 does not match reference output. MLBEDSW-9770",
     },
+    # int16 tables are not supported, but some tests happen to pass regardless.
+    # Set them to xfail but strict=False -> ok if they pass.
+    strict=False,
 )
 @common.XfailIfNoCorstone300
 def test_sigmoid_tosa_u55(test_data):
     pipeline = EthosU55PipelineBI(
-        Sigmoid(), test_data, Sigmoid.aten_op, Sigmoid.exir_op, run_on_fvp=True
+        Sigmoid(), (test_data(),), Sigmoid.aten_op, Sigmoid.exir_op, run_on_fvp=True
     )
-    pipeline.change_args("quantize", get_16bit_sigmoid_quantizer("TOSA-0.80+BI+u55"))
-    pipeline.change_args("run_method_and_compare_outputs", test_data, qtol=1)
+    pipeline.change_args("quantize", get_32bit_sigmoid_quantizer("TOSA-0.80+BI+u55"))
     pipeline.run()
 
 
@@ -153,29 +157,31 @@ def test_sigmoid_tosa_u55(test_data):
         "randn_neg": "AssertionError: Output 0 does not match reference output. MLBEDSW-9770",
         "ramp": "AssertionError: Output 0 does not match reference output. MLBEDSW-9770",
     },
+    # int16 tables are not supported, but some tests happen to pass regardless.
+    # Set them to xfail but strict=False -> ok if they pass.
+    strict=False,
 )
 @common.XfailIfNoCorstone300
 def test_sigmoid_add_sigmoid_tosa_u55(test_data):
     pipeline = EthosU55PipelineBI(
         SigmoidAddSigmoid(),
-        test_data,
+        (test_data(),),
         Sigmoid.aten_op,
         Sigmoid.exir_op,
         run_on_fvp=True,
     )
-    pipeline.change_args("quantize", get_16bit_sigmoid_quantizer("TOSA-0.80+BI+u55"))
-    pipeline.change_args("run_method_and_compare_outputs", test_data, qtol=1)
+    pipeline.change_args("quantize", get_32bit_sigmoid_quantizer("TOSA-0.80+BI+u55"))
     pipeline.run()
 
 
 @common.parametrize("test_data", test_data_suite)
+@pytest.mark.flaky(reruns=5)
 @common.XfailIfNoCorstone320
 def test_sigmoid_tosa_u85(test_data):
     pipeline = EthosU85PipelineBI(
-        Sigmoid(), test_data, Sigmoid.aten_op, Sigmoid.exir_op, run_on_fvp=True
+        Sigmoid(), (test_data(),), Sigmoid.aten_op, Sigmoid.exir_op, run_on_fvp=True
     )
-    pipeline.change_args("quantize", get_16bit_sigmoid_quantizer("TOSA-0.80+BI"))
-    pipeline.change_args("run_method_and_compare_outputs", test_data, qtol=1)
+    pipeline.change_args("quantize", get_32bit_sigmoid_quantizer("TOSA-0.80+BI"))
     pipeline.run()
 
 
@@ -186,15 +192,15 @@ def test_sigmoid_tosa_u85(test_data):
         "ramp": "AssertionError: Output 0 does not match reference output.",
     },
 )
+@pytest.mark.flaky(reruns=5)
 @common.XfailIfNoCorstone320
 def test_sigmoid_add_sigmoid_tosa_u85(test_data):
     pipeline = EthosU85PipelineBI(
         SigmoidAddSigmoid(),
-        test_data,
+        (test_data(),),
         Sigmoid.aten_op,
         Sigmoid.exir_op,
         run_on_fvp=True,
     )
-    pipeline.change_args("quantize", get_16bit_sigmoid_quantizer("TOSA-0.80+BI"))
-    pipeline.change_args("run_method_and_compare_outputs", test_data, qtol=1)
+    pipeline.change_args("quantize", get_32bit_sigmoid_quantizer("TOSA-0.80+BI"))
     pipeline.run()