diff --git a/backends/arm/scripts/parse_test_names.py b/backends/arm/scripts/parse_test_names.py index 4d4f9a21155..45dbba4c67e 100644 --- a/backends/arm/scripts/parse_test_names.py +++ b/backends/arm/scripts/parse_test_names.py @@ -14,6 +14,8 @@ "linear.default", "maximum.default", "adaptive_avg_pool2d.default", + "bitwise_right_shift.Tensor", + "bitwise_left_shift.Tensor", ] ALL_EDGE_OPS = SAMPLE_INPUT.keys() | CUSTOM_EDGE_OPS diff --git a/backends/arm/test/ops/test_scalars.py b/backends/arm/test/ops/test_scalars.py index a4748e93fdb..7a06f7dfc8d 100644 --- a/backends/arm/test/ops/test_scalars.py +++ b/backends/arm/test/ops/test_scalars.py @@ -6,13 +6,14 @@ from typing import Tuple +import pytest + import torch from executorch.backends.arm.test import common from executorch.backends.arm.test.tester.test_pipeline import ( TosaPipelineBI, TosaPipelineMI, - TransformAnnotationPassPipeline, ) """ @@ -35,247 +36,404 @@ class Add(torch.nn.Module): + aten_op = "torch.ops.aten.add.Tensor" + def forward(self, x, y): return x + y class Sub(torch.nn.Module): + aten_op = "torch.ops.aten.sub.Tensor" + def forward(self, x, y): return x - y class Div(torch.nn.Module): + aten_op = "torch.ops.aten.div.Tensor" + def forward(self, x, y): return x / y class Mul(torch.nn.Module): + aten_op = "torch.ops.aten.mul.Tensor" + def forward(self, x, y): return x * y class MulScalar(torch.nn.Module): + aten_op = "torch.ops.aten.mul.Scalar" + def forward(self, x, y): return torch.ops.aten.mul.Scalar(x, y) class DivScalar(torch.nn.Module): + aten_op = "torch.ops.aten.div.Scalar" + def forward(self, x, y): return torch.ops.aten.div.Scalar(x, y) class AddScalar(torch.nn.Module): + aten_op = "torch.ops.aten.add.Scalar" + def forward(self, x, y): return torch.ops.aten.add.Scalar(x, y) class SubScalar(torch.nn.Module): + aten_op = "torch.ops.aten.sub.Scalar" + def forward(self, x, y): return torch.ops.aten.sub.Scalar(x, y) class AddInplace(torch.nn.Module): + aten_op = "torch.ops.aten.add_.Tensor" + def forward(self, x, y): x += y return x class SubInplace(torch.nn.Module): + aten_op = "torch.ops.aten.sub_.Tensor" + def forward(self, x, y): x -= y return x class DivInplace(torch.nn.Module): + aten_op = "torch.ops.aten.div_.Tensor" + def forward(self, x, y): x /= y return x class MulInplace(torch.nn.Module): + aten_op = "torch.ops.aten.mul_.Tensor" + def forward(self, x, y): x *= y return x class AddConst(torch.nn.Module): + aten_op = "torch.ops.aten.add.Tensor" + def forward(self, x): x = 1.0 + x return x class ShiftInplaceSub(torch.nn.Module): + def forward(self, x): x = x >> 4 x -= 10 return x -# Inplace ops end with '_' (from aten naming) -ops = [ - ("Add", Add()), - ("Sub", Sub()), - ("Mul", Mul()), - ("Div", Div()), - ("Add_", AddInplace()), - ("Sub_", SubInplace()), - ("Mul_", MulInplace()), - ("Div_", DivInplace()), - ("MulScalar", MulScalar()), - ("DivScalar", DivScalar()), - ("AddScalar", AddScalar()), - ("SubScalar", SubScalar()), -] - -const_ops = [("Add", AddConst())] - dtypes = [("int", 3), ("float", 3.0)] sizes = [("r1", (1)), ("r4", (2, 4, 5, 3))] # Create combinations of tests tensor_scalar_tests = {} -for op in ops: - for dtype in dtypes: - for size in sizes: - test_name = f"{op[0]}_{dtype[0]}_{size[0]}" - tensor = torch.rand(size[1]) - scalar = dtype[1] - tensor_scalar_tests[test_name + "_ts"] = (op[1], tensor, scalar) - # Don't add (scalar, tensor) test case for .Scalar ops. - if op[0][-6:] == "Scalar": - continue - - tensor_scalar_tests[test_name + "_st"] = (op[1], scalar, tensor) - -tensor_const_tests = {} -for op in const_ops: +for dtype in dtypes: for size in sizes: - test_name = f"{op[0]}_{size[0]}" + test_name = f"{dtype[0]}_{size[0]}" tensor = torch.rand(size[1]) - tensor_const_tests[test_name] = (op[1], tensor) + scalar = dtype[1] + tensor_scalar_tests[test_name + "_ts"] = (tensor, scalar) + # # Don't add (scalar, tensor) test case for .Scalar ops. + # if op[0][-6:] == "Scalar": + # continue + tensor_scalar_tests[test_name + "_st"] = (scalar, tensor) -def _test_add_tosa_MI_pipeline(module: torch.nn.Module, test_data: tuple): - pipeline = TosaPipelineMI[input_t1](module, test_data, aten_op=[], exir_op=[]) +tensor_const_tests = {} +for size in sizes: + test_name = f"{size[0]}" + tensor = torch.rand(size[1]) + tensor_const_tests[test_name] = (tensor,) + +xfails = { + "int_r1_st": "MLETORCH-408: Arithmetic ops can't handle scalars first", + "int_r4_st": "MLETORCH-408: Arithmetic ops can't handle scalars first", + "float_r1_st": "MLETORCH-408: Arithmetic ops can't handle scalars first", + "float_r4_st": "MLETORCH-408: Arithmetic ops can't handle scalars first", +} + + +# ADD MI ------------------------------------------------------ +@common.parametrize("test_data", tensor_scalar_tests, xfails=xfails) +def test_add_tensor_tosa_MI_scalar(test_data): + """Tests regular add with one scalar input.""" + pipeline = TosaPipelineMI[input_t1](Add(), test_data, aten_op=Add.aten_op) pipeline.run() -def _test_add_tosa_BI_pipeline( - module: torch.nn.Module, test_data: tuple, check_quant_nodes=True -): - pipeline = TosaPipelineBI[input_t1](module, test_data, aten_op=[], exir_op=[]) - if not check_quant_nodes: - pipeline.pop_stage("check.quant_nodes") +@common.parametrize("test_data", tensor_scalar_tests, xfails=xfails) +def test_add_tensor_tosa_MI_inplace(test_data): + """Tests inplace add with one scalar input.""" + pipeline = TosaPipelineMI[input_t1](AddInplace(), test_data, aten_op=[]) pipeline.run() -fail_str = "MLETORCH-408: Arithmetic ops can't handle scalars first for MI" -MI_xfails = { - "Add_int_r1_st": fail_str, - "Add_int_r4_st": fail_str, - "Add_float_r1_st": fail_str, - "Add_float_r4_st": fail_str, - "Sub_int_r1_ts": fail_str, - "Sub_int_r1_st": fail_str, - "Sub_int_r4_ts": fail_str, - "Sub_int_r4_st": fail_str, - "Sub_float_r1_st": fail_str, - "Sub_float_r4_st": fail_str, - "Mul_int_r1_st": fail_str, - "Mul_int_r4_st": fail_str, - "Mul_float_r1_st": fail_str, - "Mul_float_r4_st": fail_str, - "Div_int_r1_st": fail_str, - "Div_int_r4_st": fail_str, - "Div_float_r1_st": fail_str, - "Div_float_r4_st": fail_str, - "Add__int_r1_st": fail_str, - "Add__float_r1_st": fail_str, - "Add__float_r4_st": fail_str, - "Add__int_r4_st": fail_str, - "Sub__int_r1_ts": fail_str, - "Sub__int_r1_st": fail_str, - "Sub__int_r4_ts": fail_str, - "Sub__int_r4_st": fail_str, - "Sub__float_r1_st": fail_str, - "Sub__float_r4_st": fail_str, - "Mul__int_r1_st": fail_str, - "Mul__int_r4_st": fail_str, - "Mul__float_r1_st": fail_str, - "Mul__float_r4_st": fail_str, - "Div__int_r1_st": fail_str, - "Div__int_r4_st": fail_str, - "Div__float_r1_st": fail_str, - "Div__float_r4_st": fail_str, -} +@common.parametrize("test_data", tensor_const_tests, xfails=xfails) +def test_add_tensor_tosa_MI_const(test_data): + """Tests regular add with one scalar input, with one of inputs constant.""" + pipeline = TosaPipelineMI[input_t1](AddConst(), test_data, aten_op=AddConst.aten_op) + pipeline.run() -@common.parametrize( - "tensor_scalar_tests", - tensor_scalar_tests, - MI_xfails, -) -def test_tosa_MI(tensor_scalar_tests: list): - op, x, y = tensor_scalar_tests - _test_add_tosa_MI_pipeline(op, (x, y)) +@common.parametrize("test_data", tensor_scalar_tests, xfails=xfails) +def test_add_scalar_tosa_MI(test_data): + """Tests a scalar add with one scalar input.""" + pipeline = TosaPipelineMI[input_t1]( + AddScalar(), test_data, aten_op=AddScalar.aten_op + ) + pipeline.run() + + +# ADD BI ------------------------------------------------------ +@common.parametrize("test_data", tensor_scalar_tests) +def test_add_tensor_tosa_BI_scalar(test_data): + """Tests regular add with one scalar input.""" + pipeline = TosaPipelineBI[input_t1](Add(), test_data, aten_op=[]) + pipeline.run() + + +@common.parametrize("test_data", tensor_scalar_tests) +def test_add_tensor_tosa_BI_inplace(test_data): + """Tests inplace add with one scalar input.""" + pipeline = TosaPipelineBI[input_t1](AddInplace(), test_data, aten_op=[]) + pipeline.run() -def _test_passes_tosa_BI_pipeline(module: torch.nn.Module, test_data: tuple): - pipeline = TransformAnnotationPassPipeline[input_t1](module, test_data) +@common.parametrize("test_data", tensor_const_tests) +def test_add_tensor_tosa_BI_const(test_data): + """Tests regular add with one scalar input, with one of inputs constant.""" + pipeline = TosaPipelineBI[input_t1](AddConst(), test_data, aten_op=AddConst.aten_op) pipeline.run() -fail_str = "MLETORCH-770: Numerical issues on Div Scalar." -passes_xfails = { - "Div__int_r1_ts": fail_str, - "Div__int_r4_ts": fail_str, - "Div__float_r1_ts": fail_str, - "Div__float_r4_ts": fail_str, +@common.parametrize("test_data", tensor_scalar_tests, xfails=xfails) +def test_add_scalar_tosa_BI(test_data): + """Tests a scalar add with one scalar input.""" + pipeline = TosaPipelineBI[input_t1](AddScalar(), test_data, aten_op=Add.aten_op) + pipeline.run() + + +# ADD ETHOS-U ------------------------------------------------------ +@pytest.mark.skip(reason="This is tested in test_add_scalar_tosa_BI") +def test_add_scalar_u55_BI(): + pass + + +@pytest.mark.skip(reason="This is tested in test_add_scalar_tosa_BI") +def test_add_scalar_u85_BI(): + pass + + +# SUB MI ------------------------------------------------------ +mi_sub_xfails = { + "int_r1_ts": "TypeError: All IO needs to have the same data type, got input 1: 8, input 2: 6 and output: 8", + "int_r4_ts": "TypeError: All IO needs to have the same data type, got input 1: 8, input 2: 6 and output: 8", + **xfails, } -@common.parametrize( - "tensor_scalar_tests", - tensor_scalar_tests, - passes_xfails, - strict=False, -) -def test_scalars_tosa_BI_passes(tensor_scalar_tests: list): - op, x, y = tensor_scalar_tests - _test_passes_tosa_BI_pipeline(op, (x, y)) +@common.parametrize("test_data", tensor_scalar_tests, xfails=mi_sub_xfails) +def test_sub_tensor_tosa_MI_scalar(test_data): + """Tests regular sub with one scalar input.""" + pipeline = TosaPipelineMI[input_t1](Sub(), test_data, aten_op=Sub.aten_op) + pipeline.run() + + +@common.parametrize("test_data", tensor_scalar_tests, xfails=mi_sub_xfails) +def test_sub_tensor_tosa_MI_inplace(test_data): + """Tests inplace sub with one scalar input.""" + pipeline = TosaPipelineMI[input_t1](SubInplace(), test_data, aten_op=[]) + pipeline.run() + + +@common.parametrize("test_data", tensor_scalar_tests, xfails=xfails) +def test_sub_scalar_tosa_MI(test_data): + """Tests a scalar sub with one scalar input.""" + pipeline = TosaPipelineMI[input_t1]( + SubScalar(), test_data, aten_op=SubScalar.aten_op + ) + pipeline.run() + + +# SUB BI ------------------------------------------------------ +@common.parametrize("test_data", tensor_scalar_tests) +def test_sub_tensor_tosa_BI_scalar(test_data): + """Tests regular sub with one scalar input.""" + pipeline = TosaPipelineBI[input_t1](Sub(), test_data, aten_op=[]) + pipeline.run() -# op(Scalar float, tensor) works if the scalar is constant. -@common.parametrize("tensor_const_tests", tensor_const_tests) -def test_scalars_tosa_MI(tensor_const_tests: list): - op, x = tensor_const_tests - _test_add_tosa_MI_pipeline(op, (x,)) +@common.parametrize("test_data", tensor_scalar_tests) +def test_sub_tensor_tosa_BI_inplace(test_data): + """Tests inplace sub with one scalar input.""" + pipeline = TosaPipelineBI[input_t1](SubInplace(), test_data, aten_op=[]) + pipeline.run() + + +@common.parametrize("test_data", tensor_scalar_tests, xfails=xfails) +def test_sub_scalar_tosa_BI(test_data): + """Tests a scalar sub with one scalar input.""" + pipeline = TosaPipelineBI[input_t1](SubScalar(), test_data, aten_op=Sub.aten_op) + pipeline.run() + + +# SUB ETHOS-U ------------------------------------------------------ +@pytest.mark.skip(reason="This is tested in test_sub_scalar_tosa_BI") +def test_sub_scalar_u55_BI(): + pass + + +@pytest.mark.skip(reason="This is tested in test_sub_scalar_tosa_BI") +def test_sub_scalar_u85_BI(): + pass + + +# MUL MI ------------------------------------------------------ +@common.parametrize("test_data", tensor_scalar_tests, xfails=xfails) +def test_mul_tensor_tosa_MI_scalar(test_data): + """Tests regular mul with one scalar input.""" + pipeline = TosaPipelineMI[input_t1](Mul(), test_data, aten_op=Mul.aten_op) + pipeline.run() + + +@common.parametrize("test_data", tensor_scalar_tests, xfails=xfails) +def test_mul_tensor_tosa_MI_inplace(test_data): + """Tests inplace mul with one scalar input.""" + pipeline = TosaPipelineMI[input_t1](MulInplace(), test_data, aten_op=[]) + pipeline.run() + + +@common.parametrize("test_data", tensor_scalar_tests, xfails=xfails) +def test_mul_scalar_tosa_MI(test_data): + """Tests a scalar mul with one scalar input.""" + pipeline = TosaPipelineMI[input_t1]( + MulScalar(), test_data, aten_op=MulScalar.aten_op + ) + pipeline.run() + + +# MUL BI ------------------------------------------------------ +@common.parametrize("test_data", tensor_scalar_tests) +def test_mul_tensor_tosa_BI_scalar(test_data): + """Tests regular mul with one scalar input.""" + pipeline = TosaPipelineBI[input_t1](Mul(), test_data, aten_op=[]) + pipeline.run() + + +@common.parametrize("test_data", tensor_scalar_tests) +def test_mul_tensor_tosa_BI_inplace(test_data): + """Tests inplace mul with one scalar input.""" + pipeline = TosaPipelineBI[input_t1](MulInplace(), test_data, aten_op=[]) + pipeline.run() + + +@common.parametrize("test_data", tensor_scalar_tests, xfails=xfails) +def test_mul_scalar_tosa_BI(test_data): + """Tests a scalar mul with one scalar input.""" + pipeline = TosaPipelineBI[input_t1](MulScalar(), test_data, aten_op=Mul.aten_op) + pipeline.run() + + +# MUL ETHOS-U ------------------------------------------------------ +@pytest.mark.skip(reason="This is tested in test_mul_scalar_tosa_BI") +def test_mul_scalar_u55_BI(): + pass + + +@pytest.mark.skip(reason="This is tested in test_mul_scalar_tosa_BI") +def test_mul_scalar_u85_BI(): + pass + + +# DIV MI ------------------------------------------------------ +@common.parametrize("test_data", tensor_scalar_tests, xfails=xfails) +def test_div_tensor_tosa_MI_scalar(test_data): + """Tests regular div with one scalar input.""" + pipeline = TosaPipelineMI[input_t1](Div(), test_data, aten_op=Div.aten_op) + pipeline.run() + + +@common.parametrize("test_data", tensor_scalar_tests, xfails=xfails) +def test_div_tensor_tosa_MI_inplace(test_data): + """Tests inplace div with one scalar input.""" + pipeline = TosaPipelineMI[input_t1](DivInplace(), test_data, aten_op=[]) + pipeline.run() + + +@common.parametrize("test_data", tensor_scalar_tests, xfails=xfails) +def test_div_scalar_tosa_MI(test_data): + """Tests a scalar div with one scalar input.""" + pipeline = TosaPipelineMI[input_t1]( + DivScalar(), test_data, aten_op=DivScalar.aten_op + ) + pipeline.run() -@common.parametrize("tensor_scalar_tests", tensor_scalar_tests) -def test_scalars_tosa_BI(tensor_scalar_tests: list): - op, x, y = tensor_scalar_tests - _test_add_tosa_BI_pipeline(op, (x, y)) +# DIV BI ------------------------------------------------------ +@common.parametrize("test_data", tensor_scalar_tests) +def test_div_tensor_tosa_BI_scalar(test_data): + """Tests regular div with one scalar input.""" + pipeline = TosaPipelineBI[input_t1](Div(), test_data, aten_op=[]) + pipeline.run() -# op(Scalar float, tensor) works if the scalar is constant. -@common.parametrize("tensor_const_tests", tensor_const_tests) -def test_scalars_tosa_BI_const(tensor_const_tests: list): - op, x = tensor_const_tests - _test_add_tosa_BI_pipeline(op, (x,)) +@common.parametrize("test_data", tensor_scalar_tests) +def test_div_tensor_tosa_BI_inplace(test_data): + """Tests inplace div with one scalar input.""" + pipeline = TosaPipelineBI[input_t1](DivInplace(), test_data, aten_op=[]) + pipeline.run() + + +@common.parametrize("test_data", tensor_scalar_tests, xfails=xfails) +def test_div_scalar_tosa_BI(test_data): + """Tests a scalar div with one scalar input.""" + pipeline = TosaPipelineBI[input_t1](DivScalar(), test_data, aten_op=[]) + pipeline.run() -def test_shift_sub_inplace_tosa_MI(): - _test_add_tosa_MI_pipeline( +# DIV ETHOS-U ------------------------------------------------------ +@pytest.mark.skip(reason="This is tested in test_div_scalar_tosa_BI") +def test_div_scalar_u55_BI(): + pass + + +@pytest.mark.skip(reason="This is tested in test_div_scalar_tosa_BI") +def test_div_scalar_u85_BI(): + pass + + +# SHIFT ETHOS-U ------------------------------------------------------ +def test_bitwise_right_shift_tensor_tosa_MI_inplace(): + pipeline = TosaPipelineMI[input_t1]( ShiftInplaceSub(), (torch.IntTensor(5),), + aten_op="torch.ops.aten.__rshift__.Scalar", ) + pipeline.run() -# Do not check for quant nodes in the graph for rshift. -def test_shift_sub_inplace_tosa_BI(): - _test_add_tosa_BI_pipeline( +def test_bitwise_right_shift_tensor_tosa_BI_inplace(): + pipeline = TosaPipelineBI[input_t1]( ShiftInplaceSub(), (torch.IntTensor(5),), - check_quant_nodes=False, + aten_op="torch.ops.aten.bitwise_right_shift.Tensor", ) + pipeline.pop_stage("check.quant_nodes") + pipeline.run()