diff --git a/backends/arm/test/models/test_mobilenet_v2_arm.py b/backends/arm/test/models/test_mobilenet_v2_arm.py index ac513530e04..a1f9bc0633d 100644 --- a/backends/arm/test/models/test_mobilenet_v2_arm.py +++ b/backends/arm/test/models/test_mobilenet_v2_arm.py @@ -32,6 +32,12 @@ input_t = Tuple[torch.Tensor] +quant_test_data = { + "per_channel_quantization=true": True, + "per_channel_quantization=false": False, +} + + def test_mv2_tosa_MI(): pipeline = TosaPipelineMI[input_t]( mv2, model_inputs, aten_op=[], exir_op=[], use_to_edge_transform_and_lower=True @@ -39,14 +45,15 @@ def test_mv2_tosa_MI(): pipeline.run() -def test_mv2_tosa_BI(): +@common.parametrize("per_channel_quantization", quant_test_data) +def test_mv2_tosa_BI(per_channel_quantization): pipeline = TosaPipelineBI[input_t]( mv2, model_inputs, aten_op=[], exir_op=[], use_to_edge_transform_and_lower=True, - per_channel_quantization=True, + per_channel_quantization=per_channel_quantization, atol=0.25, qtol=1, ) @@ -55,7 +62,8 @@ def test_mv2_tosa_BI(): @pytest.mark.slow @common.XfailIfNoCorstone300 -def test_mv2_u55_BI(): +@common.parametrize("per_channel_quantization", quant_test_data) +def test_mv2_u55_BI(per_channel_quantization): pipeline = EthosU55PipelineBI[input_t]( mv2, model_inputs, @@ -63,7 +71,7 @@ def test_mv2_u55_BI(): exir_ops=[], run_on_fvp=True, use_to_edge_transform_and_lower=True, - per_channel_quantization=True, + per_channel_quantization=per_channel_quantization, atol=0.25, qtol=1, ) @@ -72,7 +80,8 @@ def test_mv2_u55_BI(): @pytest.mark.slow @common.XfailIfNoCorstone320 -def test_mv2_u85_BI(): +@common.parametrize("per_channel_quantization", quant_test_data) +def test_mv2_u85_BI(per_channel_quantization): pipeline = EthosU85PipelineBI[input_t]( mv2, model_inputs, @@ -80,7 +89,7 @@ def test_mv2_u85_BI(): exir_ops=[], run_on_fvp=True, use_to_edge_transform_and_lower=True, - per_channel_quantization=True, + per_channel_quantization=per_channel_quantization, atol=0.25, qtol=1, ) diff --git a/backends/arm/test/ops/test_conv1d.py b/backends/arm/test/ops/test_conv1d.py index 768da4d5c89..cc8245ba126 100644 --- a/backends/arm/test/ops/test_conv1d.py +++ b/backends/arm/test/ops/test_conv1d.py @@ -249,7 +249,7 @@ def forward(self, x): batches=1, ) -test_modules = { +test_data_MI = { "2_3x2x40_nobias": lambda: conv1d_2_3x2x40_nobias, "3_1x3x256_st1": lambda: conv1d_3_1x3x256_st1, "3_1x3x12_st2_pd1": lambda: conv1d_3_1x3x12_st2_pd1, @@ -265,53 +265,65 @@ def forward(self, x): "two_conv1d": lambda: two_conv1d, } +test_data_BI = { + f"{k},per_channel_quant={q}": (lambda v=v, q=q: (v(), q)) + for (k, v) in test_data_MI.items() + for q in [True, False] +} + -@common.parametrize("test_module", test_modules) -def test_convolution_1d_tosa_MI(test_module): +@common.parametrize("test_data", test_data_MI) +def test_convolution_1d_tosa_MI(test_data): pipeline = TosaPipelineMI[input_t]( - test_module(), - test_module().get_inputs(), + test_data(), + test_data().get_inputs(), aten_op, exir_op, ) pipeline.run() -@common.parametrize("test_module", test_modules) -def test_convolution_1d_tosa_BI(test_module): +@common.parametrize("test_data", test_data_BI) +def test_convolution_1d_tosa_BI(test_data): + model, per_channel_quantization = test_data() pipeline = TosaPipelineBI[input_t]( - test_module(), - test_module().get_inputs(), + model, + model.get_inputs(), aten_op, exir_op, + per_channel_quantization=per_channel_quantization, + qtol=1, ) - pipeline.change_args("run_method_and_compare_outputs", qtol=1) pipeline.run() -@common.parametrize("test_module", test_modules) +@common.parametrize("test_data", test_data_BI) @common.XfailIfNoCorstone300 -def test_convolution_1d_u55_BI(test_module): +def test_convolution_1d_u55_BI(test_data): + model, per_channel_quantization = test_data() pipeline = EthosU55PipelineBI[input_t]( - test_module(), - test_module().get_inputs(), + model, + model.get_inputs(), aten_op, exir_op, run_on_fvp=True, + per_channel_quantization=per_channel_quantization, + qtol=1, ) - pipeline.change_args("run_method_and_compare_outputs", qtol=1) pipeline.run() -@common.parametrize("test_module", test_modules) +@common.parametrize("test_data", test_data_BI) @common.XfailIfNoCorstone320 -def test_convolution_1d_u85_BI(test_module): +def test_convolution_1d_u85_BI(test_data): + model, per_channel_quantization = test_data() pipeline = EthosU85PipelineBI[input_t]( - test_module(), - test_module().get_inputs(), + model, + model.get_inputs(), aten_op, exir_op, run_on_fvp=True, + per_channel_quantization=per_channel_quantization, + qtol=1, ) - pipeline.change_args("run_method_and_compare_outputs", qtol=1) pipeline.run() diff --git a/backends/arm/test/ops/test_conv2d.py b/backends/arm/test/ops/test_conv2d.py index 658978d0de8..9aa479466f8 100644 --- a/backends/arm/test/ops/test_conv2d.py +++ b/backends/arm/test/ops/test_conv2d.py @@ -357,7 +357,7 @@ def forward(self, x): # Shenanigan to get a nicer output when test fails. With unittest it looks like: # FAIL: test_convolution_2d_tosa_BI_2_3x3_1x3x12x12_st2_pd1 -test_modules = { +test_data_MI = { "2x2_3x2x40x40_nobias": lambda: conv2d_2x2_3x2x40x40_nobias, "3x3_1x3x256x256_st1": lambda: conv2d_3x3_1x3x256x256_st1, "3x3_1x3x12x12_st2_pd1": lambda: conv2d_3x3_1x3x12x12_st2_pd1, @@ -380,58 +380,79 @@ def forward(self, x): "groups_bias": lambda: conv2d_groups_bias, } +# Generate a new test set paired with per_channel_quant=True/False. +test_data_BI = { + f"{k},per_channel_quant={q}": (lambda v=v, q=q: (v(), q)) + for (k, v) in test_data_MI.items() + for q in [True, False] + # TODO: Invalid TOSA graph (MLETORCH-1144) + if (k not in ["groups", "groups_bias"]) and (q is True) +} + fvp_xfails = { - "2x2_3x2x40x40_nobias": "MLETORCH-520: Numerical issues on FVP.", - "5x5_3x2x128x128_st1": "MLETORCH-520: Numerical issues on FVP.", + f"{k},per_channel_quant={q}": reason + for k, reason in { + "2x2_3x2x40x40_nobias": "MLETORCH-520: Numerical issues on FVP.", + "5x5_3x2x128x128_st1": "MLETORCH-520: Numerical issues on FVP.", + }.items() + for q in [True, False] } + input_t = Tuple[torch.Tensor] -@common.parametrize("test_module", test_modules) -def test_convolution_2d_tosa_MI(test_module): +@common.parametrize("test_data", test_data_MI) +def test_convolution_2d_tosa_MI(test_data): + model = test_data() pipeline = TosaPipelineMI[input_t]( - test_module(), - test_module().get_inputs(), + model, + model.get_inputs(), aten_op, exir_op, ) pipeline.run() -@common.parametrize("test_module", test_modules) -def test_convolution_2d_tosa_BI(test_module): +@common.parametrize("test_data", test_data_BI) +def test_convolution_2d_tosa_BI(test_data): + model, per_channel_quantization = test_data() pipeline = TosaPipelineBI[input_t]( - test_module(), - test_module().get_inputs(), + model, + model.get_inputs(), aten_op, exir_op, + per_channel_quantization=per_channel_quantization, + qtol=1, ) - pipeline.change_args("run_method_and_compare_outputs", qtol=1) pipeline.run() -@common.parametrize("test_module", test_modules, fvp_xfails) +@common.parametrize("test_data", test_data_BI, fvp_xfails) @common.XfailIfNoCorstone300 -def test_convolution_2d_u55_BI(test_module): +def test_convolution_2d_u55_BI(test_data): + model, per_channel_quantization = test_data() pipeline = EthosU55PipelineBI[input_t]( - test_module(), - test_module().get_inputs(), + model, + model.get_inputs(), aten_op, exir_op, run_on_fvp=True, + per_channel_quantization=per_channel_quantization, ) pipeline.run() -@common.parametrize("test_module", test_modules, fvp_xfails) +@common.parametrize("test_data", test_data_BI, fvp_xfails) @common.XfailIfNoCorstone320 -def test_convolution_2d_u85_BI(test_module): +def test_convolution_u85_BI(test_data): + model, per_channel_quantization = test_data() pipeline = EthosU85PipelineBI[input_t]( - test_module(), - test_module().get_inputs(), + model, + model.get_inputs(), aten_op, exir_op, run_on_fvp=True, + per_channel_quantization=per_channel_quantization, ) pipeline.run() diff --git a/backends/arm/test/ops/test_conv3d.py b/backends/arm/test/ops/test_conv3d.py index c7bb7c55887..1a8ea5c3dd5 100644 --- a/backends/arm/test/ops/test_conv3d.py +++ b/backends/arm/test/ops/test_conv3d.py @@ -304,7 +304,7 @@ def forward(self, x): batches=1, ) -test_modules = { +test_data_MI = { "2x2_3x2x40x40_nobias": lambda: conv3d_2x2_3x2x40x40_nobias, "3x3_1x3x256x256_st1": lambda: conv3d_3x3_1x3x256x256_st1, "3x3_1x3x12x12_st2_pd1": lambda: conv3d_3x3_1x3x12x12_st2_pd1, @@ -323,50 +323,66 @@ def forward(self, x): "3x3_1x3x224x224_st2_pd1": lambda: conv3d_3x3_1x3x224x224_st2_pd1, } +# Generate a new test set paired with per_channel_quant=True/False. +test_data_BI = { + f"{k},per_channel_quant={q}": (lambda v=v, q=q: (v(), q)) + for (k, v) in test_data_MI.items() + for q in [True, False] +} + input_t = Tuple[torch.Tensor] -@common.parametrize("test_module", test_modules) +@common.parametrize("test_data", test_data_MI) @pytest.mark.skip # Not implemented, skip until it is. -def test_convolution_tosa_MI_3d(test_module): +def test_convolution_3d_tosa_MI(test_data): pipeline = TosaPipelineMI[input_t]( - test_module(), test_module().get_inputs(), aten_op, exir_op + test_data(), test_data().get_inputs(), aten_op, exir_op ) pipeline.run() -@common.parametrize("test_module", test_modules) +@common.parametrize("test_data", test_data_BI) @pytest.mark.skip # Not implemented, skip until it is. -def test_convolution_tosa_BI_3d(test_module): +def test_convolution_3d_tosa_BI(test_data): + model, per_channel_quantization = test_data() pipeline = TosaPipelineBI[input_t]( - test_module(), test_module().get_inputs(), aten_op, exir_op + model, + model.get_inputs(), + aten_op, + exir_op, + per_channel_quantization=per_channel_quantization, + qtol=1, ) - pipeline.change_args("run_method_and_compare_outputs", qtol=1) pipeline.run() -@common.parametrize("test_module", test_modules) +@common.parametrize("test_data", test_data_BI) @pytest.mark.skip # Not implemented, skip until it is. -def test_convolution_u55_BI_3d(test_module): +def test_convolution_3d_u55_BI(test_data): + model, per_channel_quantization = test_data() pipeline = EthosU55PipelineBI[input_t]( - test_module(), - test_module().get_inputs(), + model, + model.get_inputs(), aten_op, exir_op, run_on_fvp=True, + per_channel_quantization=per_channel_quantization, ) pipeline.run() -@common.parametrize("test_module", test_modules) +@common.parametrize("test_data", test_data_BI) @pytest.mark.skip # Not implemented, skip until it is. -def test_convolution_u85_BI_3d(test_module): +def test_convolution_3d_u85_BI(test_data): + model, per_channel_quantization = test_data() pipeline = EthosU85PipelineBI[input_t]( - test_module(), - test_module().get_inputs(), + model, + model.get_inputs(), aten_op, exir_op, run_on_fvp=True, + per_channel_quantization=per_channel_quantization, ) pipeline.run() diff --git a/backends/arm/test/ops/test_conv_combos.py b/backends/arm/test/ops/test_conv_combos.py index bddc30f04ab..d3218258087 100644 --- a/backends/arm/test/ops/test_conv_combos.py +++ b/backends/arm/test/ops/test_conv_combos.py @@ -36,6 +36,11 @@ class ComboBlockBottleneckResidual(torch.nn.Module): "executorch_exir_dialects_edge__ops_aten_add_Tensor", ] + test_data_BI = { + "per_channel_quant=True": True, + "per_channel_quant=False": False, + } + def __init__(self): super().__init__() # (t, c, n, s) = (6, 96, 1, 1) @@ -114,6 +119,18 @@ class ComboConvBatchnormRelu6(torch.nn.Module): "executorch_exir_dialects_edge__ops_aten_hardtanh_default", ] + test_data_MI = { + "affine=True": True, + "affine=False": False, + } + + test_data_BI = { + "affine=True,per_channel_quant=True": (True, True), + "affine=True,per_channel_quant=False": (True, False), + "affine=False,per_channel_quant=True": (False, True), + "affine=False,per_channel_quant=False": (False, False), + } + def __init__(self, affine: bool): super().__init__() self.conv2d = torch.nn.Conv2d( @@ -142,7 +159,7 @@ class ComboConvRelu6(torch.nn.Module): "executorch_exir_dialects_edge__ops_aten_hardtanh_default", ] - test_data = { + test_data_MI = { "combo_conv_relu_2_x_4d": lambda: (2 * torch.randn(1, 3, 256, 256),), "combo_conv_relu_0_5_x_4d": lambda: (0.5 * torch.randn(1, 3, 256, 256),), "combo_conv_relu_4d": lambda: (torch.randn(1, 3, 256, 256),), @@ -150,6 +167,14 @@ class ComboConvRelu6(torch.nn.Module): "combo_conv_relu_neg_2_x_4d": lambda: (-2 * torch.randn(1, 3, 256, 256),), } + # Generate a new test set paired with per_channel_quant=True/False. + test_data_BI = { + # test_name: (input, per_channel_quant) + f"{k},per_channel_quant={q}": (lambda v=v, q=q: (v(), q)) + for (k, v) in test_data_MI.items() + for q in [True, False] + } + def __init__(self): super().__init__() self.conv2d = torch.nn.Conv2d( @@ -169,13 +194,21 @@ class ComboConvAvgPool2d(torch.nn.Module): "executorch_exir_dialects_edge__ops_aten_avg_pool2d_default", ] - test_data = { + test_data_MI = { "combo_conv_avgpool_20_x_4d": lambda: (20 * torch.randn(1, 3, 64, 32),), "combo_conv_avgpool_4d": lambda: (torch.randn(1, 3, 100, 200),), "combo_conv_avgpool_5_x_4d_randn": lambda: (5 * torch.randn(1, 3, 256, 256),), "combo_conv_avgpool_2_x_4d": lambda: (torch.rand(1, 3, 512, 128),), } + # Generate a new test set paired with per_channel_quant=True/False. + test_data_BI = { + # test_name: (input, per_channel_quant) + f"{k},per_channel_quant={q}": (lambda v=v, q=q: (v(), q)) + for (k, v) in test_data_MI.items() + for q in [True, False] + } + def __init__(self): super().__init__() self.conv2d = torch.nn.Conv2d( @@ -196,7 +229,6 @@ def forward(self, x): def test_convolution_2d_tosa_MI_meandim(): model = ComboConv2dMeandim() - pipeline = TosaPipelineMI[input_t1]( model, model.get_inputs(), @@ -246,11 +278,11 @@ def test_convolution_2d_u85_BI_meandim(): ############################## ## Conv + batch norm + relu ## ############################## -affine_params = {"affine": True, "_no_affine": False} -@common.parametrize("affine", affine_params) -def test_convolution_2d_tosa_MI_batchnorm_relu6(affine): +@common.parametrize("test_data", ComboConvBatchnormRelu6.test_data_MI) +def test_convolution_2d_tosa_MI_batchnorm_relu6(test_data): + affine = test_data model = ComboConvBatchnormRelu6(affine) pipeline = TosaPipelineMI[input_t1]( model, @@ -262,21 +294,25 @@ def test_convolution_2d_tosa_MI_batchnorm_relu6(affine): @pytest.mark.flaky(reruns=5) # TODO: Investigate flakyness (MLTORCH-307) -@common.parametrize("affine", affine_params) -def test_convolution_2d_tosa_BI_batchnorm_relu6(affine): +@common.parametrize("test_data", ComboConvBatchnormRelu6.test_data_BI) +def test_convolution_2d_tosa_BI_batchnorm_relu6(test_data): + affine, per_channel_quantization = test_data model = ComboConvBatchnormRelu6(affine) pipeline = TosaPipelineBI[input_t1]( model, model.get_inputs(), aten_op=[], exir_op=ComboConvBatchnormRelu6.edge_op_list, + per_channel_quantization=per_channel_quantization, + qtol=1, ) pipeline.run() -@common.parametrize("affine", affine_params) +@common.parametrize("test_data", ComboConvBatchnormRelu6.test_data_BI) @common.XfailIfNoCorstone300 -def test_convolution_2d_u55_BI_batchnorm_relu6(affine): +def test_convolution_2d_u55_BI_batchnorm_relu6(test_data): + affine, per_channel_quantization = test_data model = ComboConvBatchnormRelu6(affine) pipeline = EthosU55PipelineBI[input_t1]( model, @@ -284,13 +320,15 @@ def test_convolution_2d_u55_BI_batchnorm_relu6(affine): aten_ops=[], exir_ops=[], run_on_fvp=True, + per_channel_quantization=per_channel_quantization, ) pipeline.run() -@common.parametrize("affine", affine_params) +@common.parametrize("test_data", ComboConvBatchnormRelu6.test_data_BI) @common.XfailIfNoCorstone320 -def test_convolution_2d_u85_BI_batchnorm_relu6(affine): +def test_convolution_2d_u85_BI_batchnorm_relu6(test_data): + affine, per_channel_quantization = test_data model = ComboConvBatchnormRelu6(affine) pipeline = EthosU85PipelineBI[input_t1]( model, @@ -298,6 +336,7 @@ def test_convolution_2d_u85_BI_batchnorm_relu6(affine): aten_ops=[], exir_ops=[], run_on_fvp=True, + per_channel_quantization=per_channel_quantization, ) pipeline.run() @@ -307,8 +346,8 @@ def test_convolution_2d_u85_BI_batchnorm_relu6(affine): ################## -@common.parametrize("test_data", ComboConvRelu6.test_data) -def test_convolution_2d_tosa_MI_relu6(test_data: torch.Tensor): +@common.parametrize("test_data", ComboConvRelu6.test_data_MI) +def test_convolution_2d_tosa_MI_relu6(test_data): model = ComboConvRelu6() pipeline = TosaPipelineMI[input_t1]( model, @@ -320,42 +359,48 @@ def test_convolution_2d_tosa_MI_relu6(test_data: torch.Tensor): @pytest.mark.flaky(reruns=5) # TODO: Investigate flakyness (MLTORCH-307) -@common.parametrize("test_data", ComboConvRelu6.test_data) -def test_convolution_2d_tosa_BI_relu6(test_data: torch.Tensor): +@common.parametrize("test_data", ComboConvRelu6.test_data_BI) +def test_convolution_2d_tosa_BI_relu6(test_data): + input, per_channel_quantization = test_data() model = ComboConvRelu6() pipeline = TosaPipelineBI[input_t1]( model, - test_data(), + input, aten_op=[], exir_op=ComboConvRelu6.edge_op_list, + per_channel_quantization=per_channel_quantization, ) pipeline.run() -@common.parametrize("test_data", ComboConvRelu6.test_data) +@common.parametrize("test_data", ComboConvRelu6.test_data_BI) @common.XfailIfNoCorstone300 -def test_convolution_2d_u55_BI_relu6(test_data: torch.Tensor): +def test_convolution_2d_u55_BI_relu6(test_data): + input, per_channel_quantization = test_data() model = ComboConvRelu6() pipeline = EthosU55PipelineBI[input_t1]( model, - test_data(), + input, aten_ops=[], exir_ops=ComboConvRelu6.edge_op_list, run_on_fvp=True, + per_channel_quantization=per_channel_quantization, ) pipeline.run() -@common.parametrize("test_data", ComboConvRelu6.test_data) +@common.parametrize("test_data", ComboConvRelu6.test_data_BI) @common.XfailIfNoCorstone320 -def test_convolution_2d_u85_BI_relu6(test_data: torch.Tensor): +def test_convolution_2d_u85_BI_relu6(test_data): + input, per_channel_quantization = test_data() model = ComboConvRelu6() pipeline = EthosU85PipelineBI[input_t1]( model, - test_data(), + input, aten_ops=[], exir_ops=ComboConvRelu6.edge_op_list, run_on_fvp=True, + per_channel_quantization=per_channel_quantization, ) pipeline.run() @@ -374,21 +419,26 @@ def test_convolution_2d_tosa_MI_block_bottleneck(): pipeline.run() +@common.parametrize("test_data", ComboBlockBottleneckResidual.test_data_BI) @pytest.mark.flaky(reruns=5) # TODO: Investigate flakyness (MLTORCH-307) -def test_convolution_2d_tosa_BI_block_bottleneck(): +def test_convolution_2d_tosa_BI_block_bottleneck(test_data): + per_channel_quantization = test_data model = ComboBlockBottleneckResidual() pipeline = TosaPipelineBI[input_t1]( model, model.get_inputs(), aten_op=[], exir_op=ComboBlockBottleneckResidual.edge_op_list, + per_channel_quantization=per_channel_quantization, ) pipeline.change_args("run_method_and_compare_outputs", model.get_inputs(), qtol=1) pipeline.run() +@common.parametrize("test_data", ComboBlockBottleneckResidual.test_data_BI) @common.XfailIfNoCorstone300 -def test_convolution_2d_u55_BI_block_bottleneck(): +def test_convolution_2d_u55_BI_block_bottleneck(test_data): + per_channel_quantization = test_data model = ComboBlockBottleneckResidual() pipeline = EthosU55PipelineBI[input_t1]( model, @@ -396,12 +446,15 @@ def test_convolution_2d_u55_BI_block_bottleneck(): aten_ops=[], exir_ops=[], run_on_fvp=True, + per_channel_quantization=per_channel_quantization, ) pipeline.run() +@common.parametrize("test_data", ComboBlockBottleneckResidual.test_data_BI) @common.XfailIfNoCorstone320 -def test_convolution_2d_u85_BI_block_bottleneck(): +def test_convolution_2d_u85_BI_block_bottleneck(test_data): + per_channel_quantization = test_data model = ComboBlockBottleneckResidual() pipeline = EthosU85PipelineBI[input_t1]( model, @@ -409,6 +462,7 @@ def test_convolution_2d_u85_BI_block_bottleneck(): aten_ops=[], exir_ops=[], run_on_fvp=True, + per_channel_quantization=per_channel_quantization, ) pipeline.run() @@ -418,8 +472,8 @@ def test_convolution_2d_u85_BI_block_bottleneck(): ###################### -@common.parametrize("test_data", ComboConvAvgPool2d.test_data) -def test_convolution_2d_tosa_MI_avgpool2d(test_data: torch.Tensor): +@common.parametrize("test_data", ComboConvAvgPool2d.test_data_MI) +def test_convolution_2d_tosa_MI_avgpool2d(test_data): model = ComboConvAvgPool2d() pipeline = TosaPipelineMI[input_t1]( model, @@ -431,41 +485,47 @@ def test_convolution_2d_tosa_MI_avgpool2d(test_data: torch.Tensor): @pytest.mark.flaky(reruns=5) # TODO: Investigate flakyness (MLTORCH-307) -@common.parametrize("test_data", ComboConvAvgPool2d.test_data) -def test_convolution_2d_tosa_BI_avgpool2d(test_data: torch.Tensor): +@common.parametrize("test_data", ComboConvAvgPool2d.test_data_BI) +def test_convolution_2d_tosa_BI_avgpool2d(test_data): + input, per_channel_quantization = test_data() model = ComboConvAvgPool2d() pipeline = TosaPipelineBI[input_t1]( model, - test_data(), + input, aten_op=[], exir_op=ComboConvAvgPool2d.edge_op_list, + per_channel_quantization=per_channel_quantization, ) pipeline.run() -@common.parametrize("test_data", ComboConvAvgPool2d.test_data) +@common.parametrize("test_data", ComboConvAvgPool2d.test_data_BI) @common.XfailIfNoCorstone300 -def test_convolution_2d_u55_BI_avgpool2d(test_data: torch.Tensor): +def test_convolution_2d_u55_BI_avgpool2d(test_data): + input, per_channel_quantization = test_data() model = ComboConvAvgPool2d() pipeline = EthosU55PipelineBI[input_t1]( model, - test_data(), + input, aten_ops=[], exir_ops=[], run_on_fvp=True, + per_channel_quantization=per_channel_quantization, ) pipeline.run() -@common.parametrize("test_data", ComboConvAvgPool2d.test_data) +@common.parametrize("test_data", ComboConvAvgPool2d.test_data_BI) @common.XfailIfNoCorstone320 -def test_convolution_2d_u85_BI_avgpool2d(test_data: torch.Tensor): +def test_convolution_2d_u85_BI_avgpool2d(test_data): + input, per_channel_quantization = test_data() model = ComboConvAvgPool2d() pipeline = EthosU85PipelineBI[input_t1]( model, - test_data(), + input, aten_ops=[], exir_ops=[], run_on_fvp=True, + per_channel_quantization=per_channel_quantization, ) pipeline.run() diff --git a/backends/arm/test/ops/test_depthwise_conv.py b/backends/arm/test/ops/test_depthwise_conv.py index 1213a04426b..4a6150317b5 100644 --- a/backends/arm/test/ops/test_depthwise_conv.py +++ b/backends/arm/test/ops/test_depthwise_conv.py @@ -154,7 +154,7 @@ ) # Shenanigan to get a nicer output when test fails. -testsuite_conv2d = { +test_data_conv2d_MI = { "2x2_1x6x4x4_gp6_st1": lambda: dw_conv2d_2x2_1x6x4x4_gp6_st1, "3x3_1x3x256x256_gp3_st1": lambda: dw_conv2d_3x3_1x3x256x256_gp3_st1, "3x3_1x4x256x256_gp4_nobias": lambda: dw_conv2d_3x3_1x4x256x256_gp4_nobias, @@ -163,26 +163,45 @@ "two_dw_conv2d": lambda: two_dw_conv2d, } -testsuite_conv2d_u85 = { - "2x2_1x6x4x4_gp6_st1": lambda: dw_conv2d_2x2_1x6x4x4_gp6_st1, - "3x3_1x3x256x256_gp3_st1": lambda: dw_conv2d_3x3_1x3x256x256_gp3_st1, - "3x3_1x4x256x256_gp4_st1": lambda: dw_conv2d_3x3_1x4x256x256_gp4_st1, - "3x3_1x4x256x256_gp4_nobias": lambda: dw_conv2d_3x3_1x4x256x256_gp4_nobias, +# Generate a new test set paired with per_channel_quant=True/False. +test_data_conv2d_BI = { + f"{k},per_channel_quant={q}": (lambda v=v, q=q: (v(), q)) + for (k, v) in test_data_conv2d_MI.items() + for q in [True, False] } -testsuite_conv1d = { +# Generate a new test set paired with per_channel_quant=True/False. +test_data_conv2d_u85 = { + f"{k},per_channel_quant={q}": (lambda v=v, q=q: (v(), q)) + for (k, v) in { + "2x2_1x6x4x4_gp6_st1": lambda: dw_conv2d_2x2_1x6x4x4_gp6_st1, + "3x3_1x3x256x256_gp3_st1": lambda: dw_conv2d_3x3_1x3x256x256_gp3_st1, + "3x3_1x4x256x256_gp4_st1": lambda: dw_conv2d_3x3_1x4x256x256_gp4_st1, + "3x3_1x4x256x256_gp4_nobias": lambda: dw_conv2d_3x3_1x4x256x256_gp4_nobias, + }.items() + for q in [True, False] +} + +test_data_conv1d_MI = { "2_1x6x4_gp6_st1": lambda: dw_conv1d_2_1x6x4_gp6_st1, "two_dw_conv1d": lambda: two_dw_conv1d, "3_1x3x256_gp3_st1": lambda: dw_conv1d_3_1x3x256_gp3_st1, "3_1x3x14_gp3_st1": lambda: dw_conv1d_3_1x3x14_gp3_st1, } +# Generate a new test set paired with per_channel_quant=True/False. +test_data_conv1d_BI = { + f"{k},per_channel_quant={q}": (lambda v=v, q=q: (v(), q)) + for (k, v) in test_data_conv1d_MI.items() + for q in [True, False] +} + -@common.parametrize("test_module", testsuite_conv1d | testsuite_conv2d) -def test_convolution_2d_tosa_MI_depth_wise(test_module: torch.nn.Module): +@common.parametrize("test_data", test_data_conv1d_MI | test_data_conv2d_MI) +def test_depthwise_convolution_2d_tosa_MI(test_data: torch.nn.Module): pipeline = TosaPipelineMI[input_t]( - test_module(), - test_module().get_inputs(), + test_data(), + test_data().get_inputs(), aten_op=[], exir_op=exir_op, ) @@ -190,70 +209,84 @@ def test_convolution_2d_tosa_MI_depth_wise(test_module: torch.nn.Module): @pytest.mark.flaky(reruns=5) # TODO: Investigate flakyness (MLTORCH-307) -@common.parametrize("test_module", testsuite_conv1d | testsuite_conv2d) -def test_convolution_2d_tosa_BI_depth_wise(test_module: torch.nn.Module): +@common.parametrize("test_data", test_data_conv1d_BI | test_data_conv2d_BI) +def test_depthwise_convolution_2d_tosa_BI(test_data): + model, per_channel_quantization = test_data() pipeline = TosaPipelineBI[input_t]( - test_module(), - test_module().get_inputs(), + model, + model.get_inputs(), aten_op=[], exir_op=exir_op, + per_channel_quantization=per_channel_quantization, ) pipeline.run() x_fails = { - "3x3_2x8x198x198_gp8_st3": "MLETORCH-517: Operators fail with batches > 1", - "two_dw_conv2d": "MLETORCH-517: Operators fail with batches > 1", + f"{k},per_channel_quant={q}": reason + for k, reason in { + "3x3_2x8x198x198_gp8_st3": "MLETORCH-517: Operators fail with batches > 1", + "two_dw_conv2d": "MLETORCH-517: Operators fail with batches > 1", + }.items() + for q in [True, False] } @common.XfailIfNoCorstone300 # TODO: MLETORCH-516 -@common.parametrize("test_module", testsuite_conv2d, x_fails) -def test_convolution_2d_u55_BI_depth_wise(test_module: torch.nn.Module): +@common.parametrize("test_data", test_data_conv2d_BI, x_fails) +def test_depthwise_convolution_2d_u55_BI(test_data): + model, per_channel_quantization = test_data() pipeline = EthosU55PipelineBI[input_t]( - test_module(), - test_module().get_inputs(), + model, + model.get_inputs(), aten_ops=[], exir_ops=exir_op, run_on_fvp=True, + per_channel_quantization=per_channel_quantization, ) pipeline.run() @common.XfailIfNoCorstone300 # TODO: MLETORCH-516 -@common.parametrize("test_module", testsuite_conv1d) -def test_convolution_1d_u55_BI_depth_wise(test_module: torch.nn.Module): +@common.parametrize("test_data", test_data_conv1d_BI) +def test_depthwise_convolution_1d_u55_BI(test_data): + model, per_channel_quantization = test_data() pipeline = EthosU55PipelineBI[input_t]( - test_module(), - test_module().get_inputs(), + model, + model.get_inputs(), aten_ops=[], exir_ops=exir_op, run_on_fvp=True, + per_channel_quantization=per_channel_quantization, ) pipeline.run() @common.XfailIfNoCorstone320 # TODO: MLETORCH-516 -@common.parametrize("test_module", testsuite_conv2d, x_fails) -def test_convolution_2d_u85_BI_depth_wise(test_module: torch.nn.Module): +@common.parametrize("test_data", test_data_conv2d_BI, x_fails) +def test_depthwise_convolution_2d_u85_BI(test_data): + model, per_channel_quantization = test_data() pipeline = EthosU85PipelineBI[input_t]( - test_module(), - test_module().get_inputs(), + model, + model.get_inputs(), aten_ops=[], exir_ops=exir_op, run_on_fvp=True, + per_channel_quantization=per_channel_quantization, ) pipeline.run() @common.XfailIfNoCorstone320 # TODO: MLETORCH-516 -@common.parametrize("test_module", testsuite_conv1d, x_fails) -def test_convolution_1d_u85_BI_depth_wise(test_module: torch.nn.Module): +@common.parametrize("test_data", test_data_conv1d_BI, x_fails) +def test_depthwise_convolution_1d_u85_BI(test_data): + model, per_channel_quantization = test_data() pipeline = EthosU85PipelineBI[input_t]( - test_module(), - test_module().get_inputs(), + model, + model.get_inputs(), aten_ops=[], exir_ops=exir_op, run_on_fvp=True, + per_channel_quantization=per_channel_quantization, ) pipeline.run() diff --git a/backends/arm/test/ops/test_linear.py b/backends/arm/test/ops/test_linear.py index 56d33097999..14f65a07192 100644 --- a/backends/arm/test/ops/test_linear.py +++ b/backends/arm/test/ops/test_linear.py @@ -24,8 +24,8 @@ input_t1 = Tuple[torch.Tensor] -test_data_suite_rank1 = { - # (test_name, test_data, out_features, has_bias) +test_data_rank1_MI = { + # test_name: (test_data, out_features, has_bias) "model_linear_rank1_zeros": lambda: ( torch.zeros(10), 15, @@ -58,8 +58,8 @@ ), } -test_data_suite_rank4 = { - # (test_name, test_data, out_features, has_bias) +test_data_rank4_MI = { + # test_name: (test_data, out_features, has_bias) "model_linear_rank4_zeros": lambda: ( torch.zeros(5, 10, 25, 20), 30, @@ -92,6 +92,20 @@ ), } +# Generate a new test set paired with per_channel_quant=True/False. +test_data_rank1_BI = { + f"{k},per_channel_quant={q}": (lambda v=v, q=q: (*v(), q)) + for (k, v) in test_data_rank1_MI.items() + for q in [True, False] +} + +# Generate a new test set paired with per_channel_quant=True/False. +test_data_rank4_BI = { + f"{k},per_channel_quant={q}": (lambda v=v, q=q: (*v(), q)) + for (k, v) in test_data_rank4_MI.items() + for q in [True, False] +} + class Linear(torch.nn.Module): def __init__( @@ -111,7 +125,7 @@ def forward(self, x): return self.fc(x) -@common.parametrize("test_data", test_data_suite_rank1 | test_data_suite_rank4) +@common.parametrize("test_data", test_data_rank1_MI | test_data_rank4_MI) def test_linear_tosa_MI(test_data: torch.Tensor): test_data, out_features, has_bias = test_data() in_features = test_data.shape[-1] @@ -129,9 +143,9 @@ def test_linear_tosa_MI(test_data: torch.Tensor): @pytest.mark.flaky(reruns=5) # TODO: Investigate flakyness. -@common.parametrize("test_data", test_data_suite_rank1 | test_data_suite_rank4) +@common.parametrize("test_data", test_data_rank1_BI | test_data_rank4_BI) def test_linear_tosa_BI(test_data: torch.Tensor): - test_data, out_features, has_bias = test_data() + test_data, out_features, has_bias, per_channel_quantization = test_data() in_features = test_data.shape[-1] pipeline = TosaPipelineBI[input_t1]( Linear( @@ -142,15 +156,16 @@ def test_linear_tosa_BI(test_data: torch.Tensor): (test_data,), aten_op, exir_op=[], + per_channel_quantization=per_channel_quantization, use_to_edge_transform_and_lower=True, ) pipeline.run() -@common.parametrize("test_data", test_data_suite_rank1) +@common.parametrize("test_data", test_data_rank1_BI) @common.XfailIfNoCorstone300 def test_linear_u55_BI(test_data: torch.Tensor): - test_data, out_features, has_bias = test_data() + test_data, out_features, has_bias, per_channel_quantization = test_data() in_features = test_data.shape[-1] EthosU55PipelineBI[input_t1]( Linear( @@ -162,28 +177,33 @@ def test_linear_u55_BI(test_data: torch.Tensor): aten_op, exir_ops=[], run_on_fvp=True, + per_channel_quantization=per_channel_quantization, use_to_edge_transform_and_lower=True, ).run() x_fail = { - "model_linear_rank4_zeros": "AssertionError: Output 0 does not match reference output.", - "model_linear_rank4_ones": "AssertionError: Output 0 does not match reference output.", - "model_linear_rank4_negative_ones": "AssertionError: Output 0 does not match reference output.", - "model_linear_rank4_rand": "AssertionError: Output 0 does not match reference output.", - "model_linear_rank4_negative_large_rand": "AssertionError: Output 0 does not match reference output.", - "model_linear_rank4_large_randn": "AssertionError: Output 0 does not match reference output.", + f"{k},per_channel_quant={q}": reason + for k, reason in { + "model_linear_rank4_zeros": "AssertionError: Output 0 does not match reference output.", + "model_linear_rank4_ones": "AssertionError: Output 0 does not match reference output.", + "model_linear_rank4_negative_ones": "AssertionError: Output 0 does not match reference output.", + "model_linear_rank4_rand": "AssertionError: Output 0 does not match reference output.", + "model_linear_rank4_negative_large_rand": "AssertionError: Output 0 does not match reference output.", + "model_linear_rank4_large_randn": "AssertionError: Output 0 does not match reference output.", + }.items() + for q in [True, False] } @common.parametrize( "test_data", - test_data_suite_rank1 | test_data_suite_rank4, + test_data_rank1_BI | test_data_rank4_BI, x_fail, ) @common.XfailIfNoCorstone320 def test_linear_u85_BI(test_data: torch.Tensor): - test_data, out_features, has_bias = test_data() + test_data, out_features, has_bias, per_channel_quantization = test_data() in_features = test_data.shape[-1] EthosU85PipelineBI[input_t1]( Linear( @@ -195,5 +215,6 @@ def test_linear_u85_BI(test_data: torch.Tensor): aten_op, exir_ops=[], run_on_fvp=True, + per_channel_quantization=per_channel_quantization, use_to_edge_transform_and_lower=True, ).run()