Skip to content

Commit 735d97b

Browse files
SaoirseARMSaoirseARM
committed
Arm backend: Refactor models to allow for TOSA 1.0
Signed-off-by: Saoirse Stewart <[email protected]> Change-Id: Ib8581f3aa9f879572b80dea8dd9e0c789b00c517
1 parent b058afb commit 735d97b

File tree

5 files changed

+355
-412
lines changed

5 files changed

+355
-412
lines changed

‎backends/arm/test/models/test_conformer.py‎

Lines changed: 99 additions & 98 deletions
Original file line numberDiff line numberDiff line change
@@ -3,32 +3,38 @@
33
# This source code is licensed under the BSD-style license found in the
44
# LICENSE file in the root directory of this source tree.
55

6-
import unittest
6+
from typing import Tuple
7+
8+
import pytest
79

810
import torch
9-
from executorch.backends.arm.test import common, conftest
1011

11-
from executorch.backends.arm.test.tester.arm_tester import ArmTester
12+
from executorch.backends.arm.test import common
13+
from executorch.backends.arm.test.tester.test_pipeline import (
14+
EthosU55PipelineBI,
15+
EthosU85PipelineBI,
16+
TosaPipelineBI,
17+
TosaPipelineMI,
18+
)
1219

1320
from torchaudio.models import Conformer
1421

22+
input_t = Tuple[torch.Tensor, torch.IntTensor] # Input x, y
23+
1524

1625
def get_test_inputs(dim, lengths, num_examples):
1726
return (torch.rand(num_examples, int(lengths.max()), dim), lengths)
1827

1928

20-
class TestConformer(unittest.TestCase):
29+
class TestConformer:
2130
"""Tests Torchaudio Conformer"""
2231

2332
# Adjust nbr below as we increase op support. Note: most of the delegates
2433
# calls are directly consecutive to each other in the .pte. The reason
2534
# for that is some assert ops are removed by passes in the
2635
# .to_executorch step, i.e. after Arm partitioner.
27-
ops_after_partitioner = {
28-
"executorch_exir_dialects_edge__ops_aten_max_default": 1,
29-
"torch.ops.aten._assert_scalar.default": 7,
30-
"torch.ops.aten._local_scalar_dense.default": 1,
31-
}
36+
aten_ops = ["torch.ops.aten._assert_scalar.default"]
37+
exir_ops = ["executorch_exir_dialects_edge__ops_aten_max_default"]
3238

3339
dim = 16
3440
num_examples = 10
@@ -43,92 +49,87 @@ class TestConformer(unittest.TestCase):
4349
)
4450
conformer = conformer.eval()
4551

46-
def test_conformer_tosa_MI(self):
47-
(
48-
ArmTester(
49-
self.conformer,
50-
example_inputs=self.model_example_inputs,
51-
compile_spec=common.get_tosa_compile_spec(tosa_spec="TOSA-0.80+MI"),
52-
)
53-
.export()
54-
.to_edge_transform_and_lower()
55-
.dump_operator_distribution()
56-
.check_count(self.ops_after_partitioner)
57-
.to_executorch()
58-
# TODO(MLETORCH-632): Fix numerical errors
59-
.run_method_and_compare_outputs(
60-
rtol=1.0,
61-
atol=5.0,
62-
inputs=get_test_inputs(self.dim, self.lengths, self.num_examples),
63-
)
64-
)
65-
66-
@unittest.expectedFailure # TODO(MLETORCH-635)
67-
def test_conformer_tosa_BI(self):
68-
(
69-
ArmTester(
70-
self.conformer,
71-
example_inputs=self.model_example_inputs,
72-
compile_spec=common.get_tosa_compile_spec(tosa_spec="TOSA-0.80+BI"),
73-
)
74-
.quantize()
75-
.export()
76-
.to_edge_transform_and_lower()
77-
.to_executorch()
78-
.run_method_and_compare_outputs(
79-
qtol=1.0,
80-
rtol=1.0,
81-
atol=5.0,
82-
inputs=get_test_inputs(self.dim, self.lengths, self.num_examples),
83-
)
84-
)
85-
86-
def test_conformer_u55_BI(self):
87-
tester = (
88-
ArmTester(
89-
self.conformer,
90-
example_inputs=self.model_example_inputs,
91-
compile_spec=common.get_u55_compile_spec(),
92-
)
93-
.quantize()
94-
.export()
95-
.to_edge_transform_and_lower()
96-
.to_executorch()
97-
.serialize()
98-
)
99-
100-
if conftest.is_option_enabled("corstone_fvp"):
101-
try:
102-
tester.run_method_and_compare_outputs(
103-
qtol=1.0,
104-
rtol=1.0,
105-
atol=5.0,
106-
inputs=get_test_inputs(self.dim, self.lengths, self.num_examples),
107-
)
108-
self.fail(
109-
"TODO(MLETORCH-635): Expected failure under FVP option, but test passed."
110-
)
111-
except Exception:
112-
pass
113-
114-
@unittest.expectedFailure # TODO(MLETORCH-635)
115-
def test_conformer_u85_BI(self):
116-
tester = (
117-
ArmTester(
118-
self.conformer,
119-
example_inputs=self.model_example_inputs,
120-
compile_spec=common.get_u85_compile_spec(),
121-
)
122-
.quantize()
123-
.export()
124-
.to_edge_transform_and_lower()
125-
.to_executorch()
126-
.serialize()
127-
)
128-
if conftest.is_option_enabled("corstone_fvp"):
129-
tester.run_method_and_compare_outputs(
130-
qtol=1.0,
131-
rtol=1.0,
132-
atol=5.0,
133-
inputs=get_test_inputs(self.dim, self.lengths, self.num_examples),
134-
)
52+
53+
def test_conformer_tosa_MI():
54+
pipeline = TosaPipelineMI[input_t](
55+
TestConformer.conformer,
56+
TestConformer.model_example_inputs,
57+
aten_op=TestConformer.aten_ops,
58+
exir_op=[],
59+
use_to_edge_transform_and_lower=True,
60+
)
61+
pipeline.change_args(
62+
"run_method_and_compare_outputs",
63+
get_test_inputs(
64+
TestConformer.dim, TestConformer.lengths, TestConformer.num_examples
65+
),
66+
rtol=1.0,
67+
atol=5.0,
68+
)
69+
pipeline.run()
70+
71+
72+
@pytest.mark.xfail(reason="All IO needs to have the same data type (MLETORCH-635)")
73+
def test_conformer_tosa_BI():
74+
pipeline = TosaPipelineBI[input_t](
75+
TestConformer.conformer,
76+
TestConformer.model_example_inputs,
77+
aten_op=TestConformer.aten_ops,
78+
exir_op=TestConformer.exir_ops,
79+
use_to_edge_transform_and_lower=True,
80+
)
81+
pipeline.change_args(
82+
"run_method_and_compare_outputs",
83+
get_test_inputs(
84+
TestConformer.dim, TestConformer.lengths, TestConformer.num_examples
85+
),
86+
rtol=1.0,
87+
atol=5.0,
88+
)
89+
pipeline.run()
90+
91+
92+
@common.XfailIfNoCorstone300
93+
@pytest.mark.xfail(
94+
reason="TODO(MLETORCH-635): Expected failure under FVP option, but test passed."
95+
)
96+
def test_conformer_u55_BI():
97+
pipeline = EthosU55PipelineBI[input_t](
98+
TestConformer.conformer,
99+
TestConformer.model_example_inputs,
100+
aten_ops=TestConformer.aten_ops,
101+
exir_ops=TestConformer.exir_ops,
102+
use_to_edge_transform_and_lower=True,
103+
run_on_fvp=True,
104+
)
105+
pipeline.change_args(
106+
"run_method_and_compare_outputs",
107+
get_test_inputs(
108+
TestConformer.dim, TestConformer.lengths, TestConformer.num_examples
109+
),
110+
rtol=1.0,
111+
atol=5.0,
112+
)
113+
pipeline.run()
114+
115+
116+
@common.XfailIfNoCorstone320
117+
@pytest.mark.xfail(reason="All IO needs to have the same data type (MLETORCH-635)")
118+
def test_conformer_u85_BI():
119+
pipeline = EthosU85PipelineBI[input_t](
120+
TestConformer.conformer,
121+
TestConformer.model_example_inputs,
122+
aten_ops=TestConformer.aten_ops,
123+
exir_ops=TestConformer.exir_ops,
124+
use_to_edge_transform_and_lower=True,
125+
run_on_fvp=True,
126+
)
127+
pipeline.change_args(
128+
"run_method_and_compare_outputs",
129+
get_test_inputs(
130+
TestConformer.dim, TestConformer.lengths, TestConformer.num_examples
131+
),
132+
rtol=1.0,
133+
atol=5.0,
134+
)
135+
pipeline.run()

‎backends/arm/test/models/test_dl3_arm.py‎

Lines changed: 65 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -3,92 +3,81 @@
33
# This source code is licensed under the BSD-style license found in the
44
# LICENSE file in the root directory of this source tree.
55

6-
import unittest
6+
from typing import Tuple
77

88
import pytest
99

10-
from executorch.backends.arm.test import common, conftest
10+
import torch
11+
12+
from executorch.backends.arm.test import common
13+
14+
from executorch.backends.arm.test.tester.test_pipeline import (
15+
EthosU55PipelineBI,
16+
EthosU85PipelineBI,
17+
TosaPipelineBI,
18+
TosaPipelineMI,
19+
)
1120

12-
from executorch.backends.arm.test.tester.arm_tester import ArmTester
1321
from executorch.examples.models import deeplab_v3
1422

23+
input_t = Tuple[torch.Tensor] # Input x
24+
1525

16-
class TestDl3(unittest.TestCase):
26+
class TestDl3:
1727
"""Tests DeepLabv3."""
1828

1929
dl3 = deeplab_v3.DeepLabV3ResNet50Model()
2030
model_example_inputs = dl3.get_example_inputs()
2131
dl3 = dl3.get_eager_model()
2232

23-
@unittest.expectedFailure
24-
def test_dl3_tosa_MI(self):
25-
(
26-
ArmTester(
27-
self.dl3,
28-
example_inputs=self.model_example_inputs,
29-
compile_spec=common.get_tosa_compile_spec("TOSA-0.80+MI"),
30-
)
31-
.export()
32-
.to_edge_transform_and_lower()
33-
.to_executorch()
34-
.run_method_and_compare_outputs(inputs=self.dl3.get_example_inputs())
35-
)
36-
37-
@unittest.expectedFailure
38-
def test_dl3_tosa_BI(self):
39-
(
40-
ArmTester(
41-
self.dl3,
42-
example_inputs=self.model_example_inputs,
43-
compile_spec=common.get_tosa_compile_spec("TOSA-0.80+BI"),
44-
)
45-
.quantize()
46-
.export()
47-
.to_edge_transform_and_lower()
48-
.to_executorch()
49-
.run_method_and_compare_outputs(
50-
atol=1.0, qtol=1, inputs=self.dl3.get_example_inputs()
51-
)
52-
)
53-
54-
@pytest.mark.slow
55-
@pytest.mark.corstone_fvp
56-
@unittest.skip
57-
def test_dl3_u55_BI(self):
58-
tester = (
59-
ArmTester(
60-
self.dl3,
61-
example_inputs=self.model_example_inputs,
62-
compile_spec=common.get_u55_compile_spec(),
63-
)
64-
.quantize()
65-
.export()
66-
.to_edge_transform_and_lower()
67-
.to_executorch()
68-
.serialize()
69-
)
70-
if conftest.is_option_enabled("corstone_fvp"):
71-
tester.run_method_and_compare_outputs(
72-
atol=1.0, qtol=1, inputs=self.dl3.get_example_inputs()
73-
)
74-
75-
@pytest.mark.slow
76-
@pytest.mark.corstone_fvp
77-
@unittest.skip
78-
def test_dl3_u85_BI(self):
79-
tester = (
80-
ArmTester(
81-
self.dl3,
82-
example_inputs=self.model_example_inputs,
83-
compile_spec=common.get_u85_compile_spec(),
84-
)
85-
.quantize()
86-
.export()
87-
.to_edge_transform_and_lower()
88-
.to_executorch()
89-
.serialize()
90-
)
91-
if conftest.is_option_enabled("corstone_fvp"):
92-
tester.run_method_and_compare_outputs(
93-
atol=1.0, qtol=1, inputs=self.dl3.get_example_inputs()
94-
)
33+
34+
@pytest.mark.flaky
35+
def test_dl3_tosa_MI():
36+
pipeline = TosaPipelineMI[input_t](
37+
TestDl3.dl3,
38+
TestDl3.model_example_inputs,
39+
aten_op=[],
40+
exir_op=[],
41+
)
42+
pipeline.change_args("run_method_and_compare_outputs", atol=1.0)
43+
pipeline.run()
44+
45+
46+
def test_dl3_tosa_BI():
47+
pipeline = TosaPipelineBI[input_t](
48+
TestDl3.dl3,
49+
TestDl3.model_example_inputs,
50+
aten_op=[],
51+
exir_op=[],
52+
)
53+
pipeline.change_args("run_method_and_compare_outputs", atol=1.0)
54+
pipeline.run()
55+
56+
57+
@pytest.mark.slow
58+
@common.XfailIfNoCorstone300
59+
@pytest.mark.xfail(reason="upsample_bilinear2d operator is not supported on U55")
60+
def test_dl3_u55_BI():
61+
pipeline = EthosU55PipelineBI[input_t](
62+
TestDl3.dl3,
63+
TestDl3.model_example_inputs,
64+
aten_ops=[],
65+
exir_ops=[],
66+
run_on_fvp=True,
67+
)
68+
pipeline.change_args("run_method_and_compare_outputs", atol=1.0)
69+
pipeline.run()
70+
71+
72+
@pytest.mark.slow
73+
@common.XfailIfNoCorstone320
74+
def test_dl3_u85_BI():
75+
pipeline = EthosU85PipelineBI[input_t](
76+
TestDl3.dl3,
77+
TestDl3.model_example_inputs,
78+
aten_ops=[],
79+
exir_ops=[],
80+
run_on_fvp=True,
81+
)
82+
pipeline.change_args("run_method_and_compare_outputs", atol=1.0)
83+
pipeline.run()

0 commit comments

Comments
 (0)