Skip to content

Commit 13de356

Browse files
Michiel-OlieslagersMichiel-Olieslagers
authored
Arm backend: Added in VGF unit tests for models (#13102)
Renamed test names to correct model names Signed-off-by: Michiel Olieslagers <[email protected]>
1 parent 4049235 commit 13de356

File tree

8 files changed

+277
-1
lines changed

8 files changed

+277
-1
lines changed

‎backends/arm/test/models/test_conformer.py‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
EthosU85PipelineINT,
1616
TosaPipelineFP,
1717
TosaPipelineINT,
18+
VgfPipeline,
1819
)
1920

2021
from torchaudio.models import Conformer
@@ -124,3 +125,40 @@ def test_conformer_u85_INT():
124125
atol=5.0,
125126
)
126127
pipeline.run()
128+
129+
130+
@common.SkipIfNoModelConverter
131+
def test_conformer_vgf_INT():
132+
pipeline = VgfPipeline[input_t](
133+
TestConformer.conformer,
134+
TestConformer.model_example_inputs,
135+
aten_op=TestConformer.aten_ops,
136+
exir_op=[],
137+
tosa_version="TOSA-1.0+INT",
138+
use_to_edge_transform_and_lower=True,
139+
)
140+
pipeline.pop_stage("check_count.exir")
141+
142+
# TODO: MLETORCH-1167 Create Vulkan backend e2e tests
143+
# pipeline.change_args(
144+
# "run_method_and_compare_outputs",
145+
# get_test_inputs(
146+
# TestConformer.dim, TestConformer.lengths, TestConformer.num_examples
147+
# ),
148+
# rtol=1.0,
149+
# atol=3.0,
150+
# )
151+
pipeline.run()
152+
153+
154+
@common.SkipIfNoModelConverter
155+
def test_conformer_vgf_FP():
156+
pipeline = VgfPipeline[input_t](
157+
TestConformer.conformer,
158+
TestConformer.model_example_inputs,
159+
aten_op=TestConformer.aten_ops,
160+
exir_op=[],
161+
tosa_version="TOSA-1.0+FP",
162+
use_to_edge_transform_and_lower=True,
163+
)
164+
pipeline.run()

‎backends/arm/test/models/test_deit_tiny_arm.py‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,12 @@
1111

1212
import torch
1313

14+
from executorch.backends.arm.test import common
15+
1416
from executorch.backends.arm.test.tester.test_pipeline import (
1517
TosaPipelineFP,
1618
TosaPipelineINT,
19+
VgfPipeline,
1720
)
1821

1922
from timm.data import IMAGENET_INCEPTION_MEAN, IMAGENET_INCEPTION_STD
@@ -56,3 +59,31 @@ def test_deit_tiny_tosa_INT():
5659
qtol=1,
5760
)
5861
pipeline.run()
62+
63+
64+
@common.SkipIfNoModelConverter
65+
def test_deit_tiny_vgf_INT():
66+
pipeline = VgfPipeline[input_t](
67+
deit_tiny,
68+
model_inputs,
69+
aten_op=[],
70+
exir_op=[],
71+
tosa_version="TOSA-1.0+INT",
72+
use_to_edge_transform_and_lower=True,
73+
atol=1.5,
74+
qtol=1,
75+
)
76+
pipeline.run()
77+
78+
79+
@common.SkipIfNoModelConverter
80+
def test_deit_tiny_vgf_FP():
81+
pipeline = VgfPipeline[input_t](
82+
deit_tiny,
83+
model_inputs,
84+
aten_op=[],
85+
exir_op=[],
86+
tosa_version="TOSA-1.0+FP",
87+
use_to_edge_transform_and_lower=True,
88+
)
89+
pipeline.run()

‎backends/arm/test/models/test_dl3_arm.py‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
EthosU85PipelineINT,
1717
TosaPipelineFP,
1818
TosaPipelineINT,
19+
VgfPipeline,
1920
)
2021

2122
from executorch.examples.models import deeplab_v3
@@ -87,3 +88,37 @@ def test_dl3_u85_INT():
8788
"run_method_and_compare_outputs", rtol=1.0, atol=1.0
8889
) # TODO: MLETORCH-1036 decrease tolerance
8990
pipeline.run()
91+
92+
93+
@common.SkipIfNoModelConverter
94+
def test_dl3_vgf_INT():
95+
pipeline = VgfPipeline[input_t](
96+
TestDl3.dl3,
97+
TestDl3.model_example_inputs,
98+
aten_op=[],
99+
exir_op=[],
100+
tosa_version="TOSA-1.0+INT",
101+
use_to_edge_transform_and_lower=True,
102+
)
103+
# TODO: MLETORCH-1167 Create Vulkan backend e2e tests
104+
# pipeline.change_args(
105+
# "run_method_and_compare_outputs", rtol=1.0, atol=1.0
106+
# )
107+
pipeline.run()
108+
109+
110+
@common.SkipIfNoModelConverter
111+
def test_dl3_vgf_FP():
112+
pipeline = VgfPipeline[input_t](
113+
TestDl3.dl3,
114+
TestDl3.model_example_inputs,
115+
aten_op=[],
116+
exir_op=[],
117+
tosa_version="TOSA-1.0+FP",
118+
use_to_edge_transform_and_lower=True,
119+
)
120+
# TODO: MLETORCH-1167 Create Vulkan backend e2e tests
121+
# pipeline.change_args(
122+
# "run_method_and_compare_outputs", rtol=1.0, atol=1.0
123+
# )
124+
pipeline.run()

‎backends/arm/test/models/test_llama.py‎

Lines changed: 41 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,11 @@
1717
import torch
1818
from executorch.backends.arm._passes import InsertCastForOpsWithInt64InputPass
1919

20-
from executorch.backends.arm.test import conftest
20+
from executorch.backends.arm.test import common, conftest
2121
from executorch.backends.arm.test.tester.test_pipeline import (
2222
TosaPipelineFP,
2323
TosaPipelineINT,
24+
VgfPipeline,
2425
)
2526
from executorch.examples.models.llama.export_llama_lib import (
2627
build_args_parser,
@@ -131,3 +132,42 @@ def test_llama_tosa_INT():
131132
use_to_edge_transform_and_lower=True,
132133
)
133134
pipeline.run()
135+
136+
137+
@common.SkipIfNoModelConverter
138+
def test_llama_vgf_FP():
139+
llama_model, llama_inputs, llama_meta = TestLlama().prepare_model()
140+
141+
if llama_model is None or llama_inputs is None:
142+
pytest.skip("Missing model and/or input files")
143+
144+
with torch.no_grad():
145+
pipeline = VgfPipeline[input_t](
146+
llama_model,
147+
llama_inputs,
148+
aten_op=[],
149+
exir_op=[],
150+
tosa_version="TOSA-1.0+FP",
151+
use_to_edge_transform_and_lower=True,
152+
)
153+
pipeline.run()
154+
155+
156+
@common.SkipIfNoModelConverter
157+
def test_llama_vgf_INT():
158+
llama_model, llama_inputs, llama_meta = TestLlama().prepare_model()
159+
160+
if llama_model is None or llama_inputs is None:
161+
pytest.skip("Missing model and/or input files")
162+
163+
with torch.no_grad():
164+
pipeline = VgfPipeline[input_t](
165+
llama_model,
166+
llama_inputs,
167+
aten_op=[],
168+
exir_op=[],
169+
tosa_version="TOSA-1.0+INT",
170+
use_to_edge_transform_and_lower=True,
171+
transform_passes=[InsertCastForOpsWithInt64InputPass()],
172+
)
173+
pipeline.run()

‎backends/arm/test/models/test_lstm_arm.py‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
EthosU85PipelineINT,
1414
TosaPipelineFP,
1515
TosaPipelineINT,
16+
VgfPipeline,
1617
)
1718

1819
from torch.nn.quantizable.modules import rnn
@@ -98,3 +99,37 @@ def test_lstm_u85_INT():
9899
"run_method_and_compare_outputs", get_test_inputs(), atol=3e-1, qtol=1.0
99100
)
100101
pipeline.run()
102+
103+
104+
@common.SkipIfNoModelConverter
105+
def test_lstm_vgf_INT():
106+
pipeline = VgfPipeline[input_t](
107+
TestLSTM.lstm,
108+
TestLSTM.model_example_inputs,
109+
aten_op=[],
110+
exir_op=[],
111+
tosa_version="TOSA-1.0+INT",
112+
use_to_edge_transform_and_lower=True,
113+
)
114+
# TODO: MLETORCH-1167 Create Vulkan backend e2e tests
115+
# pipeline.change_args(
116+
# "run_method_and_compare_outputs", get_test_inputs(), atol=3e-1, qtol=1.0
117+
# )
118+
pipeline.run()
119+
120+
121+
@common.SkipIfNoModelConverter
122+
def test_lstm_vgf_FP():
123+
pipeline = VgfPipeline[input_t](
124+
TestLSTM.lstm,
125+
TestLSTM.model_example_inputs,
126+
aten_op=[],
127+
exir_op=[],
128+
tosa_version="TOSA-1.0+FP",
129+
use_to_edge_transform_and_lower=True,
130+
)
131+
# TODO: MLETORCH-1167 Create Vulkan backend e2e tests
132+
# pipeline.change_args(
133+
# "run_method_and_compare_outputs", get_test_inputs(), atol=3e-1, qtol=1.0
134+
# )
135+
pipeline.run()

‎backends/arm/test/models/test_mobilenet_v2_arm.py‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
EthosU85PipelineINT,
1717
TosaPipelineFP,
1818
TosaPipelineINT,
19+
VgfPipeline,
1920
)
2021

2122
from torchvision import models, transforms # type: ignore[import-untyped]
@@ -94,3 +95,41 @@ def test_mv2_u85_INT(per_channel_quantization):
9495
qtol=1,
9596
)
9697
pipeline.run()
98+
99+
100+
@common.SkipIfNoModelConverter
101+
@common.parametrize("per_channel_quantization", quant_test_data)
102+
def test_mv2_vgf_INT(per_channel_quantization):
103+
pipeline = VgfPipeline[input_t](
104+
mv2,
105+
model_inputs,
106+
aten_op=[],
107+
exir_op=[],
108+
tosa_version="TOSA-1.0+INT",
109+
use_to_edge_transform_and_lower=True,
110+
per_channel_quantization=per_channel_quantization,
111+
atol=0.25,
112+
qtol=1,
113+
)
114+
# TODO: MLETORCH-1167 Create Vulkan backend e2e tests
115+
# pipeline.change_args(
116+
# "run_method_and_compare_outputs", get_test_inputs(), atol=3e-1, qtol=1.0
117+
# )
118+
pipeline.run()
119+
120+
121+
@common.SkipIfNoModelConverter
122+
def test_mv2_vgf_FP():
123+
pipeline = VgfPipeline[input_t](
124+
mv2,
125+
model_inputs,
126+
aten_op=[],
127+
exir_op=[],
128+
tosa_version="TOSA-1.0+FP",
129+
use_to_edge_transform_and_lower=True,
130+
)
131+
# TODO: MLETORCH-1167 Create Vulkan backend e2e tests
132+
# pipeline.change_args(
133+
# "run_method_and_compare_outputs", get_test_inputs(), atol=3e-1, qtol=1.0
134+
# ) # TODO: MLETORCH-1036 decrease tolerance
135+
pipeline.run()

‎backends/arm/test/models/test_mobilenet_v3_arm.py‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
EthosU85PipelineINT,
1616
TosaPipelineFP,
1717
TosaPipelineINT,
18+
VgfPipeline,
1819
)
1920

2021
from torchvision import models, transforms
@@ -82,3 +83,32 @@ def test_mv3_u85_INT():
8283
qtol=1,
8384
)
8485
pipeline.run()
86+
87+
88+
@common.SkipIfNoModelConverter
89+
@pytest.mark.slow
90+
def test_mv3_vgf_INT():
91+
pipeline = VgfPipeline[input_t](
92+
mv3,
93+
model_inputs,
94+
aten_op=[],
95+
exir_op=[],
96+
tosa_version="TOSA-1.0+INT",
97+
use_to_edge_transform_and_lower=True,
98+
atol=0.5,
99+
qtol=1,
100+
)
101+
pipeline.run()
102+
103+
104+
@common.SkipIfNoModelConverter
105+
def test_mv3_vgf_FP():
106+
pipeline = VgfPipeline[input_t](
107+
mv3,
108+
model_inputs,
109+
aten_op=[],
110+
exir_op=[],
111+
tosa_version="TOSA-1.0+FP",
112+
use_to_edge_transform_and_lower=True,
113+
)
114+
pipeline.run()

‎backends/arm/test/models/test_w2l_arm.py‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
EthosU85PipelineINT,
1818
TosaPipelineFP,
1919
TosaPipelineINT,
20+
VgfPipeline,
2021
)
2122

2223
from torchaudio import models
@@ -101,3 +102,30 @@ def test_w2l_u85_INT():
101102
run_on_fvp=True,
102103
)
103104
pipeline.run()
105+
106+
107+
@common.SkipIfNoModelConverter
108+
@pytest.mark.slow
109+
def test_w2l_vgf_INT():
110+
pipeline = VgfPipeline[input_t](
111+
TestW2L.w2l,
112+
TestW2L.model_example_inputs,
113+
aten_op=[],
114+
exir_op=TestW2L.all_operators,
115+
tosa_version="TOSA-1.0+INT",
116+
use_to_edge_transform_and_lower=True,
117+
)
118+
pipeline.run()
119+
120+
121+
@common.SkipIfNoModelConverter
122+
def test_w2l_vgf_FP():
123+
pipeline = VgfPipeline[input_t](
124+
TestW2L.w2l,
125+
TestW2L.model_example_inputs,
126+
aten_op=[],
127+
exir_op=TestW2L.all_operators,
128+
tosa_version="TOSA-1.0+FP",
129+
use_to_edge_transform_and_lower=True,
130+
)
131+
pipeline.run()

0 commit comments

Comments
 (0)