Skip to content

Commit 4b286b9

Browse files
winskuo-quicwinskuo-quic
committed
Revert "Revert "Qualcomm AI Engine Direct - CI for Non-LLM GA model" (#12166)"
This reverts commit 9554826.
1 parent b14cb22 commit 4b286b9

File tree

8 files changed

+90
-45
lines changed

8 files changed

+90
-45
lines changed

‎.ci/scripts/test_model.sh‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,22 @@ test_model_with_qnn() {
188188
EXPORT_SCRIPT=edsr
189189
# Additional deps for edsr
190190
pip install piq
191+
elif [[ "${MODEL_NAME}" == "cvt" ]]; then
192+
EXPORT_SCRIPT=cvt
193+
elif [[ "${MODEL_NAME}" == "dit" ]]; then
194+
EXPORT_SCRIPT=dit
195+
elif [[ "${MODEL_NAME}" == "efficientnet" ]]; then
196+
EXPORT_SCRIPT=efficientnet
197+
elif [[ "${MODEL_NAME}" == "focalnet" ]]; then
198+
EXPORT_SCRIPT=focalnet
199+
elif [[ "${MODEL_NAME}" == "mobilevit_v1" ]]; then
200+
EXPORT_SCRIPT=mobilevit_v1
201+
elif [[ "${MODEL_NAME}" == "mobilevit_v2" ]]; then
202+
EXPORT_SCRIPT=mobilevit_v2
203+
elif [[ "${MODEL_NAME}" == "pvt" ]]; then
204+
EXPORT_SCRIPT=pvt
205+
elif [[ "${MODEL_NAME}" == "swin" ]]; then
206+
EXPORT_SCRIPT=swin_transformer
191207
elif [[ "${MODEL_NAME}" == "albert" ]]; then
192208
EXPORT_SCRIPT=albert
193209
elif [[ "${MODEL_NAME}" == "bert" ]]; then
@@ -196,6 +212,8 @@ test_model_with_qnn() {
196212
EXPORT_SCRIPT=distilbert
197213
elif [[ "${MODEL_NAME}" == "eurobert" ]]; then
198214
EXPORT_SCRIPT=eurobert
215+
elif [[ "${MODEL_NAME}" == "roberta" ]]; then
216+
EXPORT_SCRIPT=roberta
199217
else
200218
echo "Unsupported model $MODEL_NAME"
201219
exit 1
@@ -210,10 +228,13 @@ test_model_with_qnn() {
210228
"dl3"|"mv3"|"mv2"|"ic4"|"ic3"|"vit"|"mb"|"w2l")
211229
SCRIPT_FOLDER=scripts
212230
;;
213-
"albert"|"bert"|"distilbert")
231+
"cvt"|"dit"|"focalnet"|"mobilevit_v2"|"pvt"|"swin")
232+
SCRIPT_FOLDER=oss_scripts
233+
;;
234+
"albert"|"bert"|"distilbert"|"roberta"|"efficientnet"|"mobilevit_v1")
214235
pip install evaluate
215236
SCRIPT_FOLDER=oss_scripts
216-
# Bert models running in 16bit will encounter op validation fail on some operations,
237+
# 16bit models will encounter op validation fail on some operations,
217238
# which requires CHIPSET >= SM8550.
218239
QNN_CHIPSET=SM8550
219240
;;

‎.github/workflows/trunk.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -501,7 +501,7 @@ jobs:
501501
docker-image: executorch-ubuntu-22.04-qnn-sdk
502502
submodules: 'recursive'
503503
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
504-
timeout: 900
504+
timeout: 90
505505
script: |
506506
# The generic Linux job chooses to use base env, not the one setup by the image
507507
CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
@@ -520,14 +520,14 @@ jobs:
520520
strategy:
521521
matrix:
522522
dtype: [fp32]
523-
model: [albert, bert, distilbert] # eurobert requires transfomer >= 4.48.0, skip for now
523+
model: [cvt, dit, efficientnet, focalnet, mobilevit_v1, mobilevit_v2, pvt, swin, albert, bert, distilbert, roberta] # eurobert requires transfomer >= 4.48.0, skip for now
524524
fail-fast: false
525525
with:
526526
runner: linux.2xlarge
527527
docker-image: executorch-ubuntu-22.04-qnn-sdk
528528
submodules: 'recursive'
529529
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
530-
timeout: 900
530+
timeout: 90
531531
script: |
532532
# The generic Linux job chooses to use base env, not the one setup by the image
533533
CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")

‎backends/qualcomm/builders/op_slice_copy.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ def define_node(
5656
if start < 0:
5757
start = start % input_tensor.shape[dim]
5858

59-
if len(node.args) > 3:
59+
if len(node.args) > 3 and node.args[3] is not None:
6060
end = min(cast(int, node.args[3]), input_tensor.shape[dim])
6161
if end < 0:
6262
end = end % input_tensor.shape[dim]

‎backends/qualcomm/tests/test_qnn_delegate.py‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -4698,14 +4698,13 @@ def test_gMLP(self):
46984698
self.assertGreaterEqual(msg["top_1"], 60)
46994699
self.assertGreaterEqual(msg["top_5"], 85)
47004700

4701-
@unittest.skip("Only outputs good accuracy in QNN 2.29")
4702-
def test_mobilevit_v2(self):
4701+
def test_mobilevit_v1(self):
47034702
if not self.required_envs([self.image_dataset]):
47044703
self.skipTest("missing required envs")
47054704

47064705
cmds = [
47074706
"python",
4708-
f"{self.executorch_root}/examples/qualcomm/oss_scripts/mobilevit_v2.py",
4707+
f"{self.executorch_root}/examples/qualcomm/oss_scripts/mobilevit_v1.py"
47094708
"--dataset",
47104709
self.image_dataset,
47114710
"--artifact",
@@ -4723,8 +4722,6 @@ def test_mobilevit_v2(self):
47234722
]
47244723
if self.host:
47254724
cmds.extend(["--host", self.host])
4726-
if self.shared_buffer:
4727-
cmds.extend(["--shared_buffer"])
47284725

47294726
p = subprocess.Popen(cmds, stdout=subprocess.DEVNULL)
47304727
with Listener((self.ip, self.port)) as listener:
@@ -4734,17 +4731,22 @@ def test_mobilevit_v2(self):
47344731
if "Error" in msg:
47354732
self.fail(msg["Error"])
47364733
else:
4737-
self.assertGreaterEqual(msg["top_1"], 50)
4734+
self.assertGreaterEqual(msg["top_1"], 70)
47384735
self.assertGreaterEqual(msg["top_5"], 85)
47394736

4740-
def test_pvt(self):
4737+
@unittest.skip("Only outputs good accuracy in QNN 2.29")
4738+
def test_mobilevit_v2(self):
47414739
if not self.required_envs([self.image_dataset]):
47424740
self.skipTest("missing required envs")
47434741

47444742
cmds = [
47454743
"python",
4746-
f"{self.executorch_root}/examples/qualcomm/oss_scripts/pvt.py",
4744+
f"{self.executorch_root}/examples/qualcomm/oss_scripts/mobilevit_v2.py",
4745+
"--dataset",
47474746
self.image_dataset,
4747+
"--artifact",
4748+
self.artifact_dir,
4749+
"--build_folder",
47484750
self.build_folder,
47494751
"--device",
47504752
self.device,
@@ -4757,6 +4759,8 @@ def test_pvt(self):
47574759
]
47584760
if self.host:
47594761
cmds.extend(["--host", self.host])
4762+
if self.shared_buffer:
4763+
cmds.extend(["--shared_buffer"])
47604764

47614765
p = subprocess.Popen(cmds, stdout=subprocess.DEVNULL)
47624766
with Listener((self.ip, self.port)) as listener:
@@ -4766,21 +4770,17 @@ def test_pvt(self):
47664770
if "Error" in msg:
47674771
self.fail(msg["Error"])
47684772
else:
4769-
self.assertGreaterEqual(msg["top_1"], 65)
4773+
self.assertGreaterEqual(msg["top_1"], 50)
47704774
self.assertGreaterEqual(msg["top_5"], 85)
47714775

4772-
def test_mobilevit1(self):
4776+
def test_pvt(self):
47734777
if not self.required_envs([self.image_dataset]):
47744778
self.skipTest("missing required envs")
47754779

47764780
cmds = [
47774781
"python",
4778-
f"{self.executorch_root}/examples/qualcomm/oss_scripts/mobilevit1.py"
4779-
"--dataset",
4782+
f"{self.executorch_root}/examples/qualcomm/oss_scripts/pvt.py",
47804783
self.image_dataset,
4781-
"--artifact",
4782-
self.artifact_dir,
4783-
"--build_folder",
47844784
self.build_folder,
47854785
"--device",
47864786
self.device,
@@ -4802,7 +4802,7 @@ def test_mobilevit1(self):
48024802
if "Error" in msg:
48034803
self.fail(msg["Error"])
48044804
else:
4805-
self.assertGreaterEqual(msg["top_1"], 70)
4805+
self.assertGreaterEqual(msg["top_1"], 65)
48064806
self.assertGreaterEqual(msg["top_5"], 85)
48074807

48084808
def test_regnet(self):

‎examples/qualcomm/oss_scripts/deit.py‎

Lines changed: 19 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,12 @@
66

77
import getpass
88
import json
9+
import logging
910
import os
1011
from multiprocessing.connection import Client
1112

1213
import numpy as np
14+
import torch
1315
from executorch.backends.qualcomm._passes.qnn_pass_manager import (
1416
get_capture_program_passes,
1517
)
@@ -46,16 +48,23 @@ def main(args):
4648
data_num = 100
4749
height = config.image_size
4850
width = config.image_size
49-
inputs, targets, input_list = get_imagenet_dataset(
50-
dataset_path=f"{args.dataset}",
51-
data_size=data_num,
52-
image_shape=(height, width),
53-
crop_size=(height, width),
54-
)
51+
52+
if args.ci:
53+
inputs = [(torch.rand(1, 3, height, width),)]
54+
logging.warning(
55+
"This option is for CI to verify the export flow. It uses random input and will result in poor accuracy."
56+
)
57+
else:
58+
inputs, targets, input_list = get_imagenet_dataset(
59+
dataset_path=f"{args.dataset}",
60+
data_size=data_num,
61+
image_shape=(height, width),
62+
crop_size=(height, width),
63+
)
5564

5665
# Get the Deit model.
5766
model = get_instance()
58-
pte_filename = "deit_qnn"
67+
pte_filename = "deit_qnn_q8"
5968

6069
# lower to QNN
6170
passes_job = get_capture_program_passes()
@@ -120,8 +129,8 @@ def main(args):
120129
parser.add_argument(
121130
"-a",
122131
"--artifact",
123-
help="path for storing generated artifacts and output by this example. Default ./deit_qnn",
124-
default="./deit_qnn",
132+
help="path for storing generated artifacts and output by this example. Default ./deit",
133+
default="./deit",
125134
type=str,
126135
)
127136

@@ -134,7 +143,7 @@ def main(args):
134143
"for https://www.kaggle.com/datasets/ifigotin/imagenetmini-1000)"
135144
),
136145
type=str,
137-
required=True,
146+
required=False,
138147
)
139148

140149
args = parser.parse_args()

examples/qualcomm/oss_scripts/mobilevit1.py renamed to examples/qualcomm/oss_scripts/mobilevit_v1.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ def main(args):
8484
.to("cpu")
8585
)
8686

87-
pte_filename = "mobilevit1_qnn_q16"
87+
pte_filename = "mobilevit_v1_qnn_q16"
8888
build_executorch_binary(
8989
module.eval(),
9090
inputs[0],
@@ -157,8 +157,8 @@ def main(args):
157157
"-a",
158158
"--artifact",
159159
help="path for storing generated artifacts by this example. "
160-
"Default ./mobilevit1",
161-
default="./mobilevit1",
160+
"Default ./mobilevit_v1",
161+
default="./mobilevit_v1",
162162
type=str,
163163
)
164164

‎examples/qualcomm/oss_scripts/roberta.py‎

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
import getpass
88
import json
9+
import logging
910
import os
1011
from multiprocessing.connection import Client
1112

@@ -38,16 +39,29 @@ def main(args):
3839
skip_node_id_set, skip_node_op_set = parse_skip_delegation_node(args)
3940

4041
os.makedirs(args.artifact, exist_ok=True)
41-
data_size = 100
4242

4343
tokenizer = AutoTokenizer.from_pretrained("xlm-roberta-base")
44-
inputs, targets, input_list = get_masked_language_model_dataset(
45-
args.dataset, tokenizer, data_size
46-
)
44+
data_size = 100
45+
if args.ci:
46+
random_ids = torch.randint(low=0, high=100, size=(1, 100), dtype=torch.int32)
47+
attention_mask = torch.ones((1, 100), dtype=torch.float32)
48+
inputs = [
49+
(
50+
random_ids,
51+
attention_mask,
52+
)
53+
]
54+
logging.warning(
55+
"This option is for CI to verify the export flow. It uses random input and will result in poor accuracy."
56+
)
57+
else:
58+
inputs, targets, input_list = get_masked_language_model_dataset(
59+
args.dataset, tokenizer, data_size
60+
)
4761

4862
# Get the Roberta model.
4963
model = get_instance(args)
50-
pte_filename = "roberta_qnn"
64+
pte_filename = "roberta_qnn_q16"
5165

5266
# lower to QNN
5367
passes_job = get_capture_program_passes()
@@ -137,7 +151,7 @@ def main(args):
137151
"-a",
138152
"--artifact",
139153
help="path for storing generated artifacts and output by this example. Default ./Roberta_qnn",
140-
default="./Roberta_qnn",
154+
default="./roberta",
141155
type=str,
142156
)
143157
parser.add_argument(
@@ -149,7 +163,7 @@ def main(args):
149163
"for https://www.kaggle.com/datasets/mikeortman/wikipedia-sentences"
150164
),
151165
type=str,
152-
required=True,
166+
required=False,
153167
)
154168

155169
args = parser.parse_args()

‎examples/qualcomm/oss_scripts/swin_transformer.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ def main(args):
8989

9090
data_num = 100
9191
if args.ci:
92-
inputs = [torch.rand(1, 3, 224, 224)]
92+
inputs = [(torch.rand(1, 3, 224, 224),)]
9393
logging.warning(
9494
"This option is for CI to verify the export flow. It uses random input and will result in poor accuracy."
9595
)
@@ -181,8 +181,9 @@ def main(args):
181181
parser.add_argument(
182182
"-a",
183183
"--artifact",
184-
help="path for storing generated artifacts by this example. " "Default ./swin",
185-
default="./swin",
184+
help="path for storing generated artifacts by this example. "
185+
"Default ./swin_transformer",
186+
default="./swin_transformer",
186187
type=str,
187188
)
188189

0 commit comments

Comments
 (0)