Skip to content

Commit 18ce410

Browse files
winskuo-quicwinskuo-quic
committed
Revert "Revert "Qualcomm AI Engine Direct - CI for Non-LLM GA model" (#12166)"
This reverts commit 9554826.
1 parent 45846c8 commit 18ce410

File tree

8 files changed

+90
-45
lines changed

8 files changed

+90
-45
lines changed

‎.ci/scripts/test_model.sh‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,22 @@ test_model_with_qnn() {
188188
EXPORT_SCRIPT=edsr
189189
# Additional deps for edsr
190190
pip install piq
191+
elif [[ "${MODEL_NAME}" == "cvt" ]]; then
192+
EXPORT_SCRIPT=cvt
193+
elif [[ "${MODEL_NAME}" == "dit" ]]; then
194+
EXPORT_SCRIPT=dit
195+
elif [[ "${MODEL_NAME}" == "efficientnet" ]]; then
196+
EXPORT_SCRIPT=efficientnet
197+
elif [[ "${MODEL_NAME}" == "focalnet" ]]; then
198+
EXPORT_SCRIPT=focalnet
199+
elif [[ "${MODEL_NAME}" == "mobilevit_v1" ]]; then
200+
EXPORT_SCRIPT=mobilevit_v1
201+
elif [[ "${MODEL_NAME}" == "mobilevit_v2" ]]; then
202+
EXPORT_SCRIPT=mobilevit_v2
203+
elif [[ "${MODEL_NAME}" == "pvt" ]]; then
204+
EXPORT_SCRIPT=pvt
205+
elif [[ "${MODEL_NAME}" == "swin" ]]; then
206+
EXPORT_SCRIPT=swin_transformer
191207
elif [[ "${MODEL_NAME}" == "albert" ]]; then
192208
EXPORT_SCRIPT=albert
193209
elif [[ "${MODEL_NAME}" == "bert" ]]; then
@@ -196,6 +212,8 @@ test_model_with_qnn() {
196212
EXPORT_SCRIPT=distilbert
197213
elif [[ "${MODEL_NAME}" == "eurobert" ]]; then
198214
EXPORT_SCRIPT=eurobert
215+
elif [[ "${MODEL_NAME}" == "roberta" ]]; then
216+
EXPORT_SCRIPT=roberta
199217
else
200218
echo "Unsupported model $MODEL_NAME"
201219
exit 1
@@ -210,10 +228,13 @@ test_model_with_qnn() {
210228
"dl3"|"mv3"|"mv2"|"ic4"|"ic3"|"vit"|"mb"|"w2l")
211229
SCRIPT_FOLDER=scripts
212230
;;
213-
"albert"|"bert"|"distilbert")
231+
"cvt"|"dit"|"focalnet"|"mobilevit_v2"|"pvt"|"swin")
232+
SCRIPT_FOLDER=oss_scripts
233+
;;
234+
"albert"|"bert"|"distilbert"|"roberta"|"efficientnet"|"mobilevit_v1")
214235
pip install evaluate
215236
SCRIPT_FOLDER=oss_scripts
216-
# Bert models running in 16bit will encounter op validation fail on some operations,
237+
# 16bit models will encounter op validation fail on some operations,
217238
# which requires CHIPSET >= SM8550.
218239
QNN_CHIPSET=SM8550
219240
;;

‎.github/workflows/trunk.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -532,7 +532,7 @@ jobs:
532532
docker-image: ci-image:executorch-ubuntu-22.04-qnn-sdk
533533
submodules: 'recursive'
534534
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
535-
timeout: 900
535+
timeout: 90
536536
script: |
537537
# The generic Linux job chooses to use base env, not the one setup by the image
538538
CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
@@ -551,14 +551,14 @@ jobs:
551551
strategy:
552552
matrix:
553553
dtype: [fp32]
554-
model: [albert, bert, distilbert] # eurobert requires transfomer >= 4.48.0, skip for now
554+
model: [cvt, dit, efficientnet, focalnet, mobilevit_v1, mobilevit_v2, pvt, swin, albert, bert, distilbert, roberta] # eurobert requires transfomer >= 4.48.0, skip for now
555555
fail-fast: false
556556
with:
557557
runner: linux.2xlarge
558558
docker-image: ci-image:executorch-ubuntu-22.04-qnn-sdk
559559
submodules: 'recursive'
560560
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
561-
timeout: 900
561+
timeout: 90
562562
script: |
563563
# The generic Linux job chooses to use base env, not the one setup by the image
564564
CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")

‎backends/qualcomm/builders/op_slice_copy.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ def define_node(
5656
if start < 0:
5757
start = start % input_tensor.shape[dim]
5858

59-
if len(node.args) > 3:
59+
if len(node.args) > 3 and node.args[3] is not None:
6060
end = min(cast(int, node.args[3]), input_tensor.shape[dim])
6161
if end < 0:
6262
end = end % input_tensor.shape[dim]

‎backends/qualcomm/tests/test_qnn_delegate.py‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -4885,14 +4885,13 @@ def test_gMLP(self):
48854885
self.assertGreaterEqual(msg["top_1"], 60)
48864886
self.assertGreaterEqual(msg["top_5"], 85)
48874887

4888-
@unittest.skip("Only outputs good accuracy in QNN 2.29")
4889-
def test_mobilevit_v2(self):
4888+
def test_mobilevit_v1(self):
48904889
if not self.required_envs([self.image_dataset]):
48914890
self.skipTest("missing required envs")
48924891

48934892
cmds = [
48944893
"python",
4895-
f"{self.executorch_root}/examples/qualcomm/oss_scripts/mobilevit_v2.py",
4894+
f"{self.executorch_root}/examples/qualcomm/oss_scripts/mobilevit_v1.py"
48964895
"--dataset",
48974896
self.image_dataset,
48984897
"--artifact",
@@ -4910,8 +4909,6 @@ def test_mobilevit_v2(self):
49104909
]
49114910
if self.host:
49124911
cmds.extend(["--host", self.host])
4913-
if self.shared_buffer:
4914-
cmds.extend(["--shared_buffer"])
49154912

49164913
p = subprocess.Popen(cmds, stdout=subprocess.DEVNULL)
49174914
with Listener((self.ip, self.port)) as listener:
@@ -4921,17 +4918,22 @@ def test_mobilevit_v2(self):
49214918
if "Error" in msg:
49224919
self.fail(msg["Error"])
49234920
else:
4924-
self.assertGreaterEqual(msg["top_1"], 50)
4921+
self.assertGreaterEqual(msg["top_1"], 70)
49254922
self.assertGreaterEqual(msg["top_5"], 85)
49264923

4927-
def test_pvt(self):
4924+
@unittest.skip("Only outputs good accuracy in QNN 2.29")
4925+
def test_mobilevit_v2(self):
49284926
if not self.required_envs([self.image_dataset]):
49294927
self.skipTest("missing required envs")
49304928

49314929
cmds = [
49324930
"python",
4933-
f"{self.executorch_root}/examples/qualcomm/oss_scripts/pvt.py",
4931+
f"{self.executorch_root}/examples/qualcomm/oss_scripts/mobilevit_v2.py",
4932+
"--dataset",
49344933
self.image_dataset,
4934+
"--artifact",
4935+
self.artifact_dir,
4936+
"--build_folder",
49354937
self.build_folder,
49364938
"--device",
49374939
self.device,
@@ -4944,6 +4946,8 @@ def test_pvt(self):
49444946
]
49454947
if self.host:
49464948
cmds.extend(["--host", self.host])
4949+
if self.shared_buffer:
4950+
cmds.extend(["--shared_buffer"])
49474951

49484952
p = subprocess.Popen(cmds, stdout=subprocess.DEVNULL)
49494953
with Listener((self.ip, self.port)) as listener:
@@ -4953,21 +4957,17 @@ def test_pvt(self):
49534957
if "Error" in msg:
49544958
self.fail(msg["Error"])
49554959
else:
4956-
self.assertGreaterEqual(msg["top_1"], 65)
4960+
self.assertGreaterEqual(msg["top_1"], 50)
49574961
self.assertGreaterEqual(msg["top_5"], 85)
49584962

4959-
def test_mobilevit1(self):
4963+
def test_pvt(self):
49604964
if not self.required_envs([self.image_dataset]):
49614965
self.skipTest("missing required envs")
49624966

49634967
cmds = [
49644968
"python",
4965-
f"{self.executorch_root}/examples/qualcomm/oss_scripts/mobilevit1.py"
4966-
"--dataset",
4969+
f"{self.executorch_root}/examples/qualcomm/oss_scripts/pvt.py",
49674970
self.image_dataset,
4968-
"--artifact",
4969-
self.artifact_dir,
4970-
"--build_folder",
49714971
self.build_folder,
49724972
"--device",
49734973
self.device,
@@ -4989,7 +4989,7 @@ def test_mobilevit1(self):
49894989
if "Error" in msg:
49904990
self.fail(msg["Error"])
49914991
else:
4992-
self.assertGreaterEqual(msg["top_1"], 70)
4992+
self.assertGreaterEqual(msg["top_1"], 65)
49934993
self.assertGreaterEqual(msg["top_5"], 85)
49944994

49954995
def test_regnet(self):

‎examples/qualcomm/oss_scripts/deit.py‎

Lines changed: 19 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,12 @@
66

77
import getpass
88
import json
9+
import logging
910
import os
1011
from multiprocessing.connection import Client
1112

1213
import numpy as np
14+
import torch
1315
from executorch.backends.qualcomm._passes.qnn_pass_manager import (
1416
get_capture_program_passes,
1517
)
@@ -46,16 +48,23 @@ def main(args):
4648
data_num = 100
4749
height = config.image_size
4850
width = config.image_size
49-
inputs, targets, input_list = get_imagenet_dataset(
50-
dataset_path=f"{args.dataset}",
51-
data_size=data_num,
52-
image_shape=(height, width),
53-
crop_size=(height, width),
54-
)
51+
52+
if args.ci:
53+
inputs = [(torch.rand(1, 3, height, width),)]
54+
logging.warning(
55+
"This option is for CI to verify the export flow. It uses random input and will result in poor accuracy."
56+
)
57+
else:
58+
inputs, targets, input_list = get_imagenet_dataset(
59+
dataset_path=f"{args.dataset}",
60+
data_size=data_num,
61+
image_shape=(height, width),
62+
crop_size=(height, width),
63+
)
5564

5665
# Get the Deit model.
5766
model = get_instance()
58-
pte_filename = "deit_qnn"
67+
pte_filename = "deit_qnn_q8"
5968

6069
# lower to QNN
6170
passes_job = get_capture_program_passes()
@@ -120,8 +129,8 @@ def main(args):
120129
parser.add_argument(
121130
"-a",
122131
"--artifact",
123-
help="path for storing generated artifacts and output by this example. Default ./deit_qnn",
124-
default="./deit_qnn",
132+
help="path for storing generated artifacts and output by this example. Default ./deit",
133+
default="./deit",
125134
type=str,
126135
)
127136

@@ -134,7 +143,7 @@ def main(args):
134143
"for https://www.kaggle.com/datasets/ifigotin/imagenetmini-1000)"
135144
),
136145
type=str,
137-
required=True,
146+
required=False,
138147
)
139148

140149
args = parser.parse_args()

examples/qualcomm/oss_scripts/mobilevit1.py renamed to examples/qualcomm/oss_scripts/mobilevit_v1.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ def main(args):
8484
.to("cpu")
8585
)
8686

87-
pte_filename = "mobilevit1_qnn_q16"
87+
pte_filename = "mobilevit_v1_qnn_q16"
8888
build_executorch_binary(
8989
module.eval(),
9090
inputs[0],
@@ -157,8 +157,8 @@ def main(args):
157157
"-a",
158158
"--artifact",
159159
help="path for storing generated artifacts by this example. "
160-
"Default ./mobilevit1",
161-
default="./mobilevit1",
160+
"Default ./mobilevit_v1",
161+
default="./mobilevit_v1",
162162
type=str,
163163
)
164164

‎examples/qualcomm/oss_scripts/roberta.py‎

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
import getpass
88
import json
9+
import logging
910
import os
1011
from multiprocessing.connection import Client
1112

@@ -38,16 +39,29 @@ def main(args):
3839
skip_node_id_set, skip_node_op_set = parse_skip_delegation_node(args)
3940

4041
os.makedirs(args.artifact, exist_ok=True)
41-
data_size = 100
4242

4343
tokenizer = AutoTokenizer.from_pretrained("xlm-roberta-base")
44-
inputs, targets, input_list = get_masked_language_model_dataset(
45-
args.dataset, tokenizer, data_size
46-
)
44+
data_size = 100
45+
if args.ci:
46+
random_ids = torch.randint(low=0, high=100, size=(1, 100), dtype=torch.int32)
47+
attention_mask = torch.ones((1, 100), dtype=torch.float32)
48+
inputs = [
49+
(
50+
random_ids,
51+
attention_mask,
52+
)
53+
]
54+
logging.warning(
55+
"This option is for CI to verify the export flow. It uses random input and will result in poor accuracy."
56+
)
57+
else:
58+
inputs, targets, input_list = get_masked_language_model_dataset(
59+
args.dataset, tokenizer, data_size
60+
)
4761

4862
# Get the Roberta model.
4963
model = get_instance(args)
50-
pte_filename = "roberta_qnn"
64+
pte_filename = "roberta_qnn_q16"
5165

5266
# lower to QNN
5367
passes_job = get_capture_program_passes()
@@ -137,7 +151,7 @@ def main(args):
137151
"-a",
138152
"--artifact",
139153
help="path for storing generated artifacts and output by this example. Default ./Roberta_qnn",
140-
default="./Roberta_qnn",
154+
default="./roberta",
141155
type=str,
142156
)
143157
parser.add_argument(
@@ -149,7 +163,7 @@ def main(args):
149163
"for https://www.kaggle.com/datasets/mikeortman/wikipedia-sentences"
150164
),
151165
type=str,
152-
required=True,
166+
required=False,
153167
)
154168

155169
args = parser.parse_args()

‎examples/qualcomm/oss_scripts/swin_transformer.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ def main(args):
8989

9090
data_num = 100
9191
if args.ci:
92-
inputs = [torch.rand(1, 3, 224, 224)]
92+
inputs = [(torch.rand(1, 3, 224, 224),)]
9393
logging.warning(
9494
"This option is for CI to verify the export flow. It uses random input and will result in poor accuracy."
9595
)
@@ -181,8 +181,9 @@ def main(args):
181181
parser.add_argument(
182182
"-a",
183183
"--artifact",
184-
help="path for storing generated artifacts by this example. " "Default ./swin",
185-
default="./swin",
184+
help="path for storing generated artifacts by this example. "
185+
"Default ./swin_transformer",
186+
default="./swin_transformer",
186187
type=str,
187188
)
188189

0 commit comments

Comments
 (0)