Skip to content

Commit e5874fa

Browse files
zingozingo
authored
Arm backend: Add testing/support for Inception_v4 and w2l for Ethos-U85 (pytorch#10517)
### Summary Inception_v4 and w2l for Ethos-U85 Inception_v3/4, w2l and resnet18/50 for TOSA Make it possible to override default runtime timeout per test This also split up GitHub testing in a few separate runners to speed up the testing as it takes longer then 60min. Fixes pytorch#9602 This also fix pytorch#9602 where arm unittest didn't build Executorch with EXECUTORCH_BUILD_ARM_BAREMETAL enabled ### Test plan Improve the unit and model testing for Arm backend in github with this patch. Signed-off-by: Zingo Andersen <[email protected]>
1 parent aa2c569 commit e5874fa

File tree

5 files changed

+127
-58
lines changed

5 files changed

+127
-58
lines changed

‎.github/workflows/pull.yml‎

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -481,37 +481,38 @@ jobs:
481481
build-tool: buck2
482482
docker-image: executorch-ubuntu-22.04-clang12
483483

484-
unittest-arm:
484+
unittest-arm-backend-with-no-fvp:
485+
name: unittest-arm-backend-with-no-fvp
485486
uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
486487
permissions:
487488
id-token: write
488489
contents: read
490+
strategy:
491+
matrix:
492+
include:
493+
- test_arm_baremetal: test_pytest_ops
494+
- test_arm_baremetal: test_pytest_models
495+
fail-fast: false
489496
with:
490497
runner: linux.2xlarge
491498
docker-image: executorch-ubuntu-22.04-arm-sdk
492499
submodules: 'recursive'
493500
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
494501
timeout: 90
495502
script: |
496-
set -eux
497-
498503
# The generic Linux job chooses to use base env, not the one setup by the image
499504
CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
500505
conda activate "${CONDA_ENV}"
501506
502-
BUILD_TOOL="cmake"
503-
504-
# Setup MacOS dependencies as there is no Docker support on MacOS atm
505-
PYTHON_EXECUTABLE=python \
506-
CMAKE_ARGS="-DEXECUTORCH_BUILD_PYBIND=ON" \
507-
EXECUTORCH_BUILD_ARM_BAREMETAL=ON \
508-
.ci/scripts/setup-linux.sh --build-tool "${BUILD_TOOL}"
507+
source .ci/scripts/utils.sh
508+
install_executorch "--use-pt-pinned-commit"
509509
510-
# Install Arm dependencies
511510
.ci/scripts/setup-arm-baremetal-tools.sh
512511
513-
# Run pytest without simulator
514-
backends/arm/test/test_arm_baremetal.sh test_pytest
512+
ARM_TEST=${{ matrix.test_arm_baremetal }}
513+
514+
# Test test_arm_baremetal.sh with test
515+
backends/arm/test/test_arm_baremetal.sh "${ARM_TEST}"
515516
516517
test-llama-runner-qnn-linux:
517518
name: test-llama-runner-qnn-linux

‎.github/workflows/trunk.yml‎

Lines changed: 15 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -176,12 +176,22 @@ jobs:
176176
# Test selective build
177177
PYTHON_EXECUTABLE=python bash examples/portable/scripts/test_demo_backend_delegation.sh "${BUILD_TOOL}"
178178
179-
test-arm-backend-delegation:
180-
name: test-arm-backend-delegation
179+
test-arm-backend:
180+
name: test-arm-backend
181181
uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
182182
permissions:
183183
id-token: write
184184
contents: read
185+
strategy:
186+
matrix:
187+
include:
188+
- test_arm_baremetal: test_pytest_ops_ethosu_fvp
189+
- test_arm_baremetal: test_pytest_models_ethosu_fvp
190+
- test_arm_baremetal: test_run_ethosu_fvp
191+
- test_arm_baremetal: test_models_tosa
192+
- test_arm_baremetal: test_models_ethos-u55
193+
- test_arm_baremetal: test_models_ethos-u85
194+
fail-fast: false
185195
with:
186196
runner: linux.2xlarge.memory
187197
docker-image: executorch-ubuntu-22.04-arm-sdk
@@ -202,34 +212,10 @@ jobs:
202212
# Hopefully this is high enough for this setup.
203213
sudo sysctl fs.inotify.max_user_watches=1048576 # 1024 * 1024
204214
205-
# Test ethos-u delegate examples with run.sh
206-
backends/arm/test/test_arm_baremetal.sh test_full_ethosu_fvp
207-
208-
209-
test-arm-reference-delegation:
210-
name: test-arm-reference-delegation
211-
uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
212-
permissions:
213-
id-token: write
214-
contents: read
215-
with:
216-
runner: linux.2xlarge.memory
217-
docker-image: executorch-ubuntu-22.04-arm-sdk
218-
submodules: 'recursive'
219-
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
220-
timeout: 90
221-
script: |
222-
# The generic Linux job chooses to use base env, not the one setup by the image
223-
CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
224-
conda activate "${CONDA_ENV}"
225-
226-
source .ci/scripts/utils.sh
227-
install_executorch "--use-pt-pinned-commit"
228-
229-
.ci/scripts/setup-arm-baremetal-tools.sh
215+
ARM_TEST=${{ matrix.test_arm_baremetal }}
230216
231-
# Run arm unit tests using the simulator
232-
backends/arm/test/test_arm_baremetal.sh test_pytest_ethosu_fvp
217+
# Test test_arm_baremetal.sh with test
218+
backends/arm/test/test_arm_baremetal.sh "${ARM_TEST}"
233219
234220
test-arm-cortex-m-size-test:
235221
name: test-arm-cortex-m-size-test

‎backends/arm/scripts/run_fvp.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ hash ${fvp_model} \
6565
num_macs=$(echo ${target} | cut -d - -f 3)
6666

6767
echo "--------------------------------------------------------------------------------"
68-
echo "Running ${elf_file} for ${target} run with FVP:${fvp_model} num_macs:${num_macs}"
68+
echo "Running ${elf_file} for ${target} run with FVP:${fvp_model} num_macs:${num_macs} timeout:${timeout}"
6969
echo "WARNING: Corstone FVP is not cycle accurate and should NOT be used to determine valid runtime"
7070
echo "--------------------------------------------------------------------------------"
7171

‎backends/arm/test/test_arm_baremetal.sh‎

Lines changed: 88 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -69,27 +69,71 @@ all() { # Run all tests
6969
echo "${TEST_SUITE_NAME}: PASS"
7070
}
7171

72-
test_pytest() { # Test ops and other things
72+
test_pytest_ops() { # Test ops and other things
7373
echo "${TEST_SUITE_NAME}: Run pytest"
7474

75-
./examples/models/llama3_2_vision/install_requirements.sh
75+
# Prepare for pytest
76+
backends/arm/scripts/build_executorch.sh
7677

7778
# Run arm baremetal pytest tests without FVP
78-
pytest --verbose --color=yes --numprocesses=auto backends/arm/test/
79+
pytest --verbose --color=yes --numprocesses=auto backends/arm/test/ --ignore=backends/arm/test/models
7980
echo "${TEST_SUITE_NAME}: PASS"
8081
}
8182

82-
test_pytest_ethosu_fvp() { # Same as test_pytest but also sometime verify using Corstone FVP
83+
test_pytest_models() { # Test ops and other things
84+
echo "${TEST_SUITE_NAME}: Run pytest"
85+
86+
examples/models/llama3_2_vision/install_requirements.sh
87+
88+
# Prepare for pytest
89+
backends/arm/scripts/build_executorch.sh
90+
91+
# Run arm baremetal pytest tests without FVP
92+
pytest --verbose --color=yes --numprocesses=auto backends/arm/test/models
93+
echo "${TEST_SUITE_NAME}: PASS"
94+
}
95+
96+
test_pytest() { # Test ops and other things
97+
echo "${TEST_SUITE_NAME}: Run pytest"
98+
test_pytest_ops
99+
test_pytest_models
100+
echo "${TEST_SUITE_NAME}: PASS"
101+
}
102+
103+
test_pytest_ops_ethosu_fvp() { # Same as test_pytest but also sometime verify using Corstone FVP
83104
echo "${TEST_SUITE_NAME}: Run pytest with fvp"
84105

85-
./examples/models/llama3_2_vision/install_requirements.sh
106+
# Prepare Corstone-3x0 FVP for pytest
107+
backends/arm/scripts/build_executorch.sh
108+
backends/arm/scripts/build_portable_kernels.sh
109+
# Build semihosting version of the runner used by pytest testing when using --arm_run_corstoneFVP
110+
backends/arm/test/setup_testing.sh
111+
112+
# Run arm baremetal pytest tests with FVP
113+
pytest --verbose --color=yes --numprocesses=auto backends/arm/test/ --ignore=backends/arm/test/models --arm_run_corstoneFVP
114+
echo "${TEST_SUITE_NAME}: PASS"
115+
}
116+
117+
test_pytest_models_ethosu_fvp() { # Same as test_pytest but also sometime verify using Corstone FVP
118+
echo "${TEST_SUITE_NAME}: Run pytest with fvp"
119+
120+
examples/models/llama3_2_vision/install_requirements.sh
86121

87122
# Prepare Corstone-3x0 FVP for pytest
88-
examples/arm/run.sh --model_name=add --build_only
123+
backends/arm/scripts/build_executorch.sh
124+
backends/arm/scripts/build_portable_kernels.sh
125+
# Build semihosting version of the runner used by pytest testing when using --arm_run_corstoneFVP
89126
backends/arm/test/setup_testing.sh
90127

91128
# Run arm baremetal pytest tests with FVP
92-
pytest --verbose --color=yes --numprocesses=auto backends/arm/test/ --arm_run_corstoneFVP
129+
pytest --verbose --color=yes --numprocesses=auto backends/arm/test/models --arm_run_corstoneFVP
130+
echo "${TEST_SUITE_NAME}: PASS"
131+
}
132+
133+
test_pytest_ethosu_fvp() { # Same as test_pytest but also sometime verify using Corstone FVP
134+
echo "${TEST_SUITE_NAME}: Run pytest with fvp"
135+
test_pytest_ops_ethosu_fvp
136+
test_pytest_models_ethosu_fvp
93137
echo "${TEST_SUITE_NAME}: PASS"
94138
}
95139

@@ -113,8 +157,8 @@ test_run_ethosu_fvp() { # End to End model tests using run.sh
113157
echo "${TEST_SUITE_NAME}: PASS"
114158
}
115159

116-
test_models_ethosu_fvp() { # End to End model tests using model_test.py
117-
echo "${TEST_SUITE_NAME}: Test ethos-u delegate models with test_model.py"
160+
test_models_tosa() { # End to End model tests using model_test.py
161+
echo "${TEST_SUITE_NAME}: Test TOSA delegated models with test_model.py"
118162

119163
# Build common libs once
120164
python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --build_libs
@@ -125,26 +169,57 @@ test_models_ethosu_fvp() { # End to End model tests using model_test.py
125169
python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=TOSA --model=mv3
126170
python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=TOSA --model=lstm
127171
python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=TOSA --model=edsr
172+
# python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=TOSA --model=emformer_transcribe # Takes long time to run
173+
# python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=TOSA --model=emformer_join # Takes long time to run
174+
python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=TOSA --model=w2l
175+
python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=TOSA --model=ic3
176+
python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=TOSA --model=ic4
177+
python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=TOSA --model=resnet18
178+
python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=TOSA --model=resnet50
179+
180+
echo "${TEST_SUITE_NAME}: PASS"
181+
}
182+
183+
test_models_ethos-u55() { # End to End model tests using model_test.py
184+
echo "${TEST_SUITE_NAME}: Test Ethos-U55 delegated models with test_model.py"
185+
186+
# Build common libs once
187+
python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --build_libs
128188

129189
# Ethos-U55
130190
echo "${TEST_SUITE_NAME}: Test ethos-u target Ethos-U55"
131191
python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=ethos-u55-128 --model=mv2 --extra_flags="-DET_ATOL=2.00 -DET_RTOL=2.00"
132192
python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=ethos-u55-64 --model=mv3 --extra_flags="-DET_ATOL=5.00 -DET_RTOL=5.00"
133193
python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=ethos-u55-256 --model=lstm --extra_flags="-DET_ATOL=0.03 -DET_RTOL=0.03"
134194

195+
echo "${TEST_SUITE_NAME}: PASS"
196+
}
197+
198+
test_models_ethos-u85() { # End to End model tests using model_test.py
199+
echo "${TEST_SUITE_NAME}: Test Ethos-U85 delegated models with test_model.py"
200+
201+
# Build common libs once
202+
python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --build_libs
203+
135204
# Ethos-U85
136205
echo "${TEST_SUITE_NAME}: Test ethos-u target Ethos-U85"
137-
python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=ethos-u85-256 --model=mv2 --extra_flags="-DET_ATOL=2.00 -DET_RTOL=2.00"
138-
python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=ethos-u85-1024 --model=mv3 --extra_flags="-DET_ATOL=5.00 -DET_RTOL=5.00"
139-
python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=ethos-u85-128 --model=lstm --extra_flags="-DET_ATOL=0.03 -DET_RTOL=0.03"
206+
python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=ethos-u85-256 --model=mv2 --extra_flags="-DET_ATOL=2.00 -DET_RTOL=2.00"
207+
python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=ethos-u85-512 --model=mv3 --extra_flags="-DET_ATOL=5.00 -DET_RTOL=5.00"
208+
python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=ethos-u85-128 --model=lstm --extra_flags="-DET_ATOL=0.03 -DET_RTOL=0.03"
209+
python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=ethos-u85-128 --model=w2l --extra_flags="-DET_ATOL=0.01 -DET_RTOL=0.01"
210+
python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=ethos-u85-256 --model=ic4 --extra_flags="-DET_ATOL=0.8 -DET_RTOL=0.8" --timeout=2400
211+
140212
echo "${TEST_SUITE_NAME}: PASS"
141213
}
142214

215+
143216
test_full_ethosu_fvp() { # All End to End model tests
144217
echo "${TEST_SUITE_NAME}: Test ethos-u delegate models and examples on fvp"
145218

146-
test_models_ethosu_fvp
147219
test_run_ethosu_fvp
220+
test_models_tosa
221+
test_models_ethos-u55
222+
test_models_ethos-u85
148223
echo "${TEST_SUITE_NAME}: PASS"
149224
}
150225

‎backends/arm/test/test_model.py‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,12 @@ def get_args():
6161
default=None,
6262
help="Extra cmake flags to pass the when building the executor_runner",
6363
)
64+
parser.add_argument(
65+
"--timeout",
66+
required=False,
67+
default=60 * 10,
68+
help="Timeout in seconds used when running the model",
69+
)
6470
args = parser.parse_args()
6571

6672
if args.model and "ethos-u" in args.target and args.system_config is None:
@@ -185,13 +191,14 @@ def build_ethosu_runtime(
185191
return elf_file
186192

187193

188-
def run_elf_with_fvp(script_path: str, elf_file: str, target: str):
194+
def run_elf_with_fvp(script_path: str, elf_file: str, target: str, timeout: int):
189195
run_external_cmd(
190196
[
191197
"bash",
192198
os.path.join(script_path, "run_fvp.sh"),
193199
f"--elf={elf_file}",
194200
f"--target={target}",
201+
f"--timeout={timeout}",
195202
]
196203
)
197204

@@ -243,5 +250,5 @@ def run_elf_with_fvp(script_path: str, elf_file: str, target: str):
243250
)
244251
print(f"ELF file created: {elf_file} ")
245252

246-
run_elf_with_fvp(script_path, elf_file, args.target)
253+
run_elf_with_fvp(script_path, elf_file, args.target, args.timeout)
247254
print(f"Model: {model_name} on {args.target} -> PASS")

0 commit comments

Comments
 (0)