Skip to content

Commit afb09d0

Browse files
authored
Arm backend: Add NGP unit test scripts (#13819)
Prepare scripts for CI.
1 parent deaf37f commit afb09d0

14 files changed

+393
-55
lines changed

backends/arm/CMakeLists.txt

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -73,9 +73,10 @@ if(EXECUTORCH_BUILD_VGF)
7373
# vgf backend
7474
list(TRANSFORM _vgf_backend_sources PREPEND "${EXECUTORCH_ROOT}/")
7575
add_library(vgf_backend ${_vgf_backend_sources})
76+
install(TARGETS vgf_backend EXPORT ExecuTorchTargets)
7677
target_include_directories(
77-
vgf_backend PUBLIC ${_common_include_directories} ${VULKAN_HEADERS_PATH}
78-
${VOLK_HEADERS_PATH}
78+
vgf_backend PRIVATE ${_common_include_directories} ${VULKAN_HEADERS_PATH}
79+
${VOLK_HEADERS_PATH}
7980
)
8081
target_compile_options(
8182
vgf_backend PRIVATE -DUSE_VULKAN_WRAPPER -DUSE_VULKAN_VOLK

backends/arm/README.md

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,19 @@ You can test to run some models with the full fvp test flow
8888
backends/arm/test/test_arm_baremetal.sh test_full_ethosu_fvp
8989
```
9090

91+
To run the unit test suite with VKML use the following. Note Vulkan SDK need to be installed.
92+
Have a look at install_vulkan_sdk() in .ci/scripts/setup-vulkan-linux-deps.sh on how to install Vulkan SDK.
93+
94+
```
95+
backends/arm/test/test_arm_baremetal.sh test_pytest_vkml
96+
```
97+
98+
You can test to run some models with the full VKML flow
99+
100+
```
101+
backends/arm/test/test_arm_baremetal.sh test_full_vkml
102+
```
103+
91104
## Unit tests
92105

93106
This is the structure of the test directory
@@ -102,6 +115,7 @@ test # Root test folder
102115
├── tosautil # Utility functions for TOSA artifacts
103116
├ common.py # Common functions and definitions used by many tests
104117
├ setup_testing.sh # Script to prepare testing for using the Corstone 3x0 FVP
118+
├ setup_testing_vkml.sh # Script to prepare testing for using the VKML
105119
├ test_arm_baremetal.sh # Help script to trigger testing
106120
```
107121

@@ -123,7 +137,7 @@ first you need to build and prepare some used target libs
123137

124138
```
125139
examples/arm/run.sh --model_name=add --build_only
126-
backends/arm/test/setup_testing.sh
140+
backends/arm/test/setup_testing.sh and/or backends/arm/test/setup_testing_vkml.sh
127141
```
128142

129143
The you can run the tests with
Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
#!/usr/bin/env bash
2+
# Copyright 2025 Arm Limited and/or its affiliates.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
set -eu
8+
9+
script_dir=$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd)
10+
et_root_dir=$(cd ${script_dir}/../../.. && pwd)
11+
et_root_dir=$(realpath ${et_root_dir})
12+
setup_path_script=${et_root_dir}/examples/arm/ethos-u-scratch/setup_path.sh
13+
_setup_msg="please refer to ${et_root_dir}/examples/arm/setup.sh to properly install necessary tools."
14+
15+
build_type="Release"
16+
build_with_etdump=false
17+
extra_build_flags=""
18+
output_folder="cmake-out-vkml"
19+
20+
build_with_etdump_flags=" -DEXECUTORCH_ENABLE_EVENT_TRACER=OFF "
21+
22+
help() {
23+
echo "Usage: $(basename $0) [options]"
24+
echo "Options:"
25+
echo " --build_type=<TYPE> Build with Release, Debug or RelWithDebInfo, default is ${build_type}"
26+
echo " --etdump Adds Devtools etdump support to track timing, etdump area will be base64 encoded in the log"
27+
echo " --extra_build_flags=<FLAGS> Extra flags to pass to cmake. Default: none "
28+
echo " --output=<FOLDER> Output folder Default: $(output_folder)"
29+
exit 0
30+
}
31+
32+
for arg in "$@"; do
33+
case $arg in
34+
-h|--help) help ;;
35+
--build_type=*) build_type="${arg#*=}";;
36+
--etdump) build_with_etdump=true ;;
37+
--extra_build_flags=*) extra_build_flags="${arg#*=}";;
38+
--output=*) output_folder="${arg#*=}";;
39+
--select_ops_list=*) select_ops_list="${arg#*=}";;
40+
*)
41+
;;
42+
esac
43+
done
44+
45+
# Source the tools
46+
# This should be prepared by the setup.sh
47+
[[ -f ${setup_path_script} ]] \
48+
|| { echo "Missing ${setup_path_script}. ${_setup_msg}"; exit 1; }
49+
50+
source ${setup_path_script}
51+
52+
mkdir -p "${output_folder}"
53+
output_folder=$(realpath ${output_folder})
54+
55+
echo "--------------------------------------------------------------------------------"
56+
echo "Build Arm VKML executor runner: '${output_folder}' with extra build flags: ${extra_build_flags}"
57+
echo "--------------------------------------------------------------------------------"
58+
59+
cd ${et_root_dir}/examples/arm/executor_runner
60+
61+
if [ "$build_with_etdump" = true ] ; then
62+
build_with_etdump_flags=" -DEXECUTORCH_ENABLE_EVENT_TRACER=ON "
63+
fi
64+
65+
echo "Building with extra flags: ${build_with_etdump_flags} ${extra_build_flags}"
66+
cmake \
67+
-DCMAKE_BUILD_TYPE=${build_type} \
68+
-DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
69+
-DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
70+
-DEXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR=ON \
71+
-DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \
72+
-DEXECUTORCH_BUILD_XNNPACK=OFF \
73+
-DEXECUTORCH_BUILD_VULKAN=ON \
74+
-DEXECUTORCH_BUILD_VGF=ON \
75+
-DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON \
76+
-DEXECUTORCH_BUILD_KERNELS_QUANTIZED_AOT=ON \
77+
-DEXECUTORCH_ENABLE_LOGGING=ON \
78+
-DPYTHON_EXECUTABLE=$(which python3) \
79+
${extra_build_flags} \
80+
-B ${output_folder} ${et_root_dir}
81+
82+
echo "[${BASH_SOURCE[0]}] Configured CMAKE"
83+
84+
cmake --build ${output_folder} -j$(nproc)
85+
86+
echo "[${BASH_SOURCE[0]}] Built VKML runner: "
87+
find ${output_folder} -name "executor_runner"

backends/arm/scripts/run_vkml.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,14 +19,14 @@ _setup_msg="please refer to ${et_root_dir}/examples/arm/setup.sh to properly ins
1919

2020

2121
model=""
22-
build_path="cmake-out"
22+
build_path="cmake-out-vkml"
2323
converter="model-converter"
2424

2525
help() {
2626
echo "Usage: $(basename $0) [options]"
2727
echo "Options:"
2828
echo " --model=<MODEL_FILE> .pte model file to run"
29-
echo " --build=<BUILD_PATH> Target to build and run for Default: ${build_path}"
29+
echo " --build_path=<BUILD_PATH> Path to executor_runner build. for Default: ${build_path}"
3030
exit 0
3131
}
3232

backends/arm/test/ops/test_add.py

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77

88
from typing import Tuple
99

10-
import pytest
1110
import torch
1211
from executorch.backends.arm.quantizer import arm_quantizer
1312
from executorch.backends.arm.test import common, conftest
@@ -186,13 +185,13 @@ def test_add_tensor_u85_INT_2(test_data: input_t2):
186185
pipeline.run()
187186

188187

189-
@common.parametrize("test_data", Add.test_data)
188+
# TODO/MLETORCH-1282: remove once inputs are not hard coded to ones
189+
skip_keys = {"5d_float", "1d_ones", "1d_randn"}
190+
filtered_test_data = {k: v for k, v in Add.test_data.items() if k not in skip_keys}
191+
192+
193+
@common.parametrize("test_data", filtered_test_data)
190194
@common.SkipIfNoModelConverter
191-
@common.XfailfNoVKMLEmulationLayer
192-
@pytest.mark.xfail(
193-
reason="VGF runtime is not yet fully supported for FP pipeline (MLETORCH-1234)",
194-
strict=True,
195-
)
196195
def test_add_tensor_vgf_FP(test_data: input_t1):
197196
pipeline = VgfPipeline[input_t1](
198197
Add(),
@@ -205,7 +204,7 @@ def test_add_tensor_vgf_FP(test_data: input_t1):
205204
pipeline.run()
206205

207206

208-
@common.parametrize("test_data", Add.test_data)
207+
@common.parametrize("test_data", filtered_test_data)
209208
@common.SkipIfNoModelConverter
210209
def test_add_tensor_vgf_INT(test_data: input_t1):
211210
pipeline = VgfPipeline[input_t1](
@@ -214,5 +213,6 @@ def test_add_tensor_vgf_INT(test_data: input_t1):
214213
aten_op,
215214
exir_op,
216215
tosa_version="TOSA-1.0+INT",
216+
run_on_vulkan_runtime=True,
217217
)
218218
pipeline.run()

backends/arm/test/runner_utils.py

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -257,11 +257,21 @@ def run_vkml_emulation_layer(
257257
result_stdout = result.stdout.decode() # noqa: F841
258258
# TODO: MLETORCH-1234: Support VGF e2e tests in VgfPipeline
259259
# TODO: Add regex to check for error or fault messages in stdout from Emulation Layer
260-
# TODO: Retrieve and return the output tensors once VGF runtime is able to dump them.
261-
raise NotImplementedError(
262-
"Output parsing from VKML Emulation Layer is not yet implemented. "
260+
# Regex to extract tensor values from stdout
261+
output_np = []
262+
matches = re.findall(
263+
r"Output\s+\d+:\s+tensor\(sizes=\[(.*?)\],\s+\[(.*?)\]\)",
264+
result_stdout,
265+
re.DOTALL,
263266
)
264267

268+
for shape_str, values_str in matches:
269+
shape = list(map(int, shape_str.split(",")))
270+
values = list(map(float, re.findall(r"[-+]?\d*\.\d+|\d+", values_str)))
271+
output_np.append(torch.tensor(values).reshape(shape))
272+
273+
return tuple(output_np)
274+
265275

266276
def run_corstone(
267277
executorch_program_manager: ExecutorchProgramManager,
@@ -626,7 +636,8 @@ def vkml_emulation_layer_installed() -> bool:
626636
def assert_elf_path_exists(elf_path):
627637
if not os.path.exists(elf_path):
628638
raise FileNotFoundError(
629-
f"Did not find build arm_executor_runner or executor_runner in path {elf_path}, run setup_testing.sh?"
639+
f"Did not find build arm_executor_runner or executor_runner in path {elf_path}, \
640+
run setup_testing.sh or setup_testing_vkml.sh?"
630641
)
631642

632643

@@ -643,7 +654,7 @@ def get_elf_path(target_board):
643654
assert_elf_path_exists(elf_path)
644655
elif target_board == "vkml_emulation_layer":
645656
elf_path = os.path.join(
646-
"cmake-out",
657+
"arm_test/arm_executor_runner_vkml",
647658
"executor_runner",
648659
)
649660
assert_elf_path_exists(elf_path)

backends/arm/test/setup_testing.sh

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
#!/usr/bin/env bash
22
# Copyright 2024-2025 Arm Limited and/or its affiliates.
3-
# All rights reserved.
43
#
54
# This source code is licensed under the BSD-style license found in the
65
# LICENSE file in the root directory of this source tree.
@@ -13,4 +12,4 @@ build_executor_runner=${et_root_dir}/backends/arm/scripts/build_executor_runner.
1312
build_root_test_dir=${et_root_dir}/arm_test/arm_semihosting_executor_runner
1413

1514
${build_executor_runner} --pte=semihosting --target=ethos-u55-128 --output="${build_root_test_dir}_corstone-300"
16-
${build_executor_runner} --pte=semihosting --target=ethos-u85-128 --output="${build_root_test_dir}_corstone-320"
15+
${build_executor_runner} --pte=semihosting --target=ethos-u85-128 --output="${build_root_test_dir}_corstone-320"
Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#!/usr/bin/env bash
2+
# Copyright 2024-2025 Arm Limited and/or its affiliates.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
set -eu
8+
9+
script_dir=$(realpath "$(dirname "${BASH_SOURCE[0]}")")
10+
et_root_dir=$(realpath "${script_dir}/../../..")
11+
build_executor_runner=${et_root_dir}/backends/arm/scripts/build_executor_runner_vkml.sh
12+
build_root_test_dir=${et_root_dir}/arm_test/arm_executor_runner_vkml
13+
14+
${build_executor_runner} --output="${build_root_test_dir}"

0 commit comments

Comments
 (0)