Arm backend: Test TOSA, Ethos-U55 and Ethos-U85 on github

This will run some more models in the github test flow and
also enable some unit tests to use Corston3x0 FVP.

This structure up the Arm backend testing into separate runable
scripts in the same structure as other backends.

Signed-off-by: Zingo Andersen <[email protected]>
Change-Id: I5e11b1aca19460845e330b84d0696513c400c0f0

Author: zingo

.ci/scripts/setup-arm-baremetal-tools.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+# Copyright 2024 Arm Limited and/or its affiliates.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+# NB: This function could be used to install Arm dependencies
+# Setup arm example environment (including TOSA tools)
+git config --global user.email "[email protected]"
+git config --global user.name "Github Executorch"
+bash examples/arm/setup.sh --i-agree-to-the-contained-eula

.ci/scripts/utils.sh

@@ -59,17 +59,6 @@ install_flatc_from_source() {
   popd || return
 }
 
-install_arm() {
-  # NB: This function could be used to install Arm dependencies
-  # Setup arm example environment (including TOSA tools)
-  git config --global user.email "[email protected]"
-  git config --global user.name "Github Executorch"
-  bash examples/arm/setup.sh --i-agree-to-the-contained-eula
-
-  # Test tosa_reference flow
-  source examples/arm/ethos-u-scratch/setup_path.sh
-}
-
 build_executorch_runner_buck2() {
   # Build executorch runtime with retry as this step is flaky on macos CI
   retry buck2 build //examples/portable/executor_runner:executor_runner

.github/workflows/pull.yml

@@ -354,13 +354,11 @@ jobs:
         EXECUTORCH_BUILD_ARM_BAREMETAL=ON \
         .ci/scripts/setup-linux.sh "${BUILD_TOOL}"
 
-        source .ci/scripts/utils.sh
         # Install Arm dependencies
-        install_arm
-
-        # Run pytest with coverage
-        pytest -c /dev/null -v -n auto --cov=./ --cov-report=xml backends/arm/test
+        .ci/scripts/setup-arm-baremetal-tools.sh
 
+        # Run pytest without simulator
+        backends/arm/test/test_arm_baremetal.sh test_pytest
 
   test-llama-runner-qnn-linux:
     name: test-llama-runner-qnn-linux

.github/workflows/trunk.yml

@@ -146,14 +146,15 @@ jobs:
         source .ci/scripts/utils.sh
         install_executorch
 
-        install_arm
+        .ci/scripts/setup-arm-baremetal-tools.sh
 
         # Increase number of files user can monitor to bypass buck failures.
         # Hopefully this is high enough for this setup.
         sudo sysctl fs.inotify.max_user_watches=1048576 # 1024 * 1024
 
         # Test ethos-u delegate examples with run.sh
-        PYTHON_EXECUTABLE=python bash examples/arm/run.sh examples/arm/ethos-u-scratch/
+        backends/arm/test/test_arm_baremetal.sh test_run_ethosu_fvp
+
 
   test-arm-reference-delegation:
     name: test-arm-reference-delegation
@@ -172,10 +173,10 @@ jobs:
         source .ci/scripts/utils.sh
         install_executorch
 
-        install_arm
+        .ci/scripts/setup-arm-baremetal-tools.sh
 
-        # Run arm unit tests
-        pytest -c /dev/null -v -n auto --cov=./ --cov-report=xml backends/arm/test
+        # Run arm unit tests using the simulator
+        backends/arm/test/test_arm_baremetal.sh test_pytest_ethosu_fvp
 
   test-coreml-delegate:
     name: test-coreml-delegate

backends/arm/README.md

@@ -39,6 +39,28 @@ Other:
 - `third-party/` - Dependencies on other code - in particular the TOSA serialization_lib for compiling to TOSA and the ethos-u-core-driver for the bare-metal backend supporting Ethos-U
 - `test/` - Unit test and test support functions
 
+## Testing
+
+After a setup you can run unit tests with the test_arm_baremetal.sh script.
+
+To run the pytests suite run
+
+```
+backends/arm/test/test_arm_baremetal.sh test_pytest
+```
+
+To run the unit test suite with Corstone3x0 FVP simulator support use
+
+```
+backends/arm/test/test_arm_baremetal.sh test_pytest_ethosu_fvp
+```
+
+You can test to run some models with the run.sh flow
+
+```
+backends/arm/test/test_arm_baremetal.sh test_run_ethosu_fvp
+```
+
 ## Unit tests
 This is the structure of the test directory
 
@@ -51,6 +73,8 @@ test                            #  Root test folder
 ├── tester                      #  Arm Tester class
 ├── tosautil                    #  Utility functions for TOSA artifacts
 ├ common.py                     #  Common functions and definitions used by many tests
+├ setup_testing.sh              #  Script to prepare testing for using the Corstone 3x0 FVP
+├ test_arm_baremetal.sh         #  Help script to trigger testing
 ```
 
 Some example commands to run these tests follow. Run a single test:
@@ -59,6 +83,12 @@ Some example commands to run these tests follow. Run a single test:
 python -m unittest backends.arm.test.ops.test_add.TestSimpleAdd -k test_add2_tosa_BI
 ```
 
+or with pytest
+
+```
+pytest -c /dev/null -v -n auto backends/arm/test/ops/test_add.py -k test_add2_tosa_BI
+```
+
 Or all tests in "TestSimpleAdd":
 
 ```
@@ -71,6 +101,28 @@ Or discover and run many tests:
 python -m unittest discover -s backends/arm/test/ops/
 ```
 
+or with pytest
+
+```
+pytest -c /dev/null -v -n auto backends/arm/test/ops/
+```
+
+
+You can run tests using Corstone3x0 simulators to see how it would work on something more target like
+first you need to build and prepare some used target libs
+
+```
+examples/arm/run.sh --model_name=add --build_only
+backends/arm/test/setup_testing.sh
+```
+
+The you can run the tests with
+
+```
+pytest -c /dev/null -v -n auto backends/arm/test --arm_quantize_io --arm_run_corstoneFVP
+```
+
+
 ### A note on unit tests
 
 There are currently 3 ways we unit test our code.

backends/arm/test/runner_utils.py

@@ -178,7 +178,7 @@ def __init__(
         self.output_name: str = None
         self.qp_input: list[QuantizationParams] = None
         self.qp_output: QuantizationParams = None
-        self.timeout = 120
+        self.timeout = 480
         self.target_board: str = None
 
         self._has_init_run = False
@@ -316,7 +316,7 @@ def run_corstone(
         result = _run_cmd(command_args[self.target_board], check=False)
         if result.returncode != 0:
             raise RuntimeError(
-                f"Failed to run {command_args[self.target_board]}\nError: {result.stderr.decode()}"
+                f"Failed to run {command_args[self.target_board]}\nOutput:\n{result.stdout.decode()}\nError: {result.stderr.decode()}"
             )
         result_stdout = result.stdout.decode()
 

backends/arm/test/setup_testing.sh

@@ -14,7 +14,6 @@ ethos_u_root_dir=${et_root_dir}/examples/arm/ethos-u-scratch/ethos-u
 toolchain_cmake=${et_root_dir}/examples/arm/ethos-u-setup/arm-none-eabi-gcc.cmake
 et_build_dir=${et_root_dir}/cmake-out
 build_root_test_dir=${et_build_dir}/arm_semihosting_executor_runner
-fvp_model=FVP_Corstone_SSE-300_Ethos-U55
 
 # Build Arm Baremetal executor_runner in semihosting mode.
 # Put in backends/arm/test/res to be used by unit tests.

backends/arm/test/test_arm_baremetal.sh

@@ -0,0 +1,106 @@
+#!/bin/bash
+# Copyright 2024 Arm Limited and/or its affiliates.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -e
+
+script_dir=$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd)
+
+# Executorch root
+et_root_dir=$(cd ${script_dir}/../../.. && pwd)
+cd "${et_root_dir}"
+pwd
+
+
+TEST_SUITE=$1
+
+help() {
+  echo "Usage:"
+  echo " $0 <TESTNAME>"
+  echo " where <TESTNAME> can be any of:"
+  # This will list all lines in this file that is starting with test_ remove () { and print it as a list.
+  # e,g, "test_pytest() { # Test ops and other things" -> test_pytest # Test ops and other things
+  echo "all # run all tests"
+  grep "^test_" $0 | sed 's/([^)]*)[[:space:]]*{*//g'
+  exit
+}
+
+if [[ -z "${TEST_SUITE:-}" ]]; then
+  echo "Missing test suite name, exiting..."
+  help
+else
+  echo "Run Arm baremetal test suite ${TEST_SUITE}"
+fi
+
+TEST_SUITE_NAME="$(basename "$0") ${TEST_SUITE}"
+
+all() { # Run all tests
+  # This will list all lines in this file that is starting with test_ remove () { and add this script name in 
+  # front of it and execute it in a sub shell
+  # e.g. from this file:
+  #
+  # test_pytest() { # Test ops and other things
+  #  bla bla bla
+  # }
+  # test_pytest_ethosu_fvp() { # Same as test_pytest but ...
+  #  bla bla bla
+  # }
+  #...
+  # become a small script:
+  # ----
+  # backends/arm/test/test_arm_baremetal.sh test_pytest # Test ops and other things
+  # backends/arm/test/test_arm_baremetal.sh test_pytest_ethosu_fvp # Same as test_pytest but ...
+  # ...
+  # ----
+  # That is executed
+  echo "${TEST_SUITE_NAME}: Run all tests"
+  grep "^test_" backends/arm/test/test_arm_baremetal.sh | sed 's/([^)]*)[[:space:]]*{*//g' | sed "s|^|$0 |" | sh
+}
+
+test_pytest() { # Test ops and other things
+    echo "${TEST_SUITE_NAME}: Run pytest"
+    cd "${et_root_dir}"
+    source examples/arm/ethos-u-scratch/setup_path.sh
+
+    # Run arm baremetal pytest tests without FVP
+    pytest  --verbose --color=yes --numprocesses=auto backends/arm/test/
+}
+
+test_pytest_ethosu_fvp() { # Same as test_pytest but also sometime verify using Corstone FVP
+    echo "${TEST_SUITE_NAME}: Run pytest with fvp"
+
+    source examples/arm/ethos-u-scratch/setup_path.sh
+
+    # Prepare Corstone-3x0 FVP for pytest
+    examples/arm/run.sh --model_name=add --build_only
+    backends/arm/test/setup_testing.sh
+
+    # Run arm baremetal pytest tests with FVP
+    pytest  --verbose --color=yes --numprocesses=auto backends/arm/test/ --arm_quantize_io --arm_run_corstoneFVP
+}
+
+test_run_ethosu_fvp() { # End to End model tests
+    echo "${TEST_SUITE_NAME}: Test ethos-u delegate examples with run.sh"
+
+    source examples/arm/ethos-u-scratch/setup_path.sh
+
+    # TOSA quantized
+    echo "${TEST_SUITE_NAME}: Test ethos-u target TOSA"
+    examples/arm/run.sh --target=TOSA --model_name=mv2
+    examples/arm/run.sh --target=TOSA --model_name=lstm
+    examples/arm/run.sh --target=TOSA --model_name=edsr
+
+    # Ethos-U55
+    echo "${TEST_SUITE_NAME}: Test ethos-u target Ethos-U55"
+    examples/arm/run.sh --target=ethos-u55-128 --model_name=mv2
+    examples/arm/run.sh --target=ethos-u55-128 --model_name=lstm --reorder_inputs=1,0,2
+
+    # Ethos-U85
+    echo "${TEST_SUITE_NAME}: Test ethos-u target Ethos-U85"
+    examples/arm/run.sh --target=ethos-u85-128 --model_name=mv2
+    examples/arm/run.sh --target=ethos-u85-128 --model_name=lstm --reorder_inputs=1,0,2
+    }
+
+${TEST_SUITE}

backends/arm/test/tester/arm_tester.py

@@ -251,7 +251,7 @@ def to_executorch(self, to_executorch_stage: Optional[ToExecutorch] | None = Non
         return super().to_executorch(to_executorch_stage)
 
     def serialize(
-        self, serialize_stage: Optional[Serialize] = None, timeout: int = 120
+        self, serialize_stage: Optional[Serialize] = None, timeout: int = 480
     ):
         if serialize_stage is None:
             serialize_stage = Serialize(self.runner_util, timeout=timeout)