pytorch · oscarandersson8218 · Feb 25, 2025 · Jan 27, 2025
@@ -16,18 +16,17 @@ et_root_dir=$(realpath ${et_root_dir})
 toolchain_cmake=${script_dir}/../../../examples/arm/ethos-u-setup/arm-none-eabi-gcc.cmake
 toolchain_cmake=$(realpath ${toolchain_cmake})
 
-
-
 et_build_root="${et_root_dir}/arm_test"
 build_type="Release"
+build_devtools=false
 build_with_etdump=false
 
-
 help() {
     echo "Usage: $(basename $0) [options]"
     echo "Options:"
     echo "  --et_build_root=<FOLDER>  Build output root folder to use, defaults to ${et_build_root}"
     echo "  --build_type=<TYPE>       Build with Release, Debug or RelWithDebInfo, default is ${build_type}"
+    echo "  --devtools                Build Devtools libs"
     echo "  --etdump                  Adds Devtools etdump support to track timing, etdump area will be base64 encoded in the log"
     exit 0
 }
@@ -37,32 +36,33 @@ for arg in "$@"; do
       -h|--help) help ;;
       --et_build_root=*) et_build_root="${arg#*=}";;
       --build_type=*) build_type="${arg#*=}";;
+      --devtools) build_devtools=true ;;
       --etdump) build_with_etdump=true ;;
       *)
       ;;
     esac
 done
 
 et_build_dir="${et_build_root}/cmake-out"
+
+# Used for flatcc host excutable if Devtools is used
 et_build_host_dir=${et_build_root}/cmake-out-host-tools
 
 set -x
 cd "${et_root_dir}"
 
-build_with_etdump_flags=""
 if [ "$build_with_etdump" = true ] ; then
     ( set +x ;
         echo "--------------------------------------------------------------------------------" ;
-        echo "Build ExecuTorch Libraries host flatcc bin ${build_type} into ${et_build_host_dir} - ${et_build_host_dir}/bin/flatcc" ;
+        echo "Build ExecuTorch Libraries host flatcc bin ${build_type} into ${et_build_host_dir}/bin/flatcc" ;
         echo "--------------------------------------------------------------------------------" )
 
-
     # Build host flatcc bin
     # This is a way to work around that the flatcc executable get build for target (e.g. Arm) later
     # and get replaced. flatcc is a tool used on the host for etdump and BundleIO handling.
     # The way to solve this is to generate it once for the host, then copy it to ${et_build_host_dir}/bin
     # and later point that out with -DFLATCC_EXECUTABLE=${et_build_host_dir}/bin/flatcc later.
-    mkdir -p ${et_build_host_dir}
+
     cmake                                                 \
         -DCMAKE_INSTALL_PREFIX=${et_build_host_dir}       \
         -DCMAKE_BUILD_TYPE=${build_type}                  \
@@ -79,25 +79,39 @@ if [ "$build_with_etdump" = true ] ; then
         -B"${et_build_host_dir}"                          \
         "${et_root_dir}"
 
-    # Copy host flatcc excutable to it's saved when we build for target (Arm) later
+    # third-party/flatcc/bin/flatcc gets build already in the in the cmake config step above
+    # so there is no cmake building step done
+
+    # Copy host flatcc excutable so it's saved when we build for target (Arm) later
+    et_build_host_dir=$(realpath ${et_build_host_dir})
     mkdir -p ${et_build_host_dir}/bin
     cp third-party/flatcc/bin/flatcc ${et_build_host_dir}/bin
-
-    # Add DevTools flags use in the Target build below
-    build_with_etdump_flags="-DEXECUTORCH_BUILD_DEVTOOLS=ON                    \
-                                -DEXECUTORCH_ENABLE_EVENT_TRACER=ON               \
-                                -DEXECUTORCH_SEPARATE_FLATCC_HOST_PROJECT=OFF     \
-                                -DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=OFF      \
-                                -DFLATCC_ALLOW_WERROR=OFF                         \
-                                -DFLATCC_EXECUTABLE=${et_build_host_dir}/bin/flatcc "
-    echo "build_with_etdump_flags=$build_with_etdump_flags"
 fi
 
 ( set +x ;
     echo "--------------------------------------------------------------------------------" ;
     echo "Build ExecuTorch target libs ${build_type} into '${et_build_dir}'" ;
     echo "--------------------------------------------------------------------------------" )
 
+build_devtools_flags=" -DEXECUTORCH_BUILD_DEVTOOLS=OFF "
+if [ "$build_devtools" = true ] ; then
+    build_devtools_flags=" -DEXECUTORCH_BUILD_DEVTOOLS=ON "
+fi
+
+build_with_etdump_flags=" -DEXECUTORCH_ENABLE_EVENT_TRACER=OFF "
+if [ "$build_with_etdump" = true ] ; then
+    # Add DevTools flags use in the Target build below
+    build_with_etdump_flags="-DEXECUTORCH_BUILD_DEVTOOLS=ON                    \
+                            -DEXECUTORCH_ENABLE_EVENT_TRACER=ON               \
+                            -DEXECUTORCH_SEPARATE_FLATCC_HOST_PROJECT=OFF     \
+                            -DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=OFF      \
+                            -DFLATCC_ALLOW_WERROR=OFF                         \
+                            -DFLATCC_EXECUTABLE=${et_build_host_dir}/bin/flatcc "
+fi
+
+echo "Building with Devtools: ${build_devtools_flags} ${build_with_etdump_flags}"
+
+
 # Build
 cmake                                                 \
     -DCMAKE_INSTALL_PREFIX=${et_build_dir}            \
@@ -108,6 +122,7 @@ cmake                                                 \
     -DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON           \
     -DEXECUTORCH_BUILD_EXTENSION_RUNNER_UTIL=ON       \
     -DEXECUTORCH_ENABLE_LOGGING=ON                    \
+    ${build_devtools_flags}                           \
     ${build_with_etdump_flags}                        \
     -DFLATC_EXECUTABLE="$(which flatc)"               \
     -B"${et_build_dir}"                               \

@@ -15,13 +15,17 @@ pte_file=""
 target="ethos-u55-128"
 build_type="Release"
 system_config=""
+bundleio=false
 build_with_etdump=false
 extra_build_flags=""
 output_folder_set=false
 output_folder="."
 et_build_root="${et_root_dir}/arm_test"
 ethosu_tools_dir=${et_root_dir}/examples/arm/ethos-u-scratch
 
+build_bundleio_flags=" -DET_BUNDLE_IO=OFF "
+build_with_etdump_flags=" -DEXECUTORCH_ENABLE_EVENT_TRACER=OFF "
+
 help() {
     echo "Usage: $(basename $0) [options]"
     echo "Options:"
@@ -30,6 +34,7 @@ help() {
     echo "  --build_type=<TYPE>             Build with Release, Debug or RelWithDebInfo, default is ${build_type}"
     echo "  --system_config=<CONFIG>        System configuration to select from the Vela configuration file (see vela.ini). Default: Ethos_U55_High_End_Embedded for EthosU55 targets, Ethos_U85_SYS_DRAM_Mid for EthosU85 targets."
     echo "                                     NOTE: If given, this option must match the given target. This option also sets timing adapter values customized for specific hardware, see ./executor_runner/CMakeLists.txt."
+    echo "  --bundleio                      Support both pte and Bundle IO bpte using Devtools BundelIO with Input/RefOutput included"
     echo "  --etdump                        Adds Devtools etdump support to track timing, etdump area will be base64 encoded in the log"
     echo "  --extra_build_flags=<FLAGS>     Extra flags to pass to cmake like -DET_ARM_BAREMETAL_METHOD_ALLOCATOR_POOL_SIZE=60000 Default: none "
     echo "  --output=<FOLDER>               Output folder Default: <MODEL>/<MODEL>_<TARGET INFO>.pte"
@@ -45,6 +50,7 @@ for arg in "$@"; do
       --target=*) target="${arg#*=}";;
       --build_type=*) build_type="${arg#*=}";;
       --system_config=*) system_config="${arg#*=}";;
+      --bundleio) bundleio=true ;;
       --etdump) build_with_etdump=true ;;
       --extra_build_flags=*) extra_build_flags="${arg#*=}";;
       --output=*) output_folder="${arg#*=}" ; output_folder_set=true ;;
@@ -64,9 +70,8 @@ et_build_dir=${et_build_root}/cmake-out
 et_build_dir=$(realpath ${et_build_dir})
 
 if [ "$output_folder_set" = false ] ; then
-    pte_folder=$(cd -- "$( dirname -- "${pte_file}" )" &> /dev/null && pwd)
-    pte_short_name=$(basename -- "${pte_file}" ".pte")
-    output_folder="$pte_folder/$pte_short_name"
+    # remove file ending
+    output_folder=${pte_file%.*}
 fi
 
 if [[ ${system_config} == "" ]]
@@ -86,28 +91,32 @@ else
     target_cpu=cortex-m85
 fi
 echo "--------------------------------------------------------------------------------"
-echo "Build Arm Baremetal executor_runner for ${target} with ${pte_file} using ${system_config} to '${output_folder}/cmake-out'"
+echo "Build Arm Baremetal executor_runner for ${target} with ${pte_file} using ${system_config} ${extra_build_flags} to '${output_folder}/cmake-out'"
 echo "--------------------------------------------------------------------------------"
 
 cd ${et_root_dir}/examples/arm/executor_runner
 
-build_with_etdump_flags=""
+if [ "$bundleio" = true ] ; then
+    build_bundleio_flags=" -DET_BUNDLE_IO=ON "
+fi
+
 if [ "$build_with_etdump" = true ] ; then
-    echo "Building with etdump e.g. -DEXECUTORCH_ENABLE_EVENT_TRACER=ON"
     build_with_etdump_flags=" -DEXECUTORCH_ENABLE_EVENT_TRACER=ON "
 fi
 
-mkdir -p "$output_folder"
+echo "Building with BundleIO/etdump/extra flags: ${build_bundleio_flags} ${build_with_etdump_flags} ${extra_build_flags}"
+mkdir -p "${output_folder}"
 
 cmake \
     -DCMAKE_BUILD_TYPE=${build_type}            \
     -DCMAKE_TOOLCHAIN_FILE=${toolchain_cmake}   \
     -DTARGET_CPU=${target_cpu}                  \
     -DET_DIR_PATH:PATH=${et_root_dir}           \
     -DET_BUILD_DIR_PATH:PATH=${et_build_dir}    \
-    -DET_PTE_FILE_PATH:PATH="${pte_file}"            \
+    -DET_PTE_FILE_PATH:PATH="${pte_file}"       \
     -DETHOS_SDK_PATH:PATH=${ethos_u_root_dir}   \
     -DETHOSU_TARGET_NPU_CONFIG=${target}        \
+    ${build_bundleio_flags}                     \
     ${build_with_etdump_flags}                  \
     -DPYTHON_EXECUTABLE=$(which python3)        \
     -DSYSTEM_CONFIG=${system_config}            \

@@ -19,12 +19,14 @@ _setup_msg="please refer to ${et_root_dir}/examples/arm/setup.sh to properly ins
 
 elf_file=""
 target="ethos-u55-128"
+timeout="240"
 
 help() {
     echo "Usage: $(basename $0) [options]"
     echo "Options:"
     echo "  --elf=<ELF_FILE>         elf file to run"
     echo "  --target=<TARGET>        Target to build and run for Default: ${target}"
+    echo "  --timeout=<TIME_IN_SEC>  Maximum target runtime, used to detect hanging, might need to be higer on large models Default: ${timeout}"
     exit 0
 }
 
@@ -33,6 +35,7 @@ for arg in "$@"; do
       -h|--help) help ;;
       --elf=*) elf_file="${arg#*=}";;
       --target=*) target="${arg#*=}";;
+      --timeout=*) timeout="${arg#*=}";;
       *)
       ;;
     esac
@@ -63,6 +66,7 @@ num_macs=$(echo ${target} | cut -d - -f 3)
 
 echo "--------------------------------------------------------------------------------"
 echo "Running ${elf_file} for ${target} run with FVP:${fvp_model} num_macs:${num_macs}"
+echo "WARNING: Corstone FVP is not cycle accurate and should NOT be used to determine valid runtime"
 echo "--------------------------------------------------------------------------------"
 
 log_file=$(mktemp)
@@ -75,7 +79,7 @@ if [[ ${target} == *"ethos-u55"*  ]]; then
         -C mps3_board.uart0.out_file='-'                    \
         -C mps3_board.uart0.shutdown_on_eot=1               \
         -a "${elf_file}"                                         \
-        --timelimit 220 2>&1 | tee ${log_file} || true # seconds
+        --timelimit ${timeout} 2>&1 | tee ${log_file} || true # seconds
     echo "[${BASH_SOURCE[0]}] Simulation complete, $?"
 elif [[ ${target} == *"ethos-u85"*  ]]; then
     ${fvp_model}                                            \
@@ -86,7 +90,7 @@ elif [[ ${target} == *"ethos-u85"*  ]]; then
         -C mps4_board.uart0.out_file='-'                    \
         -C mps4_board.uart0.shutdown_on_eot=1               \
         -a "${elf_file}"                                         \
-        --timelimit 220 2>&1 | tee ${log_file} || true # seconds
+        --timelimit ${timeout} 2>&1 | tee ${log_file} || true # seconds
     echo "[${BASH_SOURCE[0]}] Simulation complete, $?"
 else
     echo "Running ${elf_file} for ${target} is not supported"

@@ -92,18 +92,18 @@ test_run_ethosu_fvp() { # End to End model tests using run.sh
 
     # TOSA quantized
     echo "${TEST_SUITE_NAME}: Test ethos-u target TOSA"
-    examples/arm/run.sh --target=TOSA --model_name=add
-    examples/arm/run.sh --target=TOSA --model_name=mul
+    examples/arm/run.sh --et_build_root=arm_test/test_run --target=TOSA --model_name=add
+    examples/arm/run.sh --et_build_root=arm_test/test_run --target=TOSA --model_name=mul
 
     # Ethos-U55
     echo "${TEST_SUITE_NAME}: Test ethos-u target Ethos-U55"
-    examples/arm/run.sh --target=ethos-u55-128 --model_name=add
-    examples/arm/run.sh --target=ethos-u55-128 --model_name=mul
+    examples/arm/run.sh --et_build_root=arm_test/test_run --target=ethos-u55-128 --model_name=add
+    examples/arm/run.sh --et_build_root=arm_test/test_run --target=ethos-u55-128 --model_name=mul
 
     # Ethos-U85
     echo "${TEST_SUITE_NAME}: Test ethos-u target Ethos-U85"
-    examples/arm/run.sh --target=ethos-u85-128 --model_name=add
-    examples/arm/run.sh --target=ethos-u85-128 --model_name=mul
+    examples/arm/run.sh --et_build_root=arm_test/test_run --target=ethos-u85-128 --model_name=add
+    examples/arm/run.sh --et_build_root=arm_test/test_run --target=ethos-u85-128 --model_name=mul
     echo "${TEST_SUITE_NAME}: PASS"
     }
 
@@ -113,26 +113,26 @@ test_models_ethosu_fvp() { # End to End model tests using model_test.py
     source examples/arm/ethos-u-scratch/setup_path.sh
 
     # Build common libs once
-    python3 backends/arm/test/test_model.py --build_libs
+    python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --build_libs
 
     # TOSA quantized
     echo "${TEST_SUITE_NAME}: Test ethos-u target TOSA"
-    python3 backends/arm/test/test_model.py --target=TOSA --model=mv2
-    python3 backends/arm/test/test_model.py --target=TOSA --model=mv3
-    python3 backends/arm/test/test_model.py --target=TOSA --model=lstm
-    python3 backends/arm/test/test_model.py --target=TOSA --model=edsr
+    python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=TOSA --model=mv2
+    python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=TOSA --model=mv3
+    python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=TOSA --model=lstm
+    python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=TOSA --model=edsr
 
     # Ethos-U55
     echo "${TEST_SUITE_NAME}: Test ethos-u target Ethos-U55"
-    python3 backends/arm/test/test_model.py --target=ethos-u55-128 --model=mv2
-    python3 backends/arm/test/test_model.py --target=ethos-u55-64 --model=mv3
-    python3 backends/arm/test/test_model.py --target=ethos-u55-256 --model=lstm
+    python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=ethos-u55-128 --model=mv2  --extra_flags="-DET_ATOL=1.20 -DET_RTOL=1.20"
+    python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=ethos-u55-64  --model=mv3  --extra_flags="-DET_ATOL=5.00 -DET_RTOL=5.00"
+    python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=ethos-u55-256 --model=lstm --extra_flags="-DET_ATOL=0.02 -DET_RTOL=0.02"
 
     # Ethos-U85
     echo "${TEST_SUITE_NAME}: Test ethos-u target Ethos-U85"
-    python3 backends/arm/test/test_model.py --target=ethos-u85-256 --model=mv2
-    python3 backends/arm/test/test_model.py --target=ethos-u85-1024 --model=mv3
-    python3 backends/arm/test/test_model.py --target=ethos-u85-128 --model=lstm
+    python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=ethos-u85-256  --model=mv2  --extra_flags="-DET_ATOL=1.20 -DET_RTOL=1.20"
+    python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=ethos-u85-1024 --model=mv3  --extra_flags="-DET_ATOL=5.00 -DET_RTOL=5.00"
+    python3 backends/arm/test/test_model.py --test_output=arm_test/test_model --target=ethos-u85-128  --model=lstm --extra_flags="-DET_ATOL=0.02 -DET_RTOL=0.02"
     echo "${TEST_SUITE_NAME}: PASS"
     }
 
@@ -146,4 +146,4 @@ test_full_ethosu_fvp() { # All End to End model tests
 
 
 
-${TEST_SUITE}
+${TEST_SUITE}
@@ -56,7 +56,12 @@ def get_args():
         default=False,
         help="Don't save temporary files during compilation",
     )
-
+    parser.add_argument(
+        "--extra_flags",
+        required=False,
+        default=None,
+        help="Extra cmake flags to pass the when building the executor_runner",
+    )
     args = parser.parse_args()
 
     if args.model and "ethos-u" in args.target and args.system_config is None:
@@ -95,6 +100,8 @@ def build_libs(et_build_root: str, script_path: str):
             os.path.join(script_path, "build_executorch.sh"),
             f"--et_build_root={et_build_root}",
             "--build_type=Release",
+            "--devtools",
+            "--etdump",
         ]
     )
     run_external_cmd(
@@ -148,6 +155,7 @@ def build_pte(
             "examples.arm.aot_arm_compiler",
             "--delegate",
             "--quantize",
+            "--bundleio",
             intermediate,
             f"--model_name={model_name}",
             f"--target={target}",
@@ -158,7 +166,7 @@ def build_pte(
         ]
     )
 
-    pte_file = os.path.join(output, f"{model_name}_arm_delegate_{args.target}.pte")
+    pte_file = os.path.join(output, f"{model_name}_arm_delegate_{args.target}.bpte")
     return pte_file
 
 
@@ -168,17 +176,26 @@ def build_ethosu_runtime(
     pte_file: str,
     target: str,
     system_config: str,
+    extra_flags: str,
     elf_build_path: str,
 ):
+
+    extra_build_flag = ""
+    if extra_flags:
+        extra_build_flag = f"--extra_build_flags={extra_flags}"
+
     run_external_cmd(
         [
             "bash",
             os.path.join(script_path, "build_executorch_runner.sh"),
             f"--et_build_root={et_build_root}",
             f"--pte={pte_file}",
+            "--bundleio",
+            "--etdump",
             f"--target={target}",
             "--build_type=Release",
             f"--system_config={system_config}",
+            extra_build_flag,
             f"--output={elf_build_path}",
         ]
     )
@@ -239,6 +256,7 @@ def run_elf_with_fvp(script_path: str, elf_file: str, target: str):
                 pte_file,
                 args.target,
                 args.system_config,
+                args.extra_flags,
                 elf_build_path,
             )
             print(f"ELF file created: {elf_file} ")