Addressing some comments

Author: BujSet

.ci/scripts/setup-arm-zephyr-tools.sh renamed to .ci/scripts/setup-arm-zephyr-x86-64-tools.sh

@@ -34,6 +34,25 @@ jobs:
         ${CONDA_RUN} cmake --preset ${{ matrix.preset }}
         ${CONDA_RUN} cmake --build cmake-out -j$(( $(sysctl -n hw.ncpu) - 1 ))
 
+  zephyr:
+    uses: pytorch/test-infra/.github/workflows/zephyr_job.yml@main
+    strategy:
+      fail-fast: false
+      matrix:
+        preset: [zephyr]
+    with:
+      job-name: build
+      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
+      runner: linux.2xlarge
+      docker-image: executorch-ubuntu-22.04-arm-sdk
+      python-version: 3.12
+      submodules: recursive
+      timeout: 90
+      script: |
+        set -eux
+        ${CONDA_RUN} ./install_requirements.sh > /dev/null
+        ${CONDA_RUN} cmake --preset ${{ matrix.preset }}
+        ${CONDA_RUN} cmake --build cmake-out -j$(( $(sysctl -n hw.ncpu) - 1 ))
   linux:
     uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
     strategy:

.github/workflows/build-presets.yml

@@ -223,6 +223,10 @@ jobs:
     permissions:
       id-token: write
       contents: read
+    strategy:
+      matrix:
+        os: [bare_metal, zephyr-preset]
+      fail-fast: false
     with:
       runner: linux.2xlarge
       docker-image: executorch-ubuntu-22.04-arm-sdk
@@ -234,91 +238,59 @@ jobs:
         CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
         conda activate "${CONDA_ENV}"
 
-        source .ci/scripts/utils.sh
-        install_executorch "--use-pt-pinned-commit"
-        .ci/scripts/setup-arm-baremetal-tools.sh
-        source examples/arm/ethos-u-scratch/setup_path.sh
-
-        # User baremetal toolchain
-        arm-none-eabi-c++ --version
-        toolchain_cmake=examples/arm/ethos-u-setup/arm-none-eabi-gcc.cmake
-        toolchain_cmake=$(realpath ${toolchain_cmake})
-
-        # Build and test size test
-        bash test/build_size_test.sh "-DCMAKE_TOOLCHAIN_FILE=${toolchain_cmake} -DEXECUTORCH_BUILD_ARM_BAREMETAL=ON"
-        elf="cmake-out/test/size_test"
-
-        # Dump basic info
-        ls -al ${elf}
-        arm-none-eabi-size ${elf}
-
-        # Dump symbols
-        python .github/scripts/run_nm.py -e ${elf}
-        python .github/scripts/run_nm.py -e ${elf} -f "executorch" -p "arm-none-eabi-"
-        python .github/scripts/run_nm.py -e ${elf} -f "executorch_text" -p "arm-none-eabi-"
-
-        # Add basic guard - TODO: refine this!
-        arm-none-eabi-strip ${elf}
-        output=$(ls -la ${elf})
-        arr=($output)
-        size=${arr[4]}
-        threshold="103268" # ~100KiB
-        echo "size: $size, threshold: $threshold"
-        if [[ "$size" -le "$threshold" ]]; then
-          echo "Success $size <= $threshold"
+        cxx_flags="-fno-exceptions -fno-rtti -Wall -Werror -Wno-int-in-bool-context -DET_HAVE_PREAD=0"
+        if [[ ${{ matrix.os}} == "bare_metal" ]]; then 
+          setup_script=.ci/scripts/setup-arm-baremetal-tools.sh
+          toolchain_prefic=arm-none-eabi-
+          threshold="103268" # ~100KiB
+        elif [[ ${{ matrix.os}} == "zephyr-preset" ]]; then 
+          setup_script=.ci/scripts/setup-arm-zephyr-x86-64-tools.sh
+          toolchain_prefic=arm-zephyr-eabi-
+          threshold="153600" # should be ~138KB, set threashold to 150KB
         else
-          echo "Fail $size > $threshold"
+          echo "Fail unsupport OS selection ${{ matrix.os }}"
           exit 1
         fi
 
-  test-arm-cortex-m-zephyr-size-test:
-    name: test-arm-cortex-m-zephyr-size-test
-    uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
-    permissions:
-      id-token: write
-      contents: read
-    with:
-      runner: linux.2xlarge
-      docker-image: executorch-ubuntu-22.04-arm-sdk
-      submodules: 'recursive'
-      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
-      timeout: 90
-      script: |
-        # The generic Linux job chooses to use base env, not the one setup by the image
-        CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
-        conda activate "${CONDA_ENV}"
-
         source .ci/scripts/utils.sh
         install_executorch "--use-pt-pinned-commit"
-        .ci/scripts/setup-arm-zephyr-tools.sh
+        ${setup_script}
         source examples/arm/ethos-u-scratch/setup_path.sh
 
-        # Use arm zephyr toolchain
-        arm-zephyr-eabi-c++ --version
-        zephyr_preset=tools/cmake/preset/zephyr.cmake
-        zephyr_preset=$(realpath ${zephyr_preset})
-        toolchain_cmake=examples/arm/ethos-u-setup/arm-zephyr-eabi-gcc.cmake
+        # User toolchain
+        ${toolchain_prefix}c++ --version
+        toolchain_cmake=examples/arm/ethos-u-setup/${toolchain_prefix}gcc.cmake
         toolchain_cmake=$(realpath ${toolchain_cmake})
 
         # Build and test size test
-        bash test/build_size_test.sh "-DEXECUTORCH_BUILD_PRESET_FILE=${zephyr_preset} -DCMAKE_TOOLCHAIN_FILE=${toolchain_cmake}"
+        if [[ ${{ matrix.os}} == "bare_metal" ]]; then 
+          bash test/build_size_test.sh "-DCMAKE_TOOLCHAIN_FILE=${toolchain_cmake} -DEXECUTORCH_BUILD_ARM_BAREMETAL=ON"
+        elif [[ ${{ matrix.os}} == "zephyr-preset" ]]; then 
+          CXXFLAGS=${cxx_flags} cmake --preset zephyr -DCMAKE_INSTALL_PREFIX=cmake-out -Bcmake-out .
+          cmake --build cmake-out -j9 --target install --config Release
+          CXXFLAGS=${cxx_flags}  cmake  -DCMAKE_TOOLCHAIN_FILE=${toolchain_cmake} -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=cmake-out -Bcmake-out/test test
+          cmake --build cmake-out/test -j9 --config Release
+        else
+          echo "Fail unsupport OS selection ${{ matrix.os }}"
+          exit 1
+        fi
+
         elf="cmake-out/test/size_test"
 
         # Dump basic info
         ls -al ${elf}
-        arm-zephyr-eabi-size ${elf}
+        ${toolchain_prefix}size ${elf}
 
         # Dump symbols
         python .github/scripts/run_nm.py -e ${elf}
-        python .github/scripts/run_nm.py -e ${elf} -f "executorch" -p "arm-zephyr-eabi-"
-        python .github/scripts/run_nm.py -e ${elf} -f "executorch_text" -p "arm-zephyr-eabi-"
+        python .github/scripts/run_nm.py -e ${elf} -f "executorch" -p "${toolchain_prefix}"
+        python .github/scripts/run_nm.py -e ${elf} -f "executorch_text" -p "${toolchain_prefix}"
 
         # Add basic guard - TODO: refine this!
-        arm-zephyr-eabi-strip ${elf}
+        ${toolchain_prefix}strip ${elf}
         output=$(ls -la ${elf})
         arr=($output)
         size=${arr[4]}
-        threshold="153600" # should be ~138KB, set threashold to 150KB
         echo "size: $size, threshold: $threshold"
         if [[ "$size" -le "$threshold" ]]; then
           echo "Success $size <= $threshold"

.github/workflows/trunk.yml

@@ -112,7 +112,8 @@
             "common"
         ],
         "cacheVariables": {
-            "EXECUTORCH_BUILD_PRESET_FILE": "${sourceDir}/tools/cmake/preset/zephyr.cmake"
+            "EXECUTORCH_BUILD_PRESET_FILE": "${sourceDir}/tools/cmake/preset/zephyr.cmake",
+            "CMAKE_TOOLCHAIN_FILE": "${sourceDir}/examples/arm/ethos-u-setup/arm-zephyr-eabi-gcc.cmake"
         }
     }
   ]

CMakePresets.json

@@ -46,7 +46,7 @@ help() {
     echo "  --output=<FOLDER>               Output folder Default: <MODEL>/<MODEL>_<TARGET INFO>.pte"
     echo "  --et_build_root=<FOLDER>        Build output root folder to use, defaults to ${et_build_root}"
     echo "  --ethosu_tools_dir=<FOLDER>     Path to your Ethos-U tools dir if you not using default: ${ethosu_tools_dir}"
-    echo "  --toolchain=<TOOLCHAIN>   Toolchain can be specified (e.g. bare metal as arm-none-eabi-gcc or zpehyr as arm-zephyr-eabi-gcc"
+    echo "  --toolchain=<TOOLCHAIN>         Toolchain can be specified (e.g. bare metal as arm-none-eabi-gcc or zephyr as arm-zephyr-eabi-gcc"
     exit 0
 }
 

backends/arm/scripts/build_executor_runner.sh

@@ -28,7 +28,7 @@ help() {
     echo "  --et_build_root=<FOLDER>   Build output root folder to use, defaults to ${et_build_root}"
     echo "  --build_type=<TYPE>        Build with Release, Debug or RelWithDebInfo, default is ${build_type}"
     echo "  --portable_kernels=<OPS>   Comma separated list of portable (non delagated) kernels to include Default: ${portable_kernels}"
-    echo "  --toolchain=<TOOLCHAIN>   Toolchain can be specified (e.g. bare metal as arm-none-eabi-gcc or zpehyr as arm-zephyr-eabi-gcc"
+    echo "  --toolchain=<TOOLCHAIN>    Toolchain can be specified (e.g. bare metal as arm-none-eabi-gcc or zephyr as arm-zephyr-eabi-gcc"
     exit 0
 }
 

backends/arm/scripts/build_portable_kernels.sh

@@ -234,6 +234,8 @@ function setup_toolchain() {
         selected_toolchain_url="${user_toolchain_url}"
         selected_toolchain_dir="${user_toolchain_dir}"
         selected_toolchain_md5=""
+	# TODO: For now we skip this if its a user specified url
+        # but eventually we should verify the checksums	
         skip_md5_verify=1
     fi
     if [[ ! -e "${selected_toolchain_dir}.tar.xz" ]]; then

examples/arm/setup.sh

@@ -23,15 +23,16 @@ cmake_install_executorch_lib() {
   update_tokenizers_git_submodule
   local EXTRA_BUILD_ARGS="${@}"
 
-  CXXFLAGS="$COMMON_CXXFLAGS" retry cmake -DBUCK2="$BUCK2" \
-          -DCMAKE_CXX_STANDARD_REQUIRED=ON \
-          -DCMAKE_INSTALL_PREFIX=cmake-out \
-          -DCMAKE_BUILD_TYPE=Release \
-          -DEXECUTORCH_BUILD_EXECUTOR_RUNNER=OFF \
-          -DEXECUTORCH_OPTIMIZE_SIZE=ON \
-          -DPYTHON_EXECUTABLE="$PYTHON_EXECUTABLE" \
-          ${EXTRA_BUILD_ARGS} \
-          -Bcmake-out .
+#  CXXFLAGS="$COMMON_CXXFLAGS" retry cmake -DBUCK2="$BUCK2" \
+#          -DCMAKE_CXX_STANDARD_REQUIRED=ON \
+#          -DCMAKE_INSTALL_PREFIX=cmake-out \
+#          -DCMAKE_BUILD_TYPE=Release \
+#          -DEXECUTORCH_BUILD_EXECUTOR_RUNNER=OFF \
+#          -DEXECUTORCH_OPTIMIZE_SIZE=ON \
+#          -DPYTHON_EXECUTABLE="$PYTHON_EXECUTABLE" \
+#          ${EXTRA_BUILD_ARGS} \
+#          -Bcmake-out .
+  cmake --preset zephyr ${EXTRA_BUILD_ARGS} -DCMAKE_INSTALL_PREFIX=cmake-out -Bcmake-out .
   cmake --build cmake-out -j9 --target install --config Release
 }
 
@@ -40,7 +41,7 @@ test_cmake_size_test() {
         -DCMAKE_INSTALL_PREFIX=cmake-out \
         ${EXTRA_BUILD_ARGS} \
         -Bcmake-out/test test
-
+#    cmake --preset zephyr ${EXTRA_BUILD_ARGS} -DCMAKE_BUILD_TYPE=Release  -DCMAKE_INSTALL_PREFIX=cmake-out -Bcmake-out/test test
     echo "Build size test"
     cmake --build cmake-out/test -j9 --config Release