From 564d89c7eb08e751a647ae617271c4b322df7a31 Mon Sep 17 00:00:00 2001 From: winskuo-quic Date: Thu, 4 Sep 2025 08:53:06 +0800 Subject: [PATCH 01/26] Fix --- .github/workflows/android-perf.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/android-perf.yml b/.github/workflows/android-perf.yml index b37478686a9..33937531a01 100644 --- a/.github/workflows/android-perf.yml +++ b/.github/workflows/android-perf.yml @@ -292,7 +292,7 @@ jobs: export.output_name="${OUT_ET_MODEL_NAME}.pte" ls -lh "${OUT_ET_MODEL_NAME}.pte" elif [[ ${{ matrix.config }} == "llama3_qnn_htp" ]]; then - export QNN_SDK_ROOT=/tmp/qnn/2.37.0.25072 + export QNN_SDK_ROOT=/tmp/qnn/2.37.0.250724 export LD_LIBRARY_PATH=$QNN_SDK_ROOT/lib/x86_64-linux-clang/ export PYTHONPATH=$(pwd)/.. @@ -432,7 +432,7 @@ jobs: PYTHON_EXECUTABLE=python bash .ci/scripts/build-qnn-sdk.sh mkdir -p aar-out - PYTHON_EXECUTABLE=python ANDROID_ABIS="arm64-v8a" BUILD_AAR_DIR=aar-out EXECUTORCH_BUILD_QNN=ON QNN_SDK_ROOT=/tmp/qnn/2.37.0.25072 EXECUTORCH_ANDROID_PROFILING=ON bash scripts/build_android_library.sh + PYTHON_EXECUTABLE=python ANDROID_ABIS="arm64-v8a" BUILD_AAR_DIR=aar-out EXECUTORCH_BUILD_QNN=ON QNN_SDK_ROOT=/tmp/qnn/2.37.0.250724 EXECUTORCH_ANDROID_PROFILING=ON bash scripts/build_android_library.sh mkdir -p extension/benchmark/android/benchmark/app/libs cp aar-out/executorch.aar extension/benchmark/android/benchmark/app/libs pushd extension/benchmark/android/benchmark From a247aede57ffe7e5a7e5ebc1360b8599cacdf95c Mon Sep 17 00:00:00 2001 From: winskuo-quic Date: Thu, 4 Sep 2025 09:41:34 +0800 Subject: [PATCH 02/26] Use bash --- .ci/scripts/test_backend_linux.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.ci/scripts/test_backend_linux.sh b/.ci/scripts/test_backend_linux.sh index 254d974160a..e8be3879171 100755 --- a/.ci/scripts/test_backend_linux.sh +++ b/.ci/scripts/test_backend_linux.sh @@ -30,10 +30,12 @@ if [[ "$FLOW" == *qnn* ]]; then # Qualcomm build. TODO (gjcomer) Clean this up once the QNN pybinding integration is # cleaned up. PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool cmake - PYTHON_EXECUTABLE=python bash .ci/scripts/setup-qnn-deps.sh + PYTHON_EXECUTABLE=python source "$(dirname "${BASH_SOURCE[0]}")/../../backends/qualcomm/scripts/install_qnn_sdk.sh" + setup_libcpp 12 + setup_android_ndk + install_qnn PYTHON_EXECUTABLE=python bash .ci/scripts/build-qnn-sdk.sh QNN_X86_LIB_DIR=`realpath build-x86/lib/` - QNN_SDK_ROOT="/tmp/qnn/2.28.0.241029" export LD_LIBRARY_PATH"=$QNN_X86_LIB_DIR:$QNN_SDK_ROOT/lib/x86_64-linux-clang/:${LD_LIBRARY_PATH:-}" # TODO Get SDK root from install scripts From dcd4c2eab9827af9d7a6c9974bdade5524675994 Mon Sep 17 00:00:00 2001 From: winskuo-quic Date: Mon, 8 Sep 2025 14:59:57 +0800 Subject: [PATCH 03/26] test will pull.yml --- .github/workflows/pull.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index 379d47716c9..46862f518b4 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -523,6 +523,29 @@ jobs: # Test llama2 PYTHON_EXECUTABLE=python bash .ci/scripts/test_llama.sh -model stories110M -build_tool "${BUILD_TOOL}" -mode "${MODE}" -dtype "${DTYPE}" -pt2e_quantize "${PT2E_QUANTIZE}" + backend-test-linux: + uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + strategy: + fail-fast: false + matrix: + flow: [ + qnn, qnn_16a16w, qnn_16a8w, qnn_16a4w, qnn_16a4w_block, qnn_8a8w, + vulkan, vulkan_static_int8_per_channel, + xnnpack, xnnpack_dynamic_int8_per_channel, xnnpack_static_int8_per_channel, xnnpack_static_int8_per_tensor + ] + suite: [models, operators] + with: + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + runner: linux.4xlarge.memory + docker-image: ci-image:executorch-ubuntu-22.04-clang12 + submodules: recursive + timeout: 120 + upload-artifact: test-report-${{ matrix.flow }}-${{ matrix.suite }} + script: | + set -eux + + source .ci/scripts/test_backend_linux.sh "${{ matrix.suite }}" "${{ matrix.flow }}" "${RUNNER_ARTIFACT_DIR}" + test-static-llama-qnn-linux: name: test-static-llama-qnn-linux uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main From 38400b38a2729756efd09ec4d94208155dc62e90 Mon Sep 17 00:00:00 2001 From: winskuo-quic Date: Mon, 8 Sep 2025 15:18:33 +0800 Subject: [PATCH 04/26] t --- .ci/scripts/test_backend_linux.sh | 5 +---- backends/qualcomm/scripts/install_qnn_sdk.sh | 4 ++++ 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.ci/scripts/test_backend_linux.sh b/.ci/scripts/test_backend_linux.sh index e8be3879171..30441464eee 100755 --- a/.ci/scripts/test_backend_linux.sh +++ b/.ci/scripts/test_backend_linux.sh @@ -30,10 +30,7 @@ if [[ "$FLOW" == *qnn* ]]; then # Qualcomm build. TODO (gjcomer) Clean this up once the QNN pybinding integration is # cleaned up. PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool cmake - PYTHON_EXECUTABLE=python source "$(dirname "${BASH_SOURCE[0]}")/../../backends/qualcomm/scripts/install_qnn_sdk.sh" - setup_libcpp 12 - setup_android_ndk - install_qnn + PYTHON_EXECUTABLE=python bash "$(dirname "${BASH_SOURCE[0]}")/../../backends/qualcomm/scripts/install_qnn_sdk.sh" PYTHON_EXECUTABLE=python bash .ci/scripts/build-qnn-sdk.sh QNN_X86_LIB_DIR=`realpath build-x86/lib/` export LD_LIBRARY_PATH"=$QNN_X86_LIB_DIR:$QNN_SDK_ROOT/lib/x86_64-linux-clang/:${LD_LIBRARY_PATH:-}" diff --git a/backends/qualcomm/scripts/install_qnn_sdk.sh b/backends/qualcomm/scripts/install_qnn_sdk.sh index a97d4258770..c814b83430a 100644 --- a/backends/qualcomm/scripts/install_qnn_sdk.sh +++ b/backends/qualcomm/scripts/install_qnn_sdk.sh @@ -153,3 +153,7 @@ setup_libcpp() { echo "libc++ installed to ${INSTALL_DIR}" } + +setup_libcpp 12 +setup_android_ndk +install_qnn From 2a909da5046d3d3f20c08c9bd95f64840f4ae32f Mon Sep 17 00:00:00 2001 From: winskuo-quic Date: Mon, 8 Sep 2025 15:37:19 +0800 Subject: [PATCH 05/26] s --- .ci/scripts/setup-qnn-deps.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.ci/scripts/setup-qnn-deps.sh b/.ci/scripts/setup-qnn-deps.sh index 9af688d5cb7..f545cf726e6 100644 --- a/.ci/scripts/setup-qnn-deps.sh +++ b/.ci/scripts/setup-qnn-deps.sh @@ -7,8 +7,7 @@ set -ex -source "$(dirname "${BASH_SOURCE[0]}")/../../backends/qualcomm/scripts/install_qnn_sdk.sh" +bash "$(dirname "${BASH_SOURCE[0]}")/../../backends/qualcomm/scripts/install_qnn_sdk.sh" setup_libcpp 12 -setup_android_ndk install_qnn From 24068d18d012f97a47f63740480c52ed5c5ad05f Mon Sep 17 00:00:00 2001 From: winskuo-quic Date: Mon, 8 Sep 2025 15:50:02 +0800 Subject: [PATCH 06/26] Test --- .ci/scripts/setup-qnn-deps.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/.ci/scripts/setup-qnn-deps.sh b/.ci/scripts/setup-qnn-deps.sh index f545cf726e6..0c7e418c7ca 100644 --- a/.ci/scripts/setup-qnn-deps.sh +++ b/.ci/scripts/setup-qnn-deps.sh @@ -8,6 +8,3 @@ set -ex bash "$(dirname "${BASH_SOURCE[0]}")/../../backends/qualcomm/scripts/install_qnn_sdk.sh" - -setup_libcpp 12 -install_qnn From f090ffd38db278720e3112901e3ce04739fc8ef9 Mon Sep 17 00:00:00 2001 From: winskuo-quic Date: Mon, 8 Sep 2025 16:12:47 +0800 Subject: [PATCH 07/26] q --- .ci/scripts/setup-qnn-deps.sh | 2 ++ backends/qualcomm/scripts/install_qnn_sdk.sh | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.ci/scripts/setup-qnn-deps.sh b/.ci/scripts/setup-qnn-deps.sh index 0c7e418c7ca..039467b12a1 100644 --- a/.ci/scripts/setup-qnn-deps.sh +++ b/.ci/scripts/setup-qnn-deps.sh @@ -8,3 +8,5 @@ set -ex bash "$(dirname "${BASH_SOURCE[0]}")/../../backends/qualcomm/scripts/install_qnn_sdk.sh" + +setup_libcpp 12 \ No newline at end of file diff --git a/backends/qualcomm/scripts/install_qnn_sdk.sh b/backends/qualcomm/scripts/install_qnn_sdk.sh index c814b83430a..7f8c461bcc6 100644 --- a/backends/qualcomm/scripts/install_qnn_sdk.sh +++ b/backends/qualcomm/scripts/install_qnn_sdk.sh @@ -154,6 +154,5 @@ setup_libcpp() { echo "libc++ installed to ${INSTALL_DIR}" } -setup_libcpp 12 setup_android_ndk install_qnn From 40601a72afeccd754d15e8e22e8954b1735099bd Mon Sep 17 00:00:00 2001 From: winskuo-quic Date: Mon, 8 Sep 2025 16:23:42 +0800 Subject: [PATCH 08/26] Again --- .ci/scripts/setup-qnn-deps.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.ci/scripts/setup-qnn-deps.sh b/.ci/scripts/setup-qnn-deps.sh index 039467b12a1..a465c2fe287 100644 --- a/.ci/scripts/setup-qnn-deps.sh +++ b/.ci/scripts/setup-qnn-deps.sh @@ -7,6 +7,5 @@ set -ex -bash "$(dirname "${BASH_SOURCE[0]}")/../../backends/qualcomm/scripts/install_qnn_sdk.sh" - +source "$(dirname "${BASH_SOURCE[0]}")/../../backends/qualcomm/scripts/install_qnn_sdk.sh" setup_libcpp 12 \ No newline at end of file From 7a31c2d78615f50c2f13d918b060c5db18c9f291 Mon Sep 17 00:00:00 2001 From: winskuo-quic Date: Mon, 8 Sep 2025 16:46:28 +0800 Subject: [PATCH 09/26] Again --- .ci/scripts/test_backend_linux.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/scripts/test_backend_linux.sh b/.ci/scripts/test_backend_linux.sh index 30441464eee..abe0e8456e5 100755 --- a/.ci/scripts/test_backend_linux.sh +++ b/.ci/scripts/test_backend_linux.sh @@ -30,7 +30,7 @@ if [[ "$FLOW" == *qnn* ]]; then # Qualcomm build. TODO (gjcomer) Clean this up once the QNN pybinding integration is # cleaned up. PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool cmake - PYTHON_EXECUTABLE=python bash "$(dirname "${BASH_SOURCE[0]}")/../../backends/qualcomm/scripts/install_qnn_sdk.sh" + PYTHON_EXECUTABLE=python source .ci/scripts/setup-qnn-deps.sh PYTHON_EXECUTABLE=python bash .ci/scripts/build-qnn-sdk.sh QNN_X86_LIB_DIR=`realpath build-x86/lib/` export LD_LIBRARY_PATH"=$QNN_X86_LIB_DIR:$QNN_SDK_ROOT/lib/x86_64-linux-clang/:${LD_LIBRARY_PATH:-}" From f7144a330fde0f18108472d90239a2cf092264a4 Mon Sep 17 00:00:00 2001 From: winskuo-quic Date: Mon, 8 Sep 2025 17:06:06 +0800 Subject: [PATCH 10/26] Again --- .ci/scripts/setup-qnn-deps.sh | 3 +-- backends/qualcomm/scripts/install_qnn_sdk.sh | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/scripts/setup-qnn-deps.sh b/.ci/scripts/setup-qnn-deps.sh index a465c2fe287..5aec27fe86a 100644 --- a/.ci/scripts/setup-qnn-deps.sh +++ b/.ci/scripts/setup-qnn-deps.sh @@ -7,5 +7,4 @@ set -ex -source "$(dirname "${BASH_SOURCE[0]}")/../../backends/qualcomm/scripts/install_qnn_sdk.sh" -setup_libcpp 12 \ No newline at end of file +source "$(dirname "${BASH_SOURCE[0]}")/../../backends/qualcomm/scripts/install_qnn_sdk.sh" \ No newline at end of file diff --git a/backends/qualcomm/scripts/install_qnn_sdk.sh b/backends/qualcomm/scripts/install_qnn_sdk.sh index 7f8c461bcc6..c814b83430a 100644 --- a/backends/qualcomm/scripts/install_qnn_sdk.sh +++ b/backends/qualcomm/scripts/install_qnn_sdk.sh @@ -154,5 +154,6 @@ setup_libcpp() { echo "libc++ installed to ${INSTALL_DIR}" } +setup_libcpp 12 setup_android_ndk install_qnn From 8e9b3c23dd5815484e485153df7845d24015ffa3 Mon Sep 17 00:00:00 2001 From: winskuo-quic Date: Mon, 8 Sep 2025 17:19:35 +0800 Subject: [PATCH 11/26] Again --- .ci/scripts/test_backend_linux.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/scripts/test_backend_linux.sh b/.ci/scripts/test_backend_linux.sh index abe0e8456e5..24474cccf9d 100755 --- a/.ci/scripts/test_backend_linux.sh +++ b/.ci/scripts/test_backend_linux.sh @@ -30,7 +30,7 @@ if [[ "$FLOW" == *qnn* ]]; then # Qualcomm build. TODO (gjcomer) Clean this up once the QNN pybinding integration is # cleaned up. PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool cmake - PYTHON_EXECUTABLE=python source .ci/scripts/setup-qnn-deps.sh + PYTHON_EXECUTABLE=python source "$(dirname "${BASH_SOURCE[0]}")/../../backends/qualcomm/scripts/install_qnn_sdk.sh" PYTHON_EXECUTABLE=python bash .ci/scripts/build-qnn-sdk.sh QNN_X86_LIB_DIR=`realpath build-x86/lib/` export LD_LIBRARY_PATH"=$QNN_X86_LIB_DIR:$QNN_SDK_ROOT/lib/x86_64-linux-clang/:${LD_LIBRARY_PATH:-}" From 8ed0564ea2405f69572c4e9ef1788706980e6c14 Mon Sep 17 00:00:00 2001 From: winskuo-quic Date: Mon, 8 Sep 2025 17:39:23 +0800 Subject: [PATCH 12/26] Again --- .github/workflows/trunk.yml | 184 ++++++++++++++++++------------------ 1 file changed, 92 insertions(+), 92 deletions(-) diff --git a/.github/workflows/trunk.yml b/.github/workflows/trunk.yml index 251bb238f1b..214d2775b41 100644 --- a/.github/workflows/trunk.yml +++ b/.github/workflows/trunk.yml @@ -18,42 +18,42 @@ concurrency: cancel-in-progress: true jobs: - test-models-macos-cpu: - name: test-models-macos-cpu - uses: pytorch/test-infra/.github/workflows/macos_job.yml@main - strategy: - matrix: - # Mac runners are expensive and limited, and non reliable. - # Do some basic testing for macos jobs, and rely mostly on - # test-models-linux-aarch64 job instead. - model: [emformer_join, ic4, llama2, mobilebert, mv3, resnet50, vit, w2l] - backend: [xnnpack-quantization-delegation] - include: - - model: efficient_sam - backend: portable - - model: llama - backend: portable - - model: llama3_2_vision_encoder - backend: portable - - model: mv3 - backend: portable - fail-fast: false - with: - runner: macos-m1-stable - python-version: '3.11' - submodules: 'recursive' - ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} - timeout: 90 - script: | - MODEL_NAME=${{ matrix.model }} - BUILD_TOOL=cmake - BACKEND=${{ matrix.backend }} + # test-models-macos-cpu: + # name: test-models-macos-cpu + # uses: pytorch/test-infra/.github/workflows/macos_job.yml@main + # strategy: + # matrix: + # # Mac runners are expensive and limited, and non reliable. + # # Do some basic testing for macos jobs, and rely mostly on + # # test-models-linux-aarch64 job instead. + # model: [emformer_join, ic4, llama2, mobilebert, mv3, resnet50, vit, w2l] + # backend: [xnnpack-quantization-delegation] + # include: + # - model: efficient_sam + # backend: portable + # - model: llama + # backend: portable + # - model: llama3_2_vision_encoder + # backend: portable + # - model: mv3 + # backend: portable + # fail-fast: false + # with: + # runner: macos-m1-stable + # python-version: '3.11' + # submodules: 'recursive' + # ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + # timeout: 90 + # script: | + # MODEL_NAME=${{ matrix.model }} + # BUILD_TOOL=cmake + # BACKEND=${{ matrix.backend }} - bash .ci/scripts/setup-conda.sh - # Setup MacOS dependencies as there is no Docker support on MacOS atm - PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/setup-macos.sh --build-tool "${BUILD_TOOL}" - # Build and test executorch - PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/test_model.sh "${MODEL_NAME}" "${BUILD_TOOL}" "${BACKEND}" + # bash .ci/scripts/setup-conda.sh + # # Setup MacOS dependencies as there is no Docker support on MacOS atm + # PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/setup-macos.sh --build-tool "${BUILD_TOOL}" + # # Build and test executorch + # PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/test_model.sh "${MODEL_NAME}" "${BUILD_TOOL}" "${BACKEND}" # test-models-arm-zephyr: # name: test-models-arm-zephyr @@ -666,71 +666,71 @@ jobs: PYTHON_EXECUTABLE=python bash .ci/scripts/build-qnn-sdk.sh PYTHON_EXECUTABLE=python bash .ci/scripts/test_model.sh ${{ matrix.model }} "cmake" "qnn" - test-models-macos-coreml: - name: test-models-macos-coreml - uses: pytorch/test-infra/.github/workflows/macos_job.yml@main - strategy: - matrix: - model: [dl3, edsr, efficient_sam, emformer_join, emformer_transcribe, ic3, ic4, mobilebert, mv2, mv3, resnet50, vit, w2l] - fail-fast: false - with: - runner: macos-m1-stable - python-version: '3.11' - submodules: 'recursive' - ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} - timeout: 90 - script: | - MODEL_NAME=${{ matrix.model }} - BUILD_TOOL=cmake - BACKEND="coreml-pybind" - + # test-models-macos-coreml: + # name: test-models-macos-coreml + # uses: pytorch/test-infra/.github/workflows/macos_job.yml@main + # strategy: + # matrix: + # model: [dl3, edsr, efficient_sam, emformer_join, emformer_transcribe, ic3, ic4, mobilebert, mv2, mv3, resnet50, vit, w2l] + # fail-fast: false + # with: + # runner: macos-m1-stable + # python-version: '3.11' + # submodules: 'recursive' + # ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + # timeout: 90 + # script: | + # MODEL_NAME=${{ matrix.model }} + # BUILD_TOOL=cmake + # BACKEND="coreml-pybind" - # Set model specific overrides - if [[ "${MODEL_NAME}" == "mobilebert" ]]; then - # See https://github.com/pytorch/executorch/issues/12907 - # mobilebert has nan output on FP16, and high MSE on fp32, so we disable runtime test now - BACKEND="coreml" - fi - if [[ "${MODEL_NAME}" == "efficient_sam" ]]; then - # See https://github.com/pytorch/executorch/issues/12906 - # efficient_sam fails to run on CoreML - BACKEND="coreml" - fi + # # Set model specific overrides + # if [[ "${MODEL_NAME}" == "mobilebert" ]]; then + # # See https://github.com/pytorch/executorch/issues/12907 + # # mobilebert has nan output on FP16, and high MSE on fp32, so we disable runtime test now + # BACKEND="coreml" + # fi - bash .ci/scripts/setup-conda.sh + # if [[ "${MODEL_NAME}" == "efficient_sam" ]]; then + # # See https://github.com/pytorch/executorch/issues/12906 + # # efficient_sam fails to run on CoreML + # BACKEND="coreml" + # fi - # Setup MacOS dependencies as there is no Docker support on MacOS atm - PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/setup-macos.sh --build-tool "${BUILD_TOOL}" - PYTHON_EXECUTABLE=python ${CONDA_RUN} bash backends/apple/coreml/scripts/install_requirements.sh - echo "Finishing installing coreml." + # bash .ci/scripts/setup-conda.sh - PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/test_model.sh "${MODEL_NAME}" "${BUILD_TOOL}" "${BACKEND}" + # # Setup MacOS dependencies as there is no Docker support on MacOS atm + # PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/setup-macos.sh --build-tool "${BUILD_TOOL}" + # PYTHON_EXECUTABLE=python ${CONDA_RUN} bash backends/apple/coreml/scripts/install_requirements.sh + # echo "Finishing installing coreml." - test-models-macos-mps: - name: test-models-macos-mps - uses: pytorch/test-infra/.github/workflows/macos_job.yml@main - strategy: - fail-fast: false - with: - runner: macos-m1-stable - python-version: '3.11' - submodules: 'recursive' - ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} - timeout: 90 - script: | - BUILD_TOOL=cmake - bash .ci/scripts/setup-conda.sh + # PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/test_model.sh "${MODEL_NAME}" "${BUILD_TOOL}" "${BACKEND}" - # Setup MacOS dependencies as there is no Docker support on MacOS atm - PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/setup-macos.sh --build-tool "${BUILD_TOOL}" + # test-models-macos-mps: + # name: test-models-macos-mps + # uses: pytorch/test-infra/.github/workflows/macos_job.yml@main + # strategy: + # fail-fast: false + # with: + # runner: macos-m1-stable + # python-version: '3.11' + # submodules: 'recursive' + # ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + # timeout: 90 + # script: | + # BUILD_TOOL=cmake + # bash .ci/scripts/setup-conda.sh - # Build and test mps model - for MODEL_NAME in mv3 ic4 resnet50 edsr mobilebert w2l; do - echo "::group::Exporting mps model: $MODEL_NAME" - PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/test_model.sh "${MODEL_NAME}" "${BUILD_TOOL}" "mps" - echo "::endgroup::" - done + # # Setup MacOS dependencies as there is no Docker support on MacOS atm + # PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/setup-macos.sh --build-tool "${BUILD_TOOL}" + + # # Build and test mps model + # for MODEL_NAME in mv3 ic4 resnet50 edsr mobilebert w2l; do + # echo "::group::Exporting mps model: $MODEL_NAME" + # PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/test_model.sh "${MODEL_NAME}" "${BUILD_TOOL}" "mps" + # echo "::endgroup::" + # done test-huggingface-transformers-xnnpack: # NB: Don't run this on fork PRs because they won't have access to the secret and would fail anyway From 5cf16de1a697937fcf06dedd52cd98834fc29166 Mon Sep 17 00:00:00 2001 From: winskuo-quic Date: Mon, 8 Sep 2025 17:39:33 +0800 Subject: [PATCH 13/26] Again --- backends/qualcomm/scripts/install_qnn_sdk.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/backends/qualcomm/scripts/install_qnn_sdk.sh b/backends/qualcomm/scripts/install_qnn_sdk.sh index c814b83430a..59e85052e71 100644 --- a/backends/qualcomm/scripts/install_qnn_sdk.sh +++ b/backends/qualcomm/scripts/install_qnn_sdk.sh @@ -94,15 +94,15 @@ setup_libcpp() { LLVM_VERSION="14.0.0" INSTALL_DIR="/tmp/libcxx-${LLVM_VERSION}" - # Check if we already have a local installation - if [ -d "${INSTALL_DIR}/include" ] && [ -d "${INSTALL_DIR}/lib" ]; then - echo "Local libc++ already installed at ${INSTALL_DIR} - skipping" - # Set environment variables - export CPLUS_INCLUDE_PATH="${INSTALL_DIR}/include:$CPLUS_INCLUDE_PATH" - export LD_LIBRARY_PATH="${INSTALL_DIR}/lib:$LD_LIBRARY_PATH" - export LIBRARY_PATH="${INSTALL_DIR}/lib:$LIBRARY_PATH" - return - fi + # # Check if we already have a local installation + # if [ -d "${INSTALL_DIR}/include" ] && [ -d "${INSTALL_DIR}/lib" ]; then + # echo "Local libc++ already installed at ${INSTALL_DIR} - skipping" + # # Set environment variables + # export CPLUS_INCLUDE_PATH="${INSTALL_DIR}/include:$CPLUS_INCLUDE_PATH" + # export LD_LIBRARY_PATH="${INSTALL_DIR}/lib:$LD_LIBRARY_PATH" + # export LIBRARY_PATH="${INSTALL_DIR}/lib:$LIBRARY_PATH" + # return + # fi echo "Installing libc++ manually to ${INSTALL_DIR}" From 8c3dccdc1a744a2eec0b650415b954f64276df4d Mon Sep 17 00:00:00 2001 From: winskuo-quic Date: Mon, 8 Sep 2025 17:55:11 +0800 Subject: [PATCH 14/26] Again --- .ci/scripts/test_backend_linux.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/scripts/test_backend_linux.sh b/.ci/scripts/test_backend_linux.sh index 24474cccf9d..c7bbe7b9d29 100755 --- a/.ci/scripts/test_backend_linux.sh +++ b/.ci/scripts/test_backend_linux.sh @@ -30,7 +30,7 @@ if [[ "$FLOW" == *qnn* ]]; then # Qualcomm build. TODO (gjcomer) Clean this up once the QNN pybinding integration is # cleaned up. PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool cmake - PYTHON_EXECUTABLE=python source "$(dirname "${BASH_SOURCE[0]}")/../../backends/qualcomm/scripts/install_qnn_sdk.sh" + PYTHON_EXECUTABLE=python bash .ci/scripts/setup-qnn-deps.sh PYTHON_EXECUTABLE=python bash .ci/scripts/build-qnn-sdk.sh QNN_X86_LIB_DIR=`realpath build-x86/lib/` export LD_LIBRARY_PATH"=$QNN_X86_LIB_DIR:$QNN_SDK_ROOT/lib/x86_64-linux-clang/:${LD_LIBRARY_PATH:-}" From 67b664f6b54e417d1f5868708ff7590dee2d62b2 Mon Sep 17 00:00:00 2001 From: winskuo-quic Date: Mon, 8 Sep 2025 19:37:41 +0800 Subject: [PATCH 15/26] Debugging --- .ci/scripts/setup-qnn-deps.sh | 5 ++++- .ci/scripts/test_backend_linux.sh | 1 + .ci/scripts/test_qnn_static_llama.sh | 1 + 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.ci/scripts/setup-qnn-deps.sh b/.ci/scripts/setup-qnn-deps.sh index 5aec27fe86a..02df93769a4 100644 --- a/.ci/scripts/setup-qnn-deps.sh +++ b/.ci/scripts/setup-qnn-deps.sh @@ -7,4 +7,7 @@ set -ex -source "$(dirname "${BASH_SOURCE[0]}")/../../backends/qualcomm/scripts/install_qnn_sdk.sh" \ No newline at end of file +source "$(dirname "${BASH_SOURCE[0]}")/../../backends/qualcomm/scripts/install_qnn_sdk.sh" + +setup_libcpp 12 +install_qnn \ No newline at end of file diff --git a/.ci/scripts/test_backend_linux.sh b/.ci/scripts/test_backend_linux.sh index c7bbe7b9d29..b5830fdb82c 100755 --- a/.ci/scripts/test_backend_linux.sh +++ b/.ci/scripts/test_backend_linux.sh @@ -29,6 +29,7 @@ if [[ "$FLOW" == *qnn* ]]; then # Setup QNN sdk and deps - note that this is a bit hacky due to the nature of the # Qualcomm build. TODO (gjcomer) Clean this up once the QNN pybinding integration is # cleaned up. + echo "---------------wk----------" PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool cmake PYTHON_EXECUTABLE=python bash .ci/scripts/setup-qnn-deps.sh PYTHON_EXECUTABLE=python bash .ci/scripts/build-qnn-sdk.sh diff --git a/.ci/scripts/test_qnn_static_llama.sh b/.ci/scripts/test_qnn_static_llama.sh index 0a708fea76a..9c24070d115 100644 --- a/.ci/scripts/test_qnn_static_llama.sh +++ b/.ci/scripts/test_qnn_static_llama.sh @@ -14,6 +14,7 @@ source "$(dirname "${BASH_SOURCE[0]}")/../../backends/qualcomm/scripts/qnn_confi # Download QNN_SDK. If already downloaded, export environment path source "$(dirname "${BASH_SOURCE[0]}")/../../backends/qualcomm/scripts/install_qnn_sdk.sh" install_qnn +echo "WK checking LIBC=${CPLUS_INCLUDE_PATH}" export EXECUTORCH_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" export LD_LIBRARY_PATH="${QNN_SDK_ROOT}/lib/x86_64-linux-clang" From 7933b29c2d1e777295cd3f2d387ac872a18e006d Mon Sep 17 00:00:00 2001 From: winskuo-quic Date: Mon, 8 Sep 2025 19:58:40 +0800 Subject: [PATCH 16/26] Debugging --- .ci/scripts/test_backend_linux.sh | 2 +- .ci/scripts/test_qnn_static_llama.sh | 1 - .github/workflows/pull.yml | 100 +++++++++++++-------------- .github/workflows/trunk.yml | 72 +++++++++---------- 4 files changed, 87 insertions(+), 88 deletions(-) diff --git a/.ci/scripts/test_backend_linux.sh b/.ci/scripts/test_backend_linux.sh index b5830fdb82c..a7f7a5ae2bf 100755 --- a/.ci/scripts/test_backend_linux.sh +++ b/.ci/scripts/test_backend_linux.sh @@ -31,7 +31,7 @@ if [[ "$FLOW" == *qnn* ]]; then # cleaned up. echo "---------------wk----------" PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool cmake - PYTHON_EXECUTABLE=python bash .ci/scripts/setup-qnn-deps.sh + PYTHON_EXECUTABLE=python source "$(dirname "${BASH_SOURCE[0]}")/../../backends/qualcomm/scripts/install_qnn_sdk.sh" PYTHON_EXECUTABLE=python bash .ci/scripts/build-qnn-sdk.sh QNN_X86_LIB_DIR=`realpath build-x86/lib/` export LD_LIBRARY_PATH"=$QNN_X86_LIB_DIR:$QNN_SDK_ROOT/lib/x86_64-linux-clang/:${LD_LIBRARY_PATH:-}" diff --git a/.ci/scripts/test_qnn_static_llama.sh b/.ci/scripts/test_qnn_static_llama.sh index 9c24070d115..23eceb8f58a 100644 --- a/.ci/scripts/test_qnn_static_llama.sh +++ b/.ci/scripts/test_qnn_static_llama.sh @@ -13,7 +13,6 @@ source "$(dirname "${BASH_SOURCE[0]}")/utils.sh" source "$(dirname "${BASH_SOURCE[0]}")/../../backends/qualcomm/scripts/qnn_config.sh" # Download QNN_SDK. If already downloaded, export environment path source "$(dirname "${BASH_SOURCE[0]}")/../../backends/qualcomm/scripts/install_qnn_sdk.sh" -install_qnn echo "WK checking LIBC=${CPLUS_INCLUDE_PATH}" export EXECUTORCH_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index 46862f518b4..b0b00e014a6 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -82,58 +82,58 @@ jobs: # Build and test ExecuTorch PYTHON_EXECUTABLE=python bash .ci/scripts/test_model.sh "${MODEL_NAME}" "${BUILD_TOOL}" "${BACKEND}" - test-models-linux: - name: test-models-linux - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main - permissions: - id-token: write - contents: read - strategy: - matrix: - model: [linear, add, add_mul, ic3, mv2, resnet18, resnet50, mobilebert, emformer_transcribe] - backend: [portable, xnnpack-quantization-delegation] - runner: [linux.2xlarge] - include: - - model: ic4 - backend: portable - runner: linux.4xlarge.memory - - model: ic4 - backend: xnnpack-quantization-delegation - runner: linux.4xlarge.memory - - model: emformer_join - backend: portable - runner: linux.4xlarge.memory - - model: emformer_join - backend: xnnpack-quantization-delegation - runner: linux.4xlarge.memory - - model: phi_4_mini - backend: portable - runner: linux.4xlarge.memory - - model: llama3_2_vision_encoder - backend: portable - runner: linux.4xlarge.memory - - model: w2l - backend: portable - runner: linux.4xlarge.memory - fail-fast: false - with: - runner: ${{ matrix.runner }} - docker-image: ci-image:executorch-ubuntu-22.04-clang12 - submodules: 'recursive' - ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} - timeout: 90 - script: | - # The generic Linux job chooses to use base env, not the one setup by the image - CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") - conda activate "${CONDA_ENV}" + # test-models-linux: + # name: test-models-linux + # uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + # permissions: + # id-token: write + # contents: read + # strategy: + # matrix: + # model: [linear, add, add_mul, ic3, mv2, resnet18, resnet50, mobilebert, emformer_transcribe] + # backend: [portable, xnnpack-quantization-delegation] + # runner: [linux.2xlarge] + # include: + # - model: ic4 + # backend: portable + # runner: linux.4xlarge.memory + # - model: ic4 + # backend: xnnpack-quantization-delegation + # runner: linux.4xlarge.memory + # - model: emformer_join + # backend: portable + # runner: linux.4xlarge.memory + # - model: emformer_join + # backend: xnnpack-quantization-delegation + # runner: linux.4xlarge.memory + # - model: phi_4_mini + # backend: portable + # runner: linux.4xlarge.memory + # - model: llama3_2_vision_encoder + # backend: portable + # runner: linux.4xlarge.memory + # - model: w2l + # backend: portable + # runner: linux.4xlarge.memory + # fail-fast: false + # with: + # runner: ${{ matrix.runner }} + # docker-image: ci-image:executorch-ubuntu-22.04-clang12 + # submodules: 'recursive' + # ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + # timeout: 90 + # script: | + # # The generic Linux job chooses to use base env, not the one setup by the image + # CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") + # conda activate "${CONDA_ENV}" - MODEL_NAME=${{ matrix.model }} - BUILD_TOOL=cmake - BACKEND=${{ matrix.backend }} + # MODEL_NAME=${{ matrix.model }} + # BUILD_TOOL=cmake + # BACKEND=${{ matrix.backend }} - PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "${BUILD_TOOL}" - # Build and test ExecuTorch - PYTHON_EXECUTABLE=python bash .ci/scripts/test_model.sh "${MODEL_NAME}" "${BUILD_TOOL}" "${BACKEND}" + # PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "${BUILD_TOOL}" + # # Build and test ExecuTorch + # PYTHON_EXECUTABLE=python bash .ci/scripts/test_model.sh "${MODEL_NAME}" "${BUILD_TOOL}" "${BACKEND}" test-llama-runner-linux: # Test Both linux x86 and linux aarch64 diff --git a/.github/workflows/trunk.yml b/.github/workflows/trunk.yml index 214d2775b41..dc9b7d27f22 100644 --- a/.github/workflows/trunk.yml +++ b/.github/workflows/trunk.yml @@ -273,47 +273,47 @@ jobs: # Test selective build PYTHON_EXECUTABLE=python bash examples/portable/scripts/test_demo_backend_delegation.sh "${BUILD_TOOL}" - test-arm-backend: - name: test-arm-backend - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main - permissions: - id-token: write - contents: read - strategy: - matrix: - include: - - test_arm_baremetal: test_pytest_ops_ethosu_fvp - - test_arm_baremetal: test_pytest_models_ethosu_fvp - - test_arm_baremetal: test_run_ethosu_fvp - - test_arm_baremetal: test_models_tosa - - test_arm_baremetal: test_models_ethos-u55 - - test_arm_baremetal: test_models_ethos-u85 - - test_arm_baremetal: test_smaller_stories_llama - fail-fast: false - with: - runner: linux.2xlarge.memory - docker-image: ci-image:executorch-ubuntu-22.04-arm-sdk - submodules: 'recursive' - ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} - timeout: 120 - script: | - # The generic Linux job chooses to use base env, not the one setup by the image - CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") - conda activate "${CONDA_ENV}" + # test-arm-backend: + # name: test-arm-backend + # uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + # permissions: + # id-token: write + # contents: read + # strategy: + # matrix: + # include: + # - test_arm_baremetal: test_pytest_ops_ethosu_fvp + # - test_arm_baremetal: test_pytest_models_ethosu_fvp + # - test_arm_baremetal: test_run_ethosu_fvp + # - test_arm_baremetal: test_models_tosa + # - test_arm_baremetal: test_models_ethos-u55 + # - test_arm_baremetal: test_models_ethos-u85 + # - test_arm_baremetal: test_smaller_stories_llama + # fail-fast: false + # with: + # runner: linux.2xlarge.memory + # docker-image: ci-image:executorch-ubuntu-22.04-arm-sdk + # submodules: 'recursive' + # ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + # timeout: 120 + # script: | + # # The generic Linux job chooses to use base env, not the one setup by the image + # CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") + # conda activate "${CONDA_ENV}" - source .ci/scripts/utils.sh - install_executorch "--use-pt-pinned-commit" + # source .ci/scripts/utils.sh + # install_executorch "--use-pt-pinned-commit" - .ci/scripts/setup-arm-baremetal-tools.sh + # .ci/scripts/setup-arm-baremetal-tools.sh - # Increase number of files user can monitor to bypass buck failures. - # Hopefully this is high enough for this setup. - sudo sysctl fs.inotify.max_user_watches=1048576 # 1024 * 1024 + # # Increase number of files user can monitor to bypass buck failures. + # # Hopefully this is high enough for this setup. + # sudo sysctl fs.inotify.max_user_watches=1048576 # 1024 * 1024 - ARM_TEST=${{ matrix.test_arm_baremetal }} + # ARM_TEST=${{ matrix.test_arm_baremetal }} - # Test test_arm_baremetal.sh with test - backends/arm/test/test_arm_baremetal.sh "${ARM_TEST}" + # # Test test_arm_baremetal.sh with test + # backends/arm/test/test_arm_baremetal.sh "${ARM_TEST}" test-arm-cortex-m-size-test: name: test-arm-cortex-m-size-test From d8e1bc2ea30b5f3ce97e6f2b6f12dda640586a5d Mon Sep 17 00:00:00 2001 From: winskuo-quic Date: Mon, 8 Sep 2025 20:52:17 +0800 Subject: [PATCH 17/26] Debugging --- .ci/scripts/test_backend_linux.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/scripts/test_backend_linux.sh b/.ci/scripts/test_backend_linux.sh index a7f7a5ae2bf..517b4a27504 100755 --- a/.ci/scripts/test_backend_linux.sh +++ b/.ci/scripts/test_backend_linux.sh @@ -31,7 +31,7 @@ if [[ "$FLOW" == *qnn* ]]; then # cleaned up. echo "---------------wk----------" PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool cmake - PYTHON_EXECUTABLE=python source "$(dirname "${BASH_SOURCE[0]}")/../../backends/qualcomm/scripts/install_qnn_sdk.sh" + PYTHON_EXECUTABLE=python bash "$(dirname "${BASH_SOURCE[0]}")/../../backends/qualcomm/scripts/install_qnn_sdk.sh" PYTHON_EXECUTABLE=python bash .ci/scripts/build-qnn-sdk.sh QNN_X86_LIB_DIR=`realpath build-x86/lib/` export LD_LIBRARY_PATH"=$QNN_X86_LIB_DIR:$QNN_SDK_ROOT/lib/x86_64-linux-clang/:${LD_LIBRARY_PATH:-}" From ef839b5666ace30706b98acc571bae21be162e74 Mon Sep 17 00:00:00 2001 From: winskuo-quic Date: Mon, 8 Sep 2025 20:54:20 +0800 Subject: [PATCH 18/26] Debugging --- .ci/scripts/test_qnn_static_llama.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.ci/scripts/test_qnn_static_llama.sh b/.ci/scripts/test_qnn_static_llama.sh index 23eceb8f58a..b4b9b8de222 100644 --- a/.ci/scripts/test_qnn_static_llama.sh +++ b/.ci/scripts/test_qnn_static_llama.sh @@ -15,6 +15,7 @@ source "$(dirname "${BASH_SOURCE[0]}")/../../backends/qualcomm/scripts/qnn_confi source "$(dirname "${BASH_SOURCE[0]}")/../../backends/qualcomm/scripts/install_qnn_sdk.sh" echo "WK checking LIBC=${CPLUS_INCLUDE_PATH}" +export QNN_SDK_ROOT="/tmp/qnn/2.37.0.250724" export EXECUTORCH_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" export LD_LIBRARY_PATH="${QNN_SDK_ROOT}/lib/x86_64-linux-clang" export PYTHONPATH=".." From 8b2f07fb805e426489fc8376762e0205c9147443 Mon Sep 17 00:00:00 2001 From: winskuo-quic Date: Mon, 8 Sep 2025 21:08:10 +0800 Subject: [PATCH 19/26] Debugging --- backends/qualcomm/scripts/install_qnn_sdk.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/backends/qualcomm/scripts/install_qnn_sdk.sh b/backends/qualcomm/scripts/install_qnn_sdk.sh index 59e85052e71..c814b83430a 100644 --- a/backends/qualcomm/scripts/install_qnn_sdk.sh +++ b/backends/qualcomm/scripts/install_qnn_sdk.sh @@ -94,15 +94,15 @@ setup_libcpp() { LLVM_VERSION="14.0.0" INSTALL_DIR="/tmp/libcxx-${LLVM_VERSION}" - # # Check if we already have a local installation - # if [ -d "${INSTALL_DIR}/include" ] && [ -d "${INSTALL_DIR}/lib" ]; then - # echo "Local libc++ already installed at ${INSTALL_DIR} - skipping" - # # Set environment variables - # export CPLUS_INCLUDE_PATH="${INSTALL_DIR}/include:$CPLUS_INCLUDE_PATH" - # export LD_LIBRARY_PATH="${INSTALL_DIR}/lib:$LD_LIBRARY_PATH" - # export LIBRARY_PATH="${INSTALL_DIR}/lib:$LIBRARY_PATH" - # return - # fi + # Check if we already have a local installation + if [ -d "${INSTALL_DIR}/include" ] && [ -d "${INSTALL_DIR}/lib" ]; then + echo "Local libc++ already installed at ${INSTALL_DIR} - skipping" + # Set environment variables + export CPLUS_INCLUDE_PATH="${INSTALL_DIR}/include:$CPLUS_INCLUDE_PATH" + export LD_LIBRARY_PATH="${INSTALL_DIR}/lib:$LD_LIBRARY_PATH" + export LIBRARY_PATH="${INSTALL_DIR}/lib:$LIBRARY_PATH" + return + fi echo "Installing libc++ manually to ${INSTALL_DIR}" From 39bb89d3730ae4cc3eba9ecb23ba65e361ef1cd8 Mon Sep 17 00:00:00 2001 From: winskuo-quic Date: Mon, 8 Sep 2025 21:22:56 +0800 Subject: [PATCH 20/26] Debugging --- .ci/scripts/test_backend_linux.sh | 3 +-- backends/qualcomm/scripts/install_qnn_sdk.sh | 6 +----- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/.ci/scripts/test_backend_linux.sh b/.ci/scripts/test_backend_linux.sh index 517b4a27504..e68a082e225 100755 --- a/.ci/scripts/test_backend_linux.sh +++ b/.ci/scripts/test_backend_linux.sh @@ -31,8 +31,7 @@ if [[ "$FLOW" == *qnn* ]]; then # cleaned up. echo "---------------wk----------" PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool cmake - PYTHON_EXECUTABLE=python bash "$(dirname "${BASH_SOURCE[0]}")/../../backends/qualcomm/scripts/install_qnn_sdk.sh" - PYTHON_EXECUTABLE=python bash .ci/scripts/build-qnn-sdk.sh + PYTHON_EXECUTABLE=python source .ci/scripts/build-qnn-sdk.sh QNN_X86_LIB_DIR=`realpath build-x86/lib/` export LD_LIBRARY_PATH"=$QNN_X86_LIB_DIR:$QNN_SDK_ROOT/lib/x86_64-linux-clang/:${LD_LIBRARY_PATH:-}" diff --git a/backends/qualcomm/scripts/install_qnn_sdk.sh b/backends/qualcomm/scripts/install_qnn_sdk.sh index c814b83430a..5568159f8b7 100644 --- a/backends/qualcomm/scripts/install_qnn_sdk.sh +++ b/backends/qualcomm/scripts/install_qnn_sdk.sh @@ -152,8 +152,4 @@ setup_libcpp() { export LIBRARY_PATH="${INSTALL_DIR}/lib:${LIBRARY_PATH:-}" echo "libc++ installed to ${INSTALL_DIR}" -} - -setup_libcpp 12 -setup_android_ndk -install_qnn +} \ No newline at end of file From 231d0c041c5583c4de5db0225ee115e80e56be63 Mon Sep 17 00:00:00 2001 From: winskuo-quic Date: Mon, 8 Sep 2025 21:48:32 +0800 Subject: [PATCH 21/26] Debugging --- .ci/scripts/test_qnn_static_llama.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/.ci/scripts/test_qnn_static_llama.sh b/.ci/scripts/test_qnn_static_llama.sh index b4b9b8de222..6e1f5294e31 100644 --- a/.ci/scripts/test_qnn_static_llama.sh +++ b/.ci/scripts/test_qnn_static_llama.sh @@ -13,7 +13,6 @@ source "$(dirname "${BASH_SOURCE[0]}")/utils.sh" source "$(dirname "${BASH_SOURCE[0]}")/../../backends/qualcomm/scripts/qnn_config.sh" # Download QNN_SDK. If already downloaded, export environment path source "$(dirname "${BASH_SOURCE[0]}")/../../backends/qualcomm/scripts/install_qnn_sdk.sh" -echo "WK checking LIBC=${CPLUS_INCLUDE_PATH}" export QNN_SDK_ROOT="/tmp/qnn/2.37.0.250724" export EXECUTORCH_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" From 741f9a252da3e79427b72185cc8c6538c39a2699 Mon Sep 17 00:00:00 2001 From: winskuo-quic Date: Mon, 8 Sep 2025 21:56:36 +0800 Subject: [PATCH 22/26] Debugging --- .ci/scripts/setup-qnn-deps.sh | 1 + .ci/scripts/test_backend_linux.sh | 1 - .ci/scripts/test_qnn_static_llama.sh | 1 - .github/workflows/pull.yml | 100 +++++----- .github/workflows/trunk.yml | 184 +++++++++---------- backends/qualcomm/scripts/install_qnn_sdk.sh | 2 +- 6 files changed, 144 insertions(+), 145 deletions(-) diff --git a/.ci/scripts/setup-qnn-deps.sh b/.ci/scripts/setup-qnn-deps.sh index 02df93769a4..0de9db5c030 100644 --- a/.ci/scripts/setup-qnn-deps.sh +++ b/.ci/scripts/setup-qnn-deps.sh @@ -10,4 +10,5 @@ set -ex source "$(dirname "${BASH_SOURCE[0]}")/../../backends/qualcomm/scripts/install_qnn_sdk.sh" setup_libcpp 12 +setup_android_ndk install_qnn \ No newline at end of file diff --git a/.ci/scripts/test_backend_linux.sh b/.ci/scripts/test_backend_linux.sh index e68a082e225..243602fea21 100755 --- a/.ci/scripts/test_backend_linux.sh +++ b/.ci/scripts/test_backend_linux.sh @@ -29,7 +29,6 @@ if [[ "$FLOW" == *qnn* ]]; then # Setup QNN sdk and deps - note that this is a bit hacky due to the nature of the # Qualcomm build. TODO (gjcomer) Clean this up once the QNN pybinding integration is # cleaned up. - echo "---------------wk----------" PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool cmake PYTHON_EXECUTABLE=python source .ci/scripts/build-qnn-sdk.sh QNN_X86_LIB_DIR=`realpath build-x86/lib/` diff --git a/.ci/scripts/test_qnn_static_llama.sh b/.ci/scripts/test_qnn_static_llama.sh index 6e1f5294e31..25c44ce21c4 100644 --- a/.ci/scripts/test_qnn_static_llama.sh +++ b/.ci/scripts/test_qnn_static_llama.sh @@ -14,7 +14,6 @@ source "$(dirname "${BASH_SOURCE[0]}")/../../backends/qualcomm/scripts/qnn_confi # Download QNN_SDK. If already downloaded, export environment path source "$(dirname "${BASH_SOURCE[0]}")/../../backends/qualcomm/scripts/install_qnn_sdk.sh" -export QNN_SDK_ROOT="/tmp/qnn/2.37.0.250724" export EXECUTORCH_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" export LD_LIBRARY_PATH="${QNN_SDK_ROOT}/lib/x86_64-linux-clang" export PYTHONPATH=".." diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index b0b00e014a6..46862f518b4 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -82,58 +82,58 @@ jobs: # Build and test ExecuTorch PYTHON_EXECUTABLE=python bash .ci/scripts/test_model.sh "${MODEL_NAME}" "${BUILD_TOOL}" "${BACKEND}" - # test-models-linux: - # name: test-models-linux - # uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main - # permissions: - # id-token: write - # contents: read - # strategy: - # matrix: - # model: [linear, add, add_mul, ic3, mv2, resnet18, resnet50, mobilebert, emformer_transcribe] - # backend: [portable, xnnpack-quantization-delegation] - # runner: [linux.2xlarge] - # include: - # - model: ic4 - # backend: portable - # runner: linux.4xlarge.memory - # - model: ic4 - # backend: xnnpack-quantization-delegation - # runner: linux.4xlarge.memory - # - model: emformer_join - # backend: portable - # runner: linux.4xlarge.memory - # - model: emformer_join - # backend: xnnpack-quantization-delegation - # runner: linux.4xlarge.memory - # - model: phi_4_mini - # backend: portable - # runner: linux.4xlarge.memory - # - model: llama3_2_vision_encoder - # backend: portable - # runner: linux.4xlarge.memory - # - model: w2l - # backend: portable - # runner: linux.4xlarge.memory - # fail-fast: false - # with: - # runner: ${{ matrix.runner }} - # docker-image: ci-image:executorch-ubuntu-22.04-clang12 - # submodules: 'recursive' - # ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} - # timeout: 90 - # script: | - # # The generic Linux job chooses to use base env, not the one setup by the image - # CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") - # conda activate "${CONDA_ENV}" + test-models-linux: + name: test-models-linux + uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + permissions: + id-token: write + contents: read + strategy: + matrix: + model: [linear, add, add_mul, ic3, mv2, resnet18, resnet50, mobilebert, emformer_transcribe] + backend: [portable, xnnpack-quantization-delegation] + runner: [linux.2xlarge] + include: + - model: ic4 + backend: portable + runner: linux.4xlarge.memory + - model: ic4 + backend: xnnpack-quantization-delegation + runner: linux.4xlarge.memory + - model: emformer_join + backend: portable + runner: linux.4xlarge.memory + - model: emformer_join + backend: xnnpack-quantization-delegation + runner: linux.4xlarge.memory + - model: phi_4_mini + backend: portable + runner: linux.4xlarge.memory + - model: llama3_2_vision_encoder + backend: portable + runner: linux.4xlarge.memory + - model: w2l + backend: portable + runner: linux.4xlarge.memory + fail-fast: false + with: + runner: ${{ matrix.runner }} + docker-image: ci-image:executorch-ubuntu-22.04-clang12 + submodules: 'recursive' + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + timeout: 90 + script: | + # The generic Linux job chooses to use base env, not the one setup by the image + CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") + conda activate "${CONDA_ENV}" - # MODEL_NAME=${{ matrix.model }} - # BUILD_TOOL=cmake - # BACKEND=${{ matrix.backend }} + MODEL_NAME=${{ matrix.model }} + BUILD_TOOL=cmake + BACKEND=${{ matrix.backend }} - # PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "${BUILD_TOOL}" - # # Build and test ExecuTorch - # PYTHON_EXECUTABLE=python bash .ci/scripts/test_model.sh "${MODEL_NAME}" "${BUILD_TOOL}" "${BACKEND}" + PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "${BUILD_TOOL}" + # Build and test ExecuTorch + PYTHON_EXECUTABLE=python bash .ci/scripts/test_model.sh "${MODEL_NAME}" "${BUILD_TOOL}" "${BACKEND}" test-llama-runner-linux: # Test Both linux x86 and linux aarch64 diff --git a/.github/workflows/trunk.yml b/.github/workflows/trunk.yml index dc9b7d27f22..664acb91fb9 100644 --- a/.github/workflows/trunk.yml +++ b/.github/workflows/trunk.yml @@ -273,47 +273,47 @@ jobs: # Test selective build PYTHON_EXECUTABLE=python bash examples/portable/scripts/test_demo_backend_delegation.sh "${BUILD_TOOL}" - # test-arm-backend: - # name: test-arm-backend - # uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main - # permissions: - # id-token: write - # contents: read - # strategy: - # matrix: - # include: - # - test_arm_baremetal: test_pytest_ops_ethosu_fvp - # - test_arm_baremetal: test_pytest_models_ethosu_fvp - # - test_arm_baremetal: test_run_ethosu_fvp - # - test_arm_baremetal: test_models_tosa - # - test_arm_baremetal: test_models_ethos-u55 - # - test_arm_baremetal: test_models_ethos-u85 - # - test_arm_baremetal: test_smaller_stories_llama - # fail-fast: false - # with: - # runner: linux.2xlarge.memory - # docker-image: ci-image:executorch-ubuntu-22.04-arm-sdk - # submodules: 'recursive' - # ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} - # timeout: 120 - # script: | - # # The generic Linux job chooses to use base env, not the one setup by the image - # CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") - # conda activate "${CONDA_ENV}" + test-arm-backend: + name: test-arm-backend + uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main + permissions: + id-token: write + contents: read + strategy: + matrix: + include: + - test_arm_baremetal: test_pytest_ops_ethosu_fvp + - test_arm_baremetal: test_pytest_models_ethosu_fvp + - test_arm_baremetal: test_run_ethosu_fvp + - test_arm_baremetal: test_models_tosa + - test_arm_baremetal: test_models_ethos-u55 + - test_arm_baremetal: test_models_ethos-u85 + - test_arm_baremetal: test_smaller_stories_llama + fail-fast: false + with: + runner: linux.2xlarge.memory + docker-image: ci-image:executorch-ubuntu-22.04-arm-sdk + submodules: 'recursive' + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + timeout: 120 + script: | + # The generic Linux job chooses to use base env, not the one setup by the image + CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") + conda activate "${CONDA_ENV}" - # source .ci/scripts/utils.sh - # install_executorch "--use-pt-pinned-commit" + source .ci/scripts/utils.sh + install_executorch "--use-pt-pinned-commit" - # .ci/scripts/setup-arm-baremetal-tools.sh + .ci/scripts/setup-arm-baremetal-tools.sh - # # Increase number of files user can monitor to bypass buck failures. - # # Hopefully this is high enough for this setup. - # sudo sysctl fs.inotify.max_user_watches=1048576 # 1024 * 1024 + # Increase number of files user can monitor to bypass buck failures. + # Hopefully this is high enough for this setup. + sudo sysctl fs.inotify.max_user_watches=1048576 # 1024 * 1024 - # ARM_TEST=${{ matrix.test_arm_baremetal }} + ARM_TEST=${{ matrix.test_arm_baremetal }} - # # Test test_arm_baremetal.sh with test - # backends/arm/test/test_arm_baremetal.sh "${ARM_TEST}" + # Test test_arm_baremetal.sh with test + backends/arm/test/test_arm_baremetal.sh "${ARM_TEST}" test-arm-cortex-m-size-test: name: test-arm-cortex-m-size-test @@ -666,71 +666,71 @@ jobs: PYTHON_EXECUTABLE=python bash .ci/scripts/build-qnn-sdk.sh PYTHON_EXECUTABLE=python bash .ci/scripts/test_model.sh ${{ matrix.model }} "cmake" "qnn" - # test-models-macos-coreml: - # name: test-models-macos-coreml - # uses: pytorch/test-infra/.github/workflows/macos_job.yml@main - # strategy: - # matrix: - # model: [dl3, edsr, efficient_sam, emformer_join, emformer_transcribe, ic3, ic4, mobilebert, mv2, mv3, resnet50, vit, w2l] - # fail-fast: false - # with: - # runner: macos-m1-stable - # python-version: '3.11' - # submodules: 'recursive' - # ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} - # timeout: 90 - # script: | - # MODEL_NAME=${{ matrix.model }} - # BUILD_TOOL=cmake - # BACKEND="coreml-pybind" + test-models-macos-coreml: + name: test-models-macos-coreml + uses: pytorch/test-infra/.github/workflows/macos_job.yml@main + strategy: + matrix: + model: [dl3, edsr, efficient_sam, emformer_join, emformer_transcribe, ic3, ic4, mobilebert, mv2, mv3, resnet50, vit, w2l] + fail-fast: false + with: + runner: macos-m1-stable + python-version: '3.11' + submodules: 'recursive' + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + timeout: 90 + script: | + MODEL_NAME=${{ matrix.model }} + BUILD_TOOL=cmake + BACKEND="coreml-pybind" - # # Set model specific overrides - # if [[ "${MODEL_NAME}" == "mobilebert" ]]; then - # # See https://github.com/pytorch/executorch/issues/12907 - # # mobilebert has nan output on FP16, and high MSE on fp32, so we disable runtime test now - # BACKEND="coreml" - # fi + # Set model specific overrides + if [[ "${MODEL_NAME}" == "mobilebert" ]]; then + # See https://github.com/pytorch/executorch/issues/12907 + # mobilebert has nan output on FP16, and high MSE on fp32, so we disable runtime test now + BACKEND="coreml" + fi - # if [[ "${MODEL_NAME}" == "efficient_sam" ]]; then - # # See https://github.com/pytorch/executorch/issues/12906 - # # efficient_sam fails to run on CoreML - # BACKEND="coreml" - # fi + if [[ "${MODEL_NAME}" == "efficient_sam" ]]; then + # See https://github.com/pytorch/executorch/issues/12906 + # efficient_sam fails to run on CoreML + BACKEND="coreml" + fi - # bash .ci/scripts/setup-conda.sh + bash .ci/scripts/setup-conda.sh - # # Setup MacOS dependencies as there is no Docker support on MacOS atm - # PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/setup-macos.sh --build-tool "${BUILD_TOOL}" - # PYTHON_EXECUTABLE=python ${CONDA_RUN} bash backends/apple/coreml/scripts/install_requirements.sh - # echo "Finishing installing coreml." + # Setup MacOS dependencies as there is no Docker support on MacOS atm + PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/setup-macos.sh --build-tool "${BUILD_TOOL}" + PYTHON_EXECUTABLE=python ${CONDA_RUN} bash backends/apple/coreml/scripts/install_requirements.sh + echo "Finishing installing coreml." - # PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/test_model.sh "${MODEL_NAME}" "${BUILD_TOOL}" "${BACKEND}" + PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/test_model.sh "${MODEL_NAME}" "${BUILD_TOOL}" "${BACKEND}" - # test-models-macos-mps: - # name: test-models-macos-mps - # uses: pytorch/test-infra/.github/workflows/macos_job.yml@main - # strategy: - # fail-fast: false - # with: - # runner: macos-m1-stable - # python-version: '3.11' - # submodules: 'recursive' - # ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} - # timeout: 90 - # script: | - # BUILD_TOOL=cmake - # bash .ci/scripts/setup-conda.sh + test-models-macos-mps: + name: test-models-macos-mps + uses: pytorch/test-infra/.github/workflows/macos_job.yml@main + strategy: + fail-fast: false + with: + runner: macos-m1-stable + python-version: '3.11' + submodules: 'recursive' + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + timeout: 90 + script: | + BUILD_TOOL=cmake + bash .ci/scripts/setup-conda.sh - # # Setup MacOS dependencies as there is no Docker support on MacOS atm - # PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/setup-macos.sh --build-tool "${BUILD_TOOL}" + # Setup MacOS dependencies as there is no Docker support on MacOS atm + PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/setup-macos.sh --build-tool "${BUILD_TOOL}" - # # Build and test mps model - # for MODEL_NAME in mv3 ic4 resnet50 edsr mobilebert w2l; do - # echo "::group::Exporting mps model: $MODEL_NAME" - # PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/test_model.sh "${MODEL_NAME}" "${BUILD_TOOL}" "mps" - # echo "::endgroup::" - # done + # Build and test mps model + for MODEL_NAME in mv3 ic4 resnet50 edsr mobilebert w2l; do + echo "::group::Exporting mps model: $MODEL_NAME" + PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/test_model.sh "${MODEL_NAME}" "${BUILD_TOOL}" "mps" + echo "::endgroup::" + done test-huggingface-transformers-xnnpack: # NB: Don't run this on fork PRs because they won't have access to the secret and would fail anyway diff --git a/backends/qualcomm/scripts/install_qnn_sdk.sh b/backends/qualcomm/scripts/install_qnn_sdk.sh index 5568159f8b7..a97d4258770 100644 --- a/backends/qualcomm/scripts/install_qnn_sdk.sh +++ b/backends/qualcomm/scripts/install_qnn_sdk.sh @@ -152,4 +152,4 @@ setup_libcpp() { export LIBRARY_PATH="${INSTALL_DIR}/lib:${LIBRARY_PATH:-}" echo "libc++ installed to ${INSTALL_DIR}" -} \ No newline at end of file +} From 590f009fe01f747ed6ed1d0f6ad36b0f05b75e08 Mon Sep 17 00:00:00 2001 From: winskuo-quic Date: Mon, 8 Sep 2025 21:57:58 +0800 Subject: [PATCH 23/26] Debugging --- .github/workflows/trunk.yml | 264 ++++++++++++++++++------------------ 1 file changed, 132 insertions(+), 132 deletions(-) diff --git a/.github/workflows/trunk.yml b/.github/workflows/trunk.yml index 664acb91fb9..1e6095953f2 100644 --- a/.github/workflows/trunk.yml +++ b/.github/workflows/trunk.yml @@ -18,139 +18,139 @@ concurrency: cancel-in-progress: true jobs: - # test-models-macos-cpu: - # name: test-models-macos-cpu - # uses: pytorch/test-infra/.github/workflows/macos_job.yml@main - # strategy: - # matrix: - # # Mac runners are expensive and limited, and non reliable. - # # Do some basic testing for macos jobs, and rely mostly on - # # test-models-linux-aarch64 job instead. - # model: [emformer_join, ic4, llama2, mobilebert, mv3, resnet50, vit, w2l] - # backend: [xnnpack-quantization-delegation] - # include: - # - model: efficient_sam - # backend: portable - # - model: llama - # backend: portable - # - model: llama3_2_vision_encoder - # backend: portable - # - model: mv3 - # backend: portable - # fail-fast: false - # with: - # runner: macos-m1-stable - # python-version: '3.11' - # submodules: 'recursive' - # ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} - # timeout: 90 - # script: | - # MODEL_NAME=${{ matrix.model }} - # BUILD_TOOL=cmake - # BACKEND=${{ matrix.backend }} + test-models-macos-cpu: + name: test-models-macos-cpu + uses: pytorch/test-infra/.github/workflows/macos_job.yml@main + strategy: + matrix: + # Mac runners are expensive and limited, and non reliable. + # Do some basic testing for macos jobs, and rely mostly on + # test-models-linux-aarch64 job instead. + model: [emformer_join, ic4, llama2, mobilebert, mv3, resnet50, vit, w2l] + backend: [xnnpack-quantization-delegation] + include: + - model: efficient_sam + backend: portable + - model: llama + backend: portable + - model: llama3_2_vision_encoder + backend: portable + - model: mv3 + backend: portable + fail-fast: false + with: + runner: macos-m1-stable + python-version: '3.11' + submodules: 'recursive' + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + timeout: 90 + script: | + MODEL_NAME=${{ matrix.model }} + BUILD_TOOL=cmake + BACKEND=${{ matrix.backend }} - # bash .ci/scripts/setup-conda.sh - # # Setup MacOS dependencies as there is no Docker support on MacOS atm - # PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/setup-macos.sh --build-tool "${BUILD_TOOL}" - # # Build and test executorch - # PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/test_model.sh "${MODEL_NAME}" "${BUILD_TOOL}" "${BACKEND}" - -# test-models-arm-zephyr: -# name: test-models-arm-zephyr -# uses: pytorch/test-infra/.github/workflows/linux_job.yml@main -# strategy: -# matrix: -# model: [add, softmax, mv2] -# fail-fast: false -# with: -# runner: linux.2xlarge -# docker-image: ci-image:executorch-ubuntu-22.04-zephyr-sdk -# submodules: 'recursive' -# ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} -# timeout: 120 -# script: | -# MODEL_NAME=${{ matrix.model }} -# CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") -# conda activate "${CONDA_ENV}" -# if [[ ${{ matrix.model}} == "add" ]]; then -# SIM_LIMIT_SEC=60 -# elif [[ ${{ matrix.model}} == "softmax" ]]; then -# SIM_LIMIT_SEC=60 -# elif [[ ${{ matrix.model}} == "mv2" ]]; then -# SIM_LIMIT_SEC=5000 -# else -# echo "Failed unsupported model selection ${{ matrix.model }}" -# exit 1 -# fi -# -# source .ci/scripts/utils.sh -# source .ci/scripts/zephyr-utils.sh -# mkdir -p zephyr_scratch/ -# cd zephyr_scratch -# export ZEPHYR_PROJ_ROOT=$(realpath $(pwd)) -# export ARM_FVP_TUTORIALS_ROOT=$ZEPHYR_PROJ_ROOT/zephyr/samples/modules/executorch/arm-fvp-tutorials -# -# # TODO @Bujji: Should see if this can be moved into the docker image itself -# download_arm_zephyr_sdk -# ./zephyr-sdk-0.17.2/setup.sh -c -t arm-zephyr-eabi -# cd $ZEPHYR_PROJ_ROOT -# setup_zephyr_et_module -# -# # Run setup scripts for Arm FVP and Arm AOT Compilation -# cd $ZEPHYR_PROJ_ROOT/modules/lib/executorch -# install_executorch -# .ci/scripts/setup-arm-baremetal-tools.sh --target-toolchain zephyr -# source examples/arm/ethos-u-scratch/setup_path.sh -# source $ZEPHYR_PROJ_ROOT/zephyr/zephyr-env.sh -# -# # Get the model as PTE -# python -m examples.arm.aot_arm_compiler \ -# --model_name="${MODEL_NAME}" \ -# --output="${MODEL_NAME}.pte" -# -# # Generate the C-style header -# cd $ARM_FVP_TUTORIALS_ROOT -# python build_model.py \ -# --executorch-root $ZEPHYR_PROJ_ROOT/modules/lib/executorch \ -# --pte-file $ZEPHYR_PROJ_ROOT/modules/lib/executorch/${MODEL_NAME}.pte \ -# --output-path $ARM_FVP_TUTORIALS_ROOT/models/${MODEL_NAME}/src/ -# -# cd $ARM_FVP_TUTORIALS_ROOT/models/${MODEL_NAME}/ -# -# # Build the zephyr elf -# west build -p always -b mps3/corstone300/fvp -- \ -# -DET_PTE_FILE_PATH_FOR_SELECTIVE_BUILD=$ZEPHYR_PROJ_ROOT/modules/lib/executorch/${MODEL_NAME}.pte -# -# # Run the simulation -# FVP_Corstone_SSE-300_Ethos-U55 -a build/zephyr/zephyr.elf \ -# -C mps3_board.visualisation.disable-visualisation=1 \ -# -C mps3_board.telnetterminal0.start_telnet=0 \ -# -C mps3_board.uart0.out_file='sim.out' \ -# -C cpu0.CFGITCMSZ=15 \ -# -C cpu0.CFGDTCMSZ=15 \ -# --simlimit ${SIM_LIMIT_SEC} -# -# # Disable exit on error -# set +e -# # Report failure if any of the ouptut verification checks fail -# grep -qF "ERROR" sim.out -# exit_status=$? #store 0 if found (failure), 1 if not (success) -# if [[ "$exit_status" -eq "0" ]]; then -# cat sim.out -# set -e -# exit 1 -# fi -# -# # Report fail if simulation does not complete successfully -# grep -qF "SUCCESS: Program complete, exiting." sim.out -# exit_status=$? #store 0 if found (success), 1 if not (failure) -# if [[ "$exit_status" -eq "1" ]]; then -# cat sim.out -# set -e -# exit 1 -# fi -# # Re-enable exit on error -# set -e + bash .ci/scripts/setup-conda.sh + # Setup MacOS dependencies as there is no Docker support on MacOS atm + PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/setup-macos.sh --build-tool "${BUILD_TOOL}" + # Build and test executorch + PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/test_model.sh "${MODEL_NAME}" "${BUILD_TOOL}" "${BACKEND}" + +test-models-arm-zephyr: + name: test-models-arm-zephyr + uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + strategy: + matrix: + model: [add, softmax, mv2] + fail-fast: false + with: + runner: linux.2xlarge + docker-image: ci-image:executorch-ubuntu-22.04-zephyr-sdk + submodules: 'recursive' + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + timeout: 120 + script: | + MODEL_NAME=${{ matrix.model }} + CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") + conda activate "${CONDA_ENV}" + if [[ ${{ matrix.model}} == "add" ]]; then + SIM_LIMIT_SEC=60 + elif [[ ${{ matrix.model}} == "softmax" ]]; then + SIM_LIMIT_SEC=60 + elif [[ ${{ matrix.model}} == "mv2" ]]; then + SIM_LIMIT_SEC=5000 + else + echo "Failed unsupported model selection ${{ matrix.model }}" + exit 1 + fi + + source .ci/scripts/utils.sh + source .ci/scripts/zephyr-utils.sh + mkdir -p zephyr_scratch/ + cd zephyr_scratch + export ZEPHYR_PROJ_ROOT=$(realpath $(pwd)) + export ARM_FVP_TUTORIALS_ROOT=$ZEPHYR_PROJ_ROOT/zephyr/samples/modules/executorch/arm-fvp-tutorials + + # TODO @Bujji: Should see if this can be moved into the docker image itself + download_arm_zephyr_sdk + ./zephyr-sdk-0.17.2/setup.sh -c -t arm-zephyr-eabi + cd $ZEPHYR_PROJ_ROOT + setup_zephyr_et_module + + # Run setup scripts for Arm FVP and Arm AOT Compilation + cd $ZEPHYR_PROJ_ROOT/modules/lib/executorch + install_executorch + .ci/scripts/setup-arm-baremetal-tools.sh --target-toolchain zephyr + source examples/arm/ethos-u-scratch/setup_path.sh + source $ZEPHYR_PROJ_ROOT/zephyr/zephyr-env.sh + + # Get the model as PTE + python -m examples.arm.aot_arm_compiler \ + --model_name="${MODEL_NAME}" \ + --output="${MODEL_NAME}.pte" + + # Generate the C-style header + cd $ARM_FVP_TUTORIALS_ROOT + python build_model.py \ + --executorch-root $ZEPHYR_PROJ_ROOT/modules/lib/executorch \ + --pte-file $ZEPHYR_PROJ_ROOT/modules/lib/executorch/${MODEL_NAME}.pte \ + --output-path $ARM_FVP_TUTORIALS_ROOT/models/${MODEL_NAME}/src/ + + cd $ARM_FVP_TUTORIALS_ROOT/models/${MODEL_NAME}/ + + # Build the zephyr elf + west build -p always -b mps3/corstone300/fvp -- \ + -DET_PTE_FILE_PATH_FOR_SELECTIVE_BUILD=$ZEPHYR_PROJ_ROOT/modules/lib/executorch/${MODEL_NAME}.pte + + # Run the simulation + FVP_Corstone_SSE-300_Ethos-U55 -a build/zephyr/zephyr.elf \ + -C mps3_board.visualisation.disable-visualisation=1 \ + -C mps3_board.telnetterminal0.start_telnet=0 \ + -C mps3_board.uart0.out_file='sim.out' \ + -C cpu0.CFGITCMSZ=15 \ + -C cpu0.CFGDTCMSZ=15 \ + --simlimit ${SIM_LIMIT_SEC} + + # Disable exit on error + set +e + # Report failure if any of the ouptut verification checks fail + grep -qF "ERROR" sim.out + exit_status=$? #store 0 if found (failure), 1 if not (success) + if [[ "$exit_status" -eq "0" ]]; then + cat sim.out + set -e + exit 1 + fi + + # Report fail if simulation does not complete successfully + grep -qF "SUCCESS: Program complete, exiting." sim.out + exit_status=$? #store 0 if found (success), 1 if not (failure) + if [[ "$exit_status" -eq "1" ]]; then + cat sim.out + set -e + exit 1 + fi + # Re-enable exit on error + set -e test-models-linux-aarch64: name: test-models-linux-aarch64 From 1f758aeace8727adab5ee8031733d963a9c8bcdd Mon Sep 17 00:00:00 2001 From: winskuo-quic Date: Mon, 8 Sep 2025 21:58:51 +0800 Subject: [PATCH 24/26] Debugging --- .github/workflows/trunk.yml | 192 ++++++++++++++++++------------------ 1 file changed, 96 insertions(+), 96 deletions(-) diff --git a/.github/workflows/trunk.yml b/.github/workflows/trunk.yml index 1e6095953f2..251bb238f1b 100644 --- a/.github/workflows/trunk.yml +++ b/.github/workflows/trunk.yml @@ -55,102 +55,102 @@ jobs: # Build and test executorch PYTHON_EXECUTABLE=python ${CONDA_RUN} bash .ci/scripts/test_model.sh "${MODEL_NAME}" "${BUILD_TOOL}" "${BACKEND}" -test-models-arm-zephyr: - name: test-models-arm-zephyr - uses: pytorch/test-infra/.github/workflows/linux_job.yml@main - strategy: - matrix: - model: [add, softmax, mv2] - fail-fast: false - with: - runner: linux.2xlarge - docker-image: ci-image:executorch-ubuntu-22.04-zephyr-sdk - submodules: 'recursive' - ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} - timeout: 120 - script: | - MODEL_NAME=${{ matrix.model }} - CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") - conda activate "${CONDA_ENV}" - if [[ ${{ matrix.model}} == "add" ]]; then - SIM_LIMIT_SEC=60 - elif [[ ${{ matrix.model}} == "softmax" ]]; then - SIM_LIMIT_SEC=60 - elif [[ ${{ matrix.model}} == "mv2" ]]; then - SIM_LIMIT_SEC=5000 - else - echo "Failed unsupported model selection ${{ matrix.model }}" - exit 1 - fi - - source .ci/scripts/utils.sh - source .ci/scripts/zephyr-utils.sh - mkdir -p zephyr_scratch/ - cd zephyr_scratch - export ZEPHYR_PROJ_ROOT=$(realpath $(pwd)) - export ARM_FVP_TUTORIALS_ROOT=$ZEPHYR_PROJ_ROOT/zephyr/samples/modules/executorch/arm-fvp-tutorials - - # TODO @Bujji: Should see if this can be moved into the docker image itself - download_arm_zephyr_sdk - ./zephyr-sdk-0.17.2/setup.sh -c -t arm-zephyr-eabi - cd $ZEPHYR_PROJ_ROOT - setup_zephyr_et_module - - # Run setup scripts for Arm FVP and Arm AOT Compilation - cd $ZEPHYR_PROJ_ROOT/modules/lib/executorch - install_executorch - .ci/scripts/setup-arm-baremetal-tools.sh --target-toolchain zephyr - source examples/arm/ethos-u-scratch/setup_path.sh - source $ZEPHYR_PROJ_ROOT/zephyr/zephyr-env.sh - - # Get the model as PTE - python -m examples.arm.aot_arm_compiler \ - --model_name="${MODEL_NAME}" \ - --output="${MODEL_NAME}.pte" - - # Generate the C-style header - cd $ARM_FVP_TUTORIALS_ROOT - python build_model.py \ - --executorch-root $ZEPHYR_PROJ_ROOT/modules/lib/executorch \ - --pte-file $ZEPHYR_PROJ_ROOT/modules/lib/executorch/${MODEL_NAME}.pte \ - --output-path $ARM_FVP_TUTORIALS_ROOT/models/${MODEL_NAME}/src/ - - cd $ARM_FVP_TUTORIALS_ROOT/models/${MODEL_NAME}/ - - # Build the zephyr elf - west build -p always -b mps3/corstone300/fvp -- \ - -DET_PTE_FILE_PATH_FOR_SELECTIVE_BUILD=$ZEPHYR_PROJ_ROOT/modules/lib/executorch/${MODEL_NAME}.pte - - # Run the simulation - FVP_Corstone_SSE-300_Ethos-U55 -a build/zephyr/zephyr.elf \ - -C mps3_board.visualisation.disable-visualisation=1 \ - -C mps3_board.telnetterminal0.start_telnet=0 \ - -C mps3_board.uart0.out_file='sim.out' \ - -C cpu0.CFGITCMSZ=15 \ - -C cpu0.CFGDTCMSZ=15 \ - --simlimit ${SIM_LIMIT_SEC} - - # Disable exit on error - set +e - # Report failure if any of the ouptut verification checks fail - grep -qF "ERROR" sim.out - exit_status=$? #store 0 if found (failure), 1 if not (success) - if [[ "$exit_status" -eq "0" ]]; then - cat sim.out - set -e - exit 1 - fi - - # Report fail if simulation does not complete successfully - grep -qF "SUCCESS: Program complete, exiting." sim.out - exit_status=$? #store 0 if found (success), 1 if not (failure) - if [[ "$exit_status" -eq "1" ]]; then - cat sim.out - set -e - exit 1 - fi - # Re-enable exit on error - set -e +# test-models-arm-zephyr: +# name: test-models-arm-zephyr +# uses: pytorch/test-infra/.github/workflows/linux_job.yml@main +# strategy: +# matrix: +# model: [add, softmax, mv2] +# fail-fast: false +# with: +# runner: linux.2xlarge +# docker-image: ci-image:executorch-ubuntu-22.04-zephyr-sdk +# submodules: 'recursive' +# ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} +# timeout: 120 +# script: | +# MODEL_NAME=${{ matrix.model }} +# CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") +# conda activate "${CONDA_ENV}" +# if [[ ${{ matrix.model}} == "add" ]]; then +# SIM_LIMIT_SEC=60 +# elif [[ ${{ matrix.model}} == "softmax" ]]; then +# SIM_LIMIT_SEC=60 +# elif [[ ${{ matrix.model}} == "mv2" ]]; then +# SIM_LIMIT_SEC=5000 +# else +# echo "Failed unsupported model selection ${{ matrix.model }}" +# exit 1 +# fi +# +# source .ci/scripts/utils.sh +# source .ci/scripts/zephyr-utils.sh +# mkdir -p zephyr_scratch/ +# cd zephyr_scratch +# export ZEPHYR_PROJ_ROOT=$(realpath $(pwd)) +# export ARM_FVP_TUTORIALS_ROOT=$ZEPHYR_PROJ_ROOT/zephyr/samples/modules/executorch/arm-fvp-tutorials +# +# # TODO @Bujji: Should see if this can be moved into the docker image itself +# download_arm_zephyr_sdk +# ./zephyr-sdk-0.17.2/setup.sh -c -t arm-zephyr-eabi +# cd $ZEPHYR_PROJ_ROOT +# setup_zephyr_et_module +# +# # Run setup scripts for Arm FVP and Arm AOT Compilation +# cd $ZEPHYR_PROJ_ROOT/modules/lib/executorch +# install_executorch +# .ci/scripts/setup-arm-baremetal-tools.sh --target-toolchain zephyr +# source examples/arm/ethos-u-scratch/setup_path.sh +# source $ZEPHYR_PROJ_ROOT/zephyr/zephyr-env.sh +# +# # Get the model as PTE +# python -m examples.arm.aot_arm_compiler \ +# --model_name="${MODEL_NAME}" \ +# --output="${MODEL_NAME}.pte" +# +# # Generate the C-style header +# cd $ARM_FVP_TUTORIALS_ROOT +# python build_model.py \ +# --executorch-root $ZEPHYR_PROJ_ROOT/modules/lib/executorch \ +# --pte-file $ZEPHYR_PROJ_ROOT/modules/lib/executorch/${MODEL_NAME}.pte \ +# --output-path $ARM_FVP_TUTORIALS_ROOT/models/${MODEL_NAME}/src/ +# +# cd $ARM_FVP_TUTORIALS_ROOT/models/${MODEL_NAME}/ +# +# # Build the zephyr elf +# west build -p always -b mps3/corstone300/fvp -- \ +# -DET_PTE_FILE_PATH_FOR_SELECTIVE_BUILD=$ZEPHYR_PROJ_ROOT/modules/lib/executorch/${MODEL_NAME}.pte +# +# # Run the simulation +# FVP_Corstone_SSE-300_Ethos-U55 -a build/zephyr/zephyr.elf \ +# -C mps3_board.visualisation.disable-visualisation=1 \ +# -C mps3_board.telnetterminal0.start_telnet=0 \ +# -C mps3_board.uart0.out_file='sim.out' \ +# -C cpu0.CFGITCMSZ=15 \ +# -C cpu0.CFGDTCMSZ=15 \ +# --simlimit ${SIM_LIMIT_SEC} +# +# # Disable exit on error +# set +e +# # Report failure if any of the ouptut verification checks fail +# grep -qF "ERROR" sim.out +# exit_status=$? #store 0 if found (failure), 1 if not (success) +# if [[ "$exit_status" -eq "0" ]]; then +# cat sim.out +# set -e +# exit 1 +# fi +# +# # Report fail if simulation does not complete successfully +# grep -qF "SUCCESS: Program complete, exiting." sim.out +# exit_status=$? #store 0 if found (success), 1 if not (failure) +# if [[ "$exit_status" -eq "1" ]]; then +# cat sim.out +# set -e +# exit 1 +# fi +# # Re-enable exit on error +# set -e test-models-linux-aarch64: name: test-models-linux-aarch64 From ee037fc7653c62f7f3736d4bd3448989880988c6 Mon Sep 17 00:00:00 2001 From: winskuo-quic Date: Mon, 8 Sep 2025 22:02:08 +0800 Subject: [PATCH 25/26] Debugging --- .ci/scripts/test_qnn_static_llama.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.ci/scripts/test_qnn_static_llama.sh b/.ci/scripts/test_qnn_static_llama.sh index 25c44ce21c4..7898d03b3b9 100644 --- a/.ci/scripts/test_qnn_static_llama.sh +++ b/.ci/scripts/test_qnn_static_llama.sh @@ -9,10 +9,9 @@ set -euxo pipefail source "$(dirname "${BASH_SOURCE[0]}")/utils.sh" -# Source QNN configuration -source "$(dirname "${BASH_SOURCE[0]}")/../../backends/qualcomm/scripts/qnn_config.sh" # Download QNN_SDK. If already downloaded, export environment path source "$(dirname "${BASH_SOURCE[0]}")/../../backends/qualcomm/scripts/install_qnn_sdk.sh" +install_qnn export EXECUTORCH_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" export LD_LIBRARY_PATH="${QNN_SDK_ROOT}/lib/x86_64-linux-clang" From aad4cc79883d49edf80251fe5ee87e2a8d64c57c Mon Sep 17 00:00:00 2001 From: winskuo-quic Date: Mon, 8 Sep 2025 23:15:22 +0800 Subject: [PATCH 26/26] Working --- .github/workflows/pull.yml | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index 46862f518b4..379d47716c9 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -523,29 +523,6 @@ jobs: # Test llama2 PYTHON_EXECUTABLE=python bash .ci/scripts/test_llama.sh -model stories110M -build_tool "${BUILD_TOOL}" -mode "${MODE}" -dtype "${DTYPE}" -pt2e_quantize "${PT2E_QUANTIZE}" - backend-test-linux: - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main - strategy: - fail-fast: false - matrix: - flow: [ - qnn, qnn_16a16w, qnn_16a8w, qnn_16a4w, qnn_16a4w_block, qnn_8a8w, - vulkan, vulkan_static_int8_per_channel, - xnnpack, xnnpack_dynamic_int8_per_channel, xnnpack_static_int8_per_channel, xnnpack_static_int8_per_tensor - ] - suite: [models, operators] - with: - ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} - runner: linux.4xlarge.memory - docker-image: ci-image:executorch-ubuntu-22.04-clang12 - submodules: recursive - timeout: 120 - upload-artifact: test-report-${{ matrix.flow }}-${{ matrix.suite }} - script: | - set -eux - - source .ci/scripts/test_backend_linux.sh "${{ matrix.suite }}" "${{ matrix.flow }}" "${RUNNER_ARTIFACT_DIR}" - test-static-llama-qnn-linux: name: test-static-llama-qnn-linux uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main