Skip to content
pull

add qnn eval CI #54822

Sign in to view logs

add qnn eval CI

add qnn eval CI #54822

Workflow file for this run

.github/workflows/pull.yml at 7f0c68f
name: pull
on:
pull_request:
push:
branches:
- main
- release/*
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}-${{ github.event_name == 'schedule' }}
cancel-in-progress: true
jobs:
# test-qnn-wheel-packages-linux:
# name: test-qnn-wheel-packages-linux
# uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
# permissions:
# id-token: write
# contents: read
# strategy:
# fail-fast: false
# matrix:
# python-version: [ "3.10", "3.11", "3.12" ]
# with:
# runner: linux.2xlarge
# docker-image: ci-image:executorch-ubuntu-22.04-qnn-sdk
# submodules: 'recursive'
# ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
# timeout: 180
# script: |
# # The generic Linux job chooses to use base env, not the one setup by the image
# CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
# conda activate "${CONDA_ENV}"
# # Create a clean env for each python version
# conda create -y -n test_env_${{ matrix.python-version }} python=${{ matrix.python-version }}
# conda activate test_env_${{ matrix.python-version }}
# PYTHON_EXECUTABLE=python bash .ci/scripts/test_wheel_package_qnn.sh "${{ matrix.python-version }}"
# test-setup-linux-gcc:
# name: test-setup-linux-gcc
# uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
# permissions:
# id-token: write
# contents: read
# strategy:
# fail-fast: false
# with:
# runner: linux.2xlarge
# docker-image: ci-image:executorch-ubuntu-22.04-gcc9
# submodules: 'recursive'
# ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
# timeout: 90
# script: |
# # The generic Linux job chooses to use base env, not the one setup by the image
# CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
# conda activate "${CONDA_ENV}"
# BUILD_TOOL="cmake"
# PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "${BUILD_TOOL}"
# # Build and test ExecuTorch with the add model on portable backend.
# PYTHON_EXECUTABLE=python bash .ci/scripts/test_model.sh "add" "${BUILD_TOOL}" "portable"
# test-models-linux-basic:
# name: test-models-linux-basic
# uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
# permissions:
# id-token: write
# contents: read
# strategy:
# matrix:
# model: [mv3, vit]
# backend: [portable, xnnpack-quantization-delegation]
# build-tool: [cmake, buck2]
# runner: [linux.2xlarge, linux.arm64.2xlarge]
# docker-image: [executorch-ubuntu-22.04-clang12, executorch-ubuntu-22.04-gcc11-aarch64]
# # Excluding specific runner + docker image combinations that don't make sense:
# # - Excluding the ARM64 gcc image on the x86 runner (linux.2xlarge)
# # - Excluding the x86 clang image on the ARM64 runner (linux.arm64.2xlarge)
# exclude:
# - runner: linux.2xlarge
# docker-image: executorch-ubuntu-22.04-gcc11-aarch64
# - runner: linux.arm64.2xlarge
# docker-image: executorch-ubuntu-22.04-clang12
# # TODO: Need to figure out why buck2 doesnt work on Graviton instances.
# - runner: linux.arm64.2xlarge
# build-tool: buck2
# fail-fast: false
# with:
# runner: ${{ matrix.runner }}
# docker-image: ci-image:${{ matrix.docker-image }}
# submodules: 'recursive'
# ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
# timeout: 90
# script: |
# # The generic Linux job chooses to use base env, not the one setup by the image
# CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
# conda activate "${CONDA_ENV}"
# MODEL_NAME=${{ matrix.model }}
# BUILD_TOOL=${{ matrix.build-tool }}
# BACKEND=${{ matrix.backend }}
# PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "${BUILD_TOOL}"
# # Build and test ExecuTorch
# PYTHON_EXECUTABLE=python bash .ci/scripts/test_model.sh "${MODEL_NAME}" "${BUILD_TOOL}" "${BACKEND}"
# test-models-linux:
# name: test-models-linux
# uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
# permissions:
# id-token: write
# contents: read
# strategy:
# matrix:
# model: [linear, add, add_mul, ic3, mv2, resnet18, resnet50, mobilebert, emformer_transcribe]
# backend: [portable, xnnpack-quantization-delegation]
# runner: [linux.2xlarge]
# include:
# - model: ic4
# backend: portable
# runner: linux.4xlarge.memory
# - model: ic4
# backend: xnnpack-quantization-delegation
# runner: linux.4xlarge.memory
# - model: emformer_join
# backend: portable
# runner: linux.4xlarge.memory
# - model: emformer_join
# backend: xnnpack-quantization-delegation
# runner: linux.4xlarge.memory
# - model: phi_4_mini
# backend: portable
# runner: linux.4xlarge.memory
# - model: llama3_2_vision_encoder
# backend: portable
# runner: linux.4xlarge.memory
# - model: w2l
# backend: portable
# runner: linux.4xlarge.memory
# fail-fast: false
# with:
# runner: ${{ matrix.runner }}
# docker-image: ci-image:executorch-ubuntu-22.04-clang12
# submodules: 'recursive'
# ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
# timeout: 90
# script: |
# # The generic Linux job chooses to use base env, not the one setup by the image
# CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
# conda activate "${CONDA_ENV}"
# MODEL_NAME=${{ matrix.model }}
# BUILD_TOOL=cmake
# BACKEND=${{ matrix.backend }}
# PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "${BUILD_TOOL}"
# # Build and test ExecuTorch
# PYTHON_EXECUTABLE=python bash .ci/scripts/test_model.sh "${MODEL_NAME}" "${BUILD_TOOL}" "${BACKEND}"
# test-llama-runner-linux:
# # Test Both linux x86 and linux aarch64
# name: test-llama-runner-linux
# uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
# permissions:
# id-token: write
# contents: read
# strategy:
# matrix:
# dtype: [fp32]
# mode: [xnnpack+custom+qe,xnnpack+custom+quantize_kv,xnnpack+quantize_kv]
# runner: [linux.2xlarge, linux.arm64.2xlarge]
# docker-image: [executorch-ubuntu-22.04-clang12, executorch-ubuntu-22.04-gcc11-aarch64]
# include:
# - dtype: bf16
# mode: custom
# runner: linux.2xlarge
# docker-image: executorch-ubuntu-22.04-clang12
# # Excluding specific runner + docker image combinations that don't make sense:
# # - Excluding the ARM64 gcc image on the x86 runner (linux.2xlarge)
# # - Excluding the x86 clang image on the ARM64 runner (linux.arm64.2xlarge)
# exclude:
# - runner: linux.2xlarge
# docker-image: executorch-ubuntu-22.04-gcc11-aarch64
# - runner: linux.arm64.2xlarge
# docker-image: executorch-ubuntu-22.04-clang12
# fail-fast: false
# with:
# runner: ${{ matrix.runner }}
# docker-image: ci-image:${{ matrix.docker-image }}
# submodules: 'recursive'
# ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
# timeout: 900
# script: |
# # The generic Linux job chooses to use base env, not the one setup by the image
# CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
# conda activate "${CONDA_ENV}"
# DTYPE=${{ matrix.dtype }}
# BUILD_TOOL="cmake"
# MODE=${{ matrix.mode }}
# ARTIFACTS_DIR_NAME="artifacts-to-be-uploaded/${DTYPE}-${MODE}"
# ARTIFACTS_DIR_NAME="${ARTIFACTS_DIR_NAME/+/-}"
# # Setup executorch
# PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "${BUILD_TOOL}"
# # Install requirements for export_llama
# PYTHON_EXECUTABLE=python bash examples/models/llama/install_requirements.sh
# # Test llama2
# PYTHON_EXECUTABLE=python bash .ci/scripts/test_llama.sh -model stories110M -build_tool "${BUILD_TOOL}" -dtype "${DTYPE}" -mode "${MODE}" -upload "${ARTIFACTS_DIR_NAME}"
# test-llama-runner-linux-android:
# name: test-llama-runner-linux-android
# uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
# permissions:
# id-token: write
# contents: read
# strategy:
# fail-fast: false
# with:
# runner: linux.2xlarge
# docker-image: ci-image:executorch-ubuntu-22.04-clang12-android
# submodules: 'recursive'
# ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
# timeout: 90
# script: |
# # The generic Linux job chooses to use base env, not the one setup by the image
# CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
# conda activate "${CONDA_ENV}"
# source .ci/scripts/utils.sh
# install_executorch "--use-pt-pinned-commit"
# BUILD_TOOL="cmake"
# PYTHON_EXECUTABLE=python \
# bash .ci/scripts/build_llama_android.sh "${BUILD_TOOL}"
# test-custom-ops-linux:
# name: test-custom-ops-linux
# uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
# permissions:
# id-token: write
# contents: read
# strategy:
# fail-fast: false
# with:
# runner: linux.2xlarge
# docker-image: ci-image:executorch-ubuntu-22.04-clang12
# submodules: 'recursive'
# ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
# timeout: 90
# script: |
# # The generic Linux job chooses to use base env, not the one setup by the image
# CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
# conda activate "${CONDA_ENV}"
# BUILD_TOOL="cmake"
# PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "${BUILD_TOOL}"
# # Test custom ops
# PYTHON_EXECUTABLE=python bash examples/portable/custom_ops/test_custom_ops.sh "${BUILD_TOOL}"
# test-selective-build-linux:
# name: test-selective-build-linux
# uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
# permissions:
# id-token: write
# contents: read
# strategy:
# fail-fast: false
# with:
# runner: linux.2xlarge
# docker-image: ci-image:executorch-ubuntu-22.04-clang12
# submodules: 'recursive'
# ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
# timeout: 90
# script: |
# # The generic Linux job chooses to use base env, not the one setup by the image
# CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
# conda activate "${CONDA_ENV}"
# BUILD_TOOL="cmake"
# PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "${BUILD_TOOL}"
# # Test selective build
# PYTHON_EXECUTABLE=python bash examples/selective_build/test_selective_build.sh "${BUILD_TOOL}"
# test-multimodal-linux:
# if: ${{ !github.event.pull_request.head.repo.fork }}
# name: test-multimodal-linux
# uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
# permissions:
# id-token: write
# contents: read
# secrets: inherit
# strategy:
# fail-fast: false
# matrix:
# model: ["gemma3-4b"] # llava gives segfault so not covering.
# with:
# secrets-env: EXECUTORCH_HF_TOKEN
# runner: linux.24xlarge
# docker-image: ci-image:executorch-ubuntu-22.04-clang12
# submodules: 'recursive'
# ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
# timeout: 90
# script: |
# # The generic Linux job chooses to use base env, not the one setup by the image
# CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
# conda activate "${CONDA_ENV}"
# echo "::group::Setup ExecuTorch"
# PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "cmake"
# echo "::endgroup::"
# echo "::group::Setup Huggingface"
# pip install -U "huggingface_hub[cli]" accelerate
# huggingface-cli login --token $SECRET_EXECUTORCH_HF_TOKEN
# OPTIMUM_ET_VERSION=$(cat .ci/docker/ci_commit_pins/optimum-executorch.txt)
# pip install git+https://github.com/huggingface/optimum-executorch.git@${OPTIMUM_ET_VERSION}
# echo "::endgroup::"
# echo "::group::Test ${{ matrix.model }}"
# python .ci/scripts/test_huggingface_optimum_model.py --model ${{ matrix.model }} --quantize --recipe xnnpack
# echo "::endgroup::"
# test-moshi-linux:
# name: test-moshi-linux
# uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
# permissions:
# id-token: write
# contents: read
# strategy:
# fail-fast: false
# with:
# runner: linux.2xlarge
# docker-image: ci-image:executorch-ubuntu-22.04-clang12
# submodules: 'recursive'
# ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
# timeout: 90
# script: |
# # The generic Linux job chooses to use base env, not the one setup by the image
# CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
# conda activate "${CONDA_ENV}"
# PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "cmake"
# # install Mimi requirements
# bash examples/models/moshi/mimi/install_requirements.sh
# # reinstall executorch
# bash ./install_executorch.sh --minimal
# # run python unittest
# python -m unittest examples.models.moshi.mimi.test_mimi
# test-quantized-aot-lib-linux:
# name: test-quantized-aot-lib-linux
# uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
# permissions:
# id-token: write
# contents: read
# strategy:
# fail-fast: false
# with:
# runner: linux.2xlarge
# docker-image: ci-image:executorch-ubuntu-22.04-clang12
# submodules: 'recursive'
# ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
# timeout: 90
# script: |
# # The generic Linux job chooses to use base env, not the one setup by the image
# CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
# conda activate "${CONDA_ENV}"
# BUILD_TOOL="cmake"
# PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "${BUILD_TOOL}"
# PYTHON_EXECUTABLE=python bash examples/xnnpack/quantization/test_quantize.sh "${BUILD_TOOL}" mv2
# test-binary-size-linux-gcc:
# name: test-binary-size-linux-gcc
# uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
# permissions:
# id-token: write
# contents: read
# strategy:
# fail-fast: false
# with:
# runner: linux.2xlarge
# docker-image: ci-image:executorch-ubuntu-22.04-gcc9
# submodules: 'recursive'
# ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
# timeout: 90
# script: |
# # The generic Linux job chooses to use base env, not the one setup by the image
# CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
# conda activate "${CONDA_ENV}"
# ./install_requirements.sh --use-pt-pinned-commit
# # build module for executorch.extension.pybindings.portable_lib
# bash test/build_size_test.sh
# strip cmake-out/test/size_test
# output=$(ls -la cmake-out/test/size_test)
# arr=($output)
# size=${arr[4]}
# # threshold=48120 on devserver with gcc11.4
# # todo(lfq): update once binary size is below 50kb.
# threshold="63776"
# if [[ "$size" -le "$threshold" ]]; then
# echo "Success $size <= $threshold"
# else
# echo "Fail $size > $threshold"
# exit 1
# fi
# test-binary-size-linux:
# name: test-binary-size-linux
# uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
# permissions:
# id-token: write
# contents: read
# strategy:
# fail-fast: false
# with:
# runner: linux.2xlarge
# docker-image: ci-image:executorch-ubuntu-22.04-clang12
# submodules: 'recursive'
# ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
# timeout: 90
# script: |
# # The generic Linux job chooses to use base env, not the one setup by the image
# CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
# conda activate "${CONDA_ENV}"
# ./install_requirements.sh --use-pt-pinned-commit
# # build module for executorch.extension.pybindings.portable_lib
# bash test/build_size_test.sh
# strip cmake-out/test/size_test
# output=$(ls -la cmake-out/test/size_test)
# arr=($output)
# size=${arr[4]}
# threshold="51752"
# if [[ "$size" -le "$threshold" ]]; then
# echo "Success $size <= $threshold"
# else
# echo "Fail $size > $threshold"
# exit 1
# fi
# android:
# uses: ./.github/workflows/_android.yml
# permissions:
# id-token: write
# contents: read
# unittest:
# uses: ./.github/workflows/_unittest.yml
# permissions:
# id-token: write
# contents: read
# with:
# build-mode: Debug
# build-tool: cmake
# docker-image: ci-image:executorch-ubuntu-22.04-clang12
# unittest-editable:
# uses: ./.github/workflows/_unittest.yml
# permissions:
# id-token: write
# contents: read
# with:
# build-mode: Debug
# build-tool: cmake
# editable: true
# docker-image: ci-image:executorch-ubuntu-22.04-clang12
# unittest-buck:
# uses: ./.github/workflows/_unittest.yml
# permissions:
# id-token: write
# contents: read
# with:
# build-mode: Debug
# build-tool: buck2
# docker-image: ci-image:executorch-ubuntu-22.04-clang12
# unittest-arm-backend-with-no-fvp:
# name: unittest-arm-backend-with-no-fvp
# uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
# permissions:
# id-token: write
# contents: read
# strategy:
# matrix:
# include:
# - test_arm_baremetal: test_pytest_ops
# - test_arm_baremetal: test_pytest_models
# fail-fast: false
# with:
# runner: linux.2xlarge
# docker-image: ci-image:executorch-ubuntu-22.04-arm-sdk
# submodules: 'recursive'
# ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
# timeout: 90
# script: |
# # The generic Linux job chooses to use base env, not the one setup by the image
# CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
# conda activate "${CONDA_ENV}"
# source .ci/scripts/utils.sh
# install_executorch "--use-pt-pinned-commit"
# .ci/scripts/setup-arm-baremetal-tools.sh
# ARM_TEST=${{ matrix.test_arm_baremetal }}
# # Test test_arm_baremetal.sh with test
# backends/arm/test/test_arm_baremetal.sh "${ARM_TEST}"
# test-llama-runner-qnn-linux:
# name: test-llama-runner-qnn-linux
# uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
# permissions:
# id-token: write
# contents: read
# strategy:
# matrix:
# dtype: [fp32]
# pt2e_quantize: [qnn_16a16w, qnn_8a8w]
# mode: [qnn]
# fail-fast: false
# with:
# runner: linux.2xlarge
# docker-image: ci-image:executorch-ubuntu-22.04-qnn-sdk
# submodules: 'recursive'
# ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
# timeout: 900
# script: |
# # The generic Linux job chooses to use base env, not the one setup by the image
# CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
# conda activate "${CONDA_ENV}"
# DTYPE=${{ matrix.dtype }}
# BUILD_TOOL="cmake"
# MODE=${{ matrix.mode }}
# PT2E_QUANTIZE=${{ matrix.pt2e_quantize }}
# ./install_requirements.sh --use-pt-pinned-commit
# PYTHON_EXECUTABLE=python bash .ci/scripts/setup-qnn-deps.sh
# PYTHON_EXECUTABLE=python bash .ci/scripts/build-qnn-sdk.sh
# # Setup executorch
# PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "${BUILD_TOOL}"
# # Install requirements for export_llama
# PYTHON_EXECUTABLE=python bash examples/models/llama/install_requirements.sh
# # Test llama2
# PYTHON_EXECUTABLE=python bash .ci/scripts/test_llama.sh -model stories110M -build_tool "${BUILD_TOOL}" -mode "${MODE}" -dtype "${DTYPE}" -pt2e_quantize "${PT2E_QUANTIZE}"
# test-static-llama-qnn-linux:
# name: test-static-llama-qnn-linux
# uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
# permissions:
# id-token: write
# contents: read
# strategy:
# fail-fast: false
# with:
# runner: linux.2xlarge
# docker-image: ci-image:executorch-ubuntu-22.04-qnn-sdk
# submodules: 'recursive'
# ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
# timeout: 180
# script: |
# # The generic Linux job chooses to use base env, not the one setup by the image
# CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
# conda activate "${CONDA_ENV}"
# BUILD_TOOL="cmake"
# PYTHON_EXECUTABLE=python bash .ci/scripts/setup-qnn-deps.sh
# PYTHON_EXECUTABLE=python bash .ci/scripts/build-qnn-sdk.sh
# # Setup executorch
# PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "${BUILD_TOOL}"
# # Setup install_requirements for llama
# PYTHON_EXECUTABLE=python bash examples/models/llama/install_requirements.sh
# # Test static llama weight sharing and accuracy
# PYTHON_EXECUTABLE=python bash .ci/scripts/test_qnn_static_llama.sh
test-static-llama-qnn-eval-linux:
name: test-static-llama-qnn-eval-linux
uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
permissions:
id-token: write
contents: read
strategy:
fail-fast: false
matrix:
config:
- name: "baseline"
flags: ""
threshold: 62.0
- name: "spinquant"
flags: "--spinquant"
threshold: 62.0
- name: "range-setting"
flags: "--range_setting mse_with_act_loss"
threshold: 62.0
- name: "spinquant+range-setting"
flags: "--spinquant --range_setting mse_with_act_loss"
threshold: 62.0
with:
runner: linux.2xlarge
docker-image: ci-image:executorch-ubuntu-22.04-qnn-sdk
submodules: 'recursive'
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
timeout: 180
script: |
# The generic Linux job chooses to use base env, not the one setup by the image
CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
conda activate "${CONDA_ENV}"
BUILD_TOOL="cmake"
PYTHON_EXECUTABLE=python bash .ci/scripts/setup-qnn-deps.sh
PYTHON_EXECUTABLE=python bash .ci/scripts/build-qnn-sdk.sh
# Setup executorch
PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "${BUILD_TOOL}"
# Setup install_requirements for llama
PYTHON_EXECUTABLE=python bash examples/models/llama/install_requirements.sh
# Test static llama weight sharing and accuracy
echo ">>> Running config: ${{ matrix.config.name }}"
PYTHON_EXECUTABLE=python bash .ci/scripts/test_qnn_static_llama_eval.sh ${{ matrix.config.flags }}
# test-qnn-models-linux:
# name: test-qnn-models-linux
# uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
# permissions:
# id-token: write
# contents: read
# strategy:
# fail-fast: false
# with:
# runner: linux.2xlarge
# docker-image: ci-image:executorch-ubuntu-22.04-qnn-sdk
# submodules: 'recursive'
# ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
# timeout: 180
# script: |
# # The generic Linux job chooses to use base env, not the one setup by the image
# CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
# conda activate "${CONDA_ENV}"
# # placeholder for running test_qnn_delegate.py, can use matrix such that we can trigger different jobs, refers to test-llama-runner-qnn-linux
# # reminder: make sure each job runs fast
# test-phi-3-mini-runner-linux:
# name: test-phi-3-mini-runner-linux
# uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
# permissions:
# id-token: write
# contents: read
# strategy:
# fail-fast: false
# with:
# runner: linux.24xlarge
# docker-image: ci-image:executorch-ubuntu-22.04-clang12
# submodules: 'recursive'
# ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
# timeout: 90
# script: |
# # The generic Linux job chooses to use base env, not the one setup by the image
# CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
# conda activate "${CONDA_ENV}"
# PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "cmake"
# # install phi-3-mini requirements
# bash examples/models/phi-3-mini/install_requirements.sh
# # run e2e (export, tokenizer and runner)
# PYTHON_EXECUTABLE=python bash .ci/scripts/test_phi_3_mini.sh Release
# test-eval_llama-wikitext-linux:
# name: test-eval_llama-wikitext-linux
# uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
# permissions:
# id-token: write
# contents: read
# strategy:
# fail-fast: false
# with:
# runner: linux.24xlarge
# docker-image: ci-image:executorch-ubuntu-22.04-clang12
# submodules: 'recursive'
# ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
# timeout: 90
# script: |
# # The generic Linux job chooses to use base env, not the one setup by the image
# CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
# conda activate "${CONDA_ENV}"
# PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "cmake"
# # install llama requirements
# bash examples/models/llama/install_requirements.sh
# # run eval_llama wikitext task
# PYTHON_EXECUTABLE=python bash .ci/scripts/test_eval_llama_wikitext.sh
# # TODO(larryliu0820): Fix this issue before reenabling it: https://gist.github.com/larryliu0820/7377ecd0d79dbc06076cec8d9f2b85d2
# # test-eval_llama-mmlu-linux:
# # name: test-eval_llama-mmlu-linux
# # uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
# # permissions:
# # id-token: write
# # contents: read
# # strategy:
# # fail-fast: false
# # with:
# # runner: linux.24xlarge
# # docker-image: ci-image:executorch-ubuntu-22.04-clang12
# # submodules: 'recursive'
# # ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
# # timeout: 90
# # script: |
# # # The generic Linux job chooses to use base env, not the one setup by the image
# # CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
# # conda activate "${CONDA_ENV}"
# # PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "cmake"
# # # install llama requirements
# # bash examples/models/llama/install_requirements.sh
# # # run eval_llama mmlu task
# # PYTHON_EXECUTABLE=python bash .ci/scripts/test_eval_llama_mmlu.sh
# test-llama_runner_eager-linux:
# name: test-llama_runner_eager-linux
# uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
# permissions:
# id-token: write
# contents: read
# strategy:
# fail-fast: false
# with:
# runner: linux.24xlarge
# docker-image: ci-image:executorch-ubuntu-22.04-clang12
# submodules: 'recursive'
# ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
# timeout: 90
# script: |
# # The generic Linux job chooses to use base env, not the one setup by the image
# CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
# conda activate "${CONDA_ENV}"
# PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "cmake"
# # install llama requirements
# bash examples/models/llama/install_requirements.sh
# # run llama runner in eager mode
# PYTHON_EXECUTABLE=python bash .ci/scripts/test_llama_runner_eager.sh
# test-llama-lora-linux:
# name: test-llama-lora-linux
# uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
# permissions:
# id-token: write
# contents: read
# strategy:
# fail-fast: false
# with:
# runner: linux.24xlarge
# docker-image: ci-image:executorch-ubuntu-22.04-clang12
# submodules: 'recursive'
# ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
# timeout: 90
# script: |
# # The generic Linux job chooses to use base env, not the one setup by the image
# CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
# conda activate "${CONDA_ENV}"
# PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "cmake"
# # Install llama requirements
# bash examples/models/llama/install_requirements.sh
# # install a recent version of torchtune.
# PYTHON_EXECUTABLE=python python -m pip install torchtune==0.7.0.dev20250730 --extra-index-url https://download.pytorch.org/whl/nightly/cpu
# test-openvino-linux:
# name: test-openvino-linux
# uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
# permissions:
# id-token: write
# contents: read
# strategy:
# fail-fast: false
# if: false # TODO Re-enable after fixing timeouts (#14314)
# with:
# runner: linux.2xlarge
# docker-image: ci-image:executorch-ubuntu-22.04-gcc9
# submodules: 'recursive'
# ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
# timeout: 90
# script: |
# # The generic Linux job chooses to use base env, not the one setup by the image
# CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
# conda activate "${CONDA_ENV}"
# test-mediatek-models-linux:
# name: test-mediatek-models-linux
# uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
# permissions:
# id-token: write
# contents: read
# strategy:
# fail-fast: false
# with:
# runner: linux.24xlarge
# docker-image: ci-image:executorch-ubuntu-22.04-mediatek-sdk
# submodules: 'recursive'
# ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
# timeout: 90
# script: |
# # The generic Linux job chooses to use base env, not the one setup by the image
# CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
# conda activate "${CONDA_ENV}"
# source .ci/scripts/utils.sh
# install_executorch "--use-pt-pinned-commit"
# PYTHON_EXECUTABLE=python bash .ci/scripts/setup-mediatek-deps.sh
# PYTHON_EXECUTABLE=python bash .ci/scripts/build-mediatek-sdk.sh
# PYTHON_EXECUTABLE=python bash .ci/scripts/test_model.sh "mv3" "buck2" "mediatek"
# # placeholder for mediatek to add more tests
# test-openvino-linux:
# name: test-openvino-linux
# uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
# permissions:
# id-token: write
# contents: read
# strategy:
# fail-fast: false
# with:
# runner: linux.2xlarge
# docker-image: ci-image:executorch-ubuntu-22.04-gcc9
# submodules: 'recursive'
# ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
# timeout: 90
# script: |
# # The generic Linux job chooses to use base env, not the one setup by the image
# CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
# conda activate "${CONDA_ENV}"
# PYTHON_EXECUTABLE=python bash .ci/scripts/setup-openvino.sh
# PYTHON_EXECUTABLE=python bash .ci/scripts/test_openvino.sh
# test-build-wasm-linux:
# name: test-build-wasm-linux
# uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
# permissions:
# id-token: write
# contents: read
# strategy:
# fail-fast: false
# with:
# runner: linux.2xlarge
# docker-image: ci-image:executorch-ubuntu-22.04-clang12
# submodules: 'recursive'
# ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
# timeout: 90
# script: |
# # The generic Linux job chooses to use base env, not the one setup by the image
# CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
# conda activate "${CONDA_ENV}"
# BUILD_TOOL="cmake"
# PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "${BUILD_TOOL}"
# # Install Node.js and Emscripten
# source .ci/scripts/setup-emscripten.sh
# # Test selective build
# PYTHON_EXECUTABLE=python bash examples/wasm/test_build_wasm.sh
# unittest-wasm-bindings:
# name: unittest-wasm-bindings
# uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
# permissions:
# id-token: write
# contents: read
# strategy:
# matrix:
# enable-etdump: ['', '--enable-etdump']
# fail-fast: false
# with:
# runner: linux.2xlarge
# docker-image: ci-image:executorch-ubuntu-22.04-clang12
# submodules: 'recursive'
# ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
# timeout: 90
# script: |
# # The generic Linux job chooses to use base env, not the one setup by the image
# CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
# conda activate "${CONDA_ENV}"
# BUILD_TOOL="cmake"
# PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "${BUILD_TOOL}"
# # Install Node.js and Emscripten
# source .ci/scripts/setup-emscripten.sh
# # Test selective build
# bash scripts/build_wasm_tests.sh ${{ matrix.enable-etdump }}
# # Install Jest
# cd cmake-out-wasm/extension/wasm/test
# npm install --save-dev jest
# # Run unit test
# npm test
# unittest-nxp-neutron:
# uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
# permissions:
# id-token: write
# contents: read
# with:
# runner: linux.2xlarge
# docker-image: ci-image:executorch-ubuntu-22.04-clang12
# submodules: 'recursive'
# ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
# timeout: 90
# script: |
# set -eux
# # The generic Linux job chooses to use base env, not the one setup by the image
# CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
# conda activate "${CONDA_ENV}"
# # Build and install Executorch
# PYTHON_EXECUTABLE=python \
# CMAKE_ARGS="-DEXECUTORCH_BUILD_NXP_NEUTRON=ON" \
# .ci/scripts/setup-linux.sh --build-tool "cmake"
# # Install test requirements
# pip install -r backends/nxp/requirements-tests-pypi.txt
# pip install -r backends/nxp/requirements-tests-eiq.txt
# # Run pytest
# PYTHON_EXECUTABLE=python bash backends/nxp/run_unittests.sh
# # Run aot examples:
# PYTHON_EXECUTABLE=python bash examples/nxp/run_aot_example.sh cifar10
# PYTHON_EXECUTABLE=python bash examples/nxp/run_aot_example.sh mobilenetv2
# test-samsung-models-linux:
# name: test-samsung-models-linux
# uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
# permissions:
# id-token: write
# contents: read
# secrets: inherit
# with:
# runner: linux.2xlarge
# docker-image: ci-image:executorch-ubuntu-22.04-clang12-android
# submodules: 'recursive'
# ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
# timeout: 90
# secrets-env: SAMSUNG_AI_LITECORE_KEY
# script: |
# set -ex
# # # The generic Linux job chooses to use base env, not the one setup by the image
# # CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
# # conda activate "${CONDA_ENV}"
# # # Setup python
# # PYTHON_EXECUTABLE=python bash .ci/scripts/setup-linux.sh --build-tool "cmake"
# # Setup Samsung SDK (AI Lite Core) and install enn backend
# export SAMSUNG_AI_LITECORE_KEY=$SECRET_SAMSUNG_AI_LITECORE_KEY
# source .ci/scripts/setup-samsung-linux-deps.sh
# # # Test models serially
# # models="mv2 ic3 resnet18 resnet50 mv3 ic4 dl3 edsr vit w2l"
# # for model in $models; do
# # python -m executorch.examples.samsung.aot_compiler --model_name=$model -c E9955
# # done
# # # Test ops
# # python -m unittest discover -s backends/samsung/test/ops -p "test_*.py"
# test-vulkan-models-linux:
# name: test-vulkan-models-linux
# uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
# permissions:
# id-token: write
# contents: read
# with:
# runner: linux.2xlarge
# docker-image: ci-image:executorch-ubuntu-22.04-clang12
# submodules: 'recursive'
# ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
# timeout: 90
# script: |
# set -eux
# # The generic Linux job chooses to use base env, not the one setup by the image
# CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
# conda activate "${CONDA_ENV}"
# # Setup swiftshader and Vulkan SDK which are required to build the Vulkan delegate
# source .ci/scripts/setup-vulkan-linux-deps.sh
# # Setup python
# PYTHON_EXECUTABLE=python \
# CMAKE_ARGS="-DEXECUTORCH_BUILD_VULKAN=ON" \
# .ci/scripts/setup-linux.sh --build-tool "cmake"
# PYTHON_EXECUTABLE=python bash backends/vulkan/test/scripts/test_model.sh --build
# # Test models serially
# models="mv2 mv3 edsr resnet18 resnet50 dl3"
# for model in $models; do
# python -m examples.vulkan.export --model_name=$model --test
# done
# test-vulkan-operators-linux:
# name: test-vulkan-operators-linux
# uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
# permissions:
# id-token: write
# contents: read
# with:
# runner: linux.2xlarge
# docker-image: ci-image:executorch-ubuntu-22.04-clang12
# submodules: 'recursive'
# ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
# timeout: 90
# script: |
# set -eux
# # The generic Linux job chooses to use base env, not the one setup by the image
# CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
# conda activate "${CONDA_ENV}"
# # Setup swiftshader and Vulkan SDK which are required to build the Vulkan delegate
# source .ci/scripts/setup-vulkan-linux-deps.sh
# # Setup python
# PYTHON_EXECUTABLE=python \
# CMAKE_ARGS="-DEXECUTORCH_BUILD_VULKAN=ON" \
# .ci/scripts/setup-linux.sh --build-tool "cmake"
# # Custom operator tests
# PYTHON_EXECUTABLE=python bash backends/vulkan/test/custom_ops/build_and_run.sh add
# ./cmake-out/backends/vulkan/test/custom_ops/q8csw_linear
# ./cmake-out/backends/vulkan/test/custom_ops/q8csw_conv2d
# ./cmake-out/backends/vulkan/test/custom_ops/q4gsw_linear
# ./cmake-out/backends/vulkan/test/custom_ops/choose_qparams_per_row
# # "Classic" Operator tests
# PYTHON_EXECUTABLE=python bash backends/vulkan/test/scripts/test_op.sh --build
# # TODO(ssjia): figure out how to run custom op tests in CI. Currently, they are
# # failing due to to the libstdc++.so.6 installed with conda not supporting
# # GLIBCXX_3.4.30. These tests are still run in Meta internal CI.
# # ./cmake-out/backends/vulkan/test/op_tests/vulkan_sdpa_test
# # Run e2e testing for selected operators. More operators will be tested via this
# # route in the future.
# python -m unittest backends/vulkan/test/test_vulkan_delegate.py -k "*pt2e*"
# python -m unittest backends/vulkan/test/test_vulkan_delegate.py -k "*torchao*"
# nxp-build-test:
# name: nxp-build-test
# uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
# permissions:
# id-token: write
# contents: read
# with:
# runner: linux.2xlarge
# docker-image: ci-image:executorch-ubuntu-22.04-arm-sdk
# submodules: 'recursive'
# ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
# timeout: 90
# script: |
# # The generic Linux job chooses to use base env, not the one setup by the image
# CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
# conda activate "${CONDA_ENV}"
# # Build
# cmake -DEXECUTORCH_BUILD_NXP_NEUTRON=ON -Bcmake-out .
# cmake --build cmake-out --target executorch_delegate_neutron --config Release
# # Build check for the neutron backend library
# lib_neutron="cmake-out/backends/nxp/libexecutorch_delegate_neutron.a"
# if [ -f $lib_neutron ]; then
# echo "Neutron backend library built."
# else
# echo "Neutron backend library not found!"
# exit 1
# fi