diff --git a/.github/workflows/trunk.yml b/.github/workflows/trunk.yml index 751649ed7da..9b801801e08 100644 --- a/.github/workflows/trunk.yml +++ b/.github/workflows/trunk.yml @@ -1043,45 +1043,6 @@ jobs: build-tool: cmake docker-image: ci-image:executorch-ubuntu-22.04-clang12 - test-mcu-models: - name: test-mcu-models - uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main - strategy: - matrix: - include: - - build-tool: cmake - fail-fast: false - permissions: - id-token: write - contents: read - with: - runner: linux.2xlarge - docker-image: ci-image:executorch-ubuntu-22.04-arm-sdk - submodules: 'recursive' - ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} - timeout: 90 - script: | - BUILD_TOOL=${{ matrix.build-tool }} - - # The generic Linux job chooses to use base env, not the one setup by the image - CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") - conda activate "${CONDA_ENV}" - - # Try to mirror these as closely as possible - source .ci/scripts/utils.sh - install_executorch "--use-pt-pinned-commit" - - .ci/scripts/setup-arm-baremetal-tools.sh - source examples/arm/ethos-u-scratch/setup_path.sh - - # Run selective Build - chmod +x examples/selective_build/test_selective_build.sh - examples/selective_build/test_selective_build.sh "${BUILD_TOOL}" - - # Run MCU models - chmod +x examples/arm/run_mcu_models_fvp.sh - examples/arm/run_mcu_models_fvp.sh --target=cortex-m55 - test-models-windows: uses: pytorch/test-infra/.github/workflows/windows_job.yml@main strategy: diff --git a/examples/arm/run_mcu_models_fvp.sh b/examples/arm/run_mcu_models_fvp.sh deleted file mode 100755 index 3fa980c506b..00000000000 --- a/examples/arm/run_mcu_models_fvp.sh +++ /dev/null @@ -1,309 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# Copyright 2023-2025 Arm Limited and/or its affiliates. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. - -# Prerequisite steps: (run the following commands before running this script) -# 1. Setup your environment for Arm FVP -# a. Setup Conda environment / venv -# b. ./install_executorch.sh --clean ; ./install_executorch.sh --editable; -# c. examples/arm/setup.sh --i-agree-to-the-contained-eula; -# d. source examples/arm/ethos-u-scratch/setup_path.sh -# 2. bash examples/selective_build/test_selective_build.sh cmake - -set -u - -# Valid targets for MCU model validation -VALID_TARGETS=( - "cortex-m55" - "cortex-m85" -) - -# Default models for MCU validation with portable kernels -DEFAULT_MODELS=(mv2 mv3 lstm qadd qlinear) -# Available models (on FVP) -AVAILABLE_MODELS=(mv2 mv3 lstm qadd qlinear) -# Add the following models if you want to enable them later (atm they are not working on FVP) -# edsr w2l ic3 ic4 resnet18 resnet50 - -# Variables -TARGET="" -MODELS=() -PASSED_MODELS=() -FAILED_MODELS=() - -# Function to validate target -validate_target() { - local target=$1 - for valid_target in "${VALID_TARGETS[@]}"; do - if [[ "$target" == "$valid_target" ]]; then - return 0 - fi - done - return 1 -} - -# Function to validate models -validate_models() { - local invalid_models=() - for model in "${MODELS[@]}"; do - if [[ ! " ${AVAILABLE_MODELS[*]} " =~ " $model " ]]; then - invalid_models+=("$model") - fi - done - - if [[ ${#invalid_models[@]} -gt 0 ]]; then - echo "❌ Error: Invalid model(s): ${invalid_models[*]}" - echo "Available models: ${AVAILABLE_MODELS[*]}" - return 1 - fi - return 0 -} - -cpu_to_ethos_target() { - local cpu=$1 - case $cpu in - cortex-m55) - echo "ethos-u55-128" - ;; - cortex-m85) - echo "ethos-u85-128" - ;; - *) - echo "Unknown CPU: $cpu" >&2 - return 1 - ;; - esac -} - -# Function to show usage -show_usage() { - echo "Usage: $0 --target= [--models=]" - echo "" - echo "MCU Model Validation without delegation" - echo "" - echo "Required arguments:" - echo " --target= Target platform for validation" - echo "" - echo "Optional arguments:" - echo " --models= Comma-separated list of models to test" - echo " (overrides default model list)" - echo "" - echo "Valid targets:" - printf ' %s\n' "${VALID_TARGETS[@]}" - echo "" - echo "Available models:" - printf ' %s\n' "${AVAILABLE_MODELS[@]}" - echo "" - echo "Examples:" - echo " $0 --target=ethos-u85-128" - echo " $0 --target=ethos-u55-128 --models=mv2,mv3,resnet18" - echo "" - echo "Default behavior:" - echo " - Uses all available models: ${DEFAULT_MODELS[*]}" - echo " - Runs with portable kernels (no delegation)" -} - -# Function to display summary -show_summary() { - local total_models=${#MODELS[@]} - - echo "" - echo "════════════════════════════════════════════════════════════════" - echo "🏁 MCU MODEL VALIDATION SUMMARY - TARGET: $TARGET" - echo "════════════════════════════════════════════════════════════════" - echo "" - - # Show individual results - for model in "${MODELS[@]}"; do - if [[ " ${PASSED_MODELS[*]} " =~ " $model " ]]; then - printf "%-12s : ✅ Passed\n" "$model" - elif [[ " ${FAILED_MODELS[*]} " =~ " $model " ]]; then - printf "%-12s : ❌ Failed\n" "$model" - else - printf "%-12s : ⏭️ Skipped\n" "$model" - fi - done - - echo "" - echo "────────────────────────────────────────────────────────────────" - - # Show statistics - local passed_count=${#PASSED_MODELS[@]} - local failed_count=${#FAILED_MODELS[@]} - local success_rate=$((passed_count * 100 / total_models)) - - echo "📊 STATISTICS:" - echo " Total Models : $total_models" - echo " ✅ Passed : $passed_count" - echo " ❌ Failed : $failed_count" - echo " 📈 Success Rate : $success_rate%" - echo "" - - # Show model selection info - if [[ ${#MODELS[@]} -eq ${#DEFAULT_MODELS[@]} ]] && [[ "${MODELS[*]}" == "${DEFAULT_MODELS[*]}" ]]; then - echo "📋 Model Selection: Default (all available models)" - else - echo "📋 Model Selection: Custom (${MODELS[*]})" - fi - echo "" - - # Overall result - if [[ $failed_count -eq 0 ]]; then - echo "🎉 OVERALL RESULT: ALL TESTS PASSED!" - echo "🔧 Mode: Portable Kernels (No Delegation)" - else - echo "⚠️ OVERALL RESULT: $failed_count/$total_models TESTS FAILED" - echo "🔧 Mode: Portable Kernels (No Delegation)" - echo "" - echo "🔍 Failed models: ${FAILED_MODELS[*]}" - fi - - echo "════════════════════════════════════════════════════════════════" - echo "" -} - -# Parse command line arguments -while [[ $# -gt 0 ]]; do - case $1 in - --target=*) - TARGET="${1#*=}" - shift - ;; - --models=*) - IFS=',' read -ra MODELS <<< "${1#*=}" - shift - ;; - -h|--help) - show_usage - exit 0 - ;; - *) - echo "❌ Error: Unknown argument '$1'" - echo "" - show_usage - exit 1 - ;; - esac -done - -# Check if target is provided -if [[ -z "$TARGET" ]]; then - echo "❌ Error: --target argument is required" - echo "" - show_usage - exit 1 -fi - -# Validate target -if ! validate_target "$TARGET"; then - echo "❌ Error: Invalid target '$TARGET'" - echo "" - show_usage - exit 1 -fi - -# Use default models if none specified -if [[ ${#MODELS[@]} -eq 0 ]]; then - MODELS=("${DEFAULT_MODELS[@]}") -fi - -# Validate models -if ! validate_models; then - exit 1 -fi - -# Remove duplicates from models array -IFS=" " read -r -a MODELS <<< "$(printf '%s\n' "${MODELS[@]}" | sort -u | tr '\n' ' ')" - -echo "🎯 MCU Model Validation - Target: $TARGET" -echo "📋 Processing models: ${MODELS[*]}" -echo "🔧 Mode: Portable Kernels (No Delegation)" -echo "" - -echo "🔨 Building ExecuteTorch libraries (one-time setup)..." -if ! backends/arm/scripts/build_executorch.sh; then - echo "❌ Failed to build ExecuteTorch libraries" - exit 1 -fi -echo "✅ ExecuteTorch libraries built successfully" -echo "" - -ETHOS_TARGET=$(cpu_to_ethos_target "$TARGET") -if [[ $? -ne 0 ]]; then - echo "Invalid CPU target: $TARGET" - exit 1 -fi -echo "Using ETHOS target: $ETHOS_TARGET" - -# Process each model -for model in "${MODELS[@]}"; do - echo "=== 🚀 Processing $model for $TARGET ===" - - # Track if this model succeeds - MODEL_SUCCESS=true - - # Step 1: Create directory - echo "📁 Creating directory arm_test/$model" - mkdir -p "arm_test/$model" - - # Step 2: AOT compilation (quantized, no delegation = portable kernels) - echo "⚙️ AOT compilation for $model" - if ! python3 -m examples.arm.aot_arm_compiler \ - -m "$model" \ - --target="$ETHOS_TARGET" \ - --quantize \ - --enable_qdq_fusion_pass \ - --output="arm_test/$model"; then - echo "❌ AOT compilation failed for $model" - MODEL_SUCCESS=false - fi - - # Step 3: Build executor runner (only if AOT succeeded) - if [[ "$MODEL_SUCCESS" == true ]]; then - echo "🔨 Building executor runner for $model" - if ! backends/arm/scripts/build_executor_runner.sh \ - --pte="arm_test/$model/${model}_arm_${ETHOS_TARGET}.pte" \ - --target="$ETHOS_TARGET" \ - --output="arm_test/$model"; then - echo "❌ Executor runner build failed for $model" - MODEL_SUCCESS=false - fi - fi - - # Step 4: Run on FVP (only if build succeeded) - if [[ "$MODEL_SUCCESS" == true ]]; then - echo "🏃 Running $model on FVP with portable kernels" - if ! backends/arm/scripts/run_fvp.sh \ - --elf="arm_test/$model/arm_executor_runner" \ - --target="$ETHOS_TARGET"; then - echo "❌ FVP execution failed for $model" - MODEL_SUCCESS=false - fi - fi - - # Record result - if [[ "$MODEL_SUCCESS" == true ]]; then - echo "✅ $model completed successfully" - PASSED_MODELS+=("$model") - else - echo "❌ $model failed" - FAILED_MODELS+=("$model") - fi - - echo "" -done - -# Show comprehensive summary -show_summary - -# Exit with appropriate code for CI -if [[ ${#FAILED_MODELS[@]} -eq 0 ]]; then - exit 0 # Success -else - exit 1 # Failure -fi