Add Metal backend CI workflow with Voxtral testing #5
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test Metal Backend | |
| on: | |
| pull_request: | |
| push: | |
| branches: | |
| - main | |
| - release/* | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}-${{ github.event_name == 'schedule' }} | |
| cancel-in-progress: false | |
| jobs: | |
| test-metal-builds: | |
| name: test-executorch-metal-build | |
| runs-on: macos-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.10' | |
| - name: Test ExecuTorch Metal build | |
| run: | | |
| set -eux | |
| # Test ExecuTorch Metal build | |
| PYTHON_EXECUTABLE=python3 CMAKE_ARGS="-DEXECUTORCH_BUILD_METAL=ON" ./install_executorch.sh | |
| export-voxtral-metal-artifact: | |
| name: export-voxtral-metal-artifact | |
| runs-on: macos-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.10' | |
| - name: Export Voxtral with Metal Backend | |
| env: | |
| EXECUTORCH_HF_TOKEN: ${{ secrets.EXECUTORCH_HF_TOKEN }} | |
| run: | | |
| set -eux | |
| echo "::group::Setup ExecuTorch" | |
| PYTHON_EXECUTABLE=python3 ./install_executorch.sh | |
| echo "::endgroup::" | |
| echo "::group::Setup Huggingface" | |
| pip3 install -U "huggingface_hub[cli]" accelerate | |
| huggingface-cli login --token $EXECUTORCH_HF_TOKEN | |
| OPTIMUM_ET_VERSION=$(cat .ci/docker/ci_commit_pins/optimum-executorch.txt) | |
| pip3 install git+https://github.com/huggingface/optimum-executorch.git@${OPTIMUM_ET_VERSION} | |
| pip3 install mistral-common librosa | |
| pip3 list | |
| echo "::endgroup::" | |
| echo "::group::Export Voxtral" | |
| optimum-cli export executorch \ | |
| --model "mistralai/Voxtral-Mini-3B-2507" \ | |
| --task "multimodal-text-to-text" \ | |
| --recipe "metal" \ | |
| --dtype bfloat16 \ | |
| --max_seq_len 1024 \ | |
| --output_dir ./ | |
| python3 -m executorch.extension.audio.mel_spectrogram \ | |
| --feature_size 128 \ | |
| --stack_output \ | |
| --max_audio_len 300 \ | |
| --output_file voxtral_preprocessor.pte | |
| test -f model.pte | |
| test -f aoti_metal_blob.ptd | |
| test -f voxtral_preprocessor.pte | |
| echo "::endgroup::" | |
| echo "::group::Store Voxtral Artifacts" | |
| mkdir -p artifacts | |
| cp model.pte artifacts/ | |
| cp aoti_metal_blob.ptd artifacts/ | |
| cp voxtral_preprocessor.pte artifacts/ | |
| ls -al artifacts | |
| echo "::endgroup::" | |
| - name: Upload artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: voxtral-metal-export | |
| path: artifacts/ | |
| test-voxtral-metal-e2e: | |
| name: test-voxtral-metal-e2e | |
| needs: export-voxtral-metal-artifact | |
| runs-on: macos-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.10' | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: voxtral-metal-export | |
| path: artifacts/ | |
| - name: Test Voxtral Metal E2E | |
| run: | | |
| set -eux | |
| echo "::group::Setup ExecuTorch Requirements" | |
| CMAKE_ARGS="-DEXECUTORCH_BUILD_METAL=ON" ./install_requirements.sh | |
| pip3 list | |
| echo "::endgroup::" | |
| echo "::group::Prepare Voxtral Artifacts" | |
| cp artifacts/model.pte . | |
| cp artifacts/aoti_metal_blob.ptd . | |
| cp artifacts/voxtral_preprocessor.pte . | |
| TOKENIZER_URL="https://huggingface.co/mistralai/Voxtral-Mini-3B-2507/resolve/main/tekken.json" | |
| curl -L $TOKENIZER_URL -o tekken.json | |
| ls -al model.pte aoti_metal_blob.ptd voxtral_preprocessor.pte tekken.json | |
| echo "::endgroup::" | |
| echo "::group::Create Test Audio File" | |
| say -o call_samantha_hall.aiff "Call Samantha Hall" | |
| afconvert -f WAVE -d LEI16 call_samantha_hall.aiff call_samantha_hall.wav | |
| echo "::endgroup::" | |
| echo "::group::Build Voxtral Runner" | |
| cmake --preset llm \ | |
| -DEXECUTORCH_BUILD_METAL=ON \ | |
| -DCMAKE_INSTALL_PREFIX=cmake-out \ | |
| -DCMAKE_BUILD_TYPE=Release \ | |
| -Bcmake-out -S. | |
| cmake --build cmake-out -j$(( $(sysctl -n hw.ncpu) - 1 )) --target install --config Release | |
| cmake -DEXECUTORCH_BUILD_METAL=ON \ | |
| -DCMAKE_BUILD_TYPE=Release \ | |
| -Sexamples/models/voxtral \ | |
| -Bcmake-out/examples/models/voxtral/ | |
| cmake --build cmake-out/examples/models/voxtral --target voxtral_runner --config Release | |
| echo "::endgroup::" | |
| echo "::group::Run Voxtral Runner" | |
| set +e | |
| OUTPUT=$(cmake-out/examples/models/voxtral/voxtral_runner \ | |
| --model_path model.pte \ | |
| --data_path aoti_metal_blob.ptd \ | |
| --tokenizer_path tekken.json \ | |
| --audio_path call_samantha_hall.wav \ | |
| --processor_path voxtral_preprocessor.pte \ | |
| --temperature 0 2>&1) | |
| EXIT_CODE=$? | |
| set -e | |
| echo "$OUTPUT" | |
| if ! echo "$OUTPUT" | grep -iq "Samantha"; then | |
| echo "Expected output 'Samantha' not found in output" | |
| exit 1 | |
| fi | |
| if [ $EXIT_CODE -ne 0 ]; then | |
| echo "Unexpected exit code: $EXIT_CODE" | |
| exit $EXIT_CODE | |
| fi | |
| echo "::endgroup::" |