fix(workflow): add cmake to linux-musl x86_64 and arm64 dependency in… #391
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build, Test and Release SQLite AI | |
| on: | |
| push: | |
| workflow_dispatch: | |
| permissions: | |
| contents: write | |
| jobs: | |
| build: | |
| runs-on: ${{ matrix.os }} | |
| container: ${{ matrix.container && matrix.container || '' }} | |
| name: ${{ matrix.name }}${{ matrix.arch && format('-{0}', matrix.arch) || '' }} build${{ matrix.arch != 'arm64-v8a' && matrix.name != 'ios-sim' && matrix.name != 'ios' && matrix.name != 'apple-xcframework' && ' + test' || ''}} | |
| timeout-minutes: 120 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - os: macos-15 | |
| name: macos | |
| make: LLAMA="-DGGML_NATIVE=OFF -DGGML_METAL=ON -DGGML_ACCELERATE=ON -DGGML_BLAS=ON -DGGML_BLAS_VENDOR=Apple" WHISPER="-DWHISPER_COREML=ON -DWHISPER_COREML_ALLOW_FALLBACK=ON" | |
| - os: ubuntu-22.04 | |
| arch: x86_64 | |
| name: linux-cpu | |
| make: LLAMA="-DGGML_NATIVE=OFF -DGGML_CPU=ON -DGGML_AVX2=ON" | |
| - os: ubuntu-22.04 | |
| arch: x86_64 | |
| name: linux-gpu | |
| make: LLAMA="-DGGML_NATIVE=OFF -DGGML_CPU=ON -DGGML_VULKAN=ON -DGGML_OPENCL=ON" | |
| - os: ubuntu-22.04-arm | |
| arch: arm64 | |
| name: linux-cpu | |
| make: LLAMA="-DGGML_NATIVE=OFF -DGGML_CPU=ON -DGGML_CPU_ARM_ARCH=armv8.2-a" | |
| - os: ubuntu-22.04-arm | |
| arch: arm64 | |
| name: linux-gpu | |
| make: LLAMA="-DGGML_NATIVE=OFF -DGGML_CPU=ON -DGGML_VULKAN=ON -DGGML_OPENCL=ON" | |
| - os: ubuntu-22.04 | |
| arch: x86_64 | |
| name: linux-musl-cpu | |
| container: alpine:latest | |
| make: LLAMA="-DGGML_NATIVE=OFF -DGGML_CPU=ON -DGGML_AVX2=ON" | |
| - os: ubuntu-22.04 | |
| arch: x86_64 | |
| name: linux-musl-gpu | |
| container: alpine:latest | |
| make: LLAMA="-DGGML_NATIVE=OFF -DGGML_CPU=ON -DGGML_VULKAN=ON -DGGML_OPENCL=ON" | |
| - os: ubuntu-22.04-arm | |
| arch: arm64 | |
| name: linux-musl-cpu | |
| make: LLAMA="-DGGML_NATIVE=OFF -DGGML_CPU=ON -DGGML_CPU_ARM_ARCH=armv8.2-a" | |
| - os: ubuntu-22.04-arm | |
| arch: arm64 | |
| name: linux-musl-gpu | |
| make: LLAMA="-DGGML_NATIVE=OFF -DGGML_CPU=ON -DGGML_VULKAN=ON -DGGML_OPENCL=ON" | |
| - os: windows-2022 | |
| arch: x86_64 | |
| name: windows-cpu | |
| make: LLAMA="-DGGML_NATIVE=OFF -DGGML_CPU=ON -DGGML_AVX2=ON" | |
| - os: windows-2022 | |
| arch: x86_64 | |
| name: windows-gpu | |
| make: LLAMA="-DGGML_NATIVE=OFF -DGGML_CPU=ON -DGGML_VULKAN=ON -DGGML_OPENCL=ON" | |
| - os: ubuntu-22.04 | |
| arch: x86_64 | |
| name: android | |
| make: PLATFORM=android ARCH=x86_64 | |
| sqlite-amalgamation-zip: https://sqlite.org/2025/sqlite-amalgamation-3490100.zip | |
| - os: ubuntu-22.04 | |
| arch: arm64-v8a | |
| name: android | |
| make: PLATFORM=android ARCH=arm64-v8a LLAMA="-DGGML_CPU_ARM_ARCH=armv8.2-a+dotprod" | |
| - os: macos-15 | |
| name: ios | |
| make: PLATFORM=ios LLAMA="-DGGML_NATIVE=OFF -DGGML_METAL=ON -DGGML_ACCELERATE=ON -DGGML_BLAS=ON -DGGML_BLAS_VENDOR=Apple" WHISPER="-DWHISPER_COREML=ON -DWHISPER_COREML_ALLOW_FALLBACK=ON" | |
| - os: macos-15 | |
| name: ios-sim | |
| make: PLATFORM=ios-sim LLAMA="-DGGML_NATIVE=OFF -DGGML_METAL=ON -DGGML_ACCELERATE=ON -DGGML_BLAS=ON -DGGML_BLAS_VENDOR=Apple" WHISPER="-DWHISPER_COREML=ON -DWHISPER_COREML_ALLOW_FALLBACK=ON" | |
| - os: macos-15 | |
| name: apple-xcframework | |
| make: xcframework LLAMA="-DGGML_NATIVE=OFF -DGGML_METAL=ON -DGGML_ACCELERATE=ON -DGGML_BLAS=ON -DGGML_BLAS_VENDOR=Apple" WHISPER="-DWHISPER_COREML=ON -DWHISPER_COREML_ALLOW_FALLBACK=ON" | |
| defaults: | |
| run: | |
| shell: ${{ matrix.container && 'sh' || 'bash' }} | |
| steps: | |
| - name: linux-musl x86_64 install dependencies | |
| if: contains(matrix.name, 'linux-musl') && matrix.arch == 'x86_64' | |
| run: apk update && apk add --no-cache git gcc g++ make cmake sqlite musl-dev linux-headers | |
| - uses: actions/[email protected] | |
| with: | |
| submodules: true | |
| - name: calculate cache version hashes for modules and dependencies | |
| id: submodule-hashes | |
| env: | |
| MATRIX_MAKE: ${{ matrix.make && matrix.make || '' }} | |
| run: | | |
| LLAMA_HASH=$(git -C modules/llama.cpp rev-parse HEAD) | |
| WHISPER_HASH=$(git -C modules/whisper.cpp rev-parse HEAD) | |
| MINIAUDIO_HASH=$(git -C modules/miniaudio rev-parse HEAD) | |
| if command -v sha256sum >/dev/null 2>&1; then | |
| MAKE_HASH=$(echo "$MATRIX_MAKE" | sha256sum | cut -d' ' -f1) | |
| MAKEFILE_HASH=$(sha256sum Makefile | cut -d' ' -f1) | |
| DPCPP_MKL_HASH=$(echo "$WINDOWS_DPCPP_MKL" | sha256sum | cut -d' ' -f1) | |
| BASEKIT_URL_HASH=$(echo "$WINDOWS_BASEKIT_URL" | sha256sum | cut -d' ' -f1) | |
| elif command -v shasum >/dev/null 2>&1; then | |
| MAKE_HASH=$(echo "$MATRIX_MAKE" | shasum -a 256 | cut -d' ' -f1) | |
| MAKEFILE_HASH=$(shasum -a 256 Makefile | cut -d' ' -f1) | |
| DPCPP_MKL_HASH=$(echo "$WINDOWS_DPCPP_MKL" | shasum -a 256 | cut -d' ' -f1) | |
| BASEKIT_URL_HASH=$(echo "$WINDOWS_BASEKIT_URL" | shasum -a 256 | cut -d' ' -f1) | |
| else | |
| MAKE_HASH=$(echo "$MATRIX_MAKE" | openssl dgst -sha256 | cut -d' ' -f2) | |
| MAKEFILE_HASH=$(openssl dgst -sha256 Makefile | cut -d' ' -f2) | |
| DPCPP_MKL_HASH=$(echo "$WINDOWS_DPCPP_MKL" | openssl dgst -sha256 | cut -d' ' -f2) | |
| BASEKIT_URL_HASH=$(echo "$WINDOWS_BASEKIT_URL" | openssl dgst -sha256 | cut -d' ' -f2) | |
| fi | |
| echo "llama=$LLAMA_HASH" >> $GITHUB_OUTPUT | |
| echo "whisper=$WHISPER_HASH" >> $GITHUB_OUTPUT | |
| echo "miniaudio=$MINIAUDIO_HASH" >> $GITHUB_OUTPUT | |
| echo "make=$MAKE_HASH" >> $GITHUB_OUTPUT | |
| echo "makefile=$MAKEFILE_HASH" >> $GITHUB_OUTPUT | |
| echo "dpcpp-mkl=$DPCPP_MKL_HASH" >> $GITHUB_OUTPUT | |
| echo "basekit-url=$BASEKIT_URL_HASH" >> $GITHUB_OUTPUT | |
| - uses: msys2/[email protected] | |
| if: matrix.os == 'windows-2022' | |
| with: | |
| msystem: mingw64 | |
| install: >- | |
| git | |
| make | |
| sqlite | |
| mingw-w64-x86_64-cc | |
| mingw-w64-x86_64-cmake | |
| ${{ matrix.name == 'windows-gpu' && 'mingw-w64-x86_64-vulkan-headers' || '' }} | |
| ${{ matrix.name == 'windows-gpu' && 'mingw-w64-x86_64-vulkan-loader' || '' }} | |
| ${{ matrix.name == 'windows-gpu' && 'mingw-w64-x86_64-shaderc' || '' }} | |
| ${{ matrix.name == 'windows-gpu' && 'mingw-w64-x86_64-opencl-headers' || '' }} | |
| ${{ matrix.name == 'windows-gpu' && 'mingw-w64-x86_64-opencl-icd' || '' }} | |
| - name: macos install dependencies | |
| if: matrix.name == 'macos' | |
| run: brew link sqlite --force | |
| - name: linux-musl arm64 setup container | |
| if: contains(matrix.name, 'linux-musl') && matrix.arch == 'arm64' | |
| run: | | |
| docker run -d --name alpine \ | |
| --platform linux/arm64 \ | |
| -v ${{ github.workspace }}:/workspace \ | |
| -w /workspace \ | |
| alpine:latest \ | |
| tail -f /dev/null | |
| docker exec alpine sh -c "apk update && apk add --no-cache gcc g++ make cmake sqlite musl-dev linux-headers" | |
| - name: linux install opencl | |
| if: matrix.name == 'linux-gpu' | |
| run: sudo apt-get install -y opencl-headers ocl-icd-opencl-dev | |
| - name: linux-x86_64 install vulkan | |
| if: matrix.name == 'linux-gpu' && matrix.arch == 'x86_64' | |
| run: | | |
| wget -qO - https://packages.lunarg.com/lunarg-signing-key-pub.asc | sudo apt-key add - | |
| sudo wget -qO /etc/apt/sources.list.d/lunarg-vulkan-jammy.list https://packages.lunarg.com/vulkan/lunarg-vulkan-jammy.list | |
| sudo apt-get update -y | |
| sudo apt-get install -y mesa-vulkan-drivers | |
| # Vulkan is no longer packed for Ubuntu | |
| wget https://sdk.lunarg.com/sdk/download/latest/linux/vulkan-sdk.tar.xz?Human=true -O vulkan-sdk.tar.xz | |
| tar -xf vulkan-sdk.tar.xz | |
| cd $(ls -d 1.* | head -n1) | |
| source setup-env.sh | |
| echo "VULKAN_SDK=$VULKAN_SDK" >> $GITHUB_ENV | |
| echo "PATH=$PATH" >> $GITHUB_ENV | |
| echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" >> $GITHUB_ENV | |
| echo "VK_ADD_LAYER_PATH=$VK_ADD_LAYER_PATH" >> $GITHUB_ENV | |
| - name: linux-arm64 install vulkan | |
| if: matrix.name == 'linux-gpu' && matrix.arch == 'arm64' | |
| run: | | |
| sudo dpkg --add-architecture arm64 | |
| # Add arch-specific repositories for non-amd64 architectures | |
| cat << EOF | sudo tee /etc/apt/sources.list.d/arm64-ports.list | |
| deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ noble main universe | |
| deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ noble-updates main universe | |
| deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ noble-security main universe | |
| deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ noble-backports main universe | |
| EOF | |
| sudo apt-get update || true ;# Prevent failure due to missing URLs. | |
| sudo apt-get install -y --no-install-recommends build-essential glslc crossbuild-essential-arm64 libvulkan-dev:arm64 | |
| - name: cache llama.cpp build | |
| id: cache-llama | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| build/ggml | |
| build/llama.cpp | |
| build/llama.cpp.stamp | |
| key: llama-${{ matrix.name }}-${{ matrix.os }}-${{ matrix.arch }}-${{ steps.submodule-hashes.outputs.llama }}-${{ hashFiles('modules/llama.cpp/**') }}-${{ steps.submodule-hashes.outputs.make }}-${{ steps.submodule-hashes.outputs.makefile }} | |
| - name: cache whisper.cpp build | |
| id: cache-whisper | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| build/whisper.cpp | |
| build/whisper.cpp.stamp | |
| key: whisper-${{ matrix.name }}-${{ matrix.os }}-${{ matrix.arch }}-${{ steps.submodule-hashes.outputs.whisper }}-${{ hashFiles('modules/whisper.cpp/**') }}-${{ steps.submodule-hashes.outputs.make }}-${{ steps.submodule-hashes.outputs.makefile }} | |
| - name: cache miniaudio build | |
| id: cache-miniaudio | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| build/miniaudio | |
| build/miniaudio.stamp | |
| key: miniaudio-${{ matrix.name }}-${{ matrix.os }}-${{ matrix.arch }}-${{ steps.submodule-hashes.outputs.miniaudio }}-${{ hashFiles('modules/miniaudio/**') }}-${{ steps.submodule-hashes.outputs.make }}-${{ steps.submodule-hashes.outputs.makefile }} | |
| - name: windows build llama.cpp | |
| if: matrix.os == 'windows-2022' && steps.cache-llama.outputs.cache-hit != 'true' | |
| shell: msys2 {0} | |
| run: make build/llama.cpp.stamp ${{ matrix.make && matrix.make || ''}} | |
| env: | |
| VULKAN_SDK: "C:/msys64/mingw64" | |
| - name: unix build llama.cpp | |
| if: matrix.os != 'windows-2022' && steps.cache-llama.outputs.cache-hit != 'true' | |
| run: ${{ contains(matrix.name, 'linux-musl') && matrix.arch == 'arm64' && 'docker exec alpine' || '' }} make build/llama.cpp.stamp ${{ matrix.make && matrix.make || ''}} | |
| - name: windows build whisper.cpp | |
| if: matrix.os == 'windows-2022' && steps.cache-whisper.outputs.cache-hit != 'true' | |
| shell: msys2 {0} | |
| run: make build/whisper.cpp.stamp ${{ matrix.make && matrix.make || ''}} | |
| env: | |
| VULKAN_SDK: "C:/msys64/mingw64" | |
| - name: unix build whisper.cpp | |
| if: matrix.os != 'windows-2022' && steps.cache-whisper.outputs.cache-hit != 'true' | |
| run: ${{ contains(matrix.name, 'linux-musl') && matrix.arch == 'arm64' && 'docker exec alpine' || '' }} make build/whisper.cpp.stamp ${{ matrix.make && matrix.make || ''}} | |
| - name: windows build miniaudio | |
| if: matrix.os == 'windows-2022' && steps.cache-miniaudio.outputs.cache-hit != 'true' | |
| shell: msys2 {0} | |
| run: make build/miniaudio.stamp ${{ matrix.make && matrix.make || ''}} | |
| - name: unix build miniaudio | |
| if: matrix.os != 'windows-2022' && steps.cache-miniaudio.outputs.cache-hit != 'true' | |
| run: ${{ contains(matrix.name, 'linux-musl') && matrix.arch == 'arm64' && 'docker exec alpine' || '' }} make build/miniaudio.stamp ${{ matrix.make && matrix.make || ''}} | |
| - name: windows build sqlite-ai | |
| if: matrix.os == 'windows-2022' | |
| run: make extension ${{ matrix.make && matrix.make || ''}} | |
| shell: msys2 {0} | |
| env: | |
| VULKAN_SDK: "C:/msys64/mingw64" | |
| - name: unix build sqlite-ai | |
| if: matrix.os != 'windows-2022' | |
| run: ${{ contains(matrix.name, 'linux-musl') && matrix.arch == 'arm64' && 'docker exec alpine' || '' }} make extension ${{ matrix.make && matrix.make || ''}} | |
| - name: create keychain for codesign | |
| if: matrix.os == 'macos-15' | |
| run: | | |
| echo "${{ secrets.APPLE_CERTIFICATE }}" | base64 --decode > certificate.p12 | |
| security create-keychain -p "${{ secrets.KEYCHAIN_PASSWORD }}" build.keychain | |
| security default-keychain -s build.keychain | |
| security unlock-keychain -p "${{ secrets.KEYCHAIN_PASSWORD }}" build.keychain | |
| security import certificate.p12 -k build.keychain -P "${{ secrets.CERTIFICATE_PASSWORD }}" -T /usr/bin/codesign | |
| security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k "${{ secrets.KEYCHAIN_PASSWORD }}" build.keychain | |
| - name: codesign dylib | |
| if: matrix.os == 'macos-15' && matrix.name != 'apple-xcframework' | |
| run: codesign --sign "${{ secrets.APPLE_TEAM_ID }}" --timestamp --options runtime dist/ai.dylib | |
| - name: codesign and notarize xcframework | |
| if: matrix.name == 'apple-xcframework' | |
| run: | | |
| find dist/ai.xcframework -name "*.framework" -exec echo "Signing: {}" \; -exec codesign --sign "${{ secrets.APPLE_TEAM_ID }}" --timestamp --options runtime {} \; # Sign each individual framework FIRST | |
| codesign --sign "${{ secrets.APPLE_TEAM_ID }}" --timestamp --options runtime dist/ai.xcframework # Then sign the xcframework wrapper | |
| ditto -c -k --keepParent dist/ai.xcframework dist/ai.xcframework.zip | |
| xcrun notarytool submit dist/ai.xcframework.zip --apple-id "${{ secrets.APPLE_ID }}" --password "${{ secrets.APPLE_PASSWORD }}" --team-id "${{ secrets.APPLE_TEAM_ID }}" --wait | |
| rm dist/ai.xcframework.zip | |
| - name: cleanup keychain for codesign | |
| if: matrix.os == 'macos-15' | |
| run: | | |
| rm certificate.p12 | |
| security delete-keychain build.keychain | |
| - name: android setup test environment | |
| if: matrix.name == 'android' && matrix.arch != 'arm64-v8a' | |
| run: | | |
| echo "::group::enable kvm group perms" | |
| echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules | |
| sudo udevadm control --reload-rules | |
| sudo udevadm trigger --name-match=kvm | |
| echo "::endgroup::" | |
| echo "::group::download and build sqlite3 without SQLITE_OMIT_LOAD_EXTENSION" | |
| curl -O ${{ matrix.sqlite-amalgamation-zip }} | |
| unzip sqlite-amalgamation-*.zip | |
| export ${{ matrix.make }} | |
| $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/${{ matrix.arch }}-linux-android26-clang sqlite-amalgamation-*/shell.c sqlite-amalgamation-*/sqlite3.c -o sqlite3 -ldl | |
| # remove unused folders to save up space | |
| rm -rf sqlite-amalgamation-*.zip sqlite-amalgamation-* | |
| echo "::endgroup::" | |
| echo "::group::prepare the test script" | |
| make test PLATFORM=$PLATFORM ARCH=$ARCH || echo "It should fail. Running remaining commands in the emulator" | |
| cat > commands.sh << EOF | |
| mv -f /data/local/tmp/sqlite3 /system/xbin | |
| cd /data/local/tmp | |
| $(make test PLATFORM=$PLATFORM ARCH=$ARCH -n) | |
| EOF | |
| # remove big unused folders to avoid emulator errors | |
| echo "::endgroup::" | |
| - name: android test sqlite-ai | |
| if: matrix.name == 'android' && matrix.arch != 'arm64-v8a' | |
| uses: reactivecircus/[email protected] | |
| with: | |
| api-level: 26 | |
| arch: ${{ matrix.arch }} | |
| script: | | |
| adb root | |
| adb remount | |
| adb push ${{ github.workspace }}/commands.sh /data/local/tmp/ | |
| adb push ${{ github.workspace }}/sqlite3 /data/local/tmp/ | |
| adb push ${{ github.workspace }}/dist /data/local/tmp/ | |
| adb push ${{ github.workspace }}/Makefile /data/local/tmp/ | |
| adb shell "sh /data/local/tmp/commands.sh" | |
| - name: windows test sqlite-ai | |
| if: matrix.os == 'windows-2022' | |
| run: make test ${{ matrix.make && matrix.make || ''}} | |
| shell: msys2 {0} | |
| env: | |
| VULKAN_SDK: "C:/msys64/mingw64" | |
| - name: unix test sqlite-ai | |
| if: contains(matrix.name, 'linux') || matrix.name == 'macos' | |
| run: ${{ contains(matrix.name, 'linux-musl') && matrix.arch == 'arm64' && 'docker exec alpine' || '' }} make test ${{ matrix.make && matrix.make || ''}} | |
| - uses: actions/[email protected] | |
| if: always() | |
| with: | |
| name: ai-${{ matrix.name }}${{ matrix.arch && format('-{0}', matrix.arch) || '' }} | |
| path: dist/ai.* | |
| if-no-files-found: error | |
| release: | |
| runs-on: ubuntu-22.04 | |
| name: release | |
| needs: build | |
| if: github.ref == 'refs/heads/main' | |
| steps: | |
| - uses: actions/[email protected] | |
| - uses: actions/[email protected] | |
| with: | |
| path: artifacts | |
| - name: release tag version from sqlite-ai.h | |
| id: tag | |
| run: | | |
| VERSION=$(make version) | |
| if [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then | |
| LATEST=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" https://api.github.com/repos/${{ github.repository }}/releases/latest | jq -r '.name') | |
| if [[ "$VERSION" != "$LATEST" || "$GITHUB_EVENT_NAME" == "workflow_dispatch" ]]; then | |
| echo "version=$VERSION" >> $GITHUB_OUTPUT | |
| else | |
| echo "::warning file=src/sqlite-ai.h::To release a new version, please update the SQLITE_AI_VERSION in src/sqlite-ai.h to be different than the latest $LATEST" | |
| fi | |
| exit 0 | |
| fi | |
| echo "❌ SQLITE_AI_VERSION not found in sqlite-ai.h" | |
| exit 1 | |
| - name: zip artifacts | |
| run: | | |
| for folder in "artifacts"/*; do | |
| if [ -d "$folder" ]; then | |
| name=$(basename "$folder") | |
| if [[ "$name" != "ai-apple-xcframework" ]]; then | |
| tar -czf "${name}-${{ steps.tag.outputs.version }}.tar.gz" -C "$folder" . | |
| fi | |
| (cd "$folder" && zip -rq "../../${name}-${{ steps.tag.outputs.version }}.zip" .) | |
| fi | |
| done | |
| - uses: softprops/[email protected] | |
| if: steps.tag.outputs.version != '' | |
| with: | |
| generate_release_notes: true | |
| tag_name: ${{ steps.tag.outputs.version }} | |
| files: | | |
| ai-*-${{ steps.tag.outputs.version }}.zip | |
| ai-*-${{ steps.tag.outputs.version }}.tar.gz | |
| make_latest: true |