Build Multi-Version Packages #51
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build Multi-Version Packages | |
| on: | |
| schedule: | |
| # Run weekly on Sunday at 2am UTC | |
| - cron: '0 2 * * 0' | |
| workflow_dispatch: | |
| inputs: | |
| packages: | |
| description: 'Comma-separated package domains (empty = all)' | |
| required: false | |
| type: string | |
| max-versions: | |
| description: 'Maximum versions to build per package' | |
| required: false | |
| type: number | |
| default: 5 | |
| force: | |
| description: 'Force re-upload even if already in S3' | |
| required: false | |
| type: boolean | |
| default: false | |
| concurrency: | |
| group: build-versions | |
| cancel-in-progress: false | |
| env: | |
| AWS_REGION: us-east-1 | |
| S3_BUCKET: pantry-registry | |
| jobs: | |
| # Discover how many buildable packages exist for batching | |
| discover: | |
| name: Discover buildable packages | |
| if: github.event.inputs.packages == '' || github.event.inputs.packages == null | |
| runs-on: ubuntu-latest | |
| outputs: | |
| batches: ${{ steps.discover.outputs.batches }} | |
| total: ${{ steps.discover.outputs.total }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v6 | |
| - name: Setup Bun | |
| uses: oven-sh/setup-bun@v2 | |
| with: | |
| bun-version: latest | |
| - name: Install dependencies | |
| run: bun install | |
| - name: Discover packages | |
| id: discover | |
| run: | | |
| cd packages/ts-pantry | |
| TOTAL=$(bun scripts/build-all-packages.ts --count-only 2>/dev/null | tail -1) | |
| echo "total=$TOTAL" | |
| echo "total=$TOTAL" >> $GITHUB_OUTPUT | |
| # Use smaller batches for multi-version (more work per package) | |
| BATCH_SIZE=25 | |
| LAST_BATCH=$(( (TOTAL - 1) / BATCH_SIZE )) | |
| BATCHES=$(python3 -c "import json; print(json.dumps(list(range($LAST_BATCH + 1))))") | |
| echo "batches=$BATCHES" | |
| echo "batches=$BATCHES" >> $GITHUB_OUTPUT | |
| # Targeted multi-version build (specific packages) | |
| build-targeted: | |
| name: Build targeted (${{ matrix.platform.name }}) | |
| if: github.event.inputs.packages != '' && github.event.inputs.packages != null | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| platform: | |
| - os: macos-15 | |
| name: darwin-arm64 | |
| - os: ubuntu-latest | |
| name: linux-x86-64 | |
| runs-on: ${{ matrix.platform.os }} | |
| permissions: | |
| contents: read | |
| timeout-minutes: 120 | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v6 | |
| - name: Setup Bun | |
| uses: oven-sh/setup-bun@v2 | |
| with: | |
| bun-version: latest | |
| - name: Install dependencies | |
| run: bun install | |
| - name: Install build tools (Linux) | |
| if: runner.os == 'Linux' | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y \ | |
| build-essential libreadline-dev zlib1g-dev libssl-dev \ | |
| cmake meson ninja-build patchelf pkg-config \ | |
| autoconf automake libtool gettext intltool \ | |
| libffi-dev libexpat1-dev libbz2-dev liblzma-dev \ | |
| libncurses5-dev libsqlite3-dev \ | |
| python3-venv python3-pip python3-dev \ | |
| libcurl4-openssl-dev libxml2-dev libonig-dev \ | |
| ruby ruby-dev scons \ | |
| gperf bison flex texinfo nasm yasm \ | |
| libgmp-dev libmpfr-dev libmpc-dev \ | |
| libuv1-dev libpcre2-dev libevent-dev \ | |
| libglib2.0-dev libpixman-1-dev \ | |
| libgit2-dev libssh2-1-dev libhttp-parser-dev \ | |
| libasound2-dev libdbus-1-dev libsystemd-dev \ | |
| protobuf-compiler libprotobuf-dev \ | |
| nettle-dev libp11-kit-dev \ | |
| libboost-all-dev libsndfile1-dev \ | |
| libjpeg-dev libpng-dev libtiff-dev libwebp-dev \ | |
| libfreetype-dev libharfbuzz-dev \ | |
| libltdl-dev libacl1-dev libattr1-dev \ | |
| liblz4-dev libzstd-dev libxxhash-dev \ | |
| libyaml-dev libjansson-dev \ | |
| libedit-dev libelf-dev \ | |
| tcl-dev tk-dev \ | |
| libunwind-dev libdw-dev \ | |
| libzip-dev liblapack-dev gfortran \ | |
| python3-lxml python3-libxml2 \ | |
| libx11-dev libxext-dev libxrender-dev libxrandr-dev \ | |
| libxinerama-dev libxcomposite-dev libxdamage-dev \ | |
| libxcursor-dev libxi-dev libxtst-dev \ | |
| libxmu-dev libxt-dev libxaw7-dev libxres-dev \ | |
| libsm-dev libice-dev libxpm-dev \ | |
| libxxf86vm-dev libxshmfence-dev libxv-dev libxss-dev \ | |
| x11proto-dev xtrans-dev xutils-dev xauth \ | |
| libxkbcommon-dev libxkbfile-dev \ | |
| libxcb-render0-dev libxcb-shape0-dev libxcb-xfixes0-dev \ | |
| libxcb-image0-dev libxcb-util-dev libxcb-keysyms1-dev \ | |
| libxcb-randr0-dev libxcb-shm0-dev \ | |
| libwayland-dev libfontconfig1-dev \ | |
| gobject-introspection libgirepository1.0-dev \ | |
| libatk1.0-dev libcairo2-dev libcairo-gobject2 \ | |
| libpango1.0-dev libgtk-3-dev \ | |
| librsvg2-dev libsecret-1-dev libjson-glib-dev \ | |
| libsoup2.4-dev libgdk-pixbuf-2.0-dev \ | |
| libva-dev libegl-dev libgbm-dev libdrm-dev \ | |
| libgl-dev libglu1-mesa-dev \ | |
| clang llvm-dev libclang-dev \ | |
| libgnutls28-dev liblua5.4-dev \ | |
| libcap-dev libcap-ng-dev libslirp-dev libssh-dev \ | |
| libspeexdsp-dev libmaxminddb-dev libsmi2-dev \ | |
| libc-ares-dev libre2-dev libpcap-dev \ | |
| libnl-3-dev libnl-genl-3-dev \ | |
| libpam0g-dev libudev-dev \ | |
| libarchive-dev libusb-1.0-0-dev \ | |
| libjemalloc-dev libvterm-dev \ | |
| libxslt1-dev \ | |
| libsdl2-dev libplist-dev \ | |
| libassuan-dev libgpg-error-dev \ | |
| libfido2-dev \ | |
| libdouble-conversion-dev libgoogle-glog-dev libgflags-dev \ | |
| libsodium-dev liblzo2-dev libsnappy-dev libbrotli-dev \ | |
| doxygen xsltproc docbook-xsl xmlto | |
| - name: Setup Xcode (macOS) | |
| if: runner.os == 'macOS' | |
| run: | | |
| XCODE_PATH=$(ls -d /Applications/Xcode_26*.app 2>/dev/null | sort -V | tail -1) | |
| if [ -n "$XCODE_PATH" ]; then | |
| sudo xcode-select -s "$XCODE_PATH" | |
| echo "Selected Xcode: $XCODE_PATH" | |
| else | |
| echo "No Xcode 26.x found, using default" | |
| fi | |
| swift --version | |
| xcodebuild -version | |
| - name: Cache Homebrew packages | |
| if: runner.os == 'macOS' | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/Library/Caches/Homebrew | |
| /opt/homebrew/Cellar | |
| /opt/homebrew/opt | |
| /opt/homebrew/lib | |
| /opt/homebrew/include | |
| key: brew-build-tools-${{ runner.os }}-v2 | |
| restore-keys: | | |
| brew-build-tools-${{ runner.os }}- | |
| - name: Install build tools (macOS) | |
| if: runner.os == 'macOS' | |
| env: | |
| HOMEBREW_NO_AUTO_UPDATE: 1 | |
| HOMEBREW_NO_INSTALL_CLEANUP: 1 | |
| run: | | |
| brew install cmake meson ninja pkg-config autoconf automake libtool gettext gnu-sed scons \ | |
| gcc go boost libsndfile jpeg-turbo libpng libtiff webp freetype harfbuzz \ | |
| libyaml jansson libedit libzip lapack nasm libogg libvorbis flac \ | |
| gobject-introspection cairo pango atk gtk+3 glib gdk-pixbuf librsvg json-glib libsoup \ | |
| gnutls libgcrypt libgpg-error libassuan libksba libusb pcre2 \ | |
| libarchive jemalloc lua protobuf grpc c-ares re2 capnp \ | |
| libsecret libfido2 double-conversion glog gflags \ | |
| libsodium lzo snappy brotli \ | |
| doxygen docbook-xsl sdl2 sdl2_ttf sdl2_mixer sdl2_gfx || true | |
| - name: Fix Python setuptools (Linux) | |
| if: runner.os == 'Linux' | |
| run: | | |
| sudo rm -rf /usr/lib/python3/dist-packages/setuptools* \ | |
| /usr/lib/python3/dist-packages/wheel* \ | |
| /usr/lib/python3/dist-packages/pkg_resources* \ | |
| /usr/lib/python3/dist-packages/_distutils_hack* 2>/dev/null || true | |
| sudo pip3 install --break-system-packages setuptools wheel 2>/dev/null || true | |
| - name: Setup Rust toolchain | |
| run: | | |
| curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable | |
| source "$HOME/.cargo/env" | |
| rustup update stable | |
| cargo install cargo-c 2>/dev/null || true | |
| echo "Rust: $(rustc --version), Cargo: $(cargo --version)" | |
| echo "$HOME/.cargo/bin" >> $GITHUB_PATH | |
| python3 -m pip install --break-system-packages setuptools meson 2>/dev/null || pip3 install --break-system-packages setuptools meson 2>/dev/null || true | |
| - name: Build multi-version packages | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| cd packages/ts-pantry | |
| MAX_VERSIONS="${{ github.event.inputs.max-versions || '5' }}" | |
| FORCE="${{ github.event.inputs.force || 'false' }}" | |
| ARGS="-b $S3_BUCKET -r $AWS_REGION --platform ${{ matrix.platform.name }} --multi-version --max-versions $MAX_VERSIONS" | |
| if [ "$FORCE" = "true" ]; then | |
| ARGS="$ARGS --force" | |
| fi | |
| PACKAGES="${{ github.event.inputs.packages }}" | |
| PACKAGES="${PACKAGES// /,}" | |
| ARGS="$ARGS -p $PACKAGES" | |
| echo "Running: bun scripts/build-all-packages.ts $ARGS" | |
| bun scripts/build-all-packages.ts $ARGS | |
| timeout-minutes: 180 | |
| # Batch multi-version builds (full run) | |
| build: | |
| name: Build batch ${{ matrix.batch }} (${{ matrix.platform.name }}) | |
| needs: discover | |
| if: (github.event.inputs.packages == '' || github.event.inputs.packages == null) && needs.discover.outputs.total != '0' | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 10 | |
| matrix: | |
| batch: ${{ fromJson(needs.discover.outputs.batches) }} | |
| platform: | |
| - os: macos-15 | |
| name: darwin-arm64 | |
| - os: ubuntu-latest | |
| name: linux-x86-64 | |
| runs-on: ${{ matrix.platform.os }} | |
| permissions: | |
| contents: read | |
| timeout-minutes: 180 | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v6 | |
| - name: Setup Bun | |
| uses: oven-sh/setup-bun@v2 | |
| with: | |
| bun-version: latest | |
| - name: Install dependencies | |
| run: bun install | |
| - name: Install build tools (Linux) | |
| if: runner.os == 'Linux' | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y \ | |
| build-essential libreadline-dev zlib1g-dev libssl-dev \ | |
| cmake meson ninja-build patchelf pkg-config \ | |
| autoconf automake libtool gettext intltool \ | |
| libffi-dev libexpat1-dev libbz2-dev liblzma-dev \ | |
| libncurses5-dev libsqlite3-dev \ | |
| python3-venv python3-pip python3-dev \ | |
| libcurl4-openssl-dev libxml2-dev libonig-dev \ | |
| ruby ruby-dev scons \ | |
| gperf bison flex texinfo nasm yasm \ | |
| libgmp-dev libmpfr-dev libmpc-dev \ | |
| libuv1-dev libpcre2-dev libevent-dev \ | |
| libglib2.0-dev libpixman-1-dev \ | |
| libgit2-dev libssh2-1-dev libhttp-parser-dev \ | |
| libasound2-dev libdbus-1-dev libsystemd-dev \ | |
| protobuf-compiler libprotobuf-dev \ | |
| nettle-dev libp11-kit-dev \ | |
| libboost-all-dev libsndfile1-dev \ | |
| libjpeg-dev libpng-dev libtiff-dev libwebp-dev \ | |
| libfreetype-dev libharfbuzz-dev \ | |
| libltdl-dev libacl1-dev libattr1-dev \ | |
| liblz4-dev libzstd-dev libxxhash-dev \ | |
| libyaml-dev libjansson-dev \ | |
| libedit-dev libelf-dev \ | |
| tcl-dev tk-dev \ | |
| libunwind-dev libdw-dev \ | |
| libzip-dev liblapack-dev gfortran \ | |
| python3-lxml python3-libxml2 \ | |
| libx11-dev libxext-dev libxrender-dev libxrandr-dev \ | |
| libxinerama-dev libxcomposite-dev libxdamage-dev \ | |
| libxcursor-dev libxi-dev libxtst-dev \ | |
| libxmu-dev libxt-dev libxaw7-dev libxres-dev \ | |
| libsm-dev libice-dev libxpm-dev \ | |
| libxxf86vm-dev libxshmfence-dev libxv-dev libxss-dev \ | |
| x11proto-dev xtrans-dev xutils-dev xauth \ | |
| libxkbcommon-dev libxkbfile-dev \ | |
| libxcb-render0-dev libxcb-shape0-dev libxcb-xfixes0-dev \ | |
| libxcb-image0-dev libxcb-util-dev libxcb-keysyms1-dev \ | |
| libxcb-randr0-dev libxcb-shm0-dev \ | |
| libwayland-dev libfontconfig1-dev \ | |
| gobject-introspection libgirepository1.0-dev \ | |
| libatk1.0-dev libcairo2-dev libcairo-gobject2 \ | |
| libpango1.0-dev libgtk-3-dev \ | |
| librsvg2-dev libsecret-1-dev libjson-glib-dev \ | |
| libsoup2.4-dev libgdk-pixbuf-2.0-dev \ | |
| libva-dev libegl-dev libgbm-dev libdrm-dev \ | |
| libgl-dev libglu1-mesa-dev \ | |
| clang llvm-dev libclang-dev \ | |
| libgnutls28-dev liblua5.4-dev \ | |
| libcap-dev libcap-ng-dev libslirp-dev libssh-dev \ | |
| libspeexdsp-dev libmaxminddb-dev libsmi2-dev \ | |
| libc-ares-dev libre2-dev libpcap-dev \ | |
| libnl-3-dev libnl-genl-3-dev \ | |
| libpam0g-dev libudev-dev \ | |
| libarchive-dev libusb-1.0-0-dev \ | |
| libjemalloc-dev libvterm-dev \ | |
| libxslt1-dev \ | |
| libsdl2-dev libplist-dev \ | |
| libassuan-dev libgpg-error-dev \ | |
| libfido2-dev \ | |
| libdouble-conversion-dev libgoogle-glog-dev libgflags-dev \ | |
| libsodium-dev liblzo2-dev libsnappy-dev libbrotli-dev \ | |
| doxygen xsltproc docbook-xsl xmlto | |
| - name: Setup Xcode (macOS) | |
| if: runner.os == 'macOS' | |
| run: | | |
| XCODE_PATH=$(ls -d /Applications/Xcode_26*.app 2>/dev/null | sort -V | tail -1) | |
| if [ -n "$XCODE_PATH" ]; then | |
| sudo xcode-select -s "$XCODE_PATH" | |
| echo "Selected Xcode: $XCODE_PATH" | |
| else | |
| echo "No Xcode 26.x found, using default" | |
| fi | |
| swift --version | |
| xcodebuild -version | |
| - name: Cache Homebrew packages | |
| if: runner.os == 'macOS' | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| ~/Library/Caches/Homebrew | |
| /opt/homebrew/Cellar | |
| /opt/homebrew/opt | |
| /opt/homebrew/lib | |
| /opt/homebrew/include | |
| key: brew-build-tools-${{ runner.os }}-v2 | |
| restore-keys: | | |
| brew-build-tools-${{ runner.os }}- | |
| - name: Install build tools (macOS) | |
| if: runner.os == 'macOS' | |
| env: | |
| HOMEBREW_NO_AUTO_UPDATE: 1 | |
| HOMEBREW_NO_INSTALL_CLEANUP: 1 | |
| run: | | |
| brew install cmake meson ninja pkg-config autoconf automake libtool gettext gnu-sed scons \ | |
| gcc go boost libsndfile jpeg-turbo libpng libtiff webp freetype harfbuzz \ | |
| libyaml jansson libedit libzip lapack nasm libogg libvorbis flac \ | |
| gobject-introspection cairo pango atk gtk+3 glib gdk-pixbuf librsvg json-glib libsoup \ | |
| gnutls libgcrypt libgpg-error libassuan libksba libusb pcre2 \ | |
| libarchive jemalloc lua protobuf grpc c-ares re2 capnp \ | |
| libsecret libfido2 double-conversion glog gflags \ | |
| libsodium lzo snappy brotli \ | |
| doxygen docbook-xsl sdl2 sdl2_ttf sdl2_mixer sdl2_gfx || true | |
| - name: Fix Python setuptools (Linux) | |
| if: runner.os == 'Linux' | |
| run: | | |
| sudo rm -rf /usr/lib/python3/dist-packages/setuptools* \ | |
| /usr/lib/python3/dist-packages/wheel* \ | |
| /usr/lib/python3/dist-packages/pkg_resources* \ | |
| /usr/lib/python3/dist-packages/_distutils_hack* 2>/dev/null || true | |
| sudo pip3 install --break-system-packages setuptools wheel 2>/dev/null || true | |
| - name: Setup Rust toolchain | |
| run: | | |
| curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable | |
| source "$HOME/.cargo/env" | |
| rustup update stable | |
| cargo install cargo-c 2>/dev/null || true | |
| echo "Rust: $(rustc --version), Cargo: $(cargo --version)" | |
| echo "$HOME/.cargo/bin" >> $GITHUB_PATH | |
| python3 -m pip install --break-system-packages setuptools meson 2>/dev/null || pip3 install --break-system-packages setuptools meson 2>/dev/null || true | |
| - name: Build batch ${{ matrix.batch }} (multi-version) | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| cd packages/ts-pantry | |
| MAX_VERSIONS="${{ github.event.inputs.max-versions || '5' }}" | |
| FORCE="${{ github.event.inputs.force || 'false' }}" | |
| ARGS="-b $S3_BUCKET -r $AWS_REGION --batch ${{ matrix.batch }} --batch-size 25 --platform ${{ matrix.platform.name }} --multi-version --max-versions $MAX_VERSIONS" | |
| if [ "$FORCE" = "true" ]; then | |
| ARGS="$ARGS --force" | |
| fi | |
| echo "Running: bun scripts/build-all-packages.ts $ARGS" | |
| bun scripts/build-all-packages.ts $ARGS | |
| timeout-minutes: 170 | |
| notify: | |
| name: Discord failure alert | |
| needs: [build-targeted, build] | |
| if: always() && (needs.build-targeted.result == 'failure' || needs.build.result == 'failure') | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Build fields | |
| id: fields | |
| run: | | |
| FIELDS="[" | |
| FIRST=true | |
| for pair in "Build Targeted:${{ needs.build-targeted.result }}" "Build Batches:${{ needs.build.result }}"; do | |
| NAME="${pair%%:*}" | |
| VAL="${pair##*:}" | |
| [ "$VAL" = "skipped" ] && continue | |
| [ "$FIRST" = "true" ] && FIRST=false || FIELDS="$FIELDS," | |
| FIELDS="$FIELDS{\"name\":\"$NAME\",\"value\":\"$VAL\",\"inline\":true}" | |
| done | |
| FIELDS="$FIELDS,{\"name\":\"Max Versions\",\"value\":\"${{ github.event.inputs.max-versions || '5' }}\",\"inline\":true}]" | |
| echo "json=$FIELDS" >> "$GITHUB_OUTPUT" | |
| - uses: ./.github/actions/discord-notify | |
| with: | |
| title: Build Versions — Failed | |
| description: Multi-version build failed | |
| status: failure | |
| fields: ${{ steps.fields.outputs.json }} | |
| webhook_url: ${{ secrets.DISCORD_WEBHOOK_URL }} |