Merge pull request #64 from uyjulian/ci_improvements_20250918

uyjulian · web-flow · commit a57382c25e67 · 2025-10-04T18:09:13.000-05:00
CI improvements 20250918
diff --git a/.github/workflows/compilation.yml b/.github/workflows/compilation.yml
@@ -2,77 +2,176 @@ name: CI
 
 on:
   push:
+    paths-ignore:
+      - '**.md'
+      - '**.rst'
   pull_request:
+    paths-ignore:
+      - '**.md'
+      - '**.rst'
   repository_dispatch:
     types: [run_build, run_release]
   workflow_dispatch: {}
+env:
+  HOMEBREW_NO_ANALYTICS: 1
+  HOMEBREW_NO_AUTO_UPDATE: 1
+  HOMEBREW_NO_INSTALL_UPGRADE: 1
+  HOMEBREW_NO_INSTALL_CLEANUP: 1
+  DEBIAN_FRONTEND: noninteractive
+  TZ: Japan
 
 jobs:
   build:
-    runs-on: ${{ matrix.os[0] }}
     strategy:
       matrix:
-        os: [
-          [macos-latest, arm64, bash],
-          [macos-13, x86_64, bash],
-          [ubuntu-latest, x86_64, bash],
-          [windows-latest, x86_64, msys2]
-        ]
+        os: 
+          - runs-on: macos-latest
+            architecture: arm64
+            shell: bash
+            container: null
+            container-options: " "
+            macos-package-manager: brew
+          - runs-on: macos-13
+            architecture: x86_64
+            shell: bash
+            container: null
+            container-options: " "
+            macos-package-manager: brew
+          - runs-on: macos-latest
+            architecture: arm64
+            shell: bash
+            container: null
+            container-options: " "
+            macos-package-manager: port
+          - runs-on: ubuntu-latest
+            architecture: x86_64
+            shell: bash
+            container: "ubuntu:20.04"
+            container-options: "--user 0"
+            macos-package-manager: null
+          - runs-on: ubuntu-24.04-arm
+            architecture: arm64
+            shell: bash
+            container: "ubuntu:20.04"
+            container-options: "--user 0"
+            macos-package-manager: null
+          - runs-on: windows-latest
+            architecture: x86_64
+            shell: msys2
+            container: null
+            container-options: " "
+            macos-package-manager: null
       fail-fast: false
+    runs-on: ${{ matrix.os.runs-on }}
     defaults:
      run:
-      shell: ${{ matrix.os[2] }} {0}
+      shell: ${{ matrix.os.shell }} {0}
+    container:
+      image: ${{ matrix.os.container }}
+      options: ${{ matrix.os.container-options }}
+    timeout-minutes: 240
 
     steps:
-    - uses: actions/checkout@v4
+    - name: Checkout repository
+      uses: actions/checkout@v4
 
     - name: Install Ubuntu packages
-      if: matrix.os[0] == 'ubuntu-latest'
+      if: startsWith(matrix.os.runs-on, 'ubuntu')
       run: |
-        sudo apt-get update
-        sudo apt-get -y install texinfo bison flex gettext libgmp3-dev libmpfr-dev libmpc-dev
+        apt-get -y update
+        apt-get -y -o Dpkg::Options::='--force-confdef' -o Dpkg::Options::='--force-confold' install texinfo bison flex gettext autopoint autoconf automake libgmp3-dev libmpfr-dev libmpc-dev cmake g++ gcc git libgsl-dev make patch zlib1g-dev wget curl
 
-    - name: Install macOS packages
-      if: startsWith(matrix.os[0], 'macos')
+    - name: Install macOS packages (brew)
+      if: startsWith(matrix.os.runs-on, 'macos') && matrix.os.macos-package-manager == 'brew'
       run: |
         brew update
-        brew install texinfo bison flex gnu-sed gsl gmp mpfr libmpc
+        brew install texinfo bison flex gnu-sed autoconf automake libtool gsl gmp mpfr libmpc md5sha1sum wget curl
+
+    - name: Install macOS packages (port)
+      if: startsWith(matrix.os.runs-on, 'macos') && matrix.os.macos-package-manager == 'port'
+      run: |
+        curl -LO https://raw.githubusercontent.com/GiovanniBussi/macports-ci/master/macports-ci
+        . ./macports-ci install --remove-brew --version=2.11.5 --sync=rsync
+        sudo port install gsl gmp mpfr libmpc libiconv bison flex gsed texinfo autoconf automake libtool md5sha1sum wget curl
 
     - name: Install MSYS2 packages
-      if: matrix.os[0] == 'windows-latest'
+      if: startsWith(matrix.os.runs-on, 'windows')
       uses: msys2/setup-msys2@v2
       with:
         msystem: MINGW32
         install: |
-          base-devel git make texinfo flex bison patch mpc-devel tar
+          base-devel git make texinfo flex bison patch binutils mpc-devel tar wget curl
+          autotools gettext gettext-devel
           mingw-w64-i686-readline mingw-w64-i686-gcc mingw-w64-i686-cmake 
           mingw-w64-i686-make mingw-w64-i686-libogg
         update: true
 
     - name: Runs all the stages in the shell
+      run: |
+        . ./config/ci-env.sh
+        if test -f ./build-all.sh; then ./build-all.sh; fi
+        if test -f ./toolchain.sh; then ./toolchain.sh; fi
+
+    - name: Print version of executables (ps2toolchain-iop)
+      if: github.event.repository.name == 'ps2toolchain-iop'
       run: |
         export PS2DEV=$PWD/ps2dev
-        export PATH="$(brew --prefix gnu-sed)/libexec/gnubin:$PATH" # This is just needed for MacOS
-        export PATH=$PATH:$PS2DEV/ee/bin
-        ./toolchain.sh
+        export PATH=$PATH:$PS2DEV/iop/bin
+        mipsel-none-elf-as --version
+        mipsel-none-elf-ld --version
+        mipsel-none-elf-gcc --version
 
-    - name: Print version of executables
+    - name: Print version of executables (ps2toolchain-ee)
+      if: github.event.repository.name == 'ps2toolchain-ee'
       run: |
         export PS2DEV=$PWD/ps2dev
         export PATH=$PATH:$PS2DEV/ee/bin
         mips64r5900el-ps2-elf-as --version
         mips64r5900el-ps2-elf-ld --version
         mips64r5900el-ps2-elf-gcc --version
 
+    - name: Print version of executables (ps2toolchain-dvp)
+      if: github.event.repository.name == 'ps2toolchain-dvp'
+      run: |
+        export PS2DEV=$PWD/ps2dev
+        export PATH=$PATH:$PS2DEV/dvp/bin
+        dvp-as --version
+
     - name: Get short SHA
       id: slug
-      run: echo "sha8=${MSYSTEM}-sha[$(echo ${GITHUB_SHA} | cut -c1-8)" >> $GITHUB_OUTPUT
+      run: printf '%s\n' "sha8=$(printf '%s\n' ${GITHUB_SHA} | cut -c1-8)" >> $GITHUB_OUTPUT
 
-    - name: Prepare ps2dev folder
+    - name: Compress ps2dev folder
       run: |
-        tar -zcvf ps2dev-${{matrix.os[0]}}.tar.gz ps2dev
+        tar -zcvf ps2dev-${{ matrix.os.runs-on }}.tar.gz ps2dev
+
+    - name: Upload artifact
+      uses: actions/upload-artifact@v4
+      with:
+        name: ${{ github.event.repository.name }}-${{ matrix.os.runs-on }}-ps2dev-${{ matrix.os.architecture }}-${{ matrix.os.macos-package-manager }}-${{ steps.slug.outputs.sha8 }}
+        path: ps2dev-${{ matrix.os.runs-on }}.tar.gz
+
+    - name: Extract tag name
+      if: startsWith(github.ref, 'refs/tags/')
+      id: tag
+      run: printf 'VERSION=%s\n' "${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT
+
+    - name: Create pre-release
+      if: (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main') && github.event.repository.name == 'ps2dev' && matrix.os.macos-package-manager != 'port'
+      uses: softprops/action-gh-release@v2
+      with:
+        files: ps2dev-${{ matrix.os.runs-on }}.tar.gz
+        prerelease: true
+        name: "Development build"
+        tag_name: latest
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-    - uses: actions/upload-artifact@v4
+    - name: Release
+      if: startsWith(github.ref, 'refs/tags/') && github.event.repository.name == 'ps2dev' && matrix.os.macos-package-manager != 'port'
+      uses: softprops/action-gh-release@v2
       with:
-        name: ps2dev-${{matrix.os[0]}}-ps2dev-${{matrix.os[1]}}-${{ steps.slug.outputs.sha8 }}
-        path: ps2dev-${{matrix.os[0]}}.tar.gz
+        files: ps2dev-${{ matrix.os.runs-on }}.tar.gz
+        tag_name: ${{ steps.tag.outputs.VERSION }}
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -3,38 +3,43 @@ name: CI-Docker
 on:
   push:
     branches:
+      - master
       - main
     tags:
       - v*
+    paths-ignore:
+      - '**.md'
+      - '**.rst'
   workflow_dispatch: {}
+  repository_dispatch:
+    types: [run_build]
 
 jobs:
   build:
-    runs-on: ubuntu-latest
     env:
       DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
       DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
-      DISPATCH_TOKEN: ${{ secrets.DISPATCH_TOKEN }}
+      BASE_DOCKER_IMAGE_NAME: ${{ (github.event.repository.name == 'ps2sdk' && 'ps2toolchain') || (github.event.repository.name == 'ps2sdk-ports' && 'ps2sdk') || (github.event.repository.name == 'ps2-packer' && 'ps2sdk') || (github.event.repository.name == 'ps2dev' && 'ps2sdk-ports') || 'none' }}
+    strategy:
+      matrix:
+        runs-on: [ubuntu-latest, ubuntu-24.04-arm]
+    runs-on: ${{ matrix.runs-on }}
+    timeout-minutes: 180
 
     steps:
-    - uses: actions/checkout@v4
+    - name: Checkout repository
+      uses: actions/checkout@v4
 
     - name: Extract DOCKER_TAG using tag name
       if: startsWith(github.ref, 'refs/tags/')
       run: |
-        echo "DOCKER_TAG=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_ENV
-
+        printf 'DOCKER_TAG=%s\n' "${GITHUB_REF/refs\/tags\//}" >> $GITHUB_ENV
+    
     - name: Use default DOCKER_TAG
       if: startsWith(github.ref, 'refs/tags/') != true
       run: |
         echo "DOCKER_TAG=latest" >> $GITHUB_ENV
 
-    - name: Set up QEMU
-      uses: docker/setup-qemu-action@v3
-
-    - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v3
-
     - name: Login to DockerHub
       uses: docker/login-action@v3
       if: env.DOCKER_USERNAME != null
@@ -52,28 +57,61 @@ jobs:
     - name: Set docker tag list to include DockerHub if credentials available
       if: env.DOCKER_USERNAME != null
       run: |
-        echo "DOCKER_TAG_LIST=ghcr.io/${{ github.repository }}:${{ env.DOCKER_TAG }},${{ github.repository }}:${{ env.DOCKER_TAG }}" >> $GITHUB_ENV
+        printf 'DOCKER_TAG_LIST=%s\n' "ghcr.io/${{ github.repository }}:${{ env.DOCKER_TAG }},${{ github.repository }}:${{ env.DOCKER_TAG }}" >> $GITHUB_ENV
 
     - name: Set docker tag list to not include DockerHub if credentials not available
       if: env.DOCKER_USERNAME == null
       run: |
-        echo "DOCKER_TAG_LIST=ghcr.io/${{ github.repository }}:${{ env.DOCKER_TAG }}" >> $GITHUB_ENV
+        printf 'DOCKER_TAG_LIST=%s\n' "ghcr.io/${{ github.repository }}:${{ env.DOCKER_TAG }}" >> $GITHUB_ENV
 
     - name: Build and Push to container registry
       uses: docker/build-push-action@v5
       with:
         push: true
         tags: ${{ env.DOCKER_TAG_LIST }}
+        build-args: |
+          BASE_DOCKER_IMAGE=ghcr.io/${{ github.repository_owner }}/${{ env.BASE_DOCKER_IMAGE_NAME }}:${{ env.DOCKER_TAG }}
+          BASE_DOCKER_DVP_IMAGE=ghcr.io/${{ github.repository_owner }}/ps2toolchain-dvp:${{ env.DOCKER_TAG }}
+          BASE_DOCKER_IOP_IMAGE=ghcr.io/${{ github.repository_owner }}/ps2toolchain-iop:${{ env.DOCKER_TAG }}
+          BASE_DOCKER_EE_IMAGE=ghcr.io/${{ github.repository_owner }}/ps2toolchain-ee:${{ env.DOCKER_TAG }}
+
+    - name: Gather information for repository dispatch
+      id: dest-repo
+      run: if test -f ./config/repository-dispatch.ini; then cat ./config/repository-dispatch.ini >> $GITHUB_OUTPUT; fi
+
+  perform-repository-dispatch:
+    needs: [build]
+    runs-on: ubuntu-latest
+    container:
+      image: ubuntu:20.04
+      options: "--user 0"
+    timeout-minutes: 20
+    env:
+      DISPATCH_TOKEN: ${{ secrets.DISPATCH_TOKEN }}
+    strategy:
+      matrix:
+        dest-repo: ${{ fromJson(needs.build.outputs.DEST_REPO) }}
 
+    steps:
     - name: Send Compile action
       run: |
-        export DISPATCH_ACTION="$(echo run_build)"
-        echo "NEW_DISPATCH_ACTION=$DISPATCH_ACTION" >> $GITHUB_ENV
+        export DISPATCH_ACTION="$(printf 'run_build\n')"
+        printf 'NEW_DISPATCH_ACTION=%s\n' "$DISPATCH_ACTION" >> $GITHUB_ENV
+
+    - name: Repository Dispatch to ${{ matrix.dest-repo }}
+      uses: peter-evans/repository-dispatch@v3
+      if: env.DISPATCH_TOKEN != null && !contains(matrix.dest-repo, '/')
+      with:
+        repository: ${{ github.repository_owner }}/${{ matrix.dest-repo }}
+        token: ${{ secrets.DISPATCH_TOKEN }}
+        event-type: ${{ env.NEW_DISPATCH_ACTION }}
+        client-payload: '{"ref": "${{ github.ref }}"}'
 
-    - name: Repository Dispatch
+    - name: Repository Dispatch to specific ${{ matrix.dest-repo }}
       uses: peter-evans/repository-dispatch@v3
-      if: env.DISPATCH_TOKEN != null
+      if: env.DISPATCH_TOKEN != null && contains(matrix.dest-repo, '/')
       with:
-        repository: ${{ github.repository_owner }}/ps2toolchain
+        repository: ${{ matrix.dest-repo }}
         token: ${{ secrets.DISPATCH_TOKEN }}
         event-type: ${{ env.NEW_DISPATCH_ACTION }}
+        client-payload: '{"ref": "${{ github.ref }}"}'
diff --git a/config/ci-env.sh b/config/ci-env.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+export PS2DEV=$PWD/ps2dev
+export PATH=$PATH:$PS2DEV/bin
+export PATH=$PATH:$PS2DEV/ee/bin
diff --git a/config/repository-dispatch.ini b/config/repository-dispatch.ini
@@ -0,0 +1 @@
+DEST_REPO=["ps2toolchain"]
diff --git a/scripts/001-binutils.sh b/scripts/001-binutils.sh
@@ -41,10 +41,11 @@ OSVER=$(uname)
 if [ "$(uname -s)" = "Darwin" ]; then
   ## Check if using brew
   if command -v brew &> /dev/null; then
-    TARG_XTRA_OPTS="--with-system-zlib --with-gmp=$(brew --prefix gmp) --with-mpfr=$(brew --prefix mpfr)"
+    TARG_XTRA_OPTS="--with-system-zlib --with-gmp=$(brew --prefix gmp) --with-mpfr=$(brew --prefix mpfr) --with-mpfr=$(brew --prefix mpfr)"
   elif command -v port &> /dev/null; then
-    ## Check if using MacPorts
-    TARG_XTRA_OPTS="--with-system-zlib --with-gmp=$(port -q prefix gmp) --with-mpfr=$(port -q prefix mpfr)"
+    MACPORT_BASE=$(dirname $(port -q contents gmp|grep gmp.h)|sed s#/include##g)
+    printf 'Macport base is %s\n' "$MACPORT_BASE"
+    TARG_XTRA_OPTS="--with-system-zlib --with-libiconv_prefix=$MACPORT_BASE --with-gmp=$MACPORT_BASE --with-mpfr=$MACPORT_BASE --with-mpc=$MACPORT_BASE"
   fi
 fi
 
diff --git a/scripts/002-gcc-stage1.sh b/scripts/002-gcc-stage1.sh
@@ -45,9 +45,9 @@ if [ "$(uname -s)" = "Darwin" ]; then
     TARG_XTRA_OPTS="--with-system-zlib --with-gmp=$(brew --prefix gmp) --with-mpfr=$(brew --prefix mpfr) --with-mpc=$(brew --prefix libmpc)"
   elif command -v port &> /dev/null; then
   ## Check if using MacPorts
-    MACPORT_BASE=$(dirname `port -q contents gmp|grep gmp.h`|sed s#/include##g)
-    echo Macport base is $MACPORT_BASE
-    TARG_XTRA_OPTS="--with-system-zlib --with-gmp=$MACPORT_BASE --with-mpfr=$MACPORT_BASE --with-mpc=$MACPORT_BASE"
+    MACPORT_BASE=$(dirname $(port -q contents gmp|grep gmp.h)|sed s#/include##g)
+    printf 'Macport base is %s\n' "$MACPORT_BASE"
+    TARG_XTRA_OPTS="--with-system-zlib --with-libiconv_prefix=$MACPORT_BASE --with-gmp=$MACPORT_BASE --with-mpfr=$MACPORT_BASE --with-mpc=$MACPORT_BASE"
   fi
 fi
 
diff --git a/scripts/006-gcc-stage2.sh b/scripts/006-gcc-stage2.sh
@@ -42,15 +42,16 @@ OSVER=$(uname)
 if [ "$(uname -s)" = "Darwin" ]; then
   ## Check if using brew
   if command -v brew &> /dev/null; then
-    export PATH="$(brew --prefix gnu-sed)/libexec/gnubin:$PATH"
     TARG_XTRA_OPTS="--with-system-zlib --with-gmp=$(brew --prefix gmp) --with-mpfr=$(brew --prefix mpfr) --with-mpc=$(brew --prefix libmpc)"
   elif command -v port &> /dev/null; then
   ## Check if using MacPorts
-    MACPORT_BASE=$(dirname `port -q contents gmp|grep gmp.h`|sed s#/include##g)
-    echo Macport base is $MACPORT_BASE
-    alias sed='gsed'
-    TARG_XTRA_OPTS="--with-system-zlib --with-gmp=$MACPORT_BASE --with-mpfr=$MACPORT_BASE --with-mpc=$MACPORT_BASE"
+    MACPORT_BASE=$(dirname $(port -q contents gmp|grep gmp.h)|sed s#/include##g)
+    printf 'Macport base is %s\n' "$MACPORT_BASE"
+    TARG_XTRA_OPTS="--with-system-zlib --with-libiconv_prefix=$MACPORT_BASE --with-gmp=$MACPORT_BASE --with-mpfr=$MACPORT_BASE --with-mpc=$MACPORT_BASE"
   fi
+  mkdir -p "build-sed-wrapper"
+  ln -sf  "$(which gsed)" "build-sed-wrapper/sed"
+  export PATH="$(realpath build-sed-wrapper):$PATH"
 fi
 
 ## Determine the maximum number of processes that Make can work with.
