CI: Add Flatpak aarch64

tytan652 · tytan652 · commit 4930b3914a1f · 2023-01-24T13:04:43.000+01:00
Building through QEMU emulation is really slow. So to avoid timing out
the job (6h), the original job is split in two.

The first job build dependencies (4-5h), if the cache already exist the
build is skipped.

And the second job build CEF and OBS Studio by relying on the cache of the
first job (2-3h).

This separation is done by modifying the manifest in the first job to
build only dependencies with a fake "obs" module/command.

aarch64 builds are not enabled by default on the main workflow, it can
be enabled with a dedicated label on PRs.
diff --git a/.github/workflows/flatpak.yml b/.github/workflows/flatpak.yml
@@ -42,10 +42,71 @@ jobs:
             * ) echo 'valid_tag=false' >> $GITHUB_OUTPUT ;;
           esac
 
+  prepare:
+    name: Build Dependencies
+    runs-on: [ubuntu-latest]
+    needs: check_tag
+    if: fromJSON(needs.check_tag.outputs.valid_tag)
+    container:
+      image: bilelmoussaoui/flatpak-github-actions:kde-6.4
+      options: --privileged
+    strategy:
+      matrix:
+        arch: [x86_64, aarch64]
+    outputs:
+      cache_key: ${{ steps.setup.outputs.cache_key }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          submodules: 'recursive'
+
+      - name: 'Setup build environment'
+        id: setup
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          CACHE_KEY: flatpak-builder-${{ hashFiles('CI/flatpak/com.obsproject.Studio.json') }}
+        run: |
+          if [[ "${{ matrix.arch }}" != "x86_64" ]]; then
+            dnf install -y -q jq gh docker
+          else
+            dnf install -y -q jq gh
+          fi
+          gh extension install actions/gh-actions-cache
+
+          git config --global --add safe.directory $GITHUB_WORKSPACE
+
+          KEY="$CACHE_KEY-${{ matrix.arch }}"
+          CACHE_HIT=$(gh actions-cache list --key $KEY | grep -q $KEY && echo 'true' || echo 'false')
+
+          echo "cache_key=$CACHE_KEY" >> $GITHUB_OUTPUT
+          echo "skip_build=$CACHE_HIT" >> $GITHUB_OUTPUT
+
+      - name: Setup QEMU
+        if: ${{ (matrix.arch != 'x86_64') && !fromJSON(steps.setup.outputs.skip_build) }}
+        uses: docker/setup-qemu-action@v2
+        with:
+          platforms: arm64
+
+      - name: 'Shorten Manifest'
+        if: ${{ !fromJSON(steps.setup.outputs.skip_build) }}
+        run: |
+          jq 'del(.modules[-2:]) | .modules += [{"name": "obs", "buildsystem": "simple", "build-commands": ["cp -a /usr/bin/ls /app/bin/obs"]}]' CI/flatpak/com.obsproject.Studio.json > tmp.json
+          mv tmp.json CI/flatpak/com.obsproject.Studio.json
+
+      - name: Build Flatpak Manifest
+        if: ${{ !fromJSON(steps.setup.outputs.skip_build) }}
+        uses: flatpak/flatpak-github-actions/flatpak-builder@v5
+        with:
+          build-bundle: false
+          manifest-path: CI/flatpak/com.obsproject.Studio.json
+          cache-key: ${{ steps.setup.outputs.cache_key }}
+          arch: ${{ matrix.arch }}
+
   publish:
     name: Publish to Flathub
     runs-on: [ubuntu-latest]
-    needs: check_tag
+    needs: [check_tag, prepare]
     if: fromJSON(needs.check_tag.outputs.valid_tag)
     env:
       FLATPAK_BUILD_PATH: flatpak_app/files/share
@@ -55,6 +116,7 @@ jobs:
     strategy:
       matrix:
         branch: ${{ fromJSON(needs.check_tag.outputs.matrix) }}
+        arch: [x86_64, aarch64]
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -64,17 +126,27 @@ jobs:
       - name: 'Setup build environment'
         id: setup
         run: |
+          [[ "${{ matrix.arch }}" != "x86_64" ]] && dnf install -y -q docker
+
           git config --global --add safe.directory $GITHUB_WORKSPACE
-          echo "commitHash=$(git rev-parse --short=9 HEAD)" >> $GITHUB_OUTPUT
+
+          echo "git_hash=$(git rev-parse --short=9 HEAD)" >> $GITHUB_OUTPUT
+
+      - name: Setup QEMU
+        if: ${{ matrix.arch != 'x86_64' }}
+        uses: docker/setup-qemu-action@v2
+        with:
+          platforms: arm64
 
       - name: Build Flatpak Manifest
         uses: flatpak/flatpak-github-actions/flatpak-builder@v5
         with:
-          bundle: obs-studio-${{ steps.setup.outputs.commitHash }}.flatpak
+          bundle: obs-studio-${{ steps.setup.outputs.git_hash }}.flatpak
           manifest-path: CI/flatpak/com.obsproject.Studio.json
-          cache-key: flatpak-builder-${{ hashFiles('CI/flatpak/com.obsproject.Studio.json') }}
+          cache-key: ${{ needs.prepare.outputs.cache_key }}
           mirror-screenshots-url: https://dl.flathub.org/repo/screenshots
           branch: ${{ matrix.branch }}
+          arch: ${{ matrix.arch }}
 
       - name: Validate AppStream
         shell: bash
@@ -91,7 +163,7 @@ jobs:
 
       - name: Commit screenshots to the OSTree repository
         run: |
-          ostree commit --repo=repo --canonical-permissions --branch=screenshots/x86_64 flatpak_app/screenshots
+          ostree commit --repo=repo --canonical-permissions --branch=screenshots/${{ matrix.arch }} flatpak_app/screenshots
 
       - name: Publish to Flathub Beta
         uses: flatpak/flatpak-github-actions/flat-manager@v5
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -69,6 +69,7 @@ jobs:
     runs-on: [ubuntu-22.04]
     outputs:
       create_artifacts: ${{ steps.config.outputs.create_artifacts }}
+      flatpak_matrix: ${{ steps.config.outputs.flatpak_matrix }}
       cache_date: ${{ steps.config.outputs.cache_date }}
     steps:
       - name: 'Configure Build Jobs'
@@ -80,8 +81,15 @@ jobs:
             else
               echo 'create_artifacts=false' >> $GITHUB_OUTPUT
             fi
+
+            if test -n "$(curl -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" -s "${{ github.event.pull_request.url }}" | jq -e '.labels[] | select(.name == "Flatpak aarch64")')"; then
+              echo 'flatpak_matrix=["x86_64", "aarch64"]' >> $GITHUB_OUTPUT
+            else
+              echo 'flatpak_matrix=["x86_64"]' >> $GITHUB_OUTPUT
+            fi
           else
             echo 'create_artifacts=true' >> $GITHUB_OUTPUT
+            echo 'flatpak_matrix=["x86_64"]' >> $GITHUB_OUTPUT
           fi
           echo "cache_date=$(date +"%Y-%m-%d")" >> $GITHUB_OUTPUT
 
@@ -362,8 +370,8 @@ jobs:
           name: 'obs-studio-windows-${{ matrix.arch }}-${{ steps.setup.outputs.commitHash }}'
           path: '${{ env.FILE_NAME }}'
 
-  linux_package:
-    name: '02 - Flatpak'
+  flatpak_deps_build:
+    name: '02 - Flatpak Dependencies'
     runs-on: [ubuntu-latest]
     needs: [config, clang_check]
     defaults:
@@ -372,6 +380,74 @@ jobs:
     container:
       image: bilelmoussaoui/flatpak-github-actions:kde-6.4
       options: --privileged
+    strategy:
+      matrix:
+        arch: ${{ fromJSON(needs.config.outputs.flatpak_matrix) }}
+    outputs:
+      cache_key: ${{ steps.setup.outputs.cache_key }}
+    steps:
+
+      - name: 'Checkout'
+        uses: actions/checkout@v3
+        with:
+          submodules: 'recursive'
+          fetch-depth: 0
+
+      - name: 'Setup build environment'
+        id: setup
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          CACHE_KEY: flatpak-builder-${{ hashFiles('CI/flatpak/com.obsproject.Studio.json') }}
+        run: |
+          if [[ "${{ matrix.arch }}" != "x86_64" ]]; then
+            dnf install -y -q jq gh docker
+          else
+            dnf install -y -q jq gh
+          fi
+          gh extension install actions/gh-actions-cache
+
+          git config --global --add safe.directory $GITHUB_WORKSPACE
+
+          KEY="$CACHE_KEY-${{ matrix.arch }}"
+          CACHE_HIT=$(gh actions-cache list --key $KEY | grep -q $KEY && echo 'true' || echo 'false')
+
+          echo "cache_key=$CACHE_KEY" >> $GITHUB_OUTPUT
+          echo "skip_build=$CACHE_HIT" >> $GITHUB_OUTPUT
+
+      - name: Setup QEMU
+        if: ${{ (matrix.arch != 'x86_64') && !fromJSON(steps.setup.outputs.skip_build) }}
+        uses: docker/setup-qemu-action@v2
+        with:
+          platforms: arm64
+
+      - name: 'Shorten Manifest'
+        if: ${{ !fromJSON(steps.setup.outputs.skip_build) }}
+        run: |
+          jq 'del(.modules[-2:]) | .modules += [{"name": "obs", "buildsystem": "simple", "build-commands": ["cp -a /usr/bin/ls /app/bin/obs"]}]' CI/flatpak/com.obsproject.Studio.json > tmp.json
+          mv tmp.json CI/flatpak/com.obsproject.Studio.json
+
+      - name: Build Flatpak Manifest
+        if: ${{ !fromJSON(steps.setup.outputs.skip_build) }}
+        uses: flatpak/flatpak-github-actions/flatpak-builder@v5
+        with:
+          build-bundle: false
+          manifest-path: CI/flatpak/com.obsproject.Studio.json
+          cache-key: ${{ steps.setup.outputs.cache_key }}
+          arch: ${{ matrix.arch }}
+
+  flatpak_build:
+    name: '03 - Flatpak'
+    runs-on: [ubuntu-latest]
+    needs: [config, flatpak_deps_build]
+    defaults:
+      run:
+        shell: bash
+    container:
+      image: bilelmoussaoui/flatpak-github-actions:kde-6.4
+      options: --privileged
+    strategy:
+      matrix:
+        arch: ${{ fromJSON(needs.config.outputs.flatpak_matrix) }}
     steps:
 
       - name: 'Checkout'
@@ -381,19 +457,28 @@ jobs:
           fetch-depth: 0
 
       - name: 'Setup build environment'
+        id: setup
         run: |
+          [[ "${{ matrix.arch }}" != "x86_64" ]] && dnf install -y -q docker
+
           git config --global --add safe.directory $GITHUB_WORKSPACE
-          echo "OBS_GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD)" >> $GITHUB_ENV
-          echo "OBS_GIT_HASH=$(git rev-parse --short=9 HEAD)" >> $GITHUB_ENV
-          echo "OBS_GIT_TAG=$(git describe --tags --abbrev=0)" >> $GITHUB_ENV
+
+          echo "git_hash=$(git rev-parse --short=9 HEAD)" >> $GITHUB_OUTPUT
+
+      - name: Setup QEMU
+        if: ${{ matrix.arch != 'x86_64' }}
+        uses: docker/setup-qemu-action@v2
+        with:
+          platforms: arm64
 
       - name: Build Flatpak Manifest
         uses: flatpak/flatpak-github-actions/flatpak-builder@v5
         with:
           build-bundle: ${{ fromJSON(needs.config.outputs.create_artifacts) }}
-          bundle: obs-studio-flatpak-${{ env.OBS_GIT_HASH }}.flatpak
+          bundle: obs-studio-flatpak-${{ steps.setup.outputs.git_hash }}.flatpak
           manifest-path: CI/flatpak/com.obsproject.Studio.json
-          cache-key: flatpak-builder-${{ hashFiles('CI/flatpak/com.obsproject.Studio.json') }}
+          cache-key: ${{ needs.flatpak_deps_build.outputs.cache_key }}
+          arch: ${{ matrix.arch }}
 
   windows_package:
     name: '03 - Windows Installer'
