Build binaries for multiple targets

tstellar · tstellar · commit a60f15850385 · 2024-05-22T15:30:53.000-07:00
diff --git a/.github/workflows/continue-timeout-job.yml b/.github/workflows/continue-timeout-job.yml
@@ -0,0 +1,75 @@
+name: Continue Timeout Job
+
+on:
+  workflow_run:
+    workflows:
+      - "Windows Precommit Tests"
+      - "CI Tests"
+    types:
+      - completed
+
+permissions:
+  contents: read
+
+jobs:
+  restart:
+    name: "Restart Job"
+    permissions:
+      actions: write
+    runs-on: ubuntu-22.04
+    if: github.event.workflow_run.conclusion == 'failure'
+    steps:
+      - name: "Restart Job"
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea #v7.0.1
+        with:
+          script: |
+            const response = await github.rest.actions.listWorkflowRunArtifacts({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              run_id: context.payload.workflow_run.id
+            })
+
+            job_ids = [];
+
+            for (artifact of response.data.artifacts) {
+              console.log(artifact);
+              const match = artifact.name.match(/timeout-([0-9]+)/);
+              console.log(match);
+              if (!match) {
+                continue;
+              }
+              job_ids.push(match[1]);
+
+              // Delete the timeout artifact to prepare for the next run
+              await github.rest.actions.deleteArtifact({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                artifact_id: artifact.id
+              });
+            }
+
+            if (job_ids.length == 0) {
+              return;
+            }
+
+            if (job_ids.length > 1) {
+              // We aren't able to re-start multiple jobs individually, so our
+              // only option is to restart all failed jobs.
+              await github.rest.actions.reRunWorkflowFailedJobs({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                run_id: context.payload.workflow_run.id
+              })
+              console.log("Restarted workflow: " + context.payload.workflow_run.id);
+              return;
+            }
+
+            job_id = job_ids[0];
+            // This function does not exist even though it is in the document
+            //github.rest.actions.reRunJobForWorkflow({
+            await github.request('POST /repos/{owner}/{repo}/actions/jobs/{job_id}/rerun', {
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              job_id: job_id
+            })
+            console.log("Restarted job: " + job_id);
diff --git a/.github/workflows/get-job-id/action.yml b/.github/workflows/get-job-id/action.yml
@@ -0,0 +1,30 @@
+name: Get Job ID
+inputs:
+  job-name:
+    required: false
+    type: 'string'
+
+outputs:
+  job-id:
+    description: "A space delimited list of check-targets to pass to ninja."
+    value: ${{ steps.job-id.outputs.result }}
+
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea #v7.0.1
+      id: job-id
+      with:
+        script: |
+          const job_data = await github.rest.actions.listJobsForWorkflowRun({
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            run_id: context.runId,
+          });
+
+          for (job of job_data.data.jobs) {
+            console.log(job)
+            if (job.name == "${{ inputs.job-name }}") {
+              return job.id
+            }
+          }
diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml
@@ -85,8 +85,16 @@ jobs:
   build-stage1-linux:
     name: "Build Stage 1 Linux"
     needs: prepare
-    runs-on: ubuntu-22.04
+    runs-on: ${{ matrix.runs-on }}
     if: github.repository == 'llvm/llvm-project'
+    strategy:
+      fail-fast: false
+      matrix:
+       runs-on:
+         - ubuntu-22.04
+         - windows-2022
+         - macos-13
+         - macos-14
     steps:
     - name: Checkout LLVM
       uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
@@ -100,7 +108,7 @@ jobs:
       uses: hendrikmuhs/ccache-action@ca3acd2731eef11f1572ccb126356c2f9298d35e # v1.2.9
       with:
         max-size: 250M
-        key: sccache-${{ runner.os }}-release
+        key: sccache-${{ runner.os }}-${{ runner.arch }}-release
         variant: sccache
 
     - name: Build Stage 1 Clang
@@ -119,14 +127,14 @@ jobs:
     - name: Upload Stage 1 Source
       uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0
       with:
-        name: stage1-source
+        name: ${{ runner.os }}-${{ runner.arch }}-stage1-source
         path: llvm-project.tar.zst
         retention-days: 2
 
     - name: Upload Stage 1 Build Dir
       uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0
       with:
-        name: stage1-build
+        name: ${{ runner.os}}-${{ runner.arch }}-stage1-build
         path: build.tar.zst
         retention-days: 2
 
@@ -135,16 +143,24 @@ jobs:
     needs:
       - prepare
       - build-stage1-linux
-    runs-on: ubuntu-22.04
+    runs-on: ${{ matrix.runs-on }}
     if: github.repository == 'llvm/llvm-project'
+    strategy:
+      fail-fast: false
+      matrix:
+       runs-on:
+         - ubuntu-22.04
+         - windows-2022
+         - macos-13
+         - macos-14
     steps:
     - name: Install Ninja
       uses: llvm/actions/install-ninja@22e9f909d35b50bd1181709564bfe816eaeaae81 # main
 
     - name: Download Stage 1 Artifacts
       uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
       with:
-        pattern: stage1-*
+        pattern: ${{ runner.os }}-${{ runner.arch }}-stage1-*
         merge-multiple: true
 
     - name: Unpack Artifacts
@@ -156,6 +172,7 @@ jobs:
         rm build.tar.zst
 
     - name: Build Stage 2
+      if: false
       run: |
         ninja -C /mnt/build stage2-instrumented
 
@@ -169,14 +186,14 @@ jobs:
     - name: Upload Stage 2 Source
       uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0
       with:
-        name: stage2-source
+        name: ${{ runner.os }}-${{ runner.arch }}-stage2-source
         path: ${{ github.workspace }}/llvm-project.tar.zst
         retention-days: 2
 
     - name: Upload Stage 2 Build Dir
       uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0
       with:
-        name: stage2-build
+        name: ${{ runner.os }}-${{ runner.arch }}-stage2-build
         path: ${{ github.workspace }}/build.tar.zst
         retention-days: 2
 
@@ -197,7 +214,7 @@ jobs:
     - name: 'Download artifact'
       uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
       with:
-        pattern: stage2-*
+        pattern: ${{ runner.os }}-${{ runner.arch }}-stage2-*
         merge-multiple: true
 
     - name: Unpack Artifact
@@ -208,10 +225,26 @@ jobs:
         tar --zstd -C /mnt -xf build.tar.zst
         rm build.tar.zst
 
+    - name: Timeout Restore
+      id: timeout
+      uses: ./.github/workflows/timeout-restore
+      with:
+        artifact-name-suffix: ${{ runner.os }}-${{ runner.arch }}
+
     - name: Build Release Package
+      name: Build
       run: |
         ninja -C /mnt/build stage2-package
 
+    - name: Timeout Save
+      if: always()
+      uses: ./.github/workflows/timeout-save
+        with:
+          job-name: "Build (${{ matrix.runs-on}})"
+          artifact-name-suffix: ${{ matrix.name }}
+          timeout-step: "Build"
+          timeout-minutes: 330
+
     - id: package-info
       run: |
         filename="LLVM-${{ needs.prepare.outputs.release-version }}-Linux.tar.gz"
@@ -221,7 +254,7 @@ jobs:
     - uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0
       if: always()
       with:
-        name: release-binary
+        name: ${{ runner.os }}-${{ runner.arch }}-release-binary
         path: ${{ steps.package-info.outputs.path }}
 
     # Clean up some build files to reduce size of artifact.
@@ -239,14 +272,14 @@ jobs:
     - name: Upload Stage 3 Source
       uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0
       with:
-        name: stage3-source
+        name: ${{ runner.os }}-${{ runner.arch }}-stage3-source
         path: llvm-project.tar.zst
         retention-days: 2
 
     - name: Upload Stage 3 Build Dir
       uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0
       with:
-        name: stage3-build
+        name: ${{ runner.os }}-${{ runner.arch }}-stage3-build
         path: build.tar.zst
         retention-days: 2
 
@@ -264,7 +297,7 @@ jobs:
     - name: 'Download artifact'
       uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
       with:
-        name: release-binary
+        pattern: *-release-binary
 
     - name: Upload Release
       run: |
@@ -273,7 +306,7 @@ jobs:
         --token ${{ github.token }} \
         --release ${{ needs.prepare.outputs.release-version }} \
         upload \
-        --files ${{ needs.build-stage3-linux.outputs.release-filename }}
+        --files *.tar.gz #${{ needs.build-stage3-linux.outputs.release-filename }}
 
 
   test-stage3-linux:
@@ -283,14 +316,22 @@ jobs:
       - build-stage3-linux
     runs-on: ubuntu-22.04
     if: github.repository == 'llvm/llvm-project'
+    strategy:
+      fail-fast: false
+      matrix:
+       runs-on:
+         - ubuntu-22.04
+         - windows-2022
+         - macos-13
+         - macos-14
     steps:
     - name: Install Ninja
       uses: llvm/actions/install-ninja@22e9f909d35b50bd1181709564bfe816eaeaae81 # main
 
     - name: 'Download artifact'
       uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
       with:
-        pattern: stage3-*
+        pattern: ${{ runner.os }}-${{ runner.arch }}-stage3-*
         merge-multiple: true
 
     - name: Unpack Artifact
diff --git a/.github/workflows/timeout-restore/action.yml b/.github/workflows/timeout-restore/action.yml
@@ -0,0 +1,33 @@
+name: Timeout Restore
+description: Save build state from a timed out job.
+inputs:
+  artifact-name-suffix:
+    desciption: Suffix to add to the name of the artifact containing the build state.
+    required: true
+
+outputs:
+  exists:
+    description: "This is true if a previous timeout build was restored, false otherwise."
+    value: ${{ steps.timeout-artifact.exists }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Download Artifact
+      uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
+      with:
+        pattern: timeout-build-${{ inputs.artifact-name-suffix }}
+        merge-multiple: true
+
+    - name: Unpack Artifact
+      id: timeout-artifact
+      shell: bash
+      run: |
+        if [ -e llvm-project.tar.zst ]; then
+          tar --zstd -xf llvm-project.tar.zst
+          rm llvm-project.tar.zst
+          echo "exists=true" >> $GITHUB_OUTPUT
+        else
+          echo "exists=false" >> $GITHUB_OUTPUT
+        fi
+
diff --git a/.github/workflows/timeout-save/action.yml b/.github/workflows/timeout-save/action.yml
