fix: setup_baseline_version only runs once, and fails often on network instability

Author: iankhou

.github/actions/setup_baseline_version/action.yml

@@ -4,36 +4,86 @@ inputs:
   node_version:
     description: node version used to configure environment with
     required: true
+  max_retry_attempts:
+    description: maximum number of retry attempts for network operations
+    required: false
+    default: '5'
+  initial_wait_time:
+    description: initial wait time in seconds before first retry (increases by this amount each retry)
+    required: false
+    default: '30'
 outputs:
   baseline_dir:
     description: 'Path where baseline project directory is setup'
     value: ${{ steps.move_baseline_version.outputs.baseline_dir }}
 runs:
   using: composite
   steps:
+    - name: Setup retry function
+      shell: bash
+      run: |
+        # Create a shared retry function that can be used in other steps
+        cat > /tmp/retry_function.sh << 'EOF'
+        retry_with_backoff() {
+          local max_attempts=$1
+          local initial_wait=$2
+          shift 2
+          local command="$@"
+          
+          local attempt=1
+          local wait_time=$initial_wait
+          
+          while [ $attempt -le $max_attempts ]; do
+            echo "Attempt $attempt of $max_attempts"
+            
+            if eval "$command"; then
+              echo "Command completed successfully"
+              return 0
+            fi
+            
+            if [ $attempt -lt $max_attempts ]; then
+              echo "Command failed. Waiting ${wait_time} seconds before retry..."
+              sleep $wait_time
+              wait_time=$((wait_time + initial_wait))
+            fi
+            
+            attempt=$((attempt + 1))
+          done
+          
+          echo "All retry attempts failed"
+          return 1
+        }
+        EOF
     - name: Get baseline commit sha
       id: get_baseline_commit_sha
       shell: bash
       env:
         GH_TOKEN: ${{ github.token }}
       run: |
-        if [[ ${{ github.event_name }} == 'push' ]]; then
-          # The SHA of the most recent commit on ref before the push.
-          baseline_commit_sha="${{ github.event.before }}"
-        elif [[ ${{ github.event_name }} == 'pull_request' ]]; then
-          # The SHA of the HEAD commit on base branch.
-          baseline_commit_sha="${{ github.event.pull_request.base.sha }}"
-        elif [[ ${{ github.event_name }} == 'schedule' ]] || [[ ${{ github.event_name }} == 'workflow_dispatch' ]]; then
-          # The SHA of the parent of HEAD commit on main branch.
-          # This assumes linear history of main branch, i.e. one parent.
-          # These events have only information about HEAD commit, hence the need for lookup.
-          baseline_commit_sha=$(gh api /repos/${{ github.repository }}/commits/${{ github.sha }} | jq -r '.parents[0].sha')
-        else
-          echo Unable to determine baseline commit sha;
-          exit 1;
-        fi
-        echo baseline commit sha is $baseline_commit_sha;
-        echo "baseline_commit_sha=$baseline_commit_sha" >> "$GITHUB_OUTPUT";
+        source /tmp/retry_function.sh
+
+        get_baseline_commit() {
+          if [[ ${{ github.event_name }} == 'push' ]]; then
+            # The SHA of the most recent commit on ref before the push.
+            baseline_commit_sha="${{ github.event.before }}"
+          elif [[ ${{ github.event_name }} == 'pull_request' ]]; then
+            # The SHA of the HEAD commit on base branch.
+            baseline_commit_sha="${{ github.event.pull_request.base.sha }}"
+          elif [[ ${{ github.event_name }} == 'schedule' ]] || [[ ${{ github.event_name }} == 'workflow_dispatch' ]]; then
+            # The SHA of the parent of HEAD commit on main branch.
+            # This assumes linear history of main branch, i.e. one parent.
+            # These events have only information about HEAD commit, hence the need for lookup.
+            baseline_commit_sha=$(gh api /repos/${{ github.repository }}/commits/${{ github.sha }} | jq -r '.parents[0].sha')
+          else
+            echo "Unable to determine baseline commit sha"
+            return 1
+          fi
+        }
+
+        # Execute with retries
+        retry_with_backoff ${{ inputs.max_retry_attempts }} ${{ inputs.initial_wait_time }} get_baseline_commit
+        echo "baseline commit sha is $baseline_commit_sha"
+        echo "baseline_commit_sha=$baseline_commit_sha" >> "$GITHUB_OUTPUT"
     - name: Checkout baseline version
       uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # version 4.1.4
       with:
@@ -44,8 +94,10 @@ runs:
     - name: Install and build baseline version
       shell: bash
       run: |
-        npm ci
-        npm run build
+        source /tmp/retry_function.sh
+
+        # Execute npm ci and build with retries
+        retry_with_backoff ${{ inputs.max_retry_attempts }} ${{ inputs.initial_wait_time }} "npm ci && npm run build"
     - name: Move baseline version
       id: move_baseline_version
       shell: bash

.github/workflows/health_checks.yml

@@ -404,41 +404,11 @@ jobs:
       # See https://github.com/actions/checkout/issues/692
       - name: Checkout version for baseline
         uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # version 4.1.4
-      - name: Setup baseline version (attempt 1)
-        id: setup_baseline_version_attempt1
-        uses: ./.github/actions/setup_baseline_version
-        with:
-          node_version: 18
-        continue-on-error: true
-
-      - name: Setup baseline version (attempt 2)
-        id: setup_baseline_version_attempt2
-        if: steps.setup_baseline_version_attempt1.outcome == 'failure'
-        uses: ./.github/actions/setup_baseline_version
-        with:
-          node_version: 18
-        continue-on-error: true
-
-      - name: Setup baseline version (attempt 3)
-        id: setup_baseline_version_attempt3
-        if: steps.setup_baseline_version_attempt2.outcome == 'failure'
+      - name: Setup baseline version
+        id: setup_baseline_version
         uses: ./.github/actions/setup_baseline_version
         with:
           node_version: 18
-
-      - name: Set baseline directory output
-        id: setup_baseline_version
-        run: |
-          if [[ "${{ steps.setup_baseline_version_attempt1.outcome }}" == "success" ]]; then
-            echo "✅ Setup baseline version succeeded on attempt 1"
-            echo "baseline_dir=${{ steps.setup_baseline_version_attempt1.outputs.baseline_dir }}" >> "$GITHUB_OUTPUT"
-          elif [[ "${{ steps.setup_baseline_version_attempt2.outcome }}" == "success" ]]; then
-            echo "⚠️ Setup baseline version succeeded on attempt 2 (after 1 failure)"
-            echo "baseline_dir=${{ steps.setup_baseline_version_attempt2.outputs.baseline_dir }}" >> "$GITHUB_OUTPUT"
-          else
-            echo "⚠️ Setup baseline version succeeded on attempt 3 (after 2 failures)"
-            echo "baseline_dir=${{ steps.setup_baseline_version_attempt3.outputs.baseline_dir }}" >> "$GITHUB_OUTPUT"
-          fi
       - name: Checkout current version
         uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # version 4.1.4
       - name: Run e2e amplify outputs backwards compatibility test