fix: resolve all logical issues in Discord health monitoring

chrisbbreuer · claude · chrisbbreuer · commit fba518daa63b · 2026-03-06T08:24:19.000-08:00
- discord-notify: wrap entire action in function so jq/curl failures
  never crash parent workflow (was only protecting curl, not jq)
- health-monitor: aws s3 ls --max-items is invalid, use s3api head-bucket
- health-monitor: guard against null/empty API responses in stale sync
  check (was computing age from epoch 0 → false positive every 30min)
- health-monitor: extract last 3 chars of curl -w output for clean HTTP code
- build-package/release: map cancelled builds to yellow/warning, not red/failure
- update-pantry/update-packages: add fallback values for empty step outputs
- sync-binaries/build-versions: filter out skipped jobs from Discord fields
  (mutually exclusive jobs showed "skipped" which looks like an error)
- health-monitor: use jq '// empty' to handle null workflow run timestamps

Co-Authored-By: Claude Opus 4.6 &lt;noreply@anthropic.com&gt;
diff --git a/.github/actions/discord-notify/action.yml b/.github/actions/discord-notify/action.yml
@@ -35,35 +35,46 @@ runs:
         RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
         WORKFLOW_NAME: ${{ github.workflow }}
       run: |
-        # Map status to color
-        case "$STATUS" in
-          success) COLOR=3066993 ;;   # green
-          warning) COLOR=16776960 ;;  # yellow
-          failure) COLOR=15158332 ;;  # red
-          *) COLOR=10181046 ;;        # purple/unknown
-        esac
+        # Entire block is wrapped so notification errors never fail the parent workflow
+        _send_discord() {
+          # Map status to color
+          case "$STATUS" in
+            success) COLOR=3066993 ;;   # green
+            warning) COLOR=16776960 ;;  # yellow
+            failure) COLOR=15158332 ;;  # red
+            *) COLOR=10181046 ;;        # purple/unknown
+          esac
 
-        # Build fields JSON — append clickable workflow run link
-        FIELDS_WITH_LINK=$(echo "$FIELDS" | jq -c \
-          --arg url "$RUN_URL" \
-          --arg name "$WORKFLOW_NAME" \
-          '. + [{"name": "Workflow Run", "value": ("[" + $name + "](" + $url + ")"), "inline": false}]')
+          # Validate fields JSON, fall back to empty array
+          if ! echo "$FIELDS" | jq -e '.' >/dev/null 2>&1; then
+            FIELDS='[]'
+          fi
 
-        # Build payload safely via jq (handles all escaping)
-        jq -nc \
-          --arg title "$TITLE" \
-          --arg description "$DESCRIPTION" \
-          --argjson color "$COLOR" \
-          --argjson fields "$FIELDS_WITH_LINK" \
-          '{
-            embeds: [{
-              title: $title,
-              description: $description,
-              color: $color,
-              fields: $fields,
-              timestamp: (now | todate)
-            }]
-          }' | curl -sS -X POST \
-          -H "Content-Type: application/json" \
-          --data-binary @- \
-          "$WEBHOOK_URL" || true
+          # Append clickable workflow run link
+          FIELDS_WITH_LINK=$(echo "$FIELDS" | jq -c \
+            --arg url "$RUN_URL" \
+            --arg name "$WORKFLOW_NAME" \
+            '. + [{"name": "Workflow Run", "value": ("[" + $name + "](" + $url + ")"), "inline": false}]')
+
+          # Build payload safely via jq (handles all escaping)
+          PAYLOAD=$(jq -nc \
+            --arg title "$TITLE" \
+            --arg description "$DESCRIPTION" \
+            --argjson color "$COLOR" \
+            --argjson fields "$FIELDS_WITH_LINK" \
+            '{
+              embeds: [{
+                title: $title,
+                description: $description,
+                color: $color,
+                fields: $fields,
+                timestamp: (now | todate)
+              }]
+            }')
+
+          curl -sS -X POST \
+            -H "Content-Type: application/json" \
+            -d "$PAYLOAD" \
+            "$WEBHOOK_URL"
+        }
+        _send_discord || echo "::warning::Discord notification failed (non-fatal)"
diff --git a/.github/workflows/build-package.yml b/.github/workflows/build-package.yml
@@ -234,13 +234,13 @@ jobs:
       - uses: actions/checkout@v6
       - uses: ./.github/actions/discord-notify
         with:
-          title: "Build Package — ${{ needs.build.result == 'success' && 'Success' || 'Failed' }}"
+          title: "Build Package — ${{ needs.build.result == 'success' && 'Success' || needs.build.result == 'cancelled' && 'Cancelled' || 'Failed' }}"
           description: "Packages: ${{ github.event.inputs.packages }}"
-          status: ${{ needs.build.result == 'success' && 'success' || 'failure' }}
+          status: ${{ needs.build.result == 'success' && 'success' || needs.build.result == 'cancelled' && 'warning' || 'failure' }}
           fields: |
             [
-              {"name": "Platform", "value": "${{ github.event.inputs.platform }}", "inline": true},
-              {"name": "Force", "value": "${{ github.event.inputs.force }}", "inline": true},
+              {"name": "Platform", "value": "${{ github.event.inputs.platform || 'both' }}", "inline": true},
+              {"name": "Force", "value": "${{ github.event.inputs.force || 'true' }}", "inline": true},
               {"name": "Actor", "value": "${{ github.actor }}", "inline": true}
             ]
           webhook_url: ${{ secrets.DISCORD_WEBHOOK_URL }}
diff --git a/.github/workflows/build-versions.yml b/.github/workflows/build-versions.yml
@@ -437,15 +437,24 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v6
+      - name: Build fields
+        id: fields
+        run: |
+          FIELDS="["
+          FIRST=true
+          for pair in "Build Targeted:${{ needs.build-targeted.result }}" "Build Batches:${{ needs.build.result }}"; do
+            NAME="${pair%%:*}"
+            VAL="${pair##*:}"
+            [ "$VAL" = "skipped" ] && continue
+            [ "$FIRST" = "true" ] && FIRST=false || FIELDS="$FIELDS,"
+            FIELDS="$FIELDS{\"name\":\"$NAME\",\"value\":\"$VAL\",\"inline\":true}"
+          done
+          FIELDS="$FIELDS,{\"name\":\"Max Versions\",\"value\":\"${{ github.event.inputs.max-versions || '5' }}\",\"inline\":true}]"
+          echo "json=$FIELDS" >> "$GITHUB_OUTPUT"
       - uses: ./.github/actions/discord-notify
         with:
           title: Build Versions — Failed
           description: Multi-version build failed
           status: failure
-          fields: |
-            [
-              {"name": "Build Targeted", "value": "${{ needs.build-targeted.result }}", "inline": true},
-              {"name": "Build Batches", "value": "${{ needs.build.result }}", "inline": true},
-              {"name": "Max Versions", "value": "${{ github.event.inputs.max-versions || '5' }}", "inline": true}
-            ]
+          fields: ${{ steps.fields.outputs.json }}
           webhook_url: ${{ secrets.DISCORD_WEBHOOK_URL }}
diff --git a/.github/workflows/health-monitor.yml b/.github/workflows/health-monitor.yml
@@ -19,16 +19,17 @@ jobs:
       - name: Check registry health
         id: registry
         run: |
+          # Use a temp file to isolate the HTTP code from error output
           HTTP_CODE=$(curl -sS -o /tmp/health-response -w '%{http_code}' \
-            --max-time 15 --retry 2 --retry-delay 5 \
-            'https://registry.stacksjs.com/health' 2>/dev/null) || HTTP_CODE="000"
+            --max-time 15 \
+            'https://registry.stacksjs.com/health' 2>/dev/null) || true
+          # Extract only the last 3 chars (the HTTP code) in case of any prefix noise
+          HTTP_CODE="${HTTP_CODE: -3}"
           echo "http_code=$HTTP_CODE" >> "$GITHUB_OUTPUT"
           if [ "$HTTP_CODE" = "200" ]; then
             echo "status=healthy" >> "$GITHUB_OUTPUT"
           else
-            BODY=$(cat /tmp/health-response 2>/dev/null | head -c 200 || echo "no response")
             echo "status=unhealthy" >> "$GITHUB_OUTPUT"
-            echo "body=$BODY" >> "$GITHUB_OUTPUT"
           fi
 
       - name: Check S3 bucket accessibility
@@ -38,7 +39,7 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           AWS_REGION: us-east-1
         run: |
-          if aws s3 ls s3://pantry-registry --max-items 1 >/dev/null 2>&1; then
+          if aws s3api head-bucket --bucket pantry-registry 2>/dev/null; then
             echo "status=accessible" >> "$GITHUB_OUTPUT"
           else
             echo "status=inaccessible" >> "$GITHUB_OUTPUT"
@@ -73,27 +74,31 @@ jobs:
           # Check update-pantry — should run every 20min, alert if >2h stale
           LAST_PANTRY=$(gh api \
             "repos/${{ github.repository }}/actions/workflows/update-pantry.yml/runs?status=success&per_page=1" \
-            --jq '.workflow_runs[0].updated_at' 2>/dev/null || echo "")
-          if [ -n "$LAST_PANTRY" ]; then
-            PANTRY_TS=$(date -u -d "$LAST_PANTRY" +%s 2>/dev/null || echo "0")
-            PANTRY_AGE=$(( (NOW - PANTRY_TS) / 3600 ))
-            if [ "$PANTRY_AGE" -gt 2 ]; then
-              STALE_ITEMS="update-pantry (${PANTRY_AGE}h ago)"
+            --jq '.workflow_runs[0].updated_at // empty' 2>/dev/null || echo "")
+          if [ -n "$LAST_PANTRY" ] && [ "$LAST_PANTRY" != "null" ]; then
+            PANTRY_TS=$(date -u -d "$LAST_PANTRY" +%s 2>/dev/null || echo "")
+            if [ -n "$PANTRY_TS" ] && [ "$PANTRY_TS" -gt 0 ] 2>/dev/null; then
+              PANTRY_AGE=$(( (NOW - PANTRY_TS) / 3600 ))
+              if [ "$PANTRY_AGE" -gt 2 ]; then
+                STALE_ITEMS="update-pantry (${PANTRY_AGE}h ago)"
+              fi
             fi
           fi
 
           # Check sync-binaries — runs every 6h, alert if >12h stale
           LAST_SYNC=$(gh api \
             "repos/${{ github.repository }}/actions/workflows/sync-binaries.yml/runs?status=success&per_page=1" \
-            --jq '.workflow_runs[0].updated_at' 2>/dev/null || echo "")
-          if [ -n "$LAST_SYNC" ]; then
-            SYNC_TS=$(date -u -d "$LAST_SYNC" +%s 2>/dev/null || echo "0")
-            SYNC_AGE=$(( (NOW - SYNC_TS) / 3600 ))
-            if [ "$SYNC_AGE" -gt 12 ]; then
-              if [ -n "$STALE_ITEMS" ]; then
-                STALE_ITEMS="$STALE_ITEMS, sync-binaries (${SYNC_AGE}h ago)"
-              else
-                STALE_ITEMS="sync-binaries (${SYNC_AGE}h ago)"
+            --jq '.workflow_runs[0].updated_at // empty' 2>/dev/null || echo "")
+          if [ -n "$LAST_SYNC" ] && [ "$LAST_SYNC" != "null" ]; then
+            SYNC_TS=$(date -u -d "$LAST_SYNC" +%s 2>/dev/null || echo "")
+            if [ -n "$SYNC_TS" ] && [ "$SYNC_TS" -gt 0 ] 2>/dev/null; then
+              SYNC_AGE=$(( (NOW - SYNC_TS) / 3600 ))
+              if [ "$SYNC_AGE" -gt 12 ]; then
+                if [ -n "$STALE_ITEMS" ]; then
+                  STALE_ITEMS="$STALE_ITEMS, sync-binaries (${SYNC_AGE}h ago)"
+                else
+                  STALE_ITEMS="sync-binaries (${SYNC_AGE}h ago)"
+                fi
               fi
             fi
           fi
@@ -141,9 +146,9 @@ jobs:
           status: failure
           fields: |
             [
-              {"name": "Registry", "value": "${{ steps.registry.outputs.status }} (HTTP ${{ steps.registry.outputs.http_code }})", "inline": true},
-              {"name": "S3 Bucket", "value": "${{ steps.s3.outputs.status }}", "inline": true},
-              {"name": "Failures (6h)", "value": "${{ steps.failures.outputs.count }}", "inline": true},
+              {"name": "Registry", "value": "${{ steps.registry.outputs.status || 'unknown' }} (HTTP ${{ steps.registry.outputs.http_code || '?' }})", "inline": true},
+              {"name": "S3 Bucket", "value": "${{ steps.s3.outputs.status || 'unknown' }}", "inline": true},
+              {"name": "Failures (6h)", "value": "${{ steps.failures.outputs.count || '?' }}", "inline": true},
               {"name": "Stale Syncs", "value": "${{ steps.stale.outputs.items || 'none' }}", "inline": false}
             ]
           webhook_url: ${{ secrets.DISCORD_WEBHOOK_URL }}
@@ -171,8 +176,8 @@ jobs:
           status: success
           fields: |
             [
-              {"name": "Registry", "value": "${{ steps.registry.outputs.status }}", "inline": true},
-              {"name": "S3 Bucket", "value": "${{ steps.s3.outputs.status }}", "inline": true},
-              {"name": "Failures (6h)", "value": "${{ steps.failures.outputs.count }}", "inline": true}
+              {"name": "Registry", "value": "${{ steps.registry.outputs.status || 'unknown' }}", "inline": true},
+              {"name": "S3 Bucket", "value": "${{ steps.s3.outputs.status || 'unknown' }}", "inline": true},
+              {"name": "Failures (6h)", "value": "${{ steps.failures.outputs.count || '0' }}", "inline": true}
             ]
           webhook_url: ${{ secrets.DISCORD_WEBHOOK_URL }}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -86,9 +86,9 @@ jobs:
       - uses: actions/checkout@v6
       - uses: ./.github/actions/discord-notify
         with:
-          title: "Release — ${{ needs.pantry.result == 'success' && 'Published' || 'Failed' }}"
+          title: "Release — ${{ needs.pantry.result == 'success' && 'Published' || needs.pantry.result == 'cancelled' && 'Cancelled' || 'Failed' }}"
           description: "Tag: ${{ github.ref_name }}"
-          status: ${{ needs.pantry.result == 'success' && 'success' || 'failure' }}
+          status: ${{ needs.pantry.result == 'success' && 'success' || needs.pantry.result == 'cancelled' && 'warning' || 'failure' }}
           fields: |
             [
               {"name": "Tag", "value": "${{ github.ref_name }}", "inline": true},
diff --git a/.github/workflows/sync-binaries.yml b/.github/workflows/sync-binaries.yml
@@ -538,15 +538,25 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v6
+      - name: Build fields
+        id: fields
+        run: |
+          # Only include jobs that actually ran (not skipped)
+          FIELDS="["
+          FIRST=true
+          for pair in "Sync Pre-built:${{ needs.sync-prebuilt.result }}" "Build Targeted:${{ needs.build-targeted.result }}" "Build Batches:${{ needs.build.result }}"; do
+            NAME="${pair%%:*}"
+            VAL="${pair##*:}"
+            [ "$VAL" = "skipped" ] && continue
+            [ "$FIRST" = "true" ] && FIRST=false || FIELDS="$FIELDS,"
+            FIELDS="$FIELDS{\"name\":\"$NAME\",\"value\":\"$VAL\",\"inline\":true}"
+          done
+          FIELDS="$FIELDS]"
+          echo "json=$FIELDS" >> "$GITHUB_OUTPUT"
       - uses: ./.github/actions/discord-notify
         with:
           title: Sync Binaries — Failed
           description: One or more sync/build jobs failed
           status: failure
-          fields: |
-            [
-              {"name": "Sync Pre-built", "value": "${{ needs.sync-prebuilt.result }}", "inline": true},
-              {"name": "Build Targeted", "value": "${{ needs.build-targeted.result }}", "inline": true},
-              {"name": "Build Batches", "value": "${{ needs.build.result }}", "inline": true}
-            ]
+          fields: ${{ steps.fields.outputs.json }}
           webhook_url: ${{ secrets.DISCORD_WEBHOOK_URL }}
diff --git a/.github/workflows/update-packages.yml b/.github/workflows/update-packages.yml
@@ -393,7 +393,8 @@ jobs:
           status: failure
           fields: |
             [
-              {"name": "Fetch Result", "value": "${{ steps.fetch-packages.outputs.fetch_result && 'received' || 'missing' }}", "inline": true},
-              {"name": "Safe to Commit", "value": "${{ steps.deletion-check.outputs.safe_to_commit || 'n/a' }}", "inline": true}
+              {"name": "Fetch Success", "value": "${{ steps.commit-message.outputs.should_commit || 'unknown' }}", "inline": true},
+              {"name": "Safe to Commit", "value": "${{ steps.deletion-check.outputs.safe_to_commit || 'not reached' }}", "inline": true},
+              {"name": "Updated", "value": "${{ steps.commit-message.outputs.total_updated || '?' }}", "inline": true}
             ]
           webhook_url: ${{ secrets.DISCORD_WEBHOOK_URL }}
diff --git a/.github/workflows/update-pantry.yml b/.github/workflows/update-pantry.yml
@@ -206,7 +206,7 @@ jobs:
           status: failure
           fields: |
             [
-              {"name": "Update Success", "value": "${{ steps.update-pantry.outputs.success }}", "inline": true},
-              {"name": "Changes", "value": "${{ steps.update-pantry.outputs.total_changes }}", "inline": true}
+              {"name": "Update Success", "value": "${{ steps.update-pantry.outputs.success || 'unknown' }}", "inline": true},
+              {"name": "Changes", "value": "${{ steps.update-pantry.outputs.total_changes || '?' }}", "inline": true}
             ]
           webhook_url: ${{ secrets.DISCORD_WEBHOOK_URL }}

Original file line number	Diff line number	Diff line change
`@@ -393,7 +393,8 @@ jobs:`
`393`	`393`	`status: failure`
`394`	`394`	`fields: \|`
`395`	`395`	`[`
`396`		`- {"name": "Fetch Result", "value": "${{ steps.fetch-packages.outputs.fetch_result && 'received' \|\| 'missing' }}", "inline": true},`
`397`		`- {"name": "Safe to Commit", "value": "${{ steps.deletion-check.outputs.safe_to_commit \|\| 'n/a' }}", "inline": true}`
	`396`	`+ {"name": "Fetch Success", "value": "${{ steps.commit-message.outputs.should_commit \|\| 'unknown' }}", "inline": true},`
	`397`	`+ {"name": "Safe to Commit", "value": "${{ steps.deletion-check.outputs.safe_to_commit \|\| 'not reached' }}", "inline": true},`
	`398`	`+ {"name": "Updated", "value": "${{ steps.commit-message.outputs.total_updated \|\| '?' }}", "inline": true}`
`398`	`399`	`]`
`399`	`400`	`webhook_url: ${{ secrets.DISCORD_WEBHOOK_URL }}`
Original file line number	Diff line number	Diff line change
`@@ -206,7 +206,7 @@ jobs:`
`206`	`206`	`status: failure`
`207`	`207`	`fields: \|`
`208`	`208`	`[`
`209`		`- {"name": "Update Success", "value": "${{ steps.update-pantry.outputs.success }}", "inline": true},`
`210`		`- {"name": "Changes", "value": "${{ steps.update-pantry.outputs.total_changes }}", "inline": true}`
	`209`	`+ {"name": "Update Success", "value": "${{ steps.update-pantry.outputs.success \|\| 'unknown' }}", "inline": true},`
	`210`	`+ {"name": "Changes", "value": "${{ steps.update-pantry.outputs.total_changes \|\| '?' }}", "inline": true}`
`211`	`211`	`]`
`212`	`212`	`webhook_url: ${{ secrets.DISCORD_WEBHOOK_URL }}`