Add Slack notifications with resource counts and error reporting

james00012 · james00012 · commit db7ee0ac1371 · 2026-01-21T12:46:09.000+09:00
- Capture nuke output to parse deleted/error counts
- Upload logs as artifacts for aggregation
- Notify jobs download artifacts and aggregate counts across all regions
- Slack message includes: status, deleted count, error count (if any), run link
- Fetch webhook URL from AWS Secrets Manager (PhxDevOps account)
diff --git a/.github/workflows/nuke.yml b/.github/workflows/nuke.yml
@@ -24,7 +24,6 @@ permissions:
   contents: read
 
 env:
-  GO_VERSION: '1.21'
   MISE_VERSION: '2025.12.10'
   COMMON_EXCLUDES: >-
     --exclude-resource-type iam
@@ -40,13 +39,6 @@ env:
     --exclude-resource-type config-rules
     --exclude-resource-type nat-gateway
     --exclude-resource-type ec2-subnet
-  ALL_REGIONS: >-
-    ap-northeast-1 ap-northeast-2 ap-northeast-3
-    ap-south-1 ap-southeast-1 ap-southeast-2
-    ca-central-1 eu-central-1 eu-north-1
-    eu-west-1 eu-west-2 eu-west-3
-    me-central-1 sa-east-1
-    us-east-1 us-east-2 us-west-1 us-west-2
 
 jobs:
   # ============================================
@@ -56,6 +48,9 @@ jobs:
     name: "PhxDevOps: Global"
     runs-on: ubuntu-latest
     timeout-minutes: 30
+    outputs:
+      deleted_count: ${{ steps.nuke.outputs.deleted_count }}
+      error_count: ${{ steps.nuke.outputs.error_count }}
     if: |
       github.event_name == 'push' ||
       github.event_name == 'workflow_dispatch' && (inputs.account == 'phxdevops' || inputs.account == '') ||
@@ -81,16 +76,30 @@ jobs:
           restore-keys: ${{ runner.os }}-go-
       - run: go mod download
       - name: Nuke global resources
+        id: nuke
         run: |
+          set +e
           go run -ldflags="-X 'main.VERSION=${{ github.sha }}'" main.go aws \
             --older-than 2h --force --config ./.github/nuke_config.yml \
             --region global ${{ env.COMMON_EXCLUDES }} \
-            --delete-unaliased-kms-keys --log-level info
+            --delete-unaliased-kms-keys --log-level info 2>&1 | tee /tmp/nuke.log
+          EXIT_CODE=${PIPESTATUS[0]}
+
+          # Parse output for counts
+          DELETED=$(grep -c "^\s*INFO\s*\[Deleted\]" /tmp/nuke.log || echo "0")
+          ERRORS=$(grep -c "^\s*ERROR\s*\[Failed\]" /tmp/nuke.log || echo "0")
+          echo "deleted_count=${DELETED}" >> $GITHUB_OUTPUT
+          echo "error_count=${ERRORS}" >> $GITHUB_OUTPUT
+
+          exit $EXIT_CODE
 
   phxdevops_regional:
     name: "PhxDevOps: ${{ matrix.region }}"
     runs-on: ubuntu-latest
     timeout-minutes: 20
+    outputs:
+      deleted_count: ${{ steps.nuke.outputs.deleted_count }}
+      error_count: ${{ steps.nuke.outputs.error_count }}
     if: |
       github.event_name == 'push' ||
       github.event_name == 'workflow_dispatch' && (inputs.account == 'phxdevops' || inputs.account == '') ||
@@ -120,11 +129,28 @@ jobs:
           restore-keys: ${{ runner.os }}-go-
       - run: go mod download
       - name: Nuke ${{ matrix.region }}
+        id: nuke
         run: |
+          set +e
           go run -ldflags="-X 'main.VERSION=${{ github.sha }}'" main.go aws \
             --older-than 2h --force --config ./.github/nuke_config.yml \
             --region ${{ matrix.region }} ${{ env.COMMON_EXCLUDES }} \
-            --delete-unaliased-kms-keys --log-level info
+            --delete-unaliased-kms-keys --log-level info 2>&1 | tee /tmp/nuke.log
+          EXIT_CODE=${PIPESTATUS[0]}
+
+          DELETED=$(grep -c "^\s*INFO\s*\[Deleted\]" /tmp/nuke.log || echo "0")
+          ERRORS=$(grep -c "^\s*ERROR\s*\[Failed\]" /tmp/nuke.log || echo "0")
+          echo "deleted_count=${DELETED}" >> $GITHUB_OUTPUT
+          echo "error_count=${ERRORS}" >> $GITHUB_OUTPUT
+
+          exit $EXIT_CODE
+      - name: Upload results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: phxdevops-${{ matrix.region }}
+          path: /tmp/nuke.log
+          retention-days: 7
 
   phxdevops_notify:
     name: "PhxDevOps: Notify"
@@ -141,12 +167,32 @@ jobs:
         with:
           role-to-assume: arn:aws:iam::087285199408:role/cloud-nuke-gha
           aws-region: us-east-1
-      - name: Send Slack notification
+      - name: Download all artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: phxdevops-*
+          path: /tmp/logs
+          merge-multiple: true
+        continue-on-error: true
+      - name: Aggregate and notify
         run: |
           WEBHOOK_URL=$(aws secretsmanager get-secret-value \
             --secret-id cloud-nuke/slack-webhook \
             --query SecretString --output text)
 
+          # Aggregate counts from all logs
+          TOTAL_DELETED=0
+          TOTAL_ERRORS=0
+          if [ -d /tmp/logs ]; then
+            TOTAL_DELETED=$(grep -rh "^\s*INFO\s*\[Deleted\]" /tmp/logs 2>/dev/null | wc -l || echo "0")
+            TOTAL_ERRORS=$(grep -rh "^\s*ERROR\s*\[Failed\]" /tmp/logs 2>/dev/null | wc -l || echo "0")
+          fi
+          # Add global counts
+          GLOBAL_DELETED="${{ needs.phxdevops_global.outputs.deleted_count }}"
+          GLOBAL_ERRORS="${{ needs.phxdevops_global.outputs.error_count }}"
+          TOTAL_DELETED=$((TOTAL_DELETED + ${GLOBAL_DELETED:-0}))
+          TOTAL_ERRORS=$((TOTAL_ERRORS + ${GLOBAL_ERRORS:-0}))
+
           if [ "${{ needs.phxdevops_global.result }}" == "success" ] && \
              [ "${{ needs.phxdevops_regional.result }}" == "success" ]; then
             STATUS="✅ Success"
@@ -156,6 +202,14 @@ jobs:
             COLOR="danger"
           fi
 
+          # Build message
+          MSG="*PhxDevOps Nuke*: ${STATUS}"
+          MSG="${MSG}\nDeleted: ${TOTAL_DELETED} resources"
+          if [ "$TOTAL_ERRORS" -gt 0 ]; then
+            MSG="${MSG} | Errors: ${TOTAL_ERRORS}"
+          fi
+          MSG="${MSG}\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Run>"
+
           curl -sS -X POST "$WEBHOOK_URL" -H "Content-Type: application/json" -d @- <<EOF
           {
             "attachments": [{
@@ -164,7 +218,7 @@ jobs:
                 "type": "section",
                 "text": {
                   "type": "mrkdwn",
-                  "text": "*PhxDevOps Nuke*: ${STATUS}\nGlobal: ${{ needs.phxdevops_global.result }} | Regional: ${{ needs.phxdevops_regional.result }}\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Run>"
+                  "text": "${MSG}"
                 }
               }]
             }]
@@ -178,6 +232,9 @@ jobs:
     name: "ConfigTests: Global"
     runs-on: ubuntu-latest
     timeout-minutes: 30
+    outputs:
+      deleted_count: ${{ steps.nuke.outputs.deleted_count }}
+      error_count: ${{ steps.nuke.outputs.error_count }}
     if: |
       github.event_name == 'workflow_dispatch' && (inputs.account == 'configtests' || inputs.account == '') ||
       github.event_name == 'schedule' && github.event.schedule == '0 */3 * * *'
@@ -202,12 +259,22 @@ jobs:
           restore-keys: ${{ runner.os }}-go-
       - run: go mod download
       - name: Nuke global resources
+        id: nuke
         run: |
+          set +e
           go run -ldflags="-X 'main.VERSION=${{ github.sha }}'" main.go aws \
             --older-than 2h --force --config ./.github/nuke_config.yml \
             --region global ${{ env.COMMON_EXCLUDES }} \
             --exclude-resource-type internet-gateway \
-            --delete-unaliased-kms-keys --log-level info
+            --delete-unaliased-kms-keys --log-level info 2>&1 | tee /tmp/nuke.log
+          EXIT_CODE=${PIPESTATUS[0]}
+
+          DELETED=$(grep -c "^\s*INFO\s*\[Deleted\]" /tmp/nuke.log || echo "0")
+          ERRORS=$(grep -c "^\s*ERROR\s*\[Failed\]" /tmp/nuke.log || echo "0")
+          echo "deleted_count=${DELETED}" >> $GITHUB_OUTPUT
+          echo "error_count=${ERRORS}" >> $GITHUB_OUTPUT
+
+          exit $EXIT_CODE
 
   configtests_regional:
     name: "ConfigTests: ${{ matrix.region }}"
@@ -241,12 +308,29 @@ jobs:
           restore-keys: ${{ runner.os }}-go-
       - run: go mod download
       - name: Nuke ${{ matrix.region }}
+        id: nuke
         run: |
+          set +e
           go run -ldflags="-X 'main.VERSION=${{ github.sha }}'" main.go aws \
             --older-than 2h --force --config ./.github/nuke_config.yml \
             --region ${{ matrix.region }} ${{ env.COMMON_EXCLUDES }} \
             --exclude-resource-type internet-gateway \
-            --delete-unaliased-kms-keys --log-level info
+            --delete-unaliased-kms-keys --log-level info 2>&1 | tee /tmp/nuke.log
+          EXIT_CODE=${PIPESTATUS[0]}
+
+          DELETED=$(grep -c "^\s*INFO\s*\[Deleted\]" /tmp/nuke.log || echo "0")
+          ERRORS=$(grep -c "^\s*ERROR\s*\[Failed\]" /tmp/nuke.log || echo "0")
+          echo "deleted_count=${DELETED}" >> $GITHUB_OUTPUT
+          echo "error_count=${ERRORS}" >> $GITHUB_OUTPUT
+
+          exit $EXIT_CODE
+      - name: Upload results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: configtests-${{ matrix.region }}
+          path: /tmp/nuke.log
+          retention-days: 7
 
   configtests_notify:
     name: "ConfigTests: Notify"
@@ -262,12 +346,30 @@ jobs:
         with:
           role-to-assume: arn:aws:iam::087285199408:role/cloud-nuke-gha
           aws-region: us-east-1
-      - name: Send Slack notification
+      - name: Download all artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: configtests-*
+          path: /tmp/logs
+          merge-multiple: true
+        continue-on-error: true
+      - name: Aggregate and notify
         run: |
           WEBHOOK_URL=$(aws secretsmanager get-secret-value \
             --secret-id cloud-nuke/slack-webhook \
             --query SecretString --output text)
 
+          TOTAL_DELETED=0
+          TOTAL_ERRORS=0
+          if [ -d /tmp/logs ]; then
+            TOTAL_DELETED=$(grep -rh "^\s*INFO\s*\[Deleted\]" /tmp/logs 2>/dev/null | wc -l || echo "0")
+            TOTAL_ERRORS=$(grep -rh "^\s*ERROR\s*\[Failed\]" /tmp/logs 2>/dev/null | wc -l || echo "0")
+          fi
+          GLOBAL_DELETED="${{ needs.configtests_global.outputs.deleted_count }}"
+          GLOBAL_ERRORS="${{ needs.configtests_global.outputs.error_count }}"
+          TOTAL_DELETED=$((TOTAL_DELETED + ${GLOBAL_DELETED:-0}))
+          TOTAL_ERRORS=$((TOTAL_ERRORS + ${GLOBAL_ERRORS:-0}))
+
           if [ "${{ needs.configtests_global.result }}" == "success" ] && \
              [ "${{ needs.configtests_regional.result }}" == "success" ]; then
             STATUS="✅ Success"
@@ -277,6 +379,13 @@ jobs:
             COLOR="danger"
           fi
 
+          MSG="*ConfigTests Nuke*: ${STATUS}"
+          MSG="${MSG}\nDeleted: ${TOTAL_DELETED} resources"
+          if [ "$TOTAL_ERRORS" -gt 0 ]; then
+            MSG="${MSG} | Errors: ${TOTAL_ERRORS}"
+          fi
+          MSG="${MSG}\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Run>"
+
           curl -sS -X POST "$WEBHOOK_URL" -H "Content-Type: application/json" -d @- <<EOF
           {
             "attachments": [{
@@ -285,7 +394,7 @@ jobs:
                 "type": "section",
                 "text": {
                   "type": "mrkdwn",
-                  "text": "*ConfigTests Nuke*: ${STATUS}\nGlobal: ${{ needs.configtests_global.result }} | Regional: ${{ needs.configtests_regional.result }}\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Run>"
+                  "text": "${MSG}"
                 }
               }]
             }]
@@ -299,6 +408,9 @@ jobs:
     name: "Sandbox: Global"
     runs-on: ubuntu-latest
     timeout-minutes: 30
+    outputs:
+      deleted_count: ${{ steps.nuke.outputs.deleted_count }}
+      error_count: ${{ steps.nuke.outputs.error_count }}
     if: |
       github.event_name == 'workflow_dispatch' && (inputs.account == 'sandbox' || inputs.account == '') ||
       github.event_name == 'schedule' && github.event.schedule == '0 0 * * *'
@@ -323,12 +435,22 @@ jobs:
           restore-keys: ${{ runner.os }}-go-
       - run: go mod download
       - name: Nuke global resources
+        id: nuke
         run: |
+          set +e
           go run -ldflags="-X 'main.VERSION=${{ github.sha }}'" main.go aws \
             --older-than 24h --force --config ./.github/nuke_config.yml \
             --region global ${{ env.COMMON_EXCLUDES }} \
             --exclude-resource-type eip \
-            --delete-unaliased-kms-keys --log-level info
+            --delete-unaliased-kms-keys --log-level info 2>&1 | tee /tmp/nuke.log
+          EXIT_CODE=${PIPESTATUS[0]}
+
+          DELETED=$(grep -c "^\s*INFO\s*\[Deleted\]" /tmp/nuke.log || echo "0")
+          ERRORS=$(grep -c "^\s*ERROR\s*\[Failed\]" /tmp/nuke.log || echo "0")
+          echo "deleted_count=${DELETED}" >> $GITHUB_OUTPUT
+          echo "error_count=${ERRORS}" >> $GITHUB_OUTPUT
+
+          exit $EXIT_CODE
 
   sandbox_regional:
     name: "Sandbox: ${{ matrix.region }}"
@@ -362,12 +484,29 @@ jobs:
           restore-keys: ${{ runner.os }}-go-
       - run: go mod download
       - name: Nuke ${{ matrix.region }}
+        id: nuke
         run: |
+          set +e
           go run -ldflags="-X 'main.VERSION=${{ github.sha }}'" main.go aws \
             --older-than 24h --force --config ./.github/nuke_config.yml \
             --region ${{ matrix.region }} ${{ env.COMMON_EXCLUDES }} \
             --exclude-resource-type eip \
-            --delete-unaliased-kms-keys --log-level info
+            --delete-unaliased-kms-keys --log-level info 2>&1 | tee /tmp/nuke.log
+          EXIT_CODE=${PIPESTATUS[0]}
+
+          DELETED=$(grep -c "^\s*INFO\s*\[Deleted\]" /tmp/nuke.log || echo "0")
+          ERRORS=$(grep -c "^\s*ERROR\s*\[Failed\]" /tmp/nuke.log || echo "0")
+          echo "deleted_count=${DELETED}" >> $GITHUB_OUTPUT
+          echo "error_count=${ERRORS}" >> $GITHUB_OUTPUT
+
+          exit $EXIT_CODE
+      - name: Upload results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: sandbox-${{ matrix.region }}
+          path: /tmp/nuke.log
+          retention-days: 7
 
   sandbox_notify:
     name: "Sandbox: Notify"
@@ -383,12 +522,30 @@ jobs:
         with:
           role-to-assume: arn:aws:iam::087285199408:role/cloud-nuke-gha
           aws-region: us-east-1
-      - name: Send Slack notification
+      - name: Download all artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: sandbox-*
+          path: /tmp/logs
+          merge-multiple: true
+        continue-on-error: true
+      - name: Aggregate and notify
         run: |
           WEBHOOK_URL=$(aws secretsmanager get-secret-value \
             --secret-id cloud-nuke/slack-webhook \
             --query SecretString --output text)
 
+          TOTAL_DELETED=0
+          TOTAL_ERRORS=0
+          if [ -d /tmp/logs ]; then
+            TOTAL_DELETED=$(grep -rh "^\s*INFO\s*\[Deleted\]" /tmp/logs 2>/dev/null | wc -l || echo "0")
+            TOTAL_ERRORS=$(grep -rh "^\s*ERROR\s*\[Failed\]" /tmp/logs 2>/dev/null | wc -l || echo "0")
+          fi
+          GLOBAL_DELETED="${{ needs.sandbox_global.outputs.deleted_count }}"
+          GLOBAL_ERRORS="${{ needs.sandbox_global.outputs.error_count }}"
+          TOTAL_DELETED=$((TOTAL_DELETED + ${GLOBAL_DELETED:-0}))
+          TOTAL_ERRORS=$((TOTAL_ERRORS + ${GLOBAL_ERRORS:-0}))
+
           if [ "${{ needs.sandbox_global.result }}" == "success" ] && \
              [ "${{ needs.sandbox_regional.result }}" == "success" ]; then
             STATUS="✅ Success"
@@ -398,6 +555,13 @@ jobs:
             COLOR="danger"
           fi
 
+          MSG="*Sandbox Nuke*: ${STATUS}"
+          MSG="${MSG}\nDeleted: ${TOTAL_DELETED} resources"
+          if [ "$TOTAL_ERRORS" -gt 0 ]; then
+            MSG="${MSG} | Errors: ${TOTAL_ERRORS}"
+          fi
+          MSG="${MSG}\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Run>"
+
           curl -sS -X POST "$WEBHOOK_URL" -H "Content-Type: application/json" -d @- <<EOF
           {
             "attachments": [{
@@ -406,7 +570,7 @@ jobs:
                 "type": "section",
                 "text": {
                   "type": "mrkdwn",
-                  "text": "*Sandbox Nuke*: ${STATUS}\nGlobal: ${{ needs.sandbox_global.result }} | Regional: ${{ needs.sandbox_regional.result }}\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Run>"
+                  "text": "${MSG}"
                 }
               }]
             }]
