Skip to content

Add Slack notifications with resource counts and error reporting #7

Add Slack notifications with resource counts and error reporting

Add Slack notifications with resource counts and error reporting #7

Workflow file for this run

name: Scheduled Nuke
on:
push:
branches:
- migrate-nuke-to-gha
schedule:
- cron: '0 */3 * * *' # Every 3 hours (phxdevops, configtests)
- cron: '0 0 * * *' # Nightly at midnight UTC (sandbox)
workflow_dispatch:
inputs:
account:
description: 'Target account (leave empty for scheduled jobs)'
required: false
type: choice
options:
- ''
- phxdevops
- configtests
- sandbox
permissions:
id-token: write
contents: read
env:
MISE_VERSION: '2025.12.10'
COMMON_EXCLUDES: >-
--exclude-resource-type iam
--exclude-resource-type iam-group
--exclude-resource-type iam-policy
--exclude-resource-type iam-role
--exclude-resource-type iam-service-linked-role
--exclude-resource-type oidcprovider
--exclude-resource-type route53-hosted-zone
--exclude-resource-type route53-cidr-collection
--exclude-resource-type route53-traffic-policy
--exclude-resource-type ecr
--exclude-resource-type config-rules
--exclude-resource-type nat-gateway
--exclude-resource-type ec2-subnet
jobs:
# ============================================
# PhxDevOps Account - Every 3 hours
# ============================================
phxdevops_global:
name: "PhxDevOps: Global"
runs-on: ubuntu-latest
timeout-minutes: 30
outputs:
deleted_count: ${{ steps.nuke.outputs.deleted_count }}
error_count: ${{ steps.nuke.outputs.error_count }}
if: |
github.event_name == 'push' ||
github.event_name == 'workflow_dispatch' && (inputs.account == 'phxdevops' || inputs.account == '') ||
github.event_name == 'schedule' && github.event.schedule == '0 */3 * * *'
steps:
- uses: actions/checkout@v4
- uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: arn:aws:iam::087285199408:role/cloud-nuke-gha
aws-region: us-east-1
- uses: jdx/mise-action@v3
with:
version: ${{ env.MISE_VERSION }}
experimental: true
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- uses: actions/cache@v4
with:
path: |
~/go/pkg/mod
~/.cache/go-build
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: ${{ runner.os }}-go-
- run: go mod download
- name: Nuke global resources
id: nuke
run: |
set +e
go run -ldflags="-X 'main.VERSION=${{ github.sha }}'" main.go aws \
--older-than 2h --force --config ./.github/nuke_config.yml \
--region global ${{ env.COMMON_EXCLUDES }} \
--delete-unaliased-kms-keys --log-level info 2>&1 | tee /tmp/nuke.log
EXIT_CODE=${PIPESTATUS[0]}
# Parse output for counts
DELETED=$(grep -c "^\s*INFO\s*\[Deleted\]" /tmp/nuke.log || echo "0")
ERRORS=$(grep -c "^\s*ERROR\s*\[Failed\]" /tmp/nuke.log || echo "0")
echo "deleted_count=${DELETED}" >> $GITHUB_OUTPUT
echo "error_count=${ERRORS}" >> $GITHUB_OUTPUT
exit $EXIT_CODE
phxdevops_regional:
name: "PhxDevOps: ${{ matrix.region }}"
runs-on: ubuntu-latest
timeout-minutes: 20
outputs:
deleted_count: ${{ steps.nuke.outputs.deleted_count }}
error_count: ${{ steps.nuke.outputs.error_count }}
if: |
github.event_name == 'push' ||
github.event_name == 'workflow_dispatch' && (inputs.account == 'phxdevops' || inputs.account == '') ||
github.event_name == 'schedule' && github.event.schedule == '0 */3 * * *'
strategy:
fail-fast: false
matrix:
region: [ap-northeast-1, ap-northeast-2, ap-northeast-3, ap-south-1, ap-southeast-1, ap-southeast-2, ca-central-1, eu-central-1, eu-north-1, eu-west-1, eu-west-2, eu-west-3, me-central-1, sa-east-1, us-east-1, us-east-2, us-west-1, us-west-2]
steps:
- uses: actions/checkout@v4
- uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: arn:aws:iam::087285199408:role/cloud-nuke-gha
aws-region: ${{ matrix.region }}
- uses: jdx/mise-action@v3
with:
version: ${{ env.MISE_VERSION }}
experimental: true
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- uses: actions/cache@v4
with:
path: |
~/go/pkg/mod
~/.cache/go-build
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: ${{ runner.os }}-go-
- run: go mod download
- name: Nuke ${{ matrix.region }}
id: nuke
run: |
set +e
go run -ldflags="-X 'main.VERSION=${{ github.sha }}'" main.go aws \
--older-than 2h --force --config ./.github/nuke_config.yml \
--region ${{ matrix.region }} ${{ env.COMMON_EXCLUDES }} \
--delete-unaliased-kms-keys --log-level info 2>&1 | tee /tmp/nuke.log
EXIT_CODE=${PIPESTATUS[0]}
DELETED=$(grep -c "^\s*INFO\s*\[Deleted\]" /tmp/nuke.log || echo "0")
ERRORS=$(grep -c "^\s*ERROR\s*\[Failed\]" /tmp/nuke.log || echo "0")
echo "deleted_count=${DELETED}" >> $GITHUB_OUTPUT
echo "error_count=${ERRORS}" >> $GITHUB_OUTPUT
exit $EXIT_CODE
- name: Upload results
if: always()
uses: actions/upload-artifact@v4
with:
name: phxdevops-${{ matrix.region }}
path: /tmp/nuke.log
retention-days: 7
phxdevops_notify:
name: "PhxDevOps: Notify"
runs-on: ubuntu-latest
if: |
always() && (
github.event_name == 'push' ||
github.event_name == 'workflow_dispatch' && (inputs.account == 'phxdevops' || inputs.account == '') ||
github.event_name == 'schedule' && github.event.schedule == '0 */3 * * *'
)
needs: [phxdevops_global, phxdevops_regional]
steps:
- uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: arn:aws:iam::087285199408:role/cloud-nuke-gha
aws-region: us-east-1
- name: Download all artifacts
uses: actions/download-artifact@v4
with:
pattern: phxdevops-*
path: /tmp/logs
merge-multiple: true
continue-on-error: true
- name: Aggregate and notify
run: |
WEBHOOK_URL=$(aws secretsmanager get-secret-value \
--secret-id cloud-nuke/slack-webhook \
--query SecretString --output text)
# Aggregate counts from all logs
TOTAL_DELETED=0
TOTAL_ERRORS=0
if [ -d /tmp/logs ]; then
TOTAL_DELETED=$(grep -rh "^\s*INFO\s*\[Deleted\]" /tmp/logs 2>/dev/null | wc -l || echo "0")
TOTAL_ERRORS=$(grep -rh "^\s*ERROR\s*\[Failed\]" /tmp/logs 2>/dev/null | wc -l || echo "0")
fi
# Add global counts
GLOBAL_DELETED="${{ needs.phxdevops_global.outputs.deleted_count }}"
GLOBAL_ERRORS="${{ needs.phxdevops_global.outputs.error_count }}"
TOTAL_DELETED=$((TOTAL_DELETED + ${GLOBAL_DELETED:-0}))
TOTAL_ERRORS=$((TOTAL_ERRORS + ${GLOBAL_ERRORS:-0}))
if [ "${{ needs.phxdevops_global.result }}" == "success" ] && \
[ "${{ needs.phxdevops_regional.result }}" == "success" ]; then
STATUS="✅ Success"
COLOR="good"
else
STATUS="❌ Failed"
COLOR="danger"
fi
# Build message
MSG="*PhxDevOps Nuke*: ${STATUS}"
MSG="${MSG}\nDeleted: ${TOTAL_DELETED} resources"
if [ "$TOTAL_ERRORS" -gt 0 ]; then
MSG="${MSG} | Errors: ${TOTAL_ERRORS}"
fi
MSG="${MSG}\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Run>"
curl -sS -X POST "$WEBHOOK_URL" -H "Content-Type: application/json" -d @- <<EOF
{
"attachments": [{
"color": "${COLOR}",
"blocks": [{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "${MSG}"
}
}]
}]
}
EOF
# ============================================
# ConfigTests Account - Every 3 hours
# ============================================
configtests_global:
name: "ConfigTests: Global"
runs-on: ubuntu-latest
timeout-minutes: 30
outputs:
deleted_count: ${{ steps.nuke.outputs.deleted_count }}
error_count: ${{ steps.nuke.outputs.error_count }}
if: |
github.event_name == 'workflow_dispatch' && (inputs.account == 'configtests' || inputs.account == '') ||
github.event_name == 'schedule' && github.event.schedule == '0 */3 * * *'
steps:
- uses: actions/checkout@v4
- uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: ${{ secrets.CONFIGTESTS_ROLE_ARN }}
aws-region: us-east-1
- uses: jdx/mise-action@v3
with:
version: ${{ env.MISE_VERSION }}
experimental: true
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- uses: actions/cache@v4
with:
path: |
~/go/pkg/mod
~/.cache/go-build
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: ${{ runner.os }}-go-
- run: go mod download
- name: Nuke global resources
id: nuke
run: |
set +e
go run -ldflags="-X 'main.VERSION=${{ github.sha }}'" main.go aws \
--older-than 2h --force --config ./.github/nuke_config.yml \
--region global ${{ env.COMMON_EXCLUDES }} \
--exclude-resource-type internet-gateway \
--delete-unaliased-kms-keys --log-level info 2>&1 | tee /tmp/nuke.log
EXIT_CODE=${PIPESTATUS[0]}
DELETED=$(grep -c "^\s*INFO\s*\[Deleted\]" /tmp/nuke.log || echo "0")
ERRORS=$(grep -c "^\s*ERROR\s*\[Failed\]" /tmp/nuke.log || echo "0")
echo "deleted_count=${DELETED}" >> $GITHUB_OUTPUT
echo "error_count=${ERRORS}" >> $GITHUB_OUTPUT
exit $EXIT_CODE
configtests_regional:
name: "ConfigTests: ${{ matrix.region }}"
runs-on: ubuntu-latest
timeout-minutes: 20
if: |
github.event_name == 'workflow_dispatch' && (inputs.account == 'configtests' || inputs.account == '') ||
github.event_name == 'schedule' && github.event.schedule == '0 */3 * * *'
strategy:
fail-fast: false
matrix:
region: [ap-northeast-1, ap-northeast-2, ap-northeast-3, ap-south-1, ap-southeast-1, ap-southeast-2, ca-central-1, eu-central-1, eu-north-1, eu-west-1, eu-west-2, eu-west-3, me-central-1, sa-east-1, us-east-1, us-east-2, us-west-1, us-west-2]
steps:
- uses: actions/checkout@v4
- uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: ${{ secrets.CONFIGTESTS_ROLE_ARN }}
aws-region: ${{ matrix.region }}
- uses: jdx/mise-action@v3
with:
version: ${{ env.MISE_VERSION }}
experimental: true
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- uses: actions/cache@v4
with:
path: |
~/go/pkg/mod
~/.cache/go-build
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: ${{ runner.os }}-go-
- run: go mod download
- name: Nuke ${{ matrix.region }}
id: nuke
run: |
set +e
go run -ldflags="-X 'main.VERSION=${{ github.sha }}'" main.go aws \
--older-than 2h --force --config ./.github/nuke_config.yml \
--region ${{ matrix.region }} ${{ env.COMMON_EXCLUDES }} \
--exclude-resource-type internet-gateway \
--delete-unaliased-kms-keys --log-level info 2>&1 | tee /tmp/nuke.log
EXIT_CODE=${PIPESTATUS[0]}
DELETED=$(grep -c "^\s*INFO\s*\[Deleted\]" /tmp/nuke.log || echo "0")
ERRORS=$(grep -c "^\s*ERROR\s*\[Failed\]" /tmp/nuke.log || echo "0")
echo "deleted_count=${DELETED}" >> $GITHUB_OUTPUT
echo "error_count=${ERRORS}" >> $GITHUB_OUTPUT
exit $EXIT_CODE
- name: Upload results
if: always()
uses: actions/upload-artifact@v4
with:
name: configtests-${{ matrix.region }}
path: /tmp/nuke.log
retention-days: 7
configtests_notify:
name: "ConfigTests: Notify"
runs-on: ubuntu-latest
if: |
always() && (
github.event_name == 'workflow_dispatch' && (inputs.account == 'configtests' || inputs.account == '') ||
github.event_name == 'schedule' && github.event.schedule == '0 */3 * * *'
)
needs: [configtests_global, configtests_regional]
steps:
- uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: arn:aws:iam::087285199408:role/cloud-nuke-gha
aws-region: us-east-1
- name: Download all artifacts
uses: actions/download-artifact@v4
with:
pattern: configtests-*
path: /tmp/logs
merge-multiple: true
continue-on-error: true
- name: Aggregate and notify
run: |
WEBHOOK_URL=$(aws secretsmanager get-secret-value \
--secret-id cloud-nuke/slack-webhook \
--query SecretString --output text)
TOTAL_DELETED=0
TOTAL_ERRORS=0
if [ -d /tmp/logs ]; then
TOTAL_DELETED=$(grep -rh "^\s*INFO\s*\[Deleted\]" /tmp/logs 2>/dev/null | wc -l || echo "0")
TOTAL_ERRORS=$(grep -rh "^\s*ERROR\s*\[Failed\]" /tmp/logs 2>/dev/null | wc -l || echo "0")
fi
GLOBAL_DELETED="${{ needs.configtests_global.outputs.deleted_count }}"
GLOBAL_ERRORS="${{ needs.configtests_global.outputs.error_count }}"
TOTAL_DELETED=$((TOTAL_DELETED + ${GLOBAL_DELETED:-0}))
TOTAL_ERRORS=$((TOTAL_ERRORS + ${GLOBAL_ERRORS:-0}))
if [ "${{ needs.configtests_global.result }}" == "success" ] && \
[ "${{ needs.configtests_regional.result }}" == "success" ]; then
STATUS="✅ Success"
COLOR="good"
else
STATUS="❌ Failed"
COLOR="danger"
fi
MSG="*ConfigTests Nuke*: ${STATUS}"
MSG="${MSG}\nDeleted: ${TOTAL_DELETED} resources"
if [ "$TOTAL_ERRORS" -gt 0 ]; then
MSG="${MSG} | Errors: ${TOTAL_ERRORS}"
fi
MSG="${MSG}\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Run>"
curl -sS -X POST "$WEBHOOK_URL" -H "Content-Type: application/json" -d @- <<EOF
{
"attachments": [{
"color": "${COLOR}",
"blocks": [{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "${MSG}"
}
}]
}]
}
EOF
# ============================================
# Sandbox Account - Nightly
# ============================================
sandbox_global:
name: "Sandbox: Global"
runs-on: ubuntu-latest
timeout-minutes: 30
outputs:
deleted_count: ${{ steps.nuke.outputs.deleted_count }}
error_count: ${{ steps.nuke.outputs.error_count }}
if: |
github.event_name == 'workflow_dispatch' && (inputs.account == 'sandbox' || inputs.account == '') ||
github.event_name == 'schedule' && github.event.schedule == '0 0 * * *'
steps:
- uses: actions/checkout@v4
- uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: ${{ secrets.SANDBOX_ROLE_ARN }}
aws-region: us-east-1
- uses: jdx/mise-action@v3
with:
version: ${{ env.MISE_VERSION }}
experimental: true
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- uses: actions/cache@v4
with:
path: |
~/go/pkg/mod
~/.cache/go-build
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: ${{ runner.os }}-go-
- run: go mod download
- name: Nuke global resources
id: nuke
run: |
set +e
go run -ldflags="-X 'main.VERSION=${{ github.sha }}'" main.go aws \
--older-than 24h --force --config ./.github/nuke_config.yml \
--region global ${{ env.COMMON_EXCLUDES }} \
--exclude-resource-type eip \
--delete-unaliased-kms-keys --log-level info 2>&1 | tee /tmp/nuke.log
EXIT_CODE=${PIPESTATUS[0]}
DELETED=$(grep -c "^\s*INFO\s*\[Deleted\]" /tmp/nuke.log || echo "0")
ERRORS=$(grep -c "^\s*ERROR\s*\[Failed\]" /tmp/nuke.log || echo "0")
echo "deleted_count=${DELETED}" >> $GITHUB_OUTPUT
echo "error_count=${ERRORS}" >> $GITHUB_OUTPUT
exit $EXIT_CODE
sandbox_regional:
name: "Sandbox: ${{ matrix.region }}"
runs-on: ubuntu-latest
timeout-minutes: 20
if: |
github.event_name == 'workflow_dispatch' && (inputs.account == 'sandbox' || inputs.account == '') ||
github.event_name == 'schedule' && github.event.schedule == '0 0 * * *'
strategy:
fail-fast: false
matrix:
region: [ap-northeast-1, ap-northeast-2, ap-northeast-3, ap-south-1, ap-southeast-1, ap-southeast-2, ca-central-1, eu-central-1, eu-north-1, eu-west-1, eu-west-2, eu-west-3, me-central-1, sa-east-1, us-east-1, us-east-2, us-west-1, us-west-2]
steps:
- uses: actions/checkout@v4
- uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: ${{ secrets.SANDBOX_ROLE_ARN }}
aws-region: ${{ matrix.region }}
- uses: jdx/mise-action@v3
with:
version: ${{ env.MISE_VERSION }}
experimental: true
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- uses: actions/cache@v4
with:
path: |
~/go/pkg/mod
~/.cache/go-build
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: ${{ runner.os }}-go-
- run: go mod download
- name: Nuke ${{ matrix.region }}
id: nuke
run: |
set +e
go run -ldflags="-X 'main.VERSION=${{ github.sha }}'" main.go aws \
--older-than 24h --force --config ./.github/nuke_config.yml \
--region ${{ matrix.region }} ${{ env.COMMON_EXCLUDES }} \
--exclude-resource-type eip \
--delete-unaliased-kms-keys --log-level info 2>&1 | tee /tmp/nuke.log
EXIT_CODE=${PIPESTATUS[0]}
DELETED=$(grep -c "^\s*INFO\s*\[Deleted\]" /tmp/nuke.log || echo "0")
ERRORS=$(grep -c "^\s*ERROR\s*\[Failed\]" /tmp/nuke.log || echo "0")
echo "deleted_count=${DELETED}" >> $GITHUB_OUTPUT
echo "error_count=${ERRORS}" >> $GITHUB_OUTPUT
exit $EXIT_CODE
- name: Upload results
if: always()
uses: actions/upload-artifact@v4
with:
name: sandbox-${{ matrix.region }}
path: /tmp/nuke.log
retention-days: 7
sandbox_notify:
name: "Sandbox: Notify"
runs-on: ubuntu-latest
if: |
always() && (
github.event_name == 'workflow_dispatch' && (inputs.account == 'sandbox' || inputs.account == '') ||
github.event_name == 'schedule' && github.event.schedule == '0 0 * * *'
)
needs: [sandbox_global, sandbox_regional]
steps:
- uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: arn:aws:iam::087285199408:role/cloud-nuke-gha
aws-region: us-east-1
- name: Download all artifacts
uses: actions/download-artifact@v4
with:
pattern: sandbox-*
path: /tmp/logs
merge-multiple: true
continue-on-error: true
- name: Aggregate and notify
run: |
WEBHOOK_URL=$(aws secretsmanager get-secret-value \
--secret-id cloud-nuke/slack-webhook \
--query SecretString --output text)
TOTAL_DELETED=0
TOTAL_ERRORS=0
if [ -d /tmp/logs ]; then
TOTAL_DELETED=$(grep -rh "^\s*INFO\s*\[Deleted\]" /tmp/logs 2>/dev/null | wc -l || echo "0")
TOTAL_ERRORS=$(grep -rh "^\s*ERROR\s*\[Failed\]" /tmp/logs 2>/dev/null | wc -l || echo "0")
fi
GLOBAL_DELETED="${{ needs.sandbox_global.outputs.deleted_count }}"
GLOBAL_ERRORS="${{ needs.sandbox_global.outputs.error_count }}"
TOTAL_DELETED=$((TOTAL_DELETED + ${GLOBAL_DELETED:-0}))
TOTAL_ERRORS=$((TOTAL_ERRORS + ${GLOBAL_ERRORS:-0}))
if [ "${{ needs.sandbox_global.result }}" == "success" ] && \
[ "${{ needs.sandbox_regional.result }}" == "success" ]; then
STATUS="✅ Success"
COLOR="good"
else
STATUS="❌ Failed"
COLOR="danger"
fi
MSG="*Sandbox Nuke*: ${STATUS}"
MSG="${MSG}\nDeleted: ${TOTAL_DELETED} resources"
if [ "$TOTAL_ERRORS" -gt 0 ]; then
MSG="${MSG} | Errors: ${TOTAL_ERRORS}"
fi
MSG="${MSG}\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Run>"
curl -sS -X POST "$WEBHOOK_URL" -H "Content-Type: application/json" -d @- <<EOF
{
"attachments": [{
"color": "${COLOR}",
"blocks": [{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "${MSG}"
}
}]
}]
}
EOF