Skip to content

Commit 276fbe6

Browse files
cka-ycka-y
authored
feat: notify slack on gh action workflows failure (#810)
1 parent 8208e0a commit 276fbe6

File tree

9 files changed

+168
-25
lines changed

9 files changed

+168
-25
lines changed

‎.github/actions/notify-slack/action.yml‎

Lines changed: 52 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,18 @@
11
name: 'Notify Slack'
2-
description: 'Send a notification to a Slack channel with an error message'
2+
description: 'Send a notification to a Slack channel with an alert message based on priority level'
33
inputs:
44
OP_SERVICE_ACCOUNT_TOKEN:
55
description: 'The service account token for 1Password'
66
required: true
77
type: string
8+
PRIORITY:
9+
description: 'Priority level of the alert (medium/high)'
10+
required: true
11+
type: string
12+
STEP:
13+
description: 'The step in which the issue occurred (optional)'
14+
required: false
15+
type: string
816

917
runs:
1018
using: 'composite'
@@ -19,30 +27,52 @@ runs:
1927
CREDENTIALS: "op://rbiv7rvkkrsdlpcrz3bmv7nmcu/ifkeehu5gzi7wy5ub5qvwkaire/credential"
2028
SLACK_WEBHOOK_URL: "op://rbiv7rvkkrsdlpcrz3bmv7nmcu/Slack webhook URLs/internal-github-action-alerts/internal-github-action-alerts"
2129

30+
- name: Set Alert Message
31+
id: alert_message
32+
shell: bash
33+
run: |
34+
if [[ "${{ inputs.PRIORITY }}" == "high" ]]; then
35+
if [[ -n "${{ inputs.STEP }}" ]]; then
36+
message="🚨 High Priority Alert: [${{ github.workflow }}] failed at step \"${{ inputs.STEP }}\". Immediate attention is required to avoid production impact."
37+
else
38+
message="🚨 High Priority Alert: [${{ github.workflow }}] failed. Immediate attention is required to avoid production impact."
39+
fi
40+
else
41+
if [[ -n "${{ inputs.STEP }}" ]]; then
42+
message="🚧 Medium Priority Alert: [${{ github.workflow }}] encountered an issue at step \"${{ inputs.STEP }}\". This may affect ongoing integration processes."
43+
else
44+
message="🚧 Medium Priority Alert: [${{ github.workflow }}] encountered an issue. This may affect ongoing integration processes."
45+
fi
46+
fi
47+
48+
# Construct the JSON payload and save it to a file
49+
jq -n --arg message "$message" --arg run_url "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" '{
50+
"channel": "C074SPVCH6H",
51+
"blocks": [
52+
{
53+
"type": "header",
54+
"text": {
55+
"type": "plain_text",
56+
"text": $message,
57+
"emoji": true
58+
}
59+
},
60+
{
61+
"type": "section",
62+
"text": {
63+
"type": "mrkdwn",
64+
"text": "See <\($run_url)|GitHub Action Run Details>"
65+
}
66+
}
67+
]
68+
}' > slack_payload.json
69+
2270
- name: Post notification in Slack channel
2371
uses: slackapi/[email protected]
2472
with:
25-
payload: |
26-
{
27-
"channel": "C074SPVCH6H",
28-
"blocks": [
29-
{
30-
"type": "header",
31-
"text": {
32-
"type": "plain_text",
33-
"text": ":x: ERROR running Github action \"${{ github.workflow }}\"",
34-
"emoji": true
35-
}
36-
},
37-
{
38-
"type": "section",
39-
"text": {
40-
"type": "mrkdwn",
41-
"text": "See https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
42-
}
43-
}
44-
]
45-
}
73+
payload-file-path: ./slack_payload.json
4674
env:
4775
SLACK_WEBHOOK_URL: ${{ env.SLACK_WEBHOOK_URL }}
4876
SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
77+
78+

‎.github/workflows/datasets-batch-deployer-dev.yml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,3 +21,15 @@ jobs:
2121
DATASETS_BUCKET_NAME: 'mobilitydata-datasets-dev'
2222
secrets:
2323
GCP_MOBILITY_FEEDS_SA_KEY: ${{ secrets.DEV_GCP_MOBILITY_FEEDS_SA_KEY }}
24+
notify-slack-on-failure:
25+
needs: [ deploy ]
26+
if: always() && (needs.deploy.result == 'failure') && (github.event_name != 'workflow_dispatch')
27+
runs-on: ubuntu-latest
28+
steps:
29+
- name: Checkout code
30+
uses: actions/checkout@v4
31+
- name: Notify Slack
32+
uses: ./.github/actions/notify-slack
33+
with:
34+
OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
35+
PRIORITY: "medium"

‎.github/workflows/db-update-dev.yml‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,4 +27,16 @@ jobs:
2727
DB_GCP_MOBILITY_FEEDS_SA_KEY: ${{ secrets.QA_GCP_MOBILITY_FEEDS_SA_KEY }}
2828
OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
2929
OP_FEEDS_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_FEEDS_SERVICE_ACCOUNT_TOKEN }}
30-
POSTGRE_SQL_INSTANCE_NAME: ${{ secrets.DB_INSTANCE_NAME }}
30+
POSTGRE_SQL_INSTANCE_NAME: ${{ secrets.DB_INSTANCE_NAME }}
31+
notify-slack-on-failure:
32+
needs: [ update ]
33+
if: always() && (needs.update.result == 'failure') && (github.event_name == 'repository_dispatch')
34+
runs-on: ubuntu-latest
35+
steps:
36+
- name: Checkout code
37+
uses: actions/checkout@v4
38+
- name: Notify Slack
39+
uses: ./.github/actions/notify-slack
40+
with:
41+
OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
42+
PRIORITY: "high"

‎.github/workflows/db-update-prod.yml‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,4 +22,17 @@ jobs:
2222
DB_GCP_MOBILITY_FEEDS_SA_KEY: ${{ secrets.PROD_GCP_MOBILITY_FEEDS_SA_KEY }}
2323
OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
2424
OP_FEEDS_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_FEEDS_SERVICE_ACCOUNT_TOKEN }}
25-
POSTGRE_SQL_INSTANCE_NAME: ${{ secrets.DB_INSTANCE_NAME }}
25+
POSTGRE_SQL_INSTANCE_NAME: ${{ secrets.DB_INSTANCE_NAME }}
26+
27+
notify-slack-on-failure:
28+
needs: [ update ]
29+
if: always() && (needs.update.result == 'failure') && (github.event_name == 'repository_dispatch')
30+
runs-on: ubuntu-latest
31+
steps:
32+
- name: Checkout code
33+
uses: actions/checkout@v4
34+
- name: Notify Slack
35+
uses: ./.github/actions/notify-slack
36+
with:
37+
OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
38+
PRIORITY: "high"

‎.github/workflows/db-update-qa.yml‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,4 +22,16 @@ jobs:
2222
DB_GCP_MOBILITY_FEEDS_SA_KEY: ${{ secrets.QA_GCP_MOBILITY_FEEDS_SA_KEY }}
2323
OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
2424
OP_FEEDS_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_FEEDS_SERVICE_ACCOUNT_TOKEN }}
25-
POSTGRE_SQL_INSTANCE_NAME: ${{ secrets.DB_INSTANCE_NAME }}
25+
POSTGRE_SQL_INSTANCE_NAME: ${{ secrets.DB_INSTANCE_NAME }}
26+
notify-slack-on-failure:
27+
needs: [ update ]
28+
if: always() && (needs.update.result == 'failure') && (github.event_name == 'repository_dispatch')
29+
runs-on: ubuntu-latest
30+
steps:
31+
- name: Checkout code
32+
uses: actions/checkout@v4
33+
- name: Notify Slack
34+
uses: ./.github/actions/notify-slack
35+
with:
36+
OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
37+
PRIORITY: "high"

‎.github/workflows/release-qa.yml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,3 +39,15 @@ jobs:
3939
needs: batch-deployment
4040
uses: ./.github/workflows/web-qa.yml
4141
secrets: inherit
42+
notify-slack-on-failure:
43+
needs: [ web-deployment, integration-tests ]
44+
if: failure() && (github.event_name != 'workflow_dispatch')
45+
runs-on: ubuntu-latest
46+
steps:
47+
- name: Checkout code
48+
uses: actions/checkout@v4
49+
- name: Notify Slack
50+
uses: ./.github/actions/notify-slack
51+
with:
52+
OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
53+
PRIORITY: "high"

‎.github/workflows/release.yml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,3 +37,15 @@ jobs:
3737
needs: batch-deployment
3838
uses: ./.github/workflows/web-prod.yml
3939
secrets: inherit
40+
notify-slack-on-failure:
41+
needs: [web-deployment, integration-tests]
42+
if: failure()
43+
runs-on: ubuntu-latest
44+
steps:
45+
- name: Checkout code
46+
uses: actions/checkout@v4
47+
- name: Notify Slack
48+
uses: ./.github/actions/notify-slack
49+
with:
50+
OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
51+
PRIORITY: "high"

‎.github/workflows/validator-update.yml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,3 +37,15 @@ jobs:
3737
--schedule-time=$(date -u -d "+24 hours" +%Y-%m-%dT%H:%M:%SZ) \
3838
--oidc-service-account-email=${DEPLOYER_SERVICE_ACCOUNT} \
3939
--location=${{ vars.MOBILITY_FEEDS_REGION }}
40+
notify-slack-on-failure:
41+
needs: [ validator-update ]
42+
if: failure() && (github.event_name != 'workflow_dispatch')
43+
runs-on: ubuntu-latest
44+
steps:
45+
- name: Checkout code
46+
uses: actions/checkout@v4
47+
- name: Notify Slack
48+
uses: ./.github/actions/notify-slack
49+
with:
50+
OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
51+
PRIORITY: "high"

0 commit comments

Comments
 (0)