Skip to content

Commit b7ca0ab

Browse files
megan-bower4megan-bower4
authored
[NDR-313] Add slack notification for workflow failures (#880)
1 parent dd04967 commit b7ca0ab

File tree

7 files changed

+320
-8
lines changed

7 files changed

+320
-8
lines changed

‎.github/workflows/base-deploy-ui.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ jobs:
4949
aws-region: ${{ vars.AWS_REGION }}
5050
mask-aws-account-id: true
5151

52-
- name: Get SSM parameters from SSM
52+
- name: Get SSM parameters
5353
id: ssm-parameters
5454
run: |
5555
doc_store_api_endpoint=$(aws ssm get-parameter --name "/ndr/${{ inputs.sandbox }}/api_endpoint" --with-decryption --query "Parameter.Value" --output text)

‎.github/workflows/base-e2e-fhir-backendtest.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,9 @@ jobs:
2929
e2e-test:
3030
runs-on: ubuntu-latest
3131
environment: ${{ inputs.environment }}
32+
outputs:
33+
environment: ${{ inputs.environment }}
34+
sandbox: ${{ inputs.sandbox }}
3235
steps:
3336
- name: Checkout
3437
uses: actions/checkout@v5

‎.github/workflows/lambdas-dev-to-main-ci.yml‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,3 +78,91 @@ jobs:
7878
sandbox: ndr-dev
7979
secrets:
8080
AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }}
81+
82+
notify-slack:
83+
runs-on: ubuntu-latest
84+
needs: [check_packages, run_tests, publish_all_lambda_layers, deploy_all_lambdas, deploy_data_collection]
85+
if: failure() && github.event_name == 'push' && github.ref == 'refs/heads/main'
86+
steps:
87+
- name: Configure AWS Credentials
88+
uses: aws-actions/configure-aws-credentials@v5
89+
with:
90+
role-to-assume: ${{ secrets.AWS_ASSUME_ROLE }}
91+
role-skip-session-tagging: true
92+
aws-region: ${{ vars.AWS_REGION }}
93+
mask-aws-account-id: true
94+
95+
- name: Get slack bot token from SSM parameter store
96+
id: ssm-parameters
97+
run: |
98+
slack_bot_token=$(aws ssm get-parameter --name "/ndr/alerting/slack/bot_token" --with-decryption --query "Parameter.Value" --output text)
99+
echo "::add-mask::$slack_bot_token"
100+
echo "SLACK_BOT_TOKEN=$slack_bot_token" >> $GITHUB_ENV
101+
102+
- name: Send Slack Notification
103+
uses: slackapi/slack-github-action@v2.1.1
104+
with:
105+
method: chat.postMessage
106+
token: ${{ env.SLACK_BOT_TOKEN }}
107+
payload: |
108+
{
109+
"channel": "${{ vars.ALERTS_SLACK_CHANNEL_ID }}",
110+
"attachments": [
111+
{
112+
"color": "#ff0000",
113+
"blocks": [
114+
{
115+
"type": "header",
116+
"text": {
117+
"type": "plain_text",
118+
"text": "❌ Workflow `${{ github.workflow }}` failed"
119+
}
120+
},
121+
{
122+
"type": "section",
123+
"text": {
124+
"type": "mrkdwn",
125+
"text": "*Triggered by:* `${{ github.actor }}`\n*Workflow:* <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.workflow }}>"
126+
}
127+
},
128+
{
129+
"type": "divider"
130+
},
131+
{
132+
"type": "section",
133+
"fields": [
134+
{
135+
"type": "mrkdwn",
136+
"text": "*check_packages:* ${{ needs.check_packages.result == 'success' && ':white_check_mark:' || ':x:' }}"
137+
},
138+
{
139+
"type": "mrkdwn",
140+
"text": "*run_tests:* ${{ needs.run_tests.result == 'success' && ':white_check_mark:' || ':x:' }}"
141+
},
142+
{
143+
"type": "mrkdwn",
144+
"text": "*publish_all_lambda_layers:* ${{ needs.publish_all_lambda_layers.result == 'success' && ':white_check_mark:' || ':x:' }}"
145+
},
146+
{
147+
"type": "mrkdwn",
148+
"text": "*deploy_all_lambdas:* ${{ needs.deploy_all_lambdas.result == 'success' && ':white_check_mark:' || ':x:' }}"
149+
},
150+
{
151+
"type": "mrkdwn",
152+
"text": "*deploy_data_collection:* ${{ needs.deploy_data_collection.result == 'success' && ':white_check_mark:' || ':x:' }}"
153+
}
154+
]
155+
},
156+
{
157+
"type": "context",
158+
"elements": [
159+
{
160+
"type": "mrkdwn",
161+
"text": "Environment: `development` | Sandbox: `ndr-dev`"
162+
}
163+
]
164+
}
165+
]
166+
}
167+
]
168+
}

‎.github/workflows/ndr-e2e-backend-test-ci.yml‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ jobs:
2424
sandbox: ndr-dev
2525
secrets:
2626
AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }}
27+
2728
services-fhir-api-e2etest:
2829
uses: ./.github/workflows/base-e2e-fhir-backendtest.yml
2930
with:
@@ -32,3 +33,79 @@ jobs:
3233
sandbox: ndr-dev
3334
secrets:
3435
AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }}
36+
37+
notify-slack:
38+
runs-on: ubuntu-latest
39+
needs: [services-e2etest, services-fhir-api-e2etest]
40+
if: failure()
41+
steps:
42+
- name: Configure AWS Credentials
43+
uses: aws-actions/configure-aws-credentials@v5
44+
with:
45+
role-to-assume: ${{ secrets.AWS_ASSUME_ROLE }}
46+
role-skip-session-tagging: true
47+
aws-region: ${{ vars.AWS_REGION }}
48+
mask-aws-account-id: true
49+
50+
- name: Get slack bot token from SSM parameter store
51+
id: ssm-parameters
52+
run: |
53+
slack_bot_token=$(aws ssm get-parameter --name "/ndr/alerting/slack/bot_token" --with-decryption --query "Parameter.Value" --output text)
54+
echo "::add-mask::$slack_bot_token"
55+
echo "SLACK_BOT_TOKEN=$slack_bot_token" >> $GITHUB_ENV
56+
57+
- name: Send Slack Notification
58+
uses: slackapi/slack-github-action@v2.1.1
59+
with:
60+
method: chat.postMessage
61+
token: ${{ env.SLACK_BOT_TOKEN }}
62+
payload: |
63+
{
64+
"channel": "${{ vars.ALERTS_SLACK_CHANNEL_ID }}",
65+
"attachments": [
66+
{
67+
"color": "#ff0000",
68+
"blocks": [
69+
{
70+
"type": "header",
71+
"text": {
72+
"type": "plain_text",
73+
"text": "❌ Workflow `${{ github.workflow }}` failed"
74+
}
75+
},
76+
{
77+
"type": "section",
78+
"text": {
79+
"type": "mrkdwn",
80+
"text": "*Triggered by:* `${{ github.actor }}`\n*Workflow:* <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.workflow }}>"
81+
}
82+
},
83+
{
84+
"type": "divider"
85+
},
86+
{
87+
"type": "section",
88+
"fields": [
89+
{
90+
"type": "mrkdwn",
91+
"text": "*services-e2etest:* ${{ needs.services-e2etest.result == 'success' && ':white_check_mark:' || ':x:' }}"
92+
},
93+
{
94+
"type": "mrkdwn",
95+
"text": "*services-fhir-api-e2etest:* ${{ needs.services-fhir-api-e2etest.result == 'success' && ':white_check_mark:' || ':x:' }}"
96+
}
97+
]
98+
},
99+
{
100+
"type": "context",
101+
"elements": [
102+
{
103+
"type": "mrkdwn",
104+
"text": "Environment: `${{ needs.services-fhir-api-e2etest.outputs.environment }}` | Sandbox: `${{ needs.services-fhir-api-e2etest.outputs.sandbox }}`"
105+
}
106+
]
107+
}
108+
]
109+
}
110+
]
111+
}

‎.github/workflows/ndr-e2e-test-sandbox.yml‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,11 @@ on:
1212
description: "Sandbox environment"
1313
required: true
1414
default: "ndr"
15+
environment:
16+
description: Which environment settings to use?
17+
required: true
18+
type: string
19+
default: development
1520

1621
permissions:
1722
pull-requests: write
@@ -23,15 +28,16 @@ jobs:
2328
uses: ./.github/workflows/base-e2e-backendtest.yml
2429
with:
2530
build_branch: ${{ inputs.build_branch }}
26-
environment: development
31+
environment: ${{ inputs.environment }}
2732
sandbox: ${{ inputs.sandbox }}
2833
secrets:
2934
AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }}
35+
3036
services-fhir-api-e2etest:
3137
uses: ./.github/workflows/base-e2e-fhir-backendtest.yml
3238
with:
3339
build_branch: ${{ inputs.build_branch }}
34-
environment: development
40+
environment: ${{ inputs.environment }}
3541
sandbox: ${{ inputs.sandbox }}
3642
secrets:
37-
AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }}
43+
AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }}

‎.github/workflows/ndr-smoketest-ui-test-ci.yml‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,3 +29,72 @@ jobs:
2929
AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }}
3030
CYPRESS_ODSCODE: ${{ secrets.CYPRESS_ODSCODE }}
3131
CYPRESS_KEY: ${{ secrets.CYPRESS_KEY }}
32+
33+
notify-slack:
34+
runs-on: ubuntu-latest
35+
needs: [smoketest]
36+
if: failure()
37+
steps:
38+
- name: Configure AWS Credentials
39+
uses: aws-actions/configure-aws-credentials@v5
40+
with:
41+
role-to-assume: ${{ secrets.AWS_ASSUME_ROLE }}
42+
aws-region: ${{ vars.AWS_REGION }}
43+
44+
- name: Get slack bot token from SSM parameter store
45+
run: |
46+
slack_bot_token=$(aws ssm get-parameter --name "/ndr/alerting/slack/bot_token" --with-decryption --query "Parameter.Value" --output text)
47+
echo "::add-mask::$slack_bot_token"
48+
echo "SLACK_BOT_TOKEN=$slack_bot_token" >> $GITHUB_ENV
49+
50+
- name: Send Slack Notification
51+
uses: slackapi/slack-github-action@v2.1.1
52+
with:
53+
method: chat.postMessage
54+
token: ${{ env.SLACK_BOT_TOKEN }}
55+
payload: |
56+
{
57+
"channel": "${{ vars.ALERTS_SLACK_CHANNEL_ID }}",
58+
"attachments": [
59+
{
60+
"color": "#ff0000",
61+
"blocks": [
62+
{
63+
"type": "header",
64+
"text": {
65+
"type": "plain_text",
66+
"text": "❌ Workflow `${{ github.workflow }}` failed"
67+
}
68+
},
69+
{
70+
"type": "section",
71+
"text": {
72+
"type": "mrkdwn",
73+
"text": "*Triggered by:* `${{ github.actor }}`\n*Workflow:* <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.workflow }}>"
74+
}
75+
},
76+
{
77+
"type": "divider"
78+
},
79+
{
80+
"type": "section",
81+
"fields": [
82+
{
83+
"type": "mrkdwn",
84+
"text": "*smoketest:* ${{ needs.smoketest.result == 'success' && ':white_check_mark:' || ':x:' }}"
85+
}
86+
]
87+
},
88+
{
89+
"type": "context",
90+
"elements": [
91+
{
92+
"type": "mrkdwn",
93+
"text": "Environment: `development` | Sandbox: `ndr-dev`"
94+
}
95+
]
96+
}
97+
]
98+
}
99+
]
100+
}

0 commit comments

Comments
 (0)