[NDR-313] Add slack notification for workflow failures

.github/workflows/base-deploy-ui.yml

@@ -49,7 +49,7 @@ jobs:
           aws-region: ${{ vars.AWS_REGION }}
           mask-aws-account-id: true
 
-      - name: Get SSM parameters from SSM
+      - name: Get SSM parameters
         id: ssm-parameters
         run: | 
           doc_store_api_endpoint=$(aws ssm get-parameter --name "/ndr/${{ inputs.sandbox }}/api_endpoint" --with-decryption --query "Parameter.Value" --output text)

.github/workflows/base-e2e-fhir-backendtest.yml

@@ -29,6 +29,9 @@ jobs:
   e2e-test:
     runs-on: ubuntu-latest
     environment: ${{ inputs.environment }}
+    outputs:
+      environment: ${{ inputs.environment }}
+      sandbox: ${{ inputs.sandbox }}
     steps:
       - name: Checkout
         uses: actions/checkout@v5

.github/workflows/lambdas-dev-to-main-ci.yml

@@ -78,3 +78,91 @@ jobs:
       sandbox: ndr-dev
     secrets:
       AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }}
+
+  notify-slack:
+    runs-on: ubuntu-latest
+    needs: [check_packages, run_tests, publish_all_lambda_layers, deploy_all_lambdas, deploy_data_collection]
+    if: failure() && github.event_name == 'push' && github.ref == 'refs/heads/main'
+    steps:
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v5
+        with:
+          role-to-assume: ${{ secrets.AWS_ASSUME_ROLE }}
+          role-skip-session-tagging: true
+          aws-region: ${{ vars.AWS_REGION }}
+          mask-aws-account-id: true
+
+      - name: Get slack bot token from SSM parameter store
+        id: ssm-parameters
+        run: |
+          slack_bot_token=$(aws ssm get-parameter --name "/ndr/alerting/slack/bot_token" --with-decryption --query "Parameter.Value" --output text)
+          echo "::add-mask::$slack_bot_token"
+          echo "SLACK_BOT_TOKEN=$slack_bot_token" >> $GITHUB_ENV
+
+      - name: Send Slack Notification
+        uses: slackapi/[email protected]
+        with:
+          method: chat.postMessage
+          token: ${{ env.SLACK_BOT_TOKEN }}
+          payload: |
+            {
+              "channel": "${{ vars.ALERTS_SLACK_CHANNEL_ID }}",
+              "attachments": [
+                {
+                  "color": "#ff0000",
+                  "blocks": [
+                    {
+                      "type": "header",
+                      "text": {
+                        "type": "plain_text",
+                        "text": "❌ Workflow `${{ github.workflow }}` failed"
+                      }
+                    },
+                    {
+                      "type": "section",
+                      "text": {
+                        "type": "mrkdwn",
+                        "text": "*Triggered by:* `${{ github.actor }}`\n*Workflow:* <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.workflow }}>"
+                      }
+                    },
+                    {
+                      "type": "divider"
+                    },
+                    {
+                      "type": "section",
+                      "fields": [
+                        {
+                          "type": "mrkdwn",
+                          "text": "*check_packages:* ${{ needs.check_packages.result == 'success' && ':white_check_mark:' || ':x:' }}"
+                        },
+                        {
+                          "type": "mrkdwn",
+                          "text": "*run_tests:* ${{ needs.run_tests.result == 'success' && ':white_check_mark:' || ':x:' }}"
+                        },
+                        {
+                          "type": "mrkdwn",
+                          "text": "*publish_all_lambda_layers:* ${{ needs.publish_all_lambda_layers.result == 'success' && ':white_check_mark:' || ':x:' }}"
+                        },
+                        {
+                          "type": "mrkdwn",
+                          "text": "*deploy_all_lambdas:* ${{ needs.deploy_all_lambdas.result == 'success' && ':white_check_mark:' || ':x:' }}"
+                        },
+                        {
+                          "type": "mrkdwn",
+                          "text": "*deploy_data_collection:* ${{ needs.deploy_data_collection.result == 'success' && ':white_check_mark:' || ':x:' }}"
+                        }
+                      ]
+                    },
+                    {
+                      "type": "context",
+                      "elements": [
+                        {
+                          "type": "mrkdwn",
+                          "text": "Environment: `development` | Sandbox: `ndr-dev`"
+                        }
+                      ]
+                    }
+                  ]
+                }
+              ]
+            }

.github/workflows/ndr-e2e-backend-test-ci.yml

@@ -24,6 +24,7 @@ jobs:
       sandbox: ndr-dev
     secrets:
       AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }}
+
   services-fhir-api-e2etest:
     uses: ./.github/workflows/base-e2e-fhir-backendtest.yml
     with:
@@ -32,3 +33,79 @@ jobs:
       sandbox: ndr-dev
     secrets:
       AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }}
+
+  notify-slack:
+    runs-on: ubuntu-latest
+    needs: [services-e2etest, services-fhir-api-e2etest]
+    if: failure()
+    steps:
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v5
+        with:
+          role-to-assume: ${{ secrets.AWS_ASSUME_ROLE }}
+          role-skip-session-tagging: true
+          aws-region: ${{ vars.AWS_REGION }}
+          mask-aws-account-id: true
+
+      - name: Get slack bot token from SSM parameter store
+        id: ssm-parameters
+        run: |
+          slack_bot_token=$(aws ssm get-parameter --name "/ndr/alerting/slack/bot_token" --with-decryption --query "Parameter.Value" --output text)
+          echo "::add-mask::$slack_bot_token"
+          echo "SLACK_BOT_TOKEN=$slack_bot_token" >> $GITHUB_ENV
+
+      - name: Send Slack Notification
+        uses: slackapi/[email protected]
+        with:
+          method: chat.postMessage
+          token: ${{ env.SLACK_BOT_TOKEN }}
+          payload: |
+            {
+              "channel": "${{ vars.ALERTS_SLACK_CHANNEL_ID }}",
+              "attachments": [
+                {
+                  "color": "#ff0000",
+                  "blocks": [
+                    {
+                      "type": "header",
+                      "text": {
+                        "type": "plain_text",
+                        "text": "❌ Workflow `${{ github.workflow }}` failed"
+                      }
+                    },
+                    {
+                      "type": "section",
+                      "text": {
+                        "type": "mrkdwn",
+                        "text": "*Triggered by:* `${{ github.actor }}`\n*Workflow:* <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.workflow }}>"
+                      }
+                    },
+                    {
+                      "type": "divider"
+                    },
+                    {
+                      "type": "section",
+                      "fields": [
+                        {
+                          "type": "mrkdwn",
+                          "text": "*services-e2etest:* ${{ needs.services-e2etest.result == 'success' && ':white_check_mark:' || ':x:' }}"
+                        },
+                        {
+                          "type": "mrkdwn",
+                          "text": "*services-fhir-api-e2etest:* ${{ needs.services-fhir-api-e2etest.result == 'success' && ':white_check_mark:' || ':x:' }}"
+                        }
+                      ]
+                    },
+                    {
+                      "type": "context",
+                      "elements": [
+                        {
+                          "type": "mrkdwn",
+                          "text": "Environment: `${{ needs.services-fhir-api-e2etest.outputs.environment }}` | Sandbox: `${{ needs.services-fhir-api-e2etest.outputs.sandbox }}`"
+                        }
+                      ]
+                    }
+                  ]
+                }
+              ]
+            }

.github/workflows/ndr-e2e-test-sandbox.yml

@@ -12,6 +12,11 @@ on:
         description: "Sandbox environment"
         required: true
         default: "ndr"
+      environment:
+        description: Which environment settings to use?
+        required: true
+        type: string
+        default: development
 
 permissions:
   pull-requests: write
@@ -23,15 +28,16 @@ jobs:
     uses: ./.github/workflows/base-e2e-backendtest.yml
     with:
       build_branch: ${{ inputs.build_branch }}
-      environment: development
+      environment: ${{ inputs.environment }}
       sandbox: ${{ inputs.sandbox }}
     secrets:
       AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }}
+
   services-fhir-api-e2etest:
     uses: ./.github/workflows/base-e2e-fhir-backendtest.yml
     with:
       build_branch: ${{ inputs.build_branch }}
-      environment: development
+      environment: ${{ inputs.environment }}
       sandbox: ${{ inputs.sandbox }}
     secrets:
-      AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }}
+      AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }}

.github/workflows/ndr-smoketest-ui-test-ci.yml

@@ -29,3 +29,72 @@ jobs:
       AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }}
       CYPRESS_ODSCODE: ${{ secrets.CYPRESS_ODSCODE }}
       CYPRESS_KEY: ${{ secrets.CYPRESS_KEY }}
+
+  notify-slack:
+    runs-on: ubuntu-latest
+    needs: [smoketest]
+    if: failure()
+    steps:
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v5
+        with:
+          role-to-assume: ${{ secrets.AWS_ASSUME_ROLE }}
+          aws-region: ${{ vars.AWS_REGION }}
+
+      - name: Get slack bot token from SSM parameter store
+        run: |
+          slack_bot_token=$(aws ssm get-parameter --name "/ndr/alerting/slack/bot_token" --with-decryption --query "Parameter.Value" --output text)
+          echo "::add-mask::$slack_bot_token"
+          echo "SLACK_BOT_TOKEN=$slack_bot_token" >> $GITHUB_ENV
+
+      - name: Send Slack Notification
+        uses: slackapi/[email protected]
+        with:
+          method: chat.postMessage
+          token: ${{ env.SLACK_BOT_TOKEN }}
+          payload: |
+            {
+              "channel": "${{ vars.ALERTS_SLACK_CHANNEL_ID }}",
+              "attachments": [
+                {
+                  "color": "#ff0000",
+                  "blocks": [
+                    {
+                      "type": "header",
+                      "text": {
+                        "type": "plain_text",
+                        "text": "❌ Workflow `${{ github.workflow }}` failed"
+                      }
+                    },
+                    {
+                      "type": "section",
+                      "text": {
+                        "type": "mrkdwn",
+                        "text": "*Triggered by:* `${{ github.actor }}`\n*Workflow:* <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.workflow }}>"
+                      }
+                    },
+                    {
+                      "type": "divider"
+                    },
+                    {
+                      "type": "section",
+                      "fields": [
+                        {
+                          "type": "mrkdwn",
+                          "text": "*smoketest:* ${{ needs.smoketest.result == 'success' && ':white_check_mark:' || ':x:' }}"
+                        }
+                      ]
+                    },
+                    {
+                      "type": "context",
+                      "elements": [
+                        {
+                          "type": "mrkdwn",
+                          "text": "Environment: `development` | Sandbox: `ndr-dev`"
+                        }
+                      ]
+                    }
+                  ]
+                }
+              ]
+            }