feat: notify slack on gh action workflows failure

.github/actions/notify-slack/action.yml

@@ -1,10 +1,18 @@
 name: 'Notify Slack'
-description: 'Send a notification to a Slack channel with an error message'
+description: 'Send a notification to a Slack channel with an alert message based on priority level'
 inputs:
   OP_SERVICE_ACCOUNT_TOKEN:
     description: 'The service account token for 1Password'
     required: true
     type: string
+  PRIORITY:
+    description: 'Priority level of the alert (medium/high)'
+    required: true
+    type: string
+  STEP:
+    description: 'The step in which the issue occurred (optional)'
+    required: false
+    type: string
 
 runs:
   using: 'composite'
@@ -19,30 +27,52 @@ runs:
         CREDENTIALS: "op://rbiv7rvkkrsdlpcrz3bmv7nmcu/ifkeehu5gzi7wy5ub5qvwkaire/credential"
         SLACK_WEBHOOK_URL: "op://rbiv7rvkkrsdlpcrz3bmv7nmcu/Slack webhook URLs/internal-github-action-alerts/internal-github-action-alerts"
 
+    - name: Set Alert Message
+      id: alert_message
+      shell: bash
+      run: |
+        if [[ "${{ inputs.PRIORITY }}" == "high" ]]; then
+          if [[ -n "${{ inputs.STEP }}" ]]; then
+            message="🚨 High Priority Alert: [${{ github.workflow }}] failed at step \"${{ inputs.STEP }}\". Immediate attention is required to avoid production impact."
+          else
+            message="🚨 High Priority Alert: [${{ github.workflow }}] failed. Immediate attention is required to avoid production impact."
+          fi
+        else
+          if [[ -n "${{ inputs.STEP }}" ]]; then
+            message="🚧 Medium Priority Alert: [${{ github.workflow }}] encountered an issue at step \"${{ inputs.STEP }}\". This may affect ongoing integration processes."
+          else
+            message="🚧 Medium Priority Alert: [${{ github.workflow }}] encountered an issue. This may affect ongoing integration processes."
+          fi
+        fi
+
+        # Construct the JSON payload and save it to a file
+        jq -n --arg message "$message" --arg run_url "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" '{
+          "channel": "C074SPVCH6H",
+          "blocks": [
+            {
+              "type": "header",
+              "text": {
+                "type": "plain_text",
+                "text": $message,
+                "emoji": true
+              }
+            },
+            {
+              "type": "section",
+              "text": {
+                "type": "mrkdwn",
+                "text": "See <\($run_url)|GitHub Action Run Details>"
+              }
+            }
+          ]
+        }' > slack_payload.json
+
     - name: Post notification in Slack channel
       uses: slackapi/[email protected]
       with:
-        payload: |
-          {
-            "channel": "C074SPVCH6H",
-            "blocks": [
-              {
-                "type": "header",
-                "text": {
-                  "type": "plain_text",
-                  "text": ":x: ERROR running Github action \"${{ github.workflow }}\"",
-                  "emoji": true
-                }
-              },  
-              {
-                "type": "section",
-                "text": {
-                  "type": "mrkdwn",
-                  "text": "See https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
-                }
-              }    
-            ]
-          }
+        payload-file-path: ./slack_payload.json
       env:
         SLACK_WEBHOOK_URL: ${{ env.SLACK_WEBHOOK_URL }}
         SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
+
+

.github/workflows/datasets-batch-deployer-dev.yml

@@ -21,3 +21,15 @@ jobs:
       DATASETS_BUCKET_NAME: 'mobilitydata-datasets-dev'
     secrets:
       GCP_MOBILITY_FEEDS_SA_KEY: ${{ secrets.DEV_GCP_MOBILITY_FEEDS_SA_KEY }}
+  notify-slack-on-failure:
+    needs: [ deploy ]
+    if: always() && (needs.deploy.result == 'failure') && (github.event_name != 'workflow_dispatch')
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Notify Slack
+        uses: ./.github/actions/notify-slack
+        with:
+          OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
+          PRIORITY: "medium"

.github/workflows/db-update-dev.yml

@@ -27,4 +27,16 @@ jobs:
       DB_GCP_MOBILITY_FEEDS_SA_KEY: ${{ secrets.QA_GCP_MOBILITY_FEEDS_SA_KEY }}
       OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
       OP_FEEDS_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_FEEDS_SERVICE_ACCOUNT_TOKEN }}
-      POSTGRE_SQL_INSTANCE_NAME: ${{ secrets.DB_INSTANCE_NAME }}
+      POSTGRE_SQL_INSTANCE_NAME: ${{ secrets.DB_INSTANCE_NAME }}
+  notify-slack-on-failure:
+    needs: [ update ]
+    if: always() && (needs.update.result == 'failure') && (github.event_name == 'repository_dispatch')
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Notify Slack
+        uses: ./.github/actions/notify-slack
+        with:
+          OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
+          PRIORITY: "high"

.github/workflows/db-update-prod.yml

@@ -22,4 +22,17 @@ jobs:
       DB_GCP_MOBILITY_FEEDS_SA_KEY: ${{ secrets.PROD_GCP_MOBILITY_FEEDS_SA_KEY }}
       OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
       OP_FEEDS_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_FEEDS_SERVICE_ACCOUNT_TOKEN }}
-      POSTGRE_SQL_INSTANCE_NAME: ${{ secrets.DB_INSTANCE_NAME }}
+      POSTGRE_SQL_INSTANCE_NAME: ${{ secrets.DB_INSTANCE_NAME }}
+
+  notify-slack-on-failure:
+    needs: [ update ]
+    if: always() && (needs.update.result == 'failure') && (github.event_name == 'repository_dispatch')
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Notify Slack
+        uses: ./.github/actions/notify-slack
+        with:
+          OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
+          PRIORITY: "high"

.github/workflows/db-update-qa.yml

@@ -22,4 +22,16 @@ jobs:
       DB_GCP_MOBILITY_FEEDS_SA_KEY: ${{ secrets.QA_GCP_MOBILITY_FEEDS_SA_KEY }}
       OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
       OP_FEEDS_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_FEEDS_SERVICE_ACCOUNT_TOKEN }}
-      POSTGRE_SQL_INSTANCE_NAME: ${{ secrets.DB_INSTANCE_NAME }}
+      POSTGRE_SQL_INSTANCE_NAME: ${{ secrets.DB_INSTANCE_NAME }}
+  notify-slack-on-failure:
+    needs: [ update ]
+    if: always() && (needs.update.result == 'failure') && (github.event_name == 'repository_dispatch')
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Notify Slack
+        uses: ./.github/actions/notify-slack
+        with:
+          OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
+          PRIORITY: "high"

.github/workflows/release-qa.yml

@@ -39,3 +39,15 @@ jobs:
     needs: batch-deployment
     uses: ./.github/workflows/web-qa.yml
     secrets: inherit
+  notify-slack-on-failure:
+    needs: [ web-deployment, integration-tests ]
+    if: failure() && (github.event_name != 'workflow_dispatch')
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Notify Slack
+        uses: ./.github/actions/notify-slack
+        with:
+          OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
+          PRIORITY: "high"

.github/workflows/release.yml

@@ -37,3 +37,15 @@ jobs:
     needs: batch-deployment
     uses: ./.github/workflows/web-prod.yml
     secrets: inherit
+  notify-slack-on-failure:
+    needs: [web-deployment, integration-tests]
+    if: failure()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Notify Slack
+        uses: ./.github/actions/notify-slack
+        with:
+          OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
+          PRIORITY: "high"

.github/workflows/validator-update.yml

@@ -37,3 +37,15 @@ jobs:
             --schedule-time=$(date -u -d "+24 hours" +%Y-%m-%dT%H:%M:%SZ) \
             --oidc-service-account-email=${DEPLOYER_SERVICE_ACCOUNT} \
             --location=${{ vars.MOBILITY_FEEDS_REGION }}
+  notify-slack-on-failure:
+    needs: [ validator-update ]
+    if: failure() && (github.event_name != 'workflow_dispatch')
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Notify Slack
+        uses: ./.github/actions/notify-slack
+        with:
+          OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
+          PRIORITY: "high"