Merge pull request #5554 from raft-tech/5498-alermanager-chat-integration

Integrate PLG with Mattermost

Author: elipe17

tdrs-backend/plg/alertmanager/alertmanager.yml

@@ -4,36 +4,30 @@ global:
   smtp_from: 'no-reply@tanfdata.acf.hhs.gov'
   smtp_auth_username: 'apikey'
   smtp_auth_password: '{{ sendgrid_api_key }}'
-
+  slack_api_url: '{{ mattermost_web_hook_url }}'
 # The directory from which notification templates are read.
 templates:
   - '/etc/alertmanager/template/*.tmpl'
-
 # The root route on which each incoming alert enters.
 route:
   # The labels by which incoming alerts are grouped together. For example,
   # multiple alerts coming in for cluster=A and alertname=LatencyHigh would
   # be batched into a single group.
   group_by: ['alertname', 'env', 'service']
-
   # When a new group of alerts is created by an incoming alert, wait at
   # least 'group_wait' to send the initial notification.
   # This way ensures that you get multiple alerts for the same group that start
   # firing shortly after another are batched together on the first
   # notification.
   group_wait: 30s
-
   # When the first notification was sent, wait 'group_interval' to send a batch
   # of new alerts that started firing for that group.
   group_interval: 5m
-
   # If an alert has successfully been sent, wait 'repeat_interval' to
   # resend them.
   repeat_interval: 5m
-
   # A default receiver
-  receiver: admin-team-emails
-
+  receiver: mattermost
   # All the above attributes are inherited by all child routes and can
   # overwritten on each.
 
@@ -44,18 +38,20 @@ route:
         - alertname=~"UpTime"
       receiver: dev-team-emails
       repeat_interval: 24h
-
-    # Send all severity CRITICAL/ERROR alerts to OFA admins
+    # Send all severity CRITICAL/ERROR alerts to OFA admin emails
     - matchers:
-      - severity=~"ERROR|CRITICAL"
+        - severity=~"ERROR|CRITICAL"
       receiver: admin-team-emails
       continue: true
-    # Send all severity CRITICAL/ERROR/WARNING alerts to TDP Devs
+    # Send all severity CRITICAL/ERROR/WARNING alerts to mattermost and dev team emails
+    - matchers:
+        - severity=~"ERROR|CRITICAL|WARNING"
+      receiver: mattermost
+      continue: true
     - matchers:
-      - severity=~"ERROR|CRITICAL|WARNING"
+        - severity=~"ERROR|CRITICAL|WARNING"
       receiver: dev-team-emails
       continue: true
-
 # Inhibition rules allow to mute a set of alerts given that another alert is
 # firing.
 # We use this to mute any warning-level notifications if the same alert is
@@ -69,22 +65,34 @@ inhibit_rules:
     #   from both the source and target alerts,
     #   the inhibition rule will apply!
     equal: [alertname, env, service]
-
-  - source_matchers: [alertname=~"LocalBackendDown|DevEnvironmentBackendDown|
-                      StagingBackendDown|ProductionBackendDown"]
+  - source_matchers: [alertname=~"LocalBackendDown|DevEnvironmentBackendDown| StagingBackendDown|ProductionBackendDown"]
     target_match:
       alertname: 'UpTime'
     equal: ['environment', 'service']
-
-
-
 receivers:
   - name: 'admin-team-emails'
     email_configs:
       - to: '{{ admin_team_emails }}'
         send_resolved: true
-
   - name: 'dev-team-emails'
     email_configs:
       - to: '{{ dev_team_emails }}'
         send_resolved: true
+  - name: 'mattermost'
+    slack_configs:
+      - channel: 'guest-ofa-tdp-alerts'
+        username: 'alertmanager'
+        send_resolved: true
+        text: |-
+          {{ range .Alerts -}}
+          {{ if or (eq .Labels.severity "CRITICAL") (eq .Labels.severity "ERROR") }}
+          @here
+          {{ end }}
+          *Alert:* {{ .Annotations.title }}{{ if .Labels.severity }} - `{{ .Labels.severity }}`{{ end }}
+
+          *Description:* {{ .Annotations.description }}
+
+          *Details:*
+            {{ range .Labels.SortedPairs }} • *{{ .Name }}:* `{{ .Value }}`
+            {{ end }}
+          {{ end }}

tdrs-backend/plg/deploy.sh

@@ -102,6 +102,7 @@ deploy_alertmanager() {
     cp alertmanager.yml $CONFIG
     SENDGRID_API_KEY=$(cf env tdp-backend-prod | grep SENDGRID | cut -d " " -f2-)
     yq eval -i ".global.smtp_auth_password = \"$SENDGRID_API_KEY\"" $CONFIG
+    yq eval -i ".global.slack_api_url = \"$MATTERMOST_WEBHOOK_URL\"" $CONFIG
     yq eval -i ".receivers[0].email_configs[0].to = \"${ADMIN_EMAILS}\"" $CONFIG
     yq eval -i ".receivers[1].email_configs[0].to = \"${DEV_EMAILS}\"" $CONFIG
     cf push --no-route -f manifest.yml -t 180  --strategy rolling