Add integration for incident.io webhook

.github/workflows/tofu-apply.yml

@@ -25,3 +25,4 @@ jobs:
       opentofu_secret_variables: |
         datadog_api_key = "${{ secrets.DD_API_KEY }}"
         datadog_app_key = "${{ secrets.DD_APP_KEY }}"
+        incident_io_bearer = "${{ secrets.INCIDENT_IO_BEARER }}"

.github/workflows/tofu-check-schedule.yml

@@ -35,3 +35,4 @@ jobs:
       opentofu_secret_variables: |
         datadog_api_key = "${{ secrets.DD_API_KEY }}"
         datadog_app_key = "${{ secrets.DD_APP_KEY }}"
+        incident_io_bearer = "${{ secrets.INCIDENT_IO_BEARER }}"

.github/workflows/tofu-plan.yml

@@ -27,3 +27,4 @@ jobs:
       opentofu_secret_variables: |
         datadog_api_key = "${{ secrets.DD_API_KEY }}"
         datadog_app_key = "${{ secrets.DD_APP_KEY }}"
+        incident_io_bearer = "${{ secrets.INCIDENT_IO_BEARER }}"

integrations.tf

@@ -41,3 +41,42 @@ resource "datadog_integration_slack_channel" "pytorch-infra-alerts" {
     mute_buttons = true
   }
 }
+
+# Create a new Datadog webhook
+
+resource "datadog_webhook" "lf-incident-io" {
+  name      = "lf-inceident-io"
+  url       = "https://api.incident.io/v2/alert_events/datadog/01JKTRSFTE6H2SR4AFM4VGWZFN"
+  encode_as = "json"
+
+  custom_headers = jsonencode({ "Authorization" = "Bearer ${var.incident_io_bearer}  " })
+  payload = jsonencode({
+    alert_transition  = "$ALERT_TRANSITION",
+    deduplication_key = "$AGGREG_KEY-$ALERT_CYCLE_KEY",
+    title             = "$EVENT_TITLE",
+    description       = "$EVENT_MSG",
+    source_url        = "$LINK",
+    metadata = {
+      id             = "$ID",
+      alert_metric   = "$ALERT_METRIC",
+      alert_query    = "$ALERT_QUERY",
+      alert_scope    = "$ALERT_SCOPE",
+      alert_status   = "$ALERT_STATUS",
+      alert_title    = "$ALERT_TITLE",
+      alert_type     = "$ALERT_TYPE",
+      alert_url      = "$LINK",
+      alert_priority = "$ALERT_PRIORITY",
+      date           = "$DATE",
+      event_type     = "$EVENT_TYPE",
+      hostname       = "$HOSTNAME",
+      last_updated   = "$LAST_UPDATED",
+      logs_sample    = "$LOGS_SAMPLE",
+      org = {
+        id   = "$ORG_ID",
+        name = "$ORG_NAME"
+      },
+      snapshot_url = "$SNAPSHOT",
+      tags         = "$TAGS"
+    }
+  })
+}

monitors.tf

@@ -53,5 +53,3 @@ EOT
 The number of visible messages in `{{queuename.name}}` is outside of the typical range.
 EOT
 }
-
-

variables.tf

@@ -25,3 +25,9 @@ variable "datadog_app_key" {
   type        = string
   sensitive   = true
 }
+
+variable "incident_io_bearer" {
+  description = "Bearer token for Incident.io webhook authorization"
+  type        = string
+  sensitive   = true
+}