diff --git a/.github/workflows/tofu-apply.yml b/.github/workflows/tofu-apply.yml index 40ecd82..3035490 100644 --- a/.github/workflows/tofu-apply.yml +++ b/.github/workflows/tofu-apply.yml @@ -25,3 +25,4 @@ jobs: opentofu_secret_variables: | datadog_api_key = "${{ secrets.DD_API_KEY }}" datadog_app_key = "${{ secrets.DD_APP_KEY }}" + incident_io_bearer = "${{ secrets.INCIDENT_IO_BEARER }}" diff --git a/.github/workflows/tofu-check-schedule.yml b/.github/workflows/tofu-check-schedule.yml index e8e943d..f0aef4a 100644 --- a/.github/workflows/tofu-check-schedule.yml +++ b/.github/workflows/tofu-check-schedule.yml @@ -35,3 +35,4 @@ jobs: opentofu_secret_variables: | datadog_api_key = "${{ secrets.DD_API_KEY }}" datadog_app_key = "${{ secrets.DD_APP_KEY }}" + incident_io_bearer = "${{ secrets.INCIDENT_IO_BEARER }}" diff --git a/.github/workflows/tofu-plan.yml b/.github/workflows/tofu-plan.yml index aa22cb5..07e8b48 100644 --- a/.github/workflows/tofu-plan.yml +++ b/.github/workflows/tofu-plan.yml @@ -27,3 +27,4 @@ jobs: opentofu_secret_variables: | datadog_api_key = "${{ secrets.DD_API_KEY }}" datadog_app_key = "${{ secrets.DD_APP_KEY }}" + incident_io_bearer = "${{ secrets.INCIDENT_IO_BEARER }}" diff --git a/integrations.tf b/integrations.tf index 6de4c15..edc37d9 100644 --- a/integrations.tf +++ b/integrations.tf @@ -41,3 +41,42 @@ resource "datadog_integration_slack_channel" "pytorch-infra-alerts" { mute_buttons = true } } + +# Create a new Datadog webhook + +resource "datadog_webhook" "lf-incident-io" { + name = "lf-inceident-io" + url = "https://api.incident.io/v2/alert_events/datadog/01JKTRSFTE6H2SR4AFM4VGWZFN" + encode_as = "json" + + custom_headers = jsonencode({ "Authorization" = "Bearer ${var.incident_io_bearer} " }) + payload = jsonencode({ + alert_transition = "$ALERT_TRANSITION", + deduplication_key = "$AGGREG_KEY-$ALERT_CYCLE_KEY", + title = "$EVENT_TITLE", + description = "$EVENT_MSG", + source_url = "$LINK", + metadata = { + id = "$ID", + alert_metric = "$ALERT_METRIC", + alert_query = "$ALERT_QUERY", + alert_scope = "$ALERT_SCOPE", + alert_status = "$ALERT_STATUS", + alert_title = "$ALERT_TITLE", + alert_type = "$ALERT_TYPE", + alert_url = "$LINK", + alert_priority = "$ALERT_PRIORITY", + date = "$DATE", + event_type = "$EVENT_TYPE", + hostname = "$HOSTNAME", + last_updated = "$LAST_UPDATED", + logs_sample = "$LOGS_SAMPLE", + org = { + id = "$ORG_ID", + name = "$ORG_NAME" + }, + snapshot_url = "$SNAPSHOT", + tags = "$TAGS" + } + }) +} diff --git a/monitors.tf b/monitors.tf index 5ae8a1b..71c8ac8 100644 --- a/monitors.tf +++ b/monitors.tf @@ -53,5 +53,3 @@ EOT The number of visible messages in `{{queuename.name}}` is outside of the typical range. EOT } - - diff --git a/variables.tf b/variables.tf index 97e5dbb..51b891d 100644 --- a/variables.tf +++ b/variables.tf @@ -25,3 +25,9 @@ variable "datadog_app_key" { type = string sensitive = true } + +variable "incident_io_bearer" { + description = "Bearer token for Incident.io webhook authorization" + type = string + sensitive = true +}