Merge pull request #30 from worldcoin/limit-nr-of-renotifications

Limit nr of renotifications on slack channels

Author: danielllek

datadog.tf

@@ -30,11 +30,14 @@ module "datadog_monitoring" {
 resource "datadog_monitor" "oom" {
   count = var.monitoring_enabled ? 1 : 0
 
-  name      = "OOM kill detected on ${var.cluster_name}"
-  type      = "metric alert"
-  query     = "sum(last_1h):sum:oom_kill.oom_process.count{cluster_name:${var.cluster_name}} by {kube_namespace,kube_container_name}.as_count() >= 1"
-  timeout_h = 1
-  priority  = 3
+  name  = "OOM kill detected on ${var.cluster_name}"
+  type  = "metric alert"
+  query = "sum(last_4h):sum:oom_kill.oom_process.count{cluster_name:${var.cluster_name}} by {kube_namespace,kube_container_name}.as_count() >= 1"
+
+  on_missing_data          = "default"
+  group_retention_duration = "24h"
+
+  priority = 3
 
   monitor_thresholds {
     critical = 1