Merge pull request #54 from worldcoin/INFRA-5561-monitor-system-workload-by-default

INFRA-5561 Monitor system workload option

Author: danielllek

datadog.tf

@@ -1,3 +1,13 @@
+locals {
+  all_filter_str    = format("kube_cluster_name:%s", var.cluster_name)
+  system_filter_str = format("%s AND kube_namespace IN (kube-system, argocd, cloudflared-tunnel, node-problem-detector, policy-reporter, karpenter, kube-ops, podsteward, cluster-autoscaler, cluster-monitoring, keda, kyverno, kyverno-policy-reporter, prometheus, teleport-agent, traefik, traefik-internal, wiz)", local.all_filter_str)
+
+  oom_filter_str                               = var.monitor_system_workload_only ? local.system_filter_str : local.all_filter_str
+  deployment_multiple_restarts_filter_override = var.monitor_system_workload_only ? "${local.system_filter_str} AND kube_deployment:*" : null
+  replicaset_unavailable_filter_override       = var.monitor_system_workload_only ? local.system_filter_str : null
+  replicaset_incomplete_filter_override        = var.monitor_system_workload_only ? local.system_filter_str : null
+}
+
 module "datadog_monitoring" {
   count = var.monitoring_enabled ? 1 : 0
 
@@ -6,11 +16,18 @@ module "datadog_monitoring" {
   notification_channel = var.monitoring_notification_channel
   service              = format("EKS %s", var.cluster_name)
   env                  = var.environment
-  filter_str           = format("kube_cluster_name:%s", var.cluster_name)
+  filter_str           = local.all_filter_str
   additional_tags = [
-    "CreatedBy:terraform"
+    "CreatedBy:terraform",
+    "service:k8s",
+    "team:infrastructure",
+    "env:${var.environment}",
   ]
 
+  deployment_multiple_restarts_filter_override = local.deployment_multiple_restarts_filter_override
+  replicaset_unavailable_filter_override       = local.replicaset_unavailable_filter_override
+  replicaset_incomplete_filter_override        = local.replicaset_incomplete_filter_override
+
   # don't alert on cpu overbooking
   cpu_limits_low_perc_enabled                = false
   cpu_requests_low_perc_enabled              = false
@@ -32,7 +49,7 @@ resource "datadog_monitor" "oom" {
 
   name  = "OOM kill detected on ${var.cluster_name}"
   type  = "metric alert"
-  query = "sum(last_4h):sum:oom_kill.oom_process.count{cluster_name:${var.cluster_name}} by {kube_namespace,kube_container_name}.as_count() >= 1"
+  query = "sum(last_4h):sum:oom_kill.oom_process.count{${local.oom_filter_str}} by {kube_namespace,kube_container_name}.as_count() >= 1"
 
   on_missing_data          = "default"
   group_retention_duration = "24h"
@@ -56,7 +73,10 @@ Notify: ${var.monitoring_notification_channel}
 EOT
 
   tags = [
-    "CreatedBy:terraform"
+    "CreatedBy:terraform",
+    "service:k8s",
+    "team:infrastructure",
+    "env:${var.environment}",
   ]
 }
 
@@ -76,6 +96,8 @@ resource "datadog_synthetics_test" "cluster_monitoring" {
   tags = [
     "CreatedBy:terraform",
     "env:${var.environment}",
+    "service:k8s",
+    "team:infrastructure",
   ]
 
   request_definition {

variables.tf

@@ -151,6 +151,12 @@ variable "monitoring_enabled" {
   default     = true
 }
 
+variable "monitor_system_workload_only" {
+  description = "Monitor system workloads only."
+  type        = bool
+  default     = false
+}
+
 variable "monitoring_external_enabled" {
   description = "Whether to enable external monitoring (Datadog Synthetics)."
   type        = bool