Merge pull request #14 from aws-observability/infra-dashboard-curation

Infra dashboard curation

Author: bonclay7

examples/eks-cluster-with-vpc/main.tf

@@ -52,7 +52,7 @@ module "eks_blueprints" {
   managed_node_groups = {
     mg_5 = {
       node_group_name = "managed-ondemand"
-      instance_types  = ["m5.large"]
+      instance_types  = ["t3.xlarge"]
       min_size        = 2
       subnet_ids      = module.vpc.private_subnets
     }

modules/workloads/infra/alerts.tf

@@ -211,7 +211,7 @@ groups:
           description: The Kubelet Pod Lifecycle Event Generator has a 99th percentile duration of {{ $value }} seconds on node {{ $labels.node }}.
           summary: Kubelet Pod Lifecycle Event Generator is taking too long to relist.
       - alert: KubeletPodStartUpLatencyHigh
-        expr: histogram_quantile(0.99, sum by(cluster, instance, le) (rate(kubelet_pod_worker_duration_seconds_bucket{job="kubelet",metrics_path="/metrics"}[5m]))) * on(cluster, instance) group_left(node) kubelet_node_name{job="kubelet",metrics_path="/metrics"} > 60
+        expr: histogram_quantile(0.99, sum by(cluster, instance, le) (rate(kubelet_pod_worker_duration_seconds_bucket{job="kubelet"}[5m]))) * on(cluster, instance) group_left(node) kubelet_node_name{job="kubelet"} > 60
         for: 15m
         labels:
           severity: warning
@@ -263,7 +263,7 @@ groups:
           description: Kubelet on node {{ $labels.node }} has failed to renew its server certificate ({{ $value | humanize }} errors in the last 5 minutes).
           summary: Kubelet has failed to renew its server certificate.
       - alert: KubeletDown
-        expr: absent(up{job="kubelet",metrics_path="/metrics"} == 1)
+        expr: absent(up{job="kubelet"} == 1)
         for: 15m
         labels:
           severity: critical
@@ -350,31 +350,31 @@ groups:
           description: The kubernetes apiserver has terminated {{ $value | humanizePercentage }} of its incoming requests.
           summary: The kubernetes apiserver has terminated {{ $value | humanizePercentage }} of its incoming requests.
       - alert: KubePersistentVolumeFillingUp
-        expr: (kubelet_volume_stats_available_bytes{job="kubelet",metrics_path="/metrics",namespace=~".*"} / kubelet_volume_stats_capacity_bytes{job="kubelet",metrics_path="/metrics",namespace=~".*"}) < 0.03 and kubelet_volume_stats_used_bytes{job="kubelet",metrics_path="/metrics",namespace=~".*"} > 0 unless on(namespace, persistentvolumeclaim) kube_persistentvolumeclaim_access_mode{access_mode="ReadOnlyMany"} == 1 unless on(namespace, persistentvolumeclaim) kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1
+        expr: (kubelet_volume_stats_available_bytes{job="kubelet",namespace=~".*"} / kubelet_volume_stats_capacity_bytes{job="kubelet",namespace=~".*"}) < 0.03 and kubelet_volume_stats_used_bytes{job="kubelet",namespace=~".*"} > 0 unless on(namespace, persistentvolumeclaim) kube_persistentvolumeclaim_access_mode{access_mode="ReadOnlyMany"} == 1 unless on(namespace, persistentvolumeclaim) kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1
         for: 1m
         labels:
           severity: critical
         annotations:
           description: The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} is only {{ $value | humanizePercentage }} free.
           summary: PersistentVolume is filling up.
       - alert: KubePersistentVolumeFillingUp
-        expr: (kubelet_volume_stats_available_bytes{job="kubelet",metrics_path="/metrics",namespace=~".*"} / kubelet_volume_stats_capacity_bytes{job="kubelet",metrics_path="/metrics",namespace=~".*"}) < 0.15 and kubelet_volume_stats_used_bytes{job="kubelet",metrics_path="/metrics",namespace=~".*"} > 0 and predict_linear(kubelet_volume_stats_available_bytes{job="kubelet",metrics_path="/metrics",namespace=~".*"}[6h], 4 * 24 * 3600) < 0 unless on(namespace, persistentvolumeclaim) kube_persistentvolumeclaim_access_mode{access_mode="ReadOnlyMany"} == 1 unless on(namespace, persistentvolumeclaim) kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1
+        expr: (kubelet_volume_stats_available_bytes{job="kubelet",namespace=~".*"} / kubelet_volume_stats_capacity_bytes{job="kubelet",namespace=~".*"}) < 0.15 and kubelet_volume_stats_used_bytes{job="kubelet",namespace=~".*"} > 0 and predict_linear(kubelet_volume_stats_available_bytes{job="kubelet",namespace=~".*"}[6h], 4 * 24 * 3600) < 0 unless on(namespace, persistentvolumeclaim) kube_persistentvolumeclaim_access_mode{access_mode="ReadOnlyMany"} == 1 unless on(namespace, persistentvolumeclaim) kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1
         for: 1h
         labels:
           severity: warning
         annotations:
           description: Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} is expected to fill up within four days.
           summary: PersistentVolume is filling up.
       - alert: KubePersistentVolumeInodesFillingUp
-        expr: (kubelet_volume_stats_inodes_free{job="kubelet",metrics_path="/metrics",namespace=~".*"} / kubelet_volume_stats_inodes{job="kubelet",metrics_path="/metrics",namespace=~".*"}) < 0.03 and kubelet_volume_stats_inodes_used{job="kubelet",metrics_path="/metrics",namespace=~".*"} > 0 unless on(namespace, persistentvolumeclaim) kube_persistentvolumeclaim_access_mode{access_mode="ReadOnlyMany"} == 1 unless on(namespace, persistentvolumeclaim) kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1
+        expr: (kubelet_volume_stats_inodes_free{job="kubelet",namespace=~".*"} / kubelet_volume_stats_inodes{job="kubelet",namespace=~".*"}) < 0.03 and kubelet_volume_stats_inodes_used{job="kubelet",namespace=~".*"} > 0 unless on(namespace, persistentvolumeclaim) kube_persistentvolumeclaim_access_mode{access_mode="ReadOnlyMany"} == 1 unless on(namespace, persistentvolumeclaim) kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1
         for: 1m
         labels:
           severity: critical
         annotations:
           description: The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} only has {{ $value | humanizePercentage }} free inodes.
           summary: PersistentVolumeInodes is filling up.
       - alert: KubePersistentVolumeInodesFillingUp
-        expr: (kubelet_volume_stats_inodes_free{job="kubelet",metrics_path="/metrics",namespace=~".*"} / kubelet_volume_stats_inodes{job="kubelet",metrics_path="/metrics",namespace=~".*"}) < 0.15 and kubelet_volume_stats_inodes_used{job="kubelet",metrics_path="/metrics",namespace=~".*"} > 0 and predict_linear(kubelet_volume_stats_inodes_free{job="kubelet",metrics_path="/metrics",namespace=~".*"}[6h], 4 * 24 * 3600) < 0 unless on(namespace, persistentvolumeclaim) kube_persistentvolumeclaim_access_mode{access_mode="ReadOnlyMany"} == 1 unless on(namespace, persistentvolumeclaim) kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1
+        expr: (kubelet_volume_stats_inodes_free{job="kubelet",namespace=~".*"} / kubelet_volume_stats_inodes{job="kubelet",namespace=~".*"}) < 0.15 and kubelet_volume_stats_inodes_used{job="kubelet",namespace=~".*"} > 0 and predict_linear(kubelet_volume_stats_inodes_free{job="kubelet",namespace=~".*"}[6h], 4 * 24 * 3600) < 0 unless on(namespace, persistentvolumeclaim) kube_persistentvolumeclaim_access_mode{access_mode="ReadOnlyMany"} == 1 unless on(namespace, persistentvolumeclaim) kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1
         for: 1h
         labels:
           severity: warning

modules/workloads/infra/dashboards.tf

@@ -4,18 +4,6 @@ resource "grafana_dashboard" "workloads" {
   config_json = file("${path.module}/dashboards/workloads.json")
 }
 
-resource "grafana_dashboard" "scheduler" {
-  count       = var.enable_dashboards ? 1 : 0
-  folder      = var.dashboards_folder_id
-  config_json = file("${path.module}/dashboards/scheduler.json")
-}
-
-resource "grafana_dashboard" "proxy" {
-  count       = var.enable_dashboards ? 1 : 0
-  folder      = var.dashboards_folder_id
-  config_json = file("${path.module}/dashboards/proxy.json")
-}
-
 resource "grafana_dashboard" "podnetwork" {
   count       = var.enable_dashboards ? 1 : 0
   folder      = var.dashboards_folder_id
@@ -28,36 +16,12 @@ resource "grafana_dashboard" "pods" {
   config_json = file("${path.module}/dashboards/pods.json")
 }
 
-resource "grafana_dashboard" "pv" {
-  count       = var.enable_dashboards ? 1 : 0
-  folder      = var.dashboards_folder_id
-  config_json = file("${path.module}/dashboards/pesistentvolumes.json")
-}
-
 resource "grafana_dashboard" "nodes" {
   count       = var.enable_dashboards ? 1 : 0
   folder      = var.dashboards_folder_id
   config_json = file("${path.module}/dashboards/nodes.json")
 }
 
-resource "grafana_dashboard" "necluster" {
-  count       = var.enable_dashboards ? 1 : 0
-  folder      = var.dashboards_folder_id
-  config_json = file("${path.module}/dashboards/nodeexpoter-use-cluster.json")
-}
-
-resource "grafana_dashboard" "nenodeuse" {
-  count       = var.enable_dashboards ? 1 : 0
-  folder      = var.dashboards_folder_id
-  config_json = file("${path.module}/dashboards/nodeexporter-use-node.json")
-}
-
-resource "grafana_dashboard" "nenode" {
-  count       = var.enable_dashboards ? 1 : 0
-  folder      = var.dashboards_folder_id
-  config_json = file("${path.module}/dashboards/nodeexporter-nodes.json")
-}
-
 resource "grafana_dashboard" "nwworload" {
   count       = var.enable_dashboards ? 1 : 0
   folder      = var.dashboards_folder_id
@@ -94,24 +58,6 @@ resource "grafana_dashboard" "kubelet" {
   config_json = file("${path.module}/dashboards/kubelet.json")
 }
 
-resource "grafana_dashboard" "etcd" {
-  count       = var.enable_dashboards ? 1 : 0
-  folder      = var.dashboards_folder_id
-  config_json = file("${path.module}/dashboards/etcd.json")
-}
-
-resource "grafana_dashboard" "coredns" {
-  count       = var.enable_dashboards ? 1 : 0
-  folder      = var.dashboards_folder_id
-  config_json = file("${path.module}/dashboards/coredns.json")
-}
-
-resource "grafana_dashboard" "controller" {
-  count       = var.enable_dashboards ? 1 : 0
-  folder      = var.dashboards_folder_id
-  config_json = file("${path.module}/dashboards/controller.json")
-}
-
 resource "grafana_dashboard" "clusternw" {
   count       = var.enable_dashboards ? 1 : 0
   folder      = var.dashboards_folder_id
@@ -122,10 +68,4 @@ resource "grafana_dashboard" "cluster" {
   count       = var.enable_dashboards ? 1 : 0
   folder      = var.dashboards_folder_id
   config_json = file("${path.module}/dashboards/cluster.json")
-}
-
-resource "grafana_dashboard" "apis" {
-  count       = var.enable_dashboards ? 1 : 0
-  folder      = var.dashboards_folder_id
-  config_json = file("${path.module}/dashboards/apiserver.json")
-}
+}