High cpu usage alert now only applies to control and login nodes

Author: wtripp180901

ansible/roles/kube_prometheus_stack/defaults/main/helm.yml

@@ -51,6 +51,8 @@ kube_prometheus_stack_release_defaults:
       KubeSchedulerDown: true
       KubeProxyDown: true
       KubeControllerManagerDown: true
+      # Replaced with appliance specific versions
+      NodeCPUHighUsage: true
   prometheus:
     service:
       type: NodePort

environments/common/inventory/group_vars/all/prometheus.yml

@@ -44,3 +44,11 @@ prometheus_extra_alerting_rules:
     expr: "slurm_nodes_down > 0\n"
     labels:
       severity: critical
+  - alert: NodeCPUHighUsage
+    annotations:
+      description: '{% raw %}CPU usage at {{ $labels.instance }} has been above 90% for the last 15 minutes, is currently at {{ printf "%.2f" $value }}%.{% endraw %}'
+      summary: High CPU usage.
+    expr: sum without(mode) (avg without (cpu) (rate(node_cpu_seconds_total{job="node-exporter",mode!="idle",instance=~".+(-control|-login).*"}[2m]))) * 100 > 90
+    for: 15m
+    labels:
+      severity: info