separated prometheus recording and alerting rules

Author: wtripp180901

ansible/roles/kube_prometheus_stack/defaults/main/main.yml

@@ -63,13 +63,16 @@ prometheus_external_labels:
 
 prometheus_scrape_configs: []
 
-prometheus_extra_rules: []
+prometheus_extra_recording_rules: []
+prometheus_extra_alerting_rules: []
 
 prometheus_rules:
   appliance-rules:
     groups:
-    - name: all
-      rules: "{{ prometheus_extra_rules }}" 
+    - name: appliance-recording-rules
+      rules: "{{ prometheus_extra_recording_rules }}"
+    - name: appliance-alerting-rules
+      rules: "{{ prometheus_extra_alerting_rules }}" 
 
 # ------------------------------------------------------------------------------------------
 grafana_image_tag: 11.2.2

docs/monitoring-and-logging.md

@@ -236,7 +236,7 @@ The appliance previously used [cloudalchemy.prometheus](https://github.com/cloud
 
 See the upstream documentation for [alerting](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/) and [recording](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/) rules.
 
-In addition to the default recording and alerting rules set by kube-prometheus-stack, the appliances provides a default set of rules which can be found in the `prometheus_extra_rules` list in:
+In addition to the default recording and alerting rules set by kube-prometheus-stack, the appliance provides its own sets of default rules which can be found and modified in the `prometheus_extra_recording_rules` and `prometheus_extra_alerting_rules` lists in:
 
 > [environments/common/inventory/group_vars/all/prometheus.yml](../environments/common/inventory/group_vars/all/prometheus.yml)
 

environments/common/inventory/group_vars/all/prometheus.yml

@@ -20,14 +20,7 @@ prometheus_scrape_configs_default:
     replacement:   '${1}'
 
 prometheus_scrape_configs: "{{ prometheus_scrape_configs_default + (openondemand_scrape_configs if groups['openondemand'] | count > 0 else []) }}"
-prometheus_extra_rules:
-  - alert: SlurmNodeDown
-    annotations:
-      description: '{% raw %}{{ $value }} Slurm nodes are in down status.{% endraw %}'
-      summary: 'At least one Slurm node is down.'
-    expr: "slurm_nodes_down > 0\n"
-    labels:
-      severity: critical
+prometheus_extra_recording_rules:
   - record: node_cpu_system_seconds:record
     expr: (100 * sum by(instance)(increase(node_cpu_seconds_total{mode="system",job="node-exporter"}[60s]))) / (sum by(instance)(increase(node_cpu_seconds_total{job="node-exporter"}[60s])))
   - record: node_cpu_user_seconds:record
@@ -42,3 +35,12 @@ prometheus_extra_rules:
     expr: min by (instance) (node_cpu_scaling_frequency_hertz)
   - record: node_cpu_scaling_frequency_hertz_max:record
     expr: max by (instance) (node_cpu_scaling_frequency_hertz)
+
+prometheus_extra_alerting_rules:
+  - alert: SlurmNodeDown
+    annotations:
+      description: '{% raw %}{{ $value }} Slurm nodes are in down status.{% endraw %}'
+      summary: 'At least one Slurm node is down.'
+    expr: "slurm_nodes_down > 0\n"
+    labels:
+      severity: critical