add node failure alert

sjpb · sjpb · commit ce9ed5bf03d9 · 2025-04-10T15:45:04.000Z
diff --git a/environments/common/files/prometheus/rules/slurm.rules b/environments/common/files/prometheus/rules/slurm.rules
@@ -9,3 +9,8 @@ groups:
     expr: "slurm_nodes_down > 0\n"
     labels:
       severity: critical
+  - alert: SlurmNodeFail
+    annotations:
+      description: '{{ $value }} Slurm nodes are in fail status'
+      summary: 'At least one Slurm node is failed.'
+    expr: "slurm_nodes_fail > 0\n"
