diff --git a/etc/kayobe/kolla/config/prometheus/prometheus.rules b/etc/kayobe/kolla/config/prometheus/prometheus.rules index c9803946a..20e1b303a 100644 --- a/etc/kayobe/kolla/config/prometheus/prometheus.rules +++ b/etc/kayobe/kolla/config/prometheus/prometheus.rules @@ -7,7 +7,7 @@ groups: rules: - alert: PrometheusTargetMissing - expr: up == 0 + expr: up{job!="redfish-exporter-seed"} == 0 for: 5m labels: severity: critical @@ -15,6 +15,15 @@ groups: summary: "Prometheus target missing (instance {{ $labels.instance }})" description: "A Prometheus target has disappeared. An exporter might have crashed." + - alert: PrometheusTargetMissing + expr: up{job="redfish-exporter-seed"} == 0 + for: 15m + labels: + severity: critical + annotations: + summary: "Prometheus target missing (instance {{ $labels.instance }})" + description: "A Prometheus target has disappeared. An exporter might have crashed." + - alert: PrometheusAllTargetsMissing expr: count by (job) (up) == 0 for: 1m diff --git a/releasenotes/notes/reduces-sensitivity-of-redfish-target-alerts-a3d77a3f0c3dac8a.yaml b/releasenotes/notes/reduces-sensitivity-of-redfish-target-alerts-a3d77a3f0c3dac8a.yaml new file mode 100644 index 000000000..0ba59ea7a --- /dev/null +++ b/releasenotes/notes/reduces-sensitivity-of-redfish-target-alerts-a3d77a3f0c3dac8a.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Changes the duration for which redfish exporter must continually fail + scrapes before triggering an alert to 15 minutes. This should hopefully + reduce some alert spam.