added blackbox alert rules from capi helm

Author: wtripp180901

ansible/roles/kube_prometheus_stack/defaults/main/main.yml

@@ -67,7 +67,6 @@ prometheus_external_labels:
 prometheus_scrape_configs: []
 
 prometheus_extra_rules: []
-prometheus_extra_rules_default: []
 
 prometheus_rules:
   appliance-rules:

environments/common/inventory/group_vars/all/openondemand.yml

@@ -196,14 +196,6 @@ openondemand_scrape_configs:
       target_label: target
     - target_label: __address__ 
       replacement: "{{ kube_prometheus_stack_blackbox_exporter_release_name }}-prometheus-blackbox-exporter:9115"
-openondemand_extra_rules:
-- alert: OnDemandProbeFailed
-  annotations:
-    description: '{% raw %}Could not establish secure connection to OOD server at {{ $labels.target }}{% endraw %}'
-    summary: 'Could not establish a secure connection to an Open OnDemand server'
-  expr: "probe_success{target='ondemand.monitoring-system'} < 1\n"
-  labels:
-    severity: warning
 
 openondemand_dashboard:
   - dashboard_id: 13465

environments/common/inventory/group_vars/all/prometheus.yml

@@ -20,15 +20,62 @@ prometheus_scrape_configs_default:
     replacement:   '${1}'
 
 prometheus_scrape_configs: "{{ prometheus_scrape_configs_default + (openondemand_scrape_configs if groups['openondemand'] | count > 0 else []) }}"
-prometheus_extra_rules: "{{ prometheus_extra_rules_default + (openondemand_extra_rules if groups['openondemand'] | count > 0 else []) }}"
-prometheus_extra_rules_default:
+prometheus_extra_rules:
   - alert: SlurmNodeDown
     annotations:
       description: '{% raw %}{{ $value }} Slurm nodes are in down status.{% endraw %}'
       summary: 'At least one Slurm node is down.'
     expr: "slurm_nodes_down > 0\n"
     labels:
       severity: critical
+  - alert: BlackboxProbeFailed
+    expr: probe_success == 0
+    for: 0m
+    labels:
+      severity: critical
+    annotations:
+      summary: '{% raw %}Blackbox probe failed (target {{ $labels.target }}){% endraw %}'
+      description: "{% raw %}Blackbox probe '{{ $labels.target }}' failed{% endraw %}"
+  - alert: BlackboxSlowProbe
+    expr: avg_over_time(probe_duration_seconds[1m]) > 1.2 #around 1.14 expected due to indirection in cluster
+    for: 1m
+    labels:
+      severity: warning
+    annotations:
+      summary: '{% raw %}Blackbox slow probe (target {{ $labels.target }}){% endraw %}'
+      description: "{% raw %}Blackbox probe '{{ $labels.target }}' took more than 1s to complete - {{ $value }}{% endraw %}"
+  - alert: BlackboxProbeHttpFailure
+    expr: probe_http_status_code <= 199 OR probe_http_status_code >= 400
+    for: 0m
+    labels:
+      severity: critical
+    annotations:
+      summary: '{% raw %}Blackbox probe HTTP failure (target {{ $labels.target }}){% endraw %}'
+      description: "{% raw %}Blackbox probe '{{ $labels.target }}' returned an HTTP error status - {{ $value }}{% endraw %}"
+  - alert: BlackboxSslCertificateWillExpireSoon
+    expr: (7 * 24 * 3600) <= (last_over_time(probe_ssl_earliest_cert_expiry[10m]) - time()) < (30 * 24 * 3600)
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: '{% raw %}Blackbox SSL certificate will expire soon (target {{ $labels.target }}){% endraw %}'
+      description: "{% raw %}SSL certificate for blackbox probe '{{ $labels.target }}' expires in {{ $value | humanizeDuration }}{% endraw %}"
+  - alert: BlackboxSslCertificateWillExpireVerySoon
+    expr: 0 <= (last_over_time(probe_ssl_earliest_cert_expiry[10m]) - time()) < (7 * 24 * 3600)
+    for: 0m
+    labels:
+      severity: critical
+    annotations:
+      summary: '{% raw %}Blackbox SSL certificate will expire very soon (target {{ $labels.target }}){% endraw %}'
+      description: "{% raw %}SSL certificate for blackbox probe '{{ $labels.target }}' expires in {{ $value | humanizeDuration }}{% endraw %}"
+  - alert: BlackboxSslCertificateExpired
+    expr: (last_over_time(probe_ssl_earliest_cert_expiry[10m]) - time()) < 0
+    for: 0m
+    labels:
+      severity: critical
+    annotations:
+      summary: '{% raw %}Blackbox SSL certificate expired (target {{ $labels.target }}){% endraw %}'
+      description: "{% raw %}SSL certificate for blackbox probe '{{ $labels.target }}' has expired{% endraw %}"
   - record: node_cpu_system_seconds:record
     expr: (100 * sum by(instance)(increase(node_cpu_seconds_total{mode="system",job="node-exporter"}[60s]))) / (sum by(instance)(increase(node_cpu_seconds_total{job="node-exporter"}[60s])))
   - record: node_cpu_user_seconds:record