Added blackbox probe for OOD + alert

Author: wtripp180901

ansible/roles/kube_prometheus_stack/defaults/main/main.yml

@@ -20,6 +20,8 @@ kube_prometheus_stack_wait_timeout: 5m
 kube_prometheus_stack_metrics_image_tag: v2.12.0
 kube_prometheus_stack_patch_image_tag: v20221220-controller-v1.5.1-58-g787ea74b6
 
+kube_prometheus_stack_blackbox_exporter_release_name: blackbox-exporter
+
 control_ip: "{{ hostvars[groups['control'].0].ansible_host }}"
 
 grafana_auth_anonymous: false
@@ -64,6 +66,7 @@ prometheus_external_labels:
 prometheus_scrape_configs: []
 
 prometheus_extra_rules: []
+prometheus_extra_rules_default: []
 
 prometheus_rules:
   appliance-rules:

ansible/roles/kube_prometheus_stack/tasks/main.yml

@@ -176,6 +176,18 @@
   ansible.builtin.import_role:
     name: grafana-dashboards
 
+- name: Install blackbox exporter helm chart
+  kubernetes.core.helm:
+    chart_ref: prometheus-blackbox-exporter
+    chart_repo_url: https://prometheus-community.github.io/helm-charts
+    chart_version: 9.0.1
+    release_name: "{{ kube_prometheus_stack_blackbox_exporter_release_name }}"
+    release_namespace: "{{ kube_prometheus_stack_release_namespace }}"
+    release_values:
+      nodeSelector:
+        clusterrole: "server"
+    wait: yes
+
 - name: Install kube-prometheus-stack on target Kubernetes cluster
   kubernetes.core.helm:
     chart_ref: "{{ kube_prometheus_stack_chart_name }}"

environments/common/inventory/group_vars/all/monitoring.yml

@@ -2,5 +2,6 @@ kube_prometheus_stack_chart_version: 59.1.0
 kube_prometheus_stack_release_namespace: monitoring-system
 kube_prometheus_stack_release_name: kube-prometheus-stack
 kube_prometheus_stack_wait_timeout: 5m
+kube_prometheus_stack_blackbox_exporter_release_name: blackbox-exporter
 
 # See prometheus.yml, grafana.yml and alertmanager.yml for config of individual monitoring services

environments/common/inventory/group_vars/all/openondemand.yml

@@ -182,6 +182,28 @@ openondemand_scrape_configs:
       labels:
         environment: "{{ appliances_environment_name }}"
         service: "openondemand"
+  - job_name: "blackbox-probes"
+    metrics_path: /probe
+    params:
+      module: [http_2xx]
+    static_configs:
+      - targets:
+        - "https://{{ openondemand_address }}"
+    relabel_configs:
+    - source_labels: [__address__]
+      target_label: __param_target
+    - source_labels: [__param_target]
+      target_label: target
+    - target_label: __address__ 
+      replacement: "{{ kube_prometheus_stack_blackbox_exporter_release_name }}-prometheus-blackbox-exporter:9115"
+openondemand_extra_rules:
+- alert: OnDemandProbeFailed
+  annotations:
+    description: '{% raw %}Could not establish secure connection to OOD server at {{ $labels.target }}{% endraw %}'
+    summary: 'Could not establish a secure connection to an Open OnDemand server'
+  expr: "probe_success{target='https://{{ openondemand_address }}'} < 1\n"
+  labels:
+    severity: warning
 
 openondemand_dashboard:
   - dashboard_id: 13465

environments/common/inventory/group_vars/all/prometheus.yml

@@ -20,7 +20,8 @@ prometheus_scrape_configs_default:
     replacement:   '${1}'
 
 prometheus_scrape_configs: "{{ prometheus_scrape_configs_default + (openondemand_scrape_configs if groups['openondemand'] | count > 0 else []) }}"
-prometheus_extra_rules:
+prometheus_extra_rules: "{{ prometheus_extra_rules_default + (openondemand_extra_rules if groups['openondemand'] | count > 0 else []) }}"
+prometheus_extra_rules_default:
   - alert: SlurmNodeDown
     annotations:
       description: '{% raw %}{{ $value }} Slurm nodes are in down status.{% endraw %}'