refactored to make easier to extend with new probes

Author: wtripp180901

ansible/roles/kube_prometheus_stack/defaults/main/main.yml

@@ -24,6 +24,8 @@ kube_prometheus_stack_blackbox_exporter_release_version: 9.0.1
 kube_prometheus_stack_blackbox_exporter_image_tag: v0.25.0
 kube_prometheus_stack_blackbox_exporter_release_name: blackbox-exporter
 
+kube_prometheus_stack_blackbox_modules: {}
+
 control_ip: "{{ hostvars[groups['control'].0].ansible_host }}"
 
 grafana_auth_anonymous: false

ansible/roles/kube_prometheus_stack/tasks/main.yml

@@ -177,7 +177,7 @@
     name: grafana-dashboards
 
 - name: Install blackbox exporter helm chart
-  no_log: true
+  no_log: true # may expose testuser password
   kubernetes.core.helm:
     chart_ref: prometheus-blackbox-exporter
     chart_repo_url: https://prometheus-community.github.io/helm-charts
@@ -188,14 +188,7 @@
       nodeSelector:
         clusterrole: "server"
       config:
-        modules:
-          http_2xx:
-            http:
-              tls_config:
-                insecure_skip_verify: true
-              basic_auth:
-                username: "testuser"
-                password: "{{ vault_testuser_password }}"
+        modules: "{{ kube_prometheus_stack_blackbox_modules }}"
       configReloader:
         image:
           tag: "{{ kube_prometheus_stack_app_version }}" # keeps consistent with pre-pulled image for kube-prometheus-stack

environments/common/inventory/group_vars/all/blackbox_exporter.yml

@@ -0,0 +1,7 @@
+# Note: underlying helm chart is configured with a default module 'http_2xx', defining modules with this name here will merge the module's values with the existing module,
+# see https://github.com/prometheus-community/helm-charts/blob/main/charts/prometheus-blackbox-exporter/values.yaml#L163
+kube_prometheus_stack_blackbox_modules_defaults: {}
+
+kube_prometheus_stack_blackbox_modules: "{{ kube_prometheus_stack_blackbox_modules_defaults if ( groups['openondemand'] | count == 0 ) else ( kube_prometheus_stack_blackbox_modules_defaults | combine(openondemand_blackbox_modules) ) }}"
+
+# See prometheus_scrape_configs in prometheus.yml to add additional scrape jobs to probe services

environments/common/inventory/group_vars/all/openondemand.yml

@@ -182,10 +182,10 @@ openondemand_scrape_configs:
       labels:
         environment: "{{ appliances_environment_name }}"
         service: "openondemand"
-  - job_name: "blackbox-probes"
+  - job_name: "ood-blackbox-probe"
     metrics_path: /probe
     params:
-      module: [http_2xx]
+      module: [ood_http_2xx]
     static_configs:
       - targets:
         - "{{ openondemand_address }}"
@@ -197,6 +197,20 @@ openondemand_scrape_configs:
     - target_label: __address__ 
       replacement: "{{ kube_prometheus_stack_blackbox_exporter_release_name }}-prometheus-blackbox-exporter:9115"
 
+openondemand_blackbox_modules:
+  ood_http_2xx:
+    prober: http
+    timeout: 5s
+    http:
+      valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
+      follow_redirects: true
+      preferred_ip_protocol: "ip4"
+      tls_config:
+        insecure_skip_verify: true
+      basic_auth:
+        username: "testuser"
+        password: "{{ vault_testuser_password }}"
+
 openondemand_dashboard:
   - dashboard_id: 13465
     replacements: