fixed blackbox probes failing for ood

Author: wtripp180901

ansible/roles/kube_prometheus_stack/tasks/main.yml

@@ -113,6 +113,35 @@
                 - port: 9301
                   name: ood-exporter
                   protocol: TCP
+    
+    - name: Creating headless service for OOD server 
+      kubernetes.core.k8s:
+        namespace: "{{ kube_prometheus_stack_release_namespace }}"
+        definition:
+          kind: Service
+          metadata:
+            name: ondemand
+          spec:
+            clusterIP: None
+            ports:
+            - name: https
+              port: 443
+              protocol: TCP
+
+    - name: Binding OOD server service to host
+      kubernetes.core.k8s:
+        namespace: "{{ kube_prometheus_stack_release_namespace }}"
+        definition:
+          kind: Endpoints
+          metadata:
+            name: ondemand
+          subsets:
+            - addresses:
+                - ip: "{{ openondemand_ip }}"
+              ports:
+                - port: 443
+                  name: https
+                  protocol: TCP
 
 - name: Creating headless service for slurm exporter
   kubernetes.core.k8s:
@@ -177,6 +206,7 @@
     name: grafana-dashboards
 
 - name: Install blackbox exporter helm chart
+  no_log: true
   kubernetes.core.helm:
     chart_ref: prometheus-blackbox-exporter
     chart_repo_url: https://prometheus-community.github.io/helm-charts
@@ -186,6 +216,15 @@
     release_values:
       nodeSelector:
         clusterrole: "server"
+      config:
+        modules:
+          http_2xx:
+            http:
+              tls_config:
+                insecure_skip_verify: true
+              basic_auth:
+                username: "testuser"
+                password: "{{ vault_testuser_password }}"
     wait: yes
 
 - name: Install kube-prometheus-stack on target Kubernetes cluster

environments/common/inventory/group_vars/all/openondemand.yml

@@ -188,7 +188,7 @@ openondemand_scrape_configs:
       module: [http_2xx]
     static_configs:
       - targets:
-        - "https://{{ openondemand_address }}"
+        - ondemand.monitoring-system
     relabel_configs:
     - source_labels: [__address__]
       target_label: __param_target
@@ -201,7 +201,7 @@ openondemand_extra_rules:
   annotations:
     description: '{% raw %}Could not establish secure connection to OOD server at {{ $labels.target }}{% endraw %}'
     summary: 'Could not establish a secure connection to an Open OnDemand server'
-  expr: "probe_success{target='https://{{ openondemand_address }}'} < 1\n"
+  expr: "probe_success{target='ondemand.monitoring-system'} < 1\n"
   labels:
     severity: warning