fixed openhpc dashboard

Author: wtripp180901

ansible/roles/grafana-dashboards/files/openhpc-slurm.json

@@ -711,7 +711,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "100 - (100 * node_memory_MemAvailable_bytes{job=~\"node\", instance=~\"$instance\"} / node_memory_MemTotal_bytes{job=~\"node\", instance=~\"$instance\"})",
+          "expr": "100 - (100 * node_memory_MemAvailable_bytes{job=~\"node-exporter\", instance=~\"$instance\"} / node_memory_MemTotal_bytes{job=~\"node-exporter\", instance=~\"$instance\"})",
           "format": "time_series",
           "interval": "",
           "intervalFactor": 1,
@@ -818,7 +818,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "(100 * sum by(instance)(increase(node_cpu_seconds_total{mode=\"iowait\",job=~\"node_fast\"}[1s]))) / (sum by(instance)(increase(node_cpu_seconds_total{job=~\"node_fast\"}[1s])))",
+          "expr": "(100 * sum by(instance)(increase(node_cpu_seconds_total{mode=\"iowait\",job=~\"node-exporter_fast\"}[1s]))) / (sum by(instance)(increase(node_cpu_seconds_total{job=~\"node-exporter_fast\"}[1s])))",
           "hide": true,
           "interval": "",
           "legendFormat": "{{ instance }}",
@@ -1489,7 +1489,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "increase(node_infiniband_port_transmit_wait_total{job=\"node\",instance=~\"$instance\"}[1m]) / increase(node_infiniband_port_packets_transmitted_total{job=\"node\",instance=~\"$instance\"}[1m])",
+          "expr": "increase(node_infiniband_port_transmit_wait_total{job=\"node-exporter\",instance=~\"$instance\"}[1m]) / increase(node_infiniband_port_packets_transmitted_total{job=\"node-exporter\",instance=~\"$instance\"}[1m])",
           "hide": false,
           "interval": "",
           "legendFormat": "{{ instance }} {{ device }} {{  port }}",
@@ -1950,15 +1950,15 @@
         "allValue": null,
         "current": {},
         "datasource": "${DS_PROMETHEUS}",
-        "definition": "label_values(node_cpu_seconds_total{job=~\"node\"}, env)",
+        "definition": "label_values(node_cpu_seconds_total{job=~\"node-exporter\"}, cluster_env)",
         "error": null,
         "hide": 0,
         "includeAll": true,
         "label": "Environment",
         "multi": true,
-        "name": "env",
+        "name": "cluster_env",
         "options": [],
-        "query": "label_values(node_cpu_seconds_total{job=~\"node\"}, env)",
+        "query": "label_values(node_cpu_seconds_total{job=~\"node-exporter\"}, cluster_env)",
         "refresh": 1,
         "regex": "",
         "skipUrlSync": false,
@@ -1994,15 +1994,15 @@
         "allValue": null,
         "current": {},
         "datasource": "${DS_PROMETHEUS}",
-        "definition": "label_values(node_cpu_seconds_total{job=~\"node\", env=~\"$env\", instance=~\"$host_filter\"}, instance)",
+        "definition": "label_values(node_cpu_seconds_total{job=~\"node-exporter\", cluster_env=~\"$cluster_env\", instance=~\"$host_filter\"}, instance)",
         "error": null,
         "hide": 0,
         "includeAll": true,
         "label": null,
         "multi": true,
         "name": "instance",
         "options": [],
-        "query": "label_values(node_cpu_seconds_total{job=~\"node\", env=~\"$env\", instance=~\"$host_filter\"}, instance)",
+        "query": "label_values(node_cpu_seconds_total{job=~\"node-exporter\", cluster_env=~\"$cluster_env\", instance=~\"$host_filter\"}, instance)",
         "refresh": 1,
         "regex": "",
         "skipUrlSync": false,
@@ -2017,15 +2017,15 @@
         "allValue": null,
         "current": {},
         "datasource": "${DS_PROMETHEUS}",
-        "definition": "label_values(node_infiniband_info{job=~\"node\"}, device)",
+        "definition": "label_values(node_infiniband_info{job=~\"node-exporter\"}, device)",
         "error": null,
         "hide": 0,
         "includeAll": true,
         "label": "infiniband device",
         "multi": true,
         "name": "device",
         "options": [],
-        "query": "label_values(node_infiniband_info{job=~\"node\"}, device)",
+        "query": "label_values(node_infiniband_info{job=~\"node-exporter\"}, device)",
         "refresh": 1,
         "regex": "",
         "skipUrlSync": false,

ansible/roles/kube_prometheus_stack/defaults/main/helm.yml

@@ -172,6 +172,19 @@ kube_prometheus_stack_release_defaults:
   prometheus-node-exporter:
     image:
       tag: "{{ node_exporter_image_tag }}"
+    prometheus:
+      monitor:
+        relabelings:
+          # relabels instances to hostnames with suffixes stripped
+          - sourceLabels: [__meta_kubernetes_pod_node_name]
+            separator: ;
+            regex: ([^.]+).*
+            targetLabel: instance
+            replacement: $1
+            action: replace
+        metricRelabelings:
+          - targetLabel: cluster_env
+            replacement: ungrouped
 
 kube_prometheus_stack_release_overrides: {}