refactor(grafana): migrate dashboard provisioning to opentofu and support multi-host metrics

k3s/grafana/dashboards/homelab-monitoring.json

@@ -18,7 +18,6 @@
   "editable": true,
   "fiscalYearStartMonth": 0,
   "graphTooltip": 0,
-  "id": 4,
   "links": [],
   "panels": [
     {
@@ -138,7 +137,7 @@
           "editorMode": "code",
           "format": "table",
           "rawQuery": true,
-          "rawSql": "SELECT\n  time AS \"time\",\n  (payload->>'usage')::float AS \"CPU Usage\"\nFROM system_metrics \nWHERE\n  metric_type = 'cpu'\n  AND host = 'server'\nORDER BY time;",
+          "rawSql": "SELECT\n  time AS \"time\",\n  host,\n  (payload->>'usage')::float AS \"CPU Usage\"\nFROM system_metrics \nWHERE\n  metric_type = 'cpu'\nORDER BY time;",
           "refId": "A",
           "sql": {
             "columns": [
@@ -159,6 +158,16 @@
           }
         }
       ],
+      "transformations": [
+        {
+          "id": "partitionByValues",
+          "options": {
+            "fields": [
+              "host"
+            ]
+          }
+        }
+      ],
       "title": "CPU Usage",
       "type": "timeseries"
     },
@@ -250,7 +259,7 @@
           "editorMode": "code",
           "format": "table",
           "rawQuery": true,
-          "rawSql": "SELECT\n  time AS \"time\",\n  CASE\n    WHEN\n      LAG(time) OVER (ORDER BY time) IS NOT NULL\n      AND EXTRACT(EPOCH FROM (time - LAG(time) OVER (ORDER BY time))) > 0\n    THEN\n      (\n        ( (payload->'enp5s0'->>'rx_bytes')::bigint - LAG((payload->'enp5s0'->>'rx_bytes')::bigint) OVER (ORDER BY time) )\n        * 8.0\n      ) / (\n        EXTRACT(EPOCH FROM (time - LAG(time) OVER (ORDER BY time))) * 1000000\n      )\n    ELSE NULL\n  END AS \"Download (Mbps)\",\n  CASE\n    WHEN\n      LAG(time) OVER (ORDER BY time) IS NOT NULL\n      AND EXTRACT(EPOCH FROM (time - LAG(time) OVER (ORDER BY time))) > 0\n    THEN\n      (\n        ( (payload->'enp5s0'->>'tx_bytes')::bigint - LAG((payload->'enp5s0'->>'tx_bytes')::bigint) OVER (ORDER BY time) )\n        * 8.0\n      ) / (\n        EXTRACT(EPOCH FROM (time - LAG(time) OVER (ORDER BY time))) * 1000000\n      )\n    ELSE NULL\n  END AS \"Upload (Mbps)\"\nFROM system_metrics \nWHERE\n  metric_type = 'network'\n  AND host = 'server'\nORDER BY time;",
+          "rawSql": "SELECT\n  time AS \"time\",\n  host,\n  CASE\n    WHEN\n      LAG(time) OVER (PARTITION BY host ORDER BY time) IS NOT NULL\n      AND EXTRACT(EPOCH FROM (time - LAG(time) OVER (PARTITION BY host ORDER BY time))) > 0\n    THEN\n      (\n        ( COALESCE((payload->'enp5s0'->>'rx_bytes')::bigint, (payload->'eno1'->>'rx_bytes')::bigint) - LAG(COALESCE((payload->'enp5s0'->>'rx_bytes')::bigint, (payload->'eno1'->>'rx_bytes')::bigint)) OVER (PARTITION BY host ORDER BY time) )\n        * 8.0\n      ) / (\n        EXTRACT(EPOCH FROM (time - LAG(time) OVER (PARTITION BY host ORDER BY time))) * 1000000\n      )\n    ELSE NULL\n  END AS \"Download (Mbps)\",\n  CASE\n    WHEN\n      LAG(time) OVER (PARTITION BY host ORDER BY time) IS NOT NULL\n      AND EXTRACT(EPOCH FROM (time - LAG(time) OVER (PARTITION BY host ORDER BY time))) > 0\n    THEN\n      (\n        ( COALESCE((payload->'enp5s0'->>'tx_bytes')::bigint, (payload->'eno1'->>'tx_bytes')::bigint) - LAG(COALESCE((payload->'enp5s0'->>'tx_bytes')::bigint, (payload->'eno1'->>'tx_bytes')::bigint)) OVER (PARTITION BY host ORDER BY time) )\n        * 8.0\n      ) / (\n        EXTRACT(EPOCH FROM (time - LAG(time) OVER (PARTITION BY host ORDER BY time))) * 1000000\n      )\n    ELSE NULL\n  END AS \"Upload (Mbps)\"\nFROM system_metrics \nWHERE\n  metric_type = 'network'\nORDER BY time;",
           "refId": "A",
           "sql": {
             "columns": [
@@ -271,6 +280,16 @@
           }
         }
       ],
+      "transformations": [
+        {
+          "id": "partitionByValues",
+          "options": {
+            "fields": [
+              "host"
+            ]
+          }
+        }
+      ],
       "title": "Network Usage",
       "type": "timeseries"
     },
@@ -362,7 +381,7 @@
           "editorMode": "code",
           "format": "table",
           "rawQuery": true,
-          "rawSql": "SELECT\n  time AS \"time\",\n  (payload->>'used_percent')::float AS \"Memory Used %\"\nFROM system_metrics \nWHERE\n  metric_type = 'memory'\n  AND host = 'server'\nORDER BY time;",
+          "rawSql": "SELECT\n  time AS \"time\",\n  host,\n  (payload->>'used_percent')::float AS \"Memory Used %\"\nFROM system_metrics \nWHERE\n  metric_type = 'memory'\nORDER BY time;",
           "refId": "A",
           "sql": {
             "columns": [
@@ -383,6 +402,16 @@
           }
         }
       ],
+      "transformations": [
+        {
+          "id": "partitionByValues",
+          "options": {
+            "fields": [
+              "host"
+            ]
+          }
+        }
+      ],
       "title": "Memory Usage",
       "type": "timeseries"
     },
@@ -478,7 +507,7 @@
           "editorMode": "code",
           "format": "table",
           "rawQuery": true,
-          "rawSql": "SELECT\n  time AS \"time\",\n  (payload->'/'->>'used_percent')::float AS \"Disk Used %\"\nFROM system_metrics \nWHERE\n  metric_type = 'disk'\n  AND host = 'server'\nORDER BY time;",
+          "rawSql": "SELECT\n  time AS \"time\",\n  host,\n  (payload->'/'->>'used_percent')::float AS \"Disk Used %\"\nFROM system_metrics \nWHERE\n  metric_type = 'disk'\nORDER BY time;",
           "refId": "A",
           "sql": {
             "columns": [
@@ -499,6 +528,16 @@
           }
         }
       ],
+      "transformations": [
+        {
+          "id": "partitionByValues",
+          "options": {
+            "fields": [
+              "host"
+            ]
+          }
+        }
+      ],
       "title": "Disk Usage",
       "type": "timeseries"
     },
@@ -589,7 +628,7 @@
           "editorMode": "code",
           "format": "table",
           "rawQuery": true,
-          "rawSql": "SELECT\n  time AS \"time\",\n  (payload->'core_temps'->>'core_0')::int AS \"Core 0\",\n  (payload->'core_temps'->>'core_1')::int AS \"Core 1\",\n  (payload->'core_temps'->>'core_2')::int AS \"Core 2\",\n  (payload->'core_temps'->>'core_3')::int AS \"Core 3\"\nFROM system_metrics \nWHERE\n  metric_type = 'cpu'\n  AND host = 'server'\nORDER BY time;",
+          "rawSql": "SELECT\n  time AS \"time\",\n  host,\n  (payload->'core_temps'->>'core_0')::int AS \"Core 0\",\n  (payload->'core_temps'->>'core_1')::int AS \"Core 1\",\n  (payload->'core_temps'->>'core_2')::int AS \"Core 2\",\n  (payload->'core_temps'->>'core_3')::int AS \"Core 3\"\nFROM system_metrics \nWHERE\n  metric_type = 'cpu'\nORDER BY time;",
           "refId": "A",
           "sql": {
             "columns": [
@@ -610,6 +649,16 @@
           }
         }
       ],
+      "transformations": [
+        {
+          "id": "partitionByValues",
+          "options": {
+            "fields": [
+              "host"
+            ]
+          }
+        }
+      ],
       "title": "CPU Temperatures",
       "type": "timeseries"
     }
@@ -628,6 +677,6 @@
   "timezone": "utc",
   "title": "Homelab Monitoring",
   "uid": "adf2v2l",
-  "version": 19,
+  "version": 0,
   "weekStart": ""
 }

k3s/grafana/dashboards/reading-analytics.json

@@ -18,7 +18,6 @@
   "editable": true,
   "fiscalYearStartMonth": 0,
   "graphTooltip": 0,
-  "id": 5,
   "links": [],
   "panels": [
     {
@@ -469,6 +468,6 @@
   "timezone": "browser",
   "title": "Reading Analytics",
   "uid": "adq2m9x",
-  "version": 11,
+  "version": 0,
   "weekStart": ""
 }

k3s/grafana/values.yaml

@@ -109,7 +109,7 @@ dashboardProviders:
       folder: ''
       type: file
       disableDeletion: false
-      editable: false
+      editable: true
       options:
         path: /var/lib/grafana/dashboards
 
@@ -136,15 +136,10 @@ grafana.ini:
   check_for_updates: false
 
 extraVolumes:
-  - name: grafana-dashboards
-    configMap:
-      name: grafana-dashboards
   - name: tmp
     emptyDir: {}
 
 extraVolumeMounts:
-  - name: grafana-dashboards
-    mountPath: /var/lib/grafana/dashboards
   - name: tmp
     mountPath: /tmp
 

tofu/grafana.tf

@@ -9,3 +9,19 @@ resource "helm_release" "grafana" {
 
   depends_on = [kubernetes_namespace.observability]
 }
+
+resource "grafana_folder" "observability" {
+  title = "Observability"
+}
+
+resource "grafana_dashboard" "homelab_monitoring" {
+  folder      = grafana_folder.observability.id
+  config_json = file("${path.module}/../k3s/grafana/dashboards/homelab-monitoring.json")
+  overwrite   = true
+}
+
+resource "grafana_dashboard" "reading_analytics" {
+  folder      = grafana_folder.observability.id
+  config_json = file("${path.module}/../k3s/grafana/dashboards/reading-analytics.json")
+  overwrite   = true
+}

tofu/providers.tf

@@ -10,6 +10,10 @@ terraform {
       source  = "hashicorp/helm"
       version = "~> 2.13"
     }
+    grafana = {
+      source  = "grafana/grafana"
+      version = "~> 3.25"
+    }
   }
 
   # Step 9: MinIO state backend - uncomment AFTER MinIO pod is confirmed healthy (Step 4)
@@ -37,3 +41,15 @@ provider "helm" {
     config_context = "default"
   }
 }
+
+data "kubernetes_secret" "grafana_admin" {
+  metadata {
+    name      = "grafana-admin-secret"
+    namespace = var.observability_namespace
+  }
+}
+
+provider "grafana" {
+  url  = "http://localhost:30000"
+  auth = "${data.kubernetes_secret.grafana_admin.data["admin-user"]}:${data.kubernetes_secret.grafana_admin.data["admin-password"]}"
+}