charts: add grafana dashboards and deploy as configmap

Author: pinheadmz

docs/logging_monitoring.md

@@ -122,21 +122,7 @@ web-based interface.
 
 #### Dashboards
 
-To view the default metrics in the included default dashboard, upload the dashboard
-JSON file to the Grafana server:
-
-```sh
-curl localhost:3000/api/dashboards/db \
-  -H "Content-Type: application/json" \
-  --data "{\"dashboard\": $(cat resources/configs/grafana/default_dashboard.json)}"
-```
-
-Note the URL in the reply from the server (example):
-
-```sh
-{"folderUid":"","id":2,"slug":"default-warnet-dashboard","status":"success","uid":"fdu0pda1z6a68b","url":"/d/fdu0pda1z6a68b/default-warnet-dashboard","version":1}(
-```
-
-Open the dashboard in your browser (example):
-
-`http://localhost:3000/d/fdu0pda1z6a68b/default-warnet-dashboard`
+Grafana dashboards are described in JSON files. A default Warnet dashboard
+is included and any other json files in the `/resources/charts/grafana-dashboards/files` directory
+will also be deployed to the web server. The Grafana UI itself also has an API
+for creating, exporting, and importing other dashboard files.

resources/charts/grafana-dashboards/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

resources/charts/grafana-dashboards/Chart.yaml

@@ -0,0 +1,24 @@
+apiVersion: v2
+name: grafana-dashboards
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "1.16.0"

resources/configs/grafana/default_dashboard.json renamed to resources/charts/grafana-dashboards/files/default_dashboard.json

@@ -24,7 +24,8 @@
       ],
       "title": "Outbound connections",
       "type": "timeseries",
-      "gridPos": {
+      "gridPos":
+      {
         "h": 8,
         "w": 24,
         "x": 0,
@@ -52,7 +53,8 @@
       ],
       "title": "Inbound connections",
       "type": "timeseries",
-      "gridPos": {
+      "gridPos":
+      {
         "h": 8,
         "w": 24,
         "x": 0,
@@ -80,7 +82,8 @@
       ],
       "title": "Mempool size",
       "type": "timeseries",
-      "gridPos": {
+      "gridPos":
+      {
         "h": 8,
         "w": 24,
         "x": 0,
@@ -108,7 +111,8 @@
       ],
       "title": "Blocks",
       "type": "timeseries",
-      "gridPos": {
+      "gridPos":
+      {
         "h": 8,
         "w": 24,
         "x": 0,

resources/charts/grafana-dashboards/templates/NOTES.txt

@@ -0,0 +1 @@
+Grafana dashboards deployed

resources/charts/grafana-dashboards/templates/configmap.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: grafana-dashboards-config
+data:
+{{- $files := .Files.Glob "files/*.json" }}
+{{- range $path, $file := $files }}
+  {{ base $path }}: |-
+{{ $file | toString | indent 4 }}
+{{- end }}

resources/manifests/grafana_values.yaml

@@ -12,6 +12,25 @@ datasources:
     - name: Loki
       type: loki
       url: http://loki-gateway.warnet-logging:80
+dashboardProviders:
+  dashboardproviders.yaml:
+    apiVersion: 1
+    providers:
+    - name: 'default'
+      orgId: 1
+      folder: ''
+      type: file
+      disableDeletion: false
+      editable: true
+      options:
+        path: /var/lib/grafana/dashboards/default
+extraVolumeMounts:
+  - name: grafana-dashboards-volume
+    mountPath: /var/lib/grafana/dashboards/default
+extraVolumes:
+  - name: grafana-dashboards-volume
+    configMap:
+      name: grafana-dashboards-config
 grafana.ini:
   auth:
     disable_login_form: true

src/warnet/constants.py

@@ -95,5 +95,6 @@
     f"helm upgrade --install --namespace warnet-logging --create-namespace --values {MANIFESTS_DIR}/loki_values.yaml loki grafana/loki --version 5.47.2",
     "helm upgrade --install --namespace warnet-logging promtail grafana/promtail",
     "helm upgrade --install --namespace warnet-logging prometheus prometheus-community/kube-prometheus-stack --namespace warnet-logging --set grafana.enabled=false",
+    f"helm upgrade --install grafana-dashboards {CHARTS_DIR}/grafana-dashboards --namespace warnet-logging",
     f"helm upgrade --install --namespace warnet-logging loki-grafana grafana/grafana --values {MANIFESTS_DIR}/grafana_values.yaml",
 ]

test/logging_test.py

@@ -118,6 +118,10 @@ def get_five_values_for_metric(metric):
         self.wait_for_predicate(lambda: get_five_values_for_metric("blocks"))
         self.wait_for_predicate(lambda: get_five_values_for_metric("txrate"))
 
+        # Verify default dashboard exists
+        dbs = requests.get(f"{GRAFANA_URL}api/search").json()
+        assert dbs[0]["title"] == "Default Warnet Dashboard"
+
 
 if __name__ == "__main__":
     test = LoggingTest()