Send Prometheus metrics to Grafana Cloud

Author: ndr-ds

kubernetes/linera-validator/helmfile.yaml

@@ -1,3 +1,11 @@
+environments:
+  default:
+    values:
+      - writeToGrafanaCloud: {{ env "LINERA_WRITE_TO_GRAFANA_CLOUD" | default "false" }}
+        validatorLabel: {{ env "LINERA_VALIDATOR_LABEL" | default (printf "local-%s" (env "USER")) }}
+
+---
+
 repositories:
   - name: scylla
     url: https://scylla-operator-charts.storage.googleapis.com/stable
@@ -16,22 +24,15 @@ releases:
     needs:
       - scylla/scylla
     values:
-      - {{ env "LINERA_HELMFILE_VALUES_LINERA_CORE" | default "values-local.yaml" }}
+      {{ if .Values.writeToGrafanaCloud -}}
+      - grafanaCloudUsername: {{ fetchSecretValue (env "LINERA_GRAFANA_CLOUD_USERNAME_SECRET") | quote }}
+        grafanaCloudAPIToken: {{ fetchSecretValue (env "LINERA_GRAFANA_CLOUD_API_TOKEN_SECRET") | quote }}
+      {{- end }}
+      - writeToGrafanaCloud: {{ .Values.writeToGrafanaCloud }}
+      - {{ env "LINERA_HELMFILE_VALUES_LINERA_CORE" | default "values-local.yaml.gotmpl" }}
     set:
       - name: installCRDs
         value: "true"
-      - name: validator.serverConfig
-        value: {{ env "LINERA_HELMFILE_SET_SERVER_CONFIG" | default "working/server_1.json" }}
-      - name: validator.genesisConfig
-        value: {{ env "LINERA_HELMFILE_SET_GENESIS_CONFIG" | default "working/genesis.json" }}
-      - name: numShards
-        value: {{ env "LINERA_HELMFILE_SET_NUM_SHARDS" | default 10 }}
-      - name: lineraImage
-        value: {{ env "LINERA_HELMFILE_LINERA_IMAGE" | default "linera:latest" }}
-      - name: staticIpGcpName
-        value: {{ env "LINERA_HELMFILE_STATIC_IP_GCP_NAME" | default "" }}
-      - name: validatorDomainName
-        value: {{ env "LINERA_HELMFILE_VALIDATOR_DOMAIN_NAME" | default "" }}
   - name: scylla
     version: v1.13.0
     namespace: scylla

kubernetes/linera-validator/templates/grafana-cloud-auth-secret.yaml

@@ -0,0 +1,10 @@
+{{- if .Values.writeToGrafanaCloud }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: grafana-cloud-auth-secret
+type: kubernetes.io/basic-auth
+stringData:
+  username: {{ .Values.grafanaCloudUsername | quote }}
+  password: {{ .Values.grafanaCloudAPIToken | quote }}
+{{- end }}

kubernetes/linera-validator/templates/scylla-manager-service-monitor.yaml

@@ -1,3 +1,4 @@
+{{- if not .Values.writeToGrafanaCloud }}
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
@@ -15,3 +16,4 @@ spec:
           targetLabel: instance
           regex: (.*)
           replacement: ${1}
+{{- end }}

kubernetes/linera-validator/templates/scylla-service-monitor.yaml

@@ -1,3 +1,4 @@
+{{- if not .Values.writeToGrafanaCloud }}
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
@@ -31,3 +32,4 @@ spec:
           regex: (.*)
           replacement: ${1}
           action: replace
+{{- end }}

kubernetes/linera-validator/values-local.yaml renamed to kubernetes/linera-validator/values-local.yaml.gotmpl

@@ -1,12 +1,12 @@
 # Values for charts linera-validator for local validators.
 
 # Linera
-lineraImage: "" # Is set by helmfile.
+lineraImage: {{ env "LINERA_HELMFILE_LINERA_IMAGE" | default "linera:latest" }}
 lineraImagePullPolicy: Never
 logLevel: "debug"
 proxyPort: 19100
 metricsPort: 21100
-numShards: 10
+numShards: {{ env "LINERA_HELMFILE_SET_NUM_SHARDS" | default 10 }}
 
 # Loki
 loki-stack:
@@ -38,8 +38,29 @@ kube-prometheus-stack:
       size: 1Gi
     plugins:
       - grafana-piechart-panel
+  {{- if .Values.writeToGrafanaCloud }}
+  kubeStateMetrics:
+    enabled: false
+  {{- end }}
   prometheus:
     prometheusSpec:
+      {{- if .Values.writeToGrafanaCloud }}
+      remoteWrite:
+        - url: https://prometheus-prod-13-prod-us-east-0.grafana.net/api/prom/push
+          basicAuth:
+            username:
+              name: grafana-cloud-auth-secret
+              key: username
+            password:
+              name: grafana-cloud-auth-secret
+              key: password
+          writeRelabelConfigs:
+            - source_labels: [__name__]
+              regex: 'linera_.*'
+              action: keep
+      externalLabels:
+        validator: {{ .Values.validatorLabel }}
+      {{- end }}
       retention: 2d
       retentionSize: 1GB
       storageSpec:
@@ -56,6 +77,7 @@ kube-prometheus-stack:
       # This prevents from adding any Helm label to serviceMonitorSelector if
       # above is empty.
       serviceMonitorSelectorNilUsesHelmValues: false
+      {{- if not .Values.writeToGrafanaCloud }}
       # Relabelings needed for Scylla dashboards
       additionalScrapeConfigs:
         - job_name: scylla
@@ -95,11 +117,12 @@ kube-prometheus-stack:
               regex: '([0-9]+\.[0-9]+)(\.?[0-9]*).*'
               replacement: '$1$2'
               target_label: svr
+      {{- end }}
 
 # Environment
 environment: "kind"
 
 # Validator
 validator:
-  serverConfig: "" # Is set by helmfile.
-  genesisConfig: "" # Is set by helmfile.
+  serverConfig: {{ env "LINERA_HELMFILE_SET_SERVER_CONFIG" | default "working/server_1.json" }}
+  genesisConfig: {{ env "LINERA_HELMFILE_SET_GENESIS_CONFIG" | default "working/genesis.json" }}