Add storage for Prometheus using a PersistentVolumeClaim

Prior to this change, Prometheus' timeseries data would
be lost whenever the chart was upgraded, the Prometheus Pod
was rescheduled, or restarted due to a configuration change.

Tested with a blank storageClassName with KinD, and the local
path provisioner. After creating load with hey, and restarting
Prometheus, the invocation metrics remained available.

The name of the pvc can also be set, however this is not
documented in values.yaml, this may be required for multiple
installations of OpenFaaS within the same cluster for testing
purposes.

Signed-off-by: Alex Ellis (OpenFaaS Ltd) <[email protected]>

Author: alexellis

chart/openfaas/README.md

@@ -680,8 +680,7 @@ For legacy scaling in OpenFaaS Community Edition.
 | ----------------------- | ----------------------------------    | ---------------------------------------------------------- |
 | `alertmanager.create` | Create the AlertManager component | `true` |
 | `alertmanager.image` | Container image used for alertmanager | See [values.yaml](./values.yaml) |
-| `alertmanager.resources` | Resource limits and requests for alertmanager pods | See [values.yaml](./values.
-
+| `alertmanager.resources` | Resource limits and requests for alertmanager pods | See [values.yaml](./values.yaml) |
 
 ### Prometheus (built-in, for autoscaling and metrics)
 
@@ -693,3 +692,8 @@ For legacy scaling in OpenFaaS Community Edition.
 | `prometheus.retention.size` | The maximum number of bytes of storage blocks to retain. Units supported: B, KB, MB, GB, TB, PB, EB. 0 meaning disabled. See: [Prometheus storage](https://prometheus.io/docs/prometheus/latest/storage/#operational-aspects)| `0` |
 | `prometheus.resources` | Resource limits and requests for prometheus containers | See [values.yaml](./values.yaml) |
 | `prometheus.recordingRules` | Custom recording rules for autoscaling. | `[]` |
+| `prometheus.pvc` | Persistent volume claim for Prometheus used so that metrics survive restarts of the Pod and upgrades of the chart | `{}` |
+| `prometheus.pvc.enabled` | Enable persistent volume claim for Prometheus | `false` |
+| `prometheus.pvc.storageClassName` | Storage class for Prometheus PVC, set to `""` for the default/standard class to be picked | `""` |
+| `prometheus.pvc.size` | Size of the Prometheus PVC, 60-100Gi may be a better fit for a busy production environment | `10Gi` |
+| `prometheus.pvc.name` | Name of the Prometheus PVC, required for multiple installations within the same cluster | `""` |

chart/openfaas/templates/prometheus-dep.yaml

@@ -14,6 +14,8 @@ metadata:
   namespace: {{ .Release.Namespace | quote }}
 spec:
   replicas: 1
+  strategy:
+    type: Recreate
   selector:
     matchLabels:
       app: prometheus
@@ -119,8 +121,20 @@ spec:
                 path: prometheus-rules.yml
                 mode: 0644
 {{- end }}
+
+{{- if and .Values.prometheus.pvc.enabled .Values.openfaasPro }}
+        - name: prom-data
+          persistentVolumeClaim:
+  {{- if .Values.prometheus.pvc.name }}
+            claimName: {{.Values.prometheus.pvc.name}}
+  {{- else }}
+            claimName: prometheus-data
+  {{- end }}
+{{- else }}
         - name: prom-data
           emptyDir: {}
+{{- end }}
+
     {{- with .Values.nodeSelector }}
       nodeSelector:
 {{ toYaml . | indent 8 }}

chart/openfaas/templates/prometheus-pvc.yaml

@@ -0,0 +1,30 @@
+{{- $functionNs := default .Release.Namespace .Values.functionNamespace }}
+{{- if .Values.openfaasPro }}
+{{- if and .Values.prometheus.create .Values.prometheus.pvc.enabled }}
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  labels:
+    app: {{ template "openfaas.name" . }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+    component: prometheus
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+{{- if .Values.prometheus.pvc.name }}
+  name: {{.Values.prometheus.pvc.name}}
+{{- else }}
+  name: prometheus-data
+{{- end }}
+  namespace: {{ .Release.Namespace | quote }}
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: {{ .Values.prometheus.pvc.size | quote }}
+  {{- with .Values.prometheus.pvc.storageClassName }}
+  storageClassName: {{ . | quote }}
+  {{- end }}
+{{- end }}
+{{- end }}

chart/openfaas/values.yaml

@@ -359,6 +359,11 @@ iam:
     url: https://kubernetes.default.svc.cluster.local
     tokenExpiry: 2h
 
+## Prometheus is required for metrics and autoscaling
+##
+## It is bundled into OpenFaaS to be used only as an internal component
+## if you wish to retain the metrics for a longer period, you should
+## scrape this instance from an external Prometheus server
 prometheus:
   image: prom/prometheus:v2.54.0
   create: true
@@ -371,22 +376,19 @@ prometheus:
   annotations: {}
   recordingRules: []
 
-alertmanager:
-  image: prom/alertmanager:v0.27.0
-  create: true
-  resources:
-    requests:
-      memory: "25Mi"
-      cpu: "50m"
-    limits:
-      memory: "50Mi"
-
-stan:
-  # Image used for the NATS Streaming when using the deprecated 
-  # support in the Community Edition (CE)
-  image: nats-streaming:0.25.6
-
-# NATS (required for async)
+  # Set to true to enable persistent storage for the Prometheus Pod
+  # otherwise, the data will be lost when the Pod is restarted
+  pvc:
+    enabled: false
+    # You may want to set this higher for production, or lower for development/staging.
+    size: 30Gi
+    # Leave the storageClassName blank for the default storage class
+    # using the string "default" does not necessarily mean the default
+    # storage class
+    storageClassName:
+
+## NATS is used for OpenFaaS Pro and is required for:
+## asynchronous invocations, billing & auditing webhooks
 nats:
   channel: "faas-request"
   # Stream replication is set to 1 by default. This is only recommended for development.
@@ -405,6 +407,22 @@ nats:
       memory: "120Mi"
       cpu: "100m"
 
+## alertmanager is only used for OpenFaaS CE
+alertmanager:
+  image: prom/alertmanager:v0.27.0
+  create: true
+  resources:
+    requests:
+      memory: "25Mi"
+      cpu: "50m"
+    limits:
+      memory: "50Mi"
+
+## stan is only used for OpenFaaS CE and will be removed in
+## a fture release, having already been deprecated by the NATS team
+stan:
+  image: nats-streaming:0.25.6
+
 # ingress configuration
 ingress:
   enabled: false