SIMSBIOHUB-850: Update backup schedules and retention

infrastructure/crunchy-db/Chart.yaml

@@ -1,11 +1,7 @@
 apiVersion: v2
-name: biohub-platform-crunchy-db
-description: A Helm chart for Biohub Crunchy Postgres Cluster (DEV/TEST/PROD only)
+appVersion: 5.5.1
+description: A Helm chart for BioHub Crunchy Postgres Cluster (DEV/TEST/PROD only), based on the BCGov Crunchy Postgres Helm chart.
+name: crunchy-postgres
 type: application
 version: 0.1.0
-appVersion: "1.0.0"
-
-dependencies:
-  - name: crunchy-postgres
-    version: 0.6.6
-    repository: https://bcgov.github.io/crunchy-postgres/
+appVersion: "1.0.0"

infrastructure/crunchy-db/templates/PostgresCluster.yaml

@@ -0,0 +1,213 @@
+apiVersion: postgres-operator.crunchydata.com/v1beta1
+kind: PostgresCluster
+metadata:
+  name: {{ template "crunchy-postgres.fullname" . }}
+  labels: {{ include "crunchy-postgres.labels" . | nindent 4 }}
+spec:
+  openshift: {{ .Values.openshift }}
+  metadata:
+    labels: {{ include "crunchy-postgres.labels" . | nindent 6 }}
+  {{ if .Values.crunchyImage }}  
+  image: {{ .Values.crunchyImage }}
+  {{ end }}
+  imagePullPolicy: {{.Values.imagePullPolicy}}
+  postgresVersion: {{ .Values.postgresVersion }}
+  {{ if .Values.postGISVersion }}
+  postGISVersion: {{ .Values.postGISVersion | quote }}
+  {{ end }}
+  postgresVersion: {{ .Values.postgresVersion }}
+
+  {{ if .Values.pgmonitor.enabled }}
+
+  monitoring:
+    pgmonitor:
+      # this stuff is for the "exporter" container in the "postgres-cluster-ha" set of pods
+      exporter:
+        {{ if .Values.pgmonitor.exporter.image}}
+        image: {{ .Values.pgmonitor.exporter.image}}
+        {{ end }}
+        resources:
+          requests:
+            cpu: {{ .Values.pgmonitor.exporter.requests.cpu }}
+            memory: {{ .Values.pgmonitor.exporter.requests.memory }}
+          {{ if .Values.pgmonitor.exporter.limits }}
+          limits:
+            cpu: {{ .Values.pgmonitor.exporter.limits.cpu }}
+            memory: {{ .Values.pgmonitor.exporter.limits.memory }}
+          {{ end }}
+
+  {{ end }}
+
+  instances:
+    - name: {{ .Values.instances.name }}
+      replicas: {{ .Values.instances.replicas }}
+      resources:
+        requests:
+          cpu: {{ .Values.instances.requests.cpu }}
+          memory: {{ .Values.instances.requests.memory }}
+        {{ if .Values.instances.limits }}
+        limits:
+          cpu: {{ .Values.instances.limits.cpu }}
+          memory: {{ .Values.instances.limits.memory }}
+        {{ end }}
+      sidecars:
+        replicaCertCopy:
+          resources:
+            requests:
+              cpu: {{ .Values.instances.replicaCertCopy.requests.cpu }}
+              memory: {{ .Values.instances.replicaCertCopy.requests.memory }}
+            {{ if .Values.instances.replicaCertCopy.limits }}
+            limits:
+              cpu: {{ .Values.instances.replicaCertCopy.limits.cpu }}
+              memory: {{ .Values.instances.replicaCertCopy.limits.memory }}
+            {{ end }}
+      dataVolumeClaimSpec:
+        accessModes:
+          - "ReadWriteOnce"
+        resources:
+          requests:
+            storage: {{ .Values.instances.dataVolumeClaimSpec.storage }}
+        storageClassName: {{ .Values.instances.dataVolumeClaimSpec.storageClassName }}
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 1
+              podAffinityTerm:
+                topologyKey: topology.kubernetes.io/zone
+                labelSelector:
+                  matchLabels:
+                    postgres-operator.crunchydata.com/cluster:
+                      {{ template "crunchy-postgres.fullname" . }}
+                    postgres-operator.crunchydata.com/instance-set: {{ .Values.instances.name }}-ha
+
+  users:
+    - name: {{ template "crunchy-postgres.fullname" . }}
+      databases:
+        - {{ template "crunchy-postgres.fullname" . }}
+      options: "CREATEROLE"
+    - name: postgres
+      databases:
+        - {{ template "crunchy-postgres.fullname" . }}
+
+  {{ if .Values.dataSource.enabled }}
+  dataSource:
+    pgbackrest:
+      configuration:
+        - secret:
+            name: {{ .Values.dataSource.secretName }}
+      global:
+        {{ .Values.dataSource.repo.name }}-path: {{ .Values.dataSource.repo.path | quote }}
+      repo:
+        name: {{ .Values.dataSource.repo.name }}
+        s3:
+          bucket: {{ .Values.dataSource.repo.s3.bucket }}
+          endpoint: {{ .Values.dataSource.repo.s3.endpoint }}
+          region: {{ .Values.dataSource.repo.s3.region }}
+      stanza: {{ .Values.dataSource.stanza }}
+  {{ end }}
+
+  backups:
+    pgbackrest:
+      {{ if .Values.pgBackRest.image }}
+      image: {{ .Values.pgBackRest.image }}
+      {{ end }}
+      configuration:
+        - secret:
+            name: {{ .Values.pgBackRest.s3.s3Secret }}
+      global:
+        {{- range .Values.pgBackRest.repos }}
+        {{ .name }}-path: {{ .path | quote }}
+        {{ .name }}-retention-full: {{ .retention | quote }}
+        {{ .name }}-retention-full-type: {{ .retentionFullType }}
+        {{ .name }}-s3-uri-style: {{ $.Values.pgBackRest.s3.s3UriStyle }}
+        {{- end }}
+      repos:
+        {{- range .Values.pgBackRest.repos }}
+        - name: {{ .name }}
+          schedules:
+            full: {{ .fullSchedule | quote }}
+          s3:
+            bucket: {{ $.Values.pgBackRest.s3.bucket }}
+            endpoint: {{ $.Values.pgBackRest.s3.endpoint }}
+            region: {{ $.Values.pgBackRest.s3.region }}
+        {{- end }}
+      # this stuff is for the "pgbackrest" container (the only non-init container) in the "postgres-crunchy-repo-host" pod
+      repoHost:
+        resources:
+          requests:
+            cpu: {{ .Values.pgBackRest.repoHost.requests.cpu }}
+            memory: {{ .Values.pgBackRest.repoHost.requests.memory }}
+          {{- if .Values.pgBackRest.repoHost.limits }}
+          limits:
+            cpu: {{ .Values.pgBackRest.repoHost.limits.cpu }}
+            memory: {{ .Values.pgBackRest.repoHost.limits.memory }}
+          {{- end }}
+      sidecars:
+        # this stuff is for the "pgbackrest" container in the "postgres-crunchy-ha" set of pods
+        pgbackrest:
+          resources:
+            requests:
+              cpu: {{ .Values.pgBackRest.sidecars.requests.cpu }}
+              memory: {{ .Values.pgBackRest.sidecars.requests.memory }}
+            {{- if .Values.pgBackRest.sidecars.limits }}
+            limits:
+              cpu: {{ .Values.pgBackRest.sidecars.limits.cpu }}
+              memory: {{ .Values.pgBackRest.sidecars.limits.memory }}
+            {{- end }}
+        pgbackrestConfig:
+          resources:
+            requests:
+              cpu: {{ .Values.pgBackRest.sidecars.requests.cpu }}
+              memory: {{ .Values.pgBackRest.sidecars.requests.memory }}
+            {{- if .Values.pgBackRest.sidecars.limits }}
+            limits:
+              cpu: {{ .Values.pgBackRest.sidecars.limits.cpu }}
+              memory: {{ .Values.pgBackRest.sidecars.limits.memory }}
+            {{- end }}
+  standby:
+    enabled: {{ .Values.standby.enabled }}
+    repoName: {{ .Values.standby.repoName }}
+
+  patroni:
+    dynamicConfiguration:
+      postgresql:
+        pg_hba:
+          - {{ .Values.patroni.postgresql.pg_hba}}
+        parameters:
+          shared_buffers: {{ .Values.patroni.postgresql.parameters.shared_buffers }}
+          wal_buffers: {{ .Values.patroni.postgresql.parameters.wal_buffers }}
+          min_wal_size: {{ .Values.patroni.postgresql.parameters.min_wal_size }}
+          max_wal_size: {{ .Values.patroni.postgresql.parameters.max_wal_size }}
+          max_slot_wal_keep_size:  {{ .Values.patroni.postgresql.parameters.max_slot_wal_keep_size }}
+
+  proxy:
+    pgBouncer:
+      config:
+        global:
+          client_tls_sslmode: disable
+      {{ if .Values.proxy.pgBouncer.image }}
+      image: {{ .Values.proxy.pgBouncer.image }}
+      {{ end }}
+      replicas: {{ .Values.proxy.pgBouncer.replicas }}
+      # these resources are for the "pgbouncer" container in the "postgres-crunchy-ha-pgbouncer" set of pods
+      # there is a sidecar in these pods which are not mentioned here, but the requests/limits are teeny weeny by default so no worries there.
+      resources:
+        requests:
+          cpu: {{ .Values.proxy.pgBouncer.requests.cpu }}
+          memory: {{ .Values.proxy.pgBouncer.requests.memory }}
+        {{ if .Values.proxy.pgBouncer.limits }}
+        limits:
+          cpu: {{ .Values.proxy.pgBouncer.limits.cpu }}
+          memory: {{ .Values.proxy.pgBouncer.limits.memory }}
+        {{ end }}
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 1
+              podAffinityTerm:
+                topologyKey: topology.kubernetes.io/zone
+                labelSelector:
+                  matchLabels:
+                    postgres-operator.crunchydata.com/cluster:
+                      {{ template "crunchy-postgres.fullname" . }}
+                    postgres-operator.crunchydata.com/role: pgbouncer

infrastructure/crunchy-db/templates/_helpers.tpl

@@ -0,0 +1,62 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "crunchy-postgres.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "crunchy-postgres.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "crunchy-postgres.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "crunchy-postgres.labels" -}}
+helm.sh/chart: {{ include "crunchy-postgres.chart" . }}
+{{ include "crunchy-postgres.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "crunchy-postgres.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "crunchy-postgres.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "crunchy-postgres.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "crunchy-postgres.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}

infrastructure/crunchy-db/templates/_s3.tpl

@@ -0,0 +1,26 @@
+{{/* Output S3 credential lines for one repo (index 0 = repo1, 1 = repo2, ...). */}}
+{{- define "postgres.s3" }}
+{{- if .s3 }}
+  {{- if .s3.key }}
+repo{{ add .index 1 }}-s3-key={{ .s3.key }}
+  {{- end }}
+  {{- if .s3.keySecret }}
+repo{{ add .index 1 }}-s3-key-secret={{ .s3.keySecret }}
+  {{- end }}
+  {{- if .s3.keyType }}
+repo{{ add .index 1 }}-s3-key-type={{ .s3.keyType }}
+  {{- end }}
+  {{- if .s3.encryptionPassphrase }}
+repo{{ add .index 1 }}-cipher-pass={{ .s3.encryptionPassphrase }}
+  {{- end }}
+{{- end }}
+{{- end }}
+
+{{/* Full S3 config for all four repos (repo1-repo4). Pass root context (.). */}}
+{{- define "postgres.s3.full" -}}
+[global]
+{{- $root := . }}
+{{- range $i := until 4 }}
+{{ include "postgres.s3" (dict "s3" $root.Values.pgBackRest.s3 "index" $i) }}
+{{- end }}
+{{- end }}

infrastructure/crunchy-db/templates/s3Secret.yaml

@@ -0,0 +1,9 @@
+{{- if .Values.pgBackRest.s3.createS3Secret }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ .Values.pgBackRest.s3.s3Secret }}
+type: Opaque
+data:
+  s3.conf: {{ include "postgres.s3.full" . | b64enc }}
+{{- end }}