Skip to content

Commit bb6e944

Browse files
cwraucwrau
committed
feat: delete statefulsets if volumeClaimTemplates changes
Signed-off-by: Chris Werner Rau <[email protected]>
1 parent 24bcd88 commit bb6e944

File tree

7 files changed

+180
-2
lines changed

7 files changed

+180
-2
lines changed

‎charts/tempo-distributed/README.md‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -596,8 +596,9 @@ The memcached default args are removed and should be provided manually. The sett
596596
| ingester.image.tag | string | `nil` | Docker image tag for the ingester image. Overrides `tempo.image.tag` |
597597
| ingester.initContainers | list | `[]` | |
598598
| ingester.nodeSelector | object | `{}` | Node selector for ingester pods |
599-
| ingester.persistence | object | `{"annotations":{},"enabled":false,"inMemory":false,"labels":{},"size":"10Gi","storageClass":null}` | Persistence configuration for ingester |
599+
| ingester.persistence | object | `{"annotations":{},"enableStatefulSetRecreationForSizeChange":false,"enabled":false,"inMemory":false,"labels":{},"size":"10Gi","storageClass":null}` | Persistence configuration for ingester |
600600
| ingester.persistence.annotations | object | `{}` | Annotations for ingester's persist volume claim |
601+
| ingester.persistence.enableStatefulSetRecreationForSizeChange | bool | `false` | Enable StatefulSetRecreation for changes to PVC size |
601602
| ingester.persistence.enabled | bool | `false` | Enable creating PVCs which is required when using boltdb-shipper |
602603
| ingester.persistence.inMemory | bool | `false` | use emptyDir with ramdisk instead of PVC. **Please note that all data in ingester will be lost on pod restart** |
603604
| ingester.persistence.labels | object | `{}` | Labels for ingester's persist volume claim |
@@ -745,8 +746,9 @@ The memcached default args are removed and should be provided manually. The sett
745746
| metricsGenerator.maxUnavailable | int | `1` | Pod Disruption Budget maxUnavailable |
746747
| metricsGenerator.minReadySeconds | int | `10` | Minimum number of seconds for which a newly created Pod should be ready without any of its containers crashing/terminating |
747748
| metricsGenerator.nodeSelector | object | `{}` | Node selector for metrics-generator pods |
748-
| metricsGenerator.persistence | object | `{"annotations":{},"enabled":false,"labels":{},"size":"10Gi","storageClass":null}` | Persistence configuration for metrics-generator |
749+
| metricsGenerator.persistence | object | `{"annotations":{},"enableStatefulSetRecreationForSizeChange":false,"enabled":false,"labels":{},"size":"10Gi","storageClass":null}` | Persistence configuration for metrics-generator |
749750
| metricsGenerator.persistence.annotations | object | `{}` | Annotations for metrics generator PVCs |
751+
| metricsGenerator.persistence.enableStatefulSetRecreationForSizeChange | bool | `false` | Enable StatefulSetRecreation for changes to PVC size. This means that the StatefulSet will be deleted, recreated (with the same name) and rolled when a change to the PVC size is detected. That way the PVC can be resized without manual intervention. |
750752
| metricsGenerator.persistence.enabled | bool | `false` | Enable creating PVCs if you have kind set to StatefulSet. This disables using local disk or memory configured in walEmptyDir |
751753
| metricsGenerator.persistence.labels | object | `{}` | Labels for metrics generator PVCs |
752754
| metricsGenerator.persistence.storageClass | string | `nil` | Storage class to be used. If defined, storageClassName: <storageClass>. If set to "-", storageClassName: "", which disables dynamic provisioning. If empty or set to null, no storageClassName spec is set, choosing the default provisioner (gp2 on AWS, standard on GKE, AWS, and OpenStack). |

‎charts/tempo-distributed/templates/_helpers.tpl‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -298,3 +298,161 @@ Cluster name that shows up in dashboard metrics
298298
{{- define "tempo.clusterName" -}}
299299
{{ (include "tempo.calculatedConfig" . | fromYaml).cluster_name | default .Release.Name }}
300300
{{- end -}}
301+
302+
{{- define "tempo.statefulset.recreateOnSizeChangeHook" -}}
303+
{{- $renderedStatefulSets := list -}}
304+
{{- range $renderedStatefulSet := include (print .context.Template.BasePath .pathToStatefulsetTemplate) .context | splitList "---" -}}
305+
{{- with $renderedStatefulSet | fromYaml -}}
306+
{{- $renderedStatefulSets = append $renderedStatefulSets . -}}
307+
{{- end }}
308+
{{- end -}}
309+
{{- if $renderedStatefulSets }}
310+
{{- range $newStatefulSet := $renderedStatefulSets -}}
311+
{{- $currentStatefulset := dict -}}
312+
{{- if $newStatefulSet.spec.volumeClaimTemplates }}
313+
{{- $currentStatefulset = lookup $newStatefulSet.apiVersion $newStatefulSet.kind $newStatefulSet.metadata.namespace $newStatefulSet.metadata.name -}}
314+
{{- $needsRecreation := false -}}
315+
{{- $templates := dict -}}
316+
{{- if $currentStatefulset -}}
317+
{{- if ne (len $newStatefulSet.spec.volumeClaimTemplates) (len $currentStatefulset.spec.volumeClaimTemplates) -}}
318+
{{- $needsRecreation = true -}}
319+
{{- end -}}
320+
{{- range $index, $newVolumeClaimTemplate := $newStatefulSet.spec.volumeClaimTemplates -}}
321+
{{- $currentVolumeClaimTemplateSpec := dict -}}
322+
{{- range $oldVolumeClaimTemplate := $currentStatefulset.spec.volumeClaimTemplates -}}
323+
{{- if eq $oldVolumeClaimTemplate.metadata.name $newVolumeClaimTemplate.metadata.name -}}
324+
{{- $currentVolumeClaimTemplateSpec = $oldVolumeClaimTemplate.spec -}}
325+
{{- end -}}
326+
{{- end }}
327+
{{- $newVolumeClaimTemplateStorageSize := $newVolumeClaimTemplate.spec.resources.requests.storage -}}
328+
{{- if not $currentVolumeClaimTemplateSpec -}}
329+
{{- $needsRecreation = true -}}
330+
{{- else -}}
331+
{{- if ne $newVolumeClaimTemplateStorageSize $currentVolumeClaimTemplateSpec.resources.requests.storage -}}
332+
{{- $needsRecreation = true -}}
333+
{{- $templates = set $templates $newVolumeClaimTemplate.metadata.name $newVolumeClaimTemplateStorageSize -}}
334+
{{- end -}}
335+
{{- end -}}
336+
{{- end -}}
337+
{{- end -}}
338+
{{- if $needsRecreation }}
339+
apiVersion: batch/v1
340+
kind: Job
341+
metadata:
342+
name: {{ $newStatefulSet.metadata.name }}-recreate
343+
namespace: {{ $newStatefulSet.metadata.namespace }}
344+
labels:
345+
{{- $newStatefulSet.metadata.labels | toYaml | nindent 4 }}
346+
app.kubernetes.io/component: statefulset-recreate-job
347+
annotations:
348+
"helm.sh/hook": pre-upgrade
349+
"helm.sh/hook-weight": "-5"
350+
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
351+
spec:
352+
ttlSecondsAfterFinished: 300
353+
template:
354+
metadata:
355+
name: {{ $newStatefulSet.metadata.name }}-recreate
356+
labels:
357+
{{- $newStatefulSet.metadata.labels | toYaml | nindent 8 }}
358+
spec:
359+
serviceAccountName: {{ $newStatefulSet.metadata.name }}-recreate
360+
restartPolicy: OnFailure
361+
containers:
362+
- name: recreate
363+
image: {{ printf "%s/kubectl:%s" (coalesce $.context.Values.global.image.registry "registry.k8s.io") $.context.Capabilities.KubeVersion.Version }}
364+
command:
365+
- kubectl
366+
- delete
367+
- statefulset
368+
- {{ $newStatefulSet.metadata.name }}
369+
- --cascade=orphan
370+
{{- range $index := until (int $currentStatefulset.spec.replicas) }}
371+
{{- range $template, $size := $templates }}
372+
- name: patch-pvc-{{ $template }}-{{ $index }}
373+
image: {{ printf "%s/kubectl:%s" (coalesce $.context.Values.global.image.registry "registry.k8s.io") $.context.Capabilities.KubeVersion.Version }}
374+
command:
375+
- patch
376+
- pvc
377+
- --namespace={{ $newStatefulSet.metadata.namespace }}
378+
- {{ printf "%s-%s-%d" $template $newStatefulSet.metadata.name $index }}
379+
- --type='json'
380+
- '-p=[{"op": "replace", "path": "/spec/resources/requests/storage", "value": "{{ $size }}"}]'
381+
{{- end }}
382+
{{- end }}
383+
---
384+
apiVersion: v1
385+
kind: ServiceAccount
386+
metadata:
387+
name: {{ $newStatefulSet.metadata.name }}-recreate
388+
namespace: {{ $newStatefulSet.metadata.namespace }}
389+
labels:
390+
{{- $newStatefulSet.metadata.labels | toYaml | nindent 4 }}
391+
app.kubernetes.io/component: statefulset-recreate-job
392+
annotations:
393+
"helm.sh/hook": pre-upgrade
394+
"helm.sh/hook-weight": "-10"
395+
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
396+
---
397+
apiVersion: rbac.authorization.k8s.io/v1
398+
kind: Role
399+
metadata:
400+
name: {{ $newStatefulSet.metadata.name }}-recreate
401+
namespace: {{ $newStatefulSet.metadata.namespace }}
402+
labels:
403+
{{- $newStatefulSet.metadata.labels | toYaml | nindent 4 }}
404+
app.kubernetes.io/component: statefulset-recreate-job
405+
annotations:
406+
"helm.sh/hook": pre-upgrade
407+
"helm.sh/hook-weight": "-10"
408+
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
409+
rules:
410+
- apiGroups:
411+
- apps
412+
resources:
413+
- statefulsets
414+
resourceNames:
415+
- {{ $newStatefulSet.metadata.name }}
416+
verbs:
417+
- delete
418+
{{- if $templates }}
419+
- apiGroups:
420+
- v1
421+
resources:
422+
- persistentvolumeclaims
423+
resourceNames:
424+
{{- range $index := until (int $currentStatefulset.spec.replicas) }}
425+
{{- range $template := $templates | keys }}
426+
- {{ printf "%s-%s-%d" $template $newStatefulSet.metadata.name $index }}
427+
{{- end }}
428+
{{- end }}
429+
verbs:
430+
- patch
431+
{{- end }}
432+
---
433+
apiVersion: rbac.authorization.k8s.io/v1
434+
kind: RoleBinding
435+
metadata:
436+
name: {{ $newStatefulSet.metadata.name }}-recreate
437+
namespace: {{ $newStatefulSet.metadata.namespace }}
438+
labels:
439+
{{- $newStatefulSet.metadata.labels | toYaml | nindent 4 }}
440+
app.kubernetes.io/component: statefulset-recreate-job
441+
annotations:
442+
"helm.sh/hook": pre-upgrade
443+
"helm.sh/hook-weight": "-10"
444+
"helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
445+
subjects:
446+
- kind: ServiceAccount
447+
name: {{ $newStatefulSet.metadata.name }}-recreate
448+
namespace: {{ $newStatefulSet.metadata.namespace }}
449+
roleRef:
450+
kind: Role
451+
name: {{ $newStatefulSet.metadata.name }}-recreate
452+
apiGroup: rbac.authorization.k8s.io
453+
---
454+
{{- end }}
455+
{{ end }}
456+
{{ end }}
457+
{{ end }}
458+
{{- end -}}

‎charts/tempo-distributed/templates/ingester/statefulset-ingester-recreate-job.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{{- if and .Values.ingester.persistence.enabled .Values.ingester.persistence.enableStatefulSetRecreationForSizeChange -}}
2+
{{- include "tempo.statefulset.recreateOnSizeChangeHook" (dict "context" . "pathToStatefulsetTemplate" "/ingester/statefulset-ingester.yaml") -}}
3+
{{- end -}}

‎charts/tempo-distributed/templates/ingester/statefulset-ingester.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,9 @@ spec:
4545
{{- end }}
4646
annotations:
4747
checksum/config: {{ include (print $.Template.BasePath "/configmap-tempo.yaml") . | sha256sum }}
48+
{{- if .Values.ingester.persistence.enabled }}
49+
storage/size: {{ .Values.ingester.persistence.size | quote }}
50+
{{- end }}
4851
{{- with .Values.tempo.podAnnotations }}
4952
{{- toYaml . | nindent 8 }}
5053
{{- end }}

‎charts/tempo-distributed/templates/metrics-generator/statefulset-metrics-generator-recreate-job.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{{- if and .Values.metricsGenerator.persistence.enabled .Values.metricsGenerator.persistence.enableStatefulSetRecreationForSizeChange -}}
2+
{{- include "tempo.statefulset.recreateOnSizeChangeHook" (dict "context" . "pathToStatefulsetTemplate" "/metrics-generator/statefulset-metrics-generator.yaml") -}}
3+
{{- end -}}

‎charts/tempo-distributed/templates/metrics-generator/statefulset-metrics-generator.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,9 @@ spec:
3636
{{- end }}
3737
annotations:
3838
checksum/config: {{ include (print $.Template.BasePath "/configmap-tempo.yaml") . | sha256sum }}
39+
{{- if .Values.metricsGenerator.persistence.enabled }}
40+
storage/size: {{ .Values.metricsGenerator.persistence.size | quote }}
41+
{{- end }}
3942
{{- with .Values.tempo.podAnnotations }}
4043
{{- toYaml . | nindent 8 }}
4144
{{- end }}

‎charts/tempo-distributed/values.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -217,6 +217,8 @@ ingester:
217217
persistence:
218218
# -- Enable creating PVCs which is required when using boltdb-shipper
219219
enabled: false
220+
# -- Enable StatefulSetRecreation for changes to PVC size
221+
enableStatefulSetRecreationForSizeChange: false
220222
# -- use emptyDir with ramdisk instead of PVC. **Please note that all data in ingester will be lost on pod restart**
221223
inMemory: false
222224
# -- Size of persistent or memory disk
@@ -401,6 +403,10 @@ metricsGenerator:
401403
persistence:
402404
# -- Enable creating PVCs if you have kind set to StatefulSet. This disables using local disk or memory configured in walEmptyDir
403405
enabled: false
406+
# -- Enable StatefulSetRecreation for changes to PVC size.
407+
# This means that the StatefulSet will be deleted, recreated (with the same name) and rolled when a change to the
408+
# PVC size is detected. That way the PVC can be resized without manual intervention.
409+
enableStatefulSetRecreationForSizeChange: false
404410
size: 10Gi
405411
# -- Storage class to be used.
406412
# If defined, storageClassName: <storageClass>.

0 commit comments

Comments
 (0)