From 81b2a83d2ab0e6d6237a605da67752def3aafcd4 Mon Sep 17 00:00:00 2001 From: Viet Nguyen Duc Date: Mon, 17 Mar 2025 23:41:45 +0000 Subject: [PATCH] K8s: Improve job to clean up scaledobjects are leftover Signed-off-by: Viet Nguyen Duc --- charts/selenium-grid/CONFIGURATION.md | 7 +- .../selenium-grid/templates/_nameHelpers.tpl | 9 ++- .../patch-keda/delete-keda-objects-job.yaml | 55 +++++++++++++ .../patch-keda/patch-keda-objects-cm.yaml | 80 +++++++++++++++++++ .../patch-keda/patch-keda-objects-job.yaml | 20 +++-- charts/selenium-grid/values.yaml | 10 ++- tests/charts/make/chart_test.sh | 2 +- 7 files changed, 168 insertions(+), 15 deletions(-) create mode 100644 charts/selenium-grid/templates/patch-keda/delete-keda-objects-job.yaml create mode 100644 charts/selenium-grid/templates/patch-keda/patch-keda-objects-cm.yaml diff --git a/charts/selenium-grid/CONFIGURATION.md b/charts/selenium-grid/CONFIGURATION.md index 8026e0cde1..22d66baebd 100644 --- a/charts/selenium-grid/CONFIGURATION.md +++ b/charts/selenium-grid/CONFIGURATION.md @@ -394,8 +394,11 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes | autoscaling.annotations | object | `{"helm.sh/hook":"post-install,post-upgrade,post-rollback","helm.sh/hook-weight":"1"}` | Annotations for KEDA resources: ScaledObject and ScaledJob | | autoscaling.patchObjectFinalizers.nameOverride | string | `nil` | Override the name of the patch job | | autoscaling.patchObjectFinalizers.enabled | bool | `true` | Enable patching finalizers for KEDA scaled resources. Workaround for Hook post-upgrade selenium-grid/templates/x-node-hpa.yaml failed: object is being deleted: scaledobjects.keda.sh "x" already exists | -| autoscaling.patchObjectFinalizers.activeDeadlineSeconds | int | `120` | Deadline (in seconds) for patch job to complete | -| autoscaling.patchObjectFinalizers.annotations | object | `{"helm.sh/hook":"post-install,post-upgrade,post-rollback,pre-delete","helm.sh/hook-delete-policy":"hook-succeeded,before-hook-creation","helm.sh/hook-weight":"-1"}` | Annotations for patch job | +| autoscaling.patchObjectFinalizers.activeDeadlineSeconds | int | `300` | Deadline (in seconds) for patch job to complete | +| autoscaling.patchObjectFinalizers.annotations | object | `{"helm.sh/hook":"post-install,post-upgrade,post-rollback,pre-delete","helm.sh/hook-delete-policy":"hook-succeeded,before-hook-creation"}` | Annotations for patch job | +| autoscaling.patchObjectFinalizers.deleteObjectsScript | string | `""` | Define your custom script to replace the default script | +| autoscaling.patchObjectFinalizers.patchFinalizersScript | string | `""` | Define your custom script to replace the default script | +| autoscaling.patchObjectFinalizers.defaultMode | int | `493` | Default mode for ConfigMap is mounted as file | | autoscaling.patchObjectFinalizers.serviceAccount | string | `""` | Define an external service account name contains permissions to patch KEDA scaled resources | | autoscaling.patchObjectFinalizers.imagePullSecret | string | `""` | Custom pull secret for container in patch job | | autoscaling.patchObjectFinalizers.resources | object | `{"limits":{"cpu":"200m","memory":"500Mi"},"requests":{"cpu":"100m","memory":"200Mi"}}` | Define resources for container in patch job | diff --git a/charts/selenium-grid/templates/_nameHelpers.tpl b/charts/selenium-grid/templates/_nameHelpers.tpl index 059e15cdb8..cef0a0159c 100644 --- a/charts/selenium-grid/templates/_nameHelpers.tpl +++ b/charts/selenium-grid/templates/_nameHelpers.tpl @@ -260,11 +260,18 @@ Server ConfigMap fullname {{- tpl (default (include "seleniumGrid.component.name" (list "selenium-server-config" $)) .Values.serverConfigMap.nameOverride) $ | trunc 63 | trimSuffix "-" -}} {{- end -}} +{{/* +Delete scaledObjects leafover job fullname +*/}} +{{- define "seleniumGrid.keda.deleteObjectsJob.fullname" -}} +{{- printf "%s-scaledobjects-deletion" (tpl ( default (include "seleniumGrid.component.name" (list "selenium-patch" $)) .Values.autoscaling.patchObjectFinalizers.nameOverride) $) | trunc 63 | trimSuffix "-" -}} +{{- end -}} + {{/* Patch scaledObjects finalizers job fullname */}} {{- define "seleniumGrid.keda.patchObjectsJob.fullname" -}} -{{- tpl ( default (include "seleniumGrid.component.name" (list "selenium-patch-scaledobjects-finalizers" $)) .Values.autoscaling.patchObjectFinalizers.nameOverride) $ | trunc 63 | trimSuffix "-" -}} +{{- printf "%s-scaledobjects-finalizers" (tpl ( default (include "seleniumGrid.component.name" (list "selenium-patch" $)) .Values.autoscaling.patchObjectFinalizers.nameOverride) $) | trunc 63 | trimSuffix "-" -}} {{- end -}} {{/* diff --git a/charts/selenium-grid/templates/patch-keda/delete-keda-objects-job.yaml b/charts/selenium-grid/templates/patch-keda/delete-keda-objects-job.yaml new file mode 100644 index 0000000000..3f965fdc0a --- /dev/null +++ b/charts/selenium-grid/templates/patch-keda/delete-keda-objects-job.yaml @@ -0,0 +1,55 @@ +{{- if and (eq (include "seleniumGrid.useKEDA" $) "true") $.Values.autoscaling.patchObjectFinalizers.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "seleniumGrid.keda.deleteObjectsJob.fullname" $ }} + namespace: {{ .Release.Namespace }} + annotations: &patch_objects_job_annotations + "helm.sh/hook-weight": "-10" + {{- with $.Values.autoscaling.patchObjectFinalizers.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + labels: &patch_objects_job_labels + app: {{ template "seleniumGrid.keda.deleteObjectsJob.fullname" $ }} + app.kubernetes.io/name: {{ template "seleniumGrid.keda.deleteObjectsJob.fullname" $ }} + {{- include "seleniumGrid.commonLabels" $ | nindent 4 }} + {{- with $.Values.customLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + template: + metadata: + {{- with $.Values.autoscaling.patchObjectFinalizers.annotations }} + annotations: *patch_objects_job_annotations + {{- end }} + labels: *patch_objects_job_labels + name: {{ template "seleniumGrid.keda.deleteObjectsJob.fullname" $ }} + spec: + activeDeadlineSeconds: {{ $.Values.autoscaling.patchObjectFinalizers.activeDeadlineSeconds }} + serviceAccountName: {{ default (include "seleniumGrid.serviceAccount.fullname" $) $.Values.autoscaling.patchObjectFinalizers.serviceAccount }} + serviceAccount: {{ default (include "seleniumGrid.serviceAccount.fullname" $) $.Values.autoscaling.patchObjectFinalizers.serviceAccount }} + containers: + - name: kubectl + image: {{ $.Values.global.seleniumGrid.kubectlImage }} + command: ["/bin/bash", "-c", "/deleteObjectsScript.sh"] + volumeMounts: + - name: cleanup-script + mountPath: /deleteObjectsScript.sh + subPath: deleteObjectsScript.sh + {{- with $.Values.autoscaling.patchObjectFinalizers.resources }} + resources: {{ toYaml . | nindent 12 }} + {{- end }} + volumes: + - name: cleanup-script + configMap: + name: {{ template "seleniumGrid.keda.patchObjectsJob.fullname" $ }} + defaultMode: {{ $.Values.autoscaling.patchObjectFinalizers.defaultMode }} + {{- if or $.Values.global.seleniumGrid.imagePullSecret $.Values.autoscaling.patchObjectFinalizers.imagePullSecret }} + imagePullSecrets: + - name: {{ default $.Values.global.seleniumGrid.imagePullSecret $.Values.autoscaling.patchObjectFinalizers.imagePullSecret }} + {{- end }} + restartPolicy: Never + {{- with .Values.autoscaling.patchObjectFinalizers.nodeSelector }} + nodeSelector: {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/charts/selenium-grid/templates/patch-keda/patch-keda-objects-cm.yaml b/charts/selenium-grid/templates/patch-keda/patch-keda-objects-cm.yaml new file mode 100644 index 0000000000..7cbc212b4f --- /dev/null +++ b/charts/selenium-grid/templates/patch-keda/patch-keda-objects-cm.yaml @@ -0,0 +1,80 @@ +{{- if and (eq (include "seleniumGrid.useKEDA" $) "true") $.Values.autoscaling.patchObjectFinalizers.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "seleniumGrid.keda.patchObjectsJob.fullname" $ }} + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook-weight": "-20" + {{- with $.Values.autoscaling.patchObjectFinalizers.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + app: {{ template "seleniumGrid.keda.patchObjectsJob.fullname" $ }} + app.kubernetes.io/name: {{ template "seleniumGrid.keda.patchObjectsJob.fullname" $ }} + {{- include "seleniumGrid.commonLabels" $ | nindent 4 }} + {{- with $.Values.customLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} +data: +{{- if $.Values.autoscaling.patchObjectFinalizers.deleteObjectsScript }} + deleteObjectsScript.sh: {{- toYaml $.Values.autoscaling.patchObjectFinalizers.deleteObjectsScript | nindent 4 }} +{{- else }} + deleteObjectsScript.sh: | + #!/bin/bash + set -e + set -x + echo "Delete ScaledObjects, ScaledJobs and HPAs for {{ .Release.Name }} when upgrading or disabling autoscaling." + for i in $(kubectl get ScaledObjects -n {{ .Release.Namespace }} -l app.kubernetes.io/instance={{ .Release.Name }} \ + -o jsonpath='{.items[*].metadata.name}{"\n"}'); + do + if [ -n "$i" ]; then + kubectl delete ScaledObjects $i -n {{ .Release.Namespace }} + fi + done + for i in $(kubectl get ScaledJobs -n {{ .Release.Namespace }} -l app.kubernetes.io/instance={{ .Release.Name }} \ + -o jsonpath='{.items[*].metadata.name}{"\n"}'); + do + if [ -n "$i" ]; then + kubectl delete ScaledJobs $i -n {{ .Release.Namespace }} + fi + done + for i in $(kubectl get TriggerAuthentication -n {{ .Release.Namespace }} -l app.kubernetes.io/instance={{ .Release.Name }} \ + -o jsonpath='{.items[*].metadata.name}{"\n"}'); + do + if [ -n "$i" ]; then + kubectl delete TriggerAuthentication $i -n {{ .Release.Namespace }} + fi + done +{{- end }} +{{- if $.Values.autoscaling.patchObjectFinalizers.patchFinalizersScript }} + patchFinalizersScript.sh: {{- toYaml $.Values.autoscaling.patchObjectFinalizers.patchFinalizersScript | nindent 4 }} +{{- else }} + patchFinalizersScript.sh: | + #!/bin/bash + set -e + set -x + echo "Patch finalizers of ScaledObjects, ScaledJobs and HPAs for {{ .Release.Name }} when upgrading or disabling autoscaling." + for i in $(kubectl get ScaledObjects -n {{ .Release.Namespace }} -l app.kubernetes.io/instance={{ .Release.Name }} \ + -o jsonpath='{.items[*].metadata.name}{"\n"}'); + do + if [ -n "$i" ]; then + kubectl patch ScaledObjects $i -n {{ .Release.Namespace }} -p '{"metadata":{"finalizers":null}}' --type=merge + fi + done + for i in $(kubectl get ScaledJobs -n {{ .Release.Namespace }} -l app.kubernetes.io/instance={{ .Release.Name }} \ + -o jsonpath='{.items[*].metadata.name}{"\n"}'); + do + if [ -n "$i" ]; then + kubectl patch ScaledJobs $i -n {{ .Release.Namespace }} -p '{"metadata":{"finalizers":null}}' --type=merge + fi + done + for i in $(kubectl get TriggerAuthentication -n {{ .Release.Namespace }} -l app.kubernetes.io/instance={{ .Release.Name }} \ + -o jsonpath='{.items[*].metadata.name}{"\n"}'); + do + if [ -n "$i" ]; then + kubectl patch TriggerAuthentication $i -n {{ .Release.Namespace }} -p '{"metadata":{"finalizers":null}}' --type=merge + fi + done +{{- end }} +{{- end }} diff --git a/charts/selenium-grid/templates/patch-keda/patch-keda-objects-job.yaml b/charts/selenium-grid/templates/patch-keda/patch-keda-objects-job.yaml index 0f9d06b7a9..f4e962260b 100644 --- a/charts/selenium-grid/templates/patch-keda/patch-keda-objects-job.yaml +++ b/charts/selenium-grid/templates/patch-keda/patch-keda-objects-job.yaml @@ -4,8 +4,9 @@ kind: Job metadata: name: {{ template "seleniumGrid.keda.patchObjectsJob.fullname" $ }} namespace: {{ .Release.Namespace }} - {{- with $.Values.autoscaling.patchObjectFinalizers.annotations }} annotations: &patch_objects_job_annotations + "helm.sh/hook-weight": "-1" + {{- with $.Values.autoscaling.patchObjectFinalizers.annotations }} {{- toYaml . | nindent 4 }} {{- end }} labels: &patch_objects_job_labels @@ -30,16 +31,19 @@ spec: containers: - name: kubectl image: {{ $.Values.global.seleniumGrid.kubectlImage }} - command: ["/bin/sh", "-c"] - args: - - | - echo "Cleaning up ScaledObjects, ScaledJobs and HPAs for {{ .Release.Name }} when upgrading or disabling autoscaling." - kubectl get ScaledObjects,ScaledJobs,TriggerAuthentication -n {{ .Release.Namespace }} -l component.autoscaling={{ .Release.Name }} -o=json | jq '.metadata.finalizers = null' | kubectl apply -f - || true ; - kubectl delete ScaledObjects,ScaledJobs,TriggerAuthentication -n {{ .Release.Namespace }} -l component.autoscaling={{ .Release.Name }} --wait=false || true ; - kubectl delete hpa -n {{ .Release.Namespace }} -l component.autoscaling={{ .Release.Name }} --wait=false || true ; + command: ["/bin/bash", "-c", "/patchFinalizersScript.sh"] + volumeMounts: + - name: cleanup-script + mountPath: /patchFinalizersScript.sh + subPath: patchFinalizersScript.sh {{- with $.Values.autoscaling.patchObjectFinalizers.resources }} resources: {{ toYaml . | nindent 12 }} {{- end }} + volumes: + - name: cleanup-script + configMap: + name: {{ template "seleniumGrid.keda.patchObjectsJob.fullname" $ }} + defaultMode: {{ $.Values.autoscaling.patchObjectFinalizers.defaultMode }} {{- if or $.Values.global.seleniumGrid.imagePullSecret $.Values.autoscaling.patchObjectFinalizers.imagePullSecret }} imagePullSecrets: - name: {{ default $.Values.global.seleniumGrid.imagePullSecret $.Values.autoscaling.patchObjectFinalizers.imagePullSecret }} diff --git a/charts/selenium-grid/values.yaml b/charts/selenium-grid/values.yaml index 09509a2ebd..78cb13c196 100644 --- a/charts/selenium-grid/values.yaml +++ b/charts/selenium-grid/values.yaml @@ -1032,13 +1032,17 @@ autoscaling: # -- Enable patching finalizers for KEDA scaled resources. Workaround for Hook post-upgrade selenium-grid/templates/x-node-hpa.yaml failed: object is being deleted: scaledobjects.keda.sh "x" already exists enabled: true # -- Deadline (in seconds) for patch job to complete - activeDeadlineSeconds: 120 + activeDeadlineSeconds: 300 # -- Annotations for patch job annotations: "helm.sh/hook": post-install,post-upgrade,post-rollback,pre-delete "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation - # This should be run before all other hooks (since delete action is called), so use a negative weight - "helm.sh/hook-weight": "-1" + # -- Define your custom script to replace the default script + deleteObjectsScript: "" + # -- Define your custom script to replace the default script + patchFinalizersScript: "" + # -- Default mode for ConfigMap is mounted as file + defaultMode: 0755 # -- Define an external service account name contains permissions to patch KEDA scaled resources serviceAccount: "" # -- Custom pull secret for container in patch job diff --git a/tests/charts/make/chart_test.sh b/tests/charts/make/chart_test.sh index fa9fed7f59..7c22d552de 100755 --- a/tests/charts/make/chart_test.sh +++ b/tests/charts/make/chart_test.sh @@ -92,7 +92,7 @@ cleanup() { done if [ "${SKIP_CLEANUP}" = "false" ] || [ "${CI:-false}" != "false" ]; then echo "Clean up chart release and namespace" - helm delete ${RELEASE_NAME} --namespace ${SELENIUM_NAMESPACE} --no-hooks || true + helm delete ${RELEASE_NAME} --namespace ${SELENIUM_NAMESPACE} || true wait_for_terminated sudo chmod -R 777 ${HOST_PATH}/logs fi