K8s: Improve job to clean up scaledobjects are leftover

charts/selenium-grid/CONFIGURATION.md

@@ -394,8 +394,11 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
 | autoscaling.annotations | object | `{"helm.sh/hook":"post-install,post-upgrade,post-rollback","helm.sh/hook-weight":"1"}` | Annotations for KEDA resources: ScaledObject and ScaledJob |
 | autoscaling.patchObjectFinalizers.nameOverride | string | `nil` | Override the name of the patch job |
 | autoscaling.patchObjectFinalizers.enabled | bool | `true` | Enable patching finalizers for KEDA scaled resources. Workaround for Hook post-upgrade selenium-grid/templates/x-node-hpa.yaml failed: object is being deleted: scaledobjects.keda.sh "x" already exists |
-| autoscaling.patchObjectFinalizers.activeDeadlineSeconds | int | `120` | Deadline (in seconds) for patch job to complete |
-| autoscaling.patchObjectFinalizers.annotations | object | `{"helm.sh/hook":"post-install,post-upgrade,post-rollback,pre-delete","helm.sh/hook-delete-policy":"hook-succeeded,before-hook-creation","helm.sh/hook-weight":"-1"}` | Annotations for patch job |
+| autoscaling.patchObjectFinalizers.activeDeadlineSeconds | int | `300` | Deadline (in seconds) for patch job to complete |
+| autoscaling.patchObjectFinalizers.annotations | object | `{"helm.sh/hook":"post-install,post-upgrade,post-rollback,pre-delete","helm.sh/hook-delete-policy":"hook-succeeded,before-hook-creation"}` | Annotations for patch job |
+| autoscaling.patchObjectFinalizers.deleteObjectsScript | string | `""` | Define your custom script to replace the default script |
+| autoscaling.patchObjectFinalizers.patchFinalizersScript | string | `""` | Define your custom script to replace the default script |
+| autoscaling.patchObjectFinalizers.defaultMode | int | `493` | Default mode for ConfigMap is mounted as file |
 | autoscaling.patchObjectFinalizers.serviceAccount | string | `""` | Define an external service account name contains permissions to patch KEDA scaled resources |
 | autoscaling.patchObjectFinalizers.imagePullSecret | string | `""` | Custom pull secret for container in patch job |
 | autoscaling.patchObjectFinalizers.resources | object | `{"limits":{"cpu":"200m","memory":"500Mi"},"requests":{"cpu":"100m","memory":"200Mi"}}` | Define resources for container in patch job |

charts/selenium-grid/templates/_nameHelpers.tpl

@@ -260,11 +260,18 @@ Server ConfigMap fullname
 {{- tpl (default (include "seleniumGrid.component.name" (list "selenium-server-config" $)) .Values.serverConfigMap.nameOverride) $ | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
 
+{{/*
+Delete scaledObjects leafover job fullname
+*/}}
+{{- define "seleniumGrid.keda.deleteObjectsJob.fullname" -}}
+{{- printf "%s-scaledobjects-deletion" (tpl ( default (include "seleniumGrid.component.name" (list "selenium-patch" $)) .Values.autoscaling.patchObjectFinalizers.nameOverride) $) | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
 {{/*
 Patch scaledObjects finalizers job fullname
 */}}
 {{- define "seleniumGrid.keda.patchObjectsJob.fullname" -}}
-{{- tpl ( default (include "seleniumGrid.component.name" (list "selenium-patch-scaledobjects-finalizers" $)) .Values.autoscaling.patchObjectFinalizers.nameOverride) $ | trunc 63 | trimSuffix "-" -}}
+{{- printf "%s-scaledobjects-finalizers" (tpl ( default (include "seleniumGrid.component.name" (list "selenium-patch" $)) .Values.autoscaling.patchObjectFinalizers.nameOverride) $) | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
 
 {{/*

charts/selenium-grid/templates/patch-keda/delete-keda-objects-job.yaml

@@ -0,0 +1,55 @@
+{{- if and (eq (include "seleniumGrid.useKEDA" $) "true") $.Values.autoscaling.patchObjectFinalizers.enabled }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ template "seleniumGrid.keda.deleteObjectsJob.fullname" $ }}
+  namespace: {{ .Release.Namespace }}
+  annotations: &patch_objects_job_annotations
+    "helm.sh/hook-weight": "-10"
+  {{- with $.Values.autoscaling.patchObjectFinalizers.annotations }}
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  labels: &patch_objects_job_labels
+    app: {{ template "seleniumGrid.keda.deleteObjectsJob.fullname" $ }}
+    app.kubernetes.io/name: {{ template "seleniumGrid.keda.deleteObjectsJob.fullname" $ }}
+    {{- include "seleniumGrid.commonLabels" $ | nindent 4 }}
+  {{- with $.Values.customLabels }}
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  template:
+    metadata:
+      {{- with $.Values.autoscaling.patchObjectFinalizers.annotations }}
+      annotations: *patch_objects_job_annotations
+      {{- end }}
+      labels: *patch_objects_job_labels
+      name: {{ template "seleniumGrid.keda.deleteObjectsJob.fullname" $ }}
+    spec:
+      activeDeadlineSeconds: {{ $.Values.autoscaling.patchObjectFinalizers.activeDeadlineSeconds }}
+      serviceAccountName: {{ default (include "seleniumGrid.serviceAccount.fullname" $) $.Values.autoscaling.patchObjectFinalizers.serviceAccount }}
+      serviceAccount: {{ default (include "seleniumGrid.serviceAccount.fullname" $) $.Values.autoscaling.patchObjectFinalizers.serviceAccount }}
+      containers:
+        - name: kubectl
+          image: {{ $.Values.global.seleniumGrid.kubectlImage }}
+          command: ["/bin/bash", "-c", "/deleteObjectsScript.sh"]
+          volumeMounts:
+            - name: cleanup-script
+              mountPath: /deleteObjectsScript.sh
+              subPath: deleteObjectsScript.sh
+        {{- with $.Values.autoscaling.patchObjectFinalizers.resources }}
+          resources: {{ toYaml . | nindent 12 }}
+        {{- end }}
+      volumes:
+        - name: cleanup-script
+          configMap:
+            name: {{ template "seleniumGrid.keda.patchObjectsJob.fullname" $ }}
+            defaultMode: {{ $.Values.autoscaling.patchObjectFinalizers.defaultMode }}
+    {{- if or $.Values.global.seleniumGrid.imagePullSecret $.Values.autoscaling.patchObjectFinalizers.imagePullSecret }}
+      imagePullSecrets:
+        - name: {{ default $.Values.global.seleniumGrid.imagePullSecret $.Values.autoscaling.patchObjectFinalizers.imagePullSecret }}
+    {{- end }}
+      restartPolicy: Never
+    {{- with .Values.autoscaling.patchObjectFinalizers.nodeSelector }}
+      nodeSelector: {{- toYaml . | nindent 8 }}
+    {{- end }}
+{{- end }}

charts/selenium-grid/templates/patch-keda/patch-keda-objects-cm.yaml

@@ -0,0 +1,80 @@
+{{- if and (eq (include "seleniumGrid.useKEDA" $) "true") $.Values.autoscaling.patchObjectFinalizers.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "seleniumGrid.keda.patchObjectsJob.fullname" $ }}
+  namespace: {{ .Release.Namespace }}
+  annotations:
+    "helm.sh/hook-weight": "-20"
+  {{- with $.Values.autoscaling.patchObjectFinalizers.annotations }}
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  labels:
+    app: {{ template "seleniumGrid.keda.patchObjectsJob.fullname" $ }}
+    app.kubernetes.io/name: {{ template "seleniumGrid.keda.patchObjectsJob.fullname" $ }}
+    {{- include "seleniumGrid.commonLabels" $ | nindent 4 }}
+  {{- with $.Values.customLabels }}
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+data:
+{{- if $.Values.autoscaling.patchObjectFinalizers.deleteObjectsScript }}
+  deleteObjectsScript.sh: {{- toYaml $.Values.autoscaling.patchObjectFinalizers.deleteObjectsScript | nindent 4 }}
+{{- else }}
+  deleteObjectsScript.sh: |
+    #!/bin/bash
+    set -e
+    set -x
+    echo "Delete ScaledObjects, ScaledJobs and HPAs for {{ .Release.Name }} when upgrading or disabling autoscaling."
+    for i in $(kubectl get ScaledObjects -n {{ .Release.Namespace }} -l app.kubernetes.io/instance={{ .Release.Name }} \
+        -o jsonpath='{.items[*].metadata.name}{"\n"}');
+    do
+        if [ -n "$i" ]; then
+            kubectl delete ScaledObjects $i -n {{ .Release.Namespace }}
+        fi
+    done
+    for i in $(kubectl get ScaledJobs -n {{ .Release.Namespace }} -l app.kubernetes.io/instance={{ .Release.Name }} \
+        -o jsonpath='{.items[*].metadata.name}{"\n"}');
+    do
+        if [ -n "$i" ]; then
+            kubectl delete ScaledJobs $i -n {{ .Release.Namespace }}
+        fi
+    done
+    for i in $(kubectl get TriggerAuthentication -n {{ .Release.Namespace }} -l app.kubernetes.io/instance={{ .Release.Name }} \
+        -o jsonpath='{.items[*].metadata.name}{"\n"}');
+    do
+        if [ -n "$i" ]; then
+            kubectl delete TriggerAuthentication $i -n {{ .Release.Namespace }}
+        fi
+    done
+{{- end }}
+{{- if $.Values.autoscaling.patchObjectFinalizers.patchFinalizersScript }}
+  patchFinalizersScript.sh: {{- toYaml $.Values.autoscaling.patchObjectFinalizers.patchFinalizersScript | nindent 4 }}
+{{- else }}
+  patchFinalizersScript.sh: |
+    #!/bin/bash
+    set -e
+    set -x
+    echo "Patch finalizers of ScaledObjects, ScaledJobs and HPAs for {{ .Release.Name }} when upgrading or disabling autoscaling."
+    for i in $(kubectl get ScaledObjects -n {{ .Release.Namespace }} -l app.kubernetes.io/instance={{ .Release.Name }} \
+        -o jsonpath='{.items[*].metadata.name}{"\n"}');
+    do
+        if [ -n "$i" ]; then
+            kubectl patch ScaledObjects $i -n {{ .Release.Namespace }} -p '{"metadata":{"finalizers":null}}' --type=merge
+        fi
+    done
+    for i in $(kubectl get ScaledJobs -n {{ .Release.Namespace }} -l app.kubernetes.io/instance={{ .Release.Name }} \
+        -o jsonpath='{.items[*].metadata.name}{"\n"}');
+    do
+        if [ -n "$i" ]; then
+            kubectl patch ScaledJobs $i -n {{ .Release.Namespace }} -p '{"metadata":{"finalizers":null}}' --type=merge
+        fi
+    done
+    for i in $(kubectl get TriggerAuthentication -n {{ .Release.Namespace }} -l app.kubernetes.io/instance={{ .Release.Name }} \
+        -o jsonpath='{.items[*].metadata.name}{"\n"}');
+    do
+        if [ -n "$i" ]; then
+            kubectl patch TriggerAuthentication $i -n {{ .Release.Namespace }} -p '{"metadata":{"finalizers":null}}' --type=merge
+        fi
+    done
+{{- end }}
+{{- end }}

charts/selenium-grid/templates/patch-keda/patch-keda-objects-job.yaml

@@ -4,8 +4,9 @@ kind: Job
 metadata:
   name: {{ template "seleniumGrid.keda.patchObjectsJob.fullname" $ }}
   namespace: {{ .Release.Namespace }}
-  {{- with $.Values.autoscaling.patchObjectFinalizers.annotations }}
   annotations: &patch_objects_job_annotations
+    "helm.sh/hook-weight": "-1"
+  {{- with $.Values.autoscaling.patchObjectFinalizers.annotations }}
     {{- toYaml . | nindent 4 }}
   {{- end }}
   labels: &patch_objects_job_labels
@@ -30,16 +31,19 @@ spec:
       containers:
         - name: kubectl
           image: {{ $.Values.global.seleniumGrid.kubectlImage }}
-          command: ["/bin/sh", "-c"]
-          args:
-            - |
-              echo "Cleaning up ScaledObjects, ScaledJobs and HPAs for {{ .Release.Name }} when upgrading or disabling autoscaling."
-              kubectl get ScaledObjects,ScaledJobs,TriggerAuthentication -n {{ .Release.Namespace }} -l component.autoscaling={{ .Release.Name }} -o=json | jq '.metadata.finalizers = null' | kubectl apply -f - || true ;
-              kubectl delete ScaledObjects,ScaledJobs,TriggerAuthentication -n {{ .Release.Namespace }} -l component.autoscaling={{ .Release.Name }} --wait=false || true ;
-              kubectl delete hpa -n {{ .Release.Namespace }} -l component.autoscaling={{ .Release.Name }} --wait=false || true ;
+          command: ["/bin/bash", "-c", "/patchFinalizersScript.sh"]
+          volumeMounts:
+            - name: cleanup-script
+              mountPath: /patchFinalizersScript.sh
+              subPath: patchFinalizersScript.sh
         {{- with $.Values.autoscaling.patchObjectFinalizers.resources }}
           resources: {{ toYaml . | nindent 12 }}
         {{- end }}
+      volumes:
+        - name: cleanup-script
+          configMap:
+            name: {{ template "seleniumGrid.keda.patchObjectsJob.fullname" $ }}
+            defaultMode: {{ $.Values.autoscaling.patchObjectFinalizers.defaultMode }}
     {{- if or $.Values.global.seleniumGrid.imagePullSecret $.Values.autoscaling.patchObjectFinalizers.imagePullSecret }}
       imagePullSecrets:
         - name: {{ default $.Values.global.seleniumGrid.imagePullSecret $.Values.autoscaling.patchObjectFinalizers.imagePullSecret }}

charts/selenium-grid/values.yaml

@@ -1032,13 +1032,17 @@ autoscaling:
     # -- Enable patching finalizers for KEDA scaled resources. Workaround for Hook post-upgrade selenium-grid/templates/x-node-hpa.yaml failed: object is being deleted: scaledobjects.keda.sh "x" already exists
     enabled: true
     # -- Deadline (in seconds) for patch job to complete
-    activeDeadlineSeconds: 120
+    activeDeadlineSeconds: 300
     # -- Annotations for patch job
     annotations:
       "helm.sh/hook": post-install,post-upgrade,post-rollback,pre-delete
       "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation
-      # This should be run before all other hooks (since delete action is called), so use a negative weight
-      "helm.sh/hook-weight": "-1"
+    # -- Define your custom script to replace the default script
+    deleteObjectsScript: ""
+    # -- Define your custom script to replace the default script
+    patchFinalizersScript: ""
+    # -- Default mode for ConfigMap is mounted as file
+    defaultMode: 0755
     # -- Define an external service account name contains permissions to patch KEDA scaled resources
     serviceAccount: ""
     # -- Custom pull secret for container in patch job

tests/charts/make/chart_test.sh

@@ -92,7 +92,7 @@ cleanup() {
   done
   if [ "${SKIP_CLEANUP}" = "false" ] || [ "${CI:-false}" != "false" ]; then
     echo "Clean up chart release and namespace"
-    helm delete ${RELEASE_NAME} --namespace ${SELENIUM_NAMESPACE} --no-hooks || true
+    helm delete ${RELEASE_NAME} --namespace ${SELENIUM_NAMESPACE} || true
     wait_for_terminated
     sudo chmod -R 777 ${HOST_PATH}/logs
   fi