Skip to content

Commit dd1ff4b

Browse files
VietND96VietND96
authored
K8s: Improve job to clean up scaledobjects are leftover (#2710)
Signed-off-by: Viet Nguyen Duc <[email protected]>
1 parent c95fc27 commit dd1ff4b

File tree

7 files changed

+168
-15
lines changed

7 files changed

+168
-15
lines changed

‎charts/selenium-grid/CONFIGURATION.md‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -394,8 +394,11 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
394394
| autoscaling.annotations | object | `{"helm.sh/hook":"post-install,post-upgrade,post-rollback","helm.sh/hook-weight":"1"}` | Annotations for KEDA resources: ScaledObject and ScaledJob |
395395
| autoscaling.patchObjectFinalizers.nameOverride | string | `nil` | Override the name of the patch job |
396396
| autoscaling.patchObjectFinalizers.enabled | bool | `true` | Enable patching finalizers for KEDA scaled resources. Workaround for Hook post-upgrade selenium-grid/templates/x-node-hpa.yaml failed: object is being deleted: scaledobjects.keda.sh "x" already exists |
397-
| autoscaling.patchObjectFinalizers.activeDeadlineSeconds | int | `120` | Deadline (in seconds) for patch job to complete |
398-
| autoscaling.patchObjectFinalizers.annotations | object | `{"helm.sh/hook":"post-install,post-upgrade,post-rollback,pre-delete","helm.sh/hook-delete-policy":"hook-succeeded,before-hook-creation","helm.sh/hook-weight":"-1"}` | Annotations for patch job |
397+
| autoscaling.patchObjectFinalizers.activeDeadlineSeconds | int | `300` | Deadline (in seconds) for patch job to complete |
398+
| autoscaling.patchObjectFinalizers.annotations | object | `{"helm.sh/hook":"post-install,post-upgrade,post-rollback,pre-delete","helm.sh/hook-delete-policy":"hook-succeeded,before-hook-creation"}` | Annotations for patch job |
399+
| autoscaling.patchObjectFinalizers.deleteObjectsScript | string | `""` | Define your custom script to replace the default script |
400+
| autoscaling.patchObjectFinalizers.patchFinalizersScript | string | `""` | Define your custom script to replace the default script |
401+
| autoscaling.patchObjectFinalizers.defaultMode | int | `493` | Default mode for ConfigMap is mounted as file |
399402
| autoscaling.patchObjectFinalizers.serviceAccount | string | `""` | Define an external service account name contains permissions to patch KEDA scaled resources |
400403
| autoscaling.patchObjectFinalizers.imagePullSecret | string | `""` | Custom pull secret for container in patch job |
401404
| autoscaling.patchObjectFinalizers.resources | object | `{"limits":{"cpu":"200m","memory":"500Mi"},"requests":{"cpu":"100m","memory":"200Mi"}}` | Define resources for container in patch job |

‎charts/selenium-grid/templates/_nameHelpers.tpl‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -260,11 +260,18 @@ Server ConfigMap fullname
260260
{{- tpl (default (include "seleniumGrid.component.name" (list "selenium-server-config" $)) .Values.serverConfigMap.nameOverride) $ | trunc 63 | trimSuffix "-" -}}
261261
{{- end -}}
262262

263+
{{/*
264+
Delete scaledObjects leafover job fullname
265+
*/}}
266+
{{- define "seleniumGrid.keda.deleteObjectsJob.fullname" -}}
267+
{{- printf "%s-scaledobjects-deletion" (tpl ( default (include "seleniumGrid.component.name" (list "selenium-patch" $)) .Values.autoscaling.patchObjectFinalizers.nameOverride) $) | trunc 63 | trimSuffix "-" -}}
268+
{{- end -}}
269+
263270
{{/*
264271
Patch scaledObjects finalizers job fullname
265272
*/}}
266273
{{- define "seleniumGrid.keda.patchObjectsJob.fullname" -}}
267-
{{- tpl ( default (include "seleniumGrid.component.name" (list "selenium-patch-scaledobjects-finalizers" $)) .Values.autoscaling.patchObjectFinalizers.nameOverride) $ | trunc 63 | trimSuffix "-" -}}
274+
{{- printf "%s-scaledobjects-finalizers" (tpl ( default (include "seleniumGrid.component.name" (list "selenium-patch" $)) .Values.autoscaling.patchObjectFinalizers.nameOverride) $) | trunc 63 | trimSuffix "-" -}}
268275
{{- end -}}
269276

270277
{{/*

‎charts/selenium-grid/templates/patch-keda/delete-keda-objects-job.yaml‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
{{- if and (eq (include "seleniumGrid.useKEDA" $) "true") $.Values.autoscaling.patchObjectFinalizers.enabled }}
2+
apiVersion: batch/v1
3+
kind: Job
4+
metadata:
5+
name: {{ template "seleniumGrid.keda.deleteObjectsJob.fullname" $ }}
6+
namespace: {{ .Release.Namespace }}
7+
annotations: &patch_objects_job_annotations
8+
"helm.sh/hook-weight": "-10"
9+
{{- with $.Values.autoscaling.patchObjectFinalizers.annotations }}
10+
{{- toYaml . | nindent 4 }}
11+
{{- end }}
12+
labels: &patch_objects_job_labels
13+
app: {{ template "seleniumGrid.keda.deleteObjectsJob.fullname" $ }}
14+
app.kubernetes.io/name: {{ template "seleniumGrid.keda.deleteObjectsJob.fullname" $ }}
15+
{{- include "seleniumGrid.commonLabels" $ | nindent 4 }}
16+
{{- with $.Values.customLabels }}
17+
{{- toYaml . | nindent 4 }}
18+
{{- end }}
19+
spec:
20+
template:
21+
metadata:
22+
{{- with $.Values.autoscaling.patchObjectFinalizers.annotations }}
23+
annotations: *patch_objects_job_annotations
24+
{{- end }}
25+
labels: *patch_objects_job_labels
26+
name: {{ template "seleniumGrid.keda.deleteObjectsJob.fullname" $ }}
27+
spec:
28+
activeDeadlineSeconds: {{ $.Values.autoscaling.patchObjectFinalizers.activeDeadlineSeconds }}
29+
serviceAccountName: {{ default (include "seleniumGrid.serviceAccount.fullname" $) $.Values.autoscaling.patchObjectFinalizers.serviceAccount }}
30+
serviceAccount: {{ default (include "seleniumGrid.serviceAccount.fullname" $) $.Values.autoscaling.patchObjectFinalizers.serviceAccount }}
31+
containers:
32+
- name: kubectl
33+
image: {{ $.Values.global.seleniumGrid.kubectlImage }}
34+
command: ["/bin/bash", "-c", "/deleteObjectsScript.sh"]
35+
volumeMounts:
36+
- name: cleanup-script
37+
mountPath: /deleteObjectsScript.sh
38+
subPath: deleteObjectsScript.sh
39+
{{- with $.Values.autoscaling.patchObjectFinalizers.resources }}
40+
resources: {{ toYaml . | nindent 12 }}
41+
{{- end }}
42+
volumes:
43+
- name: cleanup-script
44+
configMap:
45+
name: {{ template "seleniumGrid.keda.patchObjectsJob.fullname" $ }}
46+
defaultMode: {{ $.Values.autoscaling.patchObjectFinalizers.defaultMode }}
47+
{{- if or $.Values.global.seleniumGrid.imagePullSecret $.Values.autoscaling.patchObjectFinalizers.imagePullSecret }}
48+
imagePullSecrets:
49+
- name: {{ default $.Values.global.seleniumGrid.imagePullSecret $.Values.autoscaling.patchObjectFinalizers.imagePullSecret }}
50+
{{- end }}
51+
restartPolicy: Never
52+
{{- with .Values.autoscaling.patchObjectFinalizers.nodeSelector }}
53+
nodeSelector: {{- toYaml . | nindent 8 }}
54+
{{- end }}
55+
{{- end }}

‎charts/selenium-grid/templates/patch-keda/patch-keda-objects-cm.yaml‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
{{- if and (eq (include "seleniumGrid.useKEDA" $) "true") $.Values.autoscaling.patchObjectFinalizers.enabled }}
2+
apiVersion: v1
3+
kind: ConfigMap
4+
metadata:
5+
name: {{ template "seleniumGrid.keda.patchObjectsJob.fullname" $ }}
6+
namespace: {{ .Release.Namespace }}
7+
annotations:
8+
"helm.sh/hook-weight": "-20"
9+
{{- with $.Values.autoscaling.patchObjectFinalizers.annotations }}
10+
{{- toYaml . | nindent 4 }}
11+
{{- end }}
12+
labels:
13+
app: {{ template "seleniumGrid.keda.patchObjectsJob.fullname" $ }}
14+
app.kubernetes.io/name: {{ template "seleniumGrid.keda.patchObjectsJob.fullname" $ }}
15+
{{- include "seleniumGrid.commonLabels" $ | nindent 4 }}
16+
{{- with $.Values.customLabels }}
17+
{{- toYaml . | nindent 4 }}
18+
{{- end }}
19+
data:
20+
{{- if $.Values.autoscaling.patchObjectFinalizers.deleteObjectsScript }}
21+
deleteObjectsScript.sh: {{- toYaml $.Values.autoscaling.patchObjectFinalizers.deleteObjectsScript | nindent 4 }}
22+
{{- else }}
23+
deleteObjectsScript.sh: |
24+
#!/bin/bash
25+
set -e
26+
set -x
27+
echo "Delete ScaledObjects, ScaledJobs and HPAs for {{ .Release.Name }} when upgrading or disabling autoscaling."
28+
for i in $(kubectl get ScaledObjects -n {{ .Release.Namespace }} -l app.kubernetes.io/instance={{ .Release.Name }} \
29+
-o jsonpath='{.items[*].metadata.name}{"\n"}');
30+
do
31+
if [ -n "$i" ]; then
32+
kubectl delete ScaledObjects $i -n {{ .Release.Namespace }}
33+
fi
34+
done
35+
for i in $(kubectl get ScaledJobs -n {{ .Release.Namespace }} -l app.kubernetes.io/instance={{ .Release.Name }} \
36+
-o jsonpath='{.items[*].metadata.name}{"\n"}');
37+
do
38+
if [ -n "$i" ]; then
39+
kubectl delete ScaledJobs $i -n {{ .Release.Namespace }}
40+
fi
41+
done
42+
for i in $(kubectl get TriggerAuthentication -n {{ .Release.Namespace }} -l app.kubernetes.io/instance={{ .Release.Name }} \
43+
-o jsonpath='{.items[*].metadata.name}{"\n"}');
44+
do
45+
if [ -n "$i" ]; then
46+
kubectl delete TriggerAuthentication $i -n {{ .Release.Namespace }}
47+
fi
48+
done
49+
{{- end }}
50+
{{- if $.Values.autoscaling.patchObjectFinalizers.patchFinalizersScript }}
51+
patchFinalizersScript.sh: {{- toYaml $.Values.autoscaling.patchObjectFinalizers.patchFinalizersScript | nindent 4 }}
52+
{{- else }}
53+
patchFinalizersScript.sh: |
54+
#!/bin/bash
55+
set -e
56+
set -x
57+
echo "Patch finalizers of ScaledObjects, ScaledJobs and HPAs for {{ .Release.Name }} when upgrading or disabling autoscaling."
58+
for i in $(kubectl get ScaledObjects -n {{ .Release.Namespace }} -l app.kubernetes.io/instance={{ .Release.Name }} \
59+
-o jsonpath='{.items[*].metadata.name}{"\n"}');
60+
do
61+
if [ -n "$i" ]; then
62+
kubectl patch ScaledObjects $i -n {{ .Release.Namespace }} -p '{"metadata":{"finalizers":null}}' --type=merge
63+
fi
64+
done
65+
for i in $(kubectl get ScaledJobs -n {{ .Release.Namespace }} -l app.kubernetes.io/instance={{ .Release.Name }} \
66+
-o jsonpath='{.items[*].metadata.name}{"\n"}');
67+
do
68+
if [ -n "$i" ]; then
69+
kubectl patch ScaledJobs $i -n {{ .Release.Namespace }} -p '{"metadata":{"finalizers":null}}' --type=merge
70+
fi
71+
done
72+
for i in $(kubectl get TriggerAuthentication -n {{ .Release.Namespace }} -l app.kubernetes.io/instance={{ .Release.Name }} \
73+
-o jsonpath='{.items[*].metadata.name}{"\n"}');
74+
do
75+
if [ -n "$i" ]; then
76+
kubectl patch TriggerAuthentication $i -n {{ .Release.Namespace }} -p '{"metadata":{"finalizers":null}}' --type=merge
77+
fi
78+
done
79+
{{- end }}
80+
{{- end }}

‎charts/selenium-grid/templates/patch-keda/patch-keda-objects-job.yaml‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,9 @@ kind: Job
44
metadata:
55
name: {{ template "seleniumGrid.keda.patchObjectsJob.fullname" $ }}
66
namespace: {{ .Release.Namespace }}
7-
{{- with $.Values.autoscaling.patchObjectFinalizers.annotations }}
87
annotations: &patch_objects_job_annotations
8+
"helm.sh/hook-weight": "-1"
9+
{{- with $.Values.autoscaling.patchObjectFinalizers.annotations }}
910
{{- toYaml . | nindent 4 }}
1011
{{- end }}
1112
labels: &patch_objects_job_labels
@@ -30,16 +31,19 @@ spec:
3031
containers:
3132
- name: kubectl
3233
image: {{ $.Values.global.seleniumGrid.kubectlImage }}
33-
command: ["/bin/sh", "-c"]
34-
args:
35-
- |
36-
echo "Cleaning up ScaledObjects, ScaledJobs and HPAs for {{ .Release.Name }} when upgrading or disabling autoscaling."
37-
kubectl get ScaledObjects,ScaledJobs,TriggerAuthentication -n {{ .Release.Namespace }} -l component.autoscaling={{ .Release.Name }} -o=json | jq '.metadata.finalizers = null' | kubectl apply -f - || true ;
38-
kubectl delete ScaledObjects,ScaledJobs,TriggerAuthentication -n {{ .Release.Namespace }} -l component.autoscaling={{ .Release.Name }} --wait=false || true ;
39-
kubectl delete hpa -n {{ .Release.Namespace }} -l component.autoscaling={{ .Release.Name }} --wait=false || true ;
34+
command: ["/bin/bash", "-c", "/patchFinalizersScript.sh"]
35+
volumeMounts:
36+
- name: cleanup-script
37+
mountPath: /patchFinalizersScript.sh
38+
subPath: patchFinalizersScript.sh
4039
{{- with $.Values.autoscaling.patchObjectFinalizers.resources }}
4140
resources: {{ toYaml . | nindent 12 }}
4241
{{- end }}
42+
volumes:
43+
- name: cleanup-script
44+
configMap:
45+
name: {{ template "seleniumGrid.keda.patchObjectsJob.fullname" $ }}
46+
defaultMode: {{ $.Values.autoscaling.patchObjectFinalizers.defaultMode }}
4347
{{- if or $.Values.global.seleniumGrid.imagePullSecret $.Values.autoscaling.patchObjectFinalizers.imagePullSecret }}
4448
imagePullSecrets:
4549
- name: {{ default $.Values.global.seleniumGrid.imagePullSecret $.Values.autoscaling.patchObjectFinalizers.imagePullSecret }}

‎charts/selenium-grid/values.yaml‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1032,13 +1032,17 @@ autoscaling:
10321032
# -- Enable patching finalizers for KEDA scaled resources. Workaround for Hook post-upgrade selenium-grid/templates/x-node-hpa.yaml failed: object is being deleted: scaledobjects.keda.sh "x" already exists
10331033
enabled: true
10341034
# -- Deadline (in seconds) for patch job to complete
1035-
activeDeadlineSeconds: 120
1035+
activeDeadlineSeconds: 300
10361036
# -- Annotations for patch job
10371037
annotations:
10381038
"helm.sh/hook": post-install,post-upgrade,post-rollback,pre-delete
10391039
"helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation
1040-
# This should be run before all other hooks (since delete action is called), so use a negative weight
1041-
"helm.sh/hook-weight": "-1"
1040+
# -- Define your custom script to replace the default script
1041+
deleteObjectsScript: ""
1042+
# -- Define your custom script to replace the default script
1043+
patchFinalizersScript: ""
1044+
# -- Default mode for ConfigMap is mounted as file
1045+
defaultMode: 0755
10421046
# -- Define an external service account name contains permissions to patch KEDA scaled resources
10431047
serviceAccount: ""
10441048
# -- Custom pull secret for container in patch job

‎tests/charts/make/chart_test.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ cleanup() {
9292
done
9393
if [ "${SKIP_CLEANUP}" = "false" ] || [ "${CI:-false}" != "false" ]; then
9494
echo "Clean up chart release and namespace"
95-
helm delete ${RELEASE_NAME} --namespace ${SELENIUM_NAMESPACE} --no-hooks || true
95+
helm delete ${RELEASE_NAME} --namespace ${SELENIUM_NAMESPACE} || true
9696
wait_for_terminated
9797
sudo chmod -R 777 ${HOST_PATH}/logs
9898
fi

0 commit comments

Comments
 (0)