feat(helm): extend helm template to support startup taints

we extend the helm chart to (1) automatically add provisionerNotReadyNodeTaintKey tolerations to the daemonset (2) pass the provisionerNotReadyNodeTaintKey and removeNodeNotReadyTaint to config map, and (3) extend the rbac to so the pod can update the node taints.

Author: saidjawad

helm/provisioner/templates/configmap.yaml

@@ -24,6 +24,10 @@ data:
 {{- if .Values.setPVOwnerRef }}
   setPVOwnerRef: "true"
 {{- end }}
+{{- if .Values.removeNodeNotReadyTaint }}
+  removeNodeNotReadyTaint: {{ .Values.removeNodeNotReadyTaint | quote }}
+  provisionerNotReadyNodeTaintKey: {{ .Values.provisionerNotReadyNodeTaintKey | quote }}
+{{- end }}
 {{- if .Values.useJobForCleaning }}
   useJobForCleaning: "yes"
 {{- end }}

helm/provisioner/templates/daemonset_linux.yaml

@@ -44,9 +44,15 @@ spec:
 {{- if .Values.nodeSelector }}
         {{ toYaml .Values.nodeSelector | nindent 8 }}
 {{- end }}
-{{- if .Values.tolerations }}
+{{- if or (.Values.tolerations) (.Values.removeNodeNotReadyTaint) }}
       tolerations:
-        {{ toYaml .Values.tolerations | nindent 8 }}
+{{- if .Values.tolerations}}
+        {{- toYaml .Values.tolerations | nindent 8 }}
+{{- end }}
+{{- if .Values.removeNodeNotReadyTaint }}
+        - key: {{ .Values.provisionerNotReadyNodeTaintKey}}
+          operator: Exists
+{{- end }}
 {{- end }}
 {{- if .Values.affinity }}
       affinity:

helm/provisioner/templates/rbac.yaml

@@ -23,7 +23,7 @@ rules:
   verbs: ["create", "update", "patch"]
 - apiGroups: [""]
   resources: ["nodes"]
-  verbs: ["get"]
+  verbs: ["get", "update"]
 {{- if .Values.rbac.extraRules }}
 {{ toYaml .Values.rbac.extraRules }}
 {{- end}}

helm/provisioner/values.yaml

@@ -133,6 +133,17 @@ nodeSelectorWindows: {}
 # Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/
 tolerations: []
 #
+# If configured, removeNodeNotReadyTaint will add provisionerNotReadyNodeTaintKey as the toleration key
+# to the DaemonSet PodSpec.
+#
+# removeNodeNotReadyTaint controls whether the provisioner should remove the provisionerNotReadyNodeTaintKey
+# once it becomes ready.
+removeNodeNotReadyTaint: false
+#
+# The key of the startup taint that provisioner will remove once it becomes ready.
+# Ref: https://karpenter.sh/docs/concepts/nodepools/#cilium-startup-taint
+provisionerNotReadyNodeTaintKey: "sig-storage-local-static-provisioner/agent-not-ready"
+#
 # If configured, affinity will add a affinity filed to the DeamonSet PodSpec.
 # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity
 affinity: {}