Merge pull request #501 from saidjawad/support_for_karpenter_start_up_taint

k8s-ci-robot · web-flow · commit 3cd5ae5d6fa7 · 2025-07-03T20:37:24.000-07:00
Support for removing start up taint
diff --git a/cmd/local-volume-provisioner/main.go b/cmd/local-volume-provisioner/main.go
@@ -17,7 +17,6 @@ limitations under the License.
 package main
 
 import (
-	"context"
 	"flag"
 	"log"
 	"math/rand"
@@ -33,15 +32,12 @@ import (
 	"sigs.k8s.io/sig-storage-local-static-provisioner/pkg/deleter"
 	"sigs.k8s.io/sig-storage-local-static-provisioner/pkg/metrics"
 	"sigs.k8s.io/sig-storage-local-static-provisioner/pkg/metrics/collectors"
+	"sigs.k8s.io/sig-storage-local-static-provisioner/pkg/util"
 	"sigs.k8s.io/sig-storage-local-static-provisioner/pkg/watcher"
 
-	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/kubernetes"
 )
 
-const maxGetNodesRetries = 3
-
 var (
 	optListenAddress string
 	optMetricsPath   string
@@ -86,7 +82,7 @@ func main() {
 	}
 
 	client := common.SetupClient()
-	node := getNode(client, nodeName)
+	node := util.GetNode(client.CoreV1(), nodeName)
 
 	configUpdate := make(chan common.ProvisionerConfiguration)
 	defer close(configUpdate)
@@ -115,22 +111,3 @@ func main() {
 	http.Handle(optMetricsPath, promhttp.Handler())
 	log.Fatal(http.ListenAndServe(optListenAddress, nil))
 }
-
-func getNode(client *kubernetes.Clientset, name string) *v1.Node {
-	var retries int
-
-	for {
-		node, err := client.CoreV1().Nodes().Get(context.TODO(), name, metav1.GetOptions{})
-		if err == nil {
-			return node
-		}
-
-		retries++
-		klog.Infof("Could not get node information (remaining retries: %d): %v", maxGetNodesRetries-retries, err)
-
-		if retries >= maxGetNodesRetries {
-			klog.Fatalf("Could not get node information: %v", err)
-		}
-		time.Sleep(time.Second)
-	}
-}
diff --git a/helm/generated_examples/additional-volumes.yaml b/helm/generated_examples/additional-volumes.yaml
@@ -59,7 +59,7 @@ rules:
   verbs: ["create", "update", "patch"]
 - apiGroups: [""]
   resources: ["nodes"]
-  verbs: ["get"]
+  verbs: ["get", "update"]
 ---
 # Source: local-static-provisioner/templates/rbac.yaml
 apiVersion: rbac.authorization.k8s.io/v1
diff --git a/helm/generated_examples/baremetal-affinity.yaml b/helm/generated_examples/baremetal-affinity.yaml
@@ -71,7 +71,7 @@ rules:
   verbs: ["create", "update", "patch"]
 - apiGroups: [""]
   resources: ["nodes"]
-  verbs: ["get"]
+  verbs: ["get", "update"]
 ---
 # Source: local-static-provisioner/templates/rbac.yaml
 apiVersion: rbac.authorization.k8s.io/v1
diff --git a/helm/generated_examples/baremetal-cleanbyjobs.yaml b/helm/generated_examples/baremetal-cleanbyjobs.yaml
@@ -71,7 +71,7 @@ rules:
   verbs: ["create", "update", "patch"]
 - apiGroups: [""]
   resources: ["nodes"]
-  verbs: ["get"]
+  verbs: ["get", "update"]
 ---
 # Source: local-static-provisioner/templates/rbac.yaml
 apiVersion: rbac.authorization.k8s.io/v1
diff --git a/helm/generated_examples/baremetal-default-storage.yaml b/helm/generated_examples/baremetal-default-storage.yaml
@@ -70,7 +70,7 @@ rules:
   verbs: ["create", "update", "patch"]
 - apiGroups: [""]
   resources: ["nodes"]
-  verbs: ["get"]
+  verbs: ["get", "update"]
 ---
 # Source: local-static-provisioner/templates/rbac.yaml
 apiVersion: rbac.authorization.k8s.io/v1
diff --git a/helm/generated_examples/baremetal-nodeselector.yaml b/helm/generated_examples/baremetal-nodeselector.yaml
@@ -71,7 +71,7 @@ rules:
   verbs: ["create", "update", "patch"]
 - apiGroups: [""]
   resources: ["nodes"]
-  verbs: ["get"]
+  verbs: ["get", "update"]
 ---
 # Source: local-static-provisioner/templates/rbac.yaml
 apiVersion: rbac.authorization.k8s.io/v1
diff --git a/helm/generated_examples/baremetal-priority-critical.yaml b/helm/generated_examples/baremetal-priority-critical.yaml
@@ -71,7 +71,7 @@ rules:
   verbs: ["create", "update", "patch"]
 - apiGroups: [""]
   resources: ["nodes"]
-  verbs: ["get"]
+  verbs: ["get", "update"]
 ---
 # Source: local-static-provisioner/templates/rbac.yaml
 apiVersion: rbac.authorization.k8s.io/v1
diff --git a/helm/generated_examples/baremetal-priority-noncritical.yaml b/helm/generated_examples/baremetal-priority-noncritical.yaml
@@ -71,7 +71,7 @@ rules:
   verbs: ["create", "update", "patch"]
 - apiGroups: [""]
   resources: ["nodes"]
-  verbs: ["get"]
+  verbs: ["get", "update"]
 ---
 # Source: local-static-provisioner/templates/rbac.yaml
 apiVersion: rbac.authorization.k8s.io/v1
diff --git a/helm/generated_examples/baremetal-prometheus.yaml b/helm/generated_examples/baremetal-prometheus.yaml
@@ -71,7 +71,7 @@ rules:
   verbs: ["create", "update", "patch"]
 - apiGroups: [""]
   resources: ["nodes"]
-  verbs: ["get"]
+  verbs: ["get", "update"]
 ---
 # Source: local-static-provisioner/templates/rbac.yaml
 apiVersion: rbac.authorization.k8s.io/v1
diff --git a/helm/generated_examples/baremetal-provisioner.yaml b/helm/generated_examples/baremetal-provisioner.yaml
@@ -70,7 +70,7 @@ rules:
   verbs: ["create", "update", "patch"]
 - apiGroups: [""]
   resources: ["nodes"]
-  verbs: ["get"]
+  verbs: ["get", "update"]
 ---
 # Source: local-static-provisioner/templates/rbac.yaml
 apiVersion: rbac.authorization.k8s.io/v1
diff --git a/helm/generated_examples/baremetal-resyncperiod.yaml b/helm/generated_examples/baremetal-resyncperiod.yaml
@@ -71,7 +71,7 @@ rules:
   verbs: ["create", "update", "patch"]
 - apiGroups: [""]
   resources: ["nodes"]
-  verbs: ["get"]
+  verbs: ["get", "update"]
 ---
 # Source: local-static-provisioner/templates/rbac.yaml
 apiVersion: rbac.authorization.k8s.io/v1
diff --git a/helm/generated_examples/baremetal-tolerations.yaml b/helm/generated_examples/baremetal-tolerations.yaml
@@ -74,7 +74,7 @@ rules:
   verbs: ["create", "update", "patch"]
 - apiGroups: [""]
   resources: ["nodes"]
-  verbs: ["get"]
+  verbs: ["get", "update"]
 ---
 # Source: local-static-provisioner/templates/rbac.yaml
 apiVersion: rbac.authorization.k8s.io/v1
@@ -128,7 +128,6 @@ spec:
       nodeSelector:
         kubernetes.io/os: linux
       tolerations:
-        
         - effect: NoSchedule
           key: node-role.kubernetes.io/master
       containers:
diff --git a/helm/generated_examples/baremetal-with-resource-limits.yaml b/helm/generated_examples/baremetal-with-resource-limits.yaml
@@ -71,7 +71,7 @@ rules:
   verbs: ["create", "update", "patch"]
 - apiGroups: [""]
   resources: ["nodes"]
-  verbs: ["get"]
+  verbs: ["get", "update"]
 ---
 # Source: local-static-provisioner/templates/rbac.yaml
 apiVersion: rbac.authorization.k8s.io/v1
diff --git a/helm/generated_examples/baremetal.yaml b/helm/generated_examples/baremetal.yaml
@@ -71,7 +71,7 @@ rules:
   verbs: ["create", "update", "patch"]
 - apiGroups: [""]
   resources: ["nodes"]
-  verbs: ["get"]
+  verbs: ["get", "update"]
 ---
 # Source: local-static-provisioner/templates/rbac.yaml
 apiVersion: rbac.authorization.k8s.io/v1
diff --git a/helm/generated_examples/development-gce.yaml b/helm/generated_examples/development-gce.yaml
@@ -68,7 +68,7 @@ rules:
   verbs: ["create", "update", "patch"]
 - apiGroups: [""]
   resources: ["nodes"]
-  verbs: ["get"]
+  verbs: ["get", "update"]
 ---
 # Source: local-static-provisioner/templates/rbac.yaml
 apiVersion: rbac.authorization.k8s.io/v1
diff --git a/helm/generated_examples/development-gke.yaml b/helm/generated_examples/development-gke.yaml
@@ -68,7 +68,7 @@ rules:
   verbs: ["create", "update", "patch"]
 - apiGroups: [""]
   resources: ["nodes"]
-  verbs: ["get"]
+  verbs: ["get", "update"]
 ---
 # Source: local-static-provisioner/templates/rbac.yaml
 apiVersion: rbac.authorization.k8s.io/v1
diff --git a/helm/generated_examples/eks-nvme-ssd.yaml b/helm/generated_examples/eks-nvme-ssd.yaml
@@ -67,7 +67,7 @@ rules:
   verbs: ["create", "update", "patch"]
 - apiGroups: [""]
   resources: ["nodes"]
-  verbs: ["get"]
+  verbs: ["get", "update"]
 ---
 # Source: local-static-provisioner/templates/rbac.yaml
 apiVersion: rbac.authorization.k8s.io/v1
diff --git a/helm/generated_examples/gce-retain.yaml b/helm/generated_examples/gce-retain.yaml
@@ -85,7 +85,7 @@ rules:
   verbs: ["create", "update", "patch"]
 - apiGroups: [""]
   resources: ["nodes"]
-  verbs: ["get"]
+  verbs: ["get", "update"]
 ---
 # Source: local-static-provisioner/templates/rbac.yaml
 apiVersion: rbac.authorization.k8s.io/v1
diff --git a/helm/generated_examples/gce.yaml b/helm/generated_examples/gce.yaml
@@ -85,7 +85,7 @@ rules:
   verbs: ["create", "update", "patch"]
 - apiGroups: [""]
   resources: ["nodes"]
-  verbs: ["get"]
+  verbs: ["get", "update"]
 ---
 # Source: local-static-provisioner/templates/rbac.yaml
 apiVersion: rbac.authorization.k8s.io/v1
diff --git a/helm/generated_examples/gke-nvme-ssd-block-raid.yaml b/helm/generated_examples/gke-nvme-ssd-block-raid.yaml
@@ -68,7 +68,7 @@ rules:
   verbs: ["create", "update", "patch"]
 - apiGroups: [""]
   resources: ["nodes"]
-  verbs: ["get"]
+  verbs: ["get", "update"]
 ---
 # Source: local-static-provisioner/templates/rbac.yaml
 apiVersion: rbac.authorization.k8s.io/v1
diff --git a/helm/generated_examples/gke.yaml b/helm/generated_examples/gke.yaml
@@ -68,7 +68,7 @@ rules:
   verbs: ["create", "update", "patch"]
 - apiGroups: [""]
   resources: ["nodes"]
-  verbs: ["get"]
+  verbs: ["get", "update"]
 ---
 # Source: local-static-provisioner/templates/rbac.yaml
 apiVersion: rbac.authorization.k8s.io/v1
diff --git a/helm/provisioner/templates/configmap.yaml b/helm/provisioner/templates/configmap.yaml
@@ -24,6 +24,10 @@ data:
 {{- if .Values.setPVOwnerRef }}
   setPVOwnerRef: "true"
 {{- end }}
+{{- if .Values.removeNodeNotReadyTaint }}
+  removeNodeNotReadyTaint: {{ .Values.removeNodeNotReadyTaint | quote }}
+  provisionerNotReadyNodeTaintKey: {{ .Values.provisionerNotReadyNodeTaintKey | quote }}
+{{- end }}
 {{- if .Values.useJobForCleaning }}
   useJobForCleaning: "yes"
 {{- end }}
diff --git a/helm/provisioner/templates/daemonset_linux.yaml b/helm/provisioner/templates/daemonset_linux.yaml
@@ -44,9 +44,15 @@ spec:
 {{- if .Values.nodeSelector }}
         {{ toYaml .Values.nodeSelector | nindent 8 }}
 {{- end }}
-{{- if .Values.tolerations }}
+{{- if or (.Values.tolerations) (.Values.removeNodeNotReadyTaint) }}
       tolerations:
-        {{ toYaml .Values.tolerations | nindent 8 }}
+{{- if .Values.tolerations}}
+        {{- toYaml .Values.tolerations | nindent 8 }}
+{{- end }}
+{{- if .Values.removeNodeNotReadyTaint }}
+        - key: {{ .Values.provisionerNotReadyNodeTaintKey}}
+          operator: Exists
+{{- end }}
 {{- end }}
 {{- if .Values.affinity }}
       affinity:
diff --git a/helm/provisioner/templates/rbac.yaml b/helm/provisioner/templates/rbac.yaml
@@ -23,7 +23,7 @@ rules:
   verbs: ["create", "update", "patch"]
 - apiGroups: [""]
   resources: ["nodes"]
-  verbs: ["get"]
+  verbs: ["get", "update"]
 {{- if .Values.rbac.extraRules }}
 {{ toYaml .Values.rbac.extraRules }}
 {{- end}}
diff --git a/helm/provisioner/values.yaml b/helm/provisioner/values.yaml
@@ -133,6 +133,17 @@ nodeSelectorWindows: {}
 # Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/
 tolerations: []
 #
+# If configured, removeNodeNotReadyTaint will add provisionerNotReadyNodeTaintKey as the toleration key
+# to the DaemonSet PodSpec.
+#
+# removeNodeNotReadyTaint controls whether the provisioner should remove the provisionerNotReadyNodeTaintKey
+# once it becomes ready.
+removeNodeNotReadyTaint: false
+#
+# The key of the startup taint that provisioner will remove once it becomes ready.
+# Ref: https://karpenter.sh/docs/concepts/nodepools/#cilium-startup-taint
+provisionerNotReadyNodeTaintKey: "sig-storage-local-static-provisioner/agent-not-ready"
+#
 # If configured, affinity will add a affinity filed to the DeamonSet PodSpec.
 # Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity
 affinity: {}
diff --git a/pkg/common/common.go b/pkg/common/common.go
@@ -119,6 +119,11 @@ type UserConfig struct {
 	LabelsForPV map[string]string
 	// SetPVOwnerRef indicates if PVs should be dependents of the owner Node
 	SetPVOwnerRef bool
+	// RemoveNodeNotReadyTaint indicates if the provisioner should remove the taint with provisionerNotReadyNodeTaintKey
+	// once it becomes ready.
+	RemoveNodeNotReadyTaint bool
+	// ProvisionerNotReadyNodeTaintKey is the key of the startup taint that provisioner will remove once it becomes ready.
+	ProvisionerNotReadyNodeTaintKey string
 }
 
 // MountConfig stores a configuration for discoverying a specific storageclass
@@ -228,6 +233,13 @@ type ProvisionerConfiguration struct {
 	LabelsForPV map[string]string `json:"labelsForPV" yaml:"labelsForPV"`
 	// SetPVOwnerRef indicates if PVs should be dependents of the owner Node, default to false
 	SetPVOwnerRef bool `json:"setPVOwnerRef" yaml:"setPVOwnerRef"`
+	// RemoveNodeNotReadyTaint controls whether the provisioner should remove the taint with provisionerNotReadyNodeTaintKey
+	// once it becomes ready.
+	// +optional
+	RemoveNodeNotReadyTaint bool `json:"removeNodeNotReadyTaint" yaml:"removeNodeNotReadyTaint"`
+	// ProvisionerNotReadyNodeTaintKey is the key of the startup taint that provisioner will remove once it becomes ready.
+	// +optional
+	ProvisionerNotReadyNodeTaintKey string `json:"provisionerNotReadyNodeTaintKey" yaml:"provisionerNotReadyNodeTaintKey"`
 }
 
 // CreateLocalPVSpec returns a PV spec that can be used for PV creation
@@ -403,18 +415,20 @@ func insertSpaces(original string) string {
 // UserConfigFromProvisionerConfig creates a UserConfig from the provided ProvisionerConfiguration struct
 func UserConfigFromProvisionerConfig(node *v1.Node, namespace, jobImage string, config ProvisionerConfiguration) *UserConfig {
 	return &UserConfig{
-		Node:              node,
-		DiscoveryMap:      config.StorageClassConfig,
-		NodeLabelsForPV:   config.NodeLabelsForPV,
-		UseAlphaAPI:       config.UseAlphaAPI,
-		UseJobForCleaning: config.UseJobForCleaning,
-		MinResyncPeriod:   config.MinResyncPeriod,
-		UseNodeNameOnly:   config.UseNodeNameOnly,
-		Namespace:         namespace,
-		JobContainerImage: jobImage,
-		JobTolerations:    config.JobTolerations,
-		LabelsForPV:       config.LabelsForPV,
-		SetPVOwnerRef:     config.SetPVOwnerRef,
+		Node:                            node,
+		DiscoveryMap:                    config.StorageClassConfig,
+		NodeLabelsForPV:                 config.NodeLabelsForPV,
+		UseAlphaAPI:                     config.UseAlphaAPI,
+		UseJobForCleaning:               config.UseJobForCleaning,
+		MinResyncPeriod:                 config.MinResyncPeriod,
+		UseNodeNameOnly:                 config.UseNodeNameOnly,
+		Namespace:                       namespace,
+		JobContainerImage:               jobImage,
+		JobTolerations:                  config.JobTolerations,
+		LabelsForPV:                     config.LabelsForPV,
+		SetPVOwnerRef:                   config.SetPVOwnerRef,
+		RemoveNodeNotReadyTaint:         config.RemoveNodeNotReadyTaint,
+		ProvisionerNotReadyNodeTaintKey: config.ProvisionerNotReadyNodeTaintKey,
 	}
 }
 
diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go
@@ -28,6 +28,7 @@ import (
 	"sigs.k8s.io/sig-storage-local-static-provisioner/pkg/common"
 	"sigs.k8s.io/sig-storage-local-static-provisioner/pkg/deleter"
 	"sigs.k8s.io/sig-storage-local-static-provisioner/pkg/discovery"
+	nodetaint "sigs.k8s.io/sig-storage-local-static-provisioner/pkg/node-taint"
 	"sigs.k8s.io/sig-storage-local-static-provisioner/pkg/populator"
 	"sigs.k8s.io/sig-storage-local-static-provisioner/pkg/util"
 
@@ -161,6 +162,9 @@ func StartLocalController(signal *signal, client *kubernetes.Clientset, ptable d
 		go jobController.Run(jobControllerStopChan)
 	}
 	klog.Info("Controller started\n")
+
+	nodeTaintRemover := nodetaint.NewRemover(runtimeConfig)
+
 	for {
 		select {
 		case stopped := <-signal.closing:
@@ -174,6 +178,9 @@ func StartLocalController(signal *signal, client *kubernetes.Clientset, ptable d
 		default:
 			deleter.DeletePVs()
 			discoverer.DiscoverLocalVolumes()
+			if !nodeTaintRemover.ShouldRemoveTaint() && discoverer.Readyz.Check(nil) == nil {
+				nodeTaintRemover.RemoveTaintWithBackoff()
+			}
 			time.Sleep(discoveryPeriod)
 		}
 	}
diff --git a/pkg/node-taint/node_taint.go b/pkg/node-taint/node_taint.go
diff --git a/pkg/node-taint/node_taint_test.go b/pkg/node-taint/node_taint_test.go
diff --git a/pkg/util/node_util.go b/pkg/util/node_util.go
