impl compensation mechanism for driver.removeNotReadyTaint()

Author: abbshr

pkg/driver/driver.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"net"
 	"strings"
+	"time"
 
 	"github.com/container-storage-interface/spec/lib/go/csi"
 	"google.golang.org/grpc"
@@ -129,10 +130,7 @@ func (d *Driver) Run() error {
 
 	// Remove taint from node to indicate driver startup success
 	// This is done at the last possible moment to prevent race conditions or false positive removals
-	err = removeNotReadyTaint(cloud.DefaultKubernetesAPIClient)
-	if err != nil {
-		klog.ErrorS(err, "Unexpected failure when attempting to remove node taint(s)")
-	}
+	go tryRemoveNotReadyTaintUntilSucceed(cloud.DefaultKubernetesAPIClient, time.Second)
 
 	klog.Infof("Listening for connections on address: %#v", listener.Addr())
 	return d.srv.Serve(listener)

pkg/driver/node.go

@@ -25,6 +25,7 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/container-storage-interface/spec/lib/go/csi"
 	"github.com/kubernetes-sigs/aws-efs-csi-driver/pkg/cloud"
@@ -452,7 +453,7 @@ type JSONPatch struct {
 	Value interface{} `json:"value"`
 }
 
-// removeNotReadyTaint removes the taint ebs.csi.aws.com/agent-not-ready from the local node
+// removeNotReadyTaint removes the taint efs.csi.aws.com/agent-not-ready from the local node
 // This taint can be optionally applied by users to prevent startup race conditions such as
 // https://github.com/kubernetes/kubernetes/issues/95911
 func removeNotReadyTaint(k8sClient cloud.KubernetesAPIClient) error {
@@ -512,3 +513,16 @@ func removeNotReadyTaint(k8sClient cloud.KubernetesAPIClient) error {
 	klog.InfoS("Removed taint(s) from local node", "node", nodeName)
 	return nil
 }
+
+// remove taint may failed, this keep retring until succeed, make sure the taint will eventually being removed
+func tryRemoveNotReadyTaintUntilSucceed(k8sClient cloud.KubernetesAPIClient, interval time.Duration) {
+	for {
+		err := removeNotReadyTaint(k8sClient)
+		if err == nil {
+			return
+		}
+
+		klog.ErrorS(err, "Unexpected failure when attempting to remove node taint(s)")
+		time.Sleep(interval)
+	}
+}