Merge pull request kubernetes#93618 from Jiawei0227/e2e-csi-mock-flaky

Fix flaky test: wait for mock CSI Driver bringup to perform e2e test

Author: k8s-ci-robot

test/e2e/storage/csi_mock_volume.go

@@ -151,11 +151,16 @@ var _ = utils.SIGDescribe("CSI mock volume", func() {
 
 		if tp.registerDriver {
 			err = waitForCSIDriver(cs, m.config.GetUniqueDriverName())
-			framework.ExpectNoError(err, "Failed to get CSIDriver : %v", err)
+			framework.ExpectNoError(err, "Failed to get CSIDriver %v", m.config.GetUniqueDriverName())
 			m.testCleanups = append(m.testCleanups, func() {
 				destroyCSIDriver(cs, m.config.GetUniqueDriverName())
 			})
 		}
+
+		// Wait for the CSIDriver actually get deployed and CSINode object to be generated.
+		// This indicates the mock CSI driver pod is up and running healthy.
+		err = drivers.WaitForCSIDriverRegistrationOnNode(m.config.ClientNodeSelection.Name, m.config.GetUniqueDriverName(), cs)
+		framework.ExpectNoError(err, "Failed to register CSIDriver %v", m.config.GetUniqueDriverName())
 	}
 
 	createPod := func(ephemeral bool) (class *storagev1.StorageClass, claim *v1.PersistentVolumeClaim, pod *v1.Pod) {

test/e2e/storage/drivers/csi.go

@@ -608,17 +608,25 @@ func waitForCSIDriverRegistrationOnAllNodes(driverName string, cs clientset.Inte
 		return err
 	}
 	for _, node := range nodes.Items {
-		if err := waitForCSIDriverRegistrationOnNode(node.Name, driverName, cs); err != nil {
+		if err := WaitForCSIDriverRegistrationOnNode(node.Name, driverName, cs); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-func waitForCSIDriverRegistrationOnNode(nodeName string, driverName string, cs clientset.Interface) error {
-	const csiNodeRegisterTimeout = 5 * time.Minute
+// WaitForCSIDriverRegistrationOnNode waits for the CSINode object generated by the node-registrar on a certain node
+func WaitForCSIDriverRegistrationOnNode(nodeName string, driverName string, cs clientset.Interface) error {
+	framework.Logf("waiting for CSIDriver %v to register on node %v", driverName, nodeName)
 
-	waitErr := wait.PollImmediate(10*time.Second, csiNodeRegisterTimeout, func() (bool, error) {
+	// About 8.6 minutes timeout
+	backoff := wait.Backoff{
+		Duration: 2 * time.Second,
+		Factor:   1.5,
+		Steps:    12,
+	}
+
+	waitErr := wait.ExponentialBackoff(backoff, func() (bool, error) {
 		csiNode, err := cs.StorageV1().CSINodes().Get(context.TODO(), nodeName, metav1.GetOptions{})
 		if err != nil && !apierrors.IsNotFound(err) {
 			return false, err