Wait for mock CSI Driver bringup to perform e2e test

In our current mock CSI driver e2e test, we are not waiting
for the CSI driver register successfully to perform test
including provision PVC. This can lead to timeout when the
csi driver takes longer to register the socket.

This change adds the waiting part so that the system will
wait for up to 10 minutes for the driver to be ready. This
normally won't take this long. However, under a resource
constraint environment it can take longer than expected time.

kubernetes#93358

Author: Jiawei0227

test/e2e/storage/csi_mock_volume.go

@@ -151,11 +151,16 @@ var _ = utils.SIGDescribe("CSI mock volume", func() {
 
 		if tp.registerDriver {
 			err = waitForCSIDriver(cs, m.config.GetUniqueDriverName())
-			framework.ExpectNoError(err, "Failed to get CSIDriver : %v", err)
+			framework.ExpectNoError(err, "Failed to get CSIDriver %v", m.config.GetUniqueDriverName())
 			m.testCleanups = append(m.testCleanups, func() {
 				destroyCSIDriver(cs, m.config.GetUniqueDriverName())
 			})
 		}
+
+		// Wait for the CSIDriver actually get deployed and CSINode object to be generated.
+		// This indicates the mock CSI driver pod is up and running healthy.
+		err = drivers.WaitForCSIDriverRegistrationOnNode(m.config.ClientNodeSelection.Name, m.config.GetUniqueDriverName(), cs)
+		framework.ExpectNoError(err, "Failed to register CSIDriver %v", m.config.GetUniqueDriverName())
 	}
 
 	createPod := func(ephemeral bool) (class *storagev1.StorageClass, claim *v1.PersistentVolumeClaim, pod *v1.Pod) {

test/e2e/storage/drivers/csi.go

@@ -608,17 +608,25 @@ func waitForCSIDriverRegistrationOnAllNodes(driverName string, cs clientset.Inte
 		return err
 	}
 	for _, node := range nodes.Items {
-		if err := waitForCSIDriverRegistrationOnNode(node.Name, driverName, cs); err != nil {
+		if err := WaitForCSIDriverRegistrationOnNode(node.Name, driverName, cs); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-func waitForCSIDriverRegistrationOnNode(nodeName string, driverName string, cs clientset.Interface) error {
-	const csiNodeRegisterTimeout = 5 * time.Minute
+// WaitForCSIDriverRegistrationOnNode waits for the CSINode object generated by the node-registrar on a certain node
+func WaitForCSIDriverRegistrationOnNode(nodeName string, driverName string, cs clientset.Interface) error {
+	framework.Logf("waiting for CSIDriver %v to register on node %v", driverName, nodeName)
 
-	waitErr := wait.PollImmediate(10*time.Second, csiNodeRegisterTimeout, func() (bool, error) {
+	// About 8.6 minutes timeout
+	backoff := wait.Backoff{
+		Duration: 2 * time.Second,
+		Factor:   1.5,
+		Steps:    12,
+	}
+
+	waitErr := wait.ExponentialBackoff(backoff, func() (bool, error) {
 		csiNode, err := cs.StorageV1().CSINodes().Get(context.TODO(), nodeName, metav1.GetOptions{})
 		if err != nil && !apierrors.IsNotFound(err) {
 			return false, err