test: restart driver e2e test

fix

Author: andyzhangx

test/e2e/dynamic_provisioning_test.go

@@ -568,4 +568,92 @@ var _ = ginkgo.Describe("[blob-csi-e2e] Dynamic Provisioning", func() {
 		}
 		test.Run(cs, ns)
 	})
+
+	ginkgo.It("nfs volume mount is still valid after driver restart [blob.csi.azure.com]", func() {
+		// print driver logs before driver restart
+		blobLog := testCmd{
+			command:  "bash",
+			args:     []string{"test/utils/blob_log.sh"},
+			startLog: "===================blob log (before restart)===================",
+			endLog:   "====================================================================",
+		}
+		execTestCmd([]testCmd{blobLog})
+
+		pod := testsuites.PodDetails{
+			Cmd: "echo 'hello world' >> /mnt/test-1/data && while true; do sleep 3600; done",
+			Volumes: []testsuites.VolumeDetails{
+				{
+					ClaimSize: "10Gi",
+					VolumeMount: testsuites.VolumeMountDetails{
+						NameGenerate:      "test-volume-",
+						MountPathGenerate: "/mnt/test-",
+					},
+				},
+			},
+		}
+
+		podCheckCmd := []string{"cat", "/mnt/test-1/data"}
+		expectedString := "hello world\n"
+		test := testsuites.DynamicallyProvisionedRestartDriverTest{
+			CSIDriver: testDriver,
+			Pod:       pod,
+			PodCheck: &testsuites.PodExecCheck{
+				Cmd:            podCheckCmd,
+				ExpectedString: expectedString,
+			},
+			StorageClassParameters: map[string]string{"protocol": "nfs"},
+			RestartDriverFunc: func() {
+				restartDriver := testCmd{
+					command:  "bash",
+					args:     []string{"test/utils/restart_driver_daemonset.sh"},
+					startLog: "Restart driver node daemonset ...",
+					endLog:   "Restart driver node daemonset done successfully",
+				}
+				execTestCmd([]testCmd{restartDriver})
+			},
+		}
+		test.Run(cs, ns)
+	})
+
+	ginkgo.It("blobfuse volume mount is still valid after driver restart [blob.csi.azure.com]", func() {
+		_, useBlobfuseProxy := os.LookupEnv("ENABLE_BLOBFUSE_PROXY")
+		if !useBlobfuseProxy {
+			ginkgo.Skip("skip this test since blobfuse-proxy is not enabled")
+		}
+
+		pod := testsuites.PodDetails{
+			Cmd: "echo 'hello world' >> /mnt/test-1/data && while true; do sleep 3600; done",
+			Volumes: []testsuites.VolumeDetails{
+				{
+					ClaimSize: "10Gi",
+					VolumeMount: testsuites.VolumeMountDetails{
+						NameGenerate:      "test-volume-",
+						MountPathGenerate: "/mnt/test-",
+					},
+				},
+			},
+		}
+
+		podCheckCmd := []string{"cat", "/mnt/test-1/data"}
+		expectedString := "hello world\n"
+		test := testsuites.DynamicallyProvisionedRestartDriverTest{
+			CSIDriver: testDriver,
+			Pod:       pod,
+			PodCheck: &testsuites.PodExecCheck{
+				Cmd:            podCheckCmd,
+				ExpectedString: expectedString,
+			},
+			StorageClassParameters: map[string]string{"skuName": "Standard_LRS"},
+			RestartDriverFunc: func() {
+				restartDriver := testCmd{
+					command:  "bash",
+					args:     []string{"test/utils/restart_driver_daemonset.sh"},
+					startLog: "Restart driver node daemonset ...",
+					endLog:   "Restart driver node daemonset done successfully",
+				}
+				execTestCmd([]testCmd{restartDriver})
+			},
+		}
+		test.Run(cs, ns)
+	})
 })

test/e2e/testsuites/dynamically_provisioned_restart_driver_tester.go

@@ -0,0 +1,64 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package testsuites
+
+import (
+	"github.com/onsi/ginkgo"
+	v1 "k8s.io/api/core/v1"
+	clientset "k8s.io/client-go/kubernetes"
+
+	"sigs.k8s.io/blob-csi-driver/test/e2e/driver"
+)
+
+// DynamicallyProvisionedRestartDriverTest will test to ensure that restarting driver doesn't affect pod mounting.
+// It will mount a pod, restart the driver daemonset and ensure that the pod still has access to original volume.
+type DynamicallyProvisionedRestartDriverTest struct {
+	CSIDriver              driver.DynamicPVTestDriver
+	Pod                    PodDetails
+	PodCheck               *PodExecCheck
+	StorageClassParameters map[string]string
+	RestartDriverFunc      func()
+}
+
+func (t *DynamicallyProvisionedRestartDriverTest) Run(client clientset.Interface, namespace *v1.Namespace) {
+	tDeployment, cleanup := t.Pod.SetupDeployment(client, namespace, t.CSIDriver, t.StorageClassParameters)
+	// defer must be called here for resources not get removed before using them
+	for i := range cleanup {
+		defer cleanup[i]()
+	}
+
+	ginkgo.By("creating the deployment for the pod")
+	tDeployment.Create()
+
+	ginkgo.By("checking that the pod is running")
+	tDeployment.WaitForPodReady()
+
+	if t.PodCheck != nil {
+		ginkgo.By("checking if pod is able to access volume")
+		tDeployment.PollForStringInPodsExec(t.PodCheck.Cmd, t.PodCheck.ExpectedString)
+	}
+
+	// restart the driver
+	ginkgo.By("restarting the driver daemonset")
+	t.RestartDriverFunc()
+
+	// check if original pod could still access volume
+	if t.PodCheck != nil {
+		ginkgo.By("checking if pod still has access to volume after driver restart")
+		tDeployment.PollForStringInPodsExec(t.PodCheck.Cmd, t.PodCheck.ExpectedString)
+	}
+}

test/e2e/testsuites/testsuites.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"fmt"
 	"math/rand"
+	"strings"
 	"time"
 
 	"sigs.k8s.io/blob-csi-driver/pkg/blob"
@@ -35,6 +36,8 @@ import (
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/util/errors"
+	"k8s.io/apimachinery/pkg/util/wait"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/pkg/kubelet/events"
 	"k8s.io/kubernetes/test/e2e/framework"
@@ -54,8 +57,9 @@ const (
 	// Description that will printed during tests
 	failedConditionDescription = "Error status code"
 
-	poll            = 2 * time.Second
-	pollLongTimeout = 5 * time.Minute
+	poll                 = 2 * time.Second
+	pollLongTimeout      = 5 * time.Minute
+	pollForStringTimeout = 1 * time.Minute
 )
 
 type TestStorageClass struct {
@@ -379,6 +383,10 @@ func (t *TestDeployment) Exec(command []string, expectedString string) {
 	framework.ExpectNoError(err)
 }
 
+func (t *TestDeployment) PollForStringInPodsExec(command []string, expectedString string) {
+	pollForStringInPodsExec(t.namespace.Name, []string{t.podName}, command, expectedString)
+}
+
 func (t *TestDeployment) DeletePodAndWait() {
 	e2elog.Logf("Deleting pod %q in namespace %q", t.podName, t.namespace.Name)
 	err := t.client.CoreV1().Pods(t.namespace.Name).Delete(context.TODO(), t.podName, metav1.DeleteOptions{})
@@ -629,3 +637,33 @@ func waitForPersistentVolumeClaimDeleted(c clientset.Interface, ns string, pvcNa
 	}
 	return fmt.Errorf("PersistentVolumeClaim %s is not removed from the system within %v", pvcName, timeout)
 }
+
+func pollForStringWorker(namespace string, pod string, command []string, expectedString string, ch chan<- error) {
+	args := append([]string{"exec", pod, "--"}, command...)
+	err := wait.PollImmediate(poll, pollForStringTimeout, func() (bool, error) {
+		stdout, err := framework.RunKubectl(namespace, args...)
+		if err != nil {
+			framework.Logf("Error waiting for output %q in pod %q: %v.", expectedString, pod, err)
+			return false, nil
+		}
+		if !strings.Contains(stdout, expectedString) {
+			framework.Logf("The stdout did not contain output %q in pod %q, found: %q.", expectedString, pod, stdout)
+			return false, nil
+		}
+		return true, nil
+	})
+	ch <- err
+}
+
+// Execute the command for all pods in the namespace, looking for expectedString in stdout
+func pollForStringInPodsExec(namespace string, pods []string, command []string, expectedString string) {
+	ch := make(chan error, len(pods))
+	for _, pod := range pods {
+		go pollForStringWorker(namespace, pod, command, expectedString, ch)
+	}
+	errs := make([]error, 0, len(pods))
+	for range pods {
+		errs = append(errs, <-ch)
+	}
+	framework.ExpectNoError(errors.NewAggregate(errs), "Failed to find %q in at least one pod's output.", expectedString)
+}

test/utils/restart_driver_daemonset.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Copyright 2021 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+
+echo "Deleting the Driver node daemonset ..."
+kubectl delete -f ./deploy/csi-blob-node.yaml --ignore-not-found
+
+sleep 15
+
+echo "Installing the Driver node daemonset ..."
+kubectl apply -f ./deploy/csi-blob-node.yaml