Skip to content

Commit 7bc338b

Browse files
andyzhangxandyzhangx
authored
Merge pull request #771 from andyzhangx/restart-driver-test
test: add restart driver e2e test
2 parents e63fd9e + 4bc76f3 commit 7bc338b

9 files changed

+225
-24
lines changed

‎test/e2e/driver/blob_csi_driver.go

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -39,9 +39,9 @@ func InitBlobCSIDriver() PVTestDriver {
3939
}
4040
}
4141

42-
func (d *blobCSIDriver) GetDynamicProvisionStorageClass(parameters map[string]string, mountOptions []string, reclaimPolicy *v1.PersistentVolumeReclaimPolicy, bindingMode *storagev1.VolumeBindingMode, allowedTopologyValues []string, namespace string) *storagev1.StorageClass {
42+
func (d *blobCSIDriver) GetProvisionStorageClass(parameters map[string]string, mountOptions []string, reclaimPolicy *v1.PersistentVolumeReclaimPolicy, bindingMode *storagev1.VolumeBindingMode, allowedTopologyValues []string, namespace string) *storagev1.StorageClass {
4343
provisioner := d.driverName
44-
generateName := fmt.Sprintf("%s-%s-dynamic-sc-", namespace, provisioner)
44+
generateName := fmt.Sprintf("%s-%s-sc-", namespace, provisioner)
4545
return getStorageClass(generateName, provisioner, parameters, mountOptions, reclaimPolicy, bindingMode, nil)
4646
}
4747

@@ -87,9 +87,3 @@ func (d *blobCSIDriver) GetPersistentVolume(volumeID string, fsType string, size
8787
},
8888
}
8989
}
90-
91-
func (d *blobCSIDriver) GetPreProvisionStorageClass(parameters map[string]string, mountOptions []string, reclaimPolicy *v1.PersistentVolumeReclaimPolicy, bindingMode *storagev1.VolumeBindingMode, allowedTopologyValues []string, namespace string) *storagev1.StorageClass {
92-
provisioner := d.driverName
93-
generateName := fmt.Sprintf("%s-%s-pre-provisioned-sc-", namespace, provisioner)
94-
return getStorageClass(generateName, provisioner, parameters, mountOptions, reclaimPolicy, bindingMode, nil)
95-
}

‎test/e2e/driver/driver.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -29,16 +29,16 @@ type PVTestDriver interface {
2929

3030
// DynamicPVTestDriver represents an interface for a CSI driver that supports DynamicPV
3131
type DynamicPVTestDriver interface {
32-
// GetDynamicProvisionStorageClass returns a StorageClass dynamic provision Persistent Volume
33-
GetDynamicProvisionStorageClass(parameters map[string]string, mountOptions []string, reclaimPolicy *v1.PersistentVolumeReclaimPolicy, bindingMode *storagev1.VolumeBindingMode, allowedTopologyValues []string, namespace string) *storagev1.StorageClass
32+
// GetProvisionStorageClass returns a StorageClass dynamic provision Persistent Volume
33+
GetProvisionStorageClass(parameters map[string]string, mountOptions []string, reclaimPolicy *v1.PersistentVolumeReclaimPolicy, bindingMode *storagev1.VolumeBindingMode, allowedTopologyValues []string, namespace string) *storagev1.StorageClass
3434
}
3535

3636
// PreProvisionedVolumeTestDriver represents an interface for a CSI driver that supports pre-provisioned volume
3737
type PreProvisionedVolumeTestDriver interface {
3838
// GetPersistentVolume returns a PersistentVolume with pre-provisioned volumeHandle
3939
GetPersistentVolume(volumeID string, fsType string, size string, reclaimPolicy *v1.PersistentVolumeReclaimPolicy, namespace string, attrib map[string]string, nodeStageSecretRef string) *v1.PersistentVolume
40-
// GetPreProvisionStorageClass returns a StorageClass with existing container
41-
GetPreProvisionStorageClass(parameters map[string]string, mountOptions []string, reclaimPolicy *v1.PersistentVolumeReclaimPolicy, bindingMode *storagev1.VolumeBindingMode, allowedTopologyValues []string, namespace string) *storagev1.StorageClass
40+
// GetProvisionStorageClass returns a StorageClass with existing container
41+
GetProvisionStorageClass(parameters map[string]string, mountOptions []string, reclaimPolicy *v1.PersistentVolumeReclaimPolicy, bindingMode *storagev1.VolumeBindingMode, allowedTopologyValues []string, namespace string) *storagev1.StorageClass
4242
}
4343

4444
func getStorageClass(

‎test/e2e/pre_provisioning_test.go

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ package e2e
1919
import (
2020
"context"
2121
"fmt"
22+
"os"
2223
"time"
2324

2425
"sigs.k8s.io/blob-csi-driver/test/e2e/driver"
@@ -330,6 +331,94 @@ var _ = ginkgo.Describe("[blob-csi-e2e] Pre-Provisioned", func() {
330331
}
331332
test.Run(cs, ns)
332333
})
334+
335+
ginkgo.It("nfs volume mount is still valid after driver restart [blob.csi.azure.com]", func() {
336+
// print driver logs before driver restart
337+
blobLog := testCmd{
338+
command: "bash",
339+
args: []string{"test/utils/blob_log.sh"},
340+
startLog: "===================blob log (before restart)===================",
341+
endLog: "====================================================================",
342+
}
343+
execTestCmd([]testCmd{blobLog})
344+
345+
pod := testsuites.PodDetails{
346+
Cmd: "echo 'hello world' >> /mnt/test-1/data && while true; do sleep 3600; done",
347+
Volumes: []testsuites.VolumeDetails{
348+
{
349+
ClaimSize: "10Gi",
350+
VolumeMount: testsuites.VolumeMountDetails{
351+
NameGenerate: "test-volume-",
352+
MountPathGenerate: "/mnt/test-",
353+
},
354+
},
355+
},
356+
}
357+
358+
podCheckCmd := []string{"cat", "/mnt/test-1/data"}
359+
expectedString := "hello world\n"
360+
test := testsuites.DynamicallyProvisionedRestartDriverTest{
361+
CSIDriver: testDriver,
362+
Pod: pod,
363+
PodCheck: &testsuites.PodExecCheck{
364+
Cmd: podCheckCmd,
365+
ExpectedString: expectedString,
366+
},
367+
StorageClassParameters: map[string]string{"protocol": "nfs"},
368+
RestartDriverFunc: func() {
369+
restartDriver := testCmd{
370+
command: "bash",
371+
args: []string{"test/utils/restart_driver_daemonset.sh"},
372+
startLog: "Restart driver node daemonset ...",
373+
endLog: "Restart driver node daemonset done successfully",
374+
}
375+
execTestCmd([]testCmd{restartDriver})
376+
},
377+
}
378+
test.Run(cs, ns)
379+
})
380+
381+
ginkgo.It("blobfuse volume mount is still valid after driver restart [blob.csi.azure.com]", func() {
382+
_, useBlobfuseProxy := os.LookupEnv("ENABLE_BLOBFUSE_PROXY")
383+
if !useBlobfuseProxy {
384+
ginkgo.Skip("skip this test since blobfuse-proxy is not enabled")
385+
}
386+
387+
pod := testsuites.PodDetails{
388+
Cmd: "echo 'hello world' >> /mnt/test-1/data && while true; do sleep 3600; done",
389+
Volumes: []testsuites.VolumeDetails{
390+
{
391+
ClaimSize: "10Gi",
392+
VolumeMount: testsuites.VolumeMountDetails{
393+
NameGenerate: "test-volume-",
394+
MountPathGenerate: "/mnt/test-",
395+
},
396+
},
397+
},
398+
}
399+
400+
podCheckCmd := []string{"cat", "/mnt/test-1/data"}
401+
expectedString := "hello world\n"
402+
test := testsuites.DynamicallyProvisionedRestartDriverTest{
403+
CSIDriver: testDriver,
404+
Pod: pod,
405+
PodCheck: &testsuites.PodExecCheck{
406+
Cmd: podCheckCmd,
407+
ExpectedString: expectedString,
408+
},
409+
StorageClassParameters: map[string]string{"skuName": "Standard_LRS"},
410+
RestartDriverFunc: func() {
411+
restartDriver := testCmd{
412+
command: "bash",
413+
args: []string{"test/utils/restart_driver_daemonset.sh"},
414+
startLog: "Restart driver node daemonset ...",
415+
endLog: "Restart driver node daemonset done successfully",
416+
}
417+
execTestCmd([]testCmd{restartDriver})
418+
},
419+
}
420+
test.Run(cs, ns)
421+
})
333422
})
334423

335424
func makeCreateVolumeReq(volumeName, secretNamespace string) *csi.CreateVolumeRequest {

‎test/e2e/suite_test.go

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -133,19 +133,13 @@ var _ = ginkgo.AfterSuite(func() {
133133
}
134134
execTestCmd([]testCmd{createExampleDeployment})
135135

136-
blobLog := testCmd{
137-
command: "bash",
138-
args: []string{"test/utils/blob_log.sh"},
139-
startLog: "===================blob log===================",
140-
endLog: "==================================================",
141-
}
142136
e2eTeardown := testCmd{
143137
command: "make",
144138
args: []string{"e2e-teardown"},
145139
startLog: "Uninstalling Azure Blob Storage CSI driver...",
146140
endLog: "Azure Blob Storage CSI driver uninstalled",
147141
}
148-
execTestCmd([]testCmd{blobLog, e2eTeardown})
142+
execTestCmd([]testCmd{e2eTeardown})
149143

150144
// install/uninstall CSI Driver deployment scripts test
151145
installDriver := testCmd{

‎test/e2e/testsuites/dynamically_provisioned_restart_driver_tester.go

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
/*
2+
Copyright 2021 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package testsuites
18+
19+
import (
20+
"github.com/onsi/ginkgo"
21+
v1 "k8s.io/api/core/v1"
22+
clientset "k8s.io/client-go/kubernetes"
23+
24+
"sigs.k8s.io/blob-csi-driver/test/e2e/driver"
25+
)
26+
27+
// DynamicallyProvisionedRestartDriverTest will test to ensure that restarting driver doesn't affect pod mounting.
28+
// It will mount a pod, restart the driver daemonset and ensure that the pod still has access to original volume.
29+
type DynamicallyProvisionedRestartDriverTest struct {
30+
CSIDriver driver.DynamicPVTestDriver
31+
Pod PodDetails
32+
PodCheck *PodExecCheck
33+
StorageClassParameters map[string]string
34+
RestartDriverFunc func()
35+
}
36+
37+
func (t *DynamicallyProvisionedRestartDriverTest) Run(client clientset.Interface, namespace *v1.Namespace) {
38+
tDeployment, cleanup := t.Pod.SetupDeployment(client, namespace, t.CSIDriver, t.StorageClassParameters)
39+
// defer must be called here for resources not get removed before using them
40+
for i := range cleanup {
41+
defer cleanup[i]()
42+
}
43+
44+
ginkgo.By("creating the deployment for the pod")
45+
tDeployment.Create()
46+
47+
ginkgo.By("checking that the pod is running")
48+
tDeployment.WaitForPodReady()
49+
50+
if t.PodCheck != nil {
51+
ginkgo.By("checking if pod is able to access volume")
52+
tDeployment.PollForStringInPodsExec(t.PodCheck.Cmd, t.PodCheck.ExpectedString)
53+
}
54+
55+
// restart the driver
56+
ginkgo.By("restarting the driver daemonset")
57+
t.RestartDriverFunc()
58+
59+
// check if original pod could still access volume
60+
if t.PodCheck != nil {
61+
ginkgo.By("checking if pod still has access to volume after driver restart")
62+
tDeployment.PollForStringInPodsExec(t.PodCheck.Cmd, t.PodCheck.ExpectedString)
63+
}
64+
}

‎test/e2e/testsuites/pre_provisioned_existing_credentials_tester.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ func (t *PreProvisionedExistingCredentialsTest) Run(client clientset.Interface,
5151
}
5252

5353
ginkgo.By("creating the storageclass with existing credentials")
54-
sc := t.CSIDriver.GetPreProvisionStorageClass(parameters, volume.MountOptions, volume.ReclaimPolicy, volume.VolumeBindingMode, volume.AllowedTopologyValues, namespace.Name)
54+
sc := t.CSIDriver.GetProvisionStorageClass(parameters, volume.MountOptions, volume.ReclaimPolicy, volume.VolumeBindingMode, volume.AllowedTopologyValues, namespace.Name)
5555
tsc := NewTestStorageClass(client, namespace, sc)
5656
createdStorageClass := tsc.Create()
5757
defer tsc.Cleanup()

‎test/e2e/testsuites/specs.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -152,7 +152,7 @@ func (pod *PodDetails) SetupDeployment(client clientset.Interface, namespace *v1
152152
cleanupFuncs := make([]func(), 0)
153153
volume := pod.Volumes[0]
154154
ginkgo.By("setting up the StorageClass")
155-
storageClass := csiDriver.GetDynamicProvisionStorageClass(storageClassParameters, volume.MountOptions, volume.ReclaimPolicy, volume.VolumeBindingMode, volume.AllowedTopologyValues, namespace.Name)
155+
storageClass := csiDriver.GetProvisionStorageClass(storageClassParameters, volume.MountOptions, volume.ReclaimPolicy, volume.VolumeBindingMode, volume.AllowedTopologyValues, namespace.Name)
156156
tsc := NewTestStorageClass(client, namespace, storageClass)
157157
createdStorageClass := tsc.Create()
158158
cleanupFuncs = append(cleanupFuncs, tsc.Cleanup)
@@ -172,7 +172,7 @@ func (pod *PodDetails) SetupDeployment(client clientset.Interface, namespace *v1
172172
func (volume *VolumeDetails) SetupDynamicPersistentVolumeClaim(client clientset.Interface, namespace *v1.Namespace, csiDriver driver.DynamicPVTestDriver, storageClassParameters map[string]string) (*TestPersistentVolumeClaim, []func()) {
173173
cleanupFuncs := make([]func(), 0)
174174
ginkgo.By("setting up the StorageClass")
175-
storageClass := csiDriver.GetDynamicProvisionStorageClass(storageClassParameters, volume.MountOptions, volume.ReclaimPolicy, volume.VolumeBindingMode, volume.AllowedTopologyValues, namespace.Name)
175+
storageClass := csiDriver.GetProvisionStorageClass(storageClassParameters, volume.MountOptions, volume.ReclaimPolicy, volume.VolumeBindingMode, volume.AllowedTopologyValues, namespace.Name)
176176
tsc := NewTestStorageClass(client, namespace, storageClass)
177177
createdStorageClass := tsc.Create()
178178
cleanupFuncs = append(cleanupFuncs, tsc.Cleanup)

‎test/e2e/testsuites/testsuites.go

Lines changed: 40 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"context"
2121
"fmt"
2222
"math/rand"
23+
"strings"
2324
"time"
2425

2526
"sigs.k8s.io/blob-csi-driver/pkg/blob"
@@ -35,6 +36,8 @@ import (
3536
"k8s.io/apimachinery/pkg/api/resource"
3637
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3738
"k8s.io/apimachinery/pkg/fields"
39+
"k8s.io/apimachinery/pkg/util/errors"
40+
"k8s.io/apimachinery/pkg/util/wait"
3841
clientset "k8s.io/client-go/kubernetes"
3942
"k8s.io/kubernetes/pkg/kubelet/events"
4043
"k8s.io/kubernetes/test/e2e/framework"
@@ -54,8 +57,9 @@ const (
5457
// Description that will printed during tests
5558
failedConditionDescription = "Error status code"
5659

57-
poll = 2 * time.Second
58-
pollLongTimeout = 5 * time.Minute
60+
poll = 2 * time.Second
61+
pollLongTimeout = 5 * time.Minute
62+
pollForStringTimeout = 1 * time.Minute
5963
)
6064

6165
type TestStorageClass struct {
@@ -379,6 +383,10 @@ func (t *TestDeployment) Exec(command []string, expectedString string) {
379383
framework.ExpectNoError(err)
380384
}
381385

386+
func (t *TestDeployment) PollForStringInPodsExec(command []string, expectedString string) {
387+
pollForStringInPodsExec(t.namespace.Name, []string{t.podName}, command, expectedString)
388+
}
389+
382390
func (t *TestDeployment) DeletePodAndWait() {
383391
e2elog.Logf("Deleting pod %q in namespace %q", t.podName, t.namespace.Name)
384392
err := t.client.CoreV1().Pods(t.namespace.Name).Delete(context.TODO(), t.podName, metav1.DeleteOptions{})
@@ -629,3 +637,33 @@ func waitForPersistentVolumeClaimDeleted(c clientset.Interface, ns string, pvcNa
629637
}
630638
return fmt.Errorf("PersistentVolumeClaim %s is not removed from the system within %v", pvcName, timeout)
631639
}
640+
641+
func pollForStringWorker(namespace string, pod string, command []string, expectedString string, ch chan<- error) {
642+
args := append([]string{"exec", pod, "--"}, command...)
643+
err := wait.PollImmediate(poll, pollForStringTimeout, func() (bool, error) {
644+
stdout, err := framework.RunKubectl(namespace, args...)
645+
if err != nil {
646+
framework.Logf("Error waiting for output %q in pod %q: %v.", expectedString, pod, err)
647+
return false, nil
648+
}
649+
if !strings.Contains(stdout, expectedString) {
650+
framework.Logf("The stdout did not contain output %q in pod %q, found: %q.", expectedString, pod, stdout)
651+
return false, nil
652+
}
653+
return true, nil
654+
})
655+
ch <- err
656+
}
657+
658+
// Execute the command for all pods in the namespace, looking for expectedString in stdout
659+
func pollForStringInPodsExec(namespace string, pods []string, command []string, expectedString string) {
660+
ch := make(chan error, len(pods))
661+
for _, pod := range pods {
662+
go pollForStringWorker(namespace, pod, command, expectedString, ch)
663+
}
664+
errs := make([]error, 0, len(pods))
665+
for range pods {
666+
errs = append(errs, <-ch)
667+
}
668+
framework.ExpectNoError(errors.NewAggregate(errs), "Failed to find %q in at least one pod's output.", expectedString)
669+
}

‎test/utils/restart_driver_daemonset.sh

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#!/bin/bash
2+
3+
# Copyright 2021 The Kubernetes Authors.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
set -euo pipefail
18+
19+
echo "restart driver node daemonset ..."
20+
kubectl rollout restart ds csi-blob-node -n kube-system
21+
22+
sleep 10

0 commit comments

Comments
 (0)