Merge pull request kubernetes#74693 from mkimuram/issue/74545

Add e2e tests for multiAttach

Author: k8s-ci-robot

test/e2e/framework/pv_util.go

@@ -93,6 +93,14 @@ type PersistentVolumeClaimConfig struct {
 	VolumeMode       *v1.PersistentVolumeMode
 }
 
+// NodeSelection specifies where to run a pod, using a combination of fixed node name,
+// node selector and/or affinity.
+type NodeSelection struct {
+	Name     string
+	Selector map[string]string
+	Affinity *v1.Affinity
+}
+
 // Clean up a pv and pvc in a single pv/pvc test case.
 // Note: delete errors are appended to []error so that we can attempt to delete both the pvc and pv.
 func PVPVCCleanup(c clientset.Interface, ns string, pv *v1.PersistentVolume, pvc *v1.PersistentVolumeClaim) []error {
@@ -873,14 +881,16 @@ func CreateNginxPod(client clientset.Interface, namespace string, nodeSelector m
 
 // create security pod with given claims
 func CreateSecPod(client clientset.Interface, namespace string, pvclaims []*v1.PersistentVolumeClaim, isPrivileged bool, command string, hostIPC bool, hostPID bool, seLinuxLabel *v1.SELinuxOptions, fsGroup *int64, timeout time.Duration) (*v1.Pod, error) {
-	return CreateSecPodWithNodeName(client, namespace, pvclaims, isPrivileged, command, hostIPC, hostPID, seLinuxLabel, fsGroup, "", timeout)
+	return CreateSecPodWithNodeSelection(client, namespace, pvclaims, isPrivileged, command, hostIPC, hostPID, seLinuxLabel, fsGroup, NodeSelection{}, timeout)
 }
 
 // create security pod with given claims
-func CreateSecPodWithNodeName(client clientset.Interface, namespace string, pvclaims []*v1.PersistentVolumeClaim, isPrivileged bool, command string, hostIPC bool, hostPID bool, seLinuxLabel *v1.SELinuxOptions, fsGroup *int64, nodeName string, timeout time.Duration) (*v1.Pod, error) {
+func CreateSecPodWithNodeSelection(client clientset.Interface, namespace string, pvclaims []*v1.PersistentVolumeClaim, isPrivileged bool, command string, hostIPC bool, hostPID bool, seLinuxLabel *v1.SELinuxOptions, fsGroup *int64, node NodeSelection, timeout time.Duration) (*v1.Pod, error) {
 	pod := MakeSecPod(namespace, pvclaims, isPrivileged, command, hostIPC, hostPID, seLinuxLabel, fsGroup)
-	// Setting nodeName
-	pod.Spec.NodeName = nodeName
+	// Setting node
+	pod.Spec.NodeName = node.Name
+	pod.Spec.NodeSelector = node.Selector
+	pod.Spec.Affinity = node.Affinity
 
 	pod, err := client.CoreV1().Pods(namespace).Create(pod)
 	if err != nil {
@@ -900,6 +910,44 @@ func CreateSecPodWithNodeName(client clientset.Interface, namespace string, pvcl
 	return pod, nil
 }
 
+// SetNodeAffinityRequirement sets affinity with specified operator to nodeName to nodeSelection
+func SetNodeAffinityRequirement(nodeSelection *NodeSelection, operator v1.NodeSelectorOperator, nodeName string) {
+	// Add node-anti-affinity.
+	if nodeSelection.Affinity == nil {
+		nodeSelection.Affinity = &v1.Affinity{}
+	}
+	if nodeSelection.Affinity.NodeAffinity == nil {
+		nodeSelection.Affinity.NodeAffinity = &v1.NodeAffinity{}
+	}
+	if nodeSelection.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil {
+		nodeSelection.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution = &v1.NodeSelector{}
+	}
+	nodeSelection.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = append(nodeSelection.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms,
+		v1.NodeSelectorTerm{
+			// https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity warns
+			// that "the value of kubernetes.io/hostname may be the same as the Node name in some environments and a different value in other environments".
+			// So this might be cleaner:
+			// MatchFields: []v1.NodeSelectorRequirement{
+			// 	{Key: "name", Operator: v1.NodeSelectorOpNotIn, Values: []string{nodeName}},
+			// },
+			// However, "name", "Name", "ObjectMeta.Name" all got rejected with "not a valid field selector key".
+
+			MatchExpressions: []v1.NodeSelectorRequirement{
+				{Key: "kubernetes.io/hostname", Operator: operator, Values: []string{nodeName}},
+			},
+		})
+}
+
+// SetAffinity sets affinity to nodeName to nodeSelection
+func SetAffinity(nodeSelection *NodeSelection, nodeName string) {
+	SetNodeAffinityRequirement(nodeSelection, v1.NodeSelectorOpIn, nodeName)
+}
+
+// SetAntiAffinity sets anti-affinity to nodeName to nodeSelection
+func SetAntiAffinity(nodeSelection *NodeSelection, nodeName string) {
+	SetNodeAffinityRequirement(nodeSelection, v1.NodeSelectorOpNotIn, nodeName)
+}
+
 // Define and create a pod with a mounted PV.  Pod runs infinite loop until killed.
 func CreateClientPod(c clientset.Interface, ns string, pvc *v1.PersistentVolumeClaim) (*v1.Pod, error) {
 	return CreatePod(c, ns, nil, []*v1.PersistentVolumeClaim{pvc}, true, "")

test/e2e/storage/csi_mock_volume.go

@@ -153,12 +153,12 @@ var _ = utils.SIGDescribe("CSI mock volume", func() {
 			scTest.AllowVolumeExpansion = true
 		}
 
-		nodeSelection := testsuites.NodeSelection{
+		nodeSelection := framework.NodeSelection{
 			// The mock driver only works when everything runs on a single node.
 			Name: nodeName,
 		}
 		if len(m.nodeLabel) > 0 {
-			nodeSelection = testsuites.NodeSelection{
+			nodeSelection = framework.NodeSelection{
 				Selector: m.nodeLabel,
 			}
 		}
@@ -177,11 +177,11 @@ var _ = utils.SIGDescribe("CSI mock volume", func() {
 
 	createPodWithPVC := func(pvc *v1.PersistentVolumeClaim) (*v1.Pod, error) {
 		nodeName := m.config.ClientNodeName
-		nodeSelection := testsuites.NodeSelection{
+		nodeSelection := framework.NodeSelection{
 			Name: nodeName,
 		}
 		if len(m.nodeLabel) > 0 {
-			nodeSelection = testsuites.NodeSelection{
+			nodeSelection = framework.NodeSelection{
 				Selector: m.nodeLabel,
 			}
 		}
@@ -597,7 +597,7 @@ func checkNodeForLimits(nodeName string, attachKey v1.ResourceName, cs clientset
 	return int(attachLimit), waitErr
 }
 
-func startPausePod(cs clientset.Interface, t testsuites.StorageClassTest, node testsuites.NodeSelection, ns string) (*storagev1.StorageClass, *v1.PersistentVolumeClaim, *v1.Pod) {
+func startPausePod(cs clientset.Interface, t testsuites.StorageClassTest, node framework.NodeSelection, ns string) (*storagev1.StorageClass, *v1.PersistentVolumeClaim, *v1.Pod) {
 	class := newStorageClass(t, ns, "")
 	var err error
 	_, err = cs.StorageV1().StorageClasses().Get(class.Name, metav1.GetOptions{})
@@ -659,7 +659,7 @@ func startPausePod(cs clientset.Interface, t testsuites.StorageClassTest, node t
 	return class, claim, pod
 }
 
-func startPausePodWithClaim(cs clientset.Interface, pvc *v1.PersistentVolumeClaim, node testsuites.NodeSelection, ns string) (*v1.Pod, error) {
+func startPausePodWithClaim(cs clientset.Interface, pvc *v1.PersistentVolumeClaim, node framework.NodeSelection, ns string) (*v1.Pod, error) {
 	pod := &v1.Pod{
 		ObjectMeta: metav1.ObjectMeta{
 			GenerateName: "pvc-volume-tester-",

test/e2e/storage/csi_volumes.go

@@ -47,6 +47,7 @@ var csiTestSuites = []func() testsuites.TestSuite{
 	testsuites.InitSubPathTestSuite,
 	testsuites.InitProvisioningTestSuite,
 	testsuites.InitSnapshottableTestSuite,
+	testsuites.InitMultiVolumeTestSuite,
 }
 
 // This executes testSuites for csi volumes.
@@ -155,7 +156,7 @@ func testTopologyNegative(cs clientset.Interface, suffix, namespace string, dela
 		test.PvCheck = func(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) {
 			// Ensure that a pod cannot be scheduled in an unsuitable zone.
 			pod := testsuites.StartInPodWithVolume(cs, namespace, claim.Name, "pvc-tester-unschedulable", "sleep 100000",
-				testsuites.NodeSelection{Selector: nodeSelector})
+				framework.NodeSelection{Selector: nodeSelector})
 			defer testsuites.StopPod(cs, pod)
 			framework.ExpectNoError(framework.WaitForPodNameUnschedulableInNamespace(cs, pod.Name, pod.Namespace), "pod should be unschedulable")
 		}

test/e2e/storage/drivers/in_tree.go

@@ -95,6 +95,7 @@ func InitNFSDriver() testsuites.TestDriver {
 			Capabilities: map[testsuites.Capability]bool{
 				testsuites.CapPersistence: true,
 				testsuites.CapExec:        true,
+				testsuites.CapRWX:         true,
 			},
 		},
 	}
@@ -232,6 +233,7 @@ func InitGlusterFSDriver() testsuites.TestDriver {
 			Capabilities: map[testsuites.Capability]bool{
 				testsuites.CapPersistence: true,
 				testsuites.CapExec:        true,
+				testsuites.CapRWX:         true,
 			},
 		},
 	}
@@ -586,6 +588,7 @@ func InitCephFSDriver() testsuites.TestDriver {
 			Capabilities: map[testsuites.Capability]bool{
 				testsuites.CapPersistence: true,
 				testsuites.CapExec:        true,
+				testsuites.CapRWX:         true,
 			},
 		},
 	}

test/e2e/storage/in_tree_volumes.go

@@ -55,6 +55,7 @@ var testSuites = []func() testsuites.TestSuite{
 	testsuites.InitVolumeModeTestSuite,
 	testsuites.InitSubPathTestSuite,
 	testsuites.InitProvisioningTestSuite,
+	testsuites.InitMultiVolumeTestSuite,
 }
 
 // This executes testSuites for in-tree volumes.

test/e2e/storage/persistent_volumes.go

@@ -30,6 +30,7 @@ import (
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/storage/testsuites"
 	"k8s.io/kubernetes/test/e2e/storage/utils"
 	imageutils "k8s.io/kubernetes/test/utils/image"
 )
@@ -308,68 +309,22 @@ var _ = utils.SIGDescribe("PersistentVolumes", func() {
 	Describe("Default StorageClass", func() {
 		Context("pods that use multiple volumes", func() {
 
-			AfterEach(func() {
-				framework.DeleteAllStatefulSets(c, ns)
-			})
-
 			It("should be reschedulable [Slow]", func() {
 				// Only run on providers with default storageclass
 				framework.SkipUnlessProviderIs("openstack", "gce", "gke", "vsphere", "azure")
 
 				numVols := 4
-				ssTester := framework.NewStatefulSetTester(c)
-
-				By("Creating a StatefulSet pod to initialize data")
-				writeCmd := "true"
-				for i := 0; i < numVols; i++ {
-					writeCmd += fmt.Sprintf("&& touch %v", getVolumeFile(i))
-				}
-				writeCmd += "&& sleep 10000"
-
-				probe := &v1.Probe{
-					Handler: v1.Handler{
-						Exec: &v1.ExecAction{
-							// Check that the last file got created
-							Command: []string{"test", "-f", getVolumeFile(numVols - 1)},
-						},
-					},
-					InitialDelaySeconds: 1,
-					PeriodSeconds:       1,
-				}
 
-				mounts := []v1.VolumeMount{}
-				claims := []v1.PersistentVolumeClaim{}
+				By("Creating pvcs")
+				claims := []*v1.PersistentVolumeClaim{}
 				for i := 0; i < numVols; i++ {
 					pvc := framework.MakePersistentVolumeClaim(framework.PersistentVolumeClaimConfig{}, ns)
-					pvc.Name = getVolName(i)
-					mounts = append(mounts, v1.VolumeMount{Name: pvc.Name, MountPath: getMountPath(i)})
-					claims = append(claims, *pvc)
+					claims = append(claims, pvc)
 				}
 
-				spec := makeStatefulSetWithPVCs(ns, writeCmd, mounts, claims, probe)
-				ss, err := c.AppsV1().StatefulSets(ns).Create(spec)
-				Expect(err).NotTo(HaveOccurred())
-				ssTester.WaitForRunningAndReady(1, ss)
-
-				By("Deleting the StatefulSet but not the volumes")
-				// Scale down to 0 first so that the Delete is quick
-				ss, err = ssTester.Scale(ss, 0)
-				Expect(err).NotTo(HaveOccurred())
-				ssTester.WaitForStatusReplicas(ss, 0)
-				err = c.AppsV1().StatefulSets(ns).Delete(ss.Name, &metav1.DeleteOptions{})
-				Expect(err).NotTo(HaveOccurred())
-
-				By("Creating a new Statefulset and validating the data")
-				validateCmd := "true"
-				for i := 0; i < numVols; i++ {
-					validateCmd += fmt.Sprintf("&& test -f %v", getVolumeFile(i))
-				}
-				validateCmd += "&& sleep 10000"
-
-				spec = makeStatefulSetWithPVCs(ns, validateCmd, mounts, claims, probe)
-				ss, err = c.AppsV1().StatefulSets(ns).Create(spec)
-				Expect(err).NotTo(HaveOccurred())
-				ssTester.WaitForRunningAndReady(1, ss)
+				By("Testing access to pvcs before and after pod recreation on differetn node")
+				testsuites.TestAccessMultipleVolumesAcrossPodRecreation(f, c, ns,
+					framework.NodeSelection{}, claims, false /* sameNode */)
 			})
 		})
 	})

test/e2e/storage/regional_pd.go

@@ -115,7 +115,7 @@ func testVolumeProvisioning(c clientset.Interface, ns string) {
 				err = verifyZonesInPV(volume, sets.NewString(cloudZones...), true /* match */)
 				Expect(err).NotTo(HaveOccurred(), "verifyZonesInPV")
 
-				testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, testsuites.NodeSelection{})
+				testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, framework.NodeSelection{})
 			},
 		},
 		{
@@ -137,7 +137,7 @@ func testVolumeProvisioning(c clientset.Interface, ns string) {
 				err = verifyZonesInPV(volume, zones, false /* match */)
 				Expect(err).NotTo(HaveOccurred(), "verifyZonesInPV")
 
-				testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, testsuites.NodeSelection{})
+				testsuites.PVWriteReadSingleNodeCheck(c, claim, volume, framework.NodeSelection{})
 			},
 		},
 	}

test/e2e/storage/testsuites/BUILD

@@ -5,6 +5,7 @@ go_library(
     srcs = [
         "base.go",
         "driveroperations.go",
+        "multivolume.go",
         "provisioning.go",
         "snapshottable.go",
         "subpath.go",
@@ -28,6 +29,7 @@ go_library(
         "//staging/src/k8s.io/apimachinery/pkg/util/rand:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//staging/src/k8s.io/apiserver/pkg/storage/names:go_default_library",
         "//staging/src/k8s.io/client-go/dynamic:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
         "//test/e2e/framework:go_default_library",

test/e2e/storage/testsuites/base.go

@@ -188,7 +188,7 @@ func createGenericVolumeTestResource(driver TestDriver, config *PerTestConfig, p
 		if pDriver, ok := driver.(PreprovisionedPVTestDriver); ok {
 			pvSource, volumeNodeAffinity := pDriver.GetPersistentVolumeSource(false, fsType, r.volume)
 			if pvSource != nil {
-				r.volSource, r.pv, r.pvc = createVolumeSourceWithPVCPV(f, dInfo.Name, pvSource, volumeNodeAffinity, false)
+				r.volSource, r.pv, r.pvc = createVolumeSourceWithPVCPV(f, dInfo.Name, pvSource, volumeNodeAffinity, false, pattern.VolMode)
 			}
 			r.volType = fmt.Sprintf("%s-preprovisionedPV", dInfo.Name)
 		}
@@ -205,7 +205,7 @@ func createGenericVolumeTestResource(driver TestDriver, config *PerTestConfig, p
 
 			if r.sc != nil {
 				r.volSource, r.pv, r.pvc = createVolumeSourceWithPVCPVFromDynamicProvisionSC(
-					f, dInfo.Name, claimSize, r.sc, false, nil)
+					f, dInfo.Name, claimSize, r.sc, false, pattern.VolMode)
 			}
 			r.volType = fmt.Sprintf("%s-dynamicPV", dInfo.Name)
 		}
@@ -269,17 +269,24 @@ func createVolumeSourceWithPVCPV(
 	pvSource *v1.PersistentVolumeSource,
 	volumeNodeAffinity *v1.VolumeNodeAffinity,
 	readOnly bool,
+	volMode v1.PersistentVolumeMode,
 ) (*v1.VolumeSource, *v1.PersistentVolume, *v1.PersistentVolumeClaim) {
 	pvConfig := framework.PersistentVolumeConfig{
 		NamePrefix:       fmt.Sprintf("%s-", name),
 		StorageClassName: f.Namespace.Name,
 		PVSource:         *pvSource,
 		NodeAffinity:     volumeNodeAffinity,
 	}
+
 	pvcConfig := framework.PersistentVolumeClaimConfig{
 		StorageClassName: &f.Namespace.Name,
 	}
 
+	if volMode != "" {
+		pvConfig.VolumeMode = &volMode
+		pvcConfig.VolumeMode = &volMode
+	}
+
 	framework.Logf("Creating PVC and PV")
 	pv, pvc, err := framework.CreatePVCPV(f.ClientSet, pvConfig, pvcConfig, f.Namespace.Name, false)
 	Expect(err).NotTo(HaveOccurred(), "PVC, PV creation failed")
@@ -302,16 +309,16 @@ func createVolumeSourceWithPVCPVFromDynamicProvisionSC(
 	claimSize string,
 	sc *storagev1.StorageClass,
 	readOnly bool,
-	volMode *v1.PersistentVolumeMode,
+	volMode v1.PersistentVolumeMode,
 ) (*v1.VolumeSource, *v1.PersistentVolume, *v1.PersistentVolumeClaim) {
 	cs := f.ClientSet
 	ns := f.Namespace.Name
 
 	By("creating a claim")
 	pvc := getClaim(claimSize, ns)
 	pvc.Spec.StorageClassName = &sc.Name
-	if volMode != nil {
-		pvc.Spec.VolumeMode = volMode
+	if volMode != "" {
+		pvc.Spec.VolumeMode = &volMode
 	}
 
 	var err error

test/e2e/storage/testsuites/driveroperations.go

@@ -22,6 +22,7 @@ import (
 	storagev1 "k8s.io/api/storage/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apiserver/pkg/storage/names"
 	"k8s.io/kubernetes/test/e2e/framework"
 	"k8s.io/kubernetes/test/e2e/storage/testpatterns"
 )
@@ -71,8 +72,8 @@ func GetStorageClass(
 			Kind: "StorageClass",
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			// Name must be unique, so let's base it on namespace name
-			Name: ns + "-" + suffix,
+			// Name must be unique, so let's base it on namespace name and use GenerateName
+			Name: names.SimpleNameGenerator.GenerateName(ns + "-" + suffix),
 		},
 		Provisioner:       provisioner,
 		Parameters:        parameters,