Merge pull request kubernetes#81960 from pohly/ephemeral-tests

ephemeral volume tests

Author: k8s-ci-robot

test/e2e/storage/csi_mock_volume.go

@@ -370,10 +370,14 @@ var _ = utils.SIGDescribe("CSI mock volume", func() {
 					framework.ExpectNoError(err, "failed to test for CSIInlineVolumes")
 				}
 
+				ginkgo.By("Deleting the previously created pod")
+				err = e2epod.DeletePodWithWait(m.cs, pod)
+				framework.ExpectNoError(err, "while deleting")
+
 				ginkgo.By("Checking CSI driver logs")
 				// The driver is deployed as a statefulset with stable pod names
 				driverPodName := "csi-mockplugin-0"
-				err = checkPodInfo(m.cs, f.Namespace.Name, driverPodName, "mock", pod, test.expectPodInfo, test.expectEphemeral, csiInlineVolumesEnabled)
+				err = checkPodLogs(m.cs, f.Namespace.Name, driverPodName, "mock", pod, test.expectPodInfo, test.expectEphemeral, csiInlineVolumesEnabled)
 				framework.ExpectNoError(err)
 			})
 		}
@@ -719,8 +723,9 @@ func startPausePodWithVolumeSource(cs clientset.Interface, volumeSource v1.Volum
 	return cs.CoreV1().Pods(ns).Create(pod)
 }
 
-// checkPodInfo tests that NodePublish was called with expected volume_context
-func checkPodInfo(cs clientset.Interface, namespace, driverPodName, driverContainerName string, pod *v1.Pod, expectPodInfo, ephemeralVolume, csiInlineVolumesEnabled bool) error {
+// checkPodLogs tests that NodePublish was called with expected volume_context and (for ephemeral inline volumes)
+// has the matching NodeUnpublish
+func checkPodLogs(cs clientset.Interface, namespace, driverPodName, driverContainerName string, pod *v1.Pod, expectPodInfo, ephemeralVolume, csiInlineVolumesEnabled bool) error {
 	expectedAttributes := map[string]string{
 		"csi.storage.k8s.io/pod.name":            pod.Name,
 		"csi.storage.k8s.io/pod.namespace":       namespace,
@@ -741,6 +746,8 @@ func checkPodInfo(cs clientset.Interface, namespace, driverPodName, driverContai
 	// Find NodePublish in the logs
 	foundAttributes := sets.NewString()
 	logLines := strings.Split(log, "\n")
+	numNodePublishVolume := 0
+	numNodeUnpublishVolume := 0
 	for _, line := range logLines {
 		if !strings.HasPrefix(line, "gRPCCall:") {
 			continue
@@ -759,19 +766,23 @@ func checkPodInfo(cs clientset.Interface, namespace, driverPodName, driverContai
 			e2elog.Logf("Could not parse CSI driver log line %q: %s", line, err)
 			continue
 		}
-		if call.Method != "/csi.v1.Node/NodePublishVolume" {
-			continue
-		}
-		// Check that NodePublish had expected attributes
-		for k, v := range expectedAttributes {
-			vv, found := call.Request.VolumeContext[k]
-			if found && v == vv {
-				foundAttributes.Insert(k)
-				e2elog.Logf("Found volume attribute %s: %s", k, v)
+		switch call.Method {
+		case "/csi.v1.Node/NodePublishVolume":
+			numNodePublishVolume++
+			if numNodePublishVolume == 1 {
+				// Check that NodePublish had expected attributes for first volume
+				for k, v := range expectedAttributes {
+					vv, found := call.Request.VolumeContext[k]
+					if found && v == vv {
+						foundAttributes.Insert(k)
+						e2elog.Logf("Found volume attribute %s: %s", k, v)
+					}
+				}
 			}
+		case "/csi.v1.Node/NodeUnpublishVolume":
+			e2elog.Logf("Found NodeUnpublishVolume: %+v", call)
+			numNodeUnpublishVolume++
 		}
-		// Process just the first NodePublish, the rest of the log is useless.
-		break
 	}
 	if expectPodInfo {
 		if foundAttributes.Len() != len(expectedAttributes) {
@@ -782,6 +793,9 @@ func checkPodInfo(cs clientset.Interface, namespace, driverPodName, driverContai
 	if foundAttributes.Len() != 0 {
 		return fmt.Errorf("some unexpected volume attributes were found: %+v", foundAttributes.List())
 	}
+	if numNodePublishVolume != numNodeUnpublishVolume {
+		return fmt.Errorf("number of NodePublishVolume %d != number of NodeUnpublishVolume %d", numNodePublishVolume, numNodeUnpublishVolume)
+	}
 	return nil
 }
 

test/e2e/storage/csi_volumes.go

@@ -42,6 +42,7 @@ var csiTestDrivers = []func() testsuites.TestDriver{
 
 // List of testSuites to be executed in below loop
 var csiTestSuites = []func() testsuites.TestSuite{
+	testsuites.InitEphemeralTestSuite,
 	testsuites.InitVolumesTestSuite,
 	testsuites.InitVolumeIOTestSuite,
 	testsuites.InitVolumeModeTestSuite,

test/e2e/storage/drivers/csi.go

@@ -62,11 +62,12 @@ const (
 
 // hostpathCSI
 type hostpathCSIDriver struct {
-	driverInfo testsuites.DriverInfo
-	manifests  []string
+	driverInfo       testsuites.DriverInfo
+	manifests        []string
+	volumeAttributes []map[string]string
 }
 
-func initHostPathCSIDriver(name string, capabilities map[testsuites.Capability]bool, manifests ...string) testsuites.TestDriver {
+func initHostPathCSIDriver(name string, capabilities map[testsuites.Capability]bool, volumeAttributes []map[string]string, manifests ...string) testsuites.TestDriver {
 	return &hostpathCSIDriver{
 		driverInfo: testsuites.DriverInfo{
 			Name:        name,
@@ -77,13 +78,15 @@ func initHostPathCSIDriver(name string, capabilities map[testsuites.Capability]b
 			),
 			Capabilities: capabilities,
 		},
-		manifests: manifests,
+		manifests:        manifests,
+		volumeAttributes: volumeAttributes,
 	}
 }
 
 var _ testsuites.TestDriver = &hostpathCSIDriver{}
 var _ testsuites.DynamicPVTestDriver = &hostpathCSIDriver{}
 var _ testsuites.SnapshottableTestDriver = &hostpathCSIDriver{}
+var _ testsuites.EphemeralTestDriver = &hostpathCSIDriver{}
 
 // InitHostPathCSIDriver returns hostpathCSIDriver that implements TestDriver interface
 func InitHostPathCSIDriver() testsuites.TestDriver {
@@ -97,16 +100,20 @@ func InitHostPathCSIDriver() testsuites.TestDriver {
 	}
 	return initHostPathCSIDriver("csi-hostpath",
 		capabilities,
-		"test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml",
+		// Volume attributes don't matter, but we have to provide at least one map.
+		[]map[string]string{
+			{"foo": "bar"},
+		},
 		"test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml",
 		"test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml",
 		"test/e2e/testing-manifests/storage-csi/external-snapshotter/rbac.yaml",
 		"test/e2e/testing-manifests/storage-csi/external-resizer/rbac.yaml",
 		"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-attacher.yaml",
+		"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-driverinfo.yaml",
+		"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-plugin.yaml",
 		"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-provisioner.yaml",
-		"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-snapshotter.yaml",
 		"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-resizer.yaml",
-		"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpathplugin.yaml",
+		"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/csi-hostpath-snapshotter.yaml",
 		"test/e2e/testing-manifests/storage-csi/hostpath/hostpath/e2e-test-rbac.yaml",
 	)
 }
@@ -116,6 +123,9 @@ func (h *hostpathCSIDriver) GetDriverInfo() *testsuites.DriverInfo {
 }
 
 func (h *hostpathCSIDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) {
+	if pattern.VolType == testpatterns.CSIInlineVolume && len(h.volumeAttributes) == 0 {
+		framework.Skipf("%s has no volume attributes defined, doesn't support ephemeral inline volumes", h.driverInfo.Name)
+	}
 }
 
 func (h *hostpathCSIDriver) GetDynamicProvisionStorageClass(config *testsuites.PerTestConfig, fsType string) *storagev1.StorageClass {
@@ -127,6 +137,14 @@ func (h *hostpathCSIDriver) GetDynamicProvisionStorageClass(config *testsuites.P
 	return testsuites.GetStorageClass(provisioner, parameters, nil, ns, suffix)
 }
 
+func (h *hostpathCSIDriver) GetVolume(config *testsuites.PerTestConfig, volumeNumber int) (map[string]string, bool, bool) {
+	return h.volumeAttributes[volumeNumber%len(h.volumeAttributes)], false /* not shared */, false /* read-write */
+}
+
+func (h *hostpathCSIDriver) GetCSIDriverName(config *testsuites.PerTestConfig) string {
+	return config.GetUniqueDriverName()
+}
+
 func (h *hostpathCSIDriver) GetSnapshotClass(config *testsuites.PerTestConfig) *unstructured.Unstructured {
 	snapshotter := config.GetUniqueDriverName()
 	parameters := map[string]string{}
@@ -205,7 +223,6 @@ var _ testsuites.DynamicPVTestDriver = &mockCSIDriver{}
 // InitMockCSIDriver returns a mockCSIDriver that implements TestDriver interface
 func InitMockCSIDriver(driverOpts CSIMockDriverOpts) testsuites.TestDriver {
 	driverManifests := []string{
-		"test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml",
 		"test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml",
 		"test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml",
 		"test/e2e/testing-manifests/storage-csi/external-resizer/rbac.yaml",
@@ -305,7 +322,7 @@ func (m *mockCSIDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTest
 		NodeName:                 config.ClientNodeName,
 		PodInfo:                  m.podInfo,
 		CanAttach:                &m.attachable,
-		VolumeLifecycleModes: []storagev1beta1.VolumeLifecycleMode{
+		VolumeLifecycleModes: &[]storagev1beta1.VolumeLifecycleMode{
 			storagev1beta1.VolumeLifecyclePersistent,
 			storagev1beta1.VolumeLifecycleEphemeral,
 		},
@@ -329,7 +346,10 @@ func (m *mockCSIDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTest
 func InitHostPathV0CSIDriver() testsuites.TestDriver {
 	return initHostPathCSIDriver("csi-hostpath-v0",
 		map[testsuites.Capability]bool{testsuites.CapPersistence: true, testsuites.CapMultiPODs: true},
-		"test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml",
+		nil, /* no volume attributes -> no ephemeral volume testing */
+		// Using the current set of rbac.yaml files is problematic here because they don't
+		// match the version of the rules that were written for the releases of external-attacher
+		// and external-provisioner that we are using here. It happens to work in practice...
 		"test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml",
 		"test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml",
 		"test/e2e/testing-manifests/storage-csi/hostpath/hostpath-v0/csi-hostpath-attacher.yaml",
@@ -423,7 +443,6 @@ func (g *gcePDCSIDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTes
 	createGCESecrets(f.ClientSet, f.Namespace.Name)
 
 	manifests := []string{
-		"test/e2e/testing-manifests/storage-csi/driver-registrar/rbac.yaml",
 		"test/e2e/testing-manifests/storage-csi/external-attacher/rbac.yaml",
 		"test/e2e/testing-manifests/storage-csi/external-provisioner/rbac.yaml",
 		"test/e2e/testing-manifests/storage-csi/gce-pd/csi-controller-rbac.yaml",

test/e2e/storage/external/external.go

@@ -179,14 +179,25 @@ type driverDefinition struct {
 		// TODO (?): load from file
 	}
 
-	// InlineVolumeAttributes defines one or more set of attributes for
-	// use as inline ephemeral volumes. At least one set of attributes
-	// has to be defined to enable testing of inline ephemeral volumes.
-	// If a test needs more volumes than defined, some of the defined
+	// InlineVolumes defines one or more volumes for use as inline
+	// ephemeral volumes. At least one such volume has to be
+	// defined to enable testing of inline ephemeral volumes.  If
+	// a test needs more volumes than defined, some of the defined
 	// volumes will be used multiple times.
 	//
 	// DriverInfo.Name is used as name of the driver in the inline volume.
-	InlineVolumeAttributes []map[string]string
+	InlineVolumes []struct {
+		// Attributes are passed as NodePublishVolumeReq.volume_context.
+		// Can be empty.
+		Attributes map[string]string
+		// Shared defines whether the resulting volume is
+		// shared between different pods (i.e.  changes made
+		// in one pod are visible in another)
+		Shared bool
+		// ReadOnly must be set to true if the driver does not
+		// support mounting as read/write.
+		ReadOnly bool
+	}
 
 	// ClaimSize defines the desired size of dynamically
 	// provisioned volumes. Default is "5GiB".
@@ -221,7 +232,7 @@ func (d *driverDefinition) SkipUnsupportedTest(pattern testpatterns.TestPattern)
 			supported = true
 		}
 	case testpatterns.CSIInlineVolume:
-		supported = len(d.InlineVolumeAttributes) != 0
+		supported = len(d.InlineVolumes) != 0
 	}
 	if !supported {
 		framework.Skipf("Driver %q does not support volume type %q - skipping", d.DriverInfo.Name, pattern.VolType)
@@ -294,11 +305,12 @@ func (d *driverDefinition) GetClaimSize() string {
 	return d.ClaimSize
 }
 
-func (d *driverDefinition) GetVolumeAttributes(config *testsuites.PerTestConfig, volumeNumber int) map[string]string {
-	if len(d.InlineVolumeAttributes) == 0 {
+func (d *driverDefinition) GetVolume(config *testsuites.PerTestConfig, volumeNumber int) (map[string]string, bool, bool) {
+	if len(d.InlineVolumes) == 0 {
 		framework.Skipf("%s does not have any InlineVolumeAttributes defined", d.DriverInfo.Name)
 	}
-	return d.InlineVolumeAttributes[volumeNumber%len(d.InlineVolumeAttributes)]
+	volume := d.InlineVolumes[volumeNumber%len(d.InlineVolumes)]
+	return volume.Attributes, volume.Shared, volume.ReadOnly
 }
 
 func (d *driverDefinition) GetCSIDriverName(config *testsuites.PerTestConfig) string {