multi-Svc test refactor (kubernetes-sigs#3469)

Author: rajguptavm

hack/run-e2e-test.sh

@@ -73,7 +73,7 @@ then
     OPTS+=(-p)
     ginkgo -mod=mod "${OPTS[@]}" --focus="csi-block-vanilla-parallelized" tests/e2e
 else
-    ginkgo -mod=mod "${OPTS[@]}" --focus="$FOCUS" tests/e2e
+    ginkgo -mod=mod "${OPTS[@]}" --focus="$FOCUS" -r tests/e2e
 fi
 
 # Checking for test status

tests/e2e/constants/kubernetes.go

@@ -78,6 +78,7 @@ const (
 	Snapshotapigroup                          = "snapshot.storage.k8s.io"
 	DefaultNginxStorageClassName              = "nginx-sc"
 	MountPath                                 = "/usr/share/nginx/html"
+	ServiceName                               = "nginx"
 	SpsServiceName                            = "sps"
 	SnapshotterContainerName                  = "csi-snapshotter"
 	SshdPort                                  = "22"

tests/e2e/e2e_common.go

@@ -464,16 +464,9 @@ var (
 
 // multiSvc env variables
 var (
-	vcSessionWaitTime                   = 5 * time.Minute
 	envStoragePolicyNameForSharedDsSvc1 = "STORAGE_POLICY_FOR_SHARED_DATASTORES_SVC1"
 	envStoragePolicyNameForSharedDsSvc2 = "STORAGE_POLICY_FOR_SHARED_DATASTORES_SVC2"
-	envSupervisorClusterNamespace1      = "SVC_NAMESPACE1"
-	envNfsDatastoreName                 = "NFS_DATASTORE_NAME"
-	envNfsDatastoreIP                   = "NFS_DATASTORE_IP"
 	pwdRotationTimeout                  = 10 * time.Minute
-	roleCnsDatastore                    = "CNS-SUPERVISOR-DATASTORE"
-	roleCnsSearchAndSpbm                = "CNS-SUPERVISOR-SEARCH-AND-SPBM"
-	roleCnsHostConfigStorageAndCnsVm    = "CNS-SUPERVISOR-HOST-CONFIG-STORAGE-AND-CNS-VM"
 )
 
 // For rwx

tests/e2e/k8testutil/util.go

@@ -2556,12 +2556,12 @@ func DeleteService(ns string, c clientset.Interface, service *v1.Service) {
 
 // GetStatefulSetFromManifest creates a StatefulSet from the statefulset.yaml
 // file present in the manifest path.
-func GetStatefulSetFromManifest(e2eTestConfig *config.E2eTestConfig, ns string) *appsv1.StatefulSet {
+func GetStatefulSetFromManifest(e2eTestConfig *config.TestInputData, ns string) *appsv1.StatefulSet {
 	ssManifestFilePath := filepath.Join(constants.ManifestPath, "statefulset.yaml")
 	framework.Logf("Parsing statefulset from %v", ssManifestFilePath)
 	ss, err := manifest.StatefulSetFromManifest(ssManifestFilePath, ns)
 	framework.ExpectNoError(err)
-	if e2eTestConfig.TestInput.TestBedInfo.WindowsEnv {
+	if e2eTestConfig.TestBedInfo.WindowsEnv {
 		ss.Spec.Template.Spec.Containers[0].Image = constants.WindowsImageOnMcr
 		ss.Spec.Template.Spec.Containers[0].Command = []string{"Powershell.exe"}
 		ss.Spec.Template.Spec.Containers[0].Args = []string{"-Command", constants.WindowsExecCmd}
@@ -3053,6 +3053,7 @@ func TrimQuotes(str string) string {
 // Returns a de-serialized structured config data
 func ReadConfigFromSecretString(cfg string) (config.E2eTestConfig, error) {
 	var config1 config.E2eTestConfig
+	var testInput config.TestInputData
 	var netPerm config.NetPermissionConfig
 	key, value := "", ""
 	var permissions vsanfstypes.VsanFileShareAccessType
@@ -3082,7 +3083,7 @@ func ReadConfigFromSecretString(cfg string) (config.E2eTestConfig, error) {
 				value = words[1]
 				// Remove trailing '"]' characters from value.
 				value = strings.TrimSuffix(value, "]")
-				config1.TestInput.Global.VCenterHostname = TrimQuotes(value)
+				testInput.Global.VCenterHostname = TrimQuotes(value)
 				fmt.Printf("Key: VirtualCenter, Value: %s\n", value)
 			}
 			continue
@@ -3093,47 +3094,47 @@ func ReadConfigFromSecretString(cfg string) (config.E2eTestConfig, error) {
 		switch key {
 		case "insecure-flag":
 			if strings.Contains(value, "true") {
-				config1.TestInput.Global.InsecureFlag = true
+				testInput.Global.InsecureFlag = true
 			} else {
-				config1.TestInput.Global.InsecureFlag = false
+				testInput.Global.InsecureFlag = false
 			}
 		case "cluster-id":
-			config1.TestInput.Global.ClusterID = value
+			testInput.Global.ClusterID = value
 		case "cluster-distribution":
-			config1.TestInput.Global.ClusterDistribution = value
+			testInput.Global.ClusterDistribution = value
 		case "user":
-			config1.TestInput.Global.User = value
+			testInput.Global.User = value
 		case "password":
-			config1.TestInput.Global.Password = value
+			testInput.Global.Password = value
 		case "datacenters":
-			config1.TestInput.Global.Datacenters = value
+			testInput.Global.Datacenters = value
 		case "port":
-			config1.TestInput.Global.VCenterPort = value
+			testInput.Global.VCenterPort = value
 		case "cnsregistervolumes-cleanup-intervalinmin":
-			config1.TestInput.Global.CnsRegisterVolumesCleanupIntervalInMin, strconvErr = strconv.Atoi(value)
+			testInput.Global.CnsRegisterVolumesCleanupIntervalInMin, strconvErr = strconv.Atoi(value)
 			gomega.Expect(strconvErr).NotTo(gomega.HaveOccurred())
 		case "topology-categories":
-			config1.TestInput.Labels.TopologyCategories = value
+			testInput.Labels.TopologyCategories = value
 		case "global-max-snapshots-per-block-volume":
-			config1.TestInput.Snapshot.GlobalMaxSnapshotsPerBlockVolume, strconvErr = strconv.Atoi(value)
+			testInput.Snapshot.GlobalMaxSnapshotsPerBlockVolume, strconvErr = strconv.Atoi(value)
 			gomega.Expect(strconvErr).NotTo(gomega.HaveOccurred())
 		case "csi-fetch-preferred-datastores-intervalinmin":
-			config1.TestInput.Global.CSIFetchPreferredDatastoresIntervalInMin, strconvErr = strconv.Atoi(value)
+			testInput.Global.CSIFetchPreferredDatastoresIntervalInMin, strconvErr = strconv.Atoi(value)
 			gomega.Expect(strconvErr).NotTo(gomega.HaveOccurred())
 		case "query-limit":
-			config1.TestInput.Global.QueryLimit, strconvErr = strconv.Atoi(value)
+			testInput.Global.QueryLimit, strconvErr = strconv.Atoi(value)
 			gomega.Expect(strconvErr).NotTo(gomega.HaveOccurred())
 		case "list-volume-threshold":
-			config1.TestInput.Global.ListVolumeThreshold, strconvErr = strconv.Atoi(value)
+			testInput.Global.ListVolumeThreshold, strconvErr = strconv.Atoi(value)
 			gomega.Expect(strconvErr).NotTo(gomega.HaveOccurred())
 		case "ca-file":
-			config1.TestInput.Global.CaFile = value
+			testInput.Global.CaFile = value
 		case "supervisor-id":
-			config1.TestInput.Global.SupervisorID = value
+			testInput.Global.SupervisorID = value
 		case "targetvSANFileShareClusters":
-			config1.TestInput.Global.TargetVsanFileShareClusters = value
+			testInput.Global.TargetVsanFileShareClusters = value
 		case "fileVolumeActivated":
-			config1.TestInput.Global.FileVolumeActivated, strconvErr = strconv.ParseBool(value)
+			testInput.Global.FileVolumeActivated, strconvErr = strconv.ParseBool(value)
 			gomega.Expect(strconvErr).NotTo(gomega.HaveOccurred())
 		case "ips":
 			netPerm.Ips = value
@@ -3145,6 +3146,7 @@ func ReadConfigFromSecretString(cfg string) (config.E2eTestConfig, error) {
 			return config1, fmt.Errorf("unknown key %s in the input string", key)
 		}
 	}
+	config1.TestInput = &testInput
 	return config1, nil
 }
 
@@ -5783,7 +5785,7 @@ func CreateParallelStatefulSets(client clientset.Interface, namespace string,
 	gomega.Expect(err).NotTo(gomega.HaveOccurred())
 }
 
-func CreateParallelStatefulSetSpec(e2eTestConfig *config.E2eTestConfig,
+func CreateParallelStatefulSetSpec(e2eTestConfig *config.TestInputData,
 	namespace string, no_of_sts int, replicas int32) []*appsv1.StatefulSet {
 	stss := []*appsv1.StatefulSet{}
 	var statefulset *appsv1.StatefulSet
@@ -7315,3 +7317,142 @@ func ListStoragePolicyUsages(ctx context.Context, c clientset.Interface, restCli
 
 	fmt.Println("All required storage policy usages are available.")
 }
+
+// ExitHostMM exits a host from maintenance mode with a particular timeout
+func ExitHostMM(ctx context.Context, host *object.HostSystem, timeout int32) {
+	task, err := host.ExitMaintenanceMode(ctx, timeout)
+	gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+	_, err = task.WaitForResultEx(ctx, nil)
+	gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+	framework.Logf("Host: %v exited from maintenance mode", host)
+}
+
+/*
+CreateStatefulSetAndVerifyPVAndPodNodeAffinty creates user specified statefulset and
+further checks the node and volumes affinities
+*/
+func CreateStatefulSetAndVerifyPVAndPodNodeAffinty(ctx context.Context, client clientset.Interface,
+	vs *config.E2eTestConfig, namespace string, parallelPodPolicy bool, replicas int32, nodeAffinityToSet bool,
+	allowedTopologies []v1.TopologySelectorLabelRequirement,
+	podAntiAffinityToSet bool, parallelStatefulSetCreation bool, modifyStsSpec bool,
+	accessMode v1.PersistentVolumeAccessMode,
+	sc *storagev1.StorageClass, verifyTopologyAffinity bool, storagePolicy string) (*v1.Service,
+	*appsv1.StatefulSet, error) {
+
+	ginkgo.By("Create service")
+	service := CreateService(namespace, client)
+
+	framework.Logf("Create StatefulSet")
+	statefulset := CreateCustomisedStatefulSets(ctx, client, vs.TestInput, namespace, parallelPodPolicy,
+		replicas, nodeAffinityToSet, allowedTopologies, podAntiAffinityToSet, modifyStsSpec,
+		"", accessMode, sc, storagePolicy)
+
+	if verifyTopologyAffinity {
+		framework.Logf("Verify PV node affinity and that the PODS are running on appropriate node")
+		err := VerifyPVnodeAffinityAndPODnodedetailsForStatefulsetsLevel5(ctx, vs, client, statefulset,
+			namespace, allowedTopologies, parallelStatefulSetCreation)
+		if err != nil {
+			return nil, nil, fmt.Errorf("error verifying PV node affinity and POD node details: %v", err)
+		}
+	}
+
+	return service, statefulset, nil
+}
+
+/*
+createCustomisedStatefulSets util methods creates statefulset as per the user's
+specific requirement and returns the customised statefulset
+*/
+func CreateCustomisedStatefulSets(ctx context.Context, client clientset.Interface, vs *config.TestInputData,
+	namespace string, isParallelPodMgmtPolicy bool, replicas int32, nodeAffinityToSet bool,
+	allowedTopologies []v1.TopologySelectorLabelRequirement,
+	podAntiAffinityToSet bool, modifyStsSpec bool, stsName string,
+	accessMode v1.PersistentVolumeAccessMode, sc *storagev1.StorageClass, storagePolicy string) *appsv1.StatefulSet {
+	framework.Logf("Preparing StatefulSet Spec")
+	statefulset := GetStatefulSetFromManifest(vs, namespace)
+
+	if accessMode == "" {
+		// If accessMode is not specified, set the default accessMode.
+		defaultAccessMode := v1.ReadWriteOnce
+		statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1].
+			Spec.AccessModes[0] = defaultAccessMode
+	} else {
+		statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1].Spec.AccessModes[0] =
+			accessMode
+	}
+
+	if modifyStsSpec {
+		if vs.TestBedInfo.MultipleSvc {
+			statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1].
+				Spec.StorageClassName = &storagePolicy
+		} else {
+			statefulset.Spec.VolumeClaimTemplates[len(statefulset.Spec.VolumeClaimTemplates)-1].
+				Spec.StorageClassName = &sc.Name
+		}
+
+		if stsName != "" {
+			statefulset.Name = stsName
+			statefulset.Spec.Template.Labels["app"] = statefulset.Name
+			statefulset.Spec.Selector.MatchLabels["app"] = statefulset.Name
+		}
+
+	}
+	if nodeAffinityToSet {
+		nodeSelectorTerms := GetNodeSelectorTerms(allowedTopologies)
+		statefulset.Spec.Template.Spec.Affinity = new(v1.Affinity)
+		statefulset.Spec.Template.Spec.Affinity.NodeAffinity = new(v1.NodeAffinity)
+		statefulset.Spec.Template.Spec.Affinity.NodeAffinity.
+			RequiredDuringSchedulingIgnoredDuringExecution = new(v1.NodeSelector)
+		statefulset.Spec.Template.Spec.Affinity.NodeAffinity.
+			RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = nodeSelectorTerms
+	}
+	if podAntiAffinityToSet {
+		statefulset.Spec.Template.Spec.Affinity = &v1.Affinity{
+			PodAntiAffinity: &v1.PodAntiAffinity{
+				RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{
+					{
+						LabelSelector: &metav1.LabelSelector{
+							MatchLabels: map[string]string{
+								"key": "app",
+							},
+						},
+						TopologyKey: "topology.kubernetes.io/zone",
+					},
+				},
+			},
+		}
+
+	}
+	if isParallelPodMgmtPolicy {
+		statefulset.Spec.PodManagementPolicy = appsv1.ParallelPodManagement
+	}
+	statefulset.Spec.Replicas = &replicas
+
+	framework.Logf("Creating statefulset")
+	CreateStatefulSet(namespace, statefulset, client)
+
+	framework.Logf("Wait for StatefulSet pods to be in up and running state")
+	fss.WaitForStatusReadyReplicas(ctx, client, statefulset, replicas)
+	gomega.Expect(fss.CheckMount(ctx, client, statefulset, constants.MountPath)).NotTo(gomega.HaveOccurred())
+	ssPodsBeforeScaleDown, err := fss.GetPodList(ctx, client, statefulset)
+	gomega.Expect(err).NotTo(gomega.HaveOccurred())
+	gomega.Expect(ssPodsBeforeScaleDown.Items).NotTo(gomega.BeEmpty(),
+		fmt.Sprintf("Unable to get list of Pods from the Statefulset: %v", statefulset.Name))
+	gomega.Expect(len(ssPodsBeforeScaleDown.Items) == int(replicas)).To(gomega.BeTrue(),
+		"Number of Pods in the statefulset should match with number of replicas")
+
+	return statefulset
+}
+
+// CreateStatefulSet creates a StatefulSet from the manifest at manifestPath in the given namespace.
+func CreateStatefulSet(ns string, ss *appsv1.StatefulSet, c clientset.Interface) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	framework.Logf("Creating statefulset %v/%v with %d replicas and selector %+v",
+		ss.Namespace, ss.Name, *(ss.Spec.Replicas), ss.Spec.Selector)
+	_, err := c.AppsV1().StatefulSets(ns).Create(ctx, ss, metav1.CreateOptions{})
+	framework.ExpectNoError(err)
+	fss.WaitForRunningAndReady(ctx, c, *ss.Spec.Replicas, ss)
+}