tkg win test fix

tests/e2e/constants/env_constants.go

@@ -213,4 +213,5 @@ const (
 	EnvVmknic4Vsan                             = "VMKNIC_FOR_VSAN"
 	EnvAccessMode                              = "ACCESS_MODE"
 	EnvVpToCsi                                 = "VCPTOCSI"
+	EnvVsphereTKGSystemNamespace               = "VSPHERE_TKG_SYSTEM_NAMESPACE"
 )

tests/e2e/constants/kubernetes.go

@@ -99,7 +99,6 @@ const (
 	VsphereCloudProviderConfiguration         = "vsphere-cloud-provider.conf"
 	VsphereControllerManager                  = "vmware-system-tkg-controller-manager"
 	VSphereCSIConf                            = "csi-vsphere.conf"
-	VsphereTKGSystemNamespace                 = "svc-tkg-domain-c10"
 	WcpServiceName                            = "wcp"
 	VmcWcpHost                                = "10.2.224.24" //This is the LB IP of VMC WCP and its constant
 	DevopsTKG                                 = "test-cluster-e2e-script"

tests/e2e/e2e_common.go

@@ -229,7 +229,7 @@ const (
 	vsphereCloudProviderConfiguration         = "vsphere-cloud-provider.conf"
 	vsphereControllerManager                  = "vmware-system-tkg-controller-manager"
 	vSphereCSIConf                            = "csi-vsphere.conf"
-	vsphereTKGSystemNamespace                 = "svc-tkg-domain-c10"
+	envVsphereTKGSystemNamespace              = "VSPHERE_TKG_SYSTEM_NAMESPACE"
 	waitTimeForCNSNodeVMAttachmentReconciler  = 30 * time.Second
 	wcpServiceName                            = "wcp"
 	vmcWcpHost                                = "10.2.224.24" //This is the LB IP of VMC WCP and its constant
@@ -349,6 +349,7 @@ const (
 	stretchedSvc          = "stretchedSvc"
 	devops                = "devops"
 	vc901                 = "vc901"
+	multiGc               = "multiGc"
 )
 
 // The following variables are required to know cluster type to run common e2e

tests/e2e/gc_block_resize_retain_policy.go

@@ -126,7 +126,8 @@ var _ = ginkgo.Describe("[csi-guest] Volume Expansion Tests with reclaimation po
 
 		// Replace second element with pod.Name.
 		if windowsEnv {
-			cmd = []string{"exec", "", "--namespace=" + namespace, "powershell.exe", "cat", "/mnt/volume1/fstype.txt"}
+			cmd = []string{"exec", "", "--namespace=" + namespace, "--", "powershell.exe", "-Command", "cat",
+				"/mnt/volume1/fstype.txt"}
 		} else {
 			cmd = []string{"exec", "", "--namespace=" + namespace, "--", "/bin/sh", "-c", "df -Tkm | grep /mnt/volume1"}
 		}
@@ -206,7 +207,8 @@ var _ = ginkgo.Describe("[csi-guest] Volume Expansion Tests with reclaimation po
 	// 18. Delete PVC created in step 6.
 	// 19. Delete PV leftover in GC.
 	// 20. Delete SC created in step 1.
-	ginkgo.It("PV with reclaim policy can be reused and resized with pod", ginkgo.Label(p0, block, tkg, vc70), func() {
+	ginkgo.It("PV with reclaim policy can be reused and resized with pod", ginkgo.Label(p0, block, tkg,
+		windows, vc70), func() {
 		ctx, cancel := context.WithCancel(context.Background())
 		defer cancel()
 		// Create a Pod to use this PVC, and verify volume has been attached.
@@ -586,7 +588,7 @@ var _ = ginkgo.Describe("[csi-guest] Volume Expansion Tests with reclaimation po
 				"exec",
 				pod.Name,
 				"--namespace=" + namespaceNewGC,
-				"powershell.exe",
+				"powershell.exe -Command",
 				"cat",
 				"/mnt/volume1/fstype.txt",
 			}

tests/e2e/gc_block_volume_expansion.go

@@ -117,7 +117,8 @@ var _ = ginkgo.Describe("[csi-guest] Volume Expansion Test", func() {
 
 		// Replace second element with pod.Name.
 		if windowsEnv {
-			cmd = []string{"exec", "", "--namespace=" + namespace, "powershell.exe", "cat", "/mnt/volume1/fstype.txt"}
+			cmd = []string{"exec", "", "--namespace=" + namespace, "--", "powershell.exe", "-Command",
+				"cat", "/mnt/volume1/fstype.txt"}
 		} else {
 			cmd = []string{"exec", "", "--namespace=" + namespace, "--", "/bin/sh", "-c", "df -Tkm | grep /mnt/volume1"}
 		}
@@ -236,9 +237,11 @@ var _ = ginkgo.Describe("[csi-guest] Volume Expansion Test", func() {
 				"exec",
 				pod.Name,
 				"--namespace=" + namespace,
+				"--",
 				"powershell.exe",
-				"$out = New-Object byte[] 536870912; (New-Object Random).NextBytes($out); " +
-					"[System.IO.File]::WriteAllBytes('/mnt/volume1/testdata2.txt', $out)",
+				"-Command",
+				"'$out = New-Object byte[] 536870912; (New-Object Random).NextBytes($out); " +
+					"[System.IO.File]::WriteAllBytes('/mnt/volume1/testdata2.txt', $out)'",
 			}
 			_ = e2ekubectl.RunKubectlOrDie(namespace, cmdTestData...)
 		} else {
@@ -360,7 +363,9 @@ var _ = ginkgo.Describe("[csi-guest] Volume Expansion Test", func() {
 				"exec",
 				pod.Name,
 				"--namespace=" + namespace,
+				"--",
 				"powershell.exe",
+				"-Command",
 				"Copy-Item -Path '/mnt/volume1/testdata2.txt' " +
 					"-Destination '/mnt/volume1/testdata2_pod.txt'",
 			}
@@ -380,7 +385,9 @@ var _ = ginkgo.Describe("[csi-guest] Volume Expansion Test", func() {
 				"exec",
 				pod.Name,
 				"--namespace=" + namespace,
+				"--",
 				"powershell.exe",
+				"-Command",
 				"((Get-FileHash '/mnt/volume1/testdata2.txt' -Algorithm SHA256).Hash -eq " +
 					"(Get-FileHash '/mnt/volume1/testdata2_pod.txt' -Algorithm SHA256).Hash)",
 			}
@@ -1527,6 +1534,11 @@ var _ = ginkgo.Describe("[csi-guest] Volume Expansion Test", func() {
 		ginkgo.By("Creating pod to attach PV to the node")
 		pod, err := createPod(ctx, client, namespace, nil, []*v1.PersistentVolumeClaim{pvclaim}, false, execCommand)
 		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+		defer func() {
+			// Delete POD.
+			ginkgo.By(fmt.Sprintf("Deleting the pod %s in namespace %s", pod.Name, namespace))
+			deletePodAndWaitForVolsToDetach(ctx, client, pod)
+		}()
 
 		ginkgo.By(fmt.Sprintf("Verify volume: %s is attached to the node: %s",
 			pv.Spec.CSI.VolumeHandle, pod.Spec.NodeName))
@@ -1564,10 +1576,13 @@ var _ = ginkgo.Describe("[csi-guest] Volume Expansion Test", func() {
 				"exec",
 				pod.Name,
 				"--namespace=" + namespace,
+				"--",
 				"powershell.exe",
+				"-Command",
 				"$out = New-Object byte[] 536870912; (New-Object Random).NextBytes($out); " +
 					"[System.IO.File]::WriteAllBytes('/mnt/volume1/testdata2.txt', $out)",
 			}
+
 			_ = e2ekubectl.RunKubectlOrDie(namespace, cmdTestData...)
 		} else {
 			_ = e2ekubectl.RunKubectlOrDie(namespace, "cp", testdataFile,
@@ -1582,7 +1597,9 @@ var _ = ginkgo.Describe("[csi-guest] Volume Expansion Test", func() {
 				"exec",
 				pod.Name,
 				"--namespace=" + namespace,
+				"--",
 				"powershell.exe",
+				"-Command",
 				"Copy-Item -Path '/mnt/volume1/testdata2.txt' " +
 					"-Destination '/mnt/volume1/testdata2_pod.txt'",
 			}
@@ -1603,7 +1620,9 @@ var _ = ginkgo.Describe("[csi-guest] Volume Expansion Test", func() {
 				"exec",
 				pod.Name,
 				"--namespace=" + namespace,
+				"--",
 				"powershell.exe",
+				"-Command",
 				"((Get-FileHash '/mnt/volume1/testdata2.txt' -Algorithm SHA256).Hash -eq " +
 					"(Get-FileHash '/mnt/volume1/testdata2_pod.txt' -Algorithm SHA256).Hash)",
 			}

tests/e2e/k8testutil/util.go

@@ -1146,8 +1146,8 @@ func BringDownCsiController(Client clientset.Interface, namespace ...string) {
 
 // bringDownTKGController helps to bring the TKG control manager pod down.
 // Its taks svc client as input.
-func BringDownTKGController(Client clientset.Interface) {
-	UpdateDeploymentReplica(Client, 0, constants.VsphereControllerManager, constants.VsphereTKGSystemNamespace)
+func BringDownTKGController(Client clientset.Interface, vsphereTKGSystemNamespace string) {
+	UpdateDeploymentReplica(Client, 0, constants.VsphereControllerManager, vsphereTKGSystemNamespace)
 	ginkgo.By("TKGControllManager replica is set to 0")
 }
 
@@ -1168,9 +1168,9 @@ func BringUpCsiController(Client clientset.Interface, csiReplicaCount int32, nam
 
 // bringUpTKGController helps to bring the TKG control manager pod up.
 // Its taks svc client as input.
-func BringUpTKGController(Client clientset.Interface, tkgReplica int32) {
+func BringUpTKGController(Client clientset.Interface, tkgReplica int32, vsphereTKGSystemNamespace string) {
 	err := UpdateDeploymentReplicawithWait(Client,
-		tkgReplica, constants.VsphereControllerManager, constants.VsphereTKGSystemNamespace)
+		tkgReplica, constants.VsphereControllerManager, vsphereTKGSystemNamespace)
 	gomega.Expect(err).NotTo(gomega.HaveOccurred())
 	ginkgo.By("TKGControllManager is up")
 }
@@ -7148,7 +7148,7 @@ func ExecCommandOnGcWorker(sshClientConfig *ssh.ClientConfig,
 	}
 
 	cmdToGetContainerInfo := fmt.Sprintf("ssh -o StrictHostKeyChecking=no -i key %s@%s "+
-		"'%s' | grep -v 'Warning'", constants.GcNodeUser, gcWorkerIp, cmd)
+		"'%s' 2> /dev/null", constants.GcNodeUser, gcWorkerIp, cmd)
 	framework.Logf("Invoking command '%v' on host %v", cmdToGetContainerInfo,
 		svcMasterIP)
 	cmdResult, err = SshExec(sshClientConfig, vs, svcMasterIP,

tests/e2e/tkgs_ha_utils.go

@@ -268,7 +268,7 @@ func verifyOnlineVolumeExpansionOnGc(client clientset.Interface, namespace strin
 			"exec",
 			pod.Name,
 			"--namespace=" + namespace,
-			"powershell.exe",
+			"-- powershell.exe -Command",
 			"$out = New-Object byte[] 536870912; (New-Object Random).NextBytes($out); " +
 				"[System.IO.File]::WriteAllBytes('/mnt/volume1/testdata2.txt', $out)",
 		}
@@ -286,7 +286,7 @@ func verifyOnlineVolumeExpansionOnGc(client clientset.Interface, namespace strin
 			"exec",
 			pod.Name,
 			"--namespace=" + namespace,
-			"powershell.exe",
+			"-- powershell.exe -Command",
 			"Copy-Item -Path '/mnt/volume1/testdata2.txt' " +
 				"-Destination '/mnt/volume1/testdata2_pod.txt'",
 		}
@@ -306,7 +306,7 @@ func verifyOnlineVolumeExpansionOnGc(client clientset.Interface, namespace strin
 			"exec",
 			pod.Name,
 			"--namespace=" + namespace,
-			"powershell.exe",
+			"-- powershell.exe -Command",
 			"((Get-FileHash '/mnt/volume1/testdata2.txt' -Algorithm SHA256).Hash -eq " +
 				"(Get-FileHash '/mnt/volume1/testdata2_pod.txt' -Algorithm SHA256).Hash)",
 		}

tests/e2e/util.go

@@ -1066,7 +1066,7 @@ func bringDownCsiController(Client clientset.Interface, namespace ...string) {
 
 // bringDownTKGController helps to bring the TKG control manager pod down.
 // Its taks svc client as input.
-func bringDownTKGController(Client clientset.Interface) {
+func bringDownTKGController(Client clientset.Interface, vsphereTKGSystemNamespace string) {
 	updateDeploymentReplica(Client, 0, vsphereControllerManager, vsphereTKGSystemNamespace)
 	ginkgo.By("TKGControllManager replica is set to 0")
 }
@@ -1088,7 +1088,7 @@ func bringUpCsiController(Client clientset.Interface, csiReplicaCount int32, nam
 
 // bringUpTKGController helps to bring the TKG control manager pod up.
 // Its taks svc client as input.
-func bringUpTKGController(Client clientset.Interface, tkgReplica int32) {
+func bringUpTKGController(Client clientset.Interface, tkgReplica int32, vsphereTKGSystemNamespace string) {
 	err := updateDeploymentReplicawithWait(Client, tkgReplica, vsphereControllerManager, vsphereTKGSystemNamespace)
 	gomega.Expect(err).NotTo(gomega.HaveOccurred())
 	ginkgo.By("TKGControllManager is up")
@@ -7598,7 +7598,7 @@ func execCommandOnGcWorker(sshClientConfig *ssh.ClientConfig, svcMasterIP string
 	}
 
 	cmdToGetContainerInfo := fmt.Sprintf("ssh -o StrictHostKeyChecking=no -i key %s@%s "+
-		"'%s' | grep -v 'Warning'", gcNodeUser, gcWorkerIp, cmd)
+		"'%s' 2> /dev/null", gcNodeUser, gcWorkerIp, cmd)
 	framework.Logf("Invoking command '%v' on host %v", cmdToGetContainerInfo,
 		svcMasterIP)
 	cmdResult, err = sshExec(sshClientConfig, svcMasterIP,

tests/e2e/volume_health_test.go

@@ -57,6 +57,7 @@ var _ = ginkgo.Describe("Volume health check", func() {
 		isVsanHealthServiceStopped bool
 		isSPSServiceStopped        bool
 		csiNamespace               string
+		vsphereTKGSystemNamespace  string
 	)
 
 	ginkgo.BeforeEach(func() {
@@ -70,6 +71,7 @@ var _ = ginkgo.Describe("Volume health check", func() {
 		csiNamespace = GetAndExpectStringEnvVar(envCSINamespace)
 		nodeList, err := fnodes.GetReadySchedulableNodes(ctx, f.ClientSet)
 		datastoreURL = GetAndExpectStringEnvVar(envSharedDatastoreURL)
+		vsphereTKGSystemNamespace = GetAndExpectStringEnvVar(envVsphereTKGSystemNamespace)
 		framework.ExpectNoError(err, "Unable to find ready and schedulable Node")
 		if guestCluster {
 			svcClient, svNamespace := getSvcClientAndNamespace()
@@ -1289,14 +1291,14 @@ var _ = ginkgo.Describe("Volume health check", func() {
 		gomega.Expect(err).NotTo(gomega.HaveOccurred())
 		csiReplicaCount := *deployment.Spec.Replicas
 
-		bringDownTKGController(svClient)
-		bringDownCsiController(gcClient)
+		bringDownTKGController(svClient, vsphereTKGSystemNamespace)
+		bringDownCsiController(gcClient, vsphereTKGSystemNamespace)
 		isControllerUP = false
 
 		defer func() {
 			if !isControllerUP {
-				bringUpTKGController(svClient, tkgReplicaCount)
-				bringUpCsiController(gcClient, csiReplicaCount)
+				bringUpTKGController(svClient, tkgReplicaCount, vsphereTKGSystemNamespace)
+				bringUpCsiController(gcClient, csiReplicaCount, vsphereTKGSystemNamespace)
 			}
 		}()
 
@@ -1313,8 +1315,8 @@ var _ = ginkgo.Describe("Volume health check", func() {
 		gomega.Expect(svPVC.Annotations[volumeHealthAnnotation]).Should(gomega.BeEquivalentTo(healthStatusAccessible))
 
 		ginkgo.By("Bring up csi-controller pod in GC")
-		bringUpTKGController(svClient, tkgReplicaCount)
-		bringUpCsiController(gcClient, csiReplicaCount)
+		bringUpTKGController(svClient, tkgReplicaCount, vsphereTKGSystemNamespace)
+		bringUpCsiController(gcClient, csiReplicaCount, vsphereTKGSystemNamespace)
 		isControllerUP = true
 
 		ginkgo.By("Verify health status of GC PVC after GC csi is up")
@@ -2123,7 +2125,6 @@ var _ = ginkgo.Describe("Volume health check", func() {
 			gcClient, err = createKubernetesClientFromConfig(k8senv)
 			gomega.Expect(err).NotTo(gomega.HaveOccurred())
 		}
-
 		tkgReplicaDeployment, err := svcClient.AppsV1().Deployments(vsphereTKGSystemNamespace).Get(ctx,
 			vsphereControllerManager, metav1.GetOptions{})
 		gomega.Expect(err).NotTo(gomega.HaveOccurred())
@@ -2140,8 +2141,8 @@ var _ = ginkgo.Describe("Volume health check", func() {
 			err := waitForHostToBeUp(hostIP)
 			gomega.Expect(err).NotTo(gomega.HaveOccurred())
 			if !isControllerUP {
-				bringUpTKGController(svcClient, tkgReplicaCount)
-				bringUpCsiController(gcClient, csiReplicaCount)
+				bringUpTKGController(svcClient, tkgReplicaCount, vsphereTKGSystemNamespace)
+				bringUpCsiController(gcClient, csiReplicaCount, vsphereTKGSystemNamespace)
 			}
 			err = fpv.DeletePersistentVolumeClaim(ctx, client, pvc.Name, namespace)
 			gomega.Expect(err).NotTo(gomega.HaveOccurred())
@@ -2160,8 +2161,8 @@ var _ = ginkgo.Describe("Volume health check", func() {
 		gomega.Expect(pvclaim.Annotations[volumeHealthAnnotation]).Should(gomega.BeEquivalentTo(healthStatusAccessible))
 
 		ginkgo.By("Bring down csi-controller pod in GC")
-		bringDownTKGController(svcClient)
-		bringDownCsiController(gcClient)
+		bringDownTKGController(svcClient, vsphereTKGSystemNamespace)
+		bringDownCsiController(gcClient, vsphereTKGSystemNamespace)
 		isControllerUP = false
 
 		// Get SV PVC before PSOD.
@@ -2181,8 +2182,8 @@ var _ = ginkgo.Describe("Volume health check", func() {
 		gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
 		ginkgo.By("Bring up csi-controller pod in GC")
-		bringUpTKGController(svcClient, tkgReplicaCount)
-		bringUpCsiController(gcClient, csiReplicaCount)
+		bringUpTKGController(svcClient, tkgReplicaCount, vsphereTKGSystemNamespace)
+		bringUpCsiController(gcClient, csiReplicaCount, vsphereTKGSystemNamespace)
 		isControllerUP = true
 
 		ginkgo.By("Verify health status of GC PVC after GC csi is up")