diff --git a/tests/e2e/constants/env_constants.go b/tests/e2e/constants/env_constants.go index 86b6e62083..663dc195b5 100644 --- a/tests/e2e/constants/env_constants.go +++ b/tests/e2e/constants/env_constants.go @@ -213,4 +213,5 @@ const ( EnvVmknic4Vsan = "VMKNIC_FOR_VSAN" EnvAccessMode = "ACCESS_MODE" EnvVpToCsi = "VCPTOCSI" + EnvVsphereTKGSystemNamespace = "VSPHERE_TKG_SYSTEM_NAMESPACE" ) diff --git a/tests/e2e/constants/kubernetes.go b/tests/e2e/constants/kubernetes.go index fc02424c26..31108eef15 100644 --- a/tests/e2e/constants/kubernetes.go +++ b/tests/e2e/constants/kubernetes.go @@ -99,7 +99,6 @@ const ( VsphereCloudProviderConfiguration = "vsphere-cloud-provider.conf" VsphereControllerManager = "vmware-system-tkg-controller-manager" VSphereCSIConf = "csi-vsphere.conf" - VsphereTKGSystemNamespace = "svc-tkg-domain-c10" WcpServiceName = "wcp" VmcWcpHost = "10.2.224.24" //This is the LB IP of VMC WCP and its constant DevopsTKG = "test-cluster-e2e-script" diff --git a/tests/e2e/e2e_common.go b/tests/e2e/e2e_common.go index b4fe559faa..9c115e6a60 100644 --- a/tests/e2e/e2e_common.go +++ b/tests/e2e/e2e_common.go @@ -229,7 +229,7 @@ const ( vsphereCloudProviderConfiguration = "vsphere-cloud-provider.conf" vsphereControllerManager = "vmware-system-tkg-controller-manager" vSphereCSIConf = "csi-vsphere.conf" - vsphereTKGSystemNamespace = "svc-tkg-domain-c10" + envVsphereTKGSystemNamespace = "VSPHERE_TKG_SYSTEM_NAMESPACE" waitTimeForCNSNodeVMAttachmentReconciler = 30 * time.Second wcpServiceName = "wcp" vmcWcpHost = "10.2.224.24" //This is the LB IP of VMC WCP and its constant @@ -349,6 +349,7 @@ const ( stretchedSvc = "stretchedSvc" devops = "devops" vc901 = "vc901" + multiGc = "multiGc" ) // The following variables are required to know cluster type to run common e2e diff --git a/tests/e2e/gc_block_resize_retain_policy.go b/tests/e2e/gc_block_resize_retain_policy.go index 266a71d375..aba8f3afe5 100644 --- a/tests/e2e/gc_block_resize_retain_policy.go +++ b/tests/e2e/gc_block_resize_retain_policy.go @@ -126,7 +126,8 @@ var _ = ginkgo.Describe("[csi-guest] Volume Expansion Tests with reclaimation po // Replace second element with pod.Name. if windowsEnv { - cmd = []string{"exec", "", "--namespace=" + namespace, "powershell.exe", "cat", "/mnt/volume1/fstype.txt"} + cmd = []string{"exec", "", "--namespace=" + namespace, "--", "powershell.exe", "-Command", "cat", + "/mnt/volume1/fstype.txt"} } else { cmd = []string{"exec", "", "--namespace=" + namespace, "--", "/bin/sh", "-c", "df -Tkm | grep /mnt/volume1"} } @@ -206,7 +207,8 @@ var _ = ginkgo.Describe("[csi-guest] Volume Expansion Tests with reclaimation po // 18. Delete PVC created in step 6. // 19. Delete PV leftover in GC. // 20. Delete SC created in step 1. - ginkgo.It("PV with reclaim policy can be reused and resized with pod", ginkgo.Label(p0, block, tkg, vc70), func() { + ginkgo.It("PV with reclaim policy can be reused and resized with pod", ginkgo.Label(p0, block, tkg, + windows, vc70), func() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() // Create a Pod to use this PVC, and verify volume has been attached. @@ -586,7 +588,7 @@ var _ = ginkgo.Describe("[csi-guest] Volume Expansion Tests with reclaimation po "exec", pod.Name, "--namespace=" + namespaceNewGC, - "powershell.exe", + "powershell.exe -Command", "cat", "/mnt/volume1/fstype.txt", } diff --git a/tests/e2e/gc_block_volume_expansion.go b/tests/e2e/gc_block_volume_expansion.go index 865791ef8e..b95a75e119 100644 --- a/tests/e2e/gc_block_volume_expansion.go +++ b/tests/e2e/gc_block_volume_expansion.go @@ -117,7 +117,8 @@ var _ = ginkgo.Describe("[csi-guest] Volume Expansion Test", func() { // Replace second element with pod.Name. if windowsEnv { - cmd = []string{"exec", "", "--namespace=" + namespace, "powershell.exe", "cat", "/mnt/volume1/fstype.txt"} + cmd = []string{"exec", "", "--namespace=" + namespace, "--", "powershell.exe", "-Command", + "cat", "/mnt/volume1/fstype.txt"} } else { cmd = []string{"exec", "", "--namespace=" + namespace, "--", "/bin/sh", "-c", "df -Tkm | grep /mnt/volume1"} } @@ -236,9 +237,11 @@ var _ = ginkgo.Describe("[csi-guest] Volume Expansion Test", func() { "exec", pod.Name, "--namespace=" + namespace, + "--", "powershell.exe", - "$out = New-Object byte[] 536870912; (New-Object Random).NextBytes($out); " + - "[System.IO.File]::WriteAllBytes('/mnt/volume1/testdata2.txt', $out)", + "-Command", + "'$out = New-Object byte[] 536870912; (New-Object Random).NextBytes($out); " + + "[System.IO.File]::WriteAllBytes('/mnt/volume1/testdata2.txt', $out)'", } _ = e2ekubectl.RunKubectlOrDie(namespace, cmdTestData...) } else { @@ -360,7 +363,9 @@ var _ = ginkgo.Describe("[csi-guest] Volume Expansion Test", func() { "exec", pod.Name, "--namespace=" + namespace, + "--", "powershell.exe", + "-Command", "Copy-Item -Path '/mnt/volume1/testdata2.txt' " + "-Destination '/mnt/volume1/testdata2_pod.txt'", } @@ -380,7 +385,9 @@ var _ = ginkgo.Describe("[csi-guest] Volume Expansion Test", func() { "exec", pod.Name, "--namespace=" + namespace, + "--", "powershell.exe", + "-Command", "((Get-FileHash '/mnt/volume1/testdata2.txt' -Algorithm SHA256).Hash -eq " + "(Get-FileHash '/mnt/volume1/testdata2_pod.txt' -Algorithm SHA256).Hash)", } @@ -1527,6 +1534,11 @@ var _ = ginkgo.Describe("[csi-guest] Volume Expansion Test", func() { ginkgo.By("Creating pod to attach PV to the node") pod, err := createPod(ctx, client, namespace, nil, []*v1.PersistentVolumeClaim{pvclaim}, false, execCommand) gomega.Expect(err).NotTo(gomega.HaveOccurred()) + defer func() { + // Delete POD. + ginkgo.By(fmt.Sprintf("Deleting the pod %s in namespace %s", pod.Name, namespace)) + deletePodAndWaitForVolsToDetach(ctx, client, pod) + }() ginkgo.By(fmt.Sprintf("Verify volume: %s is attached to the node: %s", pv.Spec.CSI.VolumeHandle, pod.Spec.NodeName)) @@ -1564,10 +1576,13 @@ var _ = ginkgo.Describe("[csi-guest] Volume Expansion Test", func() { "exec", pod.Name, "--namespace=" + namespace, + "--", "powershell.exe", + "-Command", "$out = New-Object byte[] 536870912; (New-Object Random).NextBytes($out); " + "[System.IO.File]::WriteAllBytes('/mnt/volume1/testdata2.txt', $out)", } + _ = e2ekubectl.RunKubectlOrDie(namespace, cmdTestData...) } else { _ = e2ekubectl.RunKubectlOrDie(namespace, "cp", testdataFile, @@ -1582,7 +1597,9 @@ var _ = ginkgo.Describe("[csi-guest] Volume Expansion Test", func() { "exec", pod.Name, "--namespace=" + namespace, + "--", "powershell.exe", + "-Command", "Copy-Item -Path '/mnt/volume1/testdata2.txt' " + "-Destination '/mnt/volume1/testdata2_pod.txt'", } @@ -1603,7 +1620,9 @@ var _ = ginkgo.Describe("[csi-guest] Volume Expansion Test", func() { "exec", pod.Name, "--namespace=" + namespace, + "--", "powershell.exe", + "-Command", "((Get-FileHash '/mnt/volume1/testdata2.txt' -Algorithm SHA256).Hash -eq " + "(Get-FileHash '/mnt/volume1/testdata2_pod.txt' -Algorithm SHA256).Hash)", } diff --git a/tests/e2e/k8testutil/util.go b/tests/e2e/k8testutil/util.go index 446a1329e3..541a72cf28 100644 --- a/tests/e2e/k8testutil/util.go +++ b/tests/e2e/k8testutil/util.go @@ -1146,8 +1146,8 @@ func BringDownCsiController(Client clientset.Interface, namespace ...string) { // bringDownTKGController helps to bring the TKG control manager pod down. // Its taks svc client as input. -func BringDownTKGController(Client clientset.Interface) { - UpdateDeploymentReplica(Client, 0, constants.VsphereControllerManager, constants.VsphereTKGSystemNamespace) +func BringDownTKGController(Client clientset.Interface, vsphereTKGSystemNamespace string) { + UpdateDeploymentReplica(Client, 0, constants.VsphereControllerManager, vsphereTKGSystemNamespace) ginkgo.By("TKGControllManager replica is set to 0") } @@ -1168,9 +1168,9 @@ func BringUpCsiController(Client clientset.Interface, csiReplicaCount int32, nam // bringUpTKGController helps to bring the TKG control manager pod up. // Its taks svc client as input. -func BringUpTKGController(Client clientset.Interface, tkgReplica int32) { +func BringUpTKGController(Client clientset.Interface, tkgReplica int32, vsphereTKGSystemNamespace string) { err := UpdateDeploymentReplicawithWait(Client, - tkgReplica, constants.VsphereControllerManager, constants.VsphereTKGSystemNamespace) + tkgReplica, constants.VsphereControllerManager, vsphereTKGSystemNamespace) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("TKGControllManager is up") } @@ -7148,7 +7148,7 @@ func ExecCommandOnGcWorker(sshClientConfig *ssh.ClientConfig, } cmdToGetContainerInfo := fmt.Sprintf("ssh -o StrictHostKeyChecking=no -i key %s@%s "+ - "'%s' | grep -v 'Warning'", constants.GcNodeUser, gcWorkerIp, cmd) + "'%s' 2> /dev/null", constants.GcNodeUser, gcWorkerIp, cmd) framework.Logf("Invoking command '%v' on host %v", cmdToGetContainerInfo, svcMasterIP) cmdResult, err = SshExec(sshClientConfig, vs, svcMasterIP, diff --git a/tests/e2e/tkgs_ha_utils.go b/tests/e2e/tkgs_ha_utils.go index ba0ffae688..41c4587fe5 100644 --- a/tests/e2e/tkgs_ha_utils.go +++ b/tests/e2e/tkgs_ha_utils.go @@ -268,7 +268,7 @@ func verifyOnlineVolumeExpansionOnGc(client clientset.Interface, namespace strin "exec", pod.Name, "--namespace=" + namespace, - "powershell.exe", + "-- powershell.exe -Command", "$out = New-Object byte[] 536870912; (New-Object Random).NextBytes($out); " + "[System.IO.File]::WriteAllBytes('/mnt/volume1/testdata2.txt', $out)", } @@ -286,7 +286,7 @@ func verifyOnlineVolumeExpansionOnGc(client clientset.Interface, namespace strin "exec", pod.Name, "--namespace=" + namespace, - "powershell.exe", + "-- powershell.exe -Command", "Copy-Item -Path '/mnt/volume1/testdata2.txt' " + "-Destination '/mnt/volume1/testdata2_pod.txt'", } @@ -306,7 +306,7 @@ func verifyOnlineVolumeExpansionOnGc(client clientset.Interface, namespace strin "exec", pod.Name, "--namespace=" + namespace, - "powershell.exe", + "-- powershell.exe -Command", "((Get-FileHash '/mnt/volume1/testdata2.txt' -Algorithm SHA256).Hash -eq " + "(Get-FileHash '/mnt/volume1/testdata2_pod.txt' -Algorithm SHA256).Hash)", } diff --git a/tests/e2e/util.go b/tests/e2e/util.go index eabf3f4250..4584e4f2a3 100644 --- a/tests/e2e/util.go +++ b/tests/e2e/util.go @@ -1066,7 +1066,7 @@ func bringDownCsiController(Client clientset.Interface, namespace ...string) { // bringDownTKGController helps to bring the TKG control manager pod down. // Its taks svc client as input. -func bringDownTKGController(Client clientset.Interface) { +func bringDownTKGController(Client clientset.Interface, vsphereTKGSystemNamespace string) { updateDeploymentReplica(Client, 0, vsphereControllerManager, vsphereTKGSystemNamespace) ginkgo.By("TKGControllManager replica is set to 0") } @@ -1088,7 +1088,7 @@ func bringUpCsiController(Client clientset.Interface, csiReplicaCount int32, nam // bringUpTKGController helps to bring the TKG control manager pod up. // Its taks svc client as input. -func bringUpTKGController(Client clientset.Interface, tkgReplica int32) { +func bringUpTKGController(Client clientset.Interface, tkgReplica int32, vsphereTKGSystemNamespace string) { err := updateDeploymentReplicawithWait(Client, tkgReplica, vsphereControllerManager, vsphereTKGSystemNamespace) gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("TKGControllManager is up") @@ -7598,7 +7598,7 @@ func execCommandOnGcWorker(sshClientConfig *ssh.ClientConfig, svcMasterIP string } cmdToGetContainerInfo := fmt.Sprintf("ssh -o StrictHostKeyChecking=no -i key %s@%s "+ - "'%s' | grep -v 'Warning'", gcNodeUser, gcWorkerIp, cmd) + "'%s' 2> /dev/null", gcNodeUser, gcWorkerIp, cmd) framework.Logf("Invoking command '%v' on host %v", cmdToGetContainerInfo, svcMasterIP) cmdResult, err = sshExec(sshClientConfig, svcMasterIP, diff --git a/tests/e2e/volume_health_test.go b/tests/e2e/volume_health_test.go index 82275ced50..77d2227647 100644 --- a/tests/e2e/volume_health_test.go +++ b/tests/e2e/volume_health_test.go @@ -57,6 +57,7 @@ var _ = ginkgo.Describe("Volume health check", func() { isVsanHealthServiceStopped bool isSPSServiceStopped bool csiNamespace string + vsphereTKGSystemNamespace string ) ginkgo.BeforeEach(func() { @@ -70,6 +71,7 @@ var _ = ginkgo.Describe("Volume health check", func() { csiNamespace = GetAndExpectStringEnvVar(envCSINamespace) nodeList, err := fnodes.GetReadySchedulableNodes(ctx, f.ClientSet) datastoreURL = GetAndExpectStringEnvVar(envSharedDatastoreURL) + vsphereTKGSystemNamespace = GetAndExpectStringEnvVar(envVsphereTKGSystemNamespace) framework.ExpectNoError(err, "Unable to find ready and schedulable Node") if guestCluster { svcClient, svNamespace := getSvcClientAndNamespace() @@ -1289,14 +1291,14 @@ var _ = ginkgo.Describe("Volume health check", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) csiReplicaCount := *deployment.Spec.Replicas - bringDownTKGController(svClient) - bringDownCsiController(gcClient) + bringDownTKGController(svClient, vsphereTKGSystemNamespace) + bringDownCsiController(gcClient, vsphereTKGSystemNamespace) isControllerUP = false defer func() { if !isControllerUP { - bringUpTKGController(svClient, tkgReplicaCount) - bringUpCsiController(gcClient, csiReplicaCount) + bringUpTKGController(svClient, tkgReplicaCount, vsphereTKGSystemNamespace) + bringUpCsiController(gcClient, csiReplicaCount, vsphereTKGSystemNamespace) } }() @@ -1313,8 +1315,8 @@ var _ = ginkgo.Describe("Volume health check", func() { gomega.Expect(svPVC.Annotations[volumeHealthAnnotation]).Should(gomega.BeEquivalentTo(healthStatusAccessible)) ginkgo.By("Bring up csi-controller pod in GC") - bringUpTKGController(svClient, tkgReplicaCount) - bringUpCsiController(gcClient, csiReplicaCount) + bringUpTKGController(svClient, tkgReplicaCount, vsphereTKGSystemNamespace) + bringUpCsiController(gcClient, csiReplicaCount, vsphereTKGSystemNamespace) isControllerUP = true ginkgo.By("Verify health status of GC PVC after GC csi is up") @@ -2123,7 +2125,6 @@ var _ = ginkgo.Describe("Volume health check", func() { gcClient, err = createKubernetesClientFromConfig(k8senv) gomega.Expect(err).NotTo(gomega.HaveOccurred()) } - tkgReplicaDeployment, err := svcClient.AppsV1().Deployments(vsphereTKGSystemNamespace).Get(ctx, vsphereControllerManager, metav1.GetOptions{}) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -2140,8 +2141,8 @@ var _ = ginkgo.Describe("Volume health check", func() { err := waitForHostToBeUp(hostIP) gomega.Expect(err).NotTo(gomega.HaveOccurred()) if !isControllerUP { - bringUpTKGController(svcClient, tkgReplicaCount) - bringUpCsiController(gcClient, csiReplicaCount) + bringUpTKGController(svcClient, tkgReplicaCount, vsphereTKGSystemNamespace) + bringUpCsiController(gcClient, csiReplicaCount, vsphereTKGSystemNamespace) } err = fpv.DeletePersistentVolumeClaim(ctx, client, pvc.Name, namespace) gomega.Expect(err).NotTo(gomega.HaveOccurred()) @@ -2160,8 +2161,8 @@ var _ = ginkgo.Describe("Volume health check", func() { gomega.Expect(pvclaim.Annotations[volumeHealthAnnotation]).Should(gomega.BeEquivalentTo(healthStatusAccessible)) ginkgo.By("Bring down csi-controller pod in GC") - bringDownTKGController(svcClient) - bringDownCsiController(gcClient) + bringDownTKGController(svcClient, vsphereTKGSystemNamespace) + bringDownCsiController(gcClient, vsphereTKGSystemNamespace) isControllerUP = false // Get SV PVC before PSOD. @@ -2181,8 +2182,8 @@ var _ = ginkgo.Describe("Volume health check", func() { gomega.Expect(err).NotTo(gomega.HaveOccurred()) ginkgo.By("Bring up csi-controller pod in GC") - bringUpTKGController(svcClient, tkgReplicaCount) - bringUpCsiController(gcClient, csiReplicaCount) + bringUpTKGController(svcClient, tkgReplicaCount, vsphereTKGSystemNamespace) + bringUpCsiController(gcClient, csiReplicaCount, vsphereTKGSystemNamespace) isControllerUP = true ginkgo.By("Verify health status of GC PVC after GC csi is up")