Skip to content

Commit b8b7746

Browse files
Aishwarya-HebbarAishwarya-Hebbar
authored
Utils for CSI personas testing (#3494)
1 parent 7329225 commit b8b7746

File tree

6 files changed

+287
-47
lines changed

6 files changed

+287
-47
lines changed

‎tests/e2e/csi_utils.go

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,8 @@ import (
3232
func getControllerRuntimeDetails(client clientset.Interface, nameSpace string) ([]string, []string) {
3333
ctx, cancel := context.WithCancel(context.Background())
3434
defer cancel()
35-
pods, _ := client.CoreV1().Pods(nameSpace).List(
35+
adminClient, _ := initializeClusterClientsByUserRoles(client)
36+
pods, _ := adminClient.CoreV1().Pods(nameSpace).List(
3637
ctx,
3738
metav1.ListOptions{
3839
FieldSelector: fields.SelectorFromSet(fields.Set{"status.phase": string(v1.PodRunning)}).String(),
@@ -70,7 +71,9 @@ func waitForControllerDeletion(ctx context.Context, client clientset.Interface,
7071
func mapK8sMasterNodeWithIPs(client clientset.Interface, nodeNameIPMap map[string]string) error {
7172
ctx, cancel := context.WithCancel(context.Background())
7273
defer cancel()
73-
nodes, err := client.CoreV1().Nodes().List(ctx, metav1.ListOptions{LabelSelector: "node-role.kubernetes.io/master"})
74+
adminClient, _ := initializeClusterClientsByUserRoles(client)
75+
nodes, err := adminClient.CoreV1().Nodes().List(ctx,
76+
metav1.ListOptions{LabelSelector: "node-role.kubernetes.io/master"})
7477
if err != nil {
7578
return err
7679
}

‎tests/e2e/e2e_common.go

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -508,6 +508,17 @@ var (
508508
vsanMaxFaultDomainName = "VSAN_MAX_FD_NAME"
509509
)
510510

511+
// for devops persona testing
512+
var (
513+
envAdminKubeconfig = "ADMIN_KUBECONFIG"
514+
envDevopsKubeconfig = "DEVOPS_KUBE_CONFIG"
515+
rbacApiGroup = "rbac.authorization.k8s.io"
516+
roleKeyword = "Role"
517+
audienceForSvcAccountName = "https://kubernetes.default.svc.cluster.local"
518+
envIsDevopsUser = "IS_DEVOPS_USER"
519+
serviceAccountKeyword = "ServiceAccount"
520+
)
521+
511522
// storage policy usages for storage quota validation
512523
var usageSuffixes = []string{
513524
"-pvc-usage",

‎tests/e2e/multi_master_k8s.go

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -65,12 +65,13 @@ var _ = ginkgo.Describe("[csi-multi-master-block-e2e]", func() {
6565
controllerNamespace = csiSystemNamespace
6666
}
6767
bootstrap()
68+
var err error
6869
scParameters = make(map[string]string)
6970
storagePolicyName = GetAndExpectStringEnvVar(envStoragePolicyNameForSharedDatastores)
7071

7172
nodeNameIPMap = make(map[string]string)
7273
ginkgo.By("Retrieving testbed configuration data")
73-
err := mapK8sMasterNodeWithIPs(client, nodeNameIPMap)
74+
err = mapK8sMasterNodeWithIPs(client, nodeNameIPMap)
7475
framework.ExpectNoError(err)
7576

7677
labelKey = "app"
@@ -158,7 +159,7 @@ var _ = ginkgo.Describe("[csi-multi-master-block-e2e]", func() {
158159
gomega.Expect(err).NotTo(gomega.HaveOccurred())
159160

160161
ginkgo.By(fmt.Sprintf("Waiting for claim %s to be in bound phase", pvc.Name))
161-
pvs, err = fpv.WaitForPVClaimBoundPhase(ctx, client, []*v1.PersistentVolumeClaim{pvc},
162+
pvs, err = WaitForPVClaimBoundPhase(ctx, client, []*v1.PersistentVolumeClaim{pvc},
162163
framework.ClaimProvisionTimeout)
163164
gomega.Expect(err).NotTo(gomega.HaveOccurred())
164165
gomega.Expect(pvs).NotTo(gomega.BeEmpty())
@@ -260,7 +261,7 @@ var _ = ginkgo.Describe("[csi-multi-master-block-e2e]", func() {
260261
gomega.Expect(err).NotTo(gomega.HaveOccurred())
261262

262263
ginkgo.By(fmt.Sprintf("Waiting for claim %s to be in bound phase", pvc.Name))
263-
pvs, err = fpv.WaitForPVClaimBoundPhase(ctx, client, []*v1.PersistentVolumeClaim{pvc},
264+
pvs, err = WaitForPVClaimBoundPhase(ctx, client, []*v1.PersistentVolumeClaim{pvc},
264265
framework.ClaimProvisionTimeout)
265266
gomega.Expect(err).NotTo(gomega.HaveOccurred())
266267
gomega.Expect(pvs).NotTo(gomega.BeEmpty())

‎tests/e2e/multi_vc_utils.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,8 @@ cleanup of these multiple sts creation, deleteAllStsAndPodsPVCsInNamespace is us
161161
func deleteAllStsAndPodsPVCsInNamespace(ctx context.Context, c clientset.Interface, ns string) {
162162
StatefulSetPoll := 10 * time.Second
163163
StatefulSetTimeout := 10 * time.Minute
164+
var err error
165+
adminClient, c := initializeClusterClientsByUserRoles(c)
164166
ssList, err := c.AppsV1().StatefulSets(ns).List(context.TODO(),
165167
metav1.ListOptions{LabelSelector: labels.Everything().String()})
166168
framework.ExpectNoError(err)
@@ -203,7 +205,7 @@ func deleteAllStsAndPodsPVCsInNamespace(ctx context.Context, c clientset.Interfa
203205

204206
pollErr := wait.PollUntilContextTimeout(ctx, StatefulSetPoll, StatefulSetTimeout, true,
205207
func(ctx context.Context) (bool, error) {
206-
pvList, err := c.CoreV1().PersistentVolumes().List(context.TODO(),
208+
pvList, err := adminClient.CoreV1().PersistentVolumes().List(context.TODO(),
207209
metav1.ListOptions{LabelSelector: labels.Everything().String()})
208210
if err != nil {
209211
framework.Logf("WARNING: Failed to list pvs, retrying %v", err)

‎tests/e2e/raw_block_volume.go

Lines changed: 38 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ import (
3434
"github.com/vmware/govmomi/object"
3535
"github.com/vmware/govmomi/vim25/types"
3636
corev1 "k8s.io/api/core/v1"
37+
3738
apierrors "k8s.io/apimachinery/pkg/api/errors"
3839
"k8s.io/apimachinery/pkg/api/resource"
3940
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -76,15 +77,29 @@ var _ = ginkgo.Describe("raw block volume support", func() {
7677
snapc *snapclient.Clientset
7778
restConfig *restclient.Config
7879
guestClusterRestConfig *restclient.Config
80+
adminClient clientset.Interface
7981
)
8082

8183
ginkgo.BeforeEach(func() {
8284
ctx, cancel := context.WithCancel(context.Background())
8385
defer cancel()
8486
namespace = getNamespaceToRunTests(f)
8587
client = f.ClientSet
88+
var err error
8689
bootstrap()
87-
nodeList, err := fnodes.GetReadySchedulableNodes(ctx, f.ClientSet)
90+
91+
var nodeList *corev1.NodeList
92+
runningAsDevopsUser := GetBoolEnvVarOrDefault("IS_DEVOPS_USER", false)
93+
adminClient, client = initializeClusterClientsByUserRoles(client)
94+
if guestCluster && runningAsDevopsUser {
95+
96+
saName := namespace + "sa"
97+
client, err = createScopedClient(ctx, client, namespace, saName)
98+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
99+
100+
}
101+
nodeList, err = fnodes.GetReadySchedulableNodes(ctx, f.ClientSet)
102+
88103
framework.ExpectNoError(err, "Unable to find ready and schedulable Node")
89104
if !(len(nodeList.Items) > 0) {
90105
framework.Failf("Unable to find ready and schedulable Node")
@@ -101,9 +116,10 @@ var _ = ginkgo.Describe("raw block volume support", func() {
101116
gomega.Expect(err).NotTo(gomega.HaveOccurred())
102117
}
103118

104-
sc, err := client.StorageV1().StorageClasses().Get(ctx, defaultNginxStorageClassName, metav1.GetOptions{})
119+
sc, err := adminClient.StorageV1().StorageClasses().Get(ctx, defaultNginxStorageClassName, metav1.GetOptions{})
120+
framework.Logf("err: %v", err)
105121
if err == nil && sc != nil {
106-
gomega.Expect(client.StorageV1().StorageClasses().Delete(ctx, sc.Name,
122+
gomega.Expect(adminClient.StorageV1().StorageClasses().Delete(ctx, sc.Name,
107123
*metav1.NewDeleteOptions(0))).NotTo(gomega.HaveOccurred())
108124
}
109125
scParameters = make(map[string]string)
@@ -198,10 +214,10 @@ var _ = ginkgo.Describe("raw block volume support", func() {
198214

199215
ginkgo.By("Creating StorageClass for Statefulset")
200216
scSpec := getVSphereStorageClassSpec(storageClassName, scParameters, nil, "", "", false)
201-
sc, err := client.StorageV1().StorageClasses().Create(ctx, scSpec, metav1.CreateOptions{})
217+
sc, err := adminClient.StorageV1().StorageClasses().Create(ctx, scSpec, metav1.CreateOptions{})
202218
gomega.Expect(err).NotTo(gomega.HaveOccurred())
203219
defer func() {
204-
err := client.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0))
220+
err := adminClient.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0))
205221
gomega.Expect(err).NotTo(gomega.HaveOccurred())
206222
}()
207223

@@ -232,7 +248,7 @@ var _ = ginkgo.Describe("raw block volume support", func() {
232248
replicas := *(statefulset.Spec.Replicas)
233249
defer func() {
234250
ginkgo.By(fmt.Sprintf("Deleting all statefulsets in namespace: %v", namespace))
235-
fss.DeleteAllStatefulSets(ctx, client, namespace)
251+
deleteAllStsAndPodsPVCsInNamespace(ctx, client, namespace)
236252
}()
237253

238254
// Waiting for pods status to be Ready
@@ -414,7 +430,7 @@ var _ = ginkgo.Describe("raw block volume support", func() {
414430
sc, err := createStorageClass(client, scParameters, nil, "", "", false, "")
415431
gomega.Expect(err).NotTo(gomega.HaveOccurred())
416432
defer func() {
417-
err := client.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0))
433+
err := adminClient.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0))
418434
gomega.Expect(err).NotTo(gomega.HaveOccurred())
419435
}()
420436

@@ -425,7 +441,7 @@ var _ = ginkgo.Describe("raw block volume support", func() {
425441
gomega.Expect(err).NotTo(gomega.HaveOccurred(), fmt.Sprintf("Failed to create pvc with err: %v", err))
426442

427443
ginkgo.By(fmt.Sprintf("Waiting for claim %s to be in bound phase", pvc.Name))
428-
pvs, err := fpv.WaitForPVClaimBoundPhase(ctx, client, []*corev1.PersistentVolumeClaim{pvc},
444+
pvs, err := WaitForPVClaimBoundPhase(ctx, client, []*corev1.PersistentVolumeClaim{pvc},
429445
framework.ClaimProvisionTimeout)
430446
gomega.Expect(err).NotTo(gomega.HaveOccurred())
431447
gomega.Expect(pvs).NotTo(gomega.BeEmpty())
@@ -440,7 +456,7 @@ var _ = ginkgo.Describe("raw block volume support", func() {
440456
defer func() {
441457
err := fpv.DeletePersistentVolumeClaim(ctx, client, pvc.Name, namespace)
442458
gomega.Expect(err).NotTo(gomega.HaveOccurred())
443-
err = fpv.WaitForPersistentVolumeDeleted(ctx, client, pv.Name, poll, pollTimeoutShort)
459+
err = fpv.WaitForPersistentVolumeDeleted(ctx, adminClient, pv.Name, poll, pollTimeoutShort)
444460
gomega.Expect(err).NotTo(gomega.HaveOccurred())
445461
err = e2eVSphere.waitForCNSVolumeToBeDeleted(volumeID)
446462
gomega.Expect(err).NotTo(gomega.HaveOccurred())
@@ -590,12 +606,12 @@ var _ = ginkgo.Describe("raw block volume support", func() {
590606
ginkgo.By("Creating raw block PV")
591607
pv = getPersistentVolumeSpec(fcdID, corev1.PersistentVolumeReclaimDelete, staticPVLabels, "")
592608
pv.Spec.VolumeMode = &rawBlockVolumeMode
593-
pv, err = client.CoreV1().PersistentVolumes().Create(ctx, pv, metav1.CreateOptions{})
609+
pv, err = adminClient.CoreV1().PersistentVolumes().Create(ctx, pv, metav1.CreateOptions{})
594610
gomega.Expect(err).NotTo(gomega.HaveOccurred())
595611
err = e2eVSphere.waitForCNSVolumeToBeCreated(pv.Spec.CSI.VolumeHandle)
596612
gomega.Expect(err).NotTo(gomega.HaveOccurred())
597613
defer func() {
598-
err := fpv.DeletePersistentVolume(ctx, client, pv.Name)
614+
err := fpv.DeletePersistentVolume(ctx, adminClient, pv.Name)
599615
gomega.Expect(err).NotTo(gomega.HaveOccurred())
600616
}()
601617

@@ -610,7 +626,7 @@ var _ = ginkgo.Describe("raw block volume support", func() {
610626
defer func() {
611627
err := fpv.DeletePersistentVolumeClaim(ctx, client, pvc.Name, namespace)
612628
gomega.Expect(err).NotTo(gomega.HaveOccurred())
613-
err = fpv.WaitForPersistentVolumeDeleted(ctx, client, pv.Name, poll, pollTimeoutShort)
629+
err = fpv.WaitForPersistentVolumeDeleted(ctx, adminClient, pv.Name, poll, pollTimeoutShort)
614630
gomega.Expect(err).NotTo(gomega.HaveOccurred())
615631
err = e2eVSphere.waitForCNSVolumeToBeDeleted(pv.Spec.CSI.VolumeHandle)
616632
gomega.Expect(err).NotTo(gomega.HaveOccurred())
@@ -712,7 +728,7 @@ var _ = ginkgo.Describe("raw block volume support", func() {
712728
sc, err := createStorageClass(client, scParameters, nil, "", "", true, "")
713729
gomega.Expect(err).NotTo(gomega.HaveOccurred())
714730
defer func() {
715-
err := client.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0))
731+
err := adminClient.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0))
716732
gomega.Expect(err).NotTo(gomega.HaveOccurred())
717733
}()
718734

@@ -723,7 +739,7 @@ var _ = ginkgo.Describe("raw block volume support", func() {
723739
gomega.Expect(err).NotTo(gomega.HaveOccurred(), fmt.Sprintf("Failed to create pvc with err: %v", err))
724740

725741
ginkgo.By(fmt.Sprintf("Waiting for claim %s to be in bound phase", pvc.Name))
726-
pvs, err := fpv.WaitForPVClaimBoundPhase(ctx, client, []*corev1.PersistentVolumeClaim{pvc},
742+
pvs, err := WaitForPVClaimBoundPhase(ctx, client, []*corev1.PersistentVolumeClaim{pvc},
727743
framework.ClaimProvisionTimeout)
728744
gomega.Expect(err).NotTo(gomega.HaveOccurred())
729745
gomega.Expect(pvs).NotTo(gomega.BeEmpty())
@@ -738,7 +754,7 @@ var _ = ginkgo.Describe("raw block volume support", func() {
738754
defer func() {
739755
err := fpv.DeletePersistentVolumeClaim(ctx, client, pvc.Name, namespace)
740756
gomega.Expect(err).NotTo(gomega.HaveOccurred())
741-
err = fpv.WaitForPersistentVolumeDeleted(ctx, client, pv.Name, poll, pollTimeoutShort)
757+
err = fpv.WaitForPersistentVolumeDeleted(ctx, adminClient, pv.Name, poll, pollTimeoutShort)
742758
gomega.Expect(err).NotTo(gomega.HaveOccurred())
743759
err = e2eVSphere.waitForCNSVolumeToBeDeleted(volumeID)
744760
gomega.Expect(err).NotTo(gomega.HaveOccurred())
@@ -884,7 +900,7 @@ var _ = ginkgo.Describe("raw block volume support", func() {
884900
sc, err := createStorageClass(client, scParameters, nil, "", "", true, "")
885901
gomega.Expect(err).NotTo(gomega.HaveOccurred())
886902
defer func() {
887-
err := client.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0))
903+
err := adminClient.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0))
888904
gomega.Expect(err).NotTo(gomega.HaveOccurred())
889905
}()
890906

@@ -898,7 +914,7 @@ var _ = ginkgo.Describe("raw block volume support", func() {
898914
var pvclaims []*corev1.PersistentVolumeClaim
899915
pvclaims = append(pvclaims, pvc)
900916
ginkgo.By("Waiting for all claims to be in bound state")
901-
pvs, err := fpv.WaitForPVClaimBoundPhase(ctx, client, pvclaims, framework.ClaimProvisionTimeout)
917+
pvs, err := WaitForPVClaimBoundPhase(ctx, client, pvclaims, framework.ClaimProvisionTimeout)
902918
gomega.Expect(err).NotTo(gomega.HaveOccurred())
903919
pv := pvs[0]
904920
volumeID := pv.Spec.CSI.VolumeHandle
@@ -910,7 +926,7 @@ var _ = ginkgo.Describe("raw block volume support", func() {
910926
defer func() {
911927
err := fpv.DeletePersistentVolumeClaim(ctx, client, pvc.Name, namespace)
912928
gomega.Expect(err).NotTo(gomega.HaveOccurred())
913-
err = fpv.WaitForPersistentVolumeDeleted(ctx, client, pv.Name, poll, pollTimeoutShort)
929+
err = fpv.WaitForPersistentVolumeDeleted(ctx, adminClient, pv.Name, poll, pollTimeoutShort)
914930
gomega.Expect(err).NotTo(gomega.HaveOccurred())
915931
err = e2eVSphere.waitForCNSVolumeToBeDeleted(volumeID)
916932
gomega.Expect(err).NotTo(gomega.HaveOccurred())
@@ -996,7 +1012,7 @@ var _ = ginkgo.Describe("raw block volume support", func() {
9961012
}
9971013

9981014
ginkgo.By("Waiting for controller volume resize to finish")
999-
err = waitForPvResizeForGivenPvc(pvc, client, totalResizeWaitPeriod)
1015+
err = waitForPvResizeForGivenPvc(pvc, adminClient, totalResizeWaitPeriod)
10001016
gomega.Expect(err).NotTo(gomega.HaveOccurred())
10011017

10021018
if guestCluster {
@@ -1116,7 +1132,7 @@ var _ = ginkgo.Describe("raw block volume support", func() {
11161132
sc, err := createStorageClass(client, scParameters, nil, "", "", false, "")
11171133
gomega.Expect(err).NotTo(gomega.HaveOccurred())
11181134
defer func() {
1119-
err := client.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0))
1135+
err := adminClient.StorageV1().StorageClasses().Delete(ctx, sc.Name, *metav1.NewDeleteOptions(0))
11201136
gomega.Expect(err).NotTo(gomega.HaveOccurred())
11211137
}()
11221138

@@ -1127,7 +1143,7 @@ var _ = ginkgo.Describe("raw block volume support", func() {
11271143
gomega.Expect(err).NotTo(gomega.HaveOccurred(), fmt.Sprintf("Failed to create pvc with err: %v", err))
11281144

11291145
ginkgo.By("Expect source volume claim to provision volume successfully")
1130-
pvs, err := fpv.WaitForPVClaimBoundPhase(ctx, client, []*corev1.PersistentVolumeClaim{pvc1},
1146+
pvs, err := WaitForPVClaimBoundPhase(ctx, client, []*corev1.PersistentVolumeClaim{pvc1},
11311147
framework.ClaimProvisionTimeout)
11321148
gomega.Expect(err).NotTo(gomega.HaveOccurred())
11331149
volumeID := pvs[0].Spec.CSI.VolumeHandle
@@ -1238,7 +1254,7 @@ var _ = ginkgo.Describe("raw block volume support", func() {
12381254
restoredPvc, err := fpv.CreatePVC(ctx, client, namespace, restorePvcSpec)
12391255
gomega.Expect(err).NotTo(gomega.HaveOccurred())
12401256

1241-
restoredPvs, err := fpv.WaitForPVClaimBoundPhase(ctx, client,
1257+
restoredPvs, err := WaitForPVClaimBoundPhase(ctx, client,
12421258
[]*corev1.PersistentVolumeClaim{restoredPvc},
12431259
framework.ClaimProvisionTimeout)
12441260
gomega.Expect(err).NotTo(gomega.HaveOccurred())

0 commit comments

Comments
 (0)